Skip to main content

littlebit_core/
linear.rs

1//! `LittleBitLinear` and compact 1-bit `PackedLittleBitLinear` modules.
2
3use std::collections::HashMap;
4use std::fmt;
5
6use candle_core::{DType, Device, Tensor, Var};
7use faer::Mat;
8
9use crate::binary::binarize_sign_no_grad;
10use crate::config::{
11    BiasMode, BitBudget, LayerShape, LittleBitConfig, QuantFunc, ResidualMode, TrainMode,
12    WeightState,
13};
14use crate::error::{LittleBitError, Result};
15use crate::itq::apply_joint_itq;
16use crate::packer::{binary_packer, PackedBinaryMatrix};
17use crate::svd::{rank_one_decompose, svd_lowrank, DEFAULT_LOWRANK_NITER};
18
19/// Parameter storage that supports both differentiable QAT training and frozen inference.
20#[derive(Debug, Clone)]
21pub enum Param {
22    /// Differentiable leaf variable tracked by Candle autograd.
23    Trainable(Var),
24    /// Frozen non-differentiable tensor.
25    Frozen(Tensor),
26}
27
28impl Param {
29    /// Wrap a tensor as [`Param::Trainable`] or [`Param::Frozen`] according to `mode`.
30    ///
31    /// # Errors
32    /// Returns a Candle error if constructing a [`Var`] fails.
33    pub fn from_tensor(tensor: Tensor, mode: TrainMode) -> Result<Self> {
34        match mode {
35            TrainMode::Train => Ok(Self::Trainable(Var::from_tensor(&tensor)?)),
36            TrainMode::Eval => Ok(Self::Frozen(tensor.detach())),
37        }
38    }
39
40    /// Borrow the underlying tensor regardless of mutability mode.
41    #[must_use]
42    pub fn as_tensor(&self) -> &Tensor {
43        match self {
44            Self::Trainable(v) => v.as_tensor(),
45            Self::Frozen(t) => t,
46        }
47    }
48
49    /// Return the underlying [`Var`] when trainable.
50    #[must_use]
51    pub fn as_var(&self) -> Option<&Var> {
52        match self {
53            Self::Trainable(v) => Some(v),
54            Self::Frozen(_) => None,
55        }
56    }
57
58    /// Convert to a frozen detached parameter in place.
59    pub fn freeze(&mut self) {
60        if let Self::Trainable(v) = self {
61            *self = Self::Frozen(v.as_tensor().detach());
62        }
63    }
64
65    /// Return true when backed by a trainable [`Var`].
66    #[must_use]
67    pub fn is_trainable(&self) -> bool {
68        matches!(self, Self::Trainable(_))
69    }
70
71    /// Element count of the underlying tensor.
72    #[must_use]
73    pub fn elem_count(&self) -> usize {
74        self.as_tensor().elem_count()
75    }
76}
77
78/// One latent factorization branch `(U, V, u1, u2, v1, v2)`.
79#[derive(Debug, Clone)]
80pub struct FactorBranch {
81    /// Left binary-quantized factor `(out_features, split_dim)`.
82    pub u: Param,
83    /// Right binary-quantized factor `(split_dim, in_features)`.
84    pub v: Param,
85    /// Output row scale `(1, out_features)`.
86    pub u1: Param,
87    /// Output latent rank scale `(1, split_dim)`.
88    pub u2: Param,
89    /// Input latent rank scale `(1, split_dim)`.
90    pub v1: Param,
91    /// Input column scale `(1, in_features)`.
92    pub v2: Param,
93}
94
95/// Maximum finite magnitude safely representable in IEEE-754 binary16 (`f16`).
96pub const FP16_MAX_SAFE: f64 = 65_504.0;
97
98/// Compute the smallest power of two `>= val` (clamped to `>= 1.0`), matching the
99/// `pow2_at_least` helper from `SamsungLabs/LittleBit` Issue #24.
100#[inline]
101#[must_use]
102pub fn pow2_at_least(val: f64) -> f64 {
103    if !val.is_finite() || val <= 1.0 {
104        1.0
105    } else {
106        2.0_f64.powf(val.log2().ceil())
107    }
108}
109
110impl FactorBranch {
111    fn from_raw(
112        raw: RawDecomposedBranch,
113        dtype: DType,
114        device: &Device,
115        mode: TrainMode,
116        out_features: usize,
117        in_features: usize,
118        split_dim: usize,
119    ) -> Result<Self> {
120        let make = |data: Vec<f32>, shape: (usize, usize)| -> Result<Param> {
121            let t = Tensor::from_vec(data, shape, device)?;
122            let t = if dtype == DType::F32 {
123                t
124            } else {
125                t.to_dtype(dtype)?
126            };
127            Param::from_tensor(t, mode)
128        };
129        Ok(Self {
130            u: make(raw.u, (out_features, split_dim))?,
131            v: make(raw.v, (split_dim, in_features))?,
132            u1: make(raw.u1, (1, out_features))?,
133            u2: make(raw.u2, (1, split_dim))?,
134            v1: make(raw.v1, (1, split_dim))?,
135            v2: make(raw.v2, (1, in_features))?,
136        })
137    }
138
139    fn empty(
140        out_features: usize,
141        in_features: usize,
142        split_dim: usize,
143        dtype: DType,
144        device: &Device,
145        mode: TrainMode,
146    ) -> Result<Self> {
147        let make = |shape: (usize, usize)| -> Result<Param> {
148            let t = Tensor::zeros(shape, dtype, device)?;
149            Param::from_tensor(t, mode)
150        };
151        Ok(Self {
152            u: make((out_features, split_dim))?,
153            v: make((split_dim, in_features))?,
154            u1: make((1, out_features))?,
155            u2: make((1, split_dim))?,
156            v1: make((1, split_dim))?,
157            v2: make((1, in_features))?,
158        })
159    }
160
161    fn forward(&self, x_2d: &Tensor, quant_func: QuantFunc, binarized: bool) -> Result<Tensor> {
162        let x_dtype = x_2d.dtype();
163        let cast = |t: &Tensor| -> Result<Tensor> {
164            if t.dtype() == x_dtype {
165                Ok(t.clone())
166            } else {
167                Ok(t.to_dtype(x_dtype)?)
168            }
169        };
170
171        let v = cast(self.v.as_tensor())?;
172        let u = cast(self.u.as_tensor())?;
173        let vq = if binarized { v } else { quant_func.apply(&v)? };
174        let uq = if binarized { u } else { quant_func.apply(&u)? };
175
176        let vq_t = vq.t()?.contiguous()?;
177        let uq_t = uq.t()?.contiguous()?;
178
179        if x_dtype == DType::F16 {
180            let x_f32 = x_2d.to_dtype(DType::F32)?;
181            let v2_f32 = self.v2.as_tensor().to_dtype(DType::F32)?;
182            let v1_f32 = self.v1.as_tensor().to_dtype(DType::F32)?;
183            let u2_f32 = self.u2.as_tensor().to_dtype(DType::F32)?;
184            let u1_f32 = self.u1.as_tensor().to_dtype(DType::F32)?;
185            let v1u2_f32 = v1_f32.mul(&u2_f32)?;
186
187            if x_2d.device().is_cpu() {
188                let vq_t_f32 = vq_t.to_dtype(DType::F32)?;
189                let uq_t_f32 = uq_t.to_dtype(DType::F32)?;
190                let h = x_f32.broadcast_mul(&v2_f32)?.matmul(&vq_t_f32)?;
191                let g = h.broadcast_mul(&v1u2_f32)?;
192                let out = g
193                    .matmul(&uq_t_f32)?
194                    .broadcast_mul(&u1_f32)?
195                    .to_dtype(DType::F16)?;
196                return Ok(out);
197            }
198
199            let a = x_f32.broadcast_mul(&v2_f32)?;
200            let a_max = f64::from(a.abs()?.max_all()?.to_scalar::<f32>()?);
201            let in_features = vq_t.dims()[0] as f64;
202            let s1 = pow2_at_least(a_max * in_features / FP16_MAX_SAFE);
203            let a_scaled = if s1 > 1.0 { (a / s1)? } else { a };
204
205            let h = a_scaled
206                .to_dtype(DType::F16)?
207                .matmul(&vq_t)?
208                .to_dtype(DType::F32)?;
209            let g = h.broadcast_mul(&v1u2_f32)?;
210            let g_max = f64::from(g.abs()?.max_all()?.to_scalar::<f32>()?);
211            let split_dim = uq_t.dims()[0] as f64;
212            let s2 = pow2_at_least(g_max * split_dim / FP16_MAX_SAFE);
213            let g_scaled = if s2 > 1.0 { (g / s2)? } else { g };
214
215            let y = g_scaled
216                .to_dtype(DType::F16)?
217                .matmul(&uq_t)?
218                .to_dtype(DType::F32)?;
219            let total_scale = s1 * s2;
220            let u1_scaled = if total_scale > 1.0 {
221                (u1_f32 * total_scale)?
222            } else {
223                u1_f32
224            };
225            let out = y.broadcast_mul(&u1_scaled)?.to_dtype(DType::F16)?;
226            return Ok(out);
227        }
228
229        let v2 = cast(self.v2.as_tensor())?;
230        let v1 = cast(self.v1.as_tensor())?;
231        let u2 = cast(self.u2.as_tensor())?;
232        let u1 = cast(self.u1.as_tensor())?;
233
234        let v1u2 = v1.mul(&u2)?;
235
236        // ((((x * v2) @ Vq^T) * (v1 * u2)) @ Uq^T) * u1
237        let h = x_2d.broadcast_mul(&v2)?.matmul(&vq_t)?;
238        let h = h.broadcast_mul(&v1u2)?;
239        let out = h.matmul(&uq_t)?.broadcast_mul(&u1)?;
240        Ok(out)
241    }
242
243    fn push_vars(&self, vars: &mut Vec<Var>) {
244        for p in [&self.u, &self.v, &self.u1, &self.u2, &self.v1, &self.v2] {
245            if let Some(v) = p.as_var() {
246                vars.push(v.clone());
247            }
248        }
249    }
250
251    fn freeze(&mut self) {
252        self.u.freeze();
253        self.v.freeze();
254        self.u1.freeze();
255        self.u2.freeze();
256        self.v1.freeze();
257        self.v2.freeze();
258    }
259
260    fn warm_start(&mut self, latent_mag: f64) -> Result<()> {
261        let u_scaled = (self.u.as_tensor().detach() * latent_mag)?;
262        let v_scaled = (self.v.as_tensor().detach() * latent_mag)?;
263        self.u = Param::Trainable(Var::from_tensor(&u_scaled)?);
264        self.v = Param::Trainable(Var::from_tensor(&v_scaled)?);
265        self.u1 = Param::Trainable(Var::from_tensor(&self.u1.as_tensor().detach())?);
266        self.u2 = Param::Trainable(Var::from_tensor(&self.u2.as_tensor().detach())?);
267        self.v1 = Param::Trainable(Var::from_tensor(&self.v1.as_tensor().detach())?);
268        self.v2 = Param::Trainable(Var::from_tensor(&self.v2.as_tensor().detach())?);
269        Ok(())
270    }
271
272    fn elem_count(&self) -> usize {
273        self.u.elem_count()
274            + self.v.elem_count()
275            + self.u1.elem_count()
276            + self.u2.elem_count()
277            + self.v1.elem_count()
278            + self.v2.elem_count()
279    }
280}
281
282struct RawDecomposedBranch {
283    u: Vec<f32>,
284    v: Vec<f32>,
285    u1: Vec<f32>,
286    u2: Vec<f32>,
287    v1: Vec<f32>,
288    v2: Vec<f32>,
289}
290
291fn decompose_matrix_host(
292    weight_f32: &[f32],
293    out_features: usize,
294    in_features: usize,
295    split_dim: usize,
296    config: &LittleBitConfig,
297    seed: u64,
298) -> Result<RawDecomposedBranch> {
299    let svd = svd_lowrank(
300        weight_f32,
301        out_features,
302        in_features,
303        split_dim,
304        DEFAULT_LOWRANK_NITER,
305        seed,
306    )?;
307
308    let r = svd.rank;
309    let sqrt_s: Vec<f32> = svd.s.iter().map(|&s| s.max(0.0).sqrt()).collect();
310
311    // U: (out_features, split_dim), zero-padded if r < split_dim.
312    let mut u = vec![0.0f32; out_features * split_dim];
313    for row in 0..out_features {
314        for k in 0..r {
315            u[row * split_dim + k] = svd.u[row * r + k] * sqrt_s[k];
316        }
317    }
318
319    // V: (split_dim, in_features), zero-padded if r < split_dim.
320    let mut v = vec![0.0f32; split_dim * in_features];
321    for k in 0..r {
322        let s_root = sqrt_s[k];
323        for col in 0..in_features {
324            v[k * in_features + col] = s_root * svd.vt[k * in_features + col];
325        }
326    }
327
328    if config.use_itq {
329        apply_joint_itq(
330            &mut u,
331            &mut v,
332            out_features,
333            in_features,
334            split_dim,
335            config.itq_n_iter,
336            seed.wrapping_add(1),
337        )?;
338    }
339
340    let abs_v: Vec<f32> = v.iter().map(|&x| x.abs()).collect();
341    let (v1, v2) = rank_one_decompose(&abs_v, split_dim, in_features, seed.wrapping_add(2))?;
342
343    let abs_u: Vec<f32> = u.iter().map(|&x| x.abs()).collect();
344    let (u1, u2) = rank_one_decompose(&abs_u, out_features, split_dim, seed.wrapping_add(3))?;
345
346    Ok(RawDecomposedBranch {
347        u,
348        v,
349        u1,
350        u2,
351        v1,
352        v2,
353    })
354}
355
356fn compute_residual_matrix_host(
357    orig_w: &[f32],
358    branch: &RawDecomposedBranch,
359    out_features: usize,
360    in_features: usize,
361    split_dim: usize,
362) -> Vec<f32> {
363    let u_scaled = Mat::from_fn(out_features, split_dim, |r, k| {
364        let sign = if branch.u[r * split_dim + k] < 0.0 {
365            -1.0f32
366        } else {
367            1.0f32
368        };
369        sign * (branch.u1[r] * branch.u2[k])
370    });
371    let v_scaled = Mat::from_fn(split_dim, in_features, |k, c| {
372        let sign = if branch.v[k * in_features + c] < 0.0 {
373            -1.0f32
374        } else {
375            1.0f32
376        };
377        sign * (branch.v1[k] * branch.v2[c])
378    });
379    let approx = &u_scaled * &v_scaled;
380    let mut residual = Vec::with_capacity(orig_w.len());
381    for r in 0..out_features {
382        for c in 0..in_features {
383            residual.push(orig_w[r * in_features + c] - approx[(r, c)]);
384        }
385    }
386    residual
387}
388
389/// Sub-1-bit factorized linear layer matching upstream `quantization.modules.LittleBitLinear`.
390#[derive(Debug, Clone)]
391pub struct LittleBitLinear {
392    in_features: usize,
393    out_features: usize,
394    budget: BitBudget,
395    quant_func: QuantFunc,
396    binarized: bool,
397    main: FactorBranch,
398    residual: Option<FactorBranch>,
399    bias: Option<Param>,
400}
401
402impl LittleBitLinear {
403    /// Decompose a dense weight `(out_features, in_features)` and optional bias `(out_features,)`
404    /// into a `LittleBitLinear` module via low-rank SVD (and Joint-ITQ when configured).
405    ///
406    /// # Errors
407    /// Returns an error on invalid config, non-2D weight, shape mismatch, or SVD failure.
408    pub fn from_dense_weight(
409        weight: &Tensor,
410        bias: Option<&Tensor>,
411        config: &LittleBitConfig,
412        ratio_factor: f64,
413        train_mode: TrainMode,
414        seed: u64,
415    ) -> Result<Self> {
416        config.validate()?;
417        let dims = weight.dims();
418        if dims.len() != 2 {
419            return Err(LittleBitError::InvalidShape {
420                expected: "2D weight (out_features, in_features)".to_string(),
421                actual: format!("{dims:?}"),
422            });
423        }
424        let (out_features, in_features) = (dims[0], dims[1]);
425        if let Some(b) = bias {
426            if b.dims() != [out_features] {
427                return Err(LittleBitError::InvalidShape {
428                    expected: format!("[{out_features}]"),
429                    actual: format!("{:?}", b.dims()),
430                });
431            }
432        }
433
434        let target_dtype = weight.dtype();
435        let target_device = weight.device().clone();
436        let budget = BitBudget::resolve(in_features, out_features, config, ratio_factor);
437
438        let weight_f32 = weight
439            .to_device(&Device::Cpu)?
440            .to_dtype(DType::F32)?
441            .reshape(out_features * in_features)?
442            .to_vec1::<f32>()?;
443
444        let main_raw = decompose_matrix_host(
445            &weight_f32,
446            out_features,
447            in_features,
448            budget.split_dim,
449            config,
450            seed,
451        )?;
452
453        let residual_branch = if budget.residual == ResidualMode::Dual {
454            let residual_w = compute_residual_matrix_host(
455                &weight_f32,
456                &main_raw,
457                out_features,
458                in_features,
459                budget.split_dim,
460            );
461            let res_raw = decompose_matrix_host(
462                &residual_w,
463                out_features,
464                in_features,
465                budget.split_dim,
466                config,
467                seed.wrapping_add(10_000),
468            )?;
469            Some(FactorBranch::from_raw(
470                res_raw,
471                target_dtype,
472                &target_device,
473                train_mode,
474                out_features,
475                in_features,
476                budget.split_dim,
477            )?)
478        } else {
479            None
480        };
481
482        let main = FactorBranch::from_raw(
483            main_raw,
484            target_dtype,
485            &target_device,
486            train_mode,
487            out_features,
488            in_features,
489            budget.split_dim,
490        )?;
491
492        let bias_param = match bias {
493            Some(b) => Some(Param::from_tensor(
494                b.to_device(&target_device)?.to_dtype(target_dtype)?,
495                train_mode,
496            )?),
497            None => None,
498        };
499
500        Ok(Self {
501            in_features,
502            out_features,
503            budget,
504            quant_func: config.quant_func,
505            binarized: false,
506            main,
507            residual: residual_branch,
508            bias: bias_param,
509        })
510    }
511
512    /// Allocate zero-initialized factor tensors without performing SVD (for checkpoint loading).
513    ///
514    /// # Errors
515    /// Returns an error if config validation or tensor allocation fails.
516    pub fn empty(
517        shape: LayerShape,
518        config: &LittleBitConfig,
519        ratio_factor: f64,
520        train_mode: TrainMode,
521        dtype: DType,
522        device: &Device,
523    ) -> Result<Self> {
524        config.validate()?;
525        let (in_features, out_features) = (shape.in_features, shape.out_features);
526        let budget = BitBudget::resolve(in_features, out_features, config, ratio_factor);
527        let main = FactorBranch::empty(
528            out_features,
529            in_features,
530            budget.split_dim,
531            dtype,
532            device,
533            train_mode,
534        )?;
535        let residual = if budget.residual == ResidualMode::Dual {
536            Some(FactorBranch::empty(
537                out_features,
538                in_features,
539                budget.split_dim,
540                dtype,
541                device,
542                train_mode,
543            )?)
544        } else {
545            None
546        };
547        let bias = if shape.bias == BiasMode::WithBias {
548            Some(Param::from_tensor(
549                Tensor::zeros(out_features, dtype, device)?,
550                train_mode,
551            )?)
552        } else {
553            None
554        };
555        Ok(Self {
556            in_features,
557            out_features,
558            budget,
559            quant_func: config.quant_func,
560            binarized: false,
561            main,
562            residual,
563            bias,
564        })
565    }
566
567    /// Reconstruct a [`LittleBitLinear`] layer from an unpacked state dictionary.
568    ///
569    /// # Errors
570    /// Returns [`LittleBitError::MissingTensor`] or shape mismatch errors if any factor tensor
571    /// is absent or malformed.
572    #[allow(clippy::too_many_arguments)]
573    pub fn from_state_dict(
574        prefix: &str,
575        tensors: &HashMap<String, Tensor>,
576        shape: LayerShape,
577        config: &LittleBitConfig,
578        ratio_factor: f64,
579        weight_state: WeightState,
580        dtype: DType,
581        device: &Device,
582    ) -> Result<Self> {
583        config.validate()?;
584        let (in_features, out_features) = (shape.in_features, shape.out_features);
585        let mut budget = BitBudget::resolve(in_features, out_features, config, ratio_factor);
586
587        let load_tensor = |suffix: &str| -> Result<Tensor> {
588            let key = format!("{prefix}.{suffix}");
589            let t = tensors
590                .get(&key)
591                .ok_or_else(|| LittleBitError::MissingTensor(key.clone()))?;
592            let t = if t.device().same_device(device) {
593                t.clone()
594            } else {
595                t.to_device(device)?
596            };
597            if t.dtype() == dtype {
598                Ok(t)
599            } else {
600                Ok(t.to_dtype(dtype)?)
601            }
602        };
603
604        let u = load_tensor("U")?;
605        let v = load_tensor("V")?;
606        let actual_split =
607            u.dims()
608                .get(1)
609                .copied()
610                .ok_or_else(|| LittleBitError::InvalidShape {
611                    expected: "2D U tensor".to_string(),
612                    actual: format!("{:?}", u.dims()),
613                })?;
614        if actual_split != budget.split_dim {
615            budget.split_dim = actual_split;
616            budget.eff_bit_actual = crate::config::compute_eff_bits(
617                in_features,
618                out_features,
619                actual_split,
620                budget.residual,
621            );
622            budget.total_bit_usage =
623                budget.eff_bit_actual * (in_features as f64) * (out_features as f64);
624        }
625
626        let main = FactorBranch {
627            u: Param::Frozen(u),
628            v: Param::Frozen(v),
629            u1: Param::Frozen(load_tensor("u1")?),
630            u2: Param::Frozen(load_tensor("u2")?),
631            v1: Param::Frozen(load_tensor("v1")?),
632            v2: Param::Frozen(load_tensor("v2")?),
633        };
634
635        let residual = if budget.residual == ResidualMode::Dual {
636            Some(FactorBranch {
637                u: Param::Frozen(load_tensor("U_R")?),
638                v: Param::Frozen(load_tensor("V_R")?),
639                u1: Param::Frozen(load_tensor("u1_R")?),
640                u2: Param::Frozen(load_tensor("u2_R")?),
641                v1: Param::Frozen(load_tensor("v1_R")?),
642                v2: Param::Frozen(load_tensor("v2_R")?),
643            })
644        } else {
645            None
646        };
647
648        let bias = if shape.bias == BiasMode::WithBias {
649            let b_key = format!("{prefix}.bias");
650            if tensors.contains_key(&b_key) {
651                Some(Param::Frozen(load_tensor("bias")?))
652            } else {
653                None
654            }
655        } else {
656            None
657        };
658
659        Ok(Self {
660            in_features,
661            out_features,
662            budget,
663            quant_func: config.quant_func,
664            binarized: weight_state == WeightState::PreBinarized,
665            main,
666            residual,
667            bias,
668        })
669    }
670
671    /// Forward pass accepting any leading shape `[..., in_features]` and returning `[..., out_features]`.
672    ///
673    /// # Errors
674    /// Returns an error if input trailing dimension differs from `in_features` or tensor ops fail.
675    pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
676        let dims = x.dims();
677        if dims.is_empty() || dims[dims.len() - 1] != self.in_features {
678            return Err(LittleBitError::InvalidShape {
679                expected: format!("[..., {}]", self.in_features),
680                actual: format!("{dims:?}"),
681            });
682        }
683
684        let mut out_dims = dims[..dims.len() - 1].to_vec();
685        let batch_rows: usize = out_dims.iter().product::<usize>().max(1);
686        out_dims.push(self.out_features);
687
688        let x_2d = x.reshape((batch_rows, self.in_features))?;
689        let mut y = self.main.forward(&x_2d, self.quant_func, self.binarized)?;
690        if let Some(res) = &self.residual {
691            let r = res.forward(&x_2d, self.quant_func, self.binarized)?;
692            y = y.add(&r)?;
693        }
694        if let Some(b) = &self.bias {
695            let bt = b.as_tensor();
696            let bt = if bt.dtype() == y.dtype() {
697                bt.clone()
698            } else {
699                bt.to_dtype(y.dtype())?
700            };
701            y = y.broadcast_add(&bt)?;
702        }
703        Ok(y.reshape(out_dims)?)
704    }
705
706    /// Binarize and pack `U`, `V` (and `U_R`, `V_R` when dual) into upstream safetensors keys.
707    ///
708    /// # Errors
709    /// Returns an error if binarization or packing fails.
710    pub fn pack_state_dict(&self, prefix: &str) -> Result<HashMap<String, Tensor>> {
711        let mut out = HashMap::new();
712        let mut insert_branch = |branch: &FactorBranch, suffix: &str| -> Result<()> {
713            let (u_packed, u_shape) = binary_packer(branch.u.as_tensor())?;
714            let (v_packed, v_shape) = binary_packer(branch.v.as_tensor())?;
715            out.insert(format!("{prefix}.U{suffix}_packed"), u_packed);
716            out.insert(format!("{prefix}.U{suffix}_shape"), u_shape);
717            out.insert(format!("{prefix}.V{suffix}_packed"), v_packed);
718            out.insert(format!("{prefix}.V{suffix}_shape"), v_shape);
719            out.insert(
720                format!("{prefix}.u1{suffix}"),
721                branch.u1.as_tensor().detach(),
722            );
723            out.insert(
724                format!("{prefix}.u2{suffix}"),
725                branch.u2.as_tensor().detach(),
726            );
727            out.insert(
728                format!("{prefix}.v1{suffix}"),
729                branch.v1.as_tensor().detach(),
730            );
731            out.insert(
732                format!("{prefix}.v2{suffix}"),
733                branch.v2.as_tensor().detach(),
734            );
735            Ok(())
736        };
737
738        insert_branch(&self.main, "")?;
739        if let Some(res) = &self.residual {
740            insert_branch(res, "_R")?;
741        }
742        if let Some(b) = &self.bias {
743            out.insert(format!("{prefix}.bias"), b.as_tensor().detach());
744        }
745
746        let dev = self.main.u.as_tensor().device();
747        let target_val = self.budget.eff_bit_target.unwrap_or(-1.0) as f32;
748        out.insert(
749            format!("{prefix}._eff_bit_target"),
750            Tensor::from_slice(&[target_val], (), dev)?,
751        );
752        out.insert(
753            format!("{prefix}._split_dim_final"),
754            Tensor::from_slice(&[self.budget.split_dim as i64], (), dev)?,
755        );
756        out.insert(
757            format!("{prefix}._eff_bit_actual"),
758            Tensor::from_slice(&[self.budget.eff_bit_actual as f32], (), dev)?,
759        );
760        Ok(out)
761    }
762
763    /// Convert this layer into a memory-compact 1-bit [`PackedLittleBitLinear`] runtime module.
764    ///
765    /// # Errors
766    /// Returns an error if binarization or host packing fails.
767    pub fn to_packed(&self) -> Result<PackedLittleBitLinear> {
768        let pack_branch = |branch: &FactorBranch| -> Result<PackedFactorBranch> {
769            let (u_packed_t, _) = binary_packer(branch.u.as_tensor())?;
770            let (v_packed_t, _) = binary_packer(branch.v.as_tensor())?;
771            let u_words = u_packed_t
772                .to_device(&Device::Cpu)?
773                .flatten_all()?
774                .to_vec1::<i32>()?;
775            let v_words = v_packed_t
776                .to_device(&Device::Cpu)?
777                .flatten_all()?
778                .to_vec1::<i32>()?;
779            let u =
780                PackedBinaryMatrix::from_words(self.out_features, self.budget.split_dim, u_words)?;
781            let v =
782                PackedBinaryMatrix::from_words(self.budget.split_dim, self.in_features, v_words)?;
783            let to_vec_f32 = |t: &Tensor| -> Result<Vec<f32>> {
784                Ok(t.to_device(&Device::Cpu)?
785                    .to_dtype(DType::F32)?
786                    .flatten_all()?
787                    .to_vec1::<f32>()?)
788            };
789            let u1 = to_vec_f32(branch.u1.as_tensor())?;
790            let u2 = to_vec_f32(branch.u2.as_tensor())?;
791            let v1 = to_vec_f32(branch.v1.as_tensor())?;
792            let v2 = to_vec_f32(branch.v2.as_tensor())?;
793            let v1u2: Vec<f32> = v1.iter().zip(u2.iter()).map(|(&a, &b)| a * b).collect();
794            Ok(PackedFactorBranch {
795                u,
796                v,
797                u1,
798                u2,
799                v1,
800                v2,
801                v1u2,
802            })
803        };
804
805        let main = pack_branch(&self.main)?;
806        let residual = match &self.residual {
807            Some(r) => Some(pack_branch(r)?),
808            None => None,
809        };
810        let bias = match &self.bias {
811            Some(b) => Some(
812                b.as_tensor()
813                    .to_device(&Device::Cpu)?
814                    .to_dtype(DType::F32)?
815                    .flatten_all()?
816                    .to_vec1::<f32>()?,
817            ),
818            None => None,
819        };
820
821        Ok(PackedLittleBitLinear {
822            in_features: self.in_features,
823            out_features: self.out_features,
824            budget: self.budget,
825            main,
826            residual,
827            bias,
828        })
829    }
830
831    /// Input feature dimension.
832    #[must_use]
833    pub fn in_features(&self) -> usize {
834        self.in_features
835    }
836
837    /// Output feature dimension.
838    #[must_use]
839    pub fn out_features(&self) -> usize {
840        self.out_features
841    }
842
843    /// Resolved bit budget and latent rank geometry.
844    #[must_use]
845    pub fn budget(&self) -> &BitBudget {
846        &self.budget
847    }
848
849    /// Selected latent rank `split_dim`.
850    #[must_use]
851    pub fn split_dim(&self) -> usize {
852        self.budget.split_dim
853    }
854
855    /// Whether weights are already pre-binarized (`_binarized` inference mode).
856    #[must_use]
857    pub fn is_binarized(&self) -> bool {
858        self.binarized
859    }
860
861    /// Switch module to pre-binarized inference mode, converting `U` and `V` in place to `{-1, +1}`.
862    ///
863    /// # Errors
864    /// Returns an error if binarization fails.
865    pub fn binarize_in_place(&mut self) -> Result<()> {
866        let bin_branch = |branch: &mut FactorBranch| -> Result<()> {
867            branch.u = Param::Frozen(binarize_sign_no_grad(branch.u.as_tensor())?);
868            branch.v = Param::Frozen(binarize_sign_no_grad(branch.v.as_tensor())?);
869            branch.freeze();
870            Ok(())
871        };
872        bin_branch(&mut self.main)?;
873        if let Some(res) = &mut self.residual {
874            bin_branch(res)?;
875        }
876        if let Some(b) = &mut self.bias {
877            b.freeze();
878        }
879        self.binarized = true;
880        Ok(())
881    }
882
883    /// Collect all trainable [`Var`] parameters inside this layer.
884    #[must_use]
885    pub fn trainable_vars(&self) -> Vec<Var> {
886        let mut vars = Vec::new();
887        self.main.push_vars(&mut vars);
888        if let Some(res) = &self.residual {
889            res.push_vars(&mut vars);
890        }
891        if let Some(b) = &self.bias {
892            if let Some(v) = b.as_var() {
893                vars.push(v.clone());
894            }
895        }
896        vars
897    }
898
899    /// Freeze all trainable parameters in this layer.
900    pub fn freeze(&mut self) {
901        self.main.freeze();
902        if let Some(res) = &mut self.residual {
903            res.freeze();
904        }
905        if let Some(b) = &mut self.bias {
906            b.freeze();
907        }
908    }
909
910    /// Promote a finished/pre-binarized layer back into trainable latent QAT factors by scaling
911    /// binary signs `U`/`V` (and residual `U_R`/`V_R`) by `latent_mag` (`--warm_mag`), promoting
912    /// scales and bias to trainable `Var`s, and disabling `binarized` mode.
913    ///
914    /// Matches `littlebit-bench/patches/volta-fp16.patch::warm_start`.
915    ///
916    /// # Errors
917    /// Returns [`LittleBitError::InvalidConfig`] when `latent_mag` is not finite and positive, or
918    /// Candle tensor/variable errors if scaling or allocation fails.
919    pub fn warm_start_from_binarized(&mut self, latent_mag: f64) -> Result<()> {
920        if !latent_mag.is_finite() || latent_mag <= 0.0 {
921            return Err(LittleBitError::InvalidConfig(format!(
922                "warm_start latent_mag must be finite and positive, got {latent_mag}"
923            )));
924        }
925        self.main.warm_start(latent_mag)?;
926        if let Some(res) = &mut self.residual {
927            res.warm_start(latent_mag)?;
928        }
929        if let Some(b) = &mut self.bias {
930            *b = Param::Trainable(Var::from_tensor(&b.as_tensor().detach())?);
931        }
932        self.binarized = false;
933        Ok(())
934    }
935
936    /// Total scalar parameter elements stored in this layer.
937    #[must_use]
938    pub fn param_count(&self) -> usize {
939        self.main.elem_count()
940            + self.residual.as_ref().map_or(0, FactorBranch::elem_count)
941            + self.bias.as_ref().map_or(0, Param::elem_count)
942    }
943
944    /// Borrow the main latent branch.
945    #[must_use]
946    pub fn main_branch(&self) -> &FactorBranch {
947        &self.main
948    }
949
950    /// Borrow the optional residual latent branch.
951    #[must_use]
952    pub fn residual_branch(&self) -> Option<&FactorBranch> {
953        self.residual.as_ref()
954    }
955
956    /// Borrow the optional bias parameter.
957    #[must_use]
958    pub fn bias(&self) -> Option<&Param> {
959        self.bias.as_ref()
960    }
961
962    /// Format module summary string matching upstream `LittleBitLinear.extra_repr()`.
963    #[must_use]
964    pub fn extra_repr(&self) -> String {
965        let target = match self.budget.eff_bit_target {
966            Some(v) => format!("{v:.4}"),
967            None => "N/A".to_string(),
968        };
969        format!(
970            "in_features={}, out_features={}, bias={}, split_dim={}, eff_bit_target={}, eff_bit_actual={:.4}, residual={}, total_bit_usage={:.0}",
971            self.in_features,
972            self.out_features,
973            self.bias.is_some(),
974            self.budget.split_dim,
975            target,
976            self.budget.eff_bit_actual,
977            self.budget.residual == ResidualMode::Dual,
978            self.budget.total_bit_usage,
979        )
980    }
981}
982
983impl fmt::Display for LittleBitLinear {
984    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
985        write!(f, "LittleBitLinear({})", self.extra_repr())
986    }
987}
988
989#[derive(Debug, Clone)]
990struct PackedFactorBranch {
991    u: PackedBinaryMatrix,
992    v: PackedBinaryMatrix,
993    u1: Vec<f32>,
994    u2: Vec<f32>,
995    v1: Vec<f32>,
996    v2: Vec<f32>,
997    v1u2: Vec<f32>,
998}
999
1000impl PackedFactorBranch {
1001    fn forward_f32(&self, x: &[f32], batch_rows: usize, in_features: usize) -> Result<Vec<f32>> {
1002        let split_dim = self.v.rows();
1003        let out_features = self.u.rows();
1004
1005        let mut x_scaled = Vec::with_capacity(batch_rows * in_features);
1006        for row in x.chunks_exact(in_features) {
1007            for (&xv, &sv) in row.iter().zip(self.v2.iter()) {
1008                x_scaled.push(xv * sv);
1009            }
1010        }
1011
1012        let mut hidden = self.v.matmul_transposed_f32(&x_scaled, batch_rows)?;
1013        for h_row in hidden.chunks_exact_mut(split_dim) {
1014            for (hv, &sv) in h_row.iter_mut().zip(self.v1u2.iter()) {
1015                *hv *= sv;
1016            }
1017        }
1018
1019        let mut out = self.u.matmul_transposed_f32(&hidden, batch_rows)?;
1020        for o_row in out.chunks_exact_mut(out_features) {
1021            for (ov, &sv) in o_row.iter_mut().zip(self.u1.iter()) {
1022                *ov *= sv;
1023            }
1024        }
1025        Ok(out)
1026    }
1027
1028    fn memory_bytes(&self) -> usize {
1029        self.u.memory_bytes()
1030            + self.v.memory_bytes()
1031            + (self.u1.len() + self.u2.len() + self.v1.len() + self.v2.len() + self.v1u2.len())
1032                * std::mem::size_of::<f32>()
1033    }
1034}
1035
1036/// Memory-compact 1-bit inference representation keeping `U`, `V` packed as `i32` bitfields.
1037#[derive(Debug, Clone)]
1038pub struct PackedLittleBitLinear {
1039    in_features: usize,
1040    out_features: usize,
1041    budget: BitBudget,
1042    main: PackedFactorBranch,
1043    residual: Option<PackedFactorBranch>,
1044    bias: Option<Vec<f32>>,
1045}
1046
1047impl PackedLittleBitLinear {
1048    /// Construct directly from packed checkpoint tensors without allocating dense float matrices.
1049    ///
1050    /// # Errors
1051    /// Returns [`LittleBitError::MissingTensor`] or shape mismatch errors if packed entries
1052    /// are missing or invalid.
1053    pub fn from_packed_tensors(
1054        prefix: &str,
1055        tensors: &HashMap<String, Tensor>,
1056        shape: LayerShape,
1057        config: &LittleBitConfig,
1058        ratio_factor: f64,
1059    ) -> Result<Self> {
1060        config.validate()?;
1061        let (in_features, out_features) = (shape.in_features, shape.out_features);
1062        let mut budget = BitBudget::resolve(in_features, out_features, config, ratio_factor);
1063
1064        let load_vec_f32 = |suffix: &str| -> Result<Vec<f32>> {
1065            let key = format!("{prefix}.{suffix}");
1066            let t = tensors
1067                .get(&key)
1068                .ok_or_else(|| LittleBitError::MissingTensor(key.clone()))?;
1069            Ok(t.to_device(&Device::Cpu)?
1070                .to_dtype(DType::F32)?
1071                .flatten_all()?
1072                .to_vec1::<f32>()?)
1073        };
1074
1075        let load_packed = |name: &str| -> Result<(usize, usize, Vec<i32>)> {
1076            let p_key = format!("{prefix}.{name}_packed");
1077            let s_key = format!("{prefix}.{name}_shape");
1078            let packed_t = tensors
1079                .get(&p_key)
1080                .ok_or_else(|| LittleBitError::MissingTensor(p_key.clone()))?;
1081            let shape_t = tensors
1082                .get(&s_key)
1083                .ok_or_else(|| LittleBitError::MissingTensor(s_key.clone()))?;
1084            let shape_vals = shape_t
1085                .to_device(&Device::Cpu)?
1086                .to_dtype(DType::I64)?
1087                .flatten_all()?
1088                .to_vec1::<i64>()?;
1089            if shape_vals.len() != 2 || shape_vals[0] < 0 || shape_vals[1] < 0 {
1090                return Err(LittleBitError::InvalidShape {
1091                    expected: "non-negative [rows, cols]".to_string(),
1092                    actual: format!("{shape_vals:?}"),
1093                });
1094            }
1095            let (rows, cols) = (shape_vals[0] as usize, shape_vals[1] as usize);
1096            let words = packed_t
1097                .to_device(&Device::Cpu)?
1098                .flatten_all()?
1099                .to_vec1::<i32>()?;
1100            Ok((rows, cols, words))
1101        };
1102
1103        let load_branch = |suffix: &str| -> Result<(usize, PackedFactorBranch)> {
1104            let (u_rows, u_cols, u_words) = load_packed(&format!("U{suffix}"))?;
1105            let (v_rows, v_cols, v_words) = load_packed(&format!("V{suffix}"))?;
1106            if u_rows != out_features || v_cols != in_features || u_cols != v_rows {
1107                return Err(LittleBitError::InvalidShape {
1108                    expected: format!("U=({out_features}, s), V=(s, {in_features})"),
1109                    actual: format!("U=({u_rows}, {u_cols}), V=({v_rows}, {v_cols})"),
1110                });
1111            }
1112            let u = PackedBinaryMatrix::from_words(u_rows, u_cols, u_words)?;
1113            let v = PackedBinaryMatrix::from_words(v_rows, v_cols, v_words)?;
1114            let u1 = load_vec_f32(&format!("u1{suffix}"))?;
1115            let u2 = load_vec_f32(&format!("u2{suffix}"))?;
1116            let v1 = load_vec_f32(&format!("v1{suffix}"))?;
1117            let v2 = load_vec_f32(&format!("v2{suffix}"))?;
1118            let v1u2: Vec<f32> = v1.iter().zip(u2.iter()).map(|(&a, &b)| a * b).collect();
1119            Ok((
1120                u_cols,
1121                PackedFactorBranch {
1122                    u,
1123                    v,
1124                    u1,
1125                    u2,
1126                    v1,
1127                    v2,
1128                    v1u2,
1129                },
1130            ))
1131        };
1132
1133        let (actual_split, main) = load_branch("")?;
1134        if actual_split != budget.split_dim {
1135            budget.split_dim = actual_split;
1136            budget.eff_bit_actual = crate::config::compute_eff_bits(
1137                in_features,
1138                out_features,
1139                actual_split,
1140                budget.residual,
1141            );
1142            budget.total_bit_usage =
1143                budget.eff_bit_actual * (in_features as f64) * (out_features as f64);
1144        }
1145
1146        let residual = if budget.residual == ResidualMode::Dual {
1147            let (_, res_branch) = load_branch("_R")?;
1148            Some(res_branch)
1149        } else {
1150            None
1151        };
1152
1153        let bias = if shape.bias == BiasMode::WithBias
1154            && tensors.contains_key(&format!("{prefix}.bias"))
1155        {
1156            Some(load_vec_f32("bias")?)
1157        } else {
1158            None
1159        };
1160
1161        Ok(Self {
1162            in_features,
1163            out_features,
1164            budget,
1165            main,
1166            residual,
1167            bias,
1168        })
1169    }
1170
1171    /// Execute direct 1-bit packed forward pass and return result on `x`'s device and dtype.
1172    ///
1173    /// # Errors
1174    /// Returns an error if input dimensions do not match `in_features` or tensor transfer fails.
1175    pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
1176        let dims = x.dims();
1177        if dims.is_empty() || dims[dims.len() - 1] != self.in_features {
1178            return Err(LittleBitError::InvalidShape {
1179                expected: format!("[..., {}]", self.in_features),
1180                actual: format!("{dims:?}"),
1181            });
1182        }
1183        let mut out_dims = dims[..dims.len() - 1].to_vec();
1184        let batch_rows: usize = out_dims.iter().product::<usize>().max(1);
1185        out_dims.push(self.out_features);
1186
1187        let x_flat = x
1188            .to_device(&Device::Cpu)?
1189            .to_dtype(DType::F32)?
1190            .reshape(batch_rows * self.in_features)?
1191            .to_vec1::<f32>()?;
1192
1193        let mut out = self
1194            .main
1195            .forward_f32(&x_flat, batch_rows, self.in_features)?;
1196        if let Some(res) = &self.residual {
1197            let r = res.forward_f32(&x_flat, batch_rows, self.in_features)?;
1198            for (dst, rv) in out.iter_mut().zip(r.iter()) {
1199                *dst += *rv;
1200            }
1201        }
1202        if let Some(bias) = &self.bias {
1203            for row in out.chunks_exact_mut(self.out_features) {
1204                for (dst, &bv) in row.iter_mut().zip(bias.iter()) {
1205                    *dst += bv;
1206                }
1207            }
1208        }
1209
1210        let out_t = Tensor::from_vec(out, out_dims, x.device())?;
1211        if x.dtype() == DType::F32 {
1212            Ok(out_t)
1213        } else {
1214            Ok(out_t.to_dtype(x.dtype())?)
1215        }
1216    }
1217
1218    /// Export packed tensors for checkpoint serialization.
1219    ///
1220    /// # Errors
1221    /// Returns an error if tensor creation fails.
1222    pub fn pack_state_dict(
1223        &self,
1224        prefix: &str,
1225        device: &Device,
1226    ) -> Result<HashMap<String, Tensor>> {
1227        let mut out = HashMap::new();
1228        let mut insert_branch = |branch: &PackedFactorBranch, suffix: &str| -> Result<()> {
1229            let (u_packed, u_shape) = branch.u.to_tensors(device)?;
1230            let (v_packed, v_shape) = branch.v.to_tensors(device)?;
1231            out.insert(format!("{prefix}.U{suffix}_packed"), u_packed);
1232            out.insert(format!("{prefix}.U{suffix}_shape"), u_shape);
1233            out.insert(format!("{prefix}.V{suffix}_packed"), v_packed);
1234            out.insert(format!("{prefix}.V{suffix}_shape"), v_shape);
1235            out.insert(
1236                format!("{prefix}.u1{suffix}"),
1237                Tensor::from_slice(&branch.u1, (1, self.out_features), device)?,
1238            );
1239            out.insert(
1240                format!("{prefix}.u2{suffix}"),
1241                Tensor::from_slice(&branch.u2, (1, self.budget.split_dim), device)?,
1242            );
1243            out.insert(
1244                format!("{prefix}.v1{suffix}"),
1245                Tensor::from_slice(&branch.v1, (1, self.budget.split_dim), device)?,
1246            );
1247            out.insert(
1248                format!("{prefix}.v2{suffix}"),
1249                Tensor::from_slice(&branch.v2, (1, self.in_features), device)?,
1250            );
1251            Ok(())
1252        };
1253
1254        insert_branch(&self.main, "")?;
1255        if let Some(res) = &self.residual {
1256            insert_branch(res, "_R")?;
1257        }
1258        if let Some(b) = &self.bias {
1259            out.insert(
1260                format!("{prefix}.bias"),
1261                Tensor::from_slice(b, self.out_features, device)?,
1262            );
1263        }
1264        let target_val = self.budget.eff_bit_target.unwrap_or(-1.0) as f32;
1265        out.insert(
1266            format!("{prefix}._eff_bit_target"),
1267            Tensor::from_slice(&[target_val], (), device)?,
1268        );
1269        out.insert(
1270            format!("{prefix}._split_dim_final"),
1271            Tensor::from_slice(&[self.budget.split_dim as i64], (), device)?,
1272        );
1273        out.insert(
1274            format!("{prefix}._eff_bit_actual"),
1275            Tensor::from_slice(&[self.budget.eff_bit_actual as f32], (), device)?,
1276        );
1277        Ok(out)
1278    }
1279
1280    /// Total heap memory footprint in bytes for the 1-bit packed layer.
1281    #[must_use]
1282    pub fn memory_bytes(&self) -> usize {
1283        self.main.memory_bytes()
1284            + self
1285                .residual
1286                .as_ref()
1287                .map_or(0, PackedFactorBranch::memory_bytes)
1288            + self
1289                .bias
1290                .as_ref()
1291                .map_or(0, |b| b.len() * std::mem::size_of::<f32>())
1292    }
1293
1294    /// Resolved bit budget and rank geometry.
1295    #[must_use]
1296    pub fn budget(&self) -> &BitBudget {
1297        &self.budget
1298    }
1299}
1300
1301/// Reconstruct dense approximated weight `(out_features, in_features)` from `U` and `V` factors
1302/// for diagnostics and error evaluation.
1303///
1304/// # Errors
1305/// Returns an error on shape mismatch.
1306pub fn reconstruct_branch_matrix(
1307    u: &Tensor,
1308    v: &Tensor,
1309    u1: &Tensor,
1310    u2: &Tensor,
1311    v1: &Tensor,
1312    v2: &Tensor,
1313    quant_func: QuantFunc,
1314) -> Result<Tensor> {
1315    let uq = quant_func.apply(&u.to_dtype(DType::F32)?)?;
1316    let vq = quant_func.apply(&v.to_dtype(DType::F32)?)?;
1317    let u_scale = u1
1318        .to_dtype(DType::F32)?
1319        .t()?
1320        .matmul(&u2.to_dtype(DType::F32)?)?;
1321    let v_scale = v1
1322        .to_dtype(DType::F32)?
1323        .t()?
1324        .matmul(&v2.to_dtype(DType::F32)?)?;
1325    Ok(uq.mul(&u_scale)?.matmul(&vq.mul(&v_scale)?)?)
1326}
1327
1328#[cfg(test)]
1329mod tests {
1330    use super::*;
1331    use rand::SeedableRng;
1332    use rand_chacha::ChaCha8Rng;
1333    use rand_distr::{Distribution, StandardNormal};
1334
1335    fn random_tensor(rows: usize, cols: usize, seed: u64) -> Result<Tensor> {
1336        let mut rng = ChaCha8Rng::seed_from_u64(seed);
1337        let data: Vec<f32> = (0..rows * cols)
1338            .map(|_| StandardNormal.sample(&mut rng))
1339            .collect();
1340        Ok(Tensor::from_vec(data, (rows, cols), &Device::Cpu)?)
1341    }
1342
1343    #[test]
1344    fn forward_matches_packed_and_direct_matrix_product() -> Result<()> {
1345        let out_f = 48;
1346        let in_f = 64;
1347        let w = random_tensor(out_f, in_f, 7)?;
1348        let bias = Tensor::from_vec(
1349            (0..out_f).map(|i| (i as f32) * 0.01 - 0.2).collect(),
1350            out_f,
1351            &Device::Cpu,
1352        )?;
1353        let cfg = LittleBitConfig {
1354            quant_func: QuantFunc::SmoothSign { alpha: 100.0 },
1355            eff_bit: Some(0.5),
1356            residual: true,
1357            use_itq: true,
1358            itq_n_iter: 20,
1359            ..Default::default()
1360        };
1361
1362        let layer =
1363            LittleBitLinear::from_dense_weight(&w, Some(&bias), &cfg, 1.0, TrainMode::Eval, 11)?;
1364        let packed = layer.to_packed()?;
1365
1366        assert!(packed.memory_bytes() < out_f * in_f * 4);
1367
1368        let x = random_tensor(6, in_f, 99)?.reshape((2, 3, in_f))?;
1369        let y_layer = layer.forward(&x)?;
1370        let y_packed = packed.forward(&x)?;
1371        assert_eq!(y_layer.dims(), &[2, 3, out_f]);
1372        assert_eq!(y_packed.dims(), &[2, 3, out_f]);
1373
1374        // Compare against explicit matrix reconstruction W_hat = W_main + W_res.
1375        let main = layer.main_branch();
1376        let res = layer.residual_branch().expect("residual branch");
1377        let w_main = reconstruct_branch_matrix(
1378            main.u.as_tensor(),
1379            main.v.as_tensor(),
1380            main.u1.as_tensor(),
1381            main.u2.as_tensor(),
1382            main.v1.as_tensor(),
1383            main.v2.as_tensor(),
1384            cfg.quant_func,
1385        )?;
1386        let w_res = reconstruct_branch_matrix(
1387            res.u.as_tensor(),
1388            res.v.as_tensor(),
1389            res.u1.as_tensor(),
1390            res.u2.as_tensor(),
1391            res.v1.as_tensor(),
1392            res.v2.as_tensor(),
1393            cfg.quant_func,
1394        )?;
1395        let w_hat = w_main.add(&w_res)?;
1396        let y_ref = x
1397            .reshape((6, in_f))?
1398            .matmul(&w_hat.t()?)?
1399            .broadcast_add(&bias)?
1400            .reshape((2, 3, out_f))?;
1401
1402        let diff_packed = y_layer
1403            .sub(&y_packed)?
1404            .abs()?
1405            .max_all()?
1406            .to_scalar::<f32>()?;
1407        let diff_ref = y_layer.sub(&y_ref)?.abs()?.max_all()?.to_scalar::<f32>()?;
1408        assert!(diff_packed < 1e-4, "packed drift: {diff_packed}");
1409        assert!(diff_ref < 1e-4, "reference drift: {diff_ref}");
1410        Ok(())
1411    }
1412
1413    #[test]
1414    fn autograd_flows_through_all_factors_for_both_quant_funcs() -> Result<()> {
1415        for qf in [QuantFunc::SteBinary, QuantFunc::SmoothSign { alpha: 50.0 }] {
1416            let w = random_tensor(32, 32, 19)?;
1417            let bias = Tensor::zeros(32, DType::F32, &Device::Cpu)?;
1418            let cfg = LittleBitConfig {
1419                quant_func: qf,
1420                eff_bit: None,
1421                split_dim: 8,
1422                residual: true,
1423                use_itq: true,
1424                itq_n_iter: 10,
1425                ..Default::default()
1426            };
1427            let layer = LittleBitLinear::from_dense_weight(
1428                &w,
1429                Some(&bias),
1430                &cfg,
1431                1.0,
1432                TrainMode::Train,
1433                3,
1434            )?;
1435            let x = random_tensor(4, 32, 23)?;
1436            let loss = layer.forward(&x)?.sqr()?.mean_all()?;
1437            let grads = loss.backward()?;
1438
1439            let vars = layer.trainable_vars();
1440            assert_eq!(vars.len(), 13);
1441            for var in vars {
1442                let g = grads
1443                    .get(var.as_tensor())
1444                    .expect("every factor var must receive a gradient");
1445                let norm = g.sqr()?.sum_all()?.to_scalar::<f32>()?;
1446                assert!(norm.is_finite() && norm > 0.0, "zero or NaN grad: {norm}");
1447            }
1448        }
1449        Ok(())
1450    }
1451}