1use std::collections::HashMap;
4use std::fmt;
5
6use candle_core::{DType, Device, Tensor, Var};
7use faer::Mat;
8
9use crate::binary::binarize_sign_no_grad;
10use crate::config::{
11 BiasMode, BitBudget, LayerShape, LittleBitConfig, QuantFunc, ResidualMode, TrainMode,
12 WeightState,
13};
14use crate::error::{LittleBitError, Result};
15use crate::itq::apply_joint_itq;
16use crate::packer::{binary_packer, PackedBinaryMatrix};
17use crate::svd::{rank_one_decompose, svd_lowrank, DEFAULT_LOWRANK_NITER};
18
19#[derive(Debug, Clone)]
21pub enum Param {
22 Trainable(Var),
24 Frozen(Tensor),
26}
27
28impl Param {
29 pub fn from_tensor(tensor: Tensor, mode: TrainMode) -> Result<Self> {
34 match mode {
35 TrainMode::Train => Ok(Self::Trainable(Var::from_tensor(&tensor)?)),
36 TrainMode::Eval => Ok(Self::Frozen(tensor.detach())),
37 }
38 }
39
40 #[must_use]
42 pub fn as_tensor(&self) -> &Tensor {
43 match self {
44 Self::Trainable(v) => v.as_tensor(),
45 Self::Frozen(t) => t,
46 }
47 }
48
49 #[must_use]
51 pub fn as_var(&self) -> Option<&Var> {
52 match self {
53 Self::Trainable(v) => Some(v),
54 Self::Frozen(_) => None,
55 }
56 }
57
58 pub fn freeze(&mut self) {
60 if let Self::Trainable(v) = self {
61 *self = Self::Frozen(v.as_tensor().detach());
62 }
63 }
64
65 #[must_use]
67 pub fn is_trainable(&self) -> bool {
68 matches!(self, Self::Trainable(_))
69 }
70
71 #[must_use]
73 pub fn elem_count(&self) -> usize {
74 self.as_tensor().elem_count()
75 }
76}
77
78#[derive(Debug, Clone)]
80pub struct FactorBranch {
81 pub u: Param,
83 pub v: Param,
85 pub u1: Param,
87 pub u2: Param,
89 pub v1: Param,
91 pub v2: Param,
93}
94
95pub const FP16_MAX_SAFE: f64 = 65_504.0;
97
98#[inline]
101#[must_use]
102pub fn pow2_at_least(val: f64) -> f64 {
103 if !val.is_finite() || val <= 1.0 {
104 1.0
105 } else {
106 2.0_f64.powf(val.log2().ceil())
107 }
108}
109
110impl FactorBranch {
111 fn from_raw(
112 raw: RawDecomposedBranch,
113 dtype: DType,
114 device: &Device,
115 mode: TrainMode,
116 out_features: usize,
117 in_features: usize,
118 split_dim: usize,
119 ) -> Result<Self> {
120 let make = |data: Vec<f32>, shape: (usize, usize)| -> Result<Param> {
121 let t = Tensor::from_vec(data, shape, device)?;
122 let t = if dtype == DType::F32 {
123 t
124 } else {
125 t.to_dtype(dtype)?
126 };
127 Param::from_tensor(t, mode)
128 };
129 Ok(Self {
130 u: make(raw.u, (out_features, split_dim))?,
131 v: make(raw.v, (split_dim, in_features))?,
132 u1: make(raw.u1, (1, out_features))?,
133 u2: make(raw.u2, (1, split_dim))?,
134 v1: make(raw.v1, (1, split_dim))?,
135 v2: make(raw.v2, (1, in_features))?,
136 })
137 }
138
139 fn empty(
140 out_features: usize,
141 in_features: usize,
142 split_dim: usize,
143 dtype: DType,
144 device: &Device,
145 mode: TrainMode,
146 ) -> Result<Self> {
147 let make = |shape: (usize, usize)| -> Result<Param> {
148 let t = Tensor::zeros(shape, dtype, device)?;
149 Param::from_tensor(t, mode)
150 };
151 Ok(Self {
152 u: make((out_features, split_dim))?,
153 v: make((split_dim, in_features))?,
154 u1: make((1, out_features))?,
155 u2: make((1, split_dim))?,
156 v1: make((1, split_dim))?,
157 v2: make((1, in_features))?,
158 })
159 }
160
161 fn forward(&self, x_2d: &Tensor, quant_func: QuantFunc, binarized: bool) -> Result<Tensor> {
162 let x_dtype = x_2d.dtype();
163 let cast = |t: &Tensor| -> Result<Tensor> {
164 if t.dtype() == x_dtype {
165 Ok(t.clone())
166 } else {
167 Ok(t.to_dtype(x_dtype)?)
168 }
169 };
170
171 let v = cast(self.v.as_tensor())?;
172 let u = cast(self.u.as_tensor())?;
173 let vq = if binarized { v } else { quant_func.apply(&v)? };
174 let uq = if binarized { u } else { quant_func.apply(&u)? };
175
176 let vq_t = vq.t()?.contiguous()?;
177 let uq_t = uq.t()?.contiguous()?;
178
179 if x_dtype == DType::F16 {
180 let x_f32 = x_2d.to_dtype(DType::F32)?;
181 let v2_f32 = self.v2.as_tensor().to_dtype(DType::F32)?;
182 let v1_f32 = self.v1.as_tensor().to_dtype(DType::F32)?;
183 let u2_f32 = self.u2.as_tensor().to_dtype(DType::F32)?;
184 let u1_f32 = self.u1.as_tensor().to_dtype(DType::F32)?;
185 let v1u2_f32 = v1_f32.mul(&u2_f32)?;
186
187 if x_2d.device().is_cpu() {
188 let vq_t_f32 = vq_t.to_dtype(DType::F32)?;
189 let uq_t_f32 = uq_t.to_dtype(DType::F32)?;
190 let h = x_f32.broadcast_mul(&v2_f32)?.matmul(&vq_t_f32)?;
191 let g = h.broadcast_mul(&v1u2_f32)?;
192 let out = g
193 .matmul(&uq_t_f32)?
194 .broadcast_mul(&u1_f32)?
195 .to_dtype(DType::F16)?;
196 return Ok(out);
197 }
198
199 let a = x_f32.broadcast_mul(&v2_f32)?;
200 let a_max = f64::from(a.abs()?.max_all()?.to_scalar::<f32>()?);
201 let in_features = vq_t.dims()[0] as f64;
202 let s1 = pow2_at_least(a_max * in_features / FP16_MAX_SAFE);
203 let a_scaled = if s1 > 1.0 { (a / s1)? } else { a };
204
205 let h = a_scaled
206 .to_dtype(DType::F16)?
207 .matmul(&vq_t)?
208 .to_dtype(DType::F32)?;
209 let g = h.broadcast_mul(&v1u2_f32)?;
210 let g_max = f64::from(g.abs()?.max_all()?.to_scalar::<f32>()?);
211 let split_dim = uq_t.dims()[0] as f64;
212 let s2 = pow2_at_least(g_max * split_dim / FP16_MAX_SAFE);
213 let g_scaled = if s2 > 1.0 { (g / s2)? } else { g };
214
215 let y = g_scaled
216 .to_dtype(DType::F16)?
217 .matmul(&uq_t)?
218 .to_dtype(DType::F32)?;
219 let total_scale = s1 * s2;
220 let u1_scaled = if total_scale > 1.0 {
221 (u1_f32 * total_scale)?
222 } else {
223 u1_f32
224 };
225 let out = y.broadcast_mul(&u1_scaled)?.to_dtype(DType::F16)?;
226 return Ok(out);
227 }
228
229 let v2 = cast(self.v2.as_tensor())?;
230 let v1 = cast(self.v1.as_tensor())?;
231 let u2 = cast(self.u2.as_tensor())?;
232 let u1 = cast(self.u1.as_tensor())?;
233
234 let v1u2 = v1.mul(&u2)?;
235
236 let h = x_2d.broadcast_mul(&v2)?.matmul(&vq_t)?;
238 let h = h.broadcast_mul(&v1u2)?;
239 let out = h.matmul(&uq_t)?.broadcast_mul(&u1)?;
240 Ok(out)
241 }
242
243 fn push_vars(&self, vars: &mut Vec<Var>) {
244 for p in [&self.u, &self.v, &self.u1, &self.u2, &self.v1, &self.v2] {
245 if let Some(v) = p.as_var() {
246 vars.push(v.clone());
247 }
248 }
249 }
250
251 fn freeze(&mut self) {
252 self.u.freeze();
253 self.v.freeze();
254 self.u1.freeze();
255 self.u2.freeze();
256 self.v1.freeze();
257 self.v2.freeze();
258 }
259
260 fn warm_start(&mut self, latent_mag: f64) -> Result<()> {
261 let u_scaled = (self.u.as_tensor().detach() * latent_mag)?;
262 let v_scaled = (self.v.as_tensor().detach() * latent_mag)?;
263 self.u = Param::Trainable(Var::from_tensor(&u_scaled)?);
264 self.v = Param::Trainable(Var::from_tensor(&v_scaled)?);
265 self.u1 = Param::Trainable(Var::from_tensor(&self.u1.as_tensor().detach())?);
266 self.u2 = Param::Trainable(Var::from_tensor(&self.u2.as_tensor().detach())?);
267 self.v1 = Param::Trainable(Var::from_tensor(&self.v1.as_tensor().detach())?);
268 self.v2 = Param::Trainable(Var::from_tensor(&self.v2.as_tensor().detach())?);
269 Ok(())
270 }
271
272 fn elem_count(&self) -> usize {
273 self.u.elem_count()
274 + self.v.elem_count()
275 + self.u1.elem_count()
276 + self.u2.elem_count()
277 + self.v1.elem_count()
278 + self.v2.elem_count()
279 }
280}
281
282struct RawDecomposedBranch {
283 u: Vec<f32>,
284 v: Vec<f32>,
285 u1: Vec<f32>,
286 u2: Vec<f32>,
287 v1: Vec<f32>,
288 v2: Vec<f32>,
289}
290
291fn decompose_matrix_host(
292 weight_f32: &[f32],
293 out_features: usize,
294 in_features: usize,
295 split_dim: usize,
296 config: &LittleBitConfig,
297 seed: u64,
298) -> Result<RawDecomposedBranch> {
299 let svd = svd_lowrank(
300 weight_f32,
301 out_features,
302 in_features,
303 split_dim,
304 DEFAULT_LOWRANK_NITER,
305 seed,
306 )?;
307
308 let r = svd.rank;
309 let sqrt_s: Vec<f32> = svd.s.iter().map(|&s| s.max(0.0).sqrt()).collect();
310
311 let mut u = vec![0.0f32; out_features * split_dim];
313 for row in 0..out_features {
314 for k in 0..r {
315 u[row * split_dim + k] = svd.u[row * r + k] * sqrt_s[k];
316 }
317 }
318
319 let mut v = vec![0.0f32; split_dim * in_features];
321 for k in 0..r {
322 let s_root = sqrt_s[k];
323 for col in 0..in_features {
324 v[k * in_features + col] = s_root * svd.vt[k * in_features + col];
325 }
326 }
327
328 if config.use_itq {
329 apply_joint_itq(
330 &mut u,
331 &mut v,
332 out_features,
333 in_features,
334 split_dim,
335 config.itq_n_iter,
336 seed.wrapping_add(1),
337 )?;
338 }
339
340 let abs_v: Vec<f32> = v.iter().map(|&x| x.abs()).collect();
341 let (v1, v2) = rank_one_decompose(&abs_v, split_dim, in_features, seed.wrapping_add(2))?;
342
343 let abs_u: Vec<f32> = u.iter().map(|&x| x.abs()).collect();
344 let (u1, u2) = rank_one_decompose(&abs_u, out_features, split_dim, seed.wrapping_add(3))?;
345
346 Ok(RawDecomposedBranch {
347 u,
348 v,
349 u1,
350 u2,
351 v1,
352 v2,
353 })
354}
355
356fn compute_residual_matrix_host(
357 orig_w: &[f32],
358 branch: &RawDecomposedBranch,
359 out_features: usize,
360 in_features: usize,
361 split_dim: usize,
362) -> Vec<f32> {
363 let u_scaled = Mat::from_fn(out_features, split_dim, |r, k| {
364 let sign = if branch.u[r * split_dim + k] < 0.0 {
365 -1.0f32
366 } else {
367 1.0f32
368 };
369 sign * (branch.u1[r] * branch.u2[k])
370 });
371 let v_scaled = Mat::from_fn(split_dim, in_features, |k, c| {
372 let sign = if branch.v[k * in_features + c] < 0.0 {
373 -1.0f32
374 } else {
375 1.0f32
376 };
377 sign * (branch.v1[k] * branch.v2[c])
378 });
379 let approx = &u_scaled * &v_scaled;
380 let mut residual = Vec::with_capacity(orig_w.len());
381 for r in 0..out_features {
382 for c in 0..in_features {
383 residual.push(orig_w[r * in_features + c] - approx[(r, c)]);
384 }
385 }
386 residual
387}
388
389#[derive(Debug, Clone)]
391pub struct LittleBitLinear {
392 in_features: usize,
393 out_features: usize,
394 budget: BitBudget,
395 quant_func: QuantFunc,
396 binarized: bool,
397 main: FactorBranch,
398 residual: Option<FactorBranch>,
399 bias: Option<Param>,
400}
401
402impl LittleBitLinear {
403 pub fn from_dense_weight(
409 weight: &Tensor,
410 bias: Option<&Tensor>,
411 config: &LittleBitConfig,
412 ratio_factor: f64,
413 train_mode: TrainMode,
414 seed: u64,
415 ) -> Result<Self> {
416 config.validate()?;
417 let dims = weight.dims();
418 if dims.len() != 2 {
419 return Err(LittleBitError::InvalidShape {
420 expected: "2D weight (out_features, in_features)".to_string(),
421 actual: format!("{dims:?}"),
422 });
423 }
424 let (out_features, in_features) = (dims[0], dims[1]);
425 if let Some(b) = bias {
426 if b.dims() != [out_features] {
427 return Err(LittleBitError::InvalidShape {
428 expected: format!("[{out_features}]"),
429 actual: format!("{:?}", b.dims()),
430 });
431 }
432 }
433
434 let target_dtype = weight.dtype();
435 let target_device = weight.device().clone();
436 let budget = BitBudget::resolve(in_features, out_features, config, ratio_factor);
437
438 let weight_f32 = weight
439 .to_device(&Device::Cpu)?
440 .to_dtype(DType::F32)?
441 .reshape(out_features * in_features)?
442 .to_vec1::<f32>()?;
443
444 let main_raw = decompose_matrix_host(
445 &weight_f32,
446 out_features,
447 in_features,
448 budget.split_dim,
449 config,
450 seed,
451 )?;
452
453 let residual_branch = if budget.residual == ResidualMode::Dual {
454 let residual_w = compute_residual_matrix_host(
455 &weight_f32,
456 &main_raw,
457 out_features,
458 in_features,
459 budget.split_dim,
460 );
461 let res_raw = decompose_matrix_host(
462 &residual_w,
463 out_features,
464 in_features,
465 budget.split_dim,
466 config,
467 seed.wrapping_add(10_000),
468 )?;
469 Some(FactorBranch::from_raw(
470 res_raw,
471 target_dtype,
472 &target_device,
473 train_mode,
474 out_features,
475 in_features,
476 budget.split_dim,
477 )?)
478 } else {
479 None
480 };
481
482 let main = FactorBranch::from_raw(
483 main_raw,
484 target_dtype,
485 &target_device,
486 train_mode,
487 out_features,
488 in_features,
489 budget.split_dim,
490 )?;
491
492 let bias_param = match bias {
493 Some(b) => Some(Param::from_tensor(
494 b.to_device(&target_device)?.to_dtype(target_dtype)?,
495 train_mode,
496 )?),
497 None => None,
498 };
499
500 Ok(Self {
501 in_features,
502 out_features,
503 budget,
504 quant_func: config.quant_func,
505 binarized: false,
506 main,
507 residual: residual_branch,
508 bias: bias_param,
509 })
510 }
511
512 pub fn empty(
517 shape: LayerShape,
518 config: &LittleBitConfig,
519 ratio_factor: f64,
520 train_mode: TrainMode,
521 dtype: DType,
522 device: &Device,
523 ) -> Result<Self> {
524 config.validate()?;
525 let (in_features, out_features) = (shape.in_features, shape.out_features);
526 let budget = BitBudget::resolve(in_features, out_features, config, ratio_factor);
527 let main = FactorBranch::empty(
528 out_features,
529 in_features,
530 budget.split_dim,
531 dtype,
532 device,
533 train_mode,
534 )?;
535 let residual = if budget.residual == ResidualMode::Dual {
536 Some(FactorBranch::empty(
537 out_features,
538 in_features,
539 budget.split_dim,
540 dtype,
541 device,
542 train_mode,
543 )?)
544 } else {
545 None
546 };
547 let bias = if shape.bias == BiasMode::WithBias {
548 Some(Param::from_tensor(
549 Tensor::zeros(out_features, dtype, device)?,
550 train_mode,
551 )?)
552 } else {
553 None
554 };
555 Ok(Self {
556 in_features,
557 out_features,
558 budget,
559 quant_func: config.quant_func,
560 binarized: false,
561 main,
562 residual,
563 bias,
564 })
565 }
566
567 #[allow(clippy::too_many_arguments)]
573 pub fn from_state_dict(
574 prefix: &str,
575 tensors: &HashMap<String, Tensor>,
576 shape: LayerShape,
577 config: &LittleBitConfig,
578 ratio_factor: f64,
579 weight_state: WeightState,
580 dtype: DType,
581 device: &Device,
582 ) -> Result<Self> {
583 config.validate()?;
584 let (in_features, out_features) = (shape.in_features, shape.out_features);
585 let mut budget = BitBudget::resolve(in_features, out_features, config, ratio_factor);
586
587 let load_tensor = |suffix: &str| -> Result<Tensor> {
588 let key = format!("{prefix}.{suffix}");
589 let t = tensors
590 .get(&key)
591 .ok_or_else(|| LittleBitError::MissingTensor(key.clone()))?;
592 let t = if t.device().same_device(device) {
593 t.clone()
594 } else {
595 t.to_device(device)?
596 };
597 if t.dtype() == dtype {
598 Ok(t)
599 } else {
600 Ok(t.to_dtype(dtype)?)
601 }
602 };
603
604 let u = load_tensor("U")?;
605 let v = load_tensor("V")?;
606 let actual_split =
607 u.dims()
608 .get(1)
609 .copied()
610 .ok_or_else(|| LittleBitError::InvalidShape {
611 expected: "2D U tensor".to_string(),
612 actual: format!("{:?}", u.dims()),
613 })?;
614 if actual_split != budget.split_dim {
615 budget.split_dim = actual_split;
616 budget.eff_bit_actual = crate::config::compute_eff_bits(
617 in_features,
618 out_features,
619 actual_split,
620 budget.residual,
621 );
622 budget.total_bit_usage =
623 budget.eff_bit_actual * (in_features as f64) * (out_features as f64);
624 }
625
626 let main = FactorBranch {
627 u: Param::Frozen(u),
628 v: Param::Frozen(v),
629 u1: Param::Frozen(load_tensor("u1")?),
630 u2: Param::Frozen(load_tensor("u2")?),
631 v1: Param::Frozen(load_tensor("v1")?),
632 v2: Param::Frozen(load_tensor("v2")?),
633 };
634
635 let residual = if budget.residual == ResidualMode::Dual {
636 Some(FactorBranch {
637 u: Param::Frozen(load_tensor("U_R")?),
638 v: Param::Frozen(load_tensor("V_R")?),
639 u1: Param::Frozen(load_tensor("u1_R")?),
640 u2: Param::Frozen(load_tensor("u2_R")?),
641 v1: Param::Frozen(load_tensor("v1_R")?),
642 v2: Param::Frozen(load_tensor("v2_R")?),
643 })
644 } else {
645 None
646 };
647
648 let bias = if shape.bias == BiasMode::WithBias {
649 let b_key = format!("{prefix}.bias");
650 if tensors.contains_key(&b_key) {
651 Some(Param::Frozen(load_tensor("bias")?))
652 } else {
653 None
654 }
655 } else {
656 None
657 };
658
659 Ok(Self {
660 in_features,
661 out_features,
662 budget,
663 quant_func: config.quant_func,
664 binarized: weight_state == WeightState::PreBinarized,
665 main,
666 residual,
667 bias,
668 })
669 }
670
671 pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
676 let dims = x.dims();
677 if dims.is_empty() || dims[dims.len() - 1] != self.in_features {
678 return Err(LittleBitError::InvalidShape {
679 expected: format!("[..., {}]", self.in_features),
680 actual: format!("{dims:?}"),
681 });
682 }
683
684 let mut out_dims = dims[..dims.len() - 1].to_vec();
685 let batch_rows: usize = out_dims.iter().product::<usize>().max(1);
686 out_dims.push(self.out_features);
687
688 let x_2d = x.reshape((batch_rows, self.in_features))?;
689 let mut y = self.main.forward(&x_2d, self.quant_func, self.binarized)?;
690 if let Some(res) = &self.residual {
691 let r = res.forward(&x_2d, self.quant_func, self.binarized)?;
692 y = y.add(&r)?;
693 }
694 if let Some(b) = &self.bias {
695 let bt = b.as_tensor();
696 let bt = if bt.dtype() == y.dtype() {
697 bt.clone()
698 } else {
699 bt.to_dtype(y.dtype())?
700 };
701 y = y.broadcast_add(&bt)?;
702 }
703 Ok(y.reshape(out_dims)?)
704 }
705
706 pub fn pack_state_dict(&self, prefix: &str) -> Result<HashMap<String, Tensor>> {
711 let mut out = HashMap::new();
712 let mut insert_branch = |branch: &FactorBranch, suffix: &str| -> Result<()> {
713 let (u_packed, u_shape) = binary_packer(branch.u.as_tensor())?;
714 let (v_packed, v_shape) = binary_packer(branch.v.as_tensor())?;
715 out.insert(format!("{prefix}.U{suffix}_packed"), u_packed);
716 out.insert(format!("{prefix}.U{suffix}_shape"), u_shape);
717 out.insert(format!("{prefix}.V{suffix}_packed"), v_packed);
718 out.insert(format!("{prefix}.V{suffix}_shape"), v_shape);
719 out.insert(
720 format!("{prefix}.u1{suffix}"),
721 branch.u1.as_tensor().detach(),
722 );
723 out.insert(
724 format!("{prefix}.u2{suffix}"),
725 branch.u2.as_tensor().detach(),
726 );
727 out.insert(
728 format!("{prefix}.v1{suffix}"),
729 branch.v1.as_tensor().detach(),
730 );
731 out.insert(
732 format!("{prefix}.v2{suffix}"),
733 branch.v2.as_tensor().detach(),
734 );
735 Ok(())
736 };
737
738 insert_branch(&self.main, "")?;
739 if let Some(res) = &self.residual {
740 insert_branch(res, "_R")?;
741 }
742 if let Some(b) = &self.bias {
743 out.insert(format!("{prefix}.bias"), b.as_tensor().detach());
744 }
745
746 let dev = self.main.u.as_tensor().device();
747 let target_val = self.budget.eff_bit_target.unwrap_or(-1.0) as f32;
748 out.insert(
749 format!("{prefix}._eff_bit_target"),
750 Tensor::from_slice(&[target_val], (), dev)?,
751 );
752 out.insert(
753 format!("{prefix}._split_dim_final"),
754 Tensor::from_slice(&[self.budget.split_dim as i64], (), dev)?,
755 );
756 out.insert(
757 format!("{prefix}._eff_bit_actual"),
758 Tensor::from_slice(&[self.budget.eff_bit_actual as f32], (), dev)?,
759 );
760 Ok(out)
761 }
762
763 pub fn to_packed(&self) -> Result<PackedLittleBitLinear> {
768 let pack_branch = |branch: &FactorBranch| -> Result<PackedFactorBranch> {
769 let (u_packed_t, _) = binary_packer(branch.u.as_tensor())?;
770 let (v_packed_t, _) = binary_packer(branch.v.as_tensor())?;
771 let u_words = u_packed_t
772 .to_device(&Device::Cpu)?
773 .flatten_all()?
774 .to_vec1::<i32>()?;
775 let v_words = v_packed_t
776 .to_device(&Device::Cpu)?
777 .flatten_all()?
778 .to_vec1::<i32>()?;
779 let u =
780 PackedBinaryMatrix::from_words(self.out_features, self.budget.split_dim, u_words)?;
781 let v =
782 PackedBinaryMatrix::from_words(self.budget.split_dim, self.in_features, v_words)?;
783 let to_vec_f32 = |t: &Tensor| -> Result<Vec<f32>> {
784 Ok(t.to_device(&Device::Cpu)?
785 .to_dtype(DType::F32)?
786 .flatten_all()?
787 .to_vec1::<f32>()?)
788 };
789 let u1 = to_vec_f32(branch.u1.as_tensor())?;
790 let u2 = to_vec_f32(branch.u2.as_tensor())?;
791 let v1 = to_vec_f32(branch.v1.as_tensor())?;
792 let v2 = to_vec_f32(branch.v2.as_tensor())?;
793 let v1u2: Vec<f32> = v1.iter().zip(u2.iter()).map(|(&a, &b)| a * b).collect();
794 Ok(PackedFactorBranch {
795 u,
796 v,
797 u1,
798 u2,
799 v1,
800 v2,
801 v1u2,
802 })
803 };
804
805 let main = pack_branch(&self.main)?;
806 let residual = match &self.residual {
807 Some(r) => Some(pack_branch(r)?),
808 None => None,
809 };
810 let bias = match &self.bias {
811 Some(b) => Some(
812 b.as_tensor()
813 .to_device(&Device::Cpu)?
814 .to_dtype(DType::F32)?
815 .flatten_all()?
816 .to_vec1::<f32>()?,
817 ),
818 None => None,
819 };
820
821 Ok(PackedLittleBitLinear {
822 in_features: self.in_features,
823 out_features: self.out_features,
824 budget: self.budget,
825 main,
826 residual,
827 bias,
828 })
829 }
830
831 #[must_use]
833 pub fn in_features(&self) -> usize {
834 self.in_features
835 }
836
837 #[must_use]
839 pub fn out_features(&self) -> usize {
840 self.out_features
841 }
842
843 #[must_use]
845 pub fn budget(&self) -> &BitBudget {
846 &self.budget
847 }
848
849 #[must_use]
851 pub fn split_dim(&self) -> usize {
852 self.budget.split_dim
853 }
854
855 #[must_use]
857 pub fn is_binarized(&self) -> bool {
858 self.binarized
859 }
860
861 pub fn binarize_in_place(&mut self) -> Result<()> {
866 let bin_branch = |branch: &mut FactorBranch| -> Result<()> {
867 branch.u = Param::Frozen(binarize_sign_no_grad(branch.u.as_tensor())?);
868 branch.v = Param::Frozen(binarize_sign_no_grad(branch.v.as_tensor())?);
869 branch.freeze();
870 Ok(())
871 };
872 bin_branch(&mut self.main)?;
873 if let Some(res) = &mut self.residual {
874 bin_branch(res)?;
875 }
876 if let Some(b) = &mut self.bias {
877 b.freeze();
878 }
879 self.binarized = true;
880 Ok(())
881 }
882
883 #[must_use]
885 pub fn trainable_vars(&self) -> Vec<Var> {
886 let mut vars = Vec::new();
887 self.main.push_vars(&mut vars);
888 if let Some(res) = &self.residual {
889 res.push_vars(&mut vars);
890 }
891 if let Some(b) = &self.bias {
892 if let Some(v) = b.as_var() {
893 vars.push(v.clone());
894 }
895 }
896 vars
897 }
898
899 pub fn freeze(&mut self) {
901 self.main.freeze();
902 if let Some(res) = &mut self.residual {
903 res.freeze();
904 }
905 if let Some(b) = &mut self.bias {
906 b.freeze();
907 }
908 }
909
910 pub fn warm_start_from_binarized(&mut self, latent_mag: f64) -> Result<()> {
920 if !latent_mag.is_finite() || latent_mag <= 0.0 {
921 return Err(LittleBitError::InvalidConfig(format!(
922 "warm_start latent_mag must be finite and positive, got {latent_mag}"
923 )));
924 }
925 self.main.warm_start(latent_mag)?;
926 if let Some(res) = &mut self.residual {
927 res.warm_start(latent_mag)?;
928 }
929 if let Some(b) = &mut self.bias {
930 *b = Param::Trainable(Var::from_tensor(&b.as_tensor().detach())?);
931 }
932 self.binarized = false;
933 Ok(())
934 }
935
936 #[must_use]
938 pub fn param_count(&self) -> usize {
939 self.main.elem_count()
940 + self.residual.as_ref().map_or(0, FactorBranch::elem_count)
941 + self.bias.as_ref().map_or(0, Param::elem_count)
942 }
943
944 #[must_use]
946 pub fn main_branch(&self) -> &FactorBranch {
947 &self.main
948 }
949
950 #[must_use]
952 pub fn residual_branch(&self) -> Option<&FactorBranch> {
953 self.residual.as_ref()
954 }
955
956 #[must_use]
958 pub fn bias(&self) -> Option<&Param> {
959 self.bias.as_ref()
960 }
961
962 #[must_use]
964 pub fn extra_repr(&self) -> String {
965 let target = match self.budget.eff_bit_target {
966 Some(v) => format!("{v:.4}"),
967 None => "N/A".to_string(),
968 };
969 format!(
970 "in_features={}, out_features={}, bias={}, split_dim={}, eff_bit_target={}, eff_bit_actual={:.4}, residual={}, total_bit_usage={:.0}",
971 self.in_features,
972 self.out_features,
973 self.bias.is_some(),
974 self.budget.split_dim,
975 target,
976 self.budget.eff_bit_actual,
977 self.budget.residual == ResidualMode::Dual,
978 self.budget.total_bit_usage,
979 )
980 }
981}
982
983impl fmt::Display for LittleBitLinear {
984 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
985 write!(f, "LittleBitLinear({})", self.extra_repr())
986 }
987}
988
989#[derive(Debug, Clone)]
990struct PackedFactorBranch {
991 u: PackedBinaryMatrix,
992 v: PackedBinaryMatrix,
993 u1: Vec<f32>,
994 u2: Vec<f32>,
995 v1: Vec<f32>,
996 v2: Vec<f32>,
997 v1u2: Vec<f32>,
998}
999
1000impl PackedFactorBranch {
1001 fn forward_f32(&self, x: &[f32], batch_rows: usize, in_features: usize) -> Result<Vec<f32>> {
1002 let split_dim = self.v.rows();
1003 let out_features = self.u.rows();
1004
1005 let mut x_scaled = Vec::with_capacity(batch_rows * in_features);
1006 for row in x.chunks_exact(in_features) {
1007 for (&xv, &sv) in row.iter().zip(self.v2.iter()) {
1008 x_scaled.push(xv * sv);
1009 }
1010 }
1011
1012 let mut hidden = self.v.matmul_transposed_f32(&x_scaled, batch_rows)?;
1013 for h_row in hidden.chunks_exact_mut(split_dim) {
1014 for (hv, &sv) in h_row.iter_mut().zip(self.v1u2.iter()) {
1015 *hv *= sv;
1016 }
1017 }
1018
1019 let mut out = self.u.matmul_transposed_f32(&hidden, batch_rows)?;
1020 for o_row in out.chunks_exact_mut(out_features) {
1021 for (ov, &sv) in o_row.iter_mut().zip(self.u1.iter()) {
1022 *ov *= sv;
1023 }
1024 }
1025 Ok(out)
1026 }
1027
1028 fn memory_bytes(&self) -> usize {
1029 self.u.memory_bytes()
1030 + self.v.memory_bytes()
1031 + (self.u1.len() + self.u2.len() + self.v1.len() + self.v2.len() + self.v1u2.len())
1032 * std::mem::size_of::<f32>()
1033 }
1034}
1035
1036#[derive(Debug, Clone)]
1038pub struct PackedLittleBitLinear {
1039 in_features: usize,
1040 out_features: usize,
1041 budget: BitBudget,
1042 main: PackedFactorBranch,
1043 residual: Option<PackedFactorBranch>,
1044 bias: Option<Vec<f32>>,
1045}
1046
1047impl PackedLittleBitLinear {
1048 pub fn from_packed_tensors(
1054 prefix: &str,
1055 tensors: &HashMap<String, Tensor>,
1056 shape: LayerShape,
1057 config: &LittleBitConfig,
1058 ratio_factor: f64,
1059 ) -> Result<Self> {
1060 config.validate()?;
1061 let (in_features, out_features) = (shape.in_features, shape.out_features);
1062 let mut budget = BitBudget::resolve(in_features, out_features, config, ratio_factor);
1063
1064 let load_vec_f32 = |suffix: &str| -> Result<Vec<f32>> {
1065 let key = format!("{prefix}.{suffix}");
1066 let t = tensors
1067 .get(&key)
1068 .ok_or_else(|| LittleBitError::MissingTensor(key.clone()))?;
1069 Ok(t.to_device(&Device::Cpu)?
1070 .to_dtype(DType::F32)?
1071 .flatten_all()?
1072 .to_vec1::<f32>()?)
1073 };
1074
1075 let load_packed = |name: &str| -> Result<(usize, usize, Vec<i32>)> {
1076 let p_key = format!("{prefix}.{name}_packed");
1077 let s_key = format!("{prefix}.{name}_shape");
1078 let packed_t = tensors
1079 .get(&p_key)
1080 .ok_or_else(|| LittleBitError::MissingTensor(p_key.clone()))?;
1081 let shape_t = tensors
1082 .get(&s_key)
1083 .ok_or_else(|| LittleBitError::MissingTensor(s_key.clone()))?;
1084 let shape_vals = shape_t
1085 .to_device(&Device::Cpu)?
1086 .to_dtype(DType::I64)?
1087 .flatten_all()?
1088 .to_vec1::<i64>()?;
1089 if shape_vals.len() != 2 || shape_vals[0] < 0 || shape_vals[1] < 0 {
1090 return Err(LittleBitError::InvalidShape {
1091 expected: "non-negative [rows, cols]".to_string(),
1092 actual: format!("{shape_vals:?}"),
1093 });
1094 }
1095 let (rows, cols) = (shape_vals[0] as usize, shape_vals[1] as usize);
1096 let words = packed_t
1097 .to_device(&Device::Cpu)?
1098 .flatten_all()?
1099 .to_vec1::<i32>()?;
1100 Ok((rows, cols, words))
1101 };
1102
1103 let load_branch = |suffix: &str| -> Result<(usize, PackedFactorBranch)> {
1104 let (u_rows, u_cols, u_words) = load_packed(&format!("U{suffix}"))?;
1105 let (v_rows, v_cols, v_words) = load_packed(&format!("V{suffix}"))?;
1106 if u_rows != out_features || v_cols != in_features || u_cols != v_rows {
1107 return Err(LittleBitError::InvalidShape {
1108 expected: format!("U=({out_features}, s), V=(s, {in_features})"),
1109 actual: format!("U=({u_rows}, {u_cols}), V=({v_rows}, {v_cols})"),
1110 });
1111 }
1112 let u = PackedBinaryMatrix::from_words(u_rows, u_cols, u_words)?;
1113 let v = PackedBinaryMatrix::from_words(v_rows, v_cols, v_words)?;
1114 let u1 = load_vec_f32(&format!("u1{suffix}"))?;
1115 let u2 = load_vec_f32(&format!("u2{suffix}"))?;
1116 let v1 = load_vec_f32(&format!("v1{suffix}"))?;
1117 let v2 = load_vec_f32(&format!("v2{suffix}"))?;
1118 let v1u2: Vec<f32> = v1.iter().zip(u2.iter()).map(|(&a, &b)| a * b).collect();
1119 Ok((
1120 u_cols,
1121 PackedFactorBranch {
1122 u,
1123 v,
1124 u1,
1125 u2,
1126 v1,
1127 v2,
1128 v1u2,
1129 },
1130 ))
1131 };
1132
1133 let (actual_split, main) = load_branch("")?;
1134 if actual_split != budget.split_dim {
1135 budget.split_dim = actual_split;
1136 budget.eff_bit_actual = crate::config::compute_eff_bits(
1137 in_features,
1138 out_features,
1139 actual_split,
1140 budget.residual,
1141 );
1142 budget.total_bit_usage =
1143 budget.eff_bit_actual * (in_features as f64) * (out_features as f64);
1144 }
1145
1146 let residual = if budget.residual == ResidualMode::Dual {
1147 let (_, res_branch) = load_branch("_R")?;
1148 Some(res_branch)
1149 } else {
1150 None
1151 };
1152
1153 let bias = if shape.bias == BiasMode::WithBias
1154 && tensors.contains_key(&format!("{prefix}.bias"))
1155 {
1156 Some(load_vec_f32("bias")?)
1157 } else {
1158 None
1159 };
1160
1161 Ok(Self {
1162 in_features,
1163 out_features,
1164 budget,
1165 main,
1166 residual,
1167 bias,
1168 })
1169 }
1170
1171 pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
1176 let dims = x.dims();
1177 if dims.is_empty() || dims[dims.len() - 1] != self.in_features {
1178 return Err(LittleBitError::InvalidShape {
1179 expected: format!("[..., {}]", self.in_features),
1180 actual: format!("{dims:?}"),
1181 });
1182 }
1183 let mut out_dims = dims[..dims.len() - 1].to_vec();
1184 let batch_rows: usize = out_dims.iter().product::<usize>().max(1);
1185 out_dims.push(self.out_features);
1186
1187 let x_flat = x
1188 .to_device(&Device::Cpu)?
1189 .to_dtype(DType::F32)?
1190 .reshape(batch_rows * self.in_features)?
1191 .to_vec1::<f32>()?;
1192
1193 let mut out = self
1194 .main
1195 .forward_f32(&x_flat, batch_rows, self.in_features)?;
1196 if let Some(res) = &self.residual {
1197 let r = res.forward_f32(&x_flat, batch_rows, self.in_features)?;
1198 for (dst, rv) in out.iter_mut().zip(r.iter()) {
1199 *dst += *rv;
1200 }
1201 }
1202 if let Some(bias) = &self.bias {
1203 for row in out.chunks_exact_mut(self.out_features) {
1204 for (dst, &bv) in row.iter_mut().zip(bias.iter()) {
1205 *dst += bv;
1206 }
1207 }
1208 }
1209
1210 let out_t = Tensor::from_vec(out, out_dims, x.device())?;
1211 if x.dtype() == DType::F32 {
1212 Ok(out_t)
1213 } else {
1214 Ok(out_t.to_dtype(x.dtype())?)
1215 }
1216 }
1217
1218 pub fn pack_state_dict(
1223 &self,
1224 prefix: &str,
1225 device: &Device,
1226 ) -> Result<HashMap<String, Tensor>> {
1227 let mut out = HashMap::new();
1228 let mut insert_branch = |branch: &PackedFactorBranch, suffix: &str| -> Result<()> {
1229 let (u_packed, u_shape) = branch.u.to_tensors(device)?;
1230 let (v_packed, v_shape) = branch.v.to_tensors(device)?;
1231 out.insert(format!("{prefix}.U{suffix}_packed"), u_packed);
1232 out.insert(format!("{prefix}.U{suffix}_shape"), u_shape);
1233 out.insert(format!("{prefix}.V{suffix}_packed"), v_packed);
1234 out.insert(format!("{prefix}.V{suffix}_shape"), v_shape);
1235 out.insert(
1236 format!("{prefix}.u1{suffix}"),
1237 Tensor::from_slice(&branch.u1, (1, self.out_features), device)?,
1238 );
1239 out.insert(
1240 format!("{prefix}.u2{suffix}"),
1241 Tensor::from_slice(&branch.u2, (1, self.budget.split_dim), device)?,
1242 );
1243 out.insert(
1244 format!("{prefix}.v1{suffix}"),
1245 Tensor::from_slice(&branch.v1, (1, self.budget.split_dim), device)?,
1246 );
1247 out.insert(
1248 format!("{prefix}.v2{suffix}"),
1249 Tensor::from_slice(&branch.v2, (1, self.in_features), device)?,
1250 );
1251 Ok(())
1252 };
1253
1254 insert_branch(&self.main, "")?;
1255 if let Some(res) = &self.residual {
1256 insert_branch(res, "_R")?;
1257 }
1258 if let Some(b) = &self.bias {
1259 out.insert(
1260 format!("{prefix}.bias"),
1261 Tensor::from_slice(b, self.out_features, device)?,
1262 );
1263 }
1264 let target_val = self.budget.eff_bit_target.unwrap_or(-1.0) as f32;
1265 out.insert(
1266 format!("{prefix}._eff_bit_target"),
1267 Tensor::from_slice(&[target_val], (), device)?,
1268 );
1269 out.insert(
1270 format!("{prefix}._split_dim_final"),
1271 Tensor::from_slice(&[self.budget.split_dim as i64], (), device)?,
1272 );
1273 out.insert(
1274 format!("{prefix}._eff_bit_actual"),
1275 Tensor::from_slice(&[self.budget.eff_bit_actual as f32], (), device)?,
1276 );
1277 Ok(out)
1278 }
1279
1280 #[must_use]
1282 pub fn memory_bytes(&self) -> usize {
1283 self.main.memory_bytes()
1284 + self
1285 .residual
1286 .as_ref()
1287 .map_or(0, PackedFactorBranch::memory_bytes)
1288 + self
1289 .bias
1290 .as_ref()
1291 .map_or(0, |b| b.len() * std::mem::size_of::<f32>())
1292 }
1293
1294 #[must_use]
1296 pub fn budget(&self) -> &BitBudget {
1297 &self.budget
1298 }
1299}
1300
1301pub fn reconstruct_branch_matrix(
1307 u: &Tensor,
1308 v: &Tensor,
1309 u1: &Tensor,
1310 u2: &Tensor,
1311 v1: &Tensor,
1312 v2: &Tensor,
1313 quant_func: QuantFunc,
1314) -> Result<Tensor> {
1315 let uq = quant_func.apply(&u.to_dtype(DType::F32)?)?;
1316 let vq = quant_func.apply(&v.to_dtype(DType::F32)?)?;
1317 let u_scale = u1
1318 .to_dtype(DType::F32)?
1319 .t()?
1320 .matmul(&u2.to_dtype(DType::F32)?)?;
1321 let v_scale = v1
1322 .to_dtype(DType::F32)?
1323 .t()?
1324 .matmul(&v2.to_dtype(DType::F32)?)?;
1325 Ok(uq.mul(&u_scale)?.matmul(&vq.mul(&v_scale)?)?)
1326}
1327
1328#[cfg(test)]
1329mod tests {
1330 use super::*;
1331 use rand::SeedableRng;
1332 use rand_chacha::ChaCha8Rng;
1333 use rand_distr::{Distribution, StandardNormal};
1334
1335 fn random_tensor(rows: usize, cols: usize, seed: u64) -> Result<Tensor> {
1336 let mut rng = ChaCha8Rng::seed_from_u64(seed);
1337 let data: Vec<f32> = (0..rows * cols)
1338 .map(|_| StandardNormal.sample(&mut rng))
1339 .collect();
1340 Ok(Tensor::from_vec(data, (rows, cols), &Device::Cpu)?)
1341 }
1342
1343 #[test]
1344 fn forward_matches_packed_and_direct_matrix_product() -> Result<()> {
1345 let out_f = 48;
1346 let in_f = 64;
1347 let w = random_tensor(out_f, in_f, 7)?;
1348 let bias = Tensor::from_vec(
1349 (0..out_f).map(|i| (i as f32) * 0.01 - 0.2).collect(),
1350 out_f,
1351 &Device::Cpu,
1352 )?;
1353 let cfg = LittleBitConfig {
1354 quant_func: QuantFunc::SmoothSign { alpha: 100.0 },
1355 eff_bit: Some(0.5),
1356 residual: true,
1357 use_itq: true,
1358 itq_n_iter: 20,
1359 ..Default::default()
1360 };
1361
1362 let layer =
1363 LittleBitLinear::from_dense_weight(&w, Some(&bias), &cfg, 1.0, TrainMode::Eval, 11)?;
1364 let packed = layer.to_packed()?;
1365
1366 assert!(packed.memory_bytes() < out_f * in_f * 4);
1367
1368 let x = random_tensor(6, in_f, 99)?.reshape((2, 3, in_f))?;
1369 let y_layer = layer.forward(&x)?;
1370 let y_packed = packed.forward(&x)?;
1371 assert_eq!(y_layer.dims(), &[2, 3, out_f]);
1372 assert_eq!(y_packed.dims(), &[2, 3, out_f]);
1373
1374 let main = layer.main_branch();
1376 let res = layer.residual_branch().expect("residual branch");
1377 let w_main = reconstruct_branch_matrix(
1378 main.u.as_tensor(),
1379 main.v.as_tensor(),
1380 main.u1.as_tensor(),
1381 main.u2.as_tensor(),
1382 main.v1.as_tensor(),
1383 main.v2.as_tensor(),
1384 cfg.quant_func,
1385 )?;
1386 let w_res = reconstruct_branch_matrix(
1387 res.u.as_tensor(),
1388 res.v.as_tensor(),
1389 res.u1.as_tensor(),
1390 res.u2.as_tensor(),
1391 res.v1.as_tensor(),
1392 res.v2.as_tensor(),
1393 cfg.quant_func,
1394 )?;
1395 let w_hat = w_main.add(&w_res)?;
1396 let y_ref = x
1397 .reshape((6, in_f))?
1398 .matmul(&w_hat.t()?)?
1399 .broadcast_add(&bias)?
1400 .reshape((2, 3, out_f))?;
1401
1402 let diff_packed = y_layer
1403 .sub(&y_packed)?
1404 .abs()?
1405 .max_all()?
1406 .to_scalar::<f32>()?;
1407 let diff_ref = y_layer.sub(&y_ref)?.abs()?.max_all()?.to_scalar::<f32>()?;
1408 assert!(diff_packed < 1e-4, "packed drift: {diff_packed}");
1409 assert!(diff_ref < 1e-4, "reference drift: {diff_ref}");
1410 Ok(())
1411 }
1412
1413 #[test]
1414 fn autograd_flows_through_all_factors_for_both_quant_funcs() -> Result<()> {
1415 for qf in [QuantFunc::SteBinary, QuantFunc::SmoothSign { alpha: 50.0 }] {
1416 let w = random_tensor(32, 32, 19)?;
1417 let bias = Tensor::zeros(32, DType::F32, &Device::Cpu)?;
1418 let cfg = LittleBitConfig {
1419 quant_func: qf,
1420 eff_bit: None,
1421 split_dim: 8,
1422 residual: true,
1423 use_itq: true,
1424 itq_n_iter: 10,
1425 ..Default::default()
1426 };
1427 let layer = LittleBitLinear::from_dense_weight(
1428 &w,
1429 Some(&bias),
1430 &cfg,
1431 1.0,
1432 TrainMode::Train,
1433 3,
1434 )?;
1435 let x = random_tensor(4, 32, 23)?;
1436 let loss = layer.forward(&x)?.sqr()?.mean_all()?;
1437 let grads = loss.backward()?;
1438
1439 let vars = layer.trainable_vars();
1440 assert_eq!(vars.len(), 13);
1441 for var in vars {
1442 let g = grads
1443 .get(var.as_tensor())
1444 .expect("every factor var must receive a gradient");
1445 let norm = g.sqr()?.sum_all()?.to_scalar::<f32>()?;
1446 assert!(norm.is_finite() && norm > 0.0, "zero or NaN grad: {norm}");
1447 }
1448 }
1449 Ok(())
1450 }
1451}