1use std::collections::BTreeSet;
4use std::fmt;
5use std::fs;
6use std::path::Path;
7use std::str::FromStr;
8
9use serde::{Deserialize, Deserializer, Serialize, Serializer};
10
11use crate::error::{LittleBitError, Result};
12
13pub const SMOOTH_SIGN_DEFAULT_ALPHA: f64 = 100.0;
15
16#[derive(Debug, Clone, Copy, PartialEq, Default)]
18pub enum QuantFunc {
19 #[default]
21 SteBinary,
22 SmoothSign {
25 alpha: f64,
27 },
28}
29
30impl QuantFunc {
31 #[must_use]
33 pub fn as_wire_name(self) -> &'static str {
34 match self {
35 Self::SteBinary => "STEBinary",
36 Self::SmoothSign { .. } => "SmoothSign",
37 }
38 }
39}
40
41impl fmt::Display for QuantFunc {
42 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
43 f.write_str(self.as_wire_name())
44 }
45}
46
47impl FromStr for QuantFunc {
48 type Err = LittleBitError;
49
50 fn from_str(s: &str) -> Result<Self> {
51 match s.trim().to_ascii_lowercase().as_str() {
52 "stebinary" | "ste_binary" | "ste" => Ok(Self::SteBinary),
53 "smoothsign" | "smooth_sign" | "smooth" => Ok(Self::SmoothSign {
54 alpha: SMOOTH_SIGN_DEFAULT_ALPHA,
55 }),
56 other => Err(LittleBitError::InvalidConfig(format!(
57 "unknown quant_func `{other}` (expected `STEBinary` or `SmoothSign`)"
58 ))),
59 }
60 }
61}
62
63impl Serialize for QuantFunc {
64 fn serialize<S: Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
65 serializer.serialize_str(self.as_wire_name())
66 }
67}
68
69impl<'de> Deserialize<'de> for QuantFunc {
70 fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
71 let raw = String::deserialize(deserializer)?;
72 raw.parse().map_err(serde::de::Error::custom)
73 }
74}
75
76#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
78pub enum ResidualMode {
79 Single,
81 Dual,
83}
84
85impl From<bool> for ResidualMode {
86 fn from(residual: bool) -> Self {
87 if residual {
88 Self::Dual
89 } else {
90 Self::Single
91 }
92 }
93}
94
95#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
97pub enum ItqMode {
98 Disabled,
100 Joint {
102 n_iter: usize,
104 },
105}
106
107#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
109pub enum TrainMode {
110 Train,
112 Eval,
114}
115
116#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
118pub enum InitMode {
119 Decompose,
121 Empty,
123}
124
125#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
127pub enum InferenceMode {
128 Unpacked,
130 Packed,
132}
133
134#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
136pub enum BiasMode {
137 WithBias,
139 NoBias,
141}
142
143impl From<bool> for BiasMode {
144 fn from(has_bias: bool) -> Self {
145 if has_bias {
146 Self::WithBias
147 } else {
148 Self::NoBias
149 }
150 }
151}
152
153#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
155pub enum WeightState {
156 Raw,
158 PreBinarized,
160}
161
162#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
164pub struct LayerShape {
165 pub in_features: usize,
167 pub out_features: usize,
169 pub bias: BiasMode,
171}
172
173impl LayerShape {
174 #[must_use]
176 pub const fn new(in_features: usize, out_features: usize, bias: BiasMode) -> Self {
177 Self {
178 in_features,
179 out_features,
180 bias,
181 }
182 }
183}
184
185#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
187pub enum CastPolicy {
188 Preserve,
190 ToBf16,
192}
193
194#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
196#[serde(default)]
197pub struct LittleBitConfig {
198 pub quant_func: QuantFunc,
200 pub eff_bit: Option<f64>,
202 #[serde(default, skip_serializing_if = "Option::is_none")]
204 pub eff_bit_layers: Option<String>,
205 pub split_dim: usize,
207 pub residual: bool,
209 pub kv_factor: f64,
211 pub min_split_dim: usize,
213 pub quant_mod: String,
215 pub use_itq: bool,
217 pub itq_n_iter: usize,
219}
220
221impl Default for LittleBitConfig {
222 fn default() -> Self {
223 Self {
224 quant_func: QuantFunc::SteBinary,
225 eff_bit: Some(1.0),
226 eff_bit_layers: None,
227 split_dim: 1024,
228 residual: false,
229 kv_factor: 1.0,
230 min_split_dim: 8,
231 quant_mod: "LittleBitLinear".to_string(),
232 use_itq: false,
233 itq_n_iter: 50,
234 }
235 }
236}
237
238#[derive(Debug, Clone, Default, PartialEq)]
240pub struct LittleBitConfigOverrides {
241 pub quant_func: Option<QuantFunc>,
243 pub eff_bit: Option<Option<f64>>,
245 pub eff_bit_layers: Option<Option<String>>,
247 pub split_dim: Option<usize>,
249 pub residual: Option<bool>,
251 pub kv_factor: Option<f64>,
253 pub min_split_dim: Option<usize>,
255 pub quant_mod: Option<String>,
257 pub use_itq: Option<bool>,
259 pub itq_n_iter: Option<usize>,
261}
262
263impl LittleBitConfig {
264 #[must_use]
266 pub fn residual_mode(&self) -> ResidualMode {
267 ResidualMode::from(self.residual)
268 }
269
270 #[must_use]
272 pub fn itq_mode(&self) -> ItqMode {
273 if self.use_itq {
274 ItqMode::Joint {
275 n_iter: self.itq_n_iter,
276 }
277 } else {
278 ItqMode::Disabled
279 }
280 }
281
282 pub fn validate(&self) -> Result<()> {
287 if self.min_split_dim == 0 {
288 return Err(LittleBitError::InvalidConfig(
289 "min_split_dim must be greater than 0".to_string(),
290 ));
291 }
292 if !self.kv_factor.is_finite() || self.kv_factor <= 0.0 {
293 return Err(LittleBitError::InvalidConfig(format!(
294 "kv_factor must be finite and positive, got {}",
295 self.kv_factor
296 )));
297 }
298 if let Some(bit) = self.eff_bit {
299 if !bit.is_finite() || bit <= 0.0 {
300 return Err(LittleBitError::InvalidConfig(format!(
301 "eff_bit must be finite and positive, got {bit}"
302 )));
303 }
304 } else if self.split_dim == 0 {
305 return Err(LittleBitError::InvalidConfig(
306 "split_dim must be greater than 0 when eff_bit is None".to_string(),
307 ));
308 }
309 if let QuantFunc::SmoothSign { alpha } = self.quant_func {
310 if !alpha.is_finite() || alpha <= 0.0 {
311 return Err(LittleBitError::InvalidConfig(format!(
312 "smooth_sign alpha must be finite and positive, got {alpha}"
313 )));
314 }
315 }
316 if self.use_itq && self.itq_n_iter == 0 {
317 return Err(LittleBitError::InvalidConfig(
318 "itq_n_iter must be greater than 0 when use_itq is enabled".to_string(),
319 ));
320 }
321 parse_eff_bit_layers(self.eff_bit_layers.as_deref())?;
322 Ok(())
323 }
324
325 pub fn eff_bit_for_module(&self, module_name: &str) -> Result<Option<f64>> {
331 let mut matched = None;
332 if let Some(after) = module_name
333 .find(".layers.")
334 .map(|idx| &module_name[idx + ".layers.".len()..])
335 {
336 if let Some(seg) = after.split('.').next() {
337 if let Ok(layer_idx) = seg.parse::<usize>() {
338 for (blocks, bits) in parse_eff_bit_layers(self.eff_bit_layers.as_deref())? {
339 if blocks.binary_search(&layer_idx).is_ok() {
340 matched = Some(bits);
341 }
342 }
343 }
344 }
345 }
346 Ok(matched.or(self.eff_bit))
347 }
348
349 pub fn effective_for_module(&self, module_name: &str) -> Result<Self> {
354 if self.eff_bit_layers.is_none() {
355 return Ok(self.clone());
356 }
357 let resolved = self.eff_bit_for_module(module_name)?;
358 let mut cloned = self.clone();
359 cloned.eff_bit = resolved;
360 Ok(cloned)
361 }
362
363 pub fn load_with_priority(dir: &Path, overrides: &LittleBitConfigOverrides) -> Result<Self> {
373 let default = Self::default();
374 let littlebit_path = dir.join("littlebit_config.json");
375 let mut config = if littlebit_path.exists() {
376 let raw = fs::read_to_string(&littlebit_path)?;
377 serde_json::from_str::<Self>(&raw)?
378 } else {
379 default.clone()
380 };
381
382 let model_cfg_path = dir.join("config.json");
383 if model_cfg_path.exists() {
384 let raw = fs::read_to_string(&model_cfg_path)?;
385 let value: serde_json::Value = serde_json::from_str(&raw)?;
386 if let Some(obj) = value.as_object() {
387 apply_legacy_fallback(&mut config, &default, obj)?;
388 }
389 }
390
391 apply_overrides(&mut config, overrides);
392 config.validate()?;
393 Ok(config)
394 }
395}
396
397fn apply_legacy_fallback(
398 config: &mut LittleBitConfig,
399 default: &LittleBitConfig,
400 obj: &serde_json::Map<String, serde_json::Value>,
401) -> Result<()> {
402 if config.quant_func == default.quant_func {
403 if let Some(v) = obj.get("quant_func").and_then(serde_json::Value::as_str) {
404 config.quant_func = v.parse()?;
405 }
406 }
407 if config.eff_bit == default.eff_bit {
408 if let Some(val) = obj.get("eff_bit") {
409 if val.is_null() {
410 config.eff_bit = None;
411 } else if let Some(b) = val.as_f64() {
412 config.eff_bit = Some(b);
413 }
414 }
415 }
416 if config.eff_bit_layers.is_none() {
417 if let Some(spec) = obj
418 .get("eff_bit_layers")
419 .and_then(serde_json::Value::as_str)
420 {
421 config.eff_bit_layers = Some(spec.to_string());
422 }
423 }
424 if config.split_dim == default.split_dim {
425 if let Some(s) = obj.get("split_dim").and_then(serde_json::Value::as_u64) {
426 config.split_dim = s as usize;
427 }
428 }
429 if config.residual == default.residual {
430 if let Some(r) = obj.get("residual").and_then(serde_json::Value::as_bool) {
431 config.residual = r;
432 }
433 }
434 if config.quant_mod == default.quant_mod {
435 if let Some(m) = obj.get("quant_mod").and_then(serde_json::Value::as_str) {
436 config.quant_mod = m.to_string();
437 }
438 }
439 if (config.kv_factor - default.kv_factor).abs() < f64::EPSILON {
440 if let Some(k) = obj.get("kv_factor").and_then(serde_json::Value::as_f64) {
441 config.kv_factor = k;
442 }
443 }
444 if config.min_split_dim == default.min_split_dim {
445 if let Some(m) = obj.get("min_split_dim").and_then(serde_json::Value::as_u64) {
446 config.min_split_dim = m as usize;
447 }
448 }
449 if config.use_itq == default.use_itq {
450 if let Some(u) = obj.get("use_itq").and_then(serde_json::Value::as_bool) {
451 config.use_itq = u;
452 }
453 }
454 if config.itq_n_iter == default.itq_n_iter {
455 if let Some(n) = obj.get("itq_n_iter").and_then(serde_json::Value::as_u64) {
456 config.itq_n_iter = n as usize;
457 }
458 }
459 Ok(())
460}
461
462fn apply_overrides(config: &mut LittleBitConfig, overrides: &LittleBitConfigOverrides) {
463 if let Some(qf) = overrides.quant_func {
464 config.quant_func = qf;
465 }
466 if let Some(eff_bit) = overrides.eff_bit {
467 config.eff_bit = eff_bit;
468 }
469 if let Some(spec) = &overrides.eff_bit_layers {
470 config.eff_bit_layers = spec.clone();
471 }
472 if let Some(split_dim) = overrides.split_dim {
473 config.split_dim = split_dim;
474 }
475 if let Some(residual) = overrides.residual {
476 config.residual = residual;
477 }
478 if let Some(kv_factor) = overrides.kv_factor {
479 config.kv_factor = kv_factor;
480 }
481 if let Some(min_split_dim) = overrides.min_split_dim {
482 config.min_split_dim = min_split_dim;
483 }
484 if let Some(quant_mod) = &overrides.quant_mod {
485 config.quant_mod = quant_mod.clone();
486 }
487 if let Some(use_itq) = overrides.use_itq {
488 config.use_itq = use_itq;
489 }
490 if let Some(itq_n_iter) = overrides.itq_n_iter {
491 config.itq_n_iter = itq_n_iter;
492 }
493}
494
495pub fn parse_eff_bit_layers(spec: Option<&str>) -> Result<Vec<(Vec<usize>, f64)>> {
506 let Some(raw) = spec else {
507 return Ok(Vec::new());
508 };
509 if raw.trim().is_empty() {
510 return Ok(Vec::new());
511 }
512
513 let mut out = Vec::new();
514 for raw_group in raw.split(';') {
515 let group = raw_group.trim();
516 if group.is_empty() {
517 continue;
518 }
519 let (blocks_s, bits_s) = group.rsplit_once(':').ok_or_else(|| {
520 LittleBitError::InvalidConfig(format!(
521 "invalid eff_bit_layers group `{group}` (expected `<blocks>:<eff_bit>`)"
522 ))
523 })?;
524 let bits: f64 = bits_s.trim().parse().map_err(|_| {
525 LittleBitError::InvalidConfig(format!(
526 "invalid eff_bit value `{bits_s}` in eff_bit_layers group `{group}`"
527 ))
528 })?;
529 if !bits.is_finite() || bits <= 0.0 {
530 return Err(LittleBitError::InvalidConfig(format!(
531 "eff_bit in eff_bit_layers must be finite and positive, got {bits}"
532 )));
533 }
534
535 let mut blocks = BTreeSet::new();
536 for raw_part in blocks_s.split(',') {
537 let part = raw_part.trim();
538 if part.is_empty() {
539 return Err(LittleBitError::InvalidConfig(format!(
540 "empty block specification in eff_bit_layers group `{group}`"
541 )));
542 }
543 if let Some((a_s, b_s)) = part.split_once('-') {
544 let start: usize = a_s.trim().parse().map_err(|_| {
545 LittleBitError::InvalidConfig(format!(
546 "invalid range start `{a_s}` in eff_bit_layers group `{group}`"
547 ))
548 })?;
549 let end: usize = b_s.trim().parse().map_err(|_| {
550 LittleBitError::InvalidConfig(format!(
551 "invalid range end `{b_s}` in eff_bit_layers group `{group}`"
552 ))
553 })?;
554 if start > end {
555 return Err(LittleBitError::InvalidConfig(format!(
556 "invalid descending range `{part}` in eff_bit_layers group `{group}`"
557 )));
558 }
559 for idx in start..=end {
560 blocks.insert(idx);
561 }
562 } else {
563 let idx: usize = part.parse().map_err(|_| {
564 LittleBitError::InvalidConfig(format!(
565 "invalid block index `{part}` in eff_bit_layers group `{group}`"
566 ))
567 })?;
568 blocks.insert(idx);
569 }
570 }
571
572 if blocks.is_empty() {
573 return Err(LittleBitError::InvalidConfig(format!(
574 "no block indices found in eff_bit_layers group `{group}`"
575 )));
576 }
577 out.push((blocks.into_iter().collect(), bits));
578 }
579
580 Ok(out)
581}
582
583#[derive(Debug, Clone, Copy, PartialEq)]
585pub struct BitBudget {
586 pub in_features: usize,
588 pub out_features: usize,
590 pub eff_bit_target: Option<f64>,
592 pub split_dim: usize,
594 pub eff_bit_actual: f64,
596 pub total_bit_usage: f64,
598 pub residual: ResidualMode,
600}
601
602impl BitBudget {
603 #[must_use]
605 pub fn resolve(
606 in_features: usize,
607 out_features: usize,
608 config: &LittleBitConfig,
609 ratio_factor: f64,
610 ) -> Self {
611 let residual = config.residual_mode();
612 let split_calc_float =
613 estimate_split_dim(in_features, out_features, config.eff_bit, residual)
614 .map(|v| v * ratio_factor);
615 let split_dim =
616 finalize_split_dim(split_calc_float, config.split_dim, config.min_split_dim);
617 let eff_bit_actual = compute_eff_bits(in_features, out_features, split_dim, residual);
618 let total_bit_usage = eff_bit_actual * (in_features as f64) * (out_features as f64);
619 Self {
620 in_features,
621 out_features,
622 eff_bit_target: config.eff_bit,
623 split_dim,
624 eff_bit_actual,
625 total_bit_usage,
626 residual,
627 }
628 }
629}
630
631#[must_use]
635pub fn estimate_split_dim(
636 in_features: usize,
637 out_features: usize,
638 eff_bit_target: Option<f64>,
639 residual: ResidualMode,
640) -> Option<f64> {
641 let target = eff_bit_target?;
642 let a = in_features as f64;
643 let b = out_features as f64;
644 if in_features == 0 || out_features == 0 {
645 return None;
646 }
647
648 let base = a + b + 16.0;
649 let (numerator, denominator) = match residual {
650 ResidualMode::Dual => (a * b * target - 32.0 * (a + b), 2.0 * base),
651 ResidualMode::Single => (a * b * target - 16.0 * (a + b), base),
652 };
653 if denominator == 0.0 {
654 None
655 } else {
656 Some(numerator / denominator)
657 }
658}
659
660#[must_use]
664pub fn finalize_split_dim(
665 split_float: Option<f64>,
666 split_default: usize,
667 min_split_dim: usize,
668) -> usize {
669 let cand = match split_float {
670 Some(v) => v as isize,
671 None => split_default as isize,
672 };
673 let mut rounded = cand.div_euclid(8) * 8;
674 if rounded == 0 {
675 rounded = min_split_dim as isize;
676 }
677 rounded.max(min_split_dim as isize) as usize
678}
679
680#[must_use]
686pub fn compute_eff_bits(
687 in_features: usize,
688 out_features: usize,
689 split_dim: usize,
690 residual: ResidualMode,
691) -> f64 {
692 if in_features == 0 || out_features == 0 {
693 return f64::INFINITY;
694 }
695 let a = in_features as f64;
696 let b = out_features as f64;
697 let s = split_dim as f64;
698 let num = match residual {
699 ResidualMode::Dual => s * 2.0 * (a + b + 16.0) + 32.0 * (a + b),
700 ResidualMode::Single => s * (a + b + 16.0) + 16.0 * (a + b),
701 };
702 num / (a * b)
703}
704
705#[cfg(test)]
706mod tests {
707 use super::*;
708 use tempfile::TempDir;
709
710 #[test]
711 fn bit_budget_matches_upstream_llama7b_shapes() {
712 let cfg = LittleBitConfig {
713 eff_bit: Some(0.1),
714 residual: true,
715 min_split_dim: 8,
716 ..Default::default()
717 };
718 let b1 = BitBudget::resolve(4096, 4096, &cfg, 1.0);
720 let raw = (4096.0_f64 * 4096.0 * 0.1 - 32.0 * 8192.0) / (2.0 * (8192.0 + 16.0));
721 let expected_s = ((raw as isize).div_euclid(8) * 8).max(8) as usize;
722 assert_eq!(b1.split_dim, expected_s);
723 assert!(b1.eff_bit_actual <= 0.11);
724
725 let b_kv = BitBudget::resolve(4096, 1024, &cfg, 2.0);
727 assert!(b_kv.split_dim >= 8);
728 assert_eq!(b_kv.split_dim % 8, 0);
729 }
730
731 #[test]
732 fn finalize_split_dim_floors_zero_and_negatives() {
733 assert_eq!(finalize_split_dim(Some(-42.5), 1024, 8), 8);
734 assert_eq!(finalize_split_dim(Some(7.9), 1024, 8), 8);
735 assert_eq!(finalize_split_dim(Some(63.9), 1024, 8), 56);
736 assert_eq!(finalize_split_dim(None, 1024, 8), 1024);
737 }
738
739 #[test]
740 fn config_priority_follows_overrides_then_littlebit_then_legacy() {
741 let dir = TempDir::new().expect("tempdir");
742 fs::write(
743 dir.path().join("config.json"),
744 r#"{"quant_func": "SmoothSign", "split_dim": 512, "use_itq": true}"#,
745 )
746 .expect("write config.json");
747 fs::write(
748 dir.path().join("littlebit_config.json"),
749 r#"{"eff_bit": 0.25, "residual": true}"#,
750 )
751 .expect("write littlebit_config.json");
752
753 let overrides = LittleBitConfigOverrides {
754 split_dim: Some(256),
755 ..Default::default()
756 };
757 let resolved =
758 LittleBitConfig::load_with_priority(dir.path(), &overrides).expect("load config");
759 assert_eq!(
760 resolved.quant_func,
761 QuantFunc::SmoothSign {
762 alpha: SMOOTH_SIGN_DEFAULT_ALPHA
763 }
764 );
765 assert_eq!(resolved.eff_bit, Some(0.25));
766 assert!(resolved.residual);
767 assert!(resolved.use_itq);
768 assert_eq!(resolved.split_dim, 256);
769 }
770}