Skip to main content

littlebit_models/
lib.rs

1//! LLM architectures, dynamic LittleBit patching, KV cache, and Hugging Face Hub integration.
2
3pub mod config;
4pub mod error;
5pub mod gemma;
6pub mod gemma4;
7pub mod hub;
8pub mod layer;
9pub mod llama;
10pub mod model;
11pub mod opt;
12pub mod patch;
13pub mod phi;
14pub mod phi_attn;
15pub mod streaming;
16
17pub use config::{ModelArchitecture, ModelConfig, ResolvedRopeParams, TokenIdSpec};
18pub use error::{ModelError, Result};
19pub use gemma::GemmaModel;
20pub use gemma4::{
21    Gemma4Expert, Gemma4Experts, Gemma4MoE, Gemma4Model, Gemma4Router, RouterDispatch,
22};
23pub use hub::{load_dense_model, resolve_model_dir, LittleBitModel, LoadOptions, PushOptions};
24pub use layer::{
25    apply_activation, causal_mask, repeat_kv, Embedding, KvCache, LayerNorm, LinearLayer,
26    LinearLoadSpec, NormKind, RmsNorm, RmsNormNoScale, RopeMode, RotaryEmbedding,
27};
28pub use llama::LlamaModel;
29pub use model::{CausalLm, CausalLmOutput, GenerateConfig, HiddenStatesMode, ParameterStats};
30pub use opt::OptModel;
31pub use patch::{derive_layer_seed, patch_linear_layer, PatchPolicy, PatchSummary};
32pub use phi::PhiModel;
33pub use phi_attn::PhiAttentionProjection;
34pub use streaming::{
35    stream_quantize_checkpoint, StreamingQuantizeOptions, StreamingQuantizeReport,
36};
37
38#[cfg(test)]
39mod tests {
40    use super::*;
41    use candle_core::{DType, Device, Tensor};
42    use littlebit_core::{CastPolicy, InferenceMode, LittleBitConfig, QuantFunc, TrainMode};
43    use tempfile::TempDir;
44
45    #[test]
46    fn all_six_families_forward_patch_save_and_reload_packed() -> Result<()> {
47        let dev = Device::Cpu;
48        let configs = [
49            ModelConfig::tiny_llama(128, 64, 2),
50            ModelConfig::tiny_qwen2(128, 64, 2),
51            ModelConfig::tiny_qwen3(128, 64, 2),
52            ModelConfig::tiny_phi3(128, 64, 2),
53            ModelConfig::tiny_gemma2(128, 64, 2),
54            ModelConfig::tiny_gemma3(128, 64, 2),
55            ModelConfig::tiny_gemma4_dense(128, 64, 2),
56            ModelConfig::tiny_gemma4_moe(128, 64, 2, 8, 2),
57            ModelConfig::tiny_opt(128, 64, 2),
58        ];
59
60        let lb_cfg = LittleBitConfig {
61            quant_func: QuantFunc::SmoothSign { alpha: 100.0 },
62            eff_bit: Some(0.75),
63            residual: true,
64            kv_factor: 1.5,
65            min_split_dim: 8,
66            use_itq: true,
67            itq_n_iter: 10,
68            ..Default::default()
69        };
70
71        let input = Tensor::from_slice(&[1u32, 5, 19, 42, 7, 11], (2, 3), &dev)?;
72
73        for cfg in configs {
74            let arch = cfg.detect_architecture()?;
75            let mut model = CausalLm::new_random(cfg.clone(), DType::F32, &dev, 77)?;
76
77            let dense_out = model.forward(&input, None, HiddenStatesMode::All)?;
78            assert_eq!(dense_out.logits.dims(), &[2, 3, cfg.vocab_size]);
79            assert_eq!(dense_out.hidden_states.len(), cfg.num_hidden_layers + 1);
80
81            let summary = model.apply_littlebit_patch(&lb_cfg, TrainMode::Eval, 101)?;
82            assert!(summary.patched_layers > 0, "arch {arch:?} patched 0 layers");
83
84            if let CausalLm::Phi(ref phi) = model {
85                assert!(
86                    matches!(
87                        phi.layer_qkv_projection(0),
88                        Some(PhiAttentionProjection::Split { .. })
89                    ),
90                    "Phi must split qkv_proj during patching"
91                );
92            }
93
94            let out_unpacked = model.forward(&input, None, HiddenStatesMode::None)?;
95
96            let dir = TempDir::new().expect("tempdir");
97            let wrapper = LittleBitModel::new(
98                model,
99                lb_cfg.clone(),
100                Some(format!("test/{:?}", arch)),
101                None,
102            );
103            wrapper.save_pretrained(dir.path(), None, CastPolicy::Preserve)?;
104
105            let readme = wrapper.generate_readme("owner/test-littlebit");
106            assert!(readme.contains("LittleBit-2"));
107            assert!(readme.contains(&cfg.model_type));
108
109            let reloaded_unpacked = LittleBitModel::from_pretrained(
110                dir.path().to_str().expect("utf8 path"),
111                &LoadOptions {
112                    inference_mode: InferenceMode::Unpacked,
113                    ..Default::default()
114                },
115            )?;
116            let out_reloaded =
117                reloaded_unpacked
118                    .model
119                    .forward(&input, None, HiddenStatesMode::None)?;
120            let drift_unpacked = out_unpacked
121                .logits
122                .sub(&out_reloaded.logits)?
123                .abs()?
124                .max_all()?
125                .to_scalar::<f32>()?;
126            assert!(
127                drift_unpacked < 1e-4,
128                "{arch:?} unpacked reload drift: {drift_unpacked}"
129            );
130
131            let reloaded_packed = LittleBitModel::from_pretrained(
132                dir.path().to_str().expect("utf8 path"),
133                &LoadOptions {
134                    inference_mode: InferenceMode::Packed,
135                    ..Default::default()
136                },
137            )?;
138            let out_packed = reloaded_packed
139                .model
140                .forward(&input, None, HiddenStatesMode::None)?;
141            let drift_packed = out_unpacked
142                .logits
143                .sub(&out_packed.logits)?
144                .abs()?
145                .max_all()?
146                .to_scalar::<f32>()?;
147            assert!(
148                drift_packed < 1e-4,
149                "{arch:?} packed reload drift: {drift_packed}"
150            );
151        }
152        Ok(())
153    }
154
155    #[test]
156    fn kv_cache_decoding_matches_full_prefill() -> Result<()> {
157        let dev = Device::Cpu;
158        let configs = [
159            ModelConfig::tiny_llama(96, 64, 2),
160            ModelConfig::tiny_qwen3(96, 64, 2),
161            ModelConfig::tiny_phi3(96, 64, 2),
162            ModelConfig::tiny_gemma3(96, 64, 2),
163            ModelConfig::tiny_gemma4_dense(96, 64, 2),
164            ModelConfig::tiny_gemma4_moe(96, 64, 2, 8, 2),
165            ModelConfig::tiny_opt(96, 64, 2),
166        ];
167        let prompt = [3u32, 12, 7, 25, 14];
168
169        for cfg in configs {
170            let arch = cfg.detect_architecture()?;
171            let mut model = CausalLm::new_random(cfg.clone(), DType::F32, &dev, 55)?;
172            let lb_cfg = LittleBitConfig {
173                eff_bit: Some(1.0),
174                use_itq: true,
175                itq_n_iter: 5,
176                ..Default::default()
177            };
178            model.apply_littlebit_patch(&lb_cfg, TrainMode::Eval, 13)?;
179
180            let full_t = Tensor::from_slice(&prompt, (1, prompt.len()), &dev)?;
181            let full_logits = model
182                .forward(&full_t, None, HiddenStatesMode::None)?
183                .logits
184                .get(0)?
185                .get(prompt.len() - 1)?;
186
187            let mut cache = KvCache::new(cfg.num_hidden_layers);
188            let mut step_logits = None;
189            for &tok in &prompt {
190                let tok_t = Tensor::from_slice(&[tok], (1, 1), &dev)?;
191                let out = model.forward(&tok_t, Some(&mut cache), HiddenStatesMode::None)?;
192                step_logits = Some(out.logits.get(0)?.get(0)?);
193            }
194            let cached_last = step_logits.expect("last token logits");
195            let diff = full_logits
196                .sub(&cached_last)?
197                .abs()?
198                .max_all()?
199                .to_scalar::<f32>()?;
200            assert!(diff < 1e-4, "{arch:?} KV cache drift: {diff}");
201        }
202        Ok(())
203    }
204
205    #[test]
206    fn generate_produces_requested_length() -> Result<()> {
207        let dev = Device::Cpu;
208        let cfg = ModelConfig::tiny_llama(64, 32, 2);
209        let model = CausalLm::new_random(cfg, DType::F32, &dev, 9)?;
210        let out = model.generate(
211            &[1, 2, 3],
212            &GenerateConfig {
213                max_new_tokens: 5,
214                temperature: 0.0,
215                ..Default::default()
216            },
217            &dev,
218        )?;
219        assert_eq!(out.len(), 8);
220        Ok(())
221    }
222}