1pub mod config;
4pub mod error;
5pub mod gemma;
6pub mod gemma4;
7pub mod hub;
8pub mod layer;
9pub mod llama;
10pub mod model;
11pub mod opt;
12pub mod patch;
13pub mod phi;
14pub mod phi_attn;
15pub mod streaming;
16
17pub use config::{ModelArchitecture, ModelConfig, ResolvedRopeParams, TokenIdSpec};
18pub use error::{ModelError, Result};
19pub use gemma::GemmaModel;
20pub use gemma4::{
21 Gemma4Expert, Gemma4Experts, Gemma4MoE, Gemma4Model, Gemma4Router, RouterDispatch,
22};
23pub use hub::{load_dense_model, resolve_model_dir, LittleBitModel, LoadOptions, PushOptions};
24pub use layer::{
25 apply_activation, causal_mask, repeat_kv, Embedding, KvCache, LayerNorm, LinearLayer,
26 LinearLoadSpec, NormKind, RmsNorm, RmsNormNoScale, RopeMode, RotaryEmbedding,
27};
28pub use llama::LlamaModel;
29pub use model::{CausalLm, CausalLmOutput, GenerateConfig, HiddenStatesMode, ParameterStats};
30pub use opt::OptModel;
31pub use patch::{derive_layer_seed, patch_linear_layer, PatchPolicy, PatchSummary};
32pub use phi::PhiModel;
33pub use phi_attn::PhiAttentionProjection;
34pub use streaming::{
35 stream_quantize_checkpoint, StreamingQuantizeOptions, StreamingQuantizeReport,
36};
37
38#[cfg(test)]
39mod tests {
40 use super::*;
41 use candle_core::{DType, Device, Tensor};
42 use littlebit_core::{CastPolicy, InferenceMode, LittleBitConfig, QuantFunc, TrainMode};
43 use tempfile::TempDir;
44
45 #[test]
46 fn all_six_families_forward_patch_save_and_reload_packed() -> Result<()> {
47 let dev = Device::Cpu;
48 let configs = [
49 ModelConfig::tiny_llama(128, 64, 2),
50 ModelConfig::tiny_qwen2(128, 64, 2),
51 ModelConfig::tiny_qwen3(128, 64, 2),
52 ModelConfig::tiny_phi3(128, 64, 2),
53 ModelConfig::tiny_gemma2(128, 64, 2),
54 ModelConfig::tiny_gemma3(128, 64, 2),
55 ModelConfig::tiny_gemma4_dense(128, 64, 2),
56 ModelConfig::tiny_gemma4_moe(128, 64, 2, 8, 2),
57 ModelConfig::tiny_opt(128, 64, 2),
58 ];
59
60 let lb_cfg = LittleBitConfig {
61 quant_func: QuantFunc::SmoothSign { alpha: 100.0 },
62 eff_bit: Some(0.75),
63 residual: true,
64 kv_factor: 1.5,
65 min_split_dim: 8,
66 use_itq: true,
67 itq_n_iter: 10,
68 ..Default::default()
69 };
70
71 let input = Tensor::from_slice(&[1u32, 5, 19, 42, 7, 11], (2, 3), &dev)?;
72
73 for cfg in configs {
74 let arch = cfg.detect_architecture()?;
75 let mut model = CausalLm::new_random(cfg.clone(), DType::F32, &dev, 77)?;
76
77 let dense_out = model.forward(&input, None, HiddenStatesMode::All)?;
78 assert_eq!(dense_out.logits.dims(), &[2, 3, cfg.vocab_size]);
79 assert_eq!(dense_out.hidden_states.len(), cfg.num_hidden_layers + 1);
80
81 let summary = model.apply_littlebit_patch(&lb_cfg, TrainMode::Eval, 101)?;
82 assert!(summary.patched_layers > 0, "arch {arch:?} patched 0 layers");
83
84 if let CausalLm::Phi(ref phi) = model {
85 assert!(
86 matches!(
87 phi.layer_qkv_projection(0),
88 Some(PhiAttentionProjection::Split { .. })
89 ),
90 "Phi must split qkv_proj during patching"
91 );
92 }
93
94 let out_unpacked = model.forward(&input, None, HiddenStatesMode::None)?;
95
96 let dir = TempDir::new().expect("tempdir");
97 let wrapper = LittleBitModel::new(
98 model,
99 lb_cfg.clone(),
100 Some(format!("test/{:?}", arch)),
101 None,
102 );
103 wrapper.save_pretrained(dir.path(), None, CastPolicy::Preserve)?;
104
105 let readme = wrapper.generate_readme("owner/test-littlebit");
106 assert!(readme.contains("LittleBit-2"));
107 assert!(readme.contains(&cfg.model_type));
108
109 let reloaded_unpacked = LittleBitModel::from_pretrained(
110 dir.path().to_str().expect("utf8 path"),
111 &LoadOptions {
112 inference_mode: InferenceMode::Unpacked,
113 ..Default::default()
114 },
115 )?;
116 let out_reloaded =
117 reloaded_unpacked
118 .model
119 .forward(&input, None, HiddenStatesMode::None)?;
120 let drift_unpacked = out_unpacked
121 .logits
122 .sub(&out_reloaded.logits)?
123 .abs()?
124 .max_all()?
125 .to_scalar::<f32>()?;
126 assert!(
127 drift_unpacked < 1e-4,
128 "{arch:?} unpacked reload drift: {drift_unpacked}"
129 );
130
131 let reloaded_packed = LittleBitModel::from_pretrained(
132 dir.path().to_str().expect("utf8 path"),
133 &LoadOptions {
134 inference_mode: InferenceMode::Packed,
135 ..Default::default()
136 },
137 )?;
138 let out_packed = reloaded_packed
139 .model
140 .forward(&input, None, HiddenStatesMode::None)?;
141 let drift_packed = out_unpacked
142 .logits
143 .sub(&out_packed.logits)?
144 .abs()?
145 .max_all()?
146 .to_scalar::<f32>()?;
147 assert!(
148 drift_packed < 1e-4,
149 "{arch:?} packed reload drift: {drift_packed}"
150 );
151 }
152 Ok(())
153 }
154
155 #[test]
156 fn kv_cache_decoding_matches_full_prefill() -> Result<()> {
157 let dev = Device::Cpu;
158 let configs = [
159 ModelConfig::tiny_llama(96, 64, 2),
160 ModelConfig::tiny_qwen3(96, 64, 2),
161 ModelConfig::tiny_phi3(96, 64, 2),
162 ModelConfig::tiny_gemma3(96, 64, 2),
163 ModelConfig::tiny_gemma4_dense(96, 64, 2),
164 ModelConfig::tiny_gemma4_moe(96, 64, 2, 8, 2),
165 ModelConfig::tiny_opt(96, 64, 2),
166 ];
167 let prompt = [3u32, 12, 7, 25, 14];
168
169 for cfg in configs {
170 let arch = cfg.detect_architecture()?;
171 let mut model = CausalLm::new_random(cfg.clone(), DType::F32, &dev, 55)?;
172 let lb_cfg = LittleBitConfig {
173 eff_bit: Some(1.0),
174 use_itq: true,
175 itq_n_iter: 5,
176 ..Default::default()
177 };
178 model.apply_littlebit_patch(&lb_cfg, TrainMode::Eval, 13)?;
179
180 let full_t = Tensor::from_slice(&prompt, (1, prompt.len()), &dev)?;
181 let full_logits = model
182 .forward(&full_t, None, HiddenStatesMode::None)?
183 .logits
184 .get(0)?
185 .get(prompt.len() - 1)?;
186
187 let mut cache = KvCache::new(cfg.num_hidden_layers);
188 let mut step_logits = None;
189 for &tok in &prompt {
190 let tok_t = Tensor::from_slice(&[tok], (1, 1), &dev)?;
191 let out = model.forward(&tok_t, Some(&mut cache), HiddenStatesMode::None)?;
192 step_logits = Some(out.logits.get(0)?.get(0)?);
193 }
194 let cached_last = step_logits.expect("last token logits");
195 let diff = full_logits
196 .sub(&cached_last)?
197 .abs()?
198 .max_all()?
199 .to_scalar::<f32>()?;
200 assert!(diff < 1e-4, "{arch:?} KV cache drift: {diff}");
201 }
202 Ok(())
203 }
204
205 #[test]
206 fn generate_produces_requested_length() -> Result<()> {
207 let dev = Device::Cpu;
208 let cfg = ModelConfig::tiny_llama(64, 32, 2);
209 let model = CausalLm::new_random(cfg, DType::F32, &dev, 9)?;
210 let out = model.generate(
211 &[1, 2, 3],
212 &GenerateConfig {
213 max_new_tokens: 5,
214 temperature: 0.0,
215 ..Default::default()
216 },
217 &dev,
218 )?;
219 assert_eq!(out.len(), 8);
220 Ok(())
221 }
222}