Skip to main content

littlebit_models/
model.rs

1//! Unified [`CausalLm`] abstraction dispatching across all supported LLM families.
2
3use std::collections::HashMap;
4use std::fmt;
5
6use candle_core::{DType, Device, Tensor, Var};
7use rand::Rng;
8use rand::SeedableRng;
9use rand_chacha::ChaCha8Rng;
10
11use littlebit_core::{InferenceMode, LittleBitConfig, TrainMode, WeightState};
12
13use crate::config::{ModelArchitecture, ModelConfig};
14use crate::error::Result;
15use crate::gemma::GemmaModel;
16use crate::gemma4::Gemma4Model;
17use crate::layer::{KvCache, LinearLoadSpec};
18use crate::llama::LlamaModel;
19use crate::opt::OptModel;
20use crate::patch::{PatchPolicy, PatchSummary};
21use crate::phi::PhiModel;
22
23/// Whether [`CausalLm::forward`] should collect per-layer hidden states (used during KD training).
24#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
25pub enum HiddenStatesMode {
26    /// Return only output logits (`hidden_states` vector is empty).
27    None,
28    /// Collect embeddings and per-layer hidden outputs (`len == num_hidden_layers + 1`).
29    All,
30}
31
32/// Output returned by [`CausalLm::forward`].
33#[derive(Debug, Clone)]
34pub struct CausalLmOutput {
35    /// Unnormalized vocabulary logits `(batch, seq_len, vocab_size)`.
36    pub logits: Tensor,
37    /// Optional hidden states `[embed, layer_0, ..., final_norm]` when [`HiddenStatesMode::All`].
38    pub hidden_states: Vec<Tensor>,
39}
40
41/// Trainable vs. total parameter summary matching `utils/utils.py::print_trainable_parameters`.
42#[derive(Debug, Clone, Copy, PartialEq, Eq)]
43pub struct ParameterStats {
44    /// Count of scalar elements inside trainable `Var`s.
45    pub trainable_params: usize,
46    /// Approximate total scalar parameters across the model.
47    pub all_params: usize,
48}
49
50impl ParameterStats {
51    /// Percentage of parameters that are trainable (`100 * trainable / all`).
52    #[must_use]
53    pub fn trainable_percent(self) -> f64 {
54        if self.all_params == 0 {
55            0.0
56        } else {
57            100.0 * (self.trainable_params as f64) / (self.all_params as f64)
58        }
59    }
60}
61
62impl fmt::Display for ParameterStats {
63    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
64        write!(
65            f,
66            "trainable params: {} || all params: {} || trainable%: {:.4}",
67            self.trainable_params,
68            self.all_params,
69            self.trainable_percent()
70        )
71    }
72}
73
74/// Sampling and termination configuration for [`CausalLm::generate`].
75#[derive(Debug, Clone)]
76pub struct GenerateConfig {
77    /// Maximum number of new tokens to decode.
78    pub max_new_tokens: usize,
79    /// Sampling temperature (`<= 0.0` triggers greedy argmax decoding).
80    pub temperature: f64,
81    /// Nucleus (top-p) probability threshold in `(0.0, 1.0]`.
82    pub top_p: f64,
83    /// Deterministic RNG seed for stochastic sampling.
84    pub seed: u64,
85    /// Optional stop token IDs halting generation early.
86    pub eos_token_ids: Vec<u32>,
87}
88
89impl Default for GenerateConfig {
90    fn default() -> Self {
91        Self {
92            max_new_tokens: 64,
93            temperature: 0.0,
94            top_p: 1.0,
95            seed: 42,
96            eos_token_ids: Vec::new(),
97        }
98    }
99}
100
101/// Unified causal language model supporting all architectures in `SamsungLabs/LittleBit` plus Gemma 4.
102#[derive(Debug, Clone)]
103pub enum CausalLm {
104    /// Llama, Llama 2/3, Qwen2/2.5/QwQ, and Qwen3.
105    LlamaFamily(LlamaModel),
106    /// Phi-3 and Phi-4 (including `PhiQKVSplitAttention`).
107    Phi(PhiModel),
108    /// Gemma 2 and Gemma 3.
109    Gemma(GemmaModel),
110    /// Gemma 4 Dense and Mixture-of-Experts.
111    Gemma4(Box<Gemma4Model>),
112    /// Meta OPT.
113    Opt(Box<OptModel>),
114}
115
116impl CausalLm {
117    /// Construct a model from a parsed [`ModelConfig`] and loaded checkpoint tensors.
118    ///
119    /// # Errors
120    /// Returns an error if the architecture is unsupported or required weights are missing.
121    pub fn from_tensors(
122        config: ModelConfig,
123        tensors: &HashMap<String, Tensor>,
124        littlebit: Option<&LittleBitConfig>,
125        weight_state: WeightState,
126        inference_mode: InferenceMode,
127        dtype: DType,
128        device: &Device,
129    ) -> Result<Self> {
130        let config = config.resolve_effective();
131        let arch = config.detect_architecture()?;
132        let spec = LinearLoadSpec {
133            littlebit,
134            ratio_factor: 1.0,
135            weight_state,
136            inference_mode,
137            dtype,
138            device,
139        };
140        match arch {
141            ModelArchitecture::Llama | ModelArchitecture::Qwen2 | ModelArchitecture::Qwen3 => Ok(
142                Self::LlamaFamily(LlamaModel::from_tensors(config, tensors, spec)?),
143            ),
144            ModelArchitecture::Phi3 => {
145                Ok(Self::Phi(PhiModel::from_tensors(config, tensors, spec)?))
146            }
147            ModelArchitecture::Gemma2 | ModelArchitecture::Gemma3 => Ok(Self::Gemma(
148                GemmaModel::from_tensors(config, tensors, spec)?,
149            )),
150            ModelArchitecture::Gemma4 => Ok(Self::Gemma4(Box::new(Gemma4Model::from_tensors(
151                config, tensors, spec,
152            )?))),
153            ModelArchitecture::Opt => Ok(Self::Opt(Box::new(OptModel::from_tensors(
154                config, tensors, spec,
155            )?))),
156        }
157    }
158
159    /// Initialize a compact or full-sized random model for testing and benchmarking.
160    ///
161    /// # Errors
162    /// Returns an error if architecture detection or tensor allocation fails.
163    pub fn new_random(
164        config: ModelConfig,
165        dtype: DType,
166        device: &Device,
167        seed: u64,
168    ) -> Result<Self> {
169        let config = config.resolve_effective();
170        let arch = config.detect_architecture()?;
171        let mut rng = ChaCha8Rng::seed_from_u64(seed);
172        match arch {
173            ModelArchitecture::Llama | ModelArchitecture::Qwen2 | ModelArchitecture::Qwen3 => Ok(
174                Self::LlamaFamily(LlamaModel::new_random(config, dtype, device, &mut rng)?),
175            ),
176            ModelArchitecture::Phi3 => Ok(Self::Phi(PhiModel::new_random(
177                config, dtype, device, &mut rng,
178            )?)),
179            ModelArchitecture::Gemma2 | ModelArchitecture::Gemma3 => Ok(Self::Gemma(
180                GemmaModel::new_random(config, dtype, device, &mut rng)?,
181            )),
182            ModelArchitecture::Gemma4 => Ok(Self::Gemma4(Box::new(Gemma4Model::new_random(
183                config, dtype, device, &mut rng,
184            )?))),
185            ModelArchitecture::Opt => Ok(Self::Opt(Box::new(OptModel::new_random(
186                config, dtype, device, &mut rng,
187            )?))),
188        }
189    }
190
191    /// Execute causal forward pass over `input_ids` `(batch, seq_len)`.
192    ///
193    /// # Errors
194    /// Returns an error if layer computation fails.
195    pub fn forward(
196        &self,
197        input_ids: &Tensor,
198        cache: Option<&mut KvCache>,
199        hidden_mode: HiddenStatesMode,
200    ) -> Result<CausalLmOutput> {
201        match self {
202            Self::LlamaFamily(m) => m.forward(input_ids, cache, hidden_mode),
203            Self::Phi(m) => m.forward(input_ids, cache, hidden_mode),
204            Self::Gemma(m) => m.forward(input_ids, cache, hidden_mode),
205            Self::Gemma4(m) => m.forward(input_ids, cache, hidden_mode),
206            Self::Opt(m) => m.forward(input_ids, cache, hidden_mode),
207        }
208    }
209
210    /// Apply LittleBit / LittleBit-2 quantization patch across all linear projections
211    /// (excluding `lm_head`, `router`, splitting `qkv_proj` on Phi, and scaling `k_proj`/`v_proj` by `kv_factor`).
212    ///
213    /// # Errors
214    /// Returns an error if config validation or SVD/ITQ decomposition fails.
215    pub fn apply_littlebit_patch(
216        &mut self,
217        lb_cfg: &LittleBitConfig,
218        train_mode: TrainMode,
219        base_seed: u64,
220    ) -> Result<PatchSummary> {
221        let policy = PatchPolicy::new();
222        let mut summary = PatchSummary::default();
223        match self {
224            Self::LlamaFamily(m) => {
225                m.apply_littlebit_patch(lb_cfg, &policy, train_mode, base_seed, &mut summary)?;
226            }
227            Self::Phi(m) => {
228                m.apply_littlebit_patch(lb_cfg, &policy, train_mode, base_seed, &mut summary)?;
229            }
230            Self::Gemma(m) => {
231                m.apply_littlebit_patch(lb_cfg, &policy, train_mode, base_seed, &mut summary)?;
232            }
233            Self::Gemma4(m) => {
234                m.apply_littlebit_patch(lb_cfg, &policy, train_mode, base_seed, &mut summary)?;
235            }
236            Self::Opt(m) => {
237                m.apply_littlebit_patch(lb_cfg, &policy, train_mode, base_seed, &mut summary)?;
238            }
239        }
240        Ok(summary)
241    }
242
243    /// Convert all `LittleBitLinear` layers into memory-compact 1-bit `PackedLittleBitLinear` modules.
244    ///
245    /// # Errors
246    /// Returns an error if packing fails.
247    pub fn convert_to_packed(&mut self) -> Result<()> {
248        match self {
249            Self::LlamaFamily(m) => m.convert_to_packed(),
250            Self::Phi(m) => m.convert_to_packed(),
251            Self::Gemma(m) => m.convert_to_packed(),
252            Self::Gemma4(m) => m.convert_to_packed(),
253            Self::Opt(m) => m.convert_to_packed(),
254        }
255    }
256
257    /// Prepare model parameters for QAT training, making all non-excluded parameters trainable
258    /// and freezing parameters whose module path contains any pattern in `exclude`.
259    ///
260    /// Matches upstream `utils/utils.py::prepare_model_for_training`.
261    ///
262    /// # Errors
263    /// Returns an error if wrapping parameters in [`Var`] fails.
264    pub fn prepare_for_training(&mut self, exclude: &[&str]) -> Result<()> {
265        match self {
266            Self::LlamaFamily(m) => m.make_non_quantized_trainable()?,
267            Self::Phi(m) => m.make_non_quantized_trainable()?,
268            Self::Gemma(m) => m.make_non_quantized_trainable()?,
269            Self::Gemma4(m) => m.make_non_quantized_trainable()?,
270            Self::Opt(m) => m.make_non_quantized_trainable()?,
271        }
272        self.freeze_matching(exclude);
273        Ok(())
274    }
275
276    /// Freeze all parameters matching any substring in `exclude`.
277    pub fn freeze_matching(&mut self, exclude: &[&str]) {
278        match self {
279            Self::LlamaFamily(m) => m.freeze_matching(exclude),
280            Self::Phi(m) => m.freeze_matching(exclude),
281            Self::Gemma(m) => m.freeze_matching(exclude),
282            Self::Gemma4(m) => m.freeze_matching(exclude),
283            Self::Opt(m) => m.freeze_matching(exclude),
284        }
285    }
286
287    /// Collect all active trainable [`Var`] parameters across the model.
288    #[must_use]
289    pub fn trainable_vars(&self) -> Vec<Var> {
290        match self {
291            Self::LlamaFamily(m) => m.trainable_vars(),
292            Self::Phi(m) => m.trainable_vars(),
293            Self::Gemma(m) => m.trainable_vars(),
294            Self::Gemma4(m) => m.trainable_vars(),
295            Self::Opt(m) => m.trainable_vars(),
296        }
297    }
298
299    /// Compute trainable and total scalar parameter counts.
300    #[must_use]
301    pub fn parameter_stats(&self) -> ParameterStats {
302        let trainable_params: usize = self.trainable_vars().iter().map(|v| v.elem_count()).sum();
303        let all_params = match self {
304            Self::LlamaFamily(m) => m.total_params(),
305            Self::Phi(m) => m.total_params(),
306            Self::Gemma(m) => m.total_params(),
307            Self::Gemma4(m) => m.total_params(),
308            Self::Opt(m) => m.total_params(),
309        };
310        ParameterStats {
311            trainable_params,
312            all_params,
313        }
314    }
315
316    /// Export full model state dictionary with LittleBit layers binarized & 1-bit packed.
317    ///
318    /// # Errors
319    /// Returns an error if packing any layer fails.
320    pub fn pack_state_dict(&self) -> Result<HashMap<String, Tensor>> {
321        match self {
322            Self::LlamaFamily(m) => m.pack_state_dict(),
323            Self::Phi(m) => m.pack_state_dict(),
324            Self::Gemma(m) => m.pack_state_dict(),
325            Self::Gemma4(m) => m.pack_state_dict(),
326            Self::Opt(m) => m.pack_state_dict(),
327        }
328    }
329
330    /// Borrow underlying Hugging Face [`ModelConfig`].
331    #[must_use]
332    pub fn config(&self) -> &ModelConfig {
333        match self {
334            Self::LlamaFamily(m) => m.config(),
335            Self::Phi(m) => m.config(),
336            Self::Gemma(m) => m.config(),
337            Self::Gemma4(m) => m.config(),
338            Self::Opt(m) => m.config(),
339        }
340    }
341
342    /// Autoregressive text generation using [`KvCache`] given initial `prompt_ids`.
343    ///
344    /// # Errors
345    /// Returns an error if `prompt_ids` is empty or forward/sampling fails.
346    pub fn generate(
347        &self,
348        prompt_ids: &[u32],
349        cfg: &GenerateConfig,
350        device: &Device,
351    ) -> Result<Vec<u32>> {
352        if prompt_ids.is_empty() {
353            return Err(crate::error::ModelError::InvalidArgument(
354                "prompt_ids must not be empty".to_string(),
355            ));
356        }
357
358        let mut cache = KvCache::new(self.config().num_hidden_layers);
359        let mut generated = prompt_ids.to_vec();
360        let mut rng = ChaCha8Rng::seed_from_u64(cfg.seed);
361
362        let prompt_t = Tensor::from_slice(prompt_ids, (1, prompt_ids.len()), device)?;
363        let out = self.forward(&prompt_t, Some(&mut cache), HiddenStatesMode::None)?;
364        let last_pos = prompt_ids.len() - 1;
365        let mut next_logits = out.logits.get(0)?.get(last_pos)?;
366
367        for _ in 0..cfg.max_new_tokens {
368            let next_id = sample_token(&next_logits, cfg.temperature, cfg.top_p, &mut rng)?;
369            generated.push(next_id);
370            if cfg.eos_token_ids.contains(&next_id) {
371                break;
372            }
373            let step_t = Tensor::from_slice(&[next_id], (1, 1), device)?;
374            let step_out = self.forward(&step_t, Some(&mut cache), HiddenStatesMode::None)?;
375            next_logits = step_out.logits.get(0)?.get(0)?;
376        }
377
378        Ok(generated)
379    }
380}
381
382fn sample_token(
383    logits: &Tensor,
384    temperature: f64,
385    top_p: f64,
386    rng: &mut ChaCha8Rng,
387) -> Result<u32> {
388    let probs_vec = logits.to_dtype(DType::F32)?.to_vec1::<f32>()?;
389    if probs_vec.is_empty() {
390        return Err(crate::error::ModelError::InvalidArgument(
391            "empty vocabulary logits".to_string(),
392        ));
393    }
394
395    if temperature <= 0.0 {
396        let mut best_idx = 0usize;
397        let mut best_val = f32::NEG_INFINITY;
398        for (i, &v) in probs_vec.iter().enumerate() {
399            if v > best_val {
400                best_val = v;
401                best_idx = i;
402            }
403        }
404        return Ok(best_idx as u32);
405    }
406
407    let inv_temp = 1.0 / temperature;
408    let max_logit = probs_vec.iter().copied().fold(f32::NEG_INFINITY, f32::max) as f64;
409    let mut scaled: Vec<(usize, f64)> = probs_vec
410        .iter()
411        .enumerate()
412        .map(|(i, &v)| (i, ((v as f64 - max_logit) * inv_temp).exp()))
413        .collect();
414    let sum: f64 = scaled.iter().map(|&(_, p)| p).sum();
415    if sum <= 0.0 || !sum.is_finite() {
416        return Ok(0);
417    }
418    for (_, p) in &mut scaled {
419        *p /= sum;
420    }
421
422    if top_p < 1.0 && top_p > 0.0 {
423        scaled.sort_by(|a, b| b.1.total_cmp(&a.1));
424        let mut cum = 0.0_f64;
425        let mut cutoff = scaled.len();
426        for (idx, &(_, p)) in scaled.iter().enumerate() {
427            cum += p;
428            if cum >= top_p {
429                cutoff = idx + 1;
430                break;
431            }
432        }
433        scaled.truncate(cutoff.max(1));
434        let norm: f64 = scaled.iter().map(|&(_, p)| p).sum();
435        for (_, p) in &mut scaled {
436            *p /= norm;
437        }
438    }
439
440    let draw: f64 = rng.random();
441    let mut acc = 0.0_f64;
442    for &(idx, p) in &scaled {
443        acc += p;
444        if draw <= acc {
445            return Ok(idx as u32);
446        }
447    }
448    Ok(scaled.last().map_or(0, |&(idx, _)| idx as u32))
449}