1use std::collections::HashMap;
4use std::fmt;
5
6use candle_core::{DType, Device, Tensor, Var};
7use rand::Rng;
8use rand::SeedableRng;
9use rand_chacha::ChaCha8Rng;
10
11use littlebit_core::{InferenceMode, LittleBitConfig, TrainMode, WeightState};
12
13use crate::config::{ModelArchitecture, ModelConfig};
14use crate::error::Result;
15use crate::gemma::GemmaModel;
16use crate::gemma4::Gemma4Model;
17use crate::layer::{KvCache, LinearLoadSpec};
18use crate::llama::LlamaModel;
19use crate::opt::OptModel;
20use crate::patch::{PatchPolicy, PatchSummary};
21use crate::phi::PhiModel;
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
25pub enum HiddenStatesMode {
26 None,
28 All,
30}
31
32#[derive(Debug, Clone)]
34pub struct CausalLmOutput {
35 pub logits: Tensor,
37 pub hidden_states: Vec<Tensor>,
39}
40
41#[derive(Debug, Clone, Copy, PartialEq, Eq)]
43pub struct ParameterStats {
44 pub trainable_params: usize,
46 pub all_params: usize,
48}
49
50impl ParameterStats {
51 #[must_use]
53 pub fn trainable_percent(self) -> f64 {
54 if self.all_params == 0 {
55 0.0
56 } else {
57 100.0 * (self.trainable_params as f64) / (self.all_params as f64)
58 }
59 }
60}
61
62impl fmt::Display for ParameterStats {
63 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
64 write!(
65 f,
66 "trainable params: {} || all params: {} || trainable%: {:.4}",
67 self.trainable_params,
68 self.all_params,
69 self.trainable_percent()
70 )
71 }
72}
73
74#[derive(Debug, Clone)]
76pub struct GenerateConfig {
77 pub max_new_tokens: usize,
79 pub temperature: f64,
81 pub top_p: f64,
83 pub seed: u64,
85 pub eos_token_ids: Vec<u32>,
87}
88
89impl Default for GenerateConfig {
90 fn default() -> Self {
91 Self {
92 max_new_tokens: 64,
93 temperature: 0.0,
94 top_p: 1.0,
95 seed: 42,
96 eos_token_ids: Vec::new(),
97 }
98 }
99}
100
101#[derive(Debug, Clone)]
103pub enum CausalLm {
104 LlamaFamily(LlamaModel),
106 Phi(PhiModel),
108 Gemma(GemmaModel),
110 Gemma4(Box<Gemma4Model>),
112 Opt(Box<OptModel>),
114}
115
116impl CausalLm {
117 pub fn from_tensors(
122 config: ModelConfig,
123 tensors: &HashMap<String, Tensor>,
124 littlebit: Option<&LittleBitConfig>,
125 weight_state: WeightState,
126 inference_mode: InferenceMode,
127 dtype: DType,
128 device: &Device,
129 ) -> Result<Self> {
130 let config = config.resolve_effective();
131 let arch = config.detect_architecture()?;
132 let spec = LinearLoadSpec {
133 littlebit,
134 ratio_factor: 1.0,
135 weight_state,
136 inference_mode,
137 dtype,
138 device,
139 };
140 match arch {
141 ModelArchitecture::Llama | ModelArchitecture::Qwen2 | ModelArchitecture::Qwen3 => Ok(
142 Self::LlamaFamily(LlamaModel::from_tensors(config, tensors, spec)?),
143 ),
144 ModelArchitecture::Phi3 => {
145 Ok(Self::Phi(PhiModel::from_tensors(config, tensors, spec)?))
146 }
147 ModelArchitecture::Gemma2 | ModelArchitecture::Gemma3 => Ok(Self::Gemma(
148 GemmaModel::from_tensors(config, tensors, spec)?,
149 )),
150 ModelArchitecture::Gemma4 => Ok(Self::Gemma4(Box::new(Gemma4Model::from_tensors(
151 config, tensors, spec,
152 )?))),
153 ModelArchitecture::Opt => Ok(Self::Opt(Box::new(OptModel::from_tensors(
154 config, tensors, spec,
155 )?))),
156 }
157 }
158
159 pub fn new_random(
164 config: ModelConfig,
165 dtype: DType,
166 device: &Device,
167 seed: u64,
168 ) -> Result<Self> {
169 let config = config.resolve_effective();
170 let arch = config.detect_architecture()?;
171 let mut rng = ChaCha8Rng::seed_from_u64(seed);
172 match arch {
173 ModelArchitecture::Llama | ModelArchitecture::Qwen2 | ModelArchitecture::Qwen3 => Ok(
174 Self::LlamaFamily(LlamaModel::new_random(config, dtype, device, &mut rng)?),
175 ),
176 ModelArchitecture::Phi3 => Ok(Self::Phi(PhiModel::new_random(
177 config, dtype, device, &mut rng,
178 )?)),
179 ModelArchitecture::Gemma2 | ModelArchitecture::Gemma3 => Ok(Self::Gemma(
180 GemmaModel::new_random(config, dtype, device, &mut rng)?,
181 )),
182 ModelArchitecture::Gemma4 => Ok(Self::Gemma4(Box::new(Gemma4Model::new_random(
183 config, dtype, device, &mut rng,
184 )?))),
185 ModelArchitecture::Opt => Ok(Self::Opt(Box::new(OptModel::new_random(
186 config, dtype, device, &mut rng,
187 )?))),
188 }
189 }
190
191 pub fn forward(
196 &self,
197 input_ids: &Tensor,
198 cache: Option<&mut KvCache>,
199 hidden_mode: HiddenStatesMode,
200 ) -> Result<CausalLmOutput> {
201 match self {
202 Self::LlamaFamily(m) => m.forward(input_ids, cache, hidden_mode),
203 Self::Phi(m) => m.forward(input_ids, cache, hidden_mode),
204 Self::Gemma(m) => m.forward(input_ids, cache, hidden_mode),
205 Self::Gemma4(m) => m.forward(input_ids, cache, hidden_mode),
206 Self::Opt(m) => m.forward(input_ids, cache, hidden_mode),
207 }
208 }
209
210 pub fn apply_littlebit_patch(
216 &mut self,
217 lb_cfg: &LittleBitConfig,
218 train_mode: TrainMode,
219 base_seed: u64,
220 ) -> Result<PatchSummary> {
221 let policy = PatchPolicy::new();
222 let mut summary = PatchSummary::default();
223 match self {
224 Self::LlamaFamily(m) => {
225 m.apply_littlebit_patch(lb_cfg, &policy, train_mode, base_seed, &mut summary)?;
226 }
227 Self::Phi(m) => {
228 m.apply_littlebit_patch(lb_cfg, &policy, train_mode, base_seed, &mut summary)?;
229 }
230 Self::Gemma(m) => {
231 m.apply_littlebit_patch(lb_cfg, &policy, train_mode, base_seed, &mut summary)?;
232 }
233 Self::Gemma4(m) => {
234 m.apply_littlebit_patch(lb_cfg, &policy, train_mode, base_seed, &mut summary)?;
235 }
236 Self::Opt(m) => {
237 m.apply_littlebit_patch(lb_cfg, &policy, train_mode, base_seed, &mut summary)?;
238 }
239 }
240 Ok(summary)
241 }
242
243 pub fn convert_to_packed(&mut self) -> Result<()> {
248 match self {
249 Self::LlamaFamily(m) => m.convert_to_packed(),
250 Self::Phi(m) => m.convert_to_packed(),
251 Self::Gemma(m) => m.convert_to_packed(),
252 Self::Gemma4(m) => m.convert_to_packed(),
253 Self::Opt(m) => m.convert_to_packed(),
254 }
255 }
256
257 pub fn prepare_for_training(&mut self, exclude: &[&str]) -> Result<()> {
265 match self {
266 Self::LlamaFamily(m) => m.make_non_quantized_trainable()?,
267 Self::Phi(m) => m.make_non_quantized_trainable()?,
268 Self::Gemma(m) => m.make_non_quantized_trainable()?,
269 Self::Gemma4(m) => m.make_non_quantized_trainable()?,
270 Self::Opt(m) => m.make_non_quantized_trainable()?,
271 }
272 self.freeze_matching(exclude);
273 Ok(())
274 }
275
276 pub fn freeze_matching(&mut self, exclude: &[&str]) {
278 match self {
279 Self::LlamaFamily(m) => m.freeze_matching(exclude),
280 Self::Phi(m) => m.freeze_matching(exclude),
281 Self::Gemma(m) => m.freeze_matching(exclude),
282 Self::Gemma4(m) => m.freeze_matching(exclude),
283 Self::Opt(m) => m.freeze_matching(exclude),
284 }
285 }
286
287 #[must_use]
289 pub fn trainable_vars(&self) -> Vec<Var> {
290 match self {
291 Self::LlamaFamily(m) => m.trainable_vars(),
292 Self::Phi(m) => m.trainable_vars(),
293 Self::Gemma(m) => m.trainable_vars(),
294 Self::Gemma4(m) => m.trainable_vars(),
295 Self::Opt(m) => m.trainable_vars(),
296 }
297 }
298
299 #[must_use]
301 pub fn parameter_stats(&self) -> ParameterStats {
302 let trainable_params: usize = self.trainable_vars().iter().map(|v| v.elem_count()).sum();
303 let all_params = match self {
304 Self::LlamaFamily(m) => m.total_params(),
305 Self::Phi(m) => m.total_params(),
306 Self::Gemma(m) => m.total_params(),
307 Self::Gemma4(m) => m.total_params(),
308 Self::Opt(m) => m.total_params(),
309 };
310 ParameterStats {
311 trainable_params,
312 all_params,
313 }
314 }
315
316 pub fn pack_state_dict(&self) -> Result<HashMap<String, Tensor>> {
321 match self {
322 Self::LlamaFamily(m) => m.pack_state_dict(),
323 Self::Phi(m) => m.pack_state_dict(),
324 Self::Gemma(m) => m.pack_state_dict(),
325 Self::Gemma4(m) => m.pack_state_dict(),
326 Self::Opt(m) => m.pack_state_dict(),
327 }
328 }
329
330 #[must_use]
332 pub fn config(&self) -> &ModelConfig {
333 match self {
334 Self::LlamaFamily(m) => m.config(),
335 Self::Phi(m) => m.config(),
336 Self::Gemma(m) => m.config(),
337 Self::Gemma4(m) => m.config(),
338 Self::Opt(m) => m.config(),
339 }
340 }
341
342 pub fn generate(
347 &self,
348 prompt_ids: &[u32],
349 cfg: &GenerateConfig,
350 device: &Device,
351 ) -> Result<Vec<u32>> {
352 if prompt_ids.is_empty() {
353 return Err(crate::error::ModelError::InvalidArgument(
354 "prompt_ids must not be empty".to_string(),
355 ));
356 }
357
358 let mut cache = KvCache::new(self.config().num_hidden_layers);
359 let mut generated = prompt_ids.to_vec();
360 let mut rng = ChaCha8Rng::seed_from_u64(cfg.seed);
361
362 let prompt_t = Tensor::from_slice(prompt_ids, (1, prompt_ids.len()), device)?;
363 let out = self.forward(&prompt_t, Some(&mut cache), HiddenStatesMode::None)?;
364 let last_pos = prompt_ids.len() - 1;
365 let mut next_logits = out.logits.get(0)?.get(last_pos)?;
366
367 for _ in 0..cfg.max_new_tokens {
368 let next_id = sample_token(&next_logits, cfg.temperature, cfg.top_p, &mut rng)?;
369 generated.push(next_id);
370 if cfg.eos_token_ids.contains(&next_id) {
371 break;
372 }
373 let step_t = Tensor::from_slice(&[next_id], (1, 1), device)?;
374 let step_out = self.forward(&step_t, Some(&mut cache), HiddenStatesMode::None)?;
375 next_logits = step_out.logits.get(0)?.get(0)?;
376 }
377
378 Ok(generated)
379 }
380}
381
382fn sample_token(
383 logits: &Tensor,
384 temperature: f64,
385 top_p: f64,
386 rng: &mut ChaCha8Rng,
387) -> Result<u32> {
388 let probs_vec = logits.to_dtype(DType::F32)?.to_vec1::<f32>()?;
389 if probs_vec.is_empty() {
390 return Err(crate::error::ModelError::InvalidArgument(
391 "empty vocabulary logits".to_string(),
392 ));
393 }
394
395 if temperature <= 0.0 {
396 let mut best_idx = 0usize;
397 let mut best_val = f32::NEG_INFINITY;
398 for (i, &v) in probs_vec.iter().enumerate() {
399 if v > best_val {
400 best_val = v;
401 best_idx = i;
402 }
403 }
404 return Ok(best_idx as u32);
405 }
406
407 let inv_temp = 1.0 / temperature;
408 let max_logit = probs_vec.iter().copied().fold(f32::NEG_INFINITY, f32::max) as f64;
409 let mut scaled: Vec<(usize, f64)> = probs_vec
410 .iter()
411 .enumerate()
412 .map(|(i, &v)| (i, ((v as f64 - max_logit) * inv_temp).exp()))
413 .collect();
414 let sum: f64 = scaled.iter().map(|&(_, p)| p).sum();
415 if sum <= 0.0 || !sum.is_finite() {
416 return Ok(0);
417 }
418 for (_, p) in &mut scaled {
419 *p /= sum;
420 }
421
422 if top_p < 1.0 && top_p > 0.0 {
423 scaled.sort_by(|a, b| b.1.total_cmp(&a.1));
424 let mut cum = 0.0_f64;
425 let mut cutoff = scaled.len();
426 for (idx, &(_, p)) in scaled.iter().enumerate() {
427 cum += p;
428 if cum >= top_p {
429 cutoff = idx + 1;
430 break;
431 }
432 }
433 scaled.truncate(cutoff.max(1));
434 let norm: f64 = scaled.iter().map(|&(_, p)| p).sum();
435 for (_, p) in &mut scaled {
436 *p /= norm;
437 }
438 }
439
440 let draw: f64 = rng.random();
441 let mut acc = 0.0_f64;
442 for &(idx, p) in &scaled {
443 acc += p;
444 if draw <= acc {
445 return Ok(idx as u32);
446 }
447 }
448 Ok(scaled.last().map_or(0, |&(idx, _)| idx as u32))
449}