Skip to main content

littlebit_train/
trainer.rs

1//! QAT Knowledge Distillation trainer (`main.py` & `utils/kd_utils.py` parity).
2
3use std::fs::{self, OpenOptions};
4use std::io::Write;
5use std::path::PathBuf;
6
7use candle_core::backprop::GradStore;
8use candle_core::{DType, Device, Tensor, Var};
9use candle_nn::optim::{AdamW, Optimizer, ParamsAdamW};
10use rand::seq::SliceRandom;
11use rand::SeedableRng;
12use rand_chacha::ChaCha8Rng;
13use serde::{Deserialize, Serialize};
14use tracing::info;
15
16use littlebit_core::{CastPolicy, LittleBitConfig};
17use littlebit_models::{CausalLm, HiddenStatesMode, LittleBitModel, ParameterStats};
18
19use crate::error::{Result, TrainError};
20use crate::loss::compute_kd_loss;
21use crate::scheduler::CosineWarmupScheduler;
22
23/// Hyperparameters for [`KdTrainer`].
24#[derive(Debug, Clone, Serialize, Deserialize)]
25pub struct TrainingConfig {
26    /// Random seed for batch shuffling.
27    pub seed: u64,
28    /// Number of training epochs (supports fractional epochs like `0.5` or `5.0`).
29    pub num_train_epochs: f64,
30    /// Per-step micro-batch size.
31    pub batch_size: usize,
32    /// Number of micro-batches accumulated per optimizer step.
33    pub gradient_accumulation_steps: usize,
34    /// Fraction of total optimizer steps spent warming up LR linearly.
35    pub warmup_ratio: f64,
36    /// Peak AdamW learning rate.
37    pub lr: f64,
38    /// AdamW beta1.
39    pub adam_beta1: f64,
40    /// AdamW beta2.
41    pub adam_beta2: f64,
42    /// AdamW numerical epsilon.
43    pub adam_eps: f64,
44    /// AdamW weight decay.
45    pub weight_decay: f64,
46    /// Global gradient L2 clip threshold (`0.0` disables clipping).
47    pub max_grad_norm: f64,
48    /// Minimum cosine LR ratio at end of decay (`cos_min_ratio`).
49    pub cos_min_ratio: f64,
50    /// Multiplier for summed layer-to-layer hidden-state MSE loss (`10.0` default).
51    pub l2l_loss_scale: f64,
52    /// Output directory where checkpoints and metrics logs are saved (optional).
53    pub save_dir: Option<PathBuf>,
54    /// Experiment name metadata.
55    pub exp_name: String,
56    /// Run name metadata.
57    pub run_name: String,
58    /// Module name substrings excluded from training (`["lm_head", "embed"]` default).
59    pub exclude_from_grad: Vec<String>,
60}
61
62impl Default for TrainingConfig {
63    fn default() -> Self {
64        Self {
65            seed: 42,
66            num_train_epochs: 3.0,
67            batch_size: 4,
68            gradient_accumulation_steps: 1,
69            warmup_ratio: 0.03,
70            lr: 2e-5,
71            adam_beta1: 0.9,
72            adam_beta2: 0.999,
73            adam_eps: 1e-8,
74            weight_decay: 0.0,
75            max_grad_norm: 1.0,
76            cos_min_ratio: 0.0001,
77            l2l_loss_scale: 10.0,
78            save_dir: None,
79            exp_name: "LittleBit".to_string(),
80            run_name: "littlebit_run".to_string(),
81            exclude_from_grad: vec!["lm_head".to_string(), "embed".to_string()],
82        }
83    }
84}
85
86/// Metrics logged for each optimizer step.
87#[derive(Debug, Clone, Serialize, Deserialize)]
88pub struct StepMetrics {
89    /// 1-based optimizer step index.
90    pub step: usize,
91    /// Fractional epoch progress.
92    pub epoch: f64,
93    /// Combined `kd_loss + l2l_loss`.
94    pub total_loss: f64,
95    /// KL divergence logit loss.
96    pub kd_loss: f64,
97    /// Scaled layer-to-layer MSE loss.
98    pub l2l_loss: f64,
99    /// Active learning rate used for this step.
100    pub lr: f64,
101}
102
103/// Summary returned after [`KdTrainer::train`] completes.
104#[derive(Debug, Clone, Serialize, Deserialize)]
105pub struct TrainingSummary {
106    /// Total optimizer steps executed.
107    pub total_steps: usize,
108    /// First step total loss (if at least 1 step executed).
109    pub initial_loss: Option<f64>,
110    /// Final step total loss (if at least 1 step executed).
111    pub final_loss: Option<f64>,
112    /// Final step KL divergence loss.
113    pub final_kd_loss: Option<f64>,
114    /// Final step L2L MSE loss.
115    pub final_l2l_loss: Option<f64>,
116    /// Per-step optimization metrics history.
117    pub history: Vec<StepMetrics>,
118}
119
120/// Quantization-Aware Knowledge Distillation trainer matching `KDTrainer` in `utils/kd_utils.py`.
121#[derive(Debug)]
122pub struct KdTrainer {
123    config: TrainingConfig,
124    trainable_vars: Vec<Var>,
125    optimizer: AdamW,
126    param_stats: ParameterStats,
127}
128
129impl KdTrainer {
130    /// Prepare `student` for training (freezing `exclude_from_grad`) and initialize AdamW.
131    ///
132    /// # Errors
133    /// Returns an error if no trainable parameters exist or optimizer creation fails.
134    pub fn new(student: &mut CausalLm, config: TrainingConfig) -> Result<Self> {
135        let exclude_refs: Vec<&str> = config
136            .exclude_from_grad
137            .iter()
138            .map(String::as_str)
139            .collect();
140        student.prepare_for_training(&exclude_refs)?;
141
142        let trainable_vars = student.trainable_vars();
143        if trainable_vars.is_empty() {
144            return Err(TrainError::InvalidArgument(
145                "student model has zero trainable variables".to_string(),
146            ));
147        }
148
149        let param_stats = student.parameter_stats();
150        info!("{param_stats}");
151
152        let optimizer = AdamW::new(
153            trainable_vars.clone(),
154            ParamsAdamW {
155                lr: config.lr,
156                beta1: config.adam_beta1,
157                beta2: config.adam_beta2,
158                eps: config.adam_eps,
159                weight_decay: config.weight_decay,
160            },
161        )?;
162
163        Ok(Self {
164            config,
165            trainable_vars,
166            optimizer,
167            param_stats,
168        })
169    }
170
171    /// Borrow parameter statistics logged at trainer initialization.
172    #[must_use]
173    pub fn parameter_stats(&self) -> ParameterStats {
174        self.param_stats
175    }
176
177    /// Run knowledge distillation training on `token_blocks` (`&[Vec<u32>]`) against frozen `teacher`.
178    ///
179    /// # Errors
180    /// Returns an error on invalid dataset/batch sizes, forward/backward failures, or I/O logging errors.
181    pub fn train(
182        &mut self,
183        student: &CausalLm,
184        teacher: &CausalLm,
185        token_blocks: &[Vec<u32>],
186        device: &Device,
187    ) -> Result<TrainingSummary> {
188        if token_blocks.is_empty() {
189            return Err(TrainError::InvalidArgument(
190                "training dataset contains 0 token blocks".to_string(),
191            ));
192        }
193        let batch_size = self.config.batch_size.max(1);
194        let grad_accum = self.config.gradient_accumulation_steps.max(1);
195
196        let micro_batches_per_epoch = token_blocks.len().div_ceil(batch_size);
197        let total_micro_batches =
198            ((micro_batches_per_epoch as f64) * self.config.num_train_epochs).ceil() as usize;
199        let total_micro_batches = total_micro_batches.max(1);
200        let total_opt_steps = total_micro_batches.div_ceil(grad_accum).max(1);
201
202        let scheduler = CosineWarmupScheduler::new(
203            self.config.lr,
204            total_opt_steps,
205            self.config.warmup_ratio,
206            self.config.cos_min_ratio,
207        );
208
209        let mut log_file = if let Some(dir) = &self.config.save_dir {
210            fs::create_dir_all(dir)?;
211            Some(
212                OpenOptions::new()
213                    .create(true)
214                    .append(true)
215                    .open(dir.join("metrics.jsonl"))?,
216            )
217        } else {
218            None
219        };
220
221        let mut rng = ChaCha8Rng::seed_from_u64(self.config.seed);
222        let mut indices: Vec<usize> = (0..token_blocks.len()).collect();
223        let mut history = Vec::with_capacity(total_opt_steps);
224
225        let mut micro_idx = 0usize;
226        let mut opt_step = 0usize;
227        let mut accum_grads: Option<GradStore> = None;
228        let mut accum_kd = 0.0_f64;
229        let mut accum_l2l = 0.0_f64;
230        let mut accum_count = 0usize;
231
232        while micro_idx < total_micro_batches {
233            if micro_idx.is_multiple_of(micro_batches_per_epoch) {
234                indices.shuffle(&mut rng);
235            }
236            let epoch_offset = micro_idx % micro_batches_per_epoch;
237            let start = epoch_offset * batch_size;
238            let end = (start + batch_size).min(token_blocks.len());
239            let batch_indices = &indices[start..end];
240
241            let seq_len = token_blocks[batch_indices[0]].len();
242            let mut flat_ids = Vec::with_capacity(batch_indices.len() * seq_len);
243            for &idx in batch_indices {
244                flat_ids.extend_from_slice(&token_blocks[idx]);
245            }
246            let input_ids = Tensor::from_vec(flat_ids, (batch_indices.len(), seq_len), device)?;
247
248            let teacher_out = teacher.forward(&input_ids, None, HiddenStatesMode::All)?;
249            let student_out = student.forward(&input_ids, None, HiddenStatesMode::All)?;
250
251            let loss_bd = compute_kd_loss(&student_out, &teacher_out, self.config.l2l_loss_scale)?;
252            let scaled_loss = (loss_bd.total / (grad_accum as f64))?;
253            let step_grads = scaled_loss.backward()?;
254
255            match &mut accum_grads {
256                None => accum_grads = Some(step_grads),
257                Some(acc) => acc.extend(step_grads)?,
258            }
259
260            accum_kd += loss_bd.kd_loss;
261            accum_l2l += loss_bd.l2l_loss;
262            accum_count += 1;
263            micro_idx += 1;
264
265            let ready = accum_count == grad_accum || micro_idx == total_micro_batches;
266            if ready {
267                if let Some(mut grads) = accum_grads.take() {
268                    if self.config.max_grad_norm > 0.0 {
269                        clip_grad_norm(
270                            &mut grads,
271                            &self.trainable_vars,
272                            self.config.max_grad_norm,
273                        )?;
274                    }
275                    let lr = scheduler.lr_at_step(opt_step);
276                    self.optimizer.set_learning_rate(lr);
277                    self.optimizer.step(&grads)?;
278
279                    opt_step += 1;
280                    let kd_avg = accum_kd / (accum_count as f64);
281                    let l2l_avg = accum_l2l / (accum_count as f64);
282                    let total_avg = kd_avg + l2l_avg;
283                    let epoch_progress = (micro_idx as f64) / (micro_batches_per_epoch as f64);
284
285                    let metric = StepMetrics {
286                        step: opt_step,
287                        epoch: epoch_progress,
288                        total_loss: total_avg,
289                        kd_loss: kd_avg,
290                        l2l_loss: l2l_avg,
291                        lr,
292                    };
293                    info!(
294                        step = metric.step,
295                        epoch = format!("{:.3}", metric.epoch),
296                        loss = format!("{:.6}", metric.total_loss),
297                        kd_loss = format!("{:.6}", metric.kd_loss),
298                        l2l_loss = format!("{:.6}", metric.l2l_loss),
299                        lr = format!("{:.3e}", metric.lr),
300                        "training step"
301                    );
302                    if let Some(file) = &mut log_file {
303                        writeln!(file, "{}", serde_json::to_string(&metric)?)?;
304                    }
305                    history.push(metric);
306                }
307                accum_kd = 0.0;
308                accum_l2l = 0.0;
309                accum_count = 0;
310            }
311        }
312
313        let initial_loss = history.first().map(|m| m.total_loss);
314        let final_m = history.last();
315        Ok(TrainingSummary {
316            total_steps: opt_step,
317            initial_loss,
318            final_loss: final_m.map(|m| m.total_loss),
319            final_kd_loss: final_m.map(|m| m.kd_loss),
320            final_l2l_loss: final_m.map(|m| m.l2l_loss),
321            history,
322        })
323    }
324
325    /// Save final packed LittleBit checkpoint + tokenizer artifacts matching `main.py::save_artifacts`.
326    ///
327    /// # Errors
328    /// Returns an error if saving config or safetensors fails.
329    pub fn save_artifacts(
330        student: &CausalLm,
331        lb_cfg: &LittleBitConfig,
332        base_model_id: Option<&str>,
333        save_dir: &std::path::Path,
334        tokenizer_src: Option<&std::path::Path>,
335    ) -> Result<()> {
336        let wrapper = LittleBitModel::new(
337            student.clone(),
338            lb_cfg.clone(),
339            base_model_id.map(str::to_string),
340            tokenizer_src.map(std::path::Path::to_path_buf),
341        );
342        wrapper.save_pretrained(save_dir, tokenizer_src, CastPolicy::ToBf16)?;
343        Ok(())
344    }
345}
346
347fn clip_grad_norm(grads: &mut GradStore, vars: &[Var], max_norm: f64) -> Result<()> {
348    let mut norm_sq = 0.0_f64;
349    for var in vars {
350        if let Some(g) = grads.get(var.as_tensor()) {
351            let s = g
352                .to_dtype(DType::F32)?
353                .sqr()?
354                .sum_all()?
355                .to_scalar::<f32>()? as f64;
356            norm_sq += s;
357        }
358    }
359    let total_norm = norm_sq.sqrt();
360    if total_norm > max_norm && total_norm > 0.0 {
361        let scale = max_norm / (total_norm + 1e-6);
362        for var in vars {
363            if let Some(g) = grads.get(var.as_tensor()).cloned() {
364                grads.insert(var.as_tensor(), (g * scale)?);
365            }
366        }
367    }
368    Ok(())
369}
370
371#[cfg(test)]
372mod tests {
373    use super::*;
374    use littlebit_core::{InferenceMode, QuantFunc, TrainMode};
375    use littlebit_models::{LoadOptions, ModelConfig};
376    use tempfile::TempDir;
377
378    #[test]
379    fn kd_trainer_reduces_loss_with_frozen_embed_and_lm_head() -> Result<()> {
380        let dev = Device::Cpu;
381        let model_cfg = ModelConfig::tiny_llama(48, 32, 2);
382
383        let teacher = CausalLm::new_random(model_cfg.clone(), DType::F32, &dev, 1)?;
384        let mut student = CausalLm::new_random(model_cfg.clone(), DType::F32, &dev, 1)?;
385
386        let lb_cfg = LittleBitConfig {
387            quant_func: QuantFunc::SmoothSign { alpha: 20.0 },
388            eff_bit: None,
389            split_dim: 8,
390            residual: false,
391            use_itq: true,
392            itq_n_iter: 10,
393            ..Default::default()
394        };
395        student.apply_littlebit_patch(&lb_cfg, TrainMode::Train, 7)?;
396
397        // Snapshot embed_tokens and lm_head before training to verify freeze invariant.
398        let before_state = student.pack_state_dict()?;
399        let embed_before = before_state["model.embed_tokens.weight"].to_vec2::<f32>()?;
400        let head_before = before_state["lm_head.weight"].to_vec2::<f32>()?;
401
402        let save_dir = TempDir::new().expect("tempdir");
403        let train_cfg = TrainingConfig {
404            seed: 42,
405            num_train_epochs: 6.0,
406            batch_size: 2,
407            gradient_accumulation_steps: 2,
408            warmup_ratio: 0.1,
409            lr: 5e-2,
410            l2l_loss_scale: 1.0,
411            save_dir: Some(save_dir.path().to_path_buf()),
412            ..Default::default()
413        };
414
415        let mut trainer = KdTrainer::new(&mut student, train_cfg)?;
416        let blocks = vec![
417            vec![1u32, 2, 3, 4, 5, 6],
418            vec![7u32, 8, 9, 10, 11, 12],
419            vec![13u32, 14, 15, 16, 17, 18],
420            vec![19u32, 20, 21, 22, 23, 24],
421        ];
422        let summary = trainer.train(&student, &teacher, &blocks, &dev)?;
423
424        assert!(summary.total_steps >= 6);
425        let first = summary.initial_loss.expect("initial loss");
426        let last = summary.final_loss.expect("final loss");
427        assert!(
428            last < first,
429            "KD training should lower total loss: initial={first}, final={last}"
430        );
431
432        let after_state = student.pack_state_dict()?;
433        let embed_after = after_state["model.embed_tokens.weight"].to_vec2::<f32>()?;
434        let head_after = after_state["lm_head.weight"].to_vec2::<f32>()?;
435        assert_eq!(embed_before, embed_after, "embed_tokens must remain frozen");
436        assert_eq!(head_before, head_after, "lm_head must remain frozen");
437
438        KdTrainer::save_artifacts(
439            &student,
440            &lb_cfg,
441            Some("test/tiny-llama"),
442            save_dir.path(),
443            None,
444        )?;
445        assert!(save_dir.path().join("metrics.jsonl").exists());
446
447        let reloaded = LittleBitModel::from_pretrained(
448            save_dir.path().to_str().expect("utf8 path"),
449            &LoadOptions {
450                inference_mode: InferenceMode::Packed,
451                ..Default::default()
452            },
453        )?;
454        let test_ids = Tensor::from_slice(&[1u32, 2, 3, 4], (1, 4), &dev)?;
455        let logits = reloaded
456            .model
457            .forward(&test_ids, None, HiddenStatesMode::None)?
458            .logits;
459        assert_eq!(logits.dims(), &[1, 4, model_cfg.vocab_size]);
460        Ok(())
461    }
462}