1use std::fs::{self, OpenOptions};
4use std::io::Write;
5use std::path::PathBuf;
6
7use candle_core::backprop::GradStore;
8use candle_core::{DType, Device, Tensor, Var};
9use candle_nn::optim::{AdamW, Optimizer, ParamsAdamW};
10use rand::seq::SliceRandom;
11use rand::SeedableRng;
12use rand_chacha::ChaCha8Rng;
13use serde::{Deserialize, Serialize};
14use tracing::info;
15
16use littlebit_core::{CastPolicy, LittleBitConfig};
17use littlebit_models::{CausalLm, HiddenStatesMode, LittleBitModel, ParameterStats};
18
19use crate::error::{Result, TrainError};
20use crate::loss::compute_kd_loss;
21use crate::scheduler::CosineWarmupScheduler;
22
23#[derive(Debug, Clone, Serialize, Deserialize)]
25pub struct TrainingConfig {
26 pub seed: u64,
28 pub num_train_epochs: f64,
30 pub batch_size: usize,
32 pub gradient_accumulation_steps: usize,
34 pub warmup_ratio: f64,
36 pub lr: f64,
38 pub adam_beta1: f64,
40 pub adam_beta2: f64,
42 pub adam_eps: f64,
44 pub weight_decay: f64,
46 pub max_grad_norm: f64,
48 pub cos_min_ratio: f64,
50 pub l2l_loss_scale: f64,
52 pub save_dir: Option<PathBuf>,
54 pub exp_name: String,
56 pub run_name: String,
58 pub exclude_from_grad: Vec<String>,
60}
61
62impl Default for TrainingConfig {
63 fn default() -> Self {
64 Self {
65 seed: 42,
66 num_train_epochs: 3.0,
67 batch_size: 4,
68 gradient_accumulation_steps: 1,
69 warmup_ratio: 0.03,
70 lr: 2e-5,
71 adam_beta1: 0.9,
72 adam_beta2: 0.999,
73 adam_eps: 1e-8,
74 weight_decay: 0.0,
75 max_grad_norm: 1.0,
76 cos_min_ratio: 0.0001,
77 l2l_loss_scale: 10.0,
78 save_dir: None,
79 exp_name: "LittleBit".to_string(),
80 run_name: "littlebit_run".to_string(),
81 exclude_from_grad: vec!["lm_head".to_string(), "embed".to_string()],
82 }
83 }
84}
85
86#[derive(Debug, Clone, Serialize, Deserialize)]
88pub struct StepMetrics {
89 pub step: usize,
91 pub epoch: f64,
93 pub total_loss: f64,
95 pub kd_loss: f64,
97 pub l2l_loss: f64,
99 pub lr: f64,
101}
102
103#[derive(Debug, Clone, Serialize, Deserialize)]
105pub struct TrainingSummary {
106 pub total_steps: usize,
108 pub initial_loss: Option<f64>,
110 pub final_loss: Option<f64>,
112 pub final_kd_loss: Option<f64>,
114 pub final_l2l_loss: Option<f64>,
116 pub history: Vec<StepMetrics>,
118}
119
120#[derive(Debug)]
122pub struct KdTrainer {
123 config: TrainingConfig,
124 trainable_vars: Vec<Var>,
125 optimizer: AdamW,
126 param_stats: ParameterStats,
127}
128
129impl KdTrainer {
130 pub fn new(student: &mut CausalLm, config: TrainingConfig) -> Result<Self> {
135 let exclude_refs: Vec<&str> = config
136 .exclude_from_grad
137 .iter()
138 .map(String::as_str)
139 .collect();
140 student.prepare_for_training(&exclude_refs)?;
141
142 let trainable_vars = student.trainable_vars();
143 if trainable_vars.is_empty() {
144 return Err(TrainError::InvalidArgument(
145 "student model has zero trainable variables".to_string(),
146 ));
147 }
148
149 let param_stats = student.parameter_stats();
150 info!("{param_stats}");
151
152 let optimizer = AdamW::new(
153 trainable_vars.clone(),
154 ParamsAdamW {
155 lr: config.lr,
156 beta1: config.adam_beta1,
157 beta2: config.adam_beta2,
158 eps: config.adam_eps,
159 weight_decay: config.weight_decay,
160 },
161 )?;
162
163 Ok(Self {
164 config,
165 trainable_vars,
166 optimizer,
167 param_stats,
168 })
169 }
170
171 #[must_use]
173 pub fn parameter_stats(&self) -> ParameterStats {
174 self.param_stats
175 }
176
177 pub fn train(
182 &mut self,
183 student: &CausalLm,
184 teacher: &CausalLm,
185 token_blocks: &[Vec<u32>],
186 device: &Device,
187 ) -> Result<TrainingSummary> {
188 if token_blocks.is_empty() {
189 return Err(TrainError::InvalidArgument(
190 "training dataset contains 0 token blocks".to_string(),
191 ));
192 }
193 let batch_size = self.config.batch_size.max(1);
194 let grad_accum = self.config.gradient_accumulation_steps.max(1);
195
196 let micro_batches_per_epoch = token_blocks.len().div_ceil(batch_size);
197 let total_micro_batches =
198 ((micro_batches_per_epoch as f64) * self.config.num_train_epochs).ceil() as usize;
199 let total_micro_batches = total_micro_batches.max(1);
200 let total_opt_steps = total_micro_batches.div_ceil(grad_accum).max(1);
201
202 let scheduler = CosineWarmupScheduler::new(
203 self.config.lr,
204 total_opt_steps,
205 self.config.warmup_ratio,
206 self.config.cos_min_ratio,
207 );
208
209 let mut log_file = if let Some(dir) = &self.config.save_dir {
210 fs::create_dir_all(dir)?;
211 Some(
212 OpenOptions::new()
213 .create(true)
214 .append(true)
215 .open(dir.join("metrics.jsonl"))?,
216 )
217 } else {
218 None
219 };
220
221 let mut rng = ChaCha8Rng::seed_from_u64(self.config.seed);
222 let mut indices: Vec<usize> = (0..token_blocks.len()).collect();
223 let mut history = Vec::with_capacity(total_opt_steps);
224
225 let mut micro_idx = 0usize;
226 let mut opt_step = 0usize;
227 let mut accum_grads: Option<GradStore> = None;
228 let mut accum_kd = 0.0_f64;
229 let mut accum_l2l = 0.0_f64;
230 let mut accum_count = 0usize;
231
232 while micro_idx < total_micro_batches {
233 if micro_idx.is_multiple_of(micro_batches_per_epoch) {
234 indices.shuffle(&mut rng);
235 }
236 let epoch_offset = micro_idx % micro_batches_per_epoch;
237 let start = epoch_offset * batch_size;
238 let end = (start + batch_size).min(token_blocks.len());
239 let batch_indices = &indices[start..end];
240
241 let seq_len = token_blocks[batch_indices[0]].len();
242 let mut flat_ids = Vec::with_capacity(batch_indices.len() * seq_len);
243 for &idx in batch_indices {
244 flat_ids.extend_from_slice(&token_blocks[idx]);
245 }
246 let input_ids = Tensor::from_vec(flat_ids, (batch_indices.len(), seq_len), device)?;
247
248 let teacher_out = teacher.forward(&input_ids, None, HiddenStatesMode::All)?;
249 let student_out = student.forward(&input_ids, None, HiddenStatesMode::All)?;
250
251 let loss_bd = compute_kd_loss(&student_out, &teacher_out, self.config.l2l_loss_scale)?;
252 let scaled_loss = (loss_bd.total / (grad_accum as f64))?;
253 let step_grads = scaled_loss.backward()?;
254
255 match &mut accum_grads {
256 None => accum_grads = Some(step_grads),
257 Some(acc) => acc.extend(step_grads)?,
258 }
259
260 accum_kd += loss_bd.kd_loss;
261 accum_l2l += loss_bd.l2l_loss;
262 accum_count += 1;
263 micro_idx += 1;
264
265 let ready = accum_count == grad_accum || micro_idx == total_micro_batches;
266 if ready {
267 if let Some(mut grads) = accum_grads.take() {
268 if self.config.max_grad_norm > 0.0 {
269 clip_grad_norm(
270 &mut grads,
271 &self.trainable_vars,
272 self.config.max_grad_norm,
273 )?;
274 }
275 let lr = scheduler.lr_at_step(opt_step);
276 self.optimizer.set_learning_rate(lr);
277 self.optimizer.step(&grads)?;
278
279 opt_step += 1;
280 let kd_avg = accum_kd / (accum_count as f64);
281 let l2l_avg = accum_l2l / (accum_count as f64);
282 let total_avg = kd_avg + l2l_avg;
283 let epoch_progress = (micro_idx as f64) / (micro_batches_per_epoch as f64);
284
285 let metric = StepMetrics {
286 step: opt_step,
287 epoch: epoch_progress,
288 total_loss: total_avg,
289 kd_loss: kd_avg,
290 l2l_loss: l2l_avg,
291 lr,
292 };
293 info!(
294 step = metric.step,
295 epoch = format!("{:.3}", metric.epoch),
296 loss = format!("{:.6}", metric.total_loss),
297 kd_loss = format!("{:.6}", metric.kd_loss),
298 l2l_loss = format!("{:.6}", metric.l2l_loss),
299 lr = format!("{:.3e}", metric.lr),
300 "training step"
301 );
302 if let Some(file) = &mut log_file {
303 writeln!(file, "{}", serde_json::to_string(&metric)?)?;
304 }
305 history.push(metric);
306 }
307 accum_kd = 0.0;
308 accum_l2l = 0.0;
309 accum_count = 0;
310 }
311 }
312
313 let initial_loss = history.first().map(|m| m.total_loss);
314 let final_m = history.last();
315 Ok(TrainingSummary {
316 total_steps: opt_step,
317 initial_loss,
318 final_loss: final_m.map(|m| m.total_loss),
319 final_kd_loss: final_m.map(|m| m.kd_loss),
320 final_l2l_loss: final_m.map(|m| m.l2l_loss),
321 history,
322 })
323 }
324
325 pub fn save_artifacts(
330 student: &CausalLm,
331 lb_cfg: &LittleBitConfig,
332 base_model_id: Option<&str>,
333 save_dir: &std::path::Path,
334 tokenizer_src: Option<&std::path::Path>,
335 ) -> Result<()> {
336 let wrapper = LittleBitModel::new(
337 student.clone(),
338 lb_cfg.clone(),
339 base_model_id.map(str::to_string),
340 tokenizer_src.map(std::path::Path::to_path_buf),
341 );
342 wrapper.save_pretrained(save_dir, tokenizer_src, CastPolicy::ToBf16)?;
343 Ok(())
344 }
345}
346
347fn clip_grad_norm(grads: &mut GradStore, vars: &[Var], max_norm: f64) -> Result<()> {
348 let mut norm_sq = 0.0_f64;
349 for var in vars {
350 if let Some(g) = grads.get(var.as_tensor()) {
351 let s = g
352 .to_dtype(DType::F32)?
353 .sqr()?
354 .sum_all()?
355 .to_scalar::<f32>()? as f64;
356 norm_sq += s;
357 }
358 }
359 let total_norm = norm_sq.sqrt();
360 if total_norm > max_norm && total_norm > 0.0 {
361 let scale = max_norm / (total_norm + 1e-6);
362 for var in vars {
363 if let Some(g) = grads.get(var.as_tensor()).cloned() {
364 grads.insert(var.as_tensor(), (g * scale)?);
365 }
366 }
367 }
368 Ok(())
369}
370
371#[cfg(test)]
372mod tests {
373 use super::*;
374 use littlebit_core::{InferenceMode, QuantFunc, TrainMode};
375 use littlebit_models::{LoadOptions, ModelConfig};
376 use tempfile::TempDir;
377
378 #[test]
379 fn kd_trainer_reduces_loss_with_frozen_embed_and_lm_head() -> Result<()> {
380 let dev = Device::Cpu;
381 let model_cfg = ModelConfig::tiny_llama(48, 32, 2);
382
383 let teacher = CausalLm::new_random(model_cfg.clone(), DType::F32, &dev, 1)?;
384 let mut student = CausalLm::new_random(model_cfg.clone(), DType::F32, &dev, 1)?;
385
386 let lb_cfg = LittleBitConfig {
387 quant_func: QuantFunc::SmoothSign { alpha: 20.0 },
388 eff_bit: None,
389 split_dim: 8,
390 residual: false,
391 use_itq: true,
392 itq_n_iter: 10,
393 ..Default::default()
394 };
395 student.apply_littlebit_patch(&lb_cfg, TrainMode::Train, 7)?;
396
397 let before_state = student.pack_state_dict()?;
399 let embed_before = before_state["model.embed_tokens.weight"].to_vec2::<f32>()?;
400 let head_before = before_state["lm_head.weight"].to_vec2::<f32>()?;
401
402 let save_dir = TempDir::new().expect("tempdir");
403 let train_cfg = TrainingConfig {
404 seed: 42,
405 num_train_epochs: 6.0,
406 batch_size: 2,
407 gradient_accumulation_steps: 2,
408 warmup_ratio: 0.1,
409 lr: 5e-2,
410 l2l_loss_scale: 1.0,
411 save_dir: Some(save_dir.path().to_path_buf()),
412 ..Default::default()
413 };
414
415 let mut trainer = KdTrainer::new(&mut student, train_cfg)?;
416 let blocks = vec![
417 vec![1u32, 2, 3, 4, 5, 6],
418 vec![7u32, 8, 9, 10, 11, 12],
419 vec![13u32, 14, 15, 16, 17, 18],
420 vec![19u32, 20, 21, 22, 23, 24],
421 ];
422 let summary = trainer.train(&student, &teacher, &blocks, &dev)?;
423
424 assert!(summary.total_steps >= 6);
425 let first = summary.initial_loss.expect("initial loss");
426 let last = summary.final_loss.expect("final loss");
427 assert!(
428 last < first,
429 "KD training should lower total loss: initial={first}, final={last}"
430 );
431
432 let after_state = student.pack_state_dict()?;
433 let embed_after = after_state["model.embed_tokens.weight"].to_vec2::<f32>()?;
434 let head_after = after_state["lm_head.weight"].to_vec2::<f32>()?;
435 assert_eq!(embed_before, embed_after, "embed_tokens must remain frozen");
436 assert_eq!(head_before, head_after, "lm_head must remain frozen");
437
438 KdTrainer::save_artifacts(
439 &student,
440 &lb_cfg,
441 Some("test/tiny-llama"),
442 save_dir.path(),
443 None,
444 )?;
445 assert!(save_dir.path().join("metrics.jsonl").exists());
446
447 let reloaded = LittleBitModel::from_pretrained(
448 save_dir.path().to_str().expect("utf8 path"),
449 &LoadOptions {
450 inference_mode: InferenceMode::Packed,
451 ..Default::default()
452 },
453 )?;
454 let test_ids = Tensor::from_slice(&[1u32, 2, 3, 4], (1, 4), &dev)?;
455 let logits = reloaded
456 .model
457 .forward(&test_ids, None, HiddenStatesMode::None)?
458 .logits;
459 assert_eq!(logits.dims(), &[1, 4, model_cfg.vocab_size]);
460 Ok(())
461 }
462}