Skip to main content

littlebit_train/
loss.rs

1//! Knowledge distillation KL and layer-to-layer hidden-state MSE losses (`utils/kd_utils.py`).
2
3use candle_core::{DType, Tensor, D};
4
5use littlebit_models::CausalLmOutput;
6
7use crate::error::{Result, TrainError};
8
9/// Scalar + differentiable breakdown returned by [`compute_kd_loss`].
10#[derive(Debug)]
11pub struct KdLossBreakdown {
12    /// Differentiable total loss `kd_loss + l2l_loss_scale * sum_l mse(student_l, teacher_l)`.
13    pub total: Tensor,
14    /// Detached logit KL-divergence scalar (`kd_loss`).
15    pub kd_loss: f64,
16    /// Detached scaled layer-to-layer MSE scalar (`l2l_loss`).
17    pub l2l_loss: f64,
18}
19
20/// Compute `F.kl_div(F.log_softmax(student, -1), F.softmax(teacher, -1), reduction="batchmean")`
21/// in `F32` for numerical stability.
22///
23/// # Errors
24/// Returns an error if shapes mismatch or tensor ops fail.
25pub fn kl_divergence_batchmean(student_logits: &Tensor, teacher_logits: &Tensor) -> Result<Tensor> {
26    if student_logits.dims() != teacher_logits.dims() {
27        return Err(TrainError::InvalidArgument(format!(
28            "logits shape mismatch: student {:?} vs teacher {:?}",
29            student_logits.dims(),
30            teacher_logits.dims()
31        )));
32    }
33    let batch_size = student_logits.dims().first().copied().unwrap_or(0);
34    if batch_size == 0 {
35        return Err(TrainError::InvalidArgument(
36            "batch size must be > 0 for KL divergence".to_string(),
37        ));
38    }
39
40    let s = student_logits.to_dtype(DType::F32)?;
41    let t = teacher_logits.detach().to_dtype(DType::F32)?;
42
43    let log_p = candle_nn::ops::log_softmax(&s, D::Minus1)?;
44    let log_q = candle_nn::ops::log_softmax(&t, D::Minus1)?;
45    let q = log_q.exp()?;
46
47    let elem_kl = q.mul(&log_q.sub(&log_p)?)?;
48    Ok((elem_kl.sum_all()? / (batch_size as f64))?)
49}
50
51/// Compute layer-to-layer MSE loss summed over aligned transformer hidden states and scaled by `scale`.
52///
53/// Matches `KDTrainer.mse_loss` accumulation in `utils/kd_utils.py`.
54///
55/// # Errors
56/// Returns an error if hidden state counts or tensor shapes mismatch.
57pub fn l2l_mse_loss(
58    student_reps: &[Tensor],
59    teacher_reps: &[Tensor],
60    scale: f64,
61) -> Result<Tensor> {
62    if student_reps.len() != teacher_reps.len() {
63        return Err(TrainError::InvalidArgument(format!(
64            "hidden state count mismatch: student {} vs teacher {}",
65            student_reps.len(),
66            teacher_reps.len()
67        )));
68    }
69    if student_reps.is_empty() {
70        return Err(TrainError::InvalidArgument(
71            "expected non-empty hidden states for L2L MSE loss".to_string(),
72        ));
73    }
74
75    let mut total: Option<Tensor> = None;
76    for (s_rep, t_rep) in student_reps.iter().zip(teacher_reps.iter()) {
77        let s = s_rep.to_dtype(DType::F32)?;
78        let t = t_rep.detach().to_dtype(DType::F32)?;
79        let layer_mse = candle_nn::loss::mse(&s, &t)?;
80        total = Some(match total {
81            None => layer_mse,
82            Some(acc) => acc.add(&layer_mse)?,
83        });
84    }
85
86    let summed = total.ok_or_else(|| {
87        TrainError::InvalidArgument("missing hidden states for L2L MSE loss".to_string())
88    })?;
89    Ok((summed * scale)?)
90}
91
92/// Compute combined Knowledge Distillation loss (`KD + L2L MSE`) from student and teacher outputs.
93///
94/// Slices `&hidden_states[1..]` to skip embedding outputs, matching upstream `outputs.hidden_states[1:]`.
95///
96/// # Errors
97/// Returns an error if `hidden_states` are missing or loss calculation fails.
98pub fn compute_kd_loss(
99    student_out: &CausalLmOutput,
100    teacher_out: &CausalLmOutput,
101    l2l_loss_scale: f64,
102) -> Result<KdLossBreakdown> {
103    let kd = kl_divergence_batchmean(&student_out.logits, &teacher_out.logits)?;
104    let kd_scalar = kd.to_scalar::<f32>()? as f64;
105
106    let student_reps = student_out.hidden_states.get(1..).unwrap_or(&[]);
107    let teacher_reps = teacher_out.hidden_states.get(1..).unwrap_or(&[]);
108    let l2l = if !student_reps.is_empty() && l2l_loss_scale > 0.0 {
109        l2l_mse_loss(student_reps, teacher_reps, l2l_loss_scale)?
110    } else {
111        Tensor::zeros((), DType::F32, kd.device())?
112    };
113    let l2l_scalar = l2l.to_scalar::<f32>()? as f64;
114    let total = kd.add(&l2l)?;
115
116    Ok(KdLossBreakdown {
117        total,
118        kd_loss: kd_scalar,
119        l2l_loss: l2l_scalar,
120    })
121}
122
123#[cfg(test)]
124mod tests {
125    use super::*;
126    use candle_core::{Device, Var};
127
128    #[test]
129    fn kl_divergence_is_zero_when_distributions_match() -> Result<()> {
130        let dev = Device::Cpu;
131        let logits = Tensor::from_slice(&[1.0f32, -0.5, 2.0, 0.3, -1.2, 0.8], (2, 1, 3), &dev)?;
132        let kl = kl_divergence_batchmean(&logits, &logits)?.to_scalar::<f32>()?;
133        assert!(kl.abs() < 1e-6, "expected 0 KL, got {kl}");
134        Ok(())
135    }
136
137    #[test]
138    fn kl_divergence_matches_analytical_value_and_propagates_grad() -> Result<()> {
139        let dev = Device::Cpu;
140        let s = Var::from_slice(&[0.0f32, 0.0], (1, 1, 2), &dev)?;
141        let t = Tensor::from_slice(&[1.0f32, 0.0], (1, 1, 2), &dev)?;
142        let kl = kl_divergence_batchmean(s.as_tensor(), &t)?;
143        let got = kl.to_scalar::<f32>()?;
144
145        // q = [e/(e+1), 1/(e+1)], p = [0.5, 0.5]
146        let e = 1.0f64.exp();
147        let q0 = e / (e + 1.0);
148        let q1 = 1.0 / (e + 1.0);
149        let expected = (q0 * (q0 / 0.5).ln() + q1 * (q1 / 0.5).ln()) as f32;
150        assert!((got - expected).abs() < 1e-5, "got {got}, want {expected}");
151
152        let grads = kl.backward()?;
153        let grad = grads
154            .get(s.as_tensor())
155            .expect("student logit grad")
156            .flatten_all()?
157            .to_vec1::<f32>()?;
158        // d(KL)/d(s_i) = p_i - q_i
159        assert!((grad[0] - (0.5 - q0 as f32)).abs() < 1e-5);
160        assert!((grad[1] - (0.5 - q1 as f32)).abs() < 1e-5);
161        Ok(())
162    }
163}