Skip to main content

littlebit_train/
scheduler.rs

1//! Cosine learning rate schedule with linear warmup (`WarmupCosineLR`).
2
3use std::f64::consts::PI;
4
5/// Learning rate scheduler implementing linear warmup followed by cosine decay.
6#[derive(Debug, Clone, Copy)]
7pub struct CosineWarmupScheduler {
8    base_lr: f64,
9    total_steps: usize,
10    warmup_steps: usize,
11    min_lr_ratio: f64,
12}
13
14impl CosineWarmupScheduler {
15    /// Construct a scheduler from base LR, total optimizer steps, warmup ratio, and `cos_min_ratio`.
16    #[must_use]
17    pub fn new(base_lr: f64, total_steps: usize, warmup_ratio: f64, min_lr_ratio: f64) -> Self {
18        let warmup_steps = ((total_steps as f64) * warmup_ratio.clamp(0.0, 1.0)).ceil() as usize;
19        Self {
20            base_lr,
21            total_steps: total_steps.max(1),
22            warmup_steps,
23            min_lr_ratio: min_lr_ratio.clamp(0.0, 1.0),
24        }
25    }
26
27    /// Compute the learning rate at optimizer step `step` (0-indexed).
28    #[must_use]
29    pub fn lr_at_step(&self, step: usize) -> f64 {
30        if self.warmup_steps > 0 && step < self.warmup_steps {
31            return self.base_lr * ((step + 1) as f64 / self.warmup_steps as f64);
32        }
33        let decay_steps = self.total_steps.saturating_sub(self.warmup_steps);
34        if decay_steps == 0 {
35            return self.base_lr;
36        }
37        let progress =
38            ((step.saturating_sub(self.warmup_steps)) as f64 / decay_steps as f64).clamp(0.0, 1.0);
39        let cosine = 0.5 * (1.0 + (PI * progress).cos());
40        let factor = self.min_lr_ratio + (1.0 - self.min_lr_ratio) * cosine;
41        self.base_lr * factor
42    }
43}
44
45#[cfg(test)]
46mod tests {
47    use super::*;
48
49    #[test]
50    fn cosine_warmup_hits_expected_milestones() {
51        let sched = CosineWarmupScheduler::new(1e-4, 100, 0.1, 0.0001);
52        assert!((sched.lr_at_step(0) - 1e-5).abs() < 1e-9);
53        assert!((sched.lr_at_step(9) - 1e-4).abs() < 1e-9);
54        let mid = sched.lr_at_step(55);
55        assert!(mid > 1e-5 && mid < 1e-4);
56        assert!((sched.lr_at_step(100) - 1e-8).abs() < 1e-10);
57    }
58}