littlebit_train/
scheduler.rs1use std::f64::consts::PI;
4
5#[derive(Debug, Clone, Copy)]
7pub struct CosineWarmupScheduler {
8 base_lr: f64,
9 total_steps: usize,
10 warmup_steps: usize,
11 min_lr_ratio: f64,
12}
13
14impl CosineWarmupScheduler {
15 #[must_use]
17 pub fn new(base_lr: f64, total_steps: usize, warmup_ratio: f64, min_lr_ratio: f64) -> Self {
18 let warmup_steps = ((total_steps as f64) * warmup_ratio.clamp(0.0, 1.0)).ceil() as usize;
19 Self {
20 base_lr,
21 total_steps: total_steps.max(1),
22 warmup_steps,
23 min_lr_ratio: min_lr_ratio.clamp(0.0, 1.0),
24 }
25 }
26
27 #[must_use]
29 pub fn lr_at_step(&self, step: usize) -> f64 {
30 if self.warmup_steps > 0 && step < self.warmup_steps {
31 return self.base_lr * ((step + 1) as f64 / self.warmup_steps as f64);
32 }
33 let decay_steps = self.total_steps.saturating_sub(self.warmup_steps);
34 if decay_steps == 0 {
35 return self.base_lr;
36 }
37 let progress =
38 ((step.saturating_sub(self.warmup_steps)) as f64 / decay_steps as f64).clamp(0.0, 1.0);
39 let cosine = 0.5 * (1.0 + (PI * progress).cos());
40 let factor = self.min_lr_ratio + (1.0 - self.min_lr_ratio) * cosine;
41 self.base_lr * factor
42 }
43}
44
45#[cfg(test)]
46mod tests {
47 use super::*;
48
49 #[test]
50 fn cosine_warmup_hits_expected_milestones() {
51 let sched = CosineWarmupScheduler::new(1e-4, 100, 0.1, 0.0001);
52 assert!((sched.lr_at_step(0) - 1e-5).abs() < 1e-9);
53 assert!((sched.lr_at_step(9) - 1e-4).abs() < 1e-9);
54 let mid = sched.lr_at_step(55);
55 assert!(mid > 1e-5 && mid < 1e-4);
56 assert!((sched.lr_at_step(100) - 1e-8).abs() < 1e-10);
57 }
58}