Skip to main content

tufazip_codec/
optim.rs

1use std::cell::RefCell;
2use std::collections::{BTreeMap, HashMap};
3
4use candle_core::backprop::GradStore;
5use candle_core::{CpuStorage, DType, Device, InplaceOp1, Layout, Storage, Tensor, Var};
6use rayon::prelude::*;
7
8use crate::error::CodecError;
9
10const PAR_ELEM_THRESHOLD: usize = 8_000_000;
11
12fn cpu_f32_tensor_slice(tensor: &Tensor) -> Result<(Option<Tensor>, usize, usize), CodecError> {
13    if tensor.dtype() == DType::F32 && tensor.is_contiguous() {
14        if let Some((start, end)) = tensor.layout().contiguous_offsets() {
15            return Ok((None, start, end));
16        }
17    }
18    let owned = tensor.to_dtype(DType::F32)?.contiguous()?;
19    let (start, end) = owned
20        .layout()
21        .contiguous_offsets()
22        .ok_or_else(|| CodecError::InvalidData("expected contiguous tensor".to_owned()))?;
23    Ok((Some(owned), start, end))
24}
25
26fn read_cpu_f32_to_vec(tensor: &Tensor) -> Result<Vec<f32>, CodecError> {
27    let (owned, start, end) = cpu_f32_tensor_slice(tensor)?;
28    let target = owned.as_ref().unwrap_or(tensor);
29    let (storage, _) = target.storage_and_layout();
30    match &*storage {
31        Storage::Cpu(CpuStorage::F32(slice)) => Ok(slice[start..end].to_vec()),
32        _ => Err(CodecError::InvalidData(
33            "expected F32 CPU tensor storage".to_owned(),
34        )),
35    }
36}
37
38fn cpu_grad_sq_norm(grad: &Tensor) -> Result<f64, CodecError> {
39    let (owned, start, end) = cpu_f32_tensor_slice(grad)?;
40    let target = owned.as_ref().unwrap_or(grad);
41    let (storage, _) = target.storage_and_layout();
42    match &*storage {
43        Storage::Cpu(CpuStorage::F32(slice)) => {
44            let data = &slice[start..end];
45            let mut sq_acc = [0.0f32; 8];
46            let chunks = data.chunks_exact(8);
47            let rem = chunks.remainder();
48            for c in chunks {
49                for i in 0..8 {
50                    let v = c[i];
51                    sq_acc[i] += v * v;
52                }
53            }
54            let mut acc = 0.0f64;
55            for &lane in &sq_acc {
56                acc += f64::from(lane);
57            }
58            for &v in rem {
59                let fv = f64::from(v);
60                acc += fv * fv;
61            }
62            Ok(acc)
63        }
64        _ => Err(CodecError::InvalidData(
65            "expected F32 CPU gradient storage".to_owned(),
66        )),
67    }
68}
69
70/// Computes the global L2 gradient norm across `vars` using a single host sync on GPU devices
71/// and zero intermediate allocations on CPU.
72fn compute_global_grad_norm(vars: &[(String, Var)], grads: &GradStore) -> Result<f64, CodecError> {
73    let Some((_, first)) = vars.first() else {
74        return Ok(0.0);
75    };
76    if first.device().is_cpu() {
77        let total_elems: usize = vars.iter().map(|(_, v)| v.elem_count()).sum();
78        let sum_sq: f64 = if total_elems >= PAR_ELEM_THRESHOLD {
79            let per_var: Vec<f64> = vars
80                .par_iter()
81                .map(|(_, var)| match grads.get(var) {
82                    Some(grad) => cpu_grad_sq_norm(grad),
83                    None => Ok(0.0),
84                })
85                .collect::<Result<Vec<_>, _>>()?;
86            per_var.into_iter().sum()
87        } else {
88            let mut acc = 0.0f64;
89            for (_, var) in vars {
90                if let Some(grad) = grads.get(var) {
91                    acc += cpu_grad_sq_norm(grad)?;
92                }
93            }
94            acc
95        };
96        return Ok(sum_sq.sqrt());
97    }
98
99    let mut sq_terms = Vec::with_capacity(vars.len());
100    for (_, var) in vars {
101        if let Some(grad) = grads.get(var) {
102            sq_terms.push(grad.to_dtype(DType::F32)?.sqr()?.sum_all()?);
103        }
104    }
105    if sq_terms.is_empty() {
106        return Ok(0.0);
107    }
108    let total_sq = if sq_terms.len() == 1 {
109        sq_terms[0].to_scalar::<f32>()?
110    } else {
111        Tensor::stack(&sq_terms, 0)?.sum_all()?.to_scalar::<f32>()?
112    };
113    Ok(f64::from(total_sq).sqrt())
114}
115
116/// Computes the global L2 norm across `vars`' gradients and rescales `grads` in place when above `max_norm`.
117///
118/// Matches PyTorch's `torch.nn.utils.clip_grad_norm_` (`norm_type=2.0`, `eps=1e-6`).
119///
120/// # Errors
121///
122/// Returns [`CodecError::Candle`] if gradient reduction or scaling fails.
123pub fn clip_grad_norm(
124    vars: &[(String, Var)],
125    grads: &mut GradStore,
126    max_norm: f64,
127) -> Result<f64, CodecError> {
128    let total_norm = compute_global_grad_norm(vars, grads)?;
129    if max_norm > 0.0 {
130        let clip_coef = max_norm / (total_norm + 1e-6);
131        if clip_coef < 1.0 {
132            for (_, var) in vars {
133                if let Some(grad) = grads.remove(var) {
134                    let scaled = grad.to_dtype(DType::F32)?.affine(clip_coef, 0.0)?;
135                    grads.insert(var, scaled);
136                }
137            }
138        }
139    }
140    Ok(total_norm)
141}
142
143struct CpuAdamInplaceOp<'a> {
144    grad: &'a [f32],
145    m: RefCell<&'a mut [f32]>,
146    v: RefCell<&'a mut [f32]>,
147    clip_scale: f32,
148    beta1: f32,
149    one_minus_beta1: f32,
150    beta2: f32,
151    one_minus_beta2: f32,
152    inv_bias_corr2: f32,
153    step_size: f32,
154    eps: f32,
155}
156
157impl InplaceOp1 for CpuAdamInplaceOp<'_> {
158    fn name(&self) -> &'static str {
159        "cpu-fused-adam"
160    }
161
162    fn cpu_fwd(&self, storage: &mut CpuStorage, layout: &Layout) -> candle_core::Result<()> {
163        let (start, end) = layout.contiguous_offsets().ok_or_else(|| {
164            candle_core::Error::Msg("Adam parameter variable must be contiguous".to_owned())
165        })?;
166        let slice = match storage {
167            CpuStorage::F32(data) => &mut data[start..end],
168            _ => {
169                return Err(candle_core::Error::Msg(
170                    "Adam parameter variable must have F32 storage".to_owned(),
171                ))
172            }
173        };
174        let mut m = self.m.borrow_mut();
175        let mut v = self.v.borrow_mut();
176        if slice.len() != self.grad.len() || slice.len() != m.len() || slice.len() != v.len() {
177            return Err(candle_core::Error::Msg(format!(
178                "Adam length mismatch: param={} grad={} m={} v={}",
179                slice.len(),
180                self.grad.len(),
181                m.len(),
182                v.len()
183            )));
184        }
185        for (((p_val, m_val), v_val), &g_raw) in slice
186            .iter_mut()
187            .zip(m.iter_mut())
188            .zip(v.iter_mut())
189            .zip(self.grad.iter())
190        {
191            let g = g_raw * self.clip_scale;
192            let next_m = *m_val * self.beta1 + g * self.one_minus_beta1;
193            let next_v = *v_val * self.beta2 + (g * g) * self.one_minus_beta2;
194            let denom = (next_v * self.inv_bias_corr2).sqrt() + self.eps;
195            *p_val -= (next_m / denom) * self.step_size;
196            *m_val = next_m;
197            *v_val = next_v;
198        }
199        Ok(())
200    }
201}
202
203#[derive(Debug)]
204enum MomentBuffer {
205    Cpu { m: Vec<f32>, v: Vec<f32> },
206    Device { m: Tensor, v: Tensor },
207}
208
209#[derive(Debug)]
210struct AdamParamState {
211    name: String,
212    var: Var,
213    step: u64,
214    moments: MomentBuffer,
215}
216
217impl AdamParamState {
218    #[allow(clippy::too_many_arguments)]
219    fn apply_update(
220        &mut self,
221        raw_grad: &Tensor,
222        clip_scale: f64,
223        lr: f64,
224        beta1: f64,
225        beta2: f64,
226        eps: f64,
227    ) -> Result<(), CodecError> {
228        self.step += 1;
229        let t = self.step as i32;
230        let bias_corr1 = 1.0 - beta1.powi(t);
231        let bias_corr2 = 1.0 - beta2.powi(t);
232        let step_size = lr / bias_corr1;
233
234        match &mut self.moments {
235            MomentBuffer::Cpu { m, v } => {
236                let (owned_grad, start, end) = cpu_f32_tensor_slice(raw_grad)?;
237                let grad_tensor = owned_grad.as_ref().unwrap_or(raw_grad);
238                let (storage, _) = grad_tensor.storage_and_layout();
239                let grad_slice = match &*storage {
240                    Storage::Cpu(CpuStorage::F32(data)) => &data[start..end],
241                    _ => {
242                        return Err(CodecError::InvalidData(
243                            "expected F32 CPU gradient storage".to_owned(),
244                        ))
245                    }
246                };
247                let op = CpuAdamInplaceOp {
248                    grad: grad_slice,
249                    m: RefCell::new(m.as_mut_slice()),
250                    v: RefCell::new(v.as_mut_slice()),
251                    clip_scale: clip_scale as f32,
252                    beta1: beta1 as f32,
253                    one_minus_beta1: (1.0 - beta1) as f32,
254                    beta2: beta2 as f32,
255                    one_minus_beta2: (1.0 - beta2) as f32,
256                    inv_bias_corr2: (1.0 / bias_corr2) as f32,
257                    step_size: step_size as f32,
258                    eps: eps as f32,
259                };
260                self.var.as_tensor().inplace_op1(&op)?;
261            }
262            MomentBuffer::Device { m, v } => {
263                let g = if raw_grad.dtype() == DType::F32 {
264                    raw_grad.detach()
265                } else {
266                    raw_grad.to_dtype(DType::F32)?.detach()
267                };
268                let g_b1 = (1.0 - beta1) * clip_scale;
269                let g_b2 = (1.0 - beta2) * clip_scale * clip_scale;
270                let next_m = ((&*m * beta1)? + (&g * g_b1)?)?.detach();
271                let next_v = ((&*v * beta2)? + (g.sqr()? * g_b2)?)?.detach();
272                let denom = ((&next_v / bias_corr2)?.sqrt()? + eps)?;
273                let update = (&next_m / denom)?.affine(step_size, 0.0)?;
274                let next_param = (self.var.as_tensor() - update)?;
275                self.var.set(&next_param)?;
276                *m = next_m;
277                *v = next_v;
278            }
279        }
280        Ok(())
281    }
282}
283
284/// Pure-Rust PyTorch-compatible Adam optimizer (without decoupled weight decay) with state export/import.
285#[derive(Debug)]
286pub struct Adam {
287    params: Vec<AdamParamState>,
288    total_elems: usize,
289    is_cpu: bool,
290    lr: f64,
291    beta1: f64,
292    beta2: f64,
293    eps: f64,
294}
295
296impl Adam {
297    /// Creates an [`Adam`] optimizer tracking `vars`.
298    ///
299    /// # Errors
300    ///
301    /// Returns [`CodecError::Candle`] if moment buffer allocation fails.
302    pub fn new(
303        vars: &[(String, Var)],
304        lr: f64,
305        beta1: f64,
306        beta2: f64,
307        eps: f64,
308    ) -> Result<Self, CodecError> {
309        let mut params = Vec::with_capacity(vars.len());
310        let mut total_elems = 0usize;
311        let is_cpu = vars.first().map_or(true, |(_, v)| v.device().is_cpu());
312
313        for (name, var) in vars {
314            let n = var.elem_count();
315            total_elems += n;
316            let moments = if var.device().is_cpu() {
317                MomentBuffer::Cpu {
318                    m: vec![0.0f32; n],
319                    v: vec![0.0f32; n],
320                }
321            } else {
322                MomentBuffer::Device {
323                    m: Tensor::zeros(var.shape(), DType::F32, var.device())?,
324                    v: Tensor::zeros(var.shape(), DType::F32, var.device())?,
325                }
326            };
327            params.push(AdamParamState {
328                name: name.clone(),
329                var: var.clone(),
330                step: 0,
331                moments,
332            });
333        }
334        Ok(Self {
335            params,
336            total_elems,
337            is_cpu,
338            lr,
339            beta1,
340            beta2,
341            eps,
342        })
343    }
344
345    /// Returns the active learning rate.
346    #[must_use]
347    pub fn learning_rate(&self) -> f64 {
348        self.lr
349    }
350
351    /// Updates the optimizer learning rate.
352    pub fn set_learning_rate(&mut self, lr: f64) {
353        self.lr = lr;
354    }
355
356    /// Performs one Adam update step over all tracked parameters present in `grads`.
357    ///
358    /// # Errors
359    ///
360    /// Returns [`CodecError::Candle`] if tensor updates fail.
361    pub fn step(&mut self, grads: &GradStore) -> Result<(), CodecError> {
362        self.apply_step(grads, 1.0)
363    }
364
365    /// Computes global gradient L2 norm, applies global norm clipping up to `max_norm`
366    /// (when `max_norm > 0.0`), and performs one fused Adam update step.
367    ///
368    /// # Errors
369    ///
370    /// Returns [`CodecError`] if gradient norm reduction or parameter updates fail.
371    pub fn step_clipped(&mut self, grads: &GradStore, max_norm: f64) -> Result<f64, CodecError> {
372        let total_norm = self.global_grad_norm(grads)?;
373        let clip_scale = if max_norm > 0.0 {
374            (max_norm / (total_norm + 1e-6)).min(1.0)
375        } else {
376            1.0
377        };
378        self.apply_step(grads, clip_scale)?;
379        Ok(total_norm)
380    }
381
382    fn global_grad_norm(&self, grads: &GradStore) -> Result<f64, CodecError> {
383        if self.params.is_empty() {
384            return Ok(0.0);
385        }
386        if self.is_cpu {
387            let sum_sq: f64 = if self.total_elems >= PAR_ELEM_THRESHOLD {
388                let per_param: Vec<f64> = self
389                    .params
390                    .par_iter()
391                    .map(|p| match grads.get(&p.var) {
392                        Some(grad) => cpu_grad_sq_norm(grad),
393                        None => Ok(0.0),
394                    })
395                    .collect::<Result<Vec<_>, _>>()?;
396                per_param.into_iter().sum()
397            } else {
398                let mut acc = 0.0f64;
399                for p in &self.params {
400                    if let Some(grad) = grads.get(&p.var) {
401                        acc += cpu_grad_sq_norm(grad)?;
402                    }
403                }
404                acc
405            };
406            return Ok(sum_sq.sqrt());
407        }
408
409        let mut sq_terms = Vec::with_capacity(self.params.len());
410        for p in &self.params {
411            if let Some(grad) = grads.get(&p.var) {
412                sq_terms.push(grad.to_dtype(DType::F32)?.sqr()?.sum_all()?);
413            }
414        }
415        if sq_terms.is_empty() {
416            return Ok(0.0);
417        }
418        let total_sq = if sq_terms.len() == 1 {
419            sq_terms[0].to_scalar::<f32>()?
420        } else {
421            Tensor::stack(&sq_terms, 0)?.sum_all()?.to_scalar::<f32>()?
422        };
423        Ok(f64::from(total_sq).sqrt())
424    }
425
426    fn apply_step(&mut self, grads: &GradStore, clip_scale: f64) -> Result<(), CodecError> {
427        let lr = self.lr;
428        let beta1 = self.beta1;
429        let beta2 = self.beta2;
430        let eps = self.eps;
431
432        if self.is_cpu && self.total_elems >= PAR_ELEM_THRESHOLD {
433            self.params
434                .par_iter_mut()
435                .try_for_each(|p| -> Result<(), CodecError> {
436                    let Some(raw_grad) = grads.get(&p.var) else {
437                        return Ok(());
438                    };
439                    p.apply_update(raw_grad, clip_scale, lr, beta1, beta2, eps)
440                })?;
441        } else {
442            for p in &mut self.params {
443                let Some(raw_grad) = grads.get(&p.var) else {
444                    continue;
445                };
446                p.apply_update(raw_grad, clip_scale, lr, beta1, beta2, eps)?;
447            }
448        }
449        Ok(())
450    }
451
452    /// Exports moment tensors into `out_tensors` and returns per-parameter step counts for checkpointing.
453    ///
454    /// # Errors
455    ///
456    /// Returns [`CodecError::Candle`] if transferring moments to CPU fails.
457    pub fn export_state(
458        &self,
459        prefix: &str,
460        out_tensors: &mut HashMap<String, Tensor>,
461    ) -> Result<BTreeMap<String, u64>, CodecError> {
462        let mut steps = BTreeMap::new();
463        for p in &self.params {
464            steps.insert(p.name.clone(), p.step);
465            let (m_t, v_t) = match &p.moments {
466                MomentBuffer::Cpu { m, v } => (
467                    Tensor::from_slice(m, p.var.shape(), &Device::Cpu)?,
468                    Tensor::from_slice(v, p.var.shape(), &Device::Cpu)?,
469                ),
470                MomentBuffer::Device { m, v } => (
471                    m.to_device(&Device::Cpu)?.contiguous()?,
472                    v.to_device(&Device::Cpu)?.contiguous()?,
473                ),
474            };
475            out_tensors.insert(format!("{prefix}.{}.m", p.name), m_t);
476            out_tensors.insert(format!("{prefix}.{}.v", p.name), v_t);
477        }
478        Ok(steps)
479    }
480
481    /// Restores step counts and moment buffers from a loaded checkpoint.
482    ///
483    /// # Errors
484    ///
485    /// Returns [`CodecError`] if any parameter state is missing or mismatched in shape.
486    pub fn import_state(
487        &mut self,
488        prefix: &str,
489        steps: &BTreeMap<String, u64>,
490        tensors: &HashMap<String, Tensor>,
491    ) -> Result<(), CodecError> {
492        for p in &mut self.params {
493            let step = steps.get(&p.name).copied().ok_or_else(|| {
494                CodecError::CheckpointMismatch(format!(
495                    "checkpoint missing {prefix} step count for {}",
496                    p.name
497                ))
498            })?;
499            let m_key = format!("{prefix}.{}.m", p.name);
500            let v_key = format!("{prefix}.{}.v", p.name);
501            let m_src = tensors.get(&m_key).ok_or_else(|| {
502                CodecError::CheckpointMismatch(format!("checkpoint missing tensor {m_key}"))
503            })?;
504            let v_src = tensors.get(&v_key).ok_or_else(|| {
505                CodecError::CheckpointMismatch(format!("checkpoint missing tensor {v_key}"))
506            })?;
507            if m_src.dims() != p.var.dims() || v_src.dims() != p.var.dims() {
508                return Err(CodecError::CheckpointMismatch(format!(
509                    "checkpoint optimizer moment shape mismatch for {}",
510                    p.name
511                )));
512            }
513            p.step = step;
514            match &mut p.moments {
515                MomentBuffer::Cpu { m, v } => {
516                    *m = read_cpu_f32_to_vec(m_src)?;
517                    *v = read_cpu_f32_to_vec(v_src)?;
518                }
519                MomentBuffer::Device { m, v } => {
520                    *m = m_src
521                        .to_device(p.var.device())?
522                        .to_dtype(DType::F32)?
523                        .contiguous()?;
524                    *v = v_src
525                        .to_device(p.var.device())?
526                        .to_dtype(DType::F32)?
527                        .contiguous()?;
528                }
529            }
530        }
531        Ok(())
532    }
533}
534
535#[cfg(test)]
536mod tests {
537    use super::*;
538
539    #[test]
540    fn fused_cpu_adam_step_clipped_matches_unfused_reference() {
541        let init_a = vec![0.3f32, -0.7, 1.2, -0.05, 2.0, 0.11, -1.4, 0.8];
542        let init_b = vec![-0.4f32, 0.9, -1.1, 0.6];
543        let x = Tensor::from_slice(
544            &[1.5f32, -0.5, 0.25, 0.75, -1.0, 0.2, 0.8, -0.3],
545            (1, 8),
546            &Device::Cpu,
547        )
548        .unwrap();
549
550        let ref_a = Var::from_slice(&init_a, (1, 8), &Device::Cpu).unwrap();
551        let ref_b = Var::from_slice(&init_b, (4, 1), &Device::Cpu).unwrap();
552        let opt_a = Var::from_slice(&init_a, (1, 8), &Device::Cpu).unwrap();
553        let opt_b = Var::from_slice(&init_b, (4, 1), &Device::Cpu).unwrap();
554
555        let mut opt = Adam::new(
556            &[
557                ("a".to_owned(), opt_a.clone()),
558                ("b".to_owned(), opt_b.clone()),
559            ],
560            1e-2,
561            0.9,
562            0.999,
563            1e-8,
564        )
565        .unwrap();
566
567        let mut m_a = Tensor::zeros(ref_a.shape(), DType::F32, &Device::Cpu).unwrap();
568        let mut v_a = Tensor::zeros(ref_a.shape(), DType::F32, &Device::Cpu).unwrap();
569        let mut m_b = Tensor::zeros(ref_b.shape(), DType::F32, &Device::Cpu).unwrap();
570        let mut v_b = Tensor::zeros(ref_b.shape(), DType::F32, &Device::Cpu).unwrap();
571
572        for step in 1usize..=4 {
573            let loss_ref = ((ref_a.as_tensor() * &x).unwrap().sum_all().unwrap()
574                + ref_b.as_tensor().sqr().unwrap().sum_all().unwrap())
575            .unwrap();
576            let mut grads_ref = loss_ref.backward().unwrap();
577            let ref_vars = [
578                ("a".to_owned(), ref_a.clone()),
579                ("b".to_owned(), ref_b.clone()),
580            ];
581            let norm_ref = clip_grad_norm(&ref_vars, &mut grads_ref, 0.5).unwrap();
582
583            let loss_opt = ((opt_a.as_tensor() * &x).unwrap().sum_all().unwrap()
584                + opt_b.as_tensor().sqr().unwrap().sum_all().unwrap())
585            .unwrap();
586            let grads_opt = loss_opt.backward().unwrap();
587            let norm_opt = opt.step_clipped(&grads_opt, 0.5).unwrap();
588            assert!((norm_ref - norm_opt).abs() < 1e-6);
589
590            let bc1 = 1.0 - 0.9f64.powi(step as i32);
591            let bc2 = 1.0 - 0.999f64.powi(step as i32);
592            for (var, m, v) in [(&ref_a, &mut m_a, &mut v_a), (&ref_b, &mut m_b, &mut v_b)] {
593                let g = grads_ref.get(var).unwrap().detach();
594                let nm = ((&*m * 0.9).unwrap() + (&g * 0.1).unwrap()).unwrap();
595                let nv = ((&*v * 0.999).unwrap() + (g.sqr().unwrap() * 0.001).unwrap()).unwrap();
596                let denom = ((&nv / bc2).unwrap().sqrt().unwrap() + 1e-8).unwrap();
597                let upd = (&nm / denom).unwrap().affine(1e-2 / bc1, 0.0).unwrap();
598                var.set(&(var.as_tensor() - upd).unwrap()).unwrap();
599                *m = nm;
600                *v = nv;
601            }
602        }
603
604        let got_a = opt_a
605            .as_tensor()
606            .flatten_all()
607            .unwrap()
608            .to_vec1::<f32>()
609            .unwrap();
610        let want_a = ref_a
611            .as_tensor()
612            .flatten_all()
613            .unwrap()
614            .to_vec1::<f32>()
615            .unwrap();
616        let got_b = opt_b
617            .as_tensor()
618            .flatten_all()
619            .unwrap()
620            .to_vec1::<f32>()
621            .unwrap();
622        let want_b = ref_b
623            .as_tensor()
624            .flatten_all()
625            .unwrap()
626            .to_vec1::<f32>()
627            .unwrap();
628        for (g, w) in got_a
629            .iter()
630            .zip(want_a.iter())
631            .chain(got_b.iter().zip(want_b.iter()))
632        {
633            assert!((g - w).abs() < 1e-5, "mismatch: got {g} vs want {w}");
634        }
635    }
636}