1use std::cell::RefCell;
2use std::collections::{BTreeMap, HashMap};
3
4use candle_core::backprop::GradStore;
5use candle_core::{CpuStorage, DType, Device, InplaceOp1, Layout, Storage, Tensor, Var};
6use rayon::prelude::*;
7
8use crate::error::CodecError;
9
10const PAR_ELEM_THRESHOLD: usize = 8_000_000;
11
12fn cpu_f32_tensor_slice(tensor: &Tensor) -> Result<(Option<Tensor>, usize, usize), CodecError> {
13 if tensor.dtype() == DType::F32 && tensor.is_contiguous() {
14 if let Some((start, end)) = tensor.layout().contiguous_offsets() {
15 return Ok((None, start, end));
16 }
17 }
18 let owned = tensor.to_dtype(DType::F32)?.contiguous()?;
19 let (start, end) = owned
20 .layout()
21 .contiguous_offsets()
22 .ok_or_else(|| CodecError::InvalidData("expected contiguous tensor".to_owned()))?;
23 Ok((Some(owned), start, end))
24}
25
26fn read_cpu_f32_to_vec(tensor: &Tensor) -> Result<Vec<f32>, CodecError> {
27 let (owned, start, end) = cpu_f32_tensor_slice(tensor)?;
28 let target = owned.as_ref().unwrap_or(tensor);
29 let (storage, _) = target.storage_and_layout();
30 match &*storage {
31 Storage::Cpu(CpuStorage::F32(slice)) => Ok(slice[start..end].to_vec()),
32 _ => Err(CodecError::InvalidData(
33 "expected F32 CPU tensor storage".to_owned(),
34 )),
35 }
36}
37
38fn cpu_grad_sq_norm(grad: &Tensor) -> Result<f64, CodecError> {
39 let (owned, start, end) = cpu_f32_tensor_slice(grad)?;
40 let target = owned.as_ref().unwrap_or(grad);
41 let (storage, _) = target.storage_and_layout();
42 match &*storage {
43 Storage::Cpu(CpuStorage::F32(slice)) => {
44 let data = &slice[start..end];
45 let mut sq_acc = [0.0f32; 8];
46 let chunks = data.chunks_exact(8);
47 let rem = chunks.remainder();
48 for c in chunks {
49 for i in 0..8 {
50 let v = c[i];
51 sq_acc[i] += v * v;
52 }
53 }
54 let mut acc = 0.0f64;
55 for &lane in &sq_acc {
56 acc += f64::from(lane);
57 }
58 for &v in rem {
59 let fv = f64::from(v);
60 acc += fv * fv;
61 }
62 Ok(acc)
63 }
64 _ => Err(CodecError::InvalidData(
65 "expected F32 CPU gradient storage".to_owned(),
66 )),
67 }
68}
69
70fn compute_global_grad_norm(vars: &[(String, Var)], grads: &GradStore) -> Result<f64, CodecError> {
73 let Some((_, first)) = vars.first() else {
74 return Ok(0.0);
75 };
76 if first.device().is_cpu() {
77 let total_elems: usize = vars.iter().map(|(_, v)| v.elem_count()).sum();
78 let sum_sq: f64 = if total_elems >= PAR_ELEM_THRESHOLD {
79 let per_var: Vec<f64> = vars
80 .par_iter()
81 .map(|(_, var)| match grads.get(var) {
82 Some(grad) => cpu_grad_sq_norm(grad),
83 None => Ok(0.0),
84 })
85 .collect::<Result<Vec<_>, _>>()?;
86 per_var.into_iter().sum()
87 } else {
88 let mut acc = 0.0f64;
89 for (_, var) in vars {
90 if let Some(grad) = grads.get(var) {
91 acc += cpu_grad_sq_norm(grad)?;
92 }
93 }
94 acc
95 };
96 return Ok(sum_sq.sqrt());
97 }
98
99 let mut sq_terms = Vec::with_capacity(vars.len());
100 for (_, var) in vars {
101 if let Some(grad) = grads.get(var) {
102 sq_terms.push(grad.to_dtype(DType::F32)?.sqr()?.sum_all()?);
103 }
104 }
105 if sq_terms.is_empty() {
106 return Ok(0.0);
107 }
108 let total_sq = if sq_terms.len() == 1 {
109 sq_terms[0].to_scalar::<f32>()?
110 } else {
111 Tensor::stack(&sq_terms, 0)?.sum_all()?.to_scalar::<f32>()?
112 };
113 Ok(f64::from(total_sq).sqrt())
114}
115
116pub fn clip_grad_norm(
124 vars: &[(String, Var)],
125 grads: &mut GradStore,
126 max_norm: f64,
127) -> Result<f64, CodecError> {
128 let total_norm = compute_global_grad_norm(vars, grads)?;
129 if max_norm > 0.0 {
130 let clip_coef = max_norm / (total_norm + 1e-6);
131 if clip_coef < 1.0 {
132 for (_, var) in vars {
133 if let Some(grad) = grads.remove(var) {
134 let scaled = grad.to_dtype(DType::F32)?.affine(clip_coef, 0.0)?;
135 grads.insert(var, scaled);
136 }
137 }
138 }
139 }
140 Ok(total_norm)
141}
142
143struct CpuAdamInplaceOp<'a> {
144 grad: &'a [f32],
145 m: RefCell<&'a mut [f32]>,
146 v: RefCell<&'a mut [f32]>,
147 clip_scale: f32,
148 beta1: f32,
149 one_minus_beta1: f32,
150 beta2: f32,
151 one_minus_beta2: f32,
152 inv_bias_corr2: f32,
153 step_size: f32,
154 eps: f32,
155}
156
157impl InplaceOp1 for CpuAdamInplaceOp<'_> {
158 fn name(&self) -> &'static str {
159 "cpu-fused-adam"
160 }
161
162 fn cpu_fwd(&self, storage: &mut CpuStorage, layout: &Layout) -> candle_core::Result<()> {
163 let (start, end) = layout.contiguous_offsets().ok_or_else(|| {
164 candle_core::Error::Msg("Adam parameter variable must be contiguous".to_owned())
165 })?;
166 let slice = match storage {
167 CpuStorage::F32(data) => &mut data[start..end],
168 _ => {
169 return Err(candle_core::Error::Msg(
170 "Adam parameter variable must have F32 storage".to_owned(),
171 ))
172 }
173 };
174 let mut m = self.m.borrow_mut();
175 let mut v = self.v.borrow_mut();
176 if slice.len() != self.grad.len() || slice.len() != m.len() || slice.len() != v.len() {
177 return Err(candle_core::Error::Msg(format!(
178 "Adam length mismatch: param={} grad={} m={} v={}",
179 slice.len(),
180 self.grad.len(),
181 m.len(),
182 v.len()
183 )));
184 }
185 for (((p_val, m_val), v_val), &g_raw) in slice
186 .iter_mut()
187 .zip(m.iter_mut())
188 .zip(v.iter_mut())
189 .zip(self.grad.iter())
190 {
191 let g = g_raw * self.clip_scale;
192 let next_m = *m_val * self.beta1 + g * self.one_minus_beta1;
193 let next_v = *v_val * self.beta2 + (g * g) * self.one_minus_beta2;
194 let denom = (next_v * self.inv_bias_corr2).sqrt() + self.eps;
195 *p_val -= (next_m / denom) * self.step_size;
196 *m_val = next_m;
197 *v_val = next_v;
198 }
199 Ok(())
200 }
201}
202
203#[derive(Debug)]
204enum MomentBuffer {
205 Cpu { m: Vec<f32>, v: Vec<f32> },
206 Device { m: Tensor, v: Tensor },
207}
208
209#[derive(Debug)]
210struct AdamParamState {
211 name: String,
212 var: Var,
213 step: u64,
214 moments: MomentBuffer,
215}
216
217impl AdamParamState {
218 #[allow(clippy::too_many_arguments)]
219 fn apply_update(
220 &mut self,
221 raw_grad: &Tensor,
222 clip_scale: f64,
223 lr: f64,
224 beta1: f64,
225 beta2: f64,
226 eps: f64,
227 ) -> Result<(), CodecError> {
228 self.step += 1;
229 let t = self.step as i32;
230 let bias_corr1 = 1.0 - beta1.powi(t);
231 let bias_corr2 = 1.0 - beta2.powi(t);
232 let step_size = lr / bias_corr1;
233
234 match &mut self.moments {
235 MomentBuffer::Cpu { m, v } => {
236 let (owned_grad, start, end) = cpu_f32_tensor_slice(raw_grad)?;
237 let grad_tensor = owned_grad.as_ref().unwrap_or(raw_grad);
238 let (storage, _) = grad_tensor.storage_and_layout();
239 let grad_slice = match &*storage {
240 Storage::Cpu(CpuStorage::F32(data)) => &data[start..end],
241 _ => {
242 return Err(CodecError::InvalidData(
243 "expected F32 CPU gradient storage".to_owned(),
244 ))
245 }
246 };
247 let op = CpuAdamInplaceOp {
248 grad: grad_slice,
249 m: RefCell::new(m.as_mut_slice()),
250 v: RefCell::new(v.as_mut_slice()),
251 clip_scale: clip_scale as f32,
252 beta1: beta1 as f32,
253 one_minus_beta1: (1.0 - beta1) as f32,
254 beta2: beta2 as f32,
255 one_minus_beta2: (1.0 - beta2) as f32,
256 inv_bias_corr2: (1.0 / bias_corr2) as f32,
257 step_size: step_size as f32,
258 eps: eps as f32,
259 };
260 self.var.as_tensor().inplace_op1(&op)?;
261 }
262 MomentBuffer::Device { m, v } => {
263 let g = if raw_grad.dtype() == DType::F32 {
264 raw_grad.detach()
265 } else {
266 raw_grad.to_dtype(DType::F32)?.detach()
267 };
268 let g_b1 = (1.0 - beta1) * clip_scale;
269 let g_b2 = (1.0 - beta2) * clip_scale * clip_scale;
270 let next_m = ((&*m * beta1)? + (&g * g_b1)?)?.detach();
271 let next_v = ((&*v * beta2)? + (g.sqr()? * g_b2)?)?.detach();
272 let denom = ((&next_v / bias_corr2)?.sqrt()? + eps)?;
273 let update = (&next_m / denom)?.affine(step_size, 0.0)?;
274 let next_param = (self.var.as_tensor() - update)?;
275 self.var.set(&next_param)?;
276 *m = next_m;
277 *v = next_v;
278 }
279 }
280 Ok(())
281 }
282}
283
284#[derive(Debug)]
286pub struct Adam {
287 params: Vec<AdamParamState>,
288 total_elems: usize,
289 is_cpu: bool,
290 lr: f64,
291 beta1: f64,
292 beta2: f64,
293 eps: f64,
294}
295
296impl Adam {
297 pub fn new(
303 vars: &[(String, Var)],
304 lr: f64,
305 beta1: f64,
306 beta2: f64,
307 eps: f64,
308 ) -> Result<Self, CodecError> {
309 let mut params = Vec::with_capacity(vars.len());
310 let mut total_elems = 0usize;
311 let is_cpu = vars.first().map_or(true, |(_, v)| v.device().is_cpu());
312
313 for (name, var) in vars {
314 let n = var.elem_count();
315 total_elems += n;
316 let moments = if var.device().is_cpu() {
317 MomentBuffer::Cpu {
318 m: vec![0.0f32; n],
319 v: vec![0.0f32; n],
320 }
321 } else {
322 MomentBuffer::Device {
323 m: Tensor::zeros(var.shape(), DType::F32, var.device())?,
324 v: Tensor::zeros(var.shape(), DType::F32, var.device())?,
325 }
326 };
327 params.push(AdamParamState {
328 name: name.clone(),
329 var: var.clone(),
330 step: 0,
331 moments,
332 });
333 }
334 Ok(Self {
335 params,
336 total_elems,
337 is_cpu,
338 lr,
339 beta1,
340 beta2,
341 eps,
342 })
343 }
344
345 #[must_use]
347 pub fn learning_rate(&self) -> f64 {
348 self.lr
349 }
350
351 pub fn set_learning_rate(&mut self, lr: f64) {
353 self.lr = lr;
354 }
355
356 pub fn step(&mut self, grads: &GradStore) -> Result<(), CodecError> {
362 self.apply_step(grads, 1.0)
363 }
364
365 pub fn step_clipped(&mut self, grads: &GradStore, max_norm: f64) -> Result<f64, CodecError> {
372 let total_norm = self.global_grad_norm(grads)?;
373 let clip_scale = if max_norm > 0.0 {
374 (max_norm / (total_norm + 1e-6)).min(1.0)
375 } else {
376 1.0
377 };
378 self.apply_step(grads, clip_scale)?;
379 Ok(total_norm)
380 }
381
382 fn global_grad_norm(&self, grads: &GradStore) -> Result<f64, CodecError> {
383 if self.params.is_empty() {
384 return Ok(0.0);
385 }
386 if self.is_cpu {
387 let sum_sq: f64 = if self.total_elems >= PAR_ELEM_THRESHOLD {
388 let per_param: Vec<f64> = self
389 .params
390 .par_iter()
391 .map(|p| match grads.get(&p.var) {
392 Some(grad) => cpu_grad_sq_norm(grad),
393 None => Ok(0.0),
394 })
395 .collect::<Result<Vec<_>, _>>()?;
396 per_param.into_iter().sum()
397 } else {
398 let mut acc = 0.0f64;
399 for p in &self.params {
400 if let Some(grad) = grads.get(&p.var) {
401 acc += cpu_grad_sq_norm(grad)?;
402 }
403 }
404 acc
405 };
406 return Ok(sum_sq.sqrt());
407 }
408
409 let mut sq_terms = Vec::with_capacity(self.params.len());
410 for p in &self.params {
411 if let Some(grad) = grads.get(&p.var) {
412 sq_terms.push(grad.to_dtype(DType::F32)?.sqr()?.sum_all()?);
413 }
414 }
415 if sq_terms.is_empty() {
416 return Ok(0.0);
417 }
418 let total_sq = if sq_terms.len() == 1 {
419 sq_terms[0].to_scalar::<f32>()?
420 } else {
421 Tensor::stack(&sq_terms, 0)?.sum_all()?.to_scalar::<f32>()?
422 };
423 Ok(f64::from(total_sq).sqrt())
424 }
425
426 fn apply_step(&mut self, grads: &GradStore, clip_scale: f64) -> Result<(), CodecError> {
427 let lr = self.lr;
428 let beta1 = self.beta1;
429 let beta2 = self.beta2;
430 let eps = self.eps;
431
432 if self.is_cpu && self.total_elems >= PAR_ELEM_THRESHOLD {
433 self.params
434 .par_iter_mut()
435 .try_for_each(|p| -> Result<(), CodecError> {
436 let Some(raw_grad) = grads.get(&p.var) else {
437 return Ok(());
438 };
439 p.apply_update(raw_grad, clip_scale, lr, beta1, beta2, eps)
440 })?;
441 } else {
442 for p in &mut self.params {
443 let Some(raw_grad) = grads.get(&p.var) else {
444 continue;
445 };
446 p.apply_update(raw_grad, clip_scale, lr, beta1, beta2, eps)?;
447 }
448 }
449 Ok(())
450 }
451
452 pub fn export_state(
458 &self,
459 prefix: &str,
460 out_tensors: &mut HashMap<String, Tensor>,
461 ) -> Result<BTreeMap<String, u64>, CodecError> {
462 let mut steps = BTreeMap::new();
463 for p in &self.params {
464 steps.insert(p.name.clone(), p.step);
465 let (m_t, v_t) = match &p.moments {
466 MomentBuffer::Cpu { m, v } => (
467 Tensor::from_slice(m, p.var.shape(), &Device::Cpu)?,
468 Tensor::from_slice(v, p.var.shape(), &Device::Cpu)?,
469 ),
470 MomentBuffer::Device { m, v } => (
471 m.to_device(&Device::Cpu)?.contiguous()?,
472 v.to_device(&Device::Cpu)?.contiguous()?,
473 ),
474 };
475 out_tensors.insert(format!("{prefix}.{}.m", p.name), m_t);
476 out_tensors.insert(format!("{prefix}.{}.v", p.name), v_t);
477 }
478 Ok(steps)
479 }
480
481 pub fn import_state(
487 &mut self,
488 prefix: &str,
489 steps: &BTreeMap<String, u64>,
490 tensors: &HashMap<String, Tensor>,
491 ) -> Result<(), CodecError> {
492 for p in &mut self.params {
493 let step = steps.get(&p.name).copied().ok_or_else(|| {
494 CodecError::CheckpointMismatch(format!(
495 "checkpoint missing {prefix} step count for {}",
496 p.name
497 ))
498 })?;
499 let m_key = format!("{prefix}.{}.m", p.name);
500 let v_key = format!("{prefix}.{}.v", p.name);
501 let m_src = tensors.get(&m_key).ok_or_else(|| {
502 CodecError::CheckpointMismatch(format!("checkpoint missing tensor {m_key}"))
503 })?;
504 let v_src = tensors.get(&v_key).ok_or_else(|| {
505 CodecError::CheckpointMismatch(format!("checkpoint missing tensor {v_key}"))
506 })?;
507 if m_src.dims() != p.var.dims() || v_src.dims() != p.var.dims() {
508 return Err(CodecError::CheckpointMismatch(format!(
509 "checkpoint optimizer moment shape mismatch for {}",
510 p.name
511 )));
512 }
513 p.step = step;
514 match &mut p.moments {
515 MomentBuffer::Cpu { m, v } => {
516 *m = read_cpu_f32_to_vec(m_src)?;
517 *v = read_cpu_f32_to_vec(v_src)?;
518 }
519 MomentBuffer::Device { m, v } => {
520 *m = m_src
521 .to_device(p.var.device())?
522 .to_dtype(DType::F32)?
523 .contiguous()?;
524 *v = v_src
525 .to_device(p.var.device())?
526 .to_dtype(DType::F32)?
527 .contiguous()?;
528 }
529 }
530 }
531 Ok(())
532 }
533}
534
535#[cfg(test)]
536mod tests {
537 use super::*;
538
539 #[test]
540 fn fused_cpu_adam_step_clipped_matches_unfused_reference() {
541 let init_a = vec![0.3f32, -0.7, 1.2, -0.05, 2.0, 0.11, -1.4, 0.8];
542 let init_b = vec![-0.4f32, 0.9, -1.1, 0.6];
543 let x = Tensor::from_slice(
544 &[1.5f32, -0.5, 0.25, 0.75, -1.0, 0.2, 0.8, -0.3],
545 (1, 8),
546 &Device::Cpu,
547 )
548 .unwrap();
549
550 let ref_a = Var::from_slice(&init_a, (1, 8), &Device::Cpu).unwrap();
551 let ref_b = Var::from_slice(&init_b, (4, 1), &Device::Cpu).unwrap();
552 let opt_a = Var::from_slice(&init_a, (1, 8), &Device::Cpu).unwrap();
553 let opt_b = Var::from_slice(&init_b, (4, 1), &Device::Cpu).unwrap();
554
555 let mut opt = Adam::new(
556 &[
557 ("a".to_owned(), opt_a.clone()),
558 ("b".to_owned(), opt_b.clone()),
559 ],
560 1e-2,
561 0.9,
562 0.999,
563 1e-8,
564 )
565 .unwrap();
566
567 let mut m_a = Tensor::zeros(ref_a.shape(), DType::F32, &Device::Cpu).unwrap();
568 let mut v_a = Tensor::zeros(ref_a.shape(), DType::F32, &Device::Cpu).unwrap();
569 let mut m_b = Tensor::zeros(ref_b.shape(), DType::F32, &Device::Cpu).unwrap();
570 let mut v_b = Tensor::zeros(ref_b.shape(), DType::F32, &Device::Cpu).unwrap();
571
572 for step in 1usize..=4 {
573 let loss_ref = ((ref_a.as_tensor() * &x).unwrap().sum_all().unwrap()
574 + ref_b.as_tensor().sqr().unwrap().sum_all().unwrap())
575 .unwrap();
576 let mut grads_ref = loss_ref.backward().unwrap();
577 let ref_vars = [
578 ("a".to_owned(), ref_a.clone()),
579 ("b".to_owned(), ref_b.clone()),
580 ];
581 let norm_ref = clip_grad_norm(&ref_vars, &mut grads_ref, 0.5).unwrap();
582
583 let loss_opt = ((opt_a.as_tensor() * &x).unwrap().sum_all().unwrap()
584 + opt_b.as_tensor().sqr().unwrap().sum_all().unwrap())
585 .unwrap();
586 let grads_opt = loss_opt.backward().unwrap();
587 let norm_opt = opt.step_clipped(&grads_opt, 0.5).unwrap();
588 assert!((norm_ref - norm_opt).abs() < 1e-6);
589
590 let bc1 = 1.0 - 0.9f64.powi(step as i32);
591 let bc2 = 1.0 - 0.999f64.powi(step as i32);
592 for (var, m, v) in [(&ref_a, &mut m_a, &mut v_a), (&ref_b, &mut m_b, &mut v_b)] {
593 let g = grads_ref.get(var).unwrap().detach();
594 let nm = ((&*m * 0.9).unwrap() + (&g * 0.1).unwrap()).unwrap();
595 let nv = ((&*v * 0.999).unwrap() + (g.sqr().unwrap() * 0.001).unwrap()).unwrap();
596 let denom = ((&nv / bc2).unwrap().sqrt().unwrap() + 1e-8).unwrap();
597 let upd = (&nm / denom).unwrap().affine(1e-2 / bc1, 0.0).unwrap();
598 var.set(&(var.as_tensor() - upd).unwrap()).unwrap();
599 *m = nm;
600 *v = nv;
601 }
602 }
603
604 let got_a = opt_a
605 .as_tensor()
606 .flatten_all()
607 .unwrap()
608 .to_vec1::<f32>()
609 .unwrap();
610 let want_a = ref_a
611 .as_tensor()
612 .flatten_all()
613 .unwrap()
614 .to_vec1::<f32>()
615 .unwrap();
616 let got_b = opt_b
617 .as_tensor()
618 .flatten_all()
619 .unwrap()
620 .to_vec1::<f32>()
621 .unwrap();
622 let want_b = ref_b
623 .as_tensor()
624 .flatten_all()
625 .unwrap()
626 .to_vec1::<f32>()
627 .unwrap();
628 for (g, w) in got_a
629 .iter()
630 .zip(want_a.iter())
631 .chain(got_b.iter().zip(want_b.iter()))
632 {
633 assert!((g - w).abs() < 1e-5, "mismatch: got {g} vs want {w}");
634 }
635 }
636}