Skip to main content

tufazip_codec/
model.rs

1use std::collections::{BTreeMap, HashMap};
2use std::path::Path;
3
4use candle_core::{CpuStorage, CustomOp2, DType, Device, Layout, Shape, Storage, Tensor, Var, D};
5use serde::{Deserialize, Serialize};
6use sha2::{Digest, Sha256};
7use tufazip_prior::{plan_depth_prior, read_archive, InitializationKind, PriorArchive};
8
9use crate::config::{ModelConfig, NormStyle, PositionType};
10use crate::error::CodecError;
11use crate::rng::DeterministicRng;
12
13const RMS_NORM_EPS: f64 = 1e-5;
14
15/// Forward pass execution mode.
16#[derive(Debug)]
17pub enum ForwardMode<'a> {
18    /// Deterministic evaluation mode (dropout disabled; used for online coding).
19    Eval,
20    /// Training/replay mode using the given deterministic PRNG for dropout.
21    Train(&'a mut DeterministicRng),
22}
23
24/// Selects which sequence positions are projected through the vocabulary head.
25#[derive(Debug, Clone, Copy, PartialEq, Eq)]
26pub enum OutputSlice {
27    /// Project all sequence positions `[0..seq]`.
28    All,
29    /// Project only the final sequence position (`seq - 1..seq`).
30    Last,
31    /// Project a contiguous sub-range `start..start + len`.
32    Range {
33        /// Starting token position index.
34        start: usize,
35        /// Number of positions to project.
36        len: usize,
37    },
38}
39
40/// Summary of loading a `QPI1` prior archive into a [`Transformer`].
41#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
42pub struct PriorLoadInfo {
43    /// Archive size in bytes.
44    pub archive_bytes: u64,
45    /// SHA-256 hex digest of the archive file.
46    pub archive_sha256: String,
47    /// SHA-256 hex digest of the vocabulary file.
48    pub vocab_sha256: String,
49    /// Prior expansion kind (`depth-growth` or `loop-untying`).
50    pub initialization_kind: InitializationKind,
51    /// Number of physical source layers in the archive.
52    pub source_depth: usize,
53    /// Effective target-to-source block mapping.
54    pub block_mapping: Vec<usize>,
55    /// Optional recurrent schedule for loop-untying.
56    pub block_schedule: Option<Vec<usize>>,
57    /// Canonical SHA-256 digest of the initialized model state.
58    pub initial_state_sha256: String,
59    /// Optional quantization policy label from archive metadata.
60    pub policy: Option<String>,
61}
62
63#[derive(Debug, Clone, Copy)]
64struct FusedCpuRmsNorm {
65    eps: f64,
66}
67
68impl FusedCpuRmsNorm {
69    fn compute_row_fwd(x_row: &[f32], w: &[f32], out_row: &mut [f32], eps: f64) {
70        let mut sq_acc = [0.0f32; 8];
71        let x_chunks = x_row.chunks_exact(8);
72        let x_rem = x_chunks.remainder();
73        for xc in x_chunks {
74            for i in 0..8 {
75                let x_v = xc[i];
76                sq_acc[i] += x_v * x_v;
77            }
78        }
79        let mut sum_sq = 0.0f64;
80        for &lane in &sq_acc {
81            sum_sq += f64::from(lane);
82        }
83        for &v in x_rem {
84            let fv = f64::from(v);
85            sum_sq += fv * fv;
86        }
87        let inv_rms = (1.0 / (sum_sq / (x_row.len() as f64) + eps).sqrt()) as f32;
88        for ((dst, &x_val), &w_val) in out_row.iter_mut().zip(x_row.iter()).zip(w.iter()) {
89            *dst = x_val * inv_rms * w_val;
90        }
91    }
92
93    fn compute_row_bwd(
94        x_row: &[f32],
95        dy_row: &[f32],
96        w: &[f32],
97        gx_row: &mut [f32],
98        gw_acc: &mut [f64],
99        eps: f64,
100    ) {
101        let dim = x_row.len();
102        let mut sq_acc = [0.0f32; 8];
103        let mut dot_acc = [0.0f32; 8];
104        let x_chunks = x_row.chunks_exact(8);
105        let dy_chunks = dy_row.chunks_exact(8);
106        let w_chunks = w.chunks_exact(8);
107        let x_rem = x_chunks.remainder();
108        let dy_rem = dy_chunks.remainder();
109        let w_rem = w_chunks.remainder();
110        for ((xc, dyc), wc) in x_chunks.zip(dy_chunks).zip(w_chunks) {
111            for i in 0..8 {
112                let x_v = xc[i];
113                sq_acc[i] += x_v * x_v;
114                dot_acc[i] += dyc[i] * wc[i] * x_v;
115            }
116        }
117        let mut sum_sq = 0.0f64;
118        let mut dot = 0.0f64;
119        for i in 0..8 {
120            sum_sq += f64::from(sq_acc[i]);
121            dot += f64::from(dot_acc[i]);
122        }
123        for ((&x_v, &dy_v), &w_v) in x_rem.iter().zip(dy_rem.iter()).zip(w_rem.iter()) {
124            let x_f = f64::from(x_v);
125            sum_sq += x_f * x_f;
126            dot += f64::from(dy_v) * f64::from(w_v) * x_f;
127        }
128        let inv_rms = 1.0 / (sum_sq / (dim as f64) + eps).sqrt();
129        let coeff = dot * inv_rms * inv_rms / (dim as f64);
130        let inv_rms_f32 = inv_rms as f32;
131        let coeff_f32 = coeff as f32;
132        for ((((gx_slot, &dy_val), &w_val), &x_val), gw_slot) in gx_row
133            .iter_mut()
134            .zip(dy_row.iter())
135            .zip(w.iter())
136            .zip(x_row.iter())
137            .zip(gw_acc.iter_mut())
138        {
139            *gx_slot = inv_rms_f32 * (dy_val * w_val - x_val * coeff_f32);
140            let dy_f = f64::from(dy_val);
141            let x_f = f64::from(x_val);
142            *gw_slot += dy_f * x_f * inv_rms;
143        }
144    }
145}
146
147impl CustomOp2 for FusedCpuRmsNorm {
148    fn name(&self) -> &'static str {
149        "fused-cpu-rms-norm"
150    }
151
152    fn cpu_fwd(
153        &self,
154        s1: &CpuStorage,
155        l1: &Layout,
156        s2: &CpuStorage,
157        l2: &Layout,
158    ) -> candle_core::Result<(CpuStorage, Shape)> {
159        let (x_start, x_end) = l1.contiguous_offsets().ok_or_else(|| {
160            candle_core::Error::Msg("fused-cpu-rms-norm requires contiguous x".to_owned())
161        })?;
162        let (w_start, w_end) = l2.contiguous_offsets().ok_or_else(|| {
163            candle_core::Error::Msg("fused-cpu-rms-norm requires contiguous weight".to_owned())
164        })?;
165        let x_all = match s1 {
166            CpuStorage::F32(v) => &v[x_start..x_end],
167            _ => {
168                return Err(candle_core::Error::Msg(
169                    "fused-cpu-rms-norm expects F32 x".to_owned(),
170                ))
171            }
172        };
173        let w = match s2 {
174            CpuStorage::F32(v) => &v[w_start..w_end],
175            _ => {
176                return Err(candle_core::Error::Msg(
177                    "fused-cpu-rms-norm expects F32 weight".to_owned(),
178                ))
179            }
180        };
181        let dim = w.len();
182        if dim == 0 || l1.dims().last() != Some(&dim) || x_all.len() % dim != 0 {
183            return Err(candle_core::Error::Msg(format!(
184                "fused-cpu-rms-norm shape mismatch: x={:?}, w={:?}",
185                l1.shape(),
186                l2.shape()
187            )));
188        }
189        let mut out = vec![0.0f32; x_all.len()];
190        let eps = self.eps;
191        for (out_row, x_row) in out.chunks_exact_mut(dim).zip(x_all.chunks_exact(dim)) {
192            Self::compute_row_fwd(x_row, w, out_row, eps);
193        }
194        Ok((CpuStorage::F32(out), l1.shape().clone()))
195    }
196
197    fn bwd(
198        &self,
199        arg1: &Tensor,
200        arg2: &Tensor,
201        _res: &Tensor,
202        grad_res: &Tensor,
203    ) -> candle_core::Result<(Option<Tensor>, Option<Tensor>)> {
204        let x_c = if arg1.is_contiguous() && arg1.dtype() == DType::F32 {
205            arg1.clone()
206        } else {
207            arg1.to_dtype(DType::F32)?.contiguous()?
208        };
209        let w_c = if arg2.is_contiguous() && arg2.dtype() == DType::F32 {
210            arg2.clone()
211        } else {
212            arg2.to_dtype(DType::F32)?.contiguous()?
213        };
214        let dy_c = if grad_res.is_contiguous() && grad_res.dtype() == DType::F32 {
215            grad_res.clone()
216        } else {
217            grad_res.to_dtype(DType::F32)?.contiguous()?
218        };
219
220        let (x_st, x_l) = x_c.storage_and_layout();
221        let (w_st, w_l) = w_c.storage_and_layout();
222        let (dy_st, dy_l) = dy_c.storage_and_layout();
223        let (x_s, x_e) = x_l
224            .contiguous_offsets()
225            .ok_or_else(|| candle_core::Error::Msg("non-contiguous x in RmsNorm bwd".to_owned()))?;
226        let (w_s, w_e) = w_l
227            .contiguous_offsets()
228            .ok_or_else(|| candle_core::Error::Msg("non-contiguous w in RmsNorm bwd".to_owned()))?;
229        let (dy_s, dy_e) = dy_l.contiguous_offsets().ok_or_else(|| {
230            candle_core::Error::Msg("non-contiguous dy in RmsNorm bwd".to_owned())
231        })?;
232
233        let (x_slice, w_slice, dy_slice) = match (&*x_st, &*w_st, &*dy_st) {
234            (
235                Storage::Cpu(CpuStorage::F32(xv)),
236                Storage::Cpu(CpuStorage::F32(wv)),
237                Storage::Cpu(CpuStorage::F32(dyv)),
238            ) => (&xv[x_s..x_e], &wv[w_s..w_e], &dyv[dy_s..dy_e]),
239            _ => {
240                return Err(candle_core::Error::Msg(
241                    "fused-cpu-rms-norm bwd expects F32 CPU storage".to_owned(),
242                ))
243            }
244        };
245
246        let dim = w_slice.len();
247        if dim == 0 || x_slice.is_empty() {
248            return Ok((
249                Some(Tensor::zeros(arg1.shape(), DType::F32, arg1.device())?),
250                Some(Tensor::zeros(arg2.shape(), DType::F32, arg2.device())?),
251            ));
252        }
253
254        let mut gx = vec![0.0f32; x_slice.len()];
255        let mut gw_acc = vec![0.0f64; dim];
256        let eps = self.eps;
257
258        for ((gx_row, x_row), dy_row) in gx
259            .chunks_exact_mut(dim)
260            .zip(x_slice.chunks_exact(dim))
261            .zip(dy_slice.chunks_exact(dim))
262        {
263            Self::compute_row_bwd(x_row, dy_row, w_slice, gx_row, &mut gw_acc, eps);
264        }
265
266        let gw: Vec<f32> = gw_acc.into_iter().map(|v| v as f32).collect();
267        let gx_t = Tensor::from_vec(gx, arg1.shape().clone(), arg1.device())?;
268        let gw_t = Tensor::from_vec(gw, arg2.shape().clone(), arg2.device())?;
269        Ok((Some(gx_t), Some(gw_t)))
270    }
271}
272
273#[derive(Debug, Clone)]
274struct RmsNorm {
275    weight: Option<Var>,
276    eps: f64,
277}
278
279impl RmsNorm {
280    fn new(dim: usize, enabled: bool, device: &Device) -> Result<Self, CodecError> {
281        let weight = if enabled {
282            Some(Var::ones(dim, DType::F32, device)?)
283        } else {
284            None
285        };
286        Ok(Self {
287            weight,
288            eps: RMS_NORM_EPS,
289        })
290    }
291
292    fn forward_unfused(&self, x: &Tensor, weight: &Var) -> Result<Tensor, CodecError> {
293        let x_dtype = x.dtype();
294        let x_f32 = if x_dtype == DType::F32 {
295            x.clone()
296        } else {
297            x.to_dtype(DType::F32)?
298        };
299        let var = x_f32.sqr()?.mean_keepdim(D::Minus1)?;
300        let inv_std = (var + self.eps)?.sqrt()?.recip()?;
301        let normed = x_f32.broadcast_mul(&inv_std)?;
302        let normed_cast = if x_dtype == DType::F32 {
303            normed
304        } else {
305            normed.to_dtype(x_dtype)?
306        };
307        let w = if x_dtype == DType::F32 {
308            weight.as_tensor().clone()
309        } else {
310            weight.as_tensor().to_dtype(x_dtype)?
311        };
312        Ok(normed_cast.broadcast_mul(&w)?)
313    }
314
315    fn forward(&self, x: &Tensor) -> Result<Tensor, CodecError> {
316        let Some(weight) = &self.weight else {
317            return Ok(x.clone());
318        };
319        if x.device().is_cpu() && x.dtype() == DType::F32 && weight.dtype() == DType::F32 {
320            let x_c = if x.is_contiguous() {
321                x.clone()
322            } else {
323                x.contiguous()?
324            };
325            return Ok(x_c.apply_op2(weight.as_tensor(), FusedCpuRmsNorm { eps: self.eps })?);
326        }
327        self.forward_unfused(x, weight)
328    }
329}
330
331fn linear_forward(x: &Tensor, weight: &Var, compute_dtype: DType) -> Result<Tensor, CodecError> {
332    let w = if compute_dtype == DType::F32 {
333        weight.as_tensor().clone()
334    } else {
335        weight.as_tensor().to_dtype(compute_dtype)?
336    };
337    let out_dim = w.dim(0)?;
338    match *x.dims() {
339        [batch, seq, hidden] => {
340            let x_flat = x.contiguous()?.reshape((batch * seq, hidden))?;
341            let out = x_flat.matmul(&w.t()?)?;
342            Ok(out.reshape((batch, seq, out_dim))?)
343        }
344        _ => Ok(x.matmul(&w.t()?)?),
345    }
346}
347
348fn apply_dropout(
349    x: &Tensor,
350    drop_p: f32,
351    mode: &mut ForwardMode<'_>,
352) -> Result<Tensor, CodecError> {
353    if drop_p == 0.0 {
354        return Ok(x.clone());
355    }
356    match mode {
357        ForwardMode::Eval => Ok(x.clone()),
358        ForwardMode::Train(rng) => {
359            let mask = rng.dropout_mask(x.shape(), drop_p, x.dtype(), x.device())?;
360            Ok(x.mul(&mask)?)
361        }
362    }
363}
364
365fn build_rope_cache(
366    ctx_len: usize,
367    head_dim: usize,
368    base: f64,
369    device: &Device,
370) -> Result<(Tensor, Tensor), CodecError> {
371    if head_dim % 2 != 0 {
372        return Err(CodecError::InvalidConfig(format!(
373            "head_dim ({head_dim}) must be even for RoPE"
374        )));
375    }
376    let half = head_dim / 2;
377    let mut cos_data = vec![0.0f32; ctx_len * head_dim];
378    let mut sin_data = vec![0.0f32; ctx_len * head_dim];
379    let inv_freq: Vec<f64> = (0..half)
380        .map(|i| 1.0f64 / base.powf((2 * i) as f64 / head_dim as f64))
381        .collect();
382
383    for pos in 0..ctx_len {
384        let row = pos * head_dim;
385        for (i, &freq) in inv_freq.iter().enumerate() {
386            let angle = (pos as f64) * freq;
387            let c = angle.cos() as f32;
388            let s = angle.sin() as f32;
389            cos_data[row + i] = c;
390            cos_data[row + half + i] = c;
391            sin_data[row + i] = s;
392            sin_data[row + half + i] = s;
393        }
394    }
395
396    let cos = Tensor::from_vec(cos_data, (1, 1, ctx_len, head_dim), device)?;
397    let sin = Tensor::from_vec(sin_data, (1, 1, ctx_len, head_dim), device)?;
398    Ok((cos, sin))
399}
400
401fn apply_rope(x: &Tensor, cos: &Tensor, sin: &Tensor) -> Result<Tensor, CodecError> {
402    let head_dim = x.dim(D::Minus1)?;
403    let half = head_dim / 2;
404    let x1 = x.narrow(D::Minus1, 0, half)?;
405    let x2 = x.narrow(D::Minus1, half, half)?;
406    let rotated = Tensor::cat(&[&x2.neg()?, &x1], D::Minus1)?;
407    let term1 = x.broadcast_mul(cos)?;
408    let term2 = rotated.broadcast_mul(sin)?;
409    Ok((term1 + term2)?)
410}
411
412fn build_causal_mask(ctx_len: usize, device: &Device) -> Result<Tensor, CodecError> {
413    let mut data = vec![0.0f32; ctx_len * ctx_len];
414    for i in 0..ctx_len {
415        for j in (i + 1)..ctx_len {
416            data[i * ctx_len + j] = f32::NEG_INFINITY;
417        }
418    }
419    Ok(Tensor::from_vec(data, (1, 1, ctx_len, ctx_len), device)?)
420}
421
422#[derive(Debug, Clone)]
423struct RelativePosParams {
424    r_emb: Var,
425    r_w_bias: Var,
426    r_bias: Var,
427}
428
429fn init_normal_var(
430    shape: impl Into<Shape>,
431    std: f32,
432    device: &Device,
433    rng: Option<&mut DeterministicRng>,
434) -> Result<Var, CodecError> {
435    let shape = shape.into();
436    match rng {
437        Some(rng) => {
438            let data = rng.sample_normal_vec(shape.elem_count(), 0.0, std)?;
439            Ok(Var::from_vec(data, shape, device)?)
440        }
441        None => Ok(Var::zeros(shape, DType::F32, device)?),
442    }
443}
444
445#[derive(Debug, Clone)]
446struct Attention {
447    n_head: usize,
448    d_head: usize,
449    d_model: usize,
450    dropout: f32,
451    position_type: PositionType,
452    qkv: Var,
453    proj: Var,
454    q_norm: RmsNorm,
455    k_norm: RmsNorm,
456    rel_pos: Option<RelativePosParams>,
457}
458
459impl Attention {
460    fn new(
461        cfg: &ModelConfig,
462        residual_std: f32,
463        device: &Device,
464        mut rng: Option<&mut DeterministicRng>,
465    ) -> Result<Self, CodecError> {
466        let d_model = cfg.d_model;
467        let d_head = d_model / cfg.n_head;
468        let qkv = init_normal_var((3 * d_model, d_model), 0.02, device, rng.as_deref_mut())?;
469        let proj = init_normal_var((d_model, d_model), residual_std, device, rng.as_deref_mut())?;
470        let q_norm = RmsNorm::new(d_head, cfg.qk_norm, device)?;
471        let k_norm = RmsNorm::new(d_head, cfg.qk_norm, device)?;
472
473        let rel_pos = if cfg.position_type == PositionType::LearnedRelative {
474            let r_emb = init_normal_var(
475                (cfg.ctx_len, cfg.n_head, d_head),
476                0.02,
477                device,
478                rng.as_deref_mut(),
479            )?;
480            let r_w_bias = init_normal_var((cfg.n_head, d_head), 0.02, device, rng)?;
481            let r_bias = Var::zeros((cfg.ctx_len, cfg.n_head), DType::F32, device)?;
482            Some(RelativePosParams {
483                r_emb,
484                r_w_bias,
485                r_bias,
486            })
487        } else {
488            None
489        };
490
491        Ok(Self {
492            n_head: cfg.n_head,
493            d_head,
494            d_model,
495            dropout: cfg.dropout,
496            position_type: cfg.position_type,
497            qkv,
498            proj,
499            q_norm,
500            k_norm,
501            rel_pos,
502        })
503    }
504
505    fn forward(
506        &self,
507        x: &Tensor,
508        rope_cos: Option<&Tensor>,
509        rope_sin: Option<&Tensor>,
510        causal_mask: Option<&Tensor>,
511        mode: &mut ForwardMode<'_>,
512        compute_dtype: DType,
513    ) -> Result<Tensor, CodecError> {
514        let (batch, seq, d_model) = x.dims3()?;
515        let qkv = linear_forward(x, &self.qkv, compute_dtype)?;
516        let qkv_split = qkv
517            .reshape((batch, seq, 3, self.n_head, self.d_head))?
518            .permute((2, 0, 3, 1, 4))?
519            .contiguous()?;
520        let q = qkv_split.narrow(0, 0, 1)?.squeeze(0)?;
521        let k = qkv_split.narrow(0, 1, 1)?.squeeze(0)?;
522        let v = qkv_split.narrow(0, 2, 1)?.squeeze(0)?;
523
524        let q = self.q_norm.forward(&q)?;
525        let k = self.k_norm.forward(&k)?;
526
527        let scores = if self.position_type == PositionType::Rope {
528            let cos = rope_cos
529                .ok_or_else(|| CodecError::InvalidConfig("RoPE cosine cache missing".to_owned()))?;
530            let sin = rope_sin
531                .ok_or_else(|| CodecError::InvalidConfig("RoPE sine cache missing".to_owned()))?;
532            let q = apply_rope(&q, cos, sin)?;
533            let k = apply_rope(&k, cos, sin)?;
534            let q_f32 = q.to_dtype(DType::F32)?;
535            let k_f32 = k.to_dtype(DType::F32)?.transpose(2, 3)?.contiguous()?;
536            let scale = 1.0 / (self.d_head as f64).sqrt();
537            q_f32.matmul(&k_f32)?.affine(scale, 0.0)?
538        } else {
539            let rel = self.rel_pos.as_ref().ok_or_else(|| {
540                CodecError::InvalidConfig("learned_relative parameters missing".to_owned())
541            })?;
542            let q_f32 = q.to_dtype(DType::F32)?;
543            let k_f32 = k.to_dtype(DType::F32)?.transpose(2, 3)?.contiguous()?;
544            let r_w_bias = rel.r_w_bias.as_tensor().unsqueeze(0)?.unsqueeze(2)?;
545            let q_biased = q_f32.broadcast_add(&r_w_bias)?;
546            let content_score = q_biased.matmul(&k_f32)?;
547
548            let r_emb_seq = rel
549                .r_emb
550                .as_tensor()
551                .narrow(0, 0, seq)?
552                .permute((1, 2, 0))?
553                .contiguous()?
554                .broadcast_as((batch, self.n_head, self.d_head, seq))?
555                .contiguous()?;
556            let raw_pos = q_f32.matmul(&r_emb_seq)?;
557
558            let mut dist_vec = Vec::with_capacity(seq * seq);
559            for i in 0..seq {
560                for j in 0..seq {
561                    dist_vec.push(i.saturating_sub(j) as u32);
562                }
563            }
564            let dist_flat = Tensor::from_vec(dist_vec.clone(), seq * seq, x.device())?;
565            let dist_idx = Tensor::from_vec(dist_vec, (1, 1, seq, seq), x.device())?
566                .broadcast_as((batch, self.n_head, seq, seq))?
567                .contiguous()?;
568            let pos_gathered = raw_pos.gather(&dist_idx, D::Minus1)?;
569
570            let r_bias_term = rel
571                .r_bias
572                .as_tensor()
573                .index_select(&dist_flat, 0)?
574                .reshape((seq, seq, self.n_head))?
575                .permute((2, 0, 1))?
576                .contiguous()?
577                .affine((self.d_model as f64).sqrt(), 0.0)?
578                .unsqueeze(0)?;
579            let pos_score = pos_gathered.broadcast_add(&r_bias_term)?;
580            let scale = 1.0 / (self.d_head as f64).sqrt();
581            (content_score + pos_score)?.affine(scale, 0.0)?
582        };
583
584        let masked = if let Some(mask) = causal_mask {
585            scores.broadcast_add(mask)?
586        } else {
587            scores
588        };
589
590        let prob = candle_nn::ops::softmax(&masked, D::Minus1)?;
591        let prob = apply_dropout(&prob, self.dropout, mode)?;
592        let prob = if prob.dtype() == v.dtype() {
593            prob
594        } else {
595            prob.to_dtype(v.dtype())?
596        };
597        let out = prob
598            .matmul(&v)?
599            .transpose(1, 2)?
600            .contiguous()?
601            .reshape((batch, seq, d_model))?;
602        linear_forward(&out, &self.proj, compute_dtype)
603    }
604}
605
606#[derive(Debug, Clone)]
607struct SwiGlu {
608    gate: Var,
609    up: Var,
610    down: Var,
611}
612
613impl SwiGlu {
614    fn new(
615        d_model: usize,
616        d_inner: usize,
617        residual_std: f32,
618        device: &Device,
619        mut rng: Option<&mut DeterministicRng>,
620    ) -> Result<Self, CodecError> {
621        let gate = init_normal_var((d_inner, d_model), 0.02, device, rng.as_deref_mut())?;
622        let up = init_normal_var((d_inner, d_model), 0.02, device, rng.as_deref_mut())?;
623        let down = init_normal_var((d_model, d_inner), residual_std, device, rng)?;
624        Ok(Self { gate, up, down })
625    }
626
627    fn forward(&self, x: &Tensor, compute_dtype: DType) -> Result<Tensor, CodecError> {
628        let gate = linear_forward(x, &self.gate, compute_dtype)?.silu()?;
629        let up = linear_forward(x, &self.up, compute_dtype)?;
630        linear_forward(&(gate * up)?, &self.down, compute_dtype)
631    }
632}
633
634#[derive(Debug, Clone)]
635struct Block {
636    norm_style: NormStyle,
637    dropout: f32,
638    attn_pre_norm: RmsNorm,
639    attn: Attention,
640    attn_post_norm: RmsNorm,
641    ffn_pre_norm: RmsNorm,
642    ffn: SwiGlu,
643    ffn_post_norm: RmsNorm,
644}
645
646impl Block {
647    fn new(
648        cfg: &ModelConfig,
649        residual_std: f32,
650        device: &Device,
651        mut rng: Option<&mut DeterministicRng>,
652    ) -> Result<Self, CodecError> {
653        let peri = cfg.norm_style == NormStyle::Peri;
654        Ok(Self {
655            norm_style: cfg.norm_style,
656            dropout: cfg.dropout,
657            attn_pre_norm: RmsNorm::new(cfg.d_model, peri, device)?,
658            attn: Attention::new(cfg, residual_std, device, rng.as_deref_mut())?,
659            attn_post_norm: RmsNorm::new(cfg.d_model, true, device)?,
660            ffn_pre_norm: RmsNorm::new(cfg.d_model, peri, device)?,
661            ffn: SwiGlu::new(cfg.d_model, cfg.d_inner, residual_std, device, rng)?,
662            ffn_post_norm: RmsNorm::new(cfg.d_model, true, device)?,
663        })
664    }
665
666    fn forward(
667        &self,
668        x: &Tensor,
669        rope_cos: Option<&Tensor>,
670        rope_sin: Option<&Tensor>,
671        causal_mask: Option<&Tensor>,
672        mode: &mut ForwardMode<'_>,
673        compute_dtype: DType,
674    ) -> Result<Tensor, CodecError> {
675        match self.norm_style {
676            NormStyle::Peri => {
677                let normed_attn_in = self.attn_pre_norm.forward(x)?;
678                let attn_out = self.attn.forward(
679                    &normed_attn_in,
680                    rope_cos,
681                    rope_sin,
682                    causal_mask,
683                    mode,
684                    compute_dtype,
685                )?;
686                let attn_out = self.attn_post_norm.forward(&attn_out)?;
687                let attn_out = apply_dropout(&attn_out, self.dropout, mode)?;
688                let x = (x + attn_out)?;
689
690                let normed_ffn_in = self.ffn_pre_norm.forward(&x)?;
691                let ffn_out = self.ffn.forward(&normed_ffn_in, compute_dtype)?;
692                let ffn_out = self.ffn_post_norm.forward(&ffn_out)?;
693                let ffn_out = apply_dropout(&ffn_out, self.dropout, mode)?;
694                Ok((x + ffn_out)?)
695            }
696            NormStyle::Post => {
697                let attn_out =
698                    self.attn
699                        .forward(x, rope_cos, rope_sin, causal_mask, mode, compute_dtype)?;
700                let attn_out = apply_dropout(&attn_out, self.dropout, mode)?;
701                let x = self.attn_post_norm.forward(&(x + attn_out)?)?;
702
703                let ffn_out = self.ffn.forward(&x, compute_dtype)?;
704                let ffn_out = apply_dropout(&ffn_out, self.dropout, mode)?;
705                self.ffn_post_norm.forward(&(x + ffn_out)?)
706            }
707        }
708    }
709}
710
711/// Neural Transformer language model matching `nn.py`.
712#[derive(Debug)]
713pub struct Transformer {
714    config: ModelConfig,
715    block_schedule: Vec<usize>,
716    tok_emb: Var,
717    emb_norm: RmsNorm,
718    blocks: Vec<Block>,
719    norm: RmsNorm,
720    head: Var,
721    rope_cos: Option<Tensor>,
722    rope_sin: Option<Tensor>,
723    causal_mask: Tensor,
724    named_vars: Vec<(String, Var)>,
725    state_vars: BTreeMap<String, Var>,
726}
727
728impl Transformer {
729    /// Initializes a new [`Transformer`] with deterministic weights on `device`.
730    ///
731    /// # Errors
732    ///
733    /// Returns [`CodecError`] on invalid model configuration or tensor allocation error.
734    pub fn new(
735        config: &ModelConfig,
736        device: &Device,
737        rng: &mut DeterministicRng,
738    ) -> Result<Self, CodecError> {
739        Self::build(config, device, Some(rng))
740    }
741
742    /// Constructs a [`Transformer`] and immediately loads a `QPI1` prior archive.
743    ///
744    /// For `loop-untying` priors (where every target block is copied from the archive),
745    /// parameter buffers are allocated zeroed instead of sampling discarded normal
746    /// initialization weights. For `depth-growth` priors (where unmapped layers retain
747    /// random initialization), full deterministic initialization is performed via `rng`.
748    ///
749    /// # Errors
750    ///
751    /// Returns [`CodecError`] if reading, validating, or loading the prior archive fails.
752    pub fn from_prior(
753        config: &ModelConfig,
754        archive_path: impl AsRef<Path>,
755        vocab_path: impl AsRef<Path>,
756        seed: u64,
757        device: &Device,
758        rng: &mut DeterministicRng,
759    ) -> Result<(Self, PriorLoadInfo), CodecError> {
760        let archive = read_archive(archive_path)?;
761        let is_loop_untying = archive
762            .metadata
763            .initialization
764            .as_ref()
765            .is_some_and(|init| init.kind == InitializationKind::LoopUntying);
766        let mut model = if is_loop_untying {
767            Self::build(config, device, None)?
768        } else {
769            Self::build(config, device, Some(rng))?
770        };
771        let info = model.apply_prior_archive(&archive, vocab_path, seed)?;
772        Ok((model, info))
773    }
774
775    fn build(
776        config: &ModelConfig,
777        device: &Device,
778        mut rng: Option<&mut DeterministicRng>,
779    ) -> Result<Self, CodecError> {
780        config.validate()?;
781        let block_schedule = config
782            .block_schedule
783            .clone()
784            .unwrap_or_else(|| (0..config.n_layer).collect());
785        let executed_depth = block_schedule.len();
786        let residual_std = 0.02f32 / ((2.0 * executed_depth as f64).sqrt() as f32);
787
788        let tok_emb = init_normal_var(
789            (config.vocab_size, config.d_model),
790            0.02,
791            device,
792            rng.as_deref_mut(),
793        )?;
794        let peri = config.norm_style == NormStyle::Peri;
795        let emb_norm = RmsNorm::new(config.d_model, peri, device)?;
796
797        let mut blocks = Vec::with_capacity(config.n_layer);
798        for _ in 0..config.n_layer {
799            blocks.push(Block::new(
800                config,
801                residual_std,
802                device,
803                rng.as_deref_mut(),
804            )?);
805        }
806
807        let norm = RmsNorm::new(config.d_model, peri, device)?;
808        let head = if config.tie_embeddings {
809            tok_emb.clone()
810        } else {
811            init_normal_var((config.vocab_size, config.d_model), 0.02, device, rng)?
812        };
813
814        let (rope_cos, rope_sin) = if config.position_type == PositionType::Rope {
815            let (c, s) = build_rope_cache(
816                config.ctx_len,
817                config.d_model / config.n_head,
818                config.rope_base,
819                device,
820            )?;
821            (Some(c), Some(s))
822        } else {
823            (None, None)
824        };
825
826        let mut named_vars = Vec::new();
827        let mut state_vars = BTreeMap::new();
828
829        let mut register = |name: String, var: Var, trainable: bool| {
830            state_vars.insert(name.clone(), var.clone());
831            if trainable {
832                named_vars.push((name, var));
833            }
834        };
835
836        register("tok_emb.weight".to_owned(), tok_emb.clone(), true);
837        if let Some(w) = &emb_norm.weight {
838            register("emb_norm.weight".to_owned(), w.clone(), true);
839        }
840        for (i, block) in blocks.iter().enumerate() {
841            let p = format!("blocks.{i}");
842            if let Some(w) = &block.attn_pre_norm.weight {
843                register(format!("{p}.attn_pre_norm.weight"), w.clone(), true);
844            }
845            register(format!("{p}.attn.qkv.weight"), block.attn.qkv.clone(), true);
846            if let Some(w) = &block.attn.q_norm.weight {
847                register(format!("{p}.attn.q_norm.weight"), w.clone(), true);
848            }
849            if let Some(w) = &block.attn.k_norm.weight {
850                register(format!("{p}.attn.k_norm.weight"), w.clone(), true);
851            }
852            if let Some(rel) = &block.attn.rel_pos {
853                register(format!("{p}.attn.r_emb"), rel.r_emb.clone(), true);
854                register(format!("{p}.attn.r_w_bias"), rel.r_w_bias.clone(), true);
855                register(format!("{p}.attn.r_bias"), rel.r_bias.clone(), true);
856            }
857            register(
858                format!("{p}.attn.proj.weight"),
859                block.attn.proj.clone(),
860                true,
861            );
862            if let Some(w) = &block.attn_post_norm.weight {
863                register(format!("{p}.attn_post_norm.weight"), w.clone(), true);
864            }
865            if let Some(w) = &block.ffn_pre_norm.weight {
866                register(format!("{p}.ffn_pre_norm.weight"), w.clone(), true);
867            }
868            register(format!("{p}.ffn.gate.weight"), block.ffn.gate.clone(), true);
869            register(format!("{p}.ffn.up.weight"), block.ffn.up.clone(), true);
870            register(format!("{p}.ffn.down.weight"), block.ffn.down.clone(), true);
871            if let Some(w) = &block.ffn_post_norm.weight {
872                register(format!("{p}.ffn_post_norm.weight"), w.clone(), true);
873            }
874        }
875        if let Some(w) = &norm.weight {
876            register("norm.weight".to_owned(), w.clone(), true);
877        }
878        register(
879            "head.weight".to_owned(),
880            head.clone(),
881            !config.tie_embeddings,
882        );
883
884        let causal_mask = build_causal_mask(config.ctx_len, device)?;
885
886        Ok(Self {
887            config: config.clone(),
888            block_schedule,
889            tok_emb,
890            emb_norm,
891            blocks,
892            norm,
893            head,
894            rope_cos,
895            rope_sin,
896            causal_mask,
897            named_vars,
898            state_vars,
899        })
900    }
901
902    /// Runs a forward pass over token IDs `idx` (`[batch, seq]`, `U32`), returning logits.
903    ///
904    /// # Errors
905    ///
906    /// Returns [`CodecError`] if tensor shapes, sequence lengths, or Candle operations fail.
907    pub fn forward(
908        &self,
909        idx: &Tensor,
910        output_slice: OutputSlice,
911        mode: &mut ForwardMode<'_>,
912        compute_dtype: DType,
913    ) -> Result<Tensor, CodecError> {
914        let (batch, seq) = idx.dims2()?;
915        if seq == 0 || seq > self.config.ctx_len {
916            return Err(CodecError::InvalidData(format!(
917                "sequence length {seq} must be in 1..={}",
918                self.config.ctx_len
919            )));
920        }
921
922        let (cos_slice, sin_slice) = match (&self.rope_cos, &self.rope_sin) {
923            (Some(c), Some(s)) => {
924                let cn = c.narrow(2, 0, seq)?;
925                let sn = s.narrow(2, 0, seq)?;
926                let cn = if cn.dtype() == compute_dtype {
927                    cn
928                } else {
929                    cn.to_dtype(compute_dtype)?
930                };
931                let sn = if sn.dtype() == compute_dtype {
932                    sn
933                } else {
934                    sn.to_dtype(compute_dtype)?
935                };
936                (Some(cn), Some(sn))
937            }
938            _ => (None, None),
939        };
940        let causal_mask = if seq > 1 {
941            Some(self.causal_mask.narrow(2, 0, seq)?.narrow(3, 0, seq)?)
942        } else {
943            None
944        };
945
946        let flat_idx = idx.flatten_all()?;
947        let mut x = self
948            .tok_emb
949            .as_tensor()
950            .index_select(&flat_idx, 0)?
951            .reshape((batch, seq, self.config.d_model))?;
952        if x.dtype() != compute_dtype {
953            x = x.to_dtype(compute_dtype)?;
954        }
955
956        x = self.emb_norm.forward(&x)?;
957        x = apply_dropout(&x, self.config.dropout, mode)?;
958
959        for &block_idx in &self.block_schedule {
960            x = self.blocks[block_idx].forward(
961                &x,
962                cos_slice.as_ref(),
963                sin_slice.as_ref(),
964                causal_mask.as_ref(),
965                mode,
966                compute_dtype,
967            )?;
968        }
969
970        x = self.norm.forward(&x)?;
971
972        let x = match output_slice {
973            OutputSlice::All => x,
974            OutputSlice::Last => x.narrow(1, seq - 1, 1)?,
975            OutputSlice::Range { start, len } => x.narrow(1, start, len)?,
976        };
977
978        linear_forward(&x, &self.head, compute_dtype)
979    }
980
981    /// Returns all unique trainable `(name, Var)` pairs in canonical order.
982    #[must_use]
983    pub fn named_vars(&self) -> &[(String, Var)] {
984        &self.named_vars
985    }
986
987    /// Returns `(total_params, non_embedding_params)`.
988    #[must_use]
989    pub fn param_counts(&self) -> (usize, usize) {
990        let total: usize = self.named_vars.iter().map(|(_, v)| v.elem_count()).sum();
991        let mut emb = self.tok_emb.elem_count();
992        if !self.config.tie_embeddings {
993            emb += self.head.elem_count();
994        }
995        (total, total.saturating_sub(emb))
996    }
997
998    /// Returns the expected parameter shape map matching PyTorch `state_dict()`.
999    #[must_use]
1000    pub fn state_shapes(&self) -> BTreeMap<String, Vec<usize>> {
1001        self.state_vars
1002            .iter()
1003            .map(|(name, var)| (name.clone(), var.dims().to_vec()))
1004            .collect()
1005    }
1006
1007    /// Computes a deterministic SHA-256 digest over all parameter names, shapes, dtypes, and `f32` bytes.
1008    ///
1009    /// # Errors
1010    ///
1011    /// Returns [`CodecError::Candle`] if reading parameter tensors from device fails.
1012    pub fn state_sha256(&self) -> Result<String, CodecError> {
1013        let mut hasher = Sha256::new();
1014        for (name, var) in &self.state_vars {
1015            let cpu_t = var
1016                .as_tensor()
1017                .to_device(&Device::Cpu)?
1018                .to_dtype(DType::F32)?;
1019            let dims = cpu_t.dims();
1020            let shape_str = match dims.len() {
1021                1 => format!("({},)", dims[0]),
1022                _ => {
1023                    let joined: Vec<String> = dims.iter().map(usize::to_string).collect();
1024                    format!("({})", joined.join(", "))
1025                }
1026            };
1027            hasher.update(name.as_bytes());
1028            hasher.update(b"\0");
1029            hasher.update(shape_str.as_bytes());
1030            hasher.update(b"\0");
1031            hasher.update(b"torch.float32\0");
1032
1033            let flat = cpu_t.flatten_all()?.to_vec1::<f32>()?;
1034            let mut buf = [0u8; 4096];
1035            let mut off = 0usize;
1036            for val in flat {
1037                buf[off..off + 4].copy_from_slice(&val.to_le_bytes());
1038                off += 4;
1039                if off == buf.len() {
1040                    hasher.update(buf);
1041                    off = 0;
1042                }
1043            }
1044            if off > 0 {
1045                hasher.update(&buf[..off]);
1046            }
1047        }
1048        Ok(hex::encode(hasher.finalize()))
1049    }
1050
1051    /// Loads and validates a `QPI1` depth-growth or loop-untying prior archive.
1052    ///
1053    /// # Errors
1054    ///
1055    /// Returns [`CodecError`] if reading, validating, or transferring tensors fails.
1056    pub fn load_prior(
1057        &mut self,
1058        archive_path: impl AsRef<Path>,
1059        vocab_path: impl AsRef<Path>,
1060        seed: u64,
1061    ) -> Result<PriorLoadInfo, CodecError> {
1062        let archive = read_archive(archive_path)?;
1063        self.apply_prior_archive(&archive, vocab_path, seed)
1064    }
1065
1066    fn apply_prior_archive(
1067        &mut self,
1068        archive: &PriorArchive,
1069        vocab_path: impl AsRef<Path>,
1070        seed: u64,
1071    ) -> Result<PriorLoadInfo, CodecError> {
1072        let expected_geometry = self.config.geometry();
1073        let expected_shapes = self.state_shapes();
1074        let plan = plan_depth_prior(
1075            archive,
1076            vocab_path,
1077            &expected_geometry,
1078            seed,
1079            &expected_shapes,
1080        )?;
1081        let device = self.tok_emb.device().clone();
1082
1083        for name in &plan.root_names {
1084            let src = &archive.tensors[name];
1085            let tensor = Tensor::from_slice(&src.data, Shape::from_dims(&src.shape), &device)?;
1086            self.state_vars[name].set(&tensor)?;
1087        }
1088
1089        for (&target_idx, &source_idx) in &plan.target_sources {
1090            for suffix in &plan.block_suffixes {
1091                let source_name = format!("blocks.{source_idx}.{suffix}");
1092                let target_name = format!("blocks.{target_idx}.{suffix}");
1093                let src = &archive.tensors[&source_name];
1094                let tensor = Tensor::from_slice(&src.data, Shape::from_dims(&src.shape), &device)?;
1095                self.state_vars[&target_name].set(&tensor)?;
1096            }
1097        }
1098
1099        for target_idx in &plan.zero_unmapped_targets {
1100            let proj_name = format!("blocks.{target_idx}.attn.proj.weight");
1101            let down_name = format!("blocks.{target_idx}.ffn.down.weight");
1102            for key in [&proj_name, &down_name] {
1103                let var = &self.state_vars[key];
1104                let zeros = Tensor::zeros(var.shape(), DType::F32, &device)?;
1105                var.set(&zeros)?;
1106            }
1107        }
1108
1109        let initial_state_sha256 = self.state_sha256()?;
1110        Ok(PriorLoadInfo {
1111            archive_bytes: plan.archive_bytes,
1112            archive_sha256: plan.archive_sha256,
1113            vocab_sha256: plan.vocab_sha256,
1114            initialization_kind: plan.initialization_kind,
1115            source_depth: plan.source_depth,
1116            block_mapping: plan.block_mapping,
1117            block_schedule: plan.block_schedule,
1118            initial_state_sha256,
1119            policy: plan.policy,
1120        })
1121    }
1122
1123    /// Exports all trainable parameters into `out` under `"model.<name>"` keys for safetensors checkpoints.
1124    ///
1125    /// # Errors
1126    ///
1127    /// Returns [`CodecError::Candle`] if converting tensors to contiguous CPU tensors fails.
1128    pub fn export_tensors(&self, out: &mut HashMap<String, Tensor>) -> Result<(), CodecError> {
1129        for (name, var) in &self.named_vars {
1130            let tensor = var.as_tensor().to_device(&Device::Cpu)?.contiguous()?;
1131            out.insert(format!("model.{name}"), tensor);
1132        }
1133        Ok(())
1134    }
1135
1136    /// Restores trainable parameters from checkpoint tensors loaded by `load_checkpoint`.
1137    ///
1138    /// # Errors
1139    ///
1140    /// Returns [`CodecError`] if any expected parameter is missing or has an incompatible shape.
1141    pub fn import_tensors(&mut self, tensors: &HashMap<String, Tensor>) -> Result<(), CodecError> {
1142        let device = self.tok_emb.device().clone();
1143        for (name, var) in &self.named_vars {
1144            let key = format!("model.{name}");
1145            let src = tensors.get(&key).ok_or_else(|| {
1146                CodecError::CheckpointMismatch(format!("checkpoint missing model parameter {key}"))
1147            })?;
1148            if src.dims() != var.dims() {
1149                return Err(CodecError::CheckpointMismatch(format!(
1150                    "checkpoint parameter {key} shape {:?} != expected {:?}",
1151                    src.dims(),
1152                    var.dims()
1153                )));
1154            }
1155            let aligned = src.to_device(&device)?.to_dtype(DType::F32)?.contiguous()?;
1156            var.set(&aligned)?;
1157        }
1158        Ok(())
1159    }
1160
1161    /// Computes global L2 parameter norm across all trainable parameters.
1162    ///
1163    /// # Errors
1164    ///
1165    /// Returns [`CodecError::Candle`] on tensor read failures.
1166    pub fn weight_norm(&self) -> Result<f64, CodecError> {
1167        let mut sum_sq = 0.0f64;
1168        for (_, var) in &self.named_vars {
1169            let sq = var.as_tensor().to_dtype(DType::F32)?.sqr()?.sum_all()?;
1170            sum_sq += f64::from(sq.to_scalar::<f32>()?);
1171        }
1172        Ok(sum_sq.sqrt())
1173    }
1174}
1175
1176#[cfg(test)]
1177mod tests {
1178    use super::*;
1179
1180    #[test]
1181    fn fused_cpu_rms_norm_matches_unfused_fwd_and_bwd() {
1182        for (rows, dim) in [(4usize, 16usize), (96usize, 32usize)] {
1183            let mut x_data = Vec::with_capacity(rows * dim);
1184            for i in 0..(rows * dim) {
1185                x_data.push(((i % 23) as f32 - 11.0) * 0.13);
1186            }
1187            let w_data: Vec<f32> = (0..dim).map(|j| 0.7 + (j % 7) as f32 * 0.11).collect();
1188            let dy_data: Vec<f32> = (0..(rows * dim))
1189                .map(|i| ((i * 5 % 19) as f32 - 9.0) * 0.05)
1190                .collect();
1191
1192            let dy = Tensor::from_slice(&dy_data, (rows, dim), &Device::Cpu).unwrap();
1193
1194            let x_ref = Var::from_slice(&x_data, (rows, dim), &Device::Cpu).unwrap();
1195            let w_ref = Var::from_slice(&w_data, dim, &Device::Cpu).unwrap();
1196            let norm_ref = RmsNorm {
1197                weight: Some(w_ref.clone()),
1198                eps: RMS_NORM_EPS,
1199            };
1200            let out_ref = norm_ref.forward_unfused(x_ref.as_tensor(), &w_ref).unwrap();
1201            let loss_ref = (&out_ref * &dy).unwrap().sum_all().unwrap();
1202            let grads_ref = loss_ref.backward().unwrap();
1203
1204            let x_opt = Var::from_slice(&x_data, (rows, dim), &Device::Cpu).unwrap();
1205            let w_opt = Var::from_slice(&w_data, dim, &Device::Cpu).unwrap();
1206            let norm_opt = RmsNorm {
1207                weight: Some(w_opt.clone()),
1208                eps: RMS_NORM_EPS,
1209            };
1210            let out_opt = norm_opt.forward(x_opt.as_tensor()).unwrap();
1211            let loss_opt = (&out_opt * &dy).unwrap().sum_all().unwrap();
1212            let grads_opt = loss_opt.backward().unwrap();
1213
1214            let out_r = out_ref.flatten_all().unwrap().to_vec1::<f32>().unwrap();
1215            let out_o = out_opt.flatten_all().unwrap().to_vec1::<f32>().unwrap();
1216            for (got, want) in out_o.iter().zip(out_r.iter()) {
1217                assert!((got - want).abs() < 1e-5, "fwd mismatch: {got} vs {want}");
1218            }
1219
1220            let gx_r = grads_ref
1221                .get(x_ref.as_tensor())
1222                .unwrap()
1223                .flatten_all()
1224                .unwrap()
1225                .to_vec1::<f32>()
1226                .unwrap();
1227            let gx_o = grads_opt
1228                .get(x_opt.as_tensor())
1229                .unwrap()
1230                .flatten_all()
1231                .unwrap()
1232                .to_vec1::<f32>()
1233                .unwrap();
1234            for (got, want) in gx_o.iter().zip(gx_r.iter()) {
1235                assert!((got - want).abs() < 1e-4, "gx mismatch: {got} vs {want}");
1236            }
1237
1238            let gw_r = grads_ref
1239                .get(w_ref.as_tensor())
1240                .unwrap()
1241                .to_vec1::<f32>()
1242                .unwrap();
1243            let gw_o = grads_opt
1244                .get(w_opt.as_tensor())
1245                .unwrap()
1246                .to_vec1::<f32>()
1247                .unwrap();
1248            for (got, want) in gw_o.iter().zip(gw_r.iter()) {
1249                assert!((got - want).abs() < 1e-4, "gw mismatch: {got} vs {want}");
1250            }
1251        }
1252    }
1253}