1use std::collections::{BTreeMap, HashMap};
2use std::path::Path;
3
4use candle_core::{CpuStorage, CustomOp2, DType, Device, Layout, Shape, Storage, Tensor, Var, D};
5use serde::{Deserialize, Serialize};
6use sha2::{Digest, Sha256};
7use tufazip_prior::{plan_depth_prior, read_archive, InitializationKind, PriorArchive};
8
9use crate::config::{ModelConfig, NormStyle, PositionType};
10use crate::error::CodecError;
11use crate::rng::DeterministicRng;
12
13const RMS_NORM_EPS: f64 = 1e-5;
14
15#[derive(Debug)]
17pub enum ForwardMode<'a> {
18 Eval,
20 Train(&'a mut DeterministicRng),
22}
23
24#[derive(Debug, Clone, Copy, PartialEq, Eq)]
26pub enum OutputSlice {
27 All,
29 Last,
31 Range {
33 start: usize,
35 len: usize,
37 },
38}
39
40#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
42pub struct PriorLoadInfo {
43 pub archive_bytes: u64,
45 pub archive_sha256: String,
47 pub vocab_sha256: String,
49 pub initialization_kind: InitializationKind,
51 pub source_depth: usize,
53 pub block_mapping: Vec<usize>,
55 pub block_schedule: Option<Vec<usize>>,
57 pub initial_state_sha256: String,
59 pub policy: Option<String>,
61}
62
63#[derive(Debug, Clone, Copy)]
64struct FusedCpuRmsNorm {
65 eps: f64,
66}
67
68impl FusedCpuRmsNorm {
69 fn compute_row_fwd(x_row: &[f32], w: &[f32], out_row: &mut [f32], eps: f64) {
70 let mut sq_acc = [0.0f32; 8];
71 let x_chunks = x_row.chunks_exact(8);
72 let x_rem = x_chunks.remainder();
73 for xc in x_chunks {
74 for i in 0..8 {
75 let x_v = xc[i];
76 sq_acc[i] += x_v * x_v;
77 }
78 }
79 let mut sum_sq = 0.0f64;
80 for &lane in &sq_acc {
81 sum_sq += f64::from(lane);
82 }
83 for &v in x_rem {
84 let fv = f64::from(v);
85 sum_sq += fv * fv;
86 }
87 let inv_rms = (1.0 / (sum_sq / (x_row.len() as f64) + eps).sqrt()) as f32;
88 for ((dst, &x_val), &w_val) in out_row.iter_mut().zip(x_row.iter()).zip(w.iter()) {
89 *dst = x_val * inv_rms * w_val;
90 }
91 }
92
93 fn compute_row_bwd(
94 x_row: &[f32],
95 dy_row: &[f32],
96 w: &[f32],
97 gx_row: &mut [f32],
98 gw_acc: &mut [f64],
99 eps: f64,
100 ) {
101 let dim = x_row.len();
102 let mut sq_acc = [0.0f32; 8];
103 let mut dot_acc = [0.0f32; 8];
104 let x_chunks = x_row.chunks_exact(8);
105 let dy_chunks = dy_row.chunks_exact(8);
106 let w_chunks = w.chunks_exact(8);
107 let x_rem = x_chunks.remainder();
108 let dy_rem = dy_chunks.remainder();
109 let w_rem = w_chunks.remainder();
110 for ((xc, dyc), wc) in x_chunks.zip(dy_chunks).zip(w_chunks) {
111 for i in 0..8 {
112 let x_v = xc[i];
113 sq_acc[i] += x_v * x_v;
114 dot_acc[i] += dyc[i] * wc[i] * x_v;
115 }
116 }
117 let mut sum_sq = 0.0f64;
118 let mut dot = 0.0f64;
119 for i in 0..8 {
120 sum_sq += f64::from(sq_acc[i]);
121 dot += f64::from(dot_acc[i]);
122 }
123 for ((&x_v, &dy_v), &w_v) in x_rem.iter().zip(dy_rem.iter()).zip(w_rem.iter()) {
124 let x_f = f64::from(x_v);
125 sum_sq += x_f * x_f;
126 dot += f64::from(dy_v) * f64::from(w_v) * x_f;
127 }
128 let inv_rms = 1.0 / (sum_sq / (dim as f64) + eps).sqrt();
129 let coeff = dot * inv_rms * inv_rms / (dim as f64);
130 let inv_rms_f32 = inv_rms as f32;
131 let coeff_f32 = coeff as f32;
132 for ((((gx_slot, &dy_val), &w_val), &x_val), gw_slot) in gx_row
133 .iter_mut()
134 .zip(dy_row.iter())
135 .zip(w.iter())
136 .zip(x_row.iter())
137 .zip(gw_acc.iter_mut())
138 {
139 *gx_slot = inv_rms_f32 * (dy_val * w_val - x_val * coeff_f32);
140 let dy_f = f64::from(dy_val);
141 let x_f = f64::from(x_val);
142 *gw_slot += dy_f * x_f * inv_rms;
143 }
144 }
145}
146
147impl CustomOp2 for FusedCpuRmsNorm {
148 fn name(&self) -> &'static str {
149 "fused-cpu-rms-norm"
150 }
151
152 fn cpu_fwd(
153 &self,
154 s1: &CpuStorage,
155 l1: &Layout,
156 s2: &CpuStorage,
157 l2: &Layout,
158 ) -> candle_core::Result<(CpuStorage, Shape)> {
159 let (x_start, x_end) = l1.contiguous_offsets().ok_or_else(|| {
160 candle_core::Error::Msg("fused-cpu-rms-norm requires contiguous x".to_owned())
161 })?;
162 let (w_start, w_end) = l2.contiguous_offsets().ok_or_else(|| {
163 candle_core::Error::Msg("fused-cpu-rms-norm requires contiguous weight".to_owned())
164 })?;
165 let x_all = match s1 {
166 CpuStorage::F32(v) => &v[x_start..x_end],
167 _ => {
168 return Err(candle_core::Error::Msg(
169 "fused-cpu-rms-norm expects F32 x".to_owned(),
170 ))
171 }
172 };
173 let w = match s2 {
174 CpuStorage::F32(v) => &v[w_start..w_end],
175 _ => {
176 return Err(candle_core::Error::Msg(
177 "fused-cpu-rms-norm expects F32 weight".to_owned(),
178 ))
179 }
180 };
181 let dim = w.len();
182 if dim == 0 || l1.dims().last() != Some(&dim) || x_all.len() % dim != 0 {
183 return Err(candle_core::Error::Msg(format!(
184 "fused-cpu-rms-norm shape mismatch: x={:?}, w={:?}",
185 l1.shape(),
186 l2.shape()
187 )));
188 }
189 let mut out = vec![0.0f32; x_all.len()];
190 let eps = self.eps;
191 for (out_row, x_row) in out.chunks_exact_mut(dim).zip(x_all.chunks_exact(dim)) {
192 Self::compute_row_fwd(x_row, w, out_row, eps);
193 }
194 Ok((CpuStorage::F32(out), l1.shape().clone()))
195 }
196
197 fn bwd(
198 &self,
199 arg1: &Tensor,
200 arg2: &Tensor,
201 _res: &Tensor,
202 grad_res: &Tensor,
203 ) -> candle_core::Result<(Option<Tensor>, Option<Tensor>)> {
204 let x_c = if arg1.is_contiguous() && arg1.dtype() == DType::F32 {
205 arg1.clone()
206 } else {
207 arg1.to_dtype(DType::F32)?.contiguous()?
208 };
209 let w_c = if arg2.is_contiguous() && arg2.dtype() == DType::F32 {
210 arg2.clone()
211 } else {
212 arg2.to_dtype(DType::F32)?.contiguous()?
213 };
214 let dy_c = if grad_res.is_contiguous() && grad_res.dtype() == DType::F32 {
215 grad_res.clone()
216 } else {
217 grad_res.to_dtype(DType::F32)?.contiguous()?
218 };
219
220 let (x_st, x_l) = x_c.storage_and_layout();
221 let (w_st, w_l) = w_c.storage_and_layout();
222 let (dy_st, dy_l) = dy_c.storage_and_layout();
223 let (x_s, x_e) = x_l
224 .contiguous_offsets()
225 .ok_or_else(|| candle_core::Error::Msg("non-contiguous x in RmsNorm bwd".to_owned()))?;
226 let (w_s, w_e) = w_l
227 .contiguous_offsets()
228 .ok_or_else(|| candle_core::Error::Msg("non-contiguous w in RmsNorm bwd".to_owned()))?;
229 let (dy_s, dy_e) = dy_l.contiguous_offsets().ok_or_else(|| {
230 candle_core::Error::Msg("non-contiguous dy in RmsNorm bwd".to_owned())
231 })?;
232
233 let (x_slice, w_slice, dy_slice) = match (&*x_st, &*w_st, &*dy_st) {
234 (
235 Storage::Cpu(CpuStorage::F32(xv)),
236 Storage::Cpu(CpuStorage::F32(wv)),
237 Storage::Cpu(CpuStorage::F32(dyv)),
238 ) => (&xv[x_s..x_e], &wv[w_s..w_e], &dyv[dy_s..dy_e]),
239 _ => {
240 return Err(candle_core::Error::Msg(
241 "fused-cpu-rms-norm bwd expects F32 CPU storage".to_owned(),
242 ))
243 }
244 };
245
246 let dim = w_slice.len();
247 if dim == 0 || x_slice.is_empty() {
248 return Ok((
249 Some(Tensor::zeros(arg1.shape(), DType::F32, arg1.device())?),
250 Some(Tensor::zeros(arg2.shape(), DType::F32, arg2.device())?),
251 ));
252 }
253
254 let mut gx = vec![0.0f32; x_slice.len()];
255 let mut gw_acc = vec![0.0f64; dim];
256 let eps = self.eps;
257
258 for ((gx_row, x_row), dy_row) in gx
259 .chunks_exact_mut(dim)
260 .zip(x_slice.chunks_exact(dim))
261 .zip(dy_slice.chunks_exact(dim))
262 {
263 Self::compute_row_bwd(x_row, dy_row, w_slice, gx_row, &mut gw_acc, eps);
264 }
265
266 let gw: Vec<f32> = gw_acc.into_iter().map(|v| v as f32).collect();
267 let gx_t = Tensor::from_vec(gx, arg1.shape().clone(), arg1.device())?;
268 let gw_t = Tensor::from_vec(gw, arg2.shape().clone(), arg2.device())?;
269 Ok((Some(gx_t), Some(gw_t)))
270 }
271}
272
273#[derive(Debug, Clone)]
274struct RmsNorm {
275 weight: Option<Var>,
276 eps: f64,
277}
278
279impl RmsNorm {
280 fn new(dim: usize, enabled: bool, device: &Device) -> Result<Self, CodecError> {
281 let weight = if enabled {
282 Some(Var::ones(dim, DType::F32, device)?)
283 } else {
284 None
285 };
286 Ok(Self {
287 weight,
288 eps: RMS_NORM_EPS,
289 })
290 }
291
292 fn forward_unfused(&self, x: &Tensor, weight: &Var) -> Result<Tensor, CodecError> {
293 let x_dtype = x.dtype();
294 let x_f32 = if x_dtype == DType::F32 {
295 x.clone()
296 } else {
297 x.to_dtype(DType::F32)?
298 };
299 let var = x_f32.sqr()?.mean_keepdim(D::Minus1)?;
300 let inv_std = (var + self.eps)?.sqrt()?.recip()?;
301 let normed = x_f32.broadcast_mul(&inv_std)?;
302 let normed_cast = if x_dtype == DType::F32 {
303 normed
304 } else {
305 normed.to_dtype(x_dtype)?
306 };
307 let w = if x_dtype == DType::F32 {
308 weight.as_tensor().clone()
309 } else {
310 weight.as_tensor().to_dtype(x_dtype)?
311 };
312 Ok(normed_cast.broadcast_mul(&w)?)
313 }
314
315 fn forward(&self, x: &Tensor) -> Result<Tensor, CodecError> {
316 let Some(weight) = &self.weight else {
317 return Ok(x.clone());
318 };
319 if x.device().is_cpu() && x.dtype() == DType::F32 && weight.dtype() == DType::F32 {
320 let x_c = if x.is_contiguous() {
321 x.clone()
322 } else {
323 x.contiguous()?
324 };
325 return Ok(x_c.apply_op2(weight.as_tensor(), FusedCpuRmsNorm { eps: self.eps })?);
326 }
327 self.forward_unfused(x, weight)
328 }
329}
330
331fn linear_forward(x: &Tensor, weight: &Var, compute_dtype: DType) -> Result<Tensor, CodecError> {
332 let w = if compute_dtype == DType::F32 {
333 weight.as_tensor().clone()
334 } else {
335 weight.as_tensor().to_dtype(compute_dtype)?
336 };
337 let out_dim = w.dim(0)?;
338 match *x.dims() {
339 [batch, seq, hidden] => {
340 let x_flat = x.contiguous()?.reshape((batch * seq, hidden))?;
341 let out = x_flat.matmul(&w.t()?)?;
342 Ok(out.reshape((batch, seq, out_dim))?)
343 }
344 _ => Ok(x.matmul(&w.t()?)?),
345 }
346}
347
348fn apply_dropout(
349 x: &Tensor,
350 drop_p: f32,
351 mode: &mut ForwardMode<'_>,
352) -> Result<Tensor, CodecError> {
353 if drop_p == 0.0 {
354 return Ok(x.clone());
355 }
356 match mode {
357 ForwardMode::Eval => Ok(x.clone()),
358 ForwardMode::Train(rng) => {
359 let mask = rng.dropout_mask(x.shape(), drop_p, x.dtype(), x.device())?;
360 Ok(x.mul(&mask)?)
361 }
362 }
363}
364
365fn build_rope_cache(
366 ctx_len: usize,
367 head_dim: usize,
368 base: f64,
369 device: &Device,
370) -> Result<(Tensor, Tensor), CodecError> {
371 if head_dim % 2 != 0 {
372 return Err(CodecError::InvalidConfig(format!(
373 "head_dim ({head_dim}) must be even for RoPE"
374 )));
375 }
376 let half = head_dim / 2;
377 let mut cos_data = vec![0.0f32; ctx_len * head_dim];
378 let mut sin_data = vec![0.0f32; ctx_len * head_dim];
379 let inv_freq: Vec<f64> = (0..half)
380 .map(|i| 1.0f64 / base.powf((2 * i) as f64 / head_dim as f64))
381 .collect();
382
383 for pos in 0..ctx_len {
384 let row = pos * head_dim;
385 for (i, &freq) in inv_freq.iter().enumerate() {
386 let angle = (pos as f64) * freq;
387 let c = angle.cos() as f32;
388 let s = angle.sin() as f32;
389 cos_data[row + i] = c;
390 cos_data[row + half + i] = c;
391 sin_data[row + i] = s;
392 sin_data[row + half + i] = s;
393 }
394 }
395
396 let cos = Tensor::from_vec(cos_data, (1, 1, ctx_len, head_dim), device)?;
397 let sin = Tensor::from_vec(sin_data, (1, 1, ctx_len, head_dim), device)?;
398 Ok((cos, sin))
399}
400
401fn apply_rope(x: &Tensor, cos: &Tensor, sin: &Tensor) -> Result<Tensor, CodecError> {
402 let head_dim = x.dim(D::Minus1)?;
403 let half = head_dim / 2;
404 let x1 = x.narrow(D::Minus1, 0, half)?;
405 let x2 = x.narrow(D::Minus1, half, half)?;
406 let rotated = Tensor::cat(&[&x2.neg()?, &x1], D::Minus1)?;
407 let term1 = x.broadcast_mul(cos)?;
408 let term2 = rotated.broadcast_mul(sin)?;
409 Ok((term1 + term2)?)
410}
411
412fn build_causal_mask(ctx_len: usize, device: &Device) -> Result<Tensor, CodecError> {
413 let mut data = vec![0.0f32; ctx_len * ctx_len];
414 for i in 0..ctx_len {
415 for j in (i + 1)..ctx_len {
416 data[i * ctx_len + j] = f32::NEG_INFINITY;
417 }
418 }
419 Ok(Tensor::from_vec(data, (1, 1, ctx_len, ctx_len), device)?)
420}
421
422#[derive(Debug, Clone)]
423struct RelativePosParams {
424 r_emb: Var,
425 r_w_bias: Var,
426 r_bias: Var,
427}
428
429fn init_normal_var(
430 shape: impl Into<Shape>,
431 std: f32,
432 device: &Device,
433 rng: Option<&mut DeterministicRng>,
434) -> Result<Var, CodecError> {
435 let shape = shape.into();
436 match rng {
437 Some(rng) => {
438 let data = rng.sample_normal_vec(shape.elem_count(), 0.0, std)?;
439 Ok(Var::from_vec(data, shape, device)?)
440 }
441 None => Ok(Var::zeros(shape, DType::F32, device)?),
442 }
443}
444
445#[derive(Debug, Clone)]
446struct Attention {
447 n_head: usize,
448 d_head: usize,
449 d_model: usize,
450 dropout: f32,
451 position_type: PositionType,
452 qkv: Var,
453 proj: Var,
454 q_norm: RmsNorm,
455 k_norm: RmsNorm,
456 rel_pos: Option<RelativePosParams>,
457}
458
459impl Attention {
460 fn new(
461 cfg: &ModelConfig,
462 residual_std: f32,
463 device: &Device,
464 mut rng: Option<&mut DeterministicRng>,
465 ) -> Result<Self, CodecError> {
466 let d_model = cfg.d_model;
467 let d_head = d_model / cfg.n_head;
468 let qkv = init_normal_var((3 * d_model, d_model), 0.02, device, rng.as_deref_mut())?;
469 let proj = init_normal_var((d_model, d_model), residual_std, device, rng.as_deref_mut())?;
470 let q_norm = RmsNorm::new(d_head, cfg.qk_norm, device)?;
471 let k_norm = RmsNorm::new(d_head, cfg.qk_norm, device)?;
472
473 let rel_pos = if cfg.position_type == PositionType::LearnedRelative {
474 let r_emb = init_normal_var(
475 (cfg.ctx_len, cfg.n_head, d_head),
476 0.02,
477 device,
478 rng.as_deref_mut(),
479 )?;
480 let r_w_bias = init_normal_var((cfg.n_head, d_head), 0.02, device, rng)?;
481 let r_bias = Var::zeros((cfg.ctx_len, cfg.n_head), DType::F32, device)?;
482 Some(RelativePosParams {
483 r_emb,
484 r_w_bias,
485 r_bias,
486 })
487 } else {
488 None
489 };
490
491 Ok(Self {
492 n_head: cfg.n_head,
493 d_head,
494 d_model,
495 dropout: cfg.dropout,
496 position_type: cfg.position_type,
497 qkv,
498 proj,
499 q_norm,
500 k_norm,
501 rel_pos,
502 })
503 }
504
505 fn forward(
506 &self,
507 x: &Tensor,
508 rope_cos: Option<&Tensor>,
509 rope_sin: Option<&Tensor>,
510 causal_mask: Option<&Tensor>,
511 mode: &mut ForwardMode<'_>,
512 compute_dtype: DType,
513 ) -> Result<Tensor, CodecError> {
514 let (batch, seq, d_model) = x.dims3()?;
515 let qkv = linear_forward(x, &self.qkv, compute_dtype)?;
516 let qkv_split = qkv
517 .reshape((batch, seq, 3, self.n_head, self.d_head))?
518 .permute((2, 0, 3, 1, 4))?
519 .contiguous()?;
520 let q = qkv_split.narrow(0, 0, 1)?.squeeze(0)?;
521 let k = qkv_split.narrow(0, 1, 1)?.squeeze(0)?;
522 let v = qkv_split.narrow(0, 2, 1)?.squeeze(0)?;
523
524 let q = self.q_norm.forward(&q)?;
525 let k = self.k_norm.forward(&k)?;
526
527 let scores = if self.position_type == PositionType::Rope {
528 let cos = rope_cos
529 .ok_or_else(|| CodecError::InvalidConfig("RoPE cosine cache missing".to_owned()))?;
530 let sin = rope_sin
531 .ok_or_else(|| CodecError::InvalidConfig("RoPE sine cache missing".to_owned()))?;
532 let q = apply_rope(&q, cos, sin)?;
533 let k = apply_rope(&k, cos, sin)?;
534 let q_f32 = q.to_dtype(DType::F32)?;
535 let k_f32 = k.to_dtype(DType::F32)?.transpose(2, 3)?.contiguous()?;
536 let scale = 1.0 / (self.d_head as f64).sqrt();
537 q_f32.matmul(&k_f32)?.affine(scale, 0.0)?
538 } else {
539 let rel = self.rel_pos.as_ref().ok_or_else(|| {
540 CodecError::InvalidConfig("learned_relative parameters missing".to_owned())
541 })?;
542 let q_f32 = q.to_dtype(DType::F32)?;
543 let k_f32 = k.to_dtype(DType::F32)?.transpose(2, 3)?.contiguous()?;
544 let r_w_bias = rel.r_w_bias.as_tensor().unsqueeze(0)?.unsqueeze(2)?;
545 let q_biased = q_f32.broadcast_add(&r_w_bias)?;
546 let content_score = q_biased.matmul(&k_f32)?;
547
548 let r_emb_seq = rel
549 .r_emb
550 .as_tensor()
551 .narrow(0, 0, seq)?
552 .permute((1, 2, 0))?
553 .contiguous()?
554 .broadcast_as((batch, self.n_head, self.d_head, seq))?
555 .contiguous()?;
556 let raw_pos = q_f32.matmul(&r_emb_seq)?;
557
558 let mut dist_vec = Vec::with_capacity(seq * seq);
559 for i in 0..seq {
560 for j in 0..seq {
561 dist_vec.push(i.saturating_sub(j) as u32);
562 }
563 }
564 let dist_flat = Tensor::from_vec(dist_vec.clone(), seq * seq, x.device())?;
565 let dist_idx = Tensor::from_vec(dist_vec, (1, 1, seq, seq), x.device())?
566 .broadcast_as((batch, self.n_head, seq, seq))?
567 .contiguous()?;
568 let pos_gathered = raw_pos.gather(&dist_idx, D::Minus1)?;
569
570 let r_bias_term = rel
571 .r_bias
572 .as_tensor()
573 .index_select(&dist_flat, 0)?
574 .reshape((seq, seq, self.n_head))?
575 .permute((2, 0, 1))?
576 .contiguous()?
577 .affine((self.d_model as f64).sqrt(), 0.0)?
578 .unsqueeze(0)?;
579 let pos_score = pos_gathered.broadcast_add(&r_bias_term)?;
580 let scale = 1.0 / (self.d_head as f64).sqrt();
581 (content_score + pos_score)?.affine(scale, 0.0)?
582 };
583
584 let masked = if let Some(mask) = causal_mask {
585 scores.broadcast_add(mask)?
586 } else {
587 scores
588 };
589
590 let prob = candle_nn::ops::softmax(&masked, D::Minus1)?;
591 let prob = apply_dropout(&prob, self.dropout, mode)?;
592 let prob = if prob.dtype() == v.dtype() {
593 prob
594 } else {
595 prob.to_dtype(v.dtype())?
596 };
597 let out = prob
598 .matmul(&v)?
599 .transpose(1, 2)?
600 .contiguous()?
601 .reshape((batch, seq, d_model))?;
602 linear_forward(&out, &self.proj, compute_dtype)
603 }
604}
605
606#[derive(Debug, Clone)]
607struct SwiGlu {
608 gate: Var,
609 up: Var,
610 down: Var,
611}
612
613impl SwiGlu {
614 fn new(
615 d_model: usize,
616 d_inner: usize,
617 residual_std: f32,
618 device: &Device,
619 mut rng: Option<&mut DeterministicRng>,
620 ) -> Result<Self, CodecError> {
621 let gate = init_normal_var((d_inner, d_model), 0.02, device, rng.as_deref_mut())?;
622 let up = init_normal_var((d_inner, d_model), 0.02, device, rng.as_deref_mut())?;
623 let down = init_normal_var((d_model, d_inner), residual_std, device, rng)?;
624 Ok(Self { gate, up, down })
625 }
626
627 fn forward(&self, x: &Tensor, compute_dtype: DType) -> Result<Tensor, CodecError> {
628 let gate = linear_forward(x, &self.gate, compute_dtype)?.silu()?;
629 let up = linear_forward(x, &self.up, compute_dtype)?;
630 linear_forward(&(gate * up)?, &self.down, compute_dtype)
631 }
632}
633
634#[derive(Debug, Clone)]
635struct Block {
636 norm_style: NormStyle,
637 dropout: f32,
638 attn_pre_norm: RmsNorm,
639 attn: Attention,
640 attn_post_norm: RmsNorm,
641 ffn_pre_norm: RmsNorm,
642 ffn: SwiGlu,
643 ffn_post_norm: RmsNorm,
644}
645
646impl Block {
647 fn new(
648 cfg: &ModelConfig,
649 residual_std: f32,
650 device: &Device,
651 mut rng: Option<&mut DeterministicRng>,
652 ) -> Result<Self, CodecError> {
653 let peri = cfg.norm_style == NormStyle::Peri;
654 Ok(Self {
655 norm_style: cfg.norm_style,
656 dropout: cfg.dropout,
657 attn_pre_norm: RmsNorm::new(cfg.d_model, peri, device)?,
658 attn: Attention::new(cfg, residual_std, device, rng.as_deref_mut())?,
659 attn_post_norm: RmsNorm::new(cfg.d_model, true, device)?,
660 ffn_pre_norm: RmsNorm::new(cfg.d_model, peri, device)?,
661 ffn: SwiGlu::new(cfg.d_model, cfg.d_inner, residual_std, device, rng)?,
662 ffn_post_norm: RmsNorm::new(cfg.d_model, true, device)?,
663 })
664 }
665
666 fn forward(
667 &self,
668 x: &Tensor,
669 rope_cos: Option<&Tensor>,
670 rope_sin: Option<&Tensor>,
671 causal_mask: Option<&Tensor>,
672 mode: &mut ForwardMode<'_>,
673 compute_dtype: DType,
674 ) -> Result<Tensor, CodecError> {
675 match self.norm_style {
676 NormStyle::Peri => {
677 let normed_attn_in = self.attn_pre_norm.forward(x)?;
678 let attn_out = self.attn.forward(
679 &normed_attn_in,
680 rope_cos,
681 rope_sin,
682 causal_mask,
683 mode,
684 compute_dtype,
685 )?;
686 let attn_out = self.attn_post_norm.forward(&attn_out)?;
687 let attn_out = apply_dropout(&attn_out, self.dropout, mode)?;
688 let x = (x + attn_out)?;
689
690 let normed_ffn_in = self.ffn_pre_norm.forward(&x)?;
691 let ffn_out = self.ffn.forward(&normed_ffn_in, compute_dtype)?;
692 let ffn_out = self.ffn_post_norm.forward(&ffn_out)?;
693 let ffn_out = apply_dropout(&ffn_out, self.dropout, mode)?;
694 Ok((x + ffn_out)?)
695 }
696 NormStyle::Post => {
697 let attn_out =
698 self.attn
699 .forward(x, rope_cos, rope_sin, causal_mask, mode, compute_dtype)?;
700 let attn_out = apply_dropout(&attn_out, self.dropout, mode)?;
701 let x = self.attn_post_norm.forward(&(x + attn_out)?)?;
702
703 let ffn_out = self.ffn.forward(&x, compute_dtype)?;
704 let ffn_out = apply_dropout(&ffn_out, self.dropout, mode)?;
705 self.ffn_post_norm.forward(&(x + ffn_out)?)
706 }
707 }
708 }
709}
710
711#[derive(Debug)]
713pub struct Transformer {
714 config: ModelConfig,
715 block_schedule: Vec<usize>,
716 tok_emb: Var,
717 emb_norm: RmsNorm,
718 blocks: Vec<Block>,
719 norm: RmsNorm,
720 head: Var,
721 rope_cos: Option<Tensor>,
722 rope_sin: Option<Tensor>,
723 causal_mask: Tensor,
724 named_vars: Vec<(String, Var)>,
725 state_vars: BTreeMap<String, Var>,
726}
727
728impl Transformer {
729 pub fn new(
735 config: &ModelConfig,
736 device: &Device,
737 rng: &mut DeterministicRng,
738 ) -> Result<Self, CodecError> {
739 Self::build(config, device, Some(rng))
740 }
741
742 pub fn from_prior(
753 config: &ModelConfig,
754 archive_path: impl AsRef<Path>,
755 vocab_path: impl AsRef<Path>,
756 seed: u64,
757 device: &Device,
758 rng: &mut DeterministicRng,
759 ) -> Result<(Self, PriorLoadInfo), CodecError> {
760 let archive = read_archive(archive_path)?;
761 let is_loop_untying = archive
762 .metadata
763 .initialization
764 .as_ref()
765 .is_some_and(|init| init.kind == InitializationKind::LoopUntying);
766 let mut model = if is_loop_untying {
767 Self::build(config, device, None)?
768 } else {
769 Self::build(config, device, Some(rng))?
770 };
771 let info = model.apply_prior_archive(&archive, vocab_path, seed)?;
772 Ok((model, info))
773 }
774
775 fn build(
776 config: &ModelConfig,
777 device: &Device,
778 mut rng: Option<&mut DeterministicRng>,
779 ) -> Result<Self, CodecError> {
780 config.validate()?;
781 let block_schedule = config
782 .block_schedule
783 .clone()
784 .unwrap_or_else(|| (0..config.n_layer).collect());
785 let executed_depth = block_schedule.len();
786 let residual_std = 0.02f32 / ((2.0 * executed_depth as f64).sqrt() as f32);
787
788 let tok_emb = init_normal_var(
789 (config.vocab_size, config.d_model),
790 0.02,
791 device,
792 rng.as_deref_mut(),
793 )?;
794 let peri = config.norm_style == NormStyle::Peri;
795 let emb_norm = RmsNorm::new(config.d_model, peri, device)?;
796
797 let mut blocks = Vec::with_capacity(config.n_layer);
798 for _ in 0..config.n_layer {
799 blocks.push(Block::new(
800 config,
801 residual_std,
802 device,
803 rng.as_deref_mut(),
804 )?);
805 }
806
807 let norm = RmsNorm::new(config.d_model, peri, device)?;
808 let head = if config.tie_embeddings {
809 tok_emb.clone()
810 } else {
811 init_normal_var((config.vocab_size, config.d_model), 0.02, device, rng)?
812 };
813
814 let (rope_cos, rope_sin) = if config.position_type == PositionType::Rope {
815 let (c, s) = build_rope_cache(
816 config.ctx_len,
817 config.d_model / config.n_head,
818 config.rope_base,
819 device,
820 )?;
821 (Some(c), Some(s))
822 } else {
823 (None, None)
824 };
825
826 let mut named_vars = Vec::new();
827 let mut state_vars = BTreeMap::new();
828
829 let mut register = |name: String, var: Var, trainable: bool| {
830 state_vars.insert(name.clone(), var.clone());
831 if trainable {
832 named_vars.push((name, var));
833 }
834 };
835
836 register("tok_emb.weight".to_owned(), tok_emb.clone(), true);
837 if let Some(w) = &emb_norm.weight {
838 register("emb_norm.weight".to_owned(), w.clone(), true);
839 }
840 for (i, block) in blocks.iter().enumerate() {
841 let p = format!("blocks.{i}");
842 if let Some(w) = &block.attn_pre_norm.weight {
843 register(format!("{p}.attn_pre_norm.weight"), w.clone(), true);
844 }
845 register(format!("{p}.attn.qkv.weight"), block.attn.qkv.clone(), true);
846 if let Some(w) = &block.attn.q_norm.weight {
847 register(format!("{p}.attn.q_norm.weight"), w.clone(), true);
848 }
849 if let Some(w) = &block.attn.k_norm.weight {
850 register(format!("{p}.attn.k_norm.weight"), w.clone(), true);
851 }
852 if let Some(rel) = &block.attn.rel_pos {
853 register(format!("{p}.attn.r_emb"), rel.r_emb.clone(), true);
854 register(format!("{p}.attn.r_w_bias"), rel.r_w_bias.clone(), true);
855 register(format!("{p}.attn.r_bias"), rel.r_bias.clone(), true);
856 }
857 register(
858 format!("{p}.attn.proj.weight"),
859 block.attn.proj.clone(),
860 true,
861 );
862 if let Some(w) = &block.attn_post_norm.weight {
863 register(format!("{p}.attn_post_norm.weight"), w.clone(), true);
864 }
865 if let Some(w) = &block.ffn_pre_norm.weight {
866 register(format!("{p}.ffn_pre_norm.weight"), w.clone(), true);
867 }
868 register(format!("{p}.ffn.gate.weight"), block.ffn.gate.clone(), true);
869 register(format!("{p}.ffn.up.weight"), block.ffn.up.clone(), true);
870 register(format!("{p}.ffn.down.weight"), block.ffn.down.clone(), true);
871 if let Some(w) = &block.ffn_post_norm.weight {
872 register(format!("{p}.ffn_post_norm.weight"), w.clone(), true);
873 }
874 }
875 if let Some(w) = &norm.weight {
876 register("norm.weight".to_owned(), w.clone(), true);
877 }
878 register(
879 "head.weight".to_owned(),
880 head.clone(),
881 !config.tie_embeddings,
882 );
883
884 let causal_mask = build_causal_mask(config.ctx_len, device)?;
885
886 Ok(Self {
887 config: config.clone(),
888 block_schedule,
889 tok_emb,
890 emb_norm,
891 blocks,
892 norm,
893 head,
894 rope_cos,
895 rope_sin,
896 causal_mask,
897 named_vars,
898 state_vars,
899 })
900 }
901
902 pub fn forward(
908 &self,
909 idx: &Tensor,
910 output_slice: OutputSlice,
911 mode: &mut ForwardMode<'_>,
912 compute_dtype: DType,
913 ) -> Result<Tensor, CodecError> {
914 let (batch, seq) = idx.dims2()?;
915 if seq == 0 || seq > self.config.ctx_len {
916 return Err(CodecError::InvalidData(format!(
917 "sequence length {seq} must be in 1..={}",
918 self.config.ctx_len
919 )));
920 }
921
922 let (cos_slice, sin_slice) = match (&self.rope_cos, &self.rope_sin) {
923 (Some(c), Some(s)) => {
924 let cn = c.narrow(2, 0, seq)?;
925 let sn = s.narrow(2, 0, seq)?;
926 let cn = if cn.dtype() == compute_dtype {
927 cn
928 } else {
929 cn.to_dtype(compute_dtype)?
930 };
931 let sn = if sn.dtype() == compute_dtype {
932 sn
933 } else {
934 sn.to_dtype(compute_dtype)?
935 };
936 (Some(cn), Some(sn))
937 }
938 _ => (None, None),
939 };
940 let causal_mask = if seq > 1 {
941 Some(self.causal_mask.narrow(2, 0, seq)?.narrow(3, 0, seq)?)
942 } else {
943 None
944 };
945
946 let flat_idx = idx.flatten_all()?;
947 let mut x = self
948 .tok_emb
949 .as_tensor()
950 .index_select(&flat_idx, 0)?
951 .reshape((batch, seq, self.config.d_model))?;
952 if x.dtype() != compute_dtype {
953 x = x.to_dtype(compute_dtype)?;
954 }
955
956 x = self.emb_norm.forward(&x)?;
957 x = apply_dropout(&x, self.config.dropout, mode)?;
958
959 for &block_idx in &self.block_schedule {
960 x = self.blocks[block_idx].forward(
961 &x,
962 cos_slice.as_ref(),
963 sin_slice.as_ref(),
964 causal_mask.as_ref(),
965 mode,
966 compute_dtype,
967 )?;
968 }
969
970 x = self.norm.forward(&x)?;
971
972 let x = match output_slice {
973 OutputSlice::All => x,
974 OutputSlice::Last => x.narrow(1, seq - 1, 1)?,
975 OutputSlice::Range { start, len } => x.narrow(1, start, len)?,
976 };
977
978 linear_forward(&x, &self.head, compute_dtype)
979 }
980
981 #[must_use]
983 pub fn named_vars(&self) -> &[(String, Var)] {
984 &self.named_vars
985 }
986
987 #[must_use]
989 pub fn param_counts(&self) -> (usize, usize) {
990 let total: usize = self.named_vars.iter().map(|(_, v)| v.elem_count()).sum();
991 let mut emb = self.tok_emb.elem_count();
992 if !self.config.tie_embeddings {
993 emb += self.head.elem_count();
994 }
995 (total, total.saturating_sub(emb))
996 }
997
998 #[must_use]
1000 pub fn state_shapes(&self) -> BTreeMap<String, Vec<usize>> {
1001 self.state_vars
1002 .iter()
1003 .map(|(name, var)| (name.clone(), var.dims().to_vec()))
1004 .collect()
1005 }
1006
1007 pub fn state_sha256(&self) -> Result<String, CodecError> {
1013 let mut hasher = Sha256::new();
1014 for (name, var) in &self.state_vars {
1015 let cpu_t = var
1016 .as_tensor()
1017 .to_device(&Device::Cpu)?
1018 .to_dtype(DType::F32)?;
1019 let dims = cpu_t.dims();
1020 let shape_str = match dims.len() {
1021 1 => format!("({},)", dims[0]),
1022 _ => {
1023 let joined: Vec<String> = dims.iter().map(usize::to_string).collect();
1024 format!("({})", joined.join(", "))
1025 }
1026 };
1027 hasher.update(name.as_bytes());
1028 hasher.update(b"\0");
1029 hasher.update(shape_str.as_bytes());
1030 hasher.update(b"\0");
1031 hasher.update(b"torch.float32\0");
1032
1033 let flat = cpu_t.flatten_all()?.to_vec1::<f32>()?;
1034 let mut buf = [0u8; 4096];
1035 let mut off = 0usize;
1036 for val in flat {
1037 buf[off..off + 4].copy_from_slice(&val.to_le_bytes());
1038 off += 4;
1039 if off == buf.len() {
1040 hasher.update(buf);
1041 off = 0;
1042 }
1043 }
1044 if off > 0 {
1045 hasher.update(&buf[..off]);
1046 }
1047 }
1048 Ok(hex::encode(hasher.finalize()))
1049 }
1050
1051 pub fn load_prior(
1057 &mut self,
1058 archive_path: impl AsRef<Path>,
1059 vocab_path: impl AsRef<Path>,
1060 seed: u64,
1061 ) -> Result<PriorLoadInfo, CodecError> {
1062 let archive = read_archive(archive_path)?;
1063 self.apply_prior_archive(&archive, vocab_path, seed)
1064 }
1065
1066 fn apply_prior_archive(
1067 &mut self,
1068 archive: &PriorArchive,
1069 vocab_path: impl AsRef<Path>,
1070 seed: u64,
1071 ) -> Result<PriorLoadInfo, CodecError> {
1072 let expected_geometry = self.config.geometry();
1073 let expected_shapes = self.state_shapes();
1074 let plan = plan_depth_prior(
1075 archive,
1076 vocab_path,
1077 &expected_geometry,
1078 seed,
1079 &expected_shapes,
1080 )?;
1081 let device = self.tok_emb.device().clone();
1082
1083 for name in &plan.root_names {
1084 let src = &archive.tensors[name];
1085 let tensor = Tensor::from_slice(&src.data, Shape::from_dims(&src.shape), &device)?;
1086 self.state_vars[name].set(&tensor)?;
1087 }
1088
1089 for (&target_idx, &source_idx) in &plan.target_sources {
1090 for suffix in &plan.block_suffixes {
1091 let source_name = format!("blocks.{source_idx}.{suffix}");
1092 let target_name = format!("blocks.{target_idx}.{suffix}");
1093 let src = &archive.tensors[&source_name];
1094 let tensor = Tensor::from_slice(&src.data, Shape::from_dims(&src.shape), &device)?;
1095 self.state_vars[&target_name].set(&tensor)?;
1096 }
1097 }
1098
1099 for target_idx in &plan.zero_unmapped_targets {
1100 let proj_name = format!("blocks.{target_idx}.attn.proj.weight");
1101 let down_name = format!("blocks.{target_idx}.ffn.down.weight");
1102 for key in [&proj_name, &down_name] {
1103 let var = &self.state_vars[key];
1104 let zeros = Tensor::zeros(var.shape(), DType::F32, &device)?;
1105 var.set(&zeros)?;
1106 }
1107 }
1108
1109 let initial_state_sha256 = self.state_sha256()?;
1110 Ok(PriorLoadInfo {
1111 archive_bytes: plan.archive_bytes,
1112 archive_sha256: plan.archive_sha256,
1113 vocab_sha256: plan.vocab_sha256,
1114 initialization_kind: plan.initialization_kind,
1115 source_depth: plan.source_depth,
1116 block_mapping: plan.block_mapping,
1117 block_schedule: plan.block_schedule,
1118 initial_state_sha256,
1119 policy: plan.policy,
1120 })
1121 }
1122
1123 pub fn export_tensors(&self, out: &mut HashMap<String, Tensor>) -> Result<(), CodecError> {
1129 for (name, var) in &self.named_vars {
1130 let tensor = var.as_tensor().to_device(&Device::Cpu)?.contiguous()?;
1131 out.insert(format!("model.{name}"), tensor);
1132 }
1133 Ok(())
1134 }
1135
1136 pub fn import_tensors(&mut self, tensors: &HashMap<String, Tensor>) -> Result<(), CodecError> {
1142 let device = self.tok_emb.device().clone();
1143 for (name, var) in &self.named_vars {
1144 let key = format!("model.{name}");
1145 let src = tensors.get(&key).ok_or_else(|| {
1146 CodecError::CheckpointMismatch(format!("checkpoint missing model parameter {key}"))
1147 })?;
1148 if src.dims() != var.dims() {
1149 return Err(CodecError::CheckpointMismatch(format!(
1150 "checkpoint parameter {key} shape {:?} != expected {:?}",
1151 src.dims(),
1152 var.dims()
1153 )));
1154 }
1155 let aligned = src.to_device(&device)?.to_dtype(DType::F32)?.contiguous()?;
1156 var.set(&aligned)?;
1157 }
1158 Ok(())
1159 }
1160
1161 pub fn weight_norm(&self) -> Result<f64, CodecError> {
1167 let mut sum_sq = 0.0f64;
1168 for (_, var) in &self.named_vars {
1169 let sq = var.as_tensor().to_dtype(DType::F32)?.sqr()?.sum_all()?;
1170 sum_sq += f64::from(sq.to_scalar::<f32>()?);
1171 }
1172 Ok(sum_sq.sqrt())
1173 }
1174}
1175
1176#[cfg(test)]
1177mod tests {
1178 use super::*;
1179
1180 #[test]
1181 fn fused_cpu_rms_norm_matches_unfused_fwd_and_bwd() {
1182 for (rows, dim) in [(4usize, 16usize), (96usize, 32usize)] {
1183 let mut x_data = Vec::with_capacity(rows * dim);
1184 for i in 0..(rows * dim) {
1185 x_data.push(((i % 23) as f32 - 11.0) * 0.13);
1186 }
1187 let w_data: Vec<f32> = (0..dim).map(|j| 0.7 + (j % 7) as f32 * 0.11).collect();
1188 let dy_data: Vec<f32> = (0..(rows * dim))
1189 .map(|i| ((i * 5 % 19) as f32 - 9.0) * 0.05)
1190 .collect();
1191
1192 let dy = Tensor::from_slice(&dy_data, (rows, dim), &Device::Cpu).unwrap();
1193
1194 let x_ref = Var::from_slice(&x_data, (rows, dim), &Device::Cpu).unwrap();
1195 let w_ref = Var::from_slice(&w_data, dim, &Device::Cpu).unwrap();
1196 let norm_ref = RmsNorm {
1197 weight: Some(w_ref.clone()),
1198 eps: RMS_NORM_EPS,
1199 };
1200 let out_ref = norm_ref.forward_unfused(x_ref.as_tensor(), &w_ref).unwrap();
1201 let loss_ref = (&out_ref * &dy).unwrap().sum_all().unwrap();
1202 let grads_ref = loss_ref.backward().unwrap();
1203
1204 let x_opt = Var::from_slice(&x_data, (rows, dim), &Device::Cpu).unwrap();
1205 let w_opt = Var::from_slice(&w_data, dim, &Device::Cpu).unwrap();
1206 let norm_opt = RmsNorm {
1207 weight: Some(w_opt.clone()),
1208 eps: RMS_NORM_EPS,
1209 };
1210 let out_opt = norm_opt.forward(x_opt.as_tensor()).unwrap();
1211 let loss_opt = (&out_opt * &dy).unwrap().sum_all().unwrap();
1212 let grads_opt = loss_opt.backward().unwrap();
1213
1214 let out_r = out_ref.flatten_all().unwrap().to_vec1::<f32>().unwrap();
1215 let out_o = out_opt.flatten_all().unwrap().to_vec1::<f32>().unwrap();
1216 for (got, want) in out_o.iter().zip(out_r.iter()) {
1217 assert!((got - want).abs() < 1e-5, "fwd mismatch: {got} vs {want}");
1218 }
1219
1220 let gx_r = grads_ref
1221 .get(x_ref.as_tensor())
1222 .unwrap()
1223 .flatten_all()
1224 .unwrap()
1225 .to_vec1::<f32>()
1226 .unwrap();
1227 let gx_o = grads_opt
1228 .get(x_opt.as_tensor())
1229 .unwrap()
1230 .flatten_all()
1231 .unwrap()
1232 .to_vec1::<f32>()
1233 .unwrap();
1234 for (got, want) in gx_o.iter().zip(gx_r.iter()) {
1235 assert!((got - want).abs() < 1e-4, "gx mismatch: {got} vs {want}");
1236 }
1237
1238 let gw_r = grads_ref
1239 .get(w_ref.as_tensor())
1240 .unwrap()
1241 .to_vec1::<f32>()
1242 .unwrap();
1243 let gw_o = grads_opt
1244 .get(w_opt.as_tensor())
1245 .unwrap()
1246 .to_vec1::<f32>()
1247 .unwrap();
1248 for (got, want) in gw_o.iter().zip(gw_r.iter()) {
1249 assert!((got - want).abs() < 1e-4, "gw mismatch: {got} vs {want}");
1250 }
1251 }
1252 }
1253}