Skip to main content

littlebit_models/
layer.rs

1//! Differentiable neural network building blocks for quantized and dense LLMs.
2
3use std::collections::HashMap;
4
5use candle_core::{DType, Device, Tensor, Var, D};
6use rand_chacha::ChaCha8Rng;
7use rand_distr::{Distribution, Normal};
8
9use littlebit_core::{
10    BiasMode, InferenceMode, LayerShape, LittleBitConfig, LittleBitLinear, PackedLittleBitLinear,
11    Param, TrainMode, WeightState,
12};
13
14use crate::error::{ModelError, Result};
15
16/// Bundle of parameters specifying how to load a [`LinearLayer`] from a state dict.
17#[derive(Debug, Clone, Copy)]
18pub struct LinearLoadSpec<'a> {
19    /// Optional LittleBit quantization config when loading quantized checkpoints.
20    pub littlebit: Option<&'a LittleBitConfig>,
21    /// Per-layer rank multiplier (`1.0` default, `kv_factor` for `k_proj`/`v_proj`).
22    pub ratio_factor: f64,
23    /// Whether unpacked `U`/`V` weights are already binarized.
24    pub weight_state: WeightState,
25    /// Whether to keep `U`/`V` packed at inference (`Packed`) or unpacked floats (`Unpacked`).
26    pub inference_mode: InferenceMode,
27    /// Target dtype for float parameters.
28    pub dtype: DType,
29    /// Target compute device.
30    pub device: &'a Device,
31}
32
33/// Unified linear projection supporting dense weights, differentiable LittleBit QAT layers,
34/// and compact 1-bit packed inference layers.
35#[derive(Debug, Clone)]
36pub enum LinearLayer {
37    /// Standard dense linear transformation `x @ W^T + b`.
38    Dense {
39        /// Weight tensor `(out_features, in_features)`.
40        weight: Param,
41        /// Optional bias tensor `(out_features,)`.
42        bias: Option<Param>,
43    },
44    /// Differentiable or unpacked LittleBit factorized linear layer.
45    LittleBit(Box<LittleBitLinear>),
46    /// Memory-compact 1-bit packed LittleBit inference layer.
47    Packed(Box<PackedLittleBitLinear>),
48}
49
50impl LinearLayer {
51    /// Construct a random dense linear layer with normal initialization (`std = 0.02`).
52    ///
53    /// # Errors
54    /// Returns an error if tensor creation fails.
55    pub fn new_random(
56        shape: LayerShape,
57        dtype: DType,
58        device: &Device,
59        rng: &mut ChaCha8Rng,
60    ) -> Result<Self> {
61        let dist = Normal::new(0.0f32, 0.02f32).map_err(|err| {
62            ModelError::InvalidArgument(format!("failed to construct normal dist: {err}"))
63        })?;
64        let n = shape.out_features * shape.in_features;
65        let data: Vec<f32> = (0..n).map(|_| dist.sample(rng)).collect();
66        let w = Tensor::from_vec(data, (shape.out_features, shape.in_features), device)?
67            .to_dtype(dtype)?;
68        let bias = if shape.bias == BiasMode::WithBias {
69            Some(Param::Frozen(Tensor::zeros(
70                shape.out_features,
71                dtype,
72                device,
73            )?))
74        } else {
75            None
76        };
77        Ok(Self::Dense {
78            weight: Param::Frozen(w),
79            bias,
80        })
81    }
82
83    /// Load a linear layer from a state dictionary, detecting packed LittleBit, unpacked
84    /// LittleBit, or dense weights automatically.
85    ///
86    /// # Errors
87    /// Returns [`ModelError::MissingWeight`] if expected checkpoint keys are missing.
88    pub fn from_tensors(
89        prefix: &str,
90        tensors: &HashMap<String, Tensor>,
91        shape: LayerShape,
92        spec: LinearLoadSpec<'_>,
93    ) -> Result<Self> {
94        let packed_key = format!("{prefix}.U_packed");
95        let unpacked_key = format!("{prefix}.U");
96
97        if let Some(lb_cfg) = spec.littlebit {
98            let effective_cfg = lb_cfg.effective_for_module(prefix)?;
99            if spec.inference_mode == InferenceMode::Packed && tensors.contains_key(&packed_key) {
100                let packed = PackedLittleBitLinear::from_packed_tensors(
101                    prefix,
102                    tensors,
103                    shape,
104                    &effective_cfg,
105                    spec.ratio_factor,
106                )?;
107                return Ok(Self::Packed(Box::new(packed)));
108            }
109            if tensors.contains_key(&unpacked_key) {
110                let lb = LittleBitLinear::from_state_dict(
111                    prefix,
112                    tensors,
113                    shape,
114                    &effective_cfg,
115                    spec.ratio_factor,
116                    spec.weight_state,
117                    spec.dtype,
118                    spec.device,
119                )?;
120                return Ok(Self::LittleBit(Box::new(lb)));
121            }
122        }
123
124        let w_key = format!("{prefix}.weight");
125        let w = tensors
126            .get(&w_key)
127            .ok_or_else(|| ModelError::MissingWeight(w_key.clone()))?;
128        let w = move_cast(w, spec.dtype, spec.device)?;
129
130        let bias = if shape.bias == BiasMode::WithBias {
131            let b_key = format!("{prefix}.bias");
132            match tensors.get(&b_key) {
133                Some(b) => Some(Param::Frozen(move_cast(b, spec.dtype, spec.device)?)),
134                None => None,
135            }
136        } else {
137            None
138        };
139
140        Ok(Self::Dense {
141            weight: Param::Frozen(w),
142            bias,
143        })
144    }
145
146    /// Input feature dimension.
147    #[must_use]
148    pub fn in_features(&self) -> usize {
149        match self {
150            Self::Dense { weight, .. } => weight.as_tensor().dims().get(1).copied().unwrap_or(0),
151            Self::LittleBit(lb) => lb.in_features(),
152            Self::Packed(p) => p.budget().in_features,
153        }
154    }
155
156    /// Output feature dimension.
157    #[must_use]
158    pub fn out_features(&self) -> usize {
159        match self {
160            Self::Dense { weight, .. } => weight.as_tensor().dims().first().copied().unwrap_or(0),
161            Self::LittleBit(lb) => lb.out_features(),
162            Self::Packed(p) => p.budget().out_features,
163        }
164    }
165
166    /// Return whether this layer currently carries a bias parameter.
167    #[must_use]
168    pub fn bias_mode(&self) -> BiasMode {
169        match self {
170            Self::Dense { bias, .. } => BiasMode::from(bias.is_some()),
171            Self::LittleBit(lb) => BiasMode::from(lb.bias().is_some()),
172            Self::Packed(_) => BiasMode::NoBias,
173        }
174    }
175
176    /// Execute linear forward pass on `[..., in_features]`.
177    ///
178    /// # Errors
179    /// Returns an error if shapes mismatch or tensor math fails.
180    pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
181        match self {
182            Self::Dense { weight, bias } => {
183                let dims = x.dims();
184                let in_f = self.in_features();
185                let out_f = self.out_features();
186                if dims.is_empty() || dims[dims.len() - 1] != in_f {
187                    return Err(ModelError::Core(
188                        littlebit_core::LittleBitError::InvalidShape {
189                            expected: format!("[..., {in_f}]"),
190                            actual: format!("{dims:?}"),
191                        },
192                    ));
193                }
194                let mut out_dims = dims[..dims.len() - 1].to_vec();
195                let batch_rows: usize = out_dims.iter().product::<usize>().max(1);
196                out_dims.push(out_f);
197
198                let wt = weight.as_tensor();
199                let wt = if wt.dtype() == x.dtype() {
200                    wt.clone()
201                } else {
202                    wt.to_dtype(x.dtype())?
203                };
204                let wt_t = wt.t()?.contiguous()?;
205                let mut y = x.reshape((batch_rows, in_f))?.matmul(&wt_t)?;
206                if let Some(b) = bias {
207                    let bt = b.as_tensor();
208                    let bt = if bt.dtype() == y.dtype() {
209                        bt.clone()
210                    } else {
211                        bt.to_dtype(y.dtype())?
212                    };
213                    y = y.broadcast_add(&bt)?;
214                }
215                Ok(y.reshape(out_dims)?)
216            }
217            Self::LittleBit(lb) => Ok(lb.forward(x)?),
218            Self::Packed(p) => Ok(p.forward(x)?),
219        }
220    }
221
222    /// Convert a dense layer into [`LinearLayer::LittleBit`] via SVD (and optional Joint-ITQ).
223    ///
224    /// # Errors
225    /// Returns an error if called on an already-packed layer or if factorization fails.
226    pub fn convert_to_littlebit(
227        &mut self,
228        config: &LittleBitConfig,
229        ratio_factor: f64,
230        train_mode: TrainMode,
231        seed: u64,
232    ) -> Result<()> {
233        match self {
234            Self::Dense { weight, bias } => {
235                let b_ref = bias.as_ref().map(Param::as_tensor);
236                let lb = LittleBitLinear::from_dense_weight(
237                    weight.as_tensor(),
238                    b_ref,
239                    config,
240                    ratio_factor,
241                    train_mode,
242                    seed,
243                )?;
244                *self = Self::LittleBit(Box::new(lb));
245                Ok(())
246            }
247            Self::LittleBit(_) => Ok(()),
248            Self::Packed(_) => Err(ModelError::InvalidArgument(
249                "cannot re-factorize an already packed linear layer".to_string(),
250            )),
251        }
252    }
253
254    /// Convert a [`LinearLayer::LittleBit`] layer to compact 1-bit [`LinearLayer::Packed`].
255    ///
256    /// # Errors
257    /// Returns an error if packing fails.
258    pub fn convert_to_packed(&mut self) -> Result<()> {
259        if let Self::LittleBit(lb) = self {
260            *self = Self::Packed(Box::new(lb.to_packed()?));
261        }
262        Ok(())
263    }
264
265    /// Export layer weights in packed LittleBit or dense format under `prefix`.
266    ///
267    /// # Errors
268    /// Returns an error if serialization conversion fails.
269    pub fn pack_state_dict(&self, prefix: &str) -> Result<HashMap<String, Tensor>> {
270        match self {
271            Self::Dense { weight, bias } => {
272                let mut map = HashMap::new();
273                map.insert(format!("{prefix}.weight"), weight.as_tensor().detach());
274                if let Some(b) = bias {
275                    map.insert(format!("{prefix}.bias"), b.as_tensor().detach());
276                }
277                Ok(map)
278            }
279            Self::LittleBit(lb) => Ok(lb.pack_state_dict(prefix)?),
280            Self::Packed(p) => Ok(p.pack_state_dict(prefix, &Device::Cpu)?),
281        }
282    }
283
284    /// Collect trainable [`Var`] parameters inside this layer.
285    #[must_use]
286    pub fn trainable_vars(&self) -> Vec<Var> {
287        match self {
288            Self::Dense { weight, bias } => {
289                let mut out = Vec::new();
290                if let Some(v) = weight.as_var() {
291                    out.push(v.clone());
292                }
293                if let Some(b) = bias {
294                    if let Some(v) = b.as_var() {
295                        out.push(v.clone());
296                    }
297                }
298                out
299            }
300            Self::LittleBit(lb) => lb.trainable_vars(),
301            Self::Packed(_) => Vec::new(),
302        }
303    }
304
305    /// Convert all dense weights into trainable [`Var`]s (no-op for `Packed`).
306    ///
307    /// # Errors
308    /// Returns an error if converting tensors to `Var` fails.
309    pub fn make_trainable(&mut self) -> Result<()> {
310        if let Self::Dense { weight, bias } = self {
311            *weight = Param::Trainable(Var::from_tensor(weight.as_tensor())?);
312            if let Some(b) = bias {
313                *b = Param::Trainable(Var::from_tensor(b.as_tensor())?);
314            }
315        }
316        Ok(())
317    }
318
319    /// Freeze all trainable parameters in place.
320    pub fn freeze(&mut self) {
321        match self {
322            Self::Dense { weight, bias } => {
323                weight.freeze();
324                if let Some(b) = bias {
325                    b.freeze();
326                }
327            }
328            Self::LittleBit(lb) => lb.freeze(),
329            Self::Packed(_) => {}
330        }
331    }
332
333    /// Total scalar parameter elements stored in this layer.
334    #[must_use]
335    pub fn param_count(&self) -> usize {
336        match self {
337            Self::Dense { weight, bias } => {
338                weight.elem_count() + bias.as_ref().map_or(0, Param::elem_count)
339            }
340            Self::LittleBit(lb) => lb.param_count(),
341            Self::Packed(p) => {
342                let b = p.budget();
343                let mult = if b.residual == littlebit_core::ResidualMode::Dual {
344                    2
345                } else {
346                    1
347                };
348                mult * (b.out_features * b.split_dim
349                    + b.split_dim * b.in_features
350                    + b.out_features
351                    + b.in_features
352                    + 2 * b.split_dim)
353            }
354        }
355    }
356}
357
358/// RMSNorm variant (standard vs. Gemma `+1.0` offset).
359#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
360pub enum NormKind {
361    /// Standard RMSNorm scaling by `weight`.
362    Standard,
363    /// Gemma RMSNorm scaling by `(1.0 + weight)`.
364    PlusOne,
365}
366
367/// Differentiable RMSNorm layer using `candle_nn::ops::rms_norm_slow` so QAT backprop works.
368#[derive(Debug, Clone)]
369pub struct RmsNorm {
370    /// Scale weight `(dim,)`.
371    pub weight: Param,
372    /// Numerical stability epsilon.
373    pub eps: f32,
374    /// Standard vs. `+1.0` weight offset.
375    pub kind: NormKind,
376}
377
378impl RmsNorm {
379    /// Load RMSNorm weight `{prefix}.weight`.
380    ///
381    /// # Errors
382    /// Returns [`ModelError::MissingWeight`] if absent.
383    pub fn from_tensors(
384        prefix: &str,
385        tensors: &HashMap<String, Tensor>,
386        eps: f64,
387        kind: NormKind,
388        dtype: DType,
389        device: &Device,
390    ) -> Result<Self> {
391        let key = format!("{prefix}.weight");
392        let w = tensors
393            .get(&key)
394            .ok_or_else(|| ModelError::MissingWeight(key.clone()))?;
395        Ok(Self {
396            weight: Param::Frozen(move_cast(w, dtype, device)?),
397            eps: eps as f32,
398            kind,
399        })
400    }
401
402    /// Construct unit-initialized RMSNorm (zero-initialized for [`NormKind::PlusOne`]).
403    ///
404    /// # Errors
405    /// Returns an error if tensor creation fails.
406    pub fn new_init(
407        dim: usize,
408        eps: f64,
409        kind: NormKind,
410        dtype: DType,
411        device: &Device,
412    ) -> Result<Self> {
413        let weight = match kind {
414            NormKind::Standard => Tensor::ones(dim, dtype, device)?,
415            NormKind::PlusOne => Tensor::zeros(dim, dtype, device)?,
416        };
417        Ok(Self {
418            weight: Param::Frozen(weight),
419            eps: eps as f32,
420            kind,
421        })
422    }
423
424    /// Forward pass using differentiable `rms_norm_slow`.
425    ///
426    /// # Errors
427    /// Returns an error if norm computation fails.
428    pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
429        let w = self.weight.as_tensor();
430        let w = if w.dtype() == x.dtype() {
431            w.clone()
432        } else {
433            w.to_dtype(x.dtype())?
434        };
435        let alpha = match self.kind {
436            NormKind::Standard => w,
437            NormKind::PlusOne => (w + 1.0)?,
438        };
439        Ok(candle_nn::ops::rms_norm_slow(x, &alpha, self.eps)?)
440    }
441
442    /// Collect trainable `Var` if unfrozen.
443    #[must_use]
444    pub fn trainable_vars(&self) -> Vec<Var> {
445        self.weight.as_var().into_iter().cloned().collect()
446    }
447
448    /// Enable gradients on this norm weight.
449    ///
450    /// # Errors
451    /// Returns an error if creating `Var` fails.
452    pub fn make_trainable(&mut self) -> Result<()> {
453        if let Param::Frozen(t) = &self.weight {
454            self.weight = Param::Trainable(Var::from_tensor(t)?);
455        }
456        Ok(())
457    }
458
459    /// Freeze norm weight.
460    pub fn freeze(&mut self) {
461        self.weight.freeze();
462    }
463
464    /// Export `{prefix}.weight`.
465    pub fn pack_state_dict(&self, prefix: &str, out: &mut HashMap<String, Tensor>) {
466        out.insert(format!("{prefix}.weight"), self.weight.as_tensor().detach());
467    }
468}
469
470/// Weightless RMSNorm used for Gemma 4 value normalization (`v_norm`) and router input norm.
471#[derive(Debug, Clone, Copy)]
472pub struct RmsNormNoScale {
473    /// Numerical stability epsilon.
474    pub eps: f32,
475}
476
477impl RmsNormNoScale {
478    /// Construct a weightless RMSNorm with epsilon `eps`.
479    #[must_use]
480    pub fn new(eps: f64) -> Self {
481        Self { eps: eps as f32 }
482    }
483
484    /// Normalize along the last dimension in F32 (`x * (mean(x^2) + eps)^-0.5`), casting back to `x.dtype()`.
485    ///
486    /// # Errors
487    /// Returns an error if tensor operations fail.
488    pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
489        let orig_dtype = x.dtype();
490        let x_f32 = if orig_dtype == DType::F32 {
491            x.clone()
492        } else {
493            x.to_dtype(DType::F32)?
494        };
495        let mean_sq = x_f32.sqr()?.mean_keepdim(D::Minus1)?;
496        let inv_rms = (mean_sq + f64::from(self.eps))?.powf(-0.5)?;
497        let out = x_f32.broadcast_mul(&inv_rms)?;
498        if orig_dtype == DType::F32 {
499            Ok(out)
500        } else {
501            Ok(out.to_dtype(orig_dtype)?)
502        }
503    }
504}
505
506/// Differentiable LayerNorm using `candle_nn::ops::layer_norm_slow` (for OPT).
507#[derive(Debug, Clone)]
508pub struct LayerNorm {
509    /// Optional affine weight `(dim,)`.
510    pub weight: Option<Param>,
511    /// Optional affine bias `(dim,)`.
512    pub bias: Option<Param>,
513    /// Hidden size.
514    pub dim: usize,
515    /// Stability epsilon.
516    pub eps: f32,
517}
518
519impl LayerNorm {
520    /// Load LayerNorm `{prefix}.weight` and `{prefix}.bias` when `affine` is true.
521    ///
522    /// # Errors
523    /// Returns [`ModelError::MissingWeight`] if expected weights are absent.
524    pub fn from_tensors(
525        prefix: &str,
526        tensors: &HashMap<String, Tensor>,
527        dim: usize,
528        eps: f64,
529        affine: bool,
530        dtype: DType,
531        device: &Device,
532    ) -> Result<Self> {
533        if !affine {
534            return Ok(Self {
535                weight: None,
536                bias: None,
537                dim,
538                eps: eps as f32,
539            });
540        }
541        let w_key = format!("{prefix}.weight");
542        let b_key = format!("{prefix}.bias");
543        let w = tensors
544            .get(&w_key)
545            .ok_or_else(|| ModelError::MissingWeight(w_key.clone()))?;
546        let b = tensors
547            .get(&b_key)
548            .ok_or_else(|| ModelError::MissingWeight(b_key.clone()))?;
549        Ok(Self {
550            weight: Some(Param::Frozen(move_cast(w, dtype, device)?)),
551            bias: Some(Param::Frozen(move_cast(b, dtype, device)?)),
552            dim,
553            eps: eps as f32,
554        })
555    }
556
557    /// Construct standard unit-weight, zero-bias LayerNorm.
558    ///
559    /// # Errors
560    /// Returns an error if tensor creation fails.
561    pub fn new_init(
562        dim: usize,
563        eps: f64,
564        affine: bool,
565        dtype: DType,
566        device: &Device,
567    ) -> Result<Self> {
568        let (weight, bias) = if affine {
569            (
570                Some(Param::Frozen(Tensor::ones(dim, dtype, device)?)),
571                Some(Param::Frozen(Tensor::zeros(dim, dtype, device)?)),
572            )
573        } else {
574            (None, None)
575        };
576        Ok(Self {
577            weight,
578            bias,
579            dim,
580            eps: eps as f32,
581        })
582    }
583
584    /// Forward pass using differentiable `layer_norm_slow`.
585    ///
586    /// # Errors
587    /// Returns an error if normalization fails.
588    pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
589        let alpha = match &self.weight {
590            Some(w) => {
591                let wt = w.as_tensor();
592                if wt.dtype() == x.dtype() {
593                    wt.clone()
594                } else {
595                    wt.to_dtype(x.dtype())?
596                }
597            }
598            None => Tensor::ones(self.dim, x.dtype(), x.device())?,
599        };
600        let beta = match &self.bias {
601            Some(b) => {
602                let bt = b.as_tensor();
603                if bt.dtype() == x.dtype() {
604                    bt.clone()
605                } else {
606                    bt.to_dtype(x.dtype())?
607                }
608            }
609            None => Tensor::zeros(self.dim, x.dtype(), x.device())?,
610        };
611        Ok(candle_nn::ops::layer_norm_slow(x, &alpha, &beta, self.eps)?)
612    }
613
614    /// Collect trainable variables.
615    #[must_use]
616    pub fn trainable_vars(&self) -> Vec<Var> {
617        let mut vars = Vec::new();
618        if let Some(w) = &self.weight {
619            if let Some(v) = w.as_var() {
620                vars.push(v.clone());
621            }
622        }
623        if let Some(b) = &self.bias {
624            if let Some(v) = b.as_var() {
625                vars.push(v.clone());
626            }
627        }
628        vars
629    }
630
631    /// Enable gradients on affine parameters.
632    ///
633    /// # Errors
634    /// Returns an error if creating `Var` fails.
635    pub fn make_trainable(&mut self) -> Result<()> {
636        if let Some(Param::Frozen(w)) = &self.weight {
637            self.weight = Some(Param::Trainable(Var::from_tensor(w)?));
638        }
639        if let Some(Param::Frozen(b)) = &self.bias {
640            self.bias = Some(Param::Trainable(Var::from_tensor(b)?));
641        }
642        Ok(())
643    }
644
645    /// Freeze affine parameters.
646    pub fn freeze(&mut self) {
647        if let Some(w) = &mut self.weight {
648            w.freeze();
649        }
650        if let Some(b) = &mut self.bias {
651            b.freeze();
652        }
653    }
654
655    /// Export `{prefix}.weight` and `{prefix}.bias`.
656    pub fn pack_state_dict(&self, prefix: &str, out: &mut HashMap<String, Tensor>) {
657        if let Some(w) = &self.weight {
658            out.insert(format!("{prefix}.weight"), w.as_tensor().detach());
659        }
660        if let Some(b) = &self.bias {
661            out.insert(format!("{prefix}.bias"), b.as_tensor().detach());
662        }
663    }
664
665    /// Scalar parameter count.
666    #[must_use]
667    pub fn param_count(&self) -> usize {
668        self.weight.as_ref().map_or(0, Param::elem_count)
669            + self.bias.as_ref().map_or(0, Param::elem_count)
670    }
671}
672
673/// Token or positional embedding table `(vocab_size, embed_dim)`.
674#[derive(Debug, Clone)]
675pub struct Embedding {
676    /// Weight table `(num_embeddings, embed_dim)`.
677    pub weight: Param,
678    /// Optional multiplicative embedding scale (e.g. `sqrt(hidden_size)` for Gemma).
679    pub scale: Option<f64>,
680}
681
682impl Embedding {
683    /// Load `{prefix}.weight`.
684    ///
685    /// # Errors
686    /// Returns [`ModelError::MissingWeight`] if absent.
687    pub fn from_tensors(
688        prefix: &str,
689        tensors: &HashMap<String, Tensor>,
690        scale: Option<f64>,
691        dtype: DType,
692        device: &Device,
693    ) -> Result<Self> {
694        let key = format!("{prefix}.weight");
695        let w = tensors
696            .get(&key)
697            .ok_or_else(|| ModelError::MissingWeight(key.clone()))?;
698        Ok(Self {
699            weight: Param::Frozen(move_cast(w, dtype, device)?),
700            scale,
701        })
702    }
703
704    /// Random initialization `(num_embeddings, embed_dim)` with `std = 0.02`.
705    ///
706    /// # Errors
707    /// Returns an error if tensor creation fails.
708    pub fn new_random(
709        num_embeddings: usize,
710        embed_dim: usize,
711        scale: Option<f64>,
712        dtype: DType,
713        device: &Device,
714        rng: &mut ChaCha8Rng,
715    ) -> Result<Self> {
716        let dist = Normal::new(0.0f32, 0.02f32).map_err(|err| {
717            ModelError::InvalidArgument(format!("failed to construct normal dist: {err}"))
718        })?;
719        let data: Vec<f32> = (0..num_embeddings * embed_dim)
720            .map(|_| dist.sample(rng))
721            .collect();
722        let w = Tensor::from_vec(data, (num_embeddings, embed_dim), device)?.to_dtype(dtype)?;
723        Ok(Self {
724            weight: Param::Frozen(w),
725            scale,
726        })
727    }
728
729    /// Embed token indices `[..., seq_len]` into `[..., seq_len, embed_dim]`.
730    ///
731    /// # Errors
732    /// Returns an error if indexing or reshaping fails.
733    pub fn forward(&self, ids: &Tensor) -> Result<Tensor> {
734        let mut out_dims = ids.dims().to_vec();
735        let embed_dim = self.weight.as_tensor().dims().get(1).copied().unwrap_or(0);
736        out_dims.push(embed_dim);
737
738        let flat_ids = ids.flatten_all()?.to_dtype(DType::U32)?;
739        let selected = self
740            .weight
741            .as_tensor()
742            .index_select(&flat_ids, 0)?
743            .reshape(out_dims)?;
744        match self.scale {
745            Some(s) => Ok((selected * s)?),
746            None => Ok(selected),
747        }
748    }
749
750    /// Project hidden states `[..., embed_dim]` through tied embedding weights -> `[..., num_embeddings]`.
751    ///
752    /// # Errors
753    /// Returns an error if matrix multiplication fails.
754    pub fn forward_lm_head(&self, hidden: &Tensor) -> Result<Tensor> {
755        let dims = hidden.dims();
756        let embed_dim = self.weight.as_tensor().dims().get(1).copied().unwrap_or(0);
757        let vocab_size = self.weight.as_tensor().dims().first().copied().unwrap_or(0);
758        let mut out_dims = dims[..dims.len() - 1].to_vec();
759        let batch_rows: usize = out_dims.iter().product::<usize>().max(1);
760        out_dims.push(vocab_size);
761
762        let w = self.weight.as_tensor();
763        let w = if w.dtype() == hidden.dtype() {
764            w.clone()
765        } else {
766            w.to_dtype(hidden.dtype())?
767        };
768        let wt = w.t()?.contiguous()?;
769        let out = hidden.reshape((batch_rows, embed_dim))?.matmul(&wt)?;
770        Ok(out.reshape(out_dims)?)
771    }
772
773    /// Freeze embedding weights.
774    pub fn freeze(&mut self) {
775        self.weight.freeze();
776    }
777
778    /// Collect trainable `Var` if unfrozen.
779    #[must_use]
780    pub fn trainable_vars(&self) -> Vec<Var> {
781        self.weight.as_var().into_iter().cloned().collect()
782    }
783
784    /// Enable gradients on embedding weights.
785    ///
786    /// # Errors
787    /// Returns an error if creating `Var` fails.
788    pub fn make_trainable(&mut self) -> Result<()> {
789        if let Param::Frozen(w) = &self.weight {
790            self.weight = Param::Trainable(Var::from_tensor(w)?);
791        }
792        Ok(())
793    }
794
795    /// Export `{prefix}.weight`.
796    pub fn pack_state_dict(&self, prefix: &str, out: &mut HashMap<String, Tensor>) {
797        out.insert(format!("{prefix}.weight"), self.weight.as_tensor().detach());
798    }
799}
800
801/// Rotary Position Embedding frequency parameterization mode.
802#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
803pub enum RopeMode {
804    /// Standard RoPE denominator `2 * rope_angles` (equal to `head_dim` when `partial_rotary_factor == 1.0`).
805    Default,
806    /// Gemma 4 proportional RoPE (p-RoPE): always uses full `head_dim` as exponent denominator
807    /// even when `partial_rotary_factor < 1.0`, zero-padding non-rotated NOPE pairs.
808    Proportional,
809}
810
811/// Precomputed Rotary Position Embedding (`cos`, `sin`) supporting differentiable `rope_slow`
812/// as well as partial and proportional RoPE (`p-RoPE`).
813#[derive(Debug, Clone)]
814pub struct RotaryEmbedding {
815    head_dim: usize,
816    rope_theta: f64,
817    partial_rotary_factor: f64,
818    scale_factor: f64,
819    mode: RopeMode,
820    cos: Tensor,
821    sin: Tensor,
822}
823
824impl RotaryEmbedding {
825    /// Precompute `(max_seq, head_dim / 2)` cosine and sine tables using standard full-head RoPE.
826    ///
827    /// # Errors
828    /// Returns an error if `head_dim` is odd or tensor creation fails.
829    pub fn new(
830        head_dim: usize,
831        max_seq: usize,
832        rope_theta: f64,
833        dtype: DType,
834        device: &Device,
835    ) -> Result<Self> {
836        Self::new_with_config(
837            head_dim,
838            max_seq,
839            rope_theta,
840            1.0,
841            1.0,
842            RopeMode::Default,
843            dtype,
844            device,
845        )
846    }
847
848    /// Precompute `(max_seq, head_dim / 2)` cosine and sine tables with partial/proportional RoPE options.
849    ///
850    /// # Errors
851    /// Returns an error if `head_dim` is odd, `partial_rotary_factor` is outside `(0.0, 1.0]`,
852    /// `scale_factor <= 0.0`, or tensor creation fails.
853    #[allow(clippy::too_many_arguments)]
854    pub fn new_with_config(
855        head_dim: usize,
856        max_seq: usize,
857        rope_theta: f64,
858        partial_rotary_factor: f64,
859        scale_factor: f64,
860        mode: RopeMode,
861        dtype: DType,
862        device: &Device,
863    ) -> Result<Self> {
864        if !head_dim.is_multiple_of(2) {
865            return Err(ModelError::InvalidArgument(format!(
866                "rotary embedding head_dim must be even, got {head_dim}"
867            )));
868        }
869        if !partial_rotary_factor.is_finite()
870            || partial_rotary_factor <= 0.0
871            || partial_rotary_factor > 1.0
872        {
873            return Err(ModelError::InvalidArgument(format!(
874                "partial_rotary_factor must be in (0.0, 1.0], got {partial_rotary_factor}"
875            )));
876        }
877        if !scale_factor.is_finite() || scale_factor <= 0.0 {
878            return Err(ModelError::InvalidArgument(format!(
879                "rope scale_factor must be finite and positive, got {scale_factor}"
880            )));
881        }
882        let (cos, sin) = compute_rope_tables_with_config(
883            head_dim,
884            max_seq,
885            rope_theta,
886            partial_rotary_factor,
887            scale_factor,
888            mode,
889            dtype,
890            device,
891        )?;
892        Ok(Self {
893            head_dim,
894            rope_theta,
895            partial_rotary_factor,
896            scale_factor,
897            mode,
898            cos,
899            sin,
900        })
901    }
902
903    /// Apply RoPE to `(batch, heads, seq_len, head_dim)` starting at `pos_offset`.
904    ///
905    /// # Errors
906    /// Returns an error if `rope_slow` or dynamic table recomputation fails.
907    pub fn apply(&self, x: &Tensor, pos_offset: usize) -> Result<Tensor> {
908        let (_, _, seq_len, _) = x.dims4()?;
909        let needed = pos_offset + seq_len;
910        let (cos_full, sin_full) = if needed <= self.cos.dims()[0] {
911            (self.cos.clone(), self.sin.clone())
912        } else {
913            compute_rope_tables_with_config(
914                self.head_dim,
915                needed,
916                self.rope_theta,
917                self.partial_rotary_factor,
918                self.scale_factor,
919                self.mode,
920                x.dtype(),
921                x.device(),
922            )?
923        };
924        let cos = cos_full
925            .narrow(0, pos_offset, seq_len)?
926            .to_dtype(x.dtype())?;
927        let sin = sin_full
928            .narrow(0, pos_offset, seq_len)?
929            .to_dtype(x.dtype())?;
930        Ok(candle_nn::rotary_emb::rope_slow(x, &cos, &sin)?)
931    }
932}
933
934#[allow(clippy::too_many_arguments)]
935fn compute_rope_tables_with_config(
936    head_dim: usize,
937    max_seq: usize,
938    rope_theta: f64,
939    partial_rotary_factor: f64,
940    scale_factor: f64,
941    mode: RopeMode,
942    dtype: DType,
943    device: &Device,
944) -> Result<(Tensor, Tensor)> {
945    let half_dim = head_dim / 2;
946    let rope_angles =
947        (((partial_rotary_factor * head_dim as f64) / 2.0).floor() as usize).min(half_dim);
948    let denom = match mode {
949        RopeMode::Proportional => head_dim as f64,
950        RopeMode::Default => (2 * rope_angles).max(2) as f64,
951    };
952    let mut inv_freq = vec![0.0f32; half_dim];
953    for (i, slot) in inv_freq.iter_mut().take(rope_angles).enumerate() {
954        *slot = (1.0f64 / rope_theta.powf((2 * i) as f64 / denom) / scale_factor) as f32;
955    }
956    let inv_freq_t = Tensor::from_vec(inv_freq, (1, half_dim), device)?;
957    let positions: Vec<f32> = (0..max_seq).map(|p| p as f32).collect();
958    let pos_t = Tensor::from_vec(positions, (max_seq, 1), device)?;
959    let freqs = pos_t.matmul(&inv_freq_t)?;
960    let cos = freqs.cos()?.to_dtype(dtype)?;
961    let sin = freqs.sin()?.to_dtype(dtype)?;
962    Ok((cos, sin))
963}
964
965/// Layer-indexed key/value cache for autoregressive generation.
966#[derive(Debug, Clone)]
967pub struct KvCache {
968    layers: Vec<Option<(Tensor, Tensor)>>,
969    seen_tokens: Vec<usize>,
970}
971
972impl KvCache {
973    /// Allocate an empty cache for `num_layers` transformer blocks.
974    #[must_use]
975    pub fn new(num_layers: usize) -> Self {
976        Self {
977            layers: vec![None; num_layers],
978            seen_tokens: vec![0; num_layers],
979        }
980    }
981
982    /// Number of tokens currently retained in the cache window at `layer_idx`.
983    #[must_use]
984    pub fn past_len(&self, layer_idx: usize) -> usize {
985        self.layers
986            .get(layer_idx)
987            .and_then(|entry| entry.as_ref())
988            .and_then(|(k, _)| k.dims().get(2).copied())
989            .unwrap_or(0)
990    }
991
992    /// Total number of sequence positions processed so far at `layer_idx` (unaffected by sliding-window eviction).
993    #[must_use]
994    pub fn seq_pos(&self, layer_idx: usize) -> usize {
995        self.seen_tokens.get(layer_idx).copied().unwrap_or(0)
996    }
997
998    /// Append `(k, v)` of shape `(batch, kv_heads, seq_len, head_dim)` and return the concatenated history.
999    ///
1000    /// # Errors
1001    /// Returns an error if `layer_idx` is out of bounds or concatenation fails.
1002    pub fn append(
1003        &mut self,
1004        layer_idx: usize,
1005        k: &Tensor,
1006        v: &Tensor,
1007        sliding_window: Option<usize>,
1008    ) -> Result<(Tensor, Tensor)> {
1009        let added = k.dims()[2];
1010        let slot = self.layers.get_mut(layer_idx).ok_or_else(|| {
1011            ModelError::InvalidArgument(format!("kv cache layer index {layer_idx} out of bounds"))
1012        })?;
1013        let (mut next_k, mut next_v) = match slot.take() {
1014            Some((prev_k, prev_v)) => (
1015                Tensor::cat(&[&prev_k, k], 2)?,
1016                Tensor::cat(&[&prev_v, v], 2)?,
1017            ),
1018            None => (k.clone(), v.clone()),
1019        };
1020
1021        if let Some(seen) = self.seen_tokens.get_mut(layer_idx) {
1022            *seen += added;
1023        }
1024
1025        if let Some(win) = sliding_window {
1026            let cur_len = next_k.dims()[2];
1027            if cur_len > win {
1028                next_k = next_k.narrow(2, cur_len - win, win)?;
1029                next_v = next_v.narrow(2, cur_len - win, win)?;
1030            }
1031        }
1032
1033        *slot = Some((next_k.clone(), next_v.clone()));
1034        Ok((next_k, next_v))
1035    }
1036
1037    /// Reset all cached key/value tensors and processed position counters.
1038    pub fn clear(&mut self) {
1039        for slot in &mut self.layers {
1040            *slot = None;
1041        }
1042        self.seen_tokens.fill(0);
1043    }
1044}
1045
1046/// Repeat key/value heads `n_rep` times along dimension 1 for Grouped-Query Attention (GQA).
1047///
1048/// # Errors
1049/// Returns an error if tensor is not rank-4 or reshape fails.
1050pub fn repeat_kv(x: &Tensor, n_rep: usize) -> Result<Tensor> {
1051    if n_rep == 1 {
1052        return Ok(x.clone());
1053    }
1054    let (batch, kv_heads, seq_len, head_dim) = x.dims4()?;
1055    let expanded = x
1056        .unsqueeze(2)?
1057        .expand((batch, kv_heads, n_rep, seq_len, head_dim))?
1058        .reshape((batch, kv_heads * n_rep, seq_len, head_dim))?;
1059    Ok(expanded)
1060}
1061
1062/// Build a causal additive attention mask `(1, 1, seq_len, total_len)` with `-1e9` on masked positions.
1063///
1064/// # Errors
1065/// Returns an error if tensor creation fails.
1066pub fn causal_mask(
1067    seq_len: usize,
1068    past_len: usize,
1069    sliding_window: Option<usize>,
1070    dtype: DType,
1071    device: &Device,
1072) -> Result<Tensor> {
1073    let total_len = past_len + seq_len;
1074    let mut mask = vec![0.0f32; seq_len * total_len];
1075    for q in 0..seq_len {
1076        let q_pos = past_len + q;
1077        for k in 0..total_len {
1078            let masked_future = k > q_pos;
1079            let masked_window = match sliding_window {
1080                Some(win) if win > 0 => q_pos >= win && k <= q_pos - win,
1081                _ => false,
1082            };
1083            if masked_future || masked_window {
1084                mask[q * total_len + k] = -1.0e9;
1085            }
1086        }
1087    }
1088    let t = Tensor::from_vec(mask, (1, 1, seq_len, total_len), device)?;
1089    if dtype == DType::F32 {
1090        Ok(t)
1091    } else {
1092        Ok(t.to_dtype(dtype)?)
1093    }
1094}
1095
1096/// Apply activation function matching Hugging Face `hidden_act`.
1097///
1098/// # Errors
1099/// Returns [`ModelError::UnsupportedArchitecture`] for unknown activation names.
1100pub fn apply_activation(x: &Tensor, name: &str) -> Result<Tensor> {
1101    match name.to_ascii_lowercase().as_str() {
1102        "silu" | "swish" => Ok(candle_nn::ops::silu(x)?),
1103        "gelu" => Ok(x.gelu_erf()?),
1104        "gelu_pytorch_tanh" | "gelu_new" | "gelu_fast" => Ok(x.gelu()?),
1105        "relu" => Ok(x.relu()?),
1106        other => Err(ModelError::UnsupportedArchitecture(format!(
1107            "unsupported activation function `{other}`"
1108        ))),
1109    }
1110}
1111
1112/// Parameters controlling scaled dot-product attention inside decoder layers.
1113#[derive(Debug, Clone, Copy)]
1114pub struct AttentionParams {
1115    /// Multiplicative logit scale (typically `1.0 / sqrt(head_dim)`).
1116    pub scale: f64,
1117    /// Optional logit softcap (`softcap * tanh(scores / softcap)`), used by Gemma 2/3.
1118    pub softcap: Option<f64>,
1119    /// Optional sliding window attention span.
1120    pub sliding_window: Option<usize>,
1121}
1122
1123/// Differentiable multi-head / grouped-query causal scaled dot-product attention.
1124///
1125/// Expects `q` `(batch, num_heads, seq_len, head_dim)` and `k, v` `(batch, num_kv_heads, seq_len, head_dim)`.
1126/// Returns `(batch, seq_len, num_heads * head_dim)`.
1127///
1128/// # Errors
1129/// Returns an error if GQA head counts are incompatible or tensor ops fail.
1130pub(crate) fn compute_attention(
1131    q: &Tensor,
1132    k: &Tensor,
1133    v: &Tensor,
1134    layer_idx: usize,
1135    cache: Option<&mut KvCache>,
1136    params: AttentionParams,
1137) -> Result<Tensor> {
1138    let (batch, num_heads, seq_len, head_dim) = q.dims4()?;
1139    let (_, num_kv_heads, _, _) = k.dims4()?;
1140    if num_kv_heads == 0 || num_heads % num_kv_heads != 0 {
1141        return Err(ModelError::InvalidArgument(format!(
1142            "num_attention_heads ({num_heads}) must be divisible by num_key_value_heads ({num_kv_heads})"
1143        )));
1144    }
1145    let n_rep = num_heads / num_kv_heads;
1146
1147    let (k_all, v_all) = match cache {
1148        Some(c) => c.append(layer_idx, k, v, params.sliding_window)?,
1149        None => (k.clone(), v.clone()),
1150    };
1151    let effective_past_len = k_all.dims()[2].saturating_sub(seq_len);
1152
1153    let k_rep = repeat_kv(&k_all, n_rep)?;
1154    let v_rep = repeat_kv(&v_all, n_rep)?;
1155
1156    let mut scores = (q.matmul(&k_rep.t()?.contiguous()?)? * params.scale)?;
1157    if let Some(cap) = params.softcap {
1158        scores = ((scores / cap)?.tanh()? * cap)?;
1159    }
1160    if seq_len > 1 {
1161        let mask = causal_mask(
1162            seq_len,
1163            effective_past_len,
1164            params.sliding_window,
1165            scores.dtype(),
1166            scores.device(),
1167        )?;
1168        scores = scores.broadcast_add(&mask)?;
1169    }
1170
1171    // Use differentiable softmax (`candle_nn::ops::softmax`) rather than `softmax_last_dim`.
1172    let probs = candle_nn::ops::softmax(&scores, D::Minus1)?;
1173    let ctx = probs.matmul(&v_rep.contiguous()?)?;
1174    Ok(ctx
1175        .transpose(1, 2)?
1176        .contiguous()?
1177        .reshape((batch, seq_len, num_heads * head_dim))?)
1178}
1179
1180pub(crate) fn move_cast(t: &Tensor, dtype: DType, device: &Device) -> Result<Tensor> {
1181    let moved = if t.device().same_device(device) {
1182        t.clone()
1183    } else {
1184        t.to_device(device)?
1185    };
1186    if moved.dtype() == dtype {
1187        Ok(moved)
1188    } else {
1189        Ok(moved.to_dtype(dtype)?)
1190    }
1191}