Skip to main content

littlebit_models/
gemma4.rs

1//! Gemma 4 Dense and Mixture-of-Experts (MoE) decoder architectures.
2//!
3//! Implements Hugging Face `Gemma4TextModel` / `Gemma4ForConditionalGeneration` text backbone:
4//! - Hybrid per-layer `sliding_attention` and `full_attention` schedules
5//! - Heterogeneous sliding vs. global head dimensions and KV head counts
6//! - Optional `attention_k_eq_v` on global (`full_attention`) layers (sharing `k_proj` for `V` prior to `k_norm`/RoPE + scaleless `v_norm`)
7//! - Standard (non-offset) `Gemma4RMSNorm` (`NormKind::Standard`) plus weightless `RmsNormNoScale` for `v_norm` and router input norm
8//! - Unit attention scaling (`scale = 1.0`) and Proportional RoPE (`p-RoPE`)
9//! - Optional dual-path FFN combining a shared dense MLP with sparse top-k routed MoE (`Gemma4Router` + `Gemma4Experts`)
10//! - Per-layer residual `layer_scalar` multiplier
11
12use std::collections::{BTreeMap, HashMap};
13
14use candle_core::{DType, Device, Tensor, Var, D};
15use rand_chacha::ChaCha8Rng;
16use rand_distr::{Distribution, Normal};
17
18use littlebit_core::{BiasMode, LayerShape, LittleBitConfig, Param, TrainMode};
19
20use crate::config::{ModelConfig, ResolvedRopeParams};
21use crate::error::{ModelError, Result};
22use crate::layer::{
23    apply_activation, compute_attention, move_cast, AttentionParams, Embedding, KvCache,
24    LinearLayer, LinearLoadSpec, NormKind, RmsNorm, RmsNormNoScale, RopeMode, RotaryEmbedding,
25};
26use crate::model::{CausalLmOutput, HiddenStatesMode};
27use crate::patch::{patch_linear_layer, PatchPolicy, PatchSummary};
28
29/// Sparse expert routing decision emitted by [`Gemma4Router`].
30#[derive(Debug, Clone)]
31pub struct RouterDispatch {
32    /// Selected expert indices per flattened token `[num_tokens][top_k]`.
33    pub top_k_indices: Vec<Vec<usize>>,
34    /// Normalized & scaled routing weights per flattened token `[num_tokens][top_k]`.
35    pub top_k_weights: Vec<Vec<f32>>,
36}
37
38/// Top-k softmax router for Gemma 4 Mixture-of-Experts blocks.
39///
40/// The router linear projection (`proj`) always remains unquantized (`LinearLayer::Dense`)
41/// so top-k routing fidelity is preserved at sub-1-bit expert compression.
42#[derive(Debug, Clone)]
43pub struct Gemma4Router {
44    norm: RmsNormNoScale,
45    scale: Param,
46    /// Unquantized router linear projection `(hidden_size -> num_experts)`.
47    pub proj: LinearLayer,
48    per_expert_scale: Param,
49    scalar_root_size: f64,
50    num_experts: usize,
51    top_k: usize,
52}
53
54impl Gemma4Router {
55    /// Load [`Gemma4Router`] weights under prefix `{layer_prefix}.router`.
56    ///
57    /// # Errors
58    /// Returns [`ModelError::MissingWeight`] if required router parameters are absent.
59    pub fn from_tensors(
60        prefix: &str,
61        config: &ModelConfig,
62        tensors: &HashMap<String, Tensor>,
63        spec: LinearLoadSpec<'_>,
64    ) -> Result<Self> {
65        let num_experts = config.num_experts.ok_or_else(|| {
66            ModelError::InvalidArgument("Gemma 4 MoE requires config.num_experts".to_string())
67        })?;
68        let top_k = config.top_k_experts.unwrap_or(8).min(num_experts).max(1);
69
70        let scale_key = format!("{prefix}.scale");
71        let scale_t = tensors
72            .get(&scale_key)
73            .ok_or_else(|| ModelError::MissingWeight(scale_key.clone()))?;
74        let scale = Param::Frozen(move_cast(scale_t, spec.dtype, spec.device)?);
75
76        let per_expert_key = format!("{prefix}.per_expert_scale");
77        let per_expert_t = tensors
78            .get(&per_expert_key)
79            .ok_or_else(|| ModelError::MissingWeight(per_expert_key.clone()))?;
80        let per_expert_scale = Param::Frozen(move_cast(per_expert_t, spec.dtype, spec.device)?);
81
82        // Keep router projection dense even during quantized checkpoint loading.
83        let router_spec = LinearLoadSpec {
84            littlebit: None,
85            ratio_factor: 1.0,
86            ..spec
87        };
88        let proj = LinearLayer::from_tensors(
89            &format!("{prefix}.proj"),
90            tensors,
91            LayerShape::new(config.hidden_size, num_experts, BiasMode::NoBias),
92            router_spec,
93        )?;
94
95        Ok(Self {
96            norm: RmsNormNoScale::new(config.rms_norm_eps),
97            scale,
98            proj,
99            per_expert_scale,
100            scalar_root_size: (config.hidden_size as f64).powf(-0.5),
101            num_experts,
102            top_k,
103        })
104    }
105
106    /// Initialize random [`Gemma4Router`] weights for tests.
107    ///
108    /// # Errors
109    /// Returns an error if tensor creation fails.
110    pub fn new_random(
111        config: &ModelConfig,
112        dtype: DType,
113        device: &Device,
114        rng: &mut ChaCha8Rng,
115    ) -> Result<Self> {
116        let num_experts = config.num_experts.unwrap_or(8).max(1);
117        let top_k = config.top_k_experts.unwrap_or(2).min(num_experts).max(1);
118        Ok(Self {
119            norm: RmsNormNoScale::new(config.rms_norm_eps),
120            scale: Param::Frozen(Tensor::ones(config.hidden_size, dtype, device)?),
121            proj: LinearLayer::new_random(
122                LayerShape::new(config.hidden_size, num_experts, BiasMode::NoBias),
123                dtype,
124                device,
125                rng,
126            )?,
127            per_expert_scale: Param::Frozen(Tensor::ones(num_experts, dtype, device)?),
128            scalar_root_size: (config.hidden_size as f64).powf(-0.5),
129            num_experts,
130            top_k,
131        })
132    }
133
134    /// Total number of experts supported by this router.
135    #[must_use]
136    pub const fn num_experts(&self) -> usize {
137        self.num_experts
138    }
139
140    /// Number of top-ranked experts activated per token.
141    #[must_use]
142    pub const fn top_k(&self) -> usize {
143        self.top_k
144    }
145
146    /// Compute top-k expert assignments and scaled probabilities for `hidden_flat` `(num_tokens, hidden_size)`.
147    ///
148    /// # Errors
149    /// Returns an error if tensor shapes are invalid or projection fails.
150    pub fn forward(&self, hidden_flat: &Tensor) -> Result<RouterDispatch> {
151        let normed = self.norm.forward(hidden_flat)?;
152        let scale_raw = self.scale.as_tensor();
153        let scale_t = if scale_raw.dtype() == normed.dtype() {
154            scale_raw.clone()
155        } else {
156            scale_raw.to_dtype(normed.dtype())?
157        };
158        let scaled = (normed * self.scalar_root_size)?.broadcast_mul(&scale_t)?;
159        let logits = self.proj.forward(&scaled)?.to_dtype(DType::F32)?;
160        let probs = candle_nn::ops::softmax(&logits, D::Minus1)?;
161
162        let probs_vec = probs.to_vec2::<f32>()?;
163        let per_expert = self
164            .per_expert_scale
165            .as_tensor()
166            .to_dtype(DType::F32)?
167            .to_vec1::<f32>()?;
168
169        let mut top_k_indices = Vec::with_capacity(probs_vec.len());
170        let mut top_k_weights = Vec::with_capacity(probs_vec.len());
171
172        for row in probs_vec {
173            let mut ranked: Vec<(usize, f32)> = row.into_iter().enumerate().collect();
174            ranked.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
175            ranked.truncate(self.top_k);
176
177            let denom: f32 = ranked.iter().map(|&(_, p)| p).sum();
178            let mut idxs = Vec::with_capacity(self.top_k);
179            let mut weights = Vec::with_capacity(self.top_k);
180            for (eid, p) in ranked {
181                let norm_w = if denom > 0.0 {
182                    p / denom
183                } else {
184                    1.0 / (self.top_k as f32)
185                };
186                let expert_scale = per_expert.get(eid).copied().unwrap_or(1.0);
187                idxs.push(eid);
188                weights.push(norm_w * expert_scale);
189            }
190            top_k_indices.push(idxs);
191            top_k_weights.push(weights);
192        }
193
194        Ok(RouterDispatch {
195            top_k_indices,
196            top_k_weights,
197        })
198    }
199}
200
201/// Single routed SwiGLU / GeGLU expert MLP inside [`Gemma4Experts`].
202#[derive(Debug, Clone)]
203pub struct Gemma4Expert {
204    /// Fused `[gate_proj; up_proj]` projection `(hidden_size -> 2 * moe_intermediate_size)`.
205    pub gate_up_proj: LinearLayer,
206    /// Down projection `(moe_intermediate_size -> hidden_size)`.
207    pub down_proj: LinearLayer,
208}
209
210/// Collection of routed Mixture-of-Experts feed-forward networks with sparse dispatch.
211#[derive(Debug, Clone)]
212pub struct Gemma4Experts {
213    /// Individual routed experts (`len == num_experts`).
214    pub experts: Vec<Gemma4Expert>,
215    intermediate_size: usize,
216    hidden_act: String,
217}
218
219impl Gemma4Experts {
220    /// Load experts from either per-expert keys (`{prefix}.{e}.gate_up_proj.*`) or
221    /// Hugging Face fused 3D parameter tensors (`{prefix}.gate_up_proj` / `{prefix}.down_proj`).
222    ///
223    /// # Errors
224    /// Returns an error if expert weights are missing or shapes do not match `config`.
225    pub fn from_tensors(
226        prefix: &str,
227        config: &ModelConfig,
228        tensors: &HashMap<String, Tensor>,
229        spec: LinearLoadSpec<'_>,
230    ) -> Result<Self> {
231        let num_experts = config.num_experts.ok_or_else(|| {
232            ModelError::InvalidArgument("Gemma 4 MoE requires config.num_experts".to_string())
233        })?;
234        let inter = config
235            .moe_intermediate_size
236            .unwrap_or_else(|| config.effective_intermediate_size());
237
238        let has_per_expert_keys = tensors
239            .contains_key(&format!("{prefix}.0.gate_up_proj.U_packed"))
240            || tensors.contains_key(&format!("{prefix}.0.gate_up_proj.U"))
241            || tensors.contains_key(&format!("{prefix}.0.gate_up_proj.weight"));
242
243        let mut experts = Vec::with_capacity(num_experts);
244        if has_per_expert_keys {
245            for e in 0..num_experts {
246                let gate_up_proj = LinearLayer::from_tensors(
247                    &format!("{prefix}.{e}.gate_up_proj"),
248                    tensors,
249                    LayerShape::new(config.hidden_size, 2 * inter, BiasMode::NoBias),
250                    spec,
251                )?;
252                let down_proj = LinearLayer::from_tensors(
253                    &format!("{prefix}.{e}.down_proj"),
254                    tensors,
255                    LayerShape::new(inter, config.hidden_size, BiasMode::NoBias),
256                    spec,
257                )?;
258                experts.push(Gemma4Expert {
259                    gate_up_proj,
260                    down_proj,
261                });
262            }
263        } else {
264            let gu_key = format!("{prefix}.gate_up_proj");
265            let dn_key = format!("{prefix}.down_proj");
266            let gu_3d = tensors
267                .get(&gu_key)
268                .ok_or_else(|| ModelError::MissingWeight(gu_key.clone()))?;
269            let dn_3d = tensors
270                .get(&dn_key)
271                .ok_or_else(|| ModelError::MissingWeight(dn_key.clone()))?;
272
273            let gu_dims = gu_3d.dims();
274            let dn_dims = dn_3d.dims();
275            if gu_dims.len() != 3 || gu_dims[0] != num_experts {
276                return Err(ModelError::InvalidArgument(format!(
277                    "expected `{gu_key}` with shape [{num_experts}, 2*{inter}, {}], got {gu_dims:?}",
278                    config.hidden_size
279                )));
280            }
281            if dn_dims.len() != 3 || dn_dims[0] != num_experts {
282                return Err(ModelError::InvalidArgument(format!(
283                    "expected `{dn_key}` with shape [{num_experts}, {}, {inter}], got {dn_dims:?}",
284                    config.hidden_size
285                )));
286            }
287
288            for e in 0..num_experts {
289                let w_gu = move_cast(&gu_3d.get(e)?.contiguous()?, spec.dtype, spec.device)?;
290                let w_dn = move_cast(&dn_3d.get(e)?.contiguous()?, spec.dtype, spec.device)?;
291                experts.push(Gemma4Expert {
292                    gate_up_proj: LinearLayer::Dense {
293                        weight: Param::Frozen(w_gu),
294                        bias: None,
295                    },
296                    down_proj: LinearLayer::Dense {
297                        weight: Param::Frozen(w_dn),
298                        bias: None,
299                    },
300                });
301            }
302        }
303
304        Ok(Self {
305            experts,
306            intermediate_size: inter,
307            hidden_act: config.effective_hidden_act().to_string(),
308        })
309    }
310
311    /// Initialize random [`Gemma4Experts`] for tests.
312    ///
313    /// # Errors
314    /// Returns an error if tensor creation fails.
315    pub fn new_random(
316        config: &ModelConfig,
317        dtype: DType,
318        device: &Device,
319        rng: &mut ChaCha8Rng,
320    ) -> Result<Self> {
321        let num_experts = config.num_experts.unwrap_or(8).max(1);
322        let inter = config
323            .moe_intermediate_size
324            .unwrap_or_else(|| config.effective_intermediate_size());
325        let mut experts = Vec::with_capacity(num_experts);
326        for _ in 0..num_experts {
327            experts.push(Gemma4Expert {
328                gate_up_proj: LinearLayer::new_random(
329                    LayerShape::new(config.hidden_size, 2 * inter, BiasMode::NoBias),
330                    dtype,
331                    device,
332                    rng,
333                )?,
334                down_proj: LinearLayer::new_random(
335                    LayerShape::new(inter, config.hidden_size, BiasMode::NoBias),
336                    dtype,
337                    device,
338                    rng,
339                )?,
340            });
341        }
342        Ok(Self {
343            experts,
344            intermediate_size: inter,
345            hidden_act: config.effective_hidden_act().to_string(),
346        })
347    }
348
349    /// Execute ONLY the experts selected in `dispatch` across their assigned tokens.
350    ///
351    /// # Errors
352    /// Returns an error if an assigned expert ID is out of bounds or tensor ops fail.
353    pub fn forward(&self, hidden_flat: &Tensor, dispatch: &RouterDispatch) -> Result<Tensor> {
354        let (num_tokens, _) = hidden_flat.dims2()?;
355        if num_tokens == 0 {
356            return Ok(Tensor::zeros_like(hidden_flat)?);
357        }
358
359        let mut buckets: BTreeMap<usize, Vec<(u32, f32)>> = BTreeMap::new();
360        for (tok_idx, (eids, weights)) in dispatch
361            .top_k_indices
362            .iter()
363            .zip(dispatch.top_k_weights.iter())
364            .enumerate()
365        {
366            let tok_u32 = tok_idx as u32;
367            for (&eid, &w) in eids.iter().zip(weights.iter()) {
368                buckets.entry(eid).or_default().push((tok_u32, w));
369            }
370        }
371
372        let device = hidden_flat.device();
373        let dtype = hidden_flat.dtype();
374        let mut acc = Tensor::zeros_like(hidden_flat)?;
375
376        for (expert_id, entries) in buckets {
377            let expert = self.experts.get(expert_id).ok_or_else(|| {
378                ModelError::InvalidArgument(format!(
379                    "routed expert index {expert_id} out of bounds (num_experts = {})",
380                    self.experts.len()
381                ))
382            })?;
383            let n_tok = entries.len();
384            let tok_indices: Vec<u32> = entries.iter().map(|&(t, _)| t).collect();
385            let weights: Vec<f32> = entries.iter().map(|&(_, w)| w).collect();
386
387            let tok_t = Tensor::from_vec(tok_indices, n_tok, device)?;
388            let w_col = Tensor::from_vec(weights, (n_tok, 1), device)?.to_dtype(dtype)?;
389
390            let x_e = hidden_flat.index_select(&tok_t, 0)?;
391            let gate_up = expert.gate_up_proj.forward(&x_e)?;
392            let gate = gate_up.narrow(1, 0, self.intermediate_size)?;
393            let up = gate_up.narrow(1, self.intermediate_size, self.intermediate_size)?;
394            let act = apply_activation(&gate, &self.hidden_act)?.mul(&up)?;
395            let y_e = expert.down_proj.forward(&act)?.broadcast_mul(&w_col)?;
396
397            acc = acc.index_add(&tok_t, &y_e, 0)?;
398        }
399
400        Ok(acc)
401    }
402}
403
404/// Complete Mixture-of-Experts sub-block inside a [`Gemma4Model`] decoder layer.
405#[derive(Debug, Clone)]
406pub struct Gemma4MoE {
407    /// Post-normalization on the parallel dense shared MLP output.
408    pub post_feedforward_layernorm_1: RmsNorm,
409    /// Pre-normalization before routed expert execution.
410    pub pre_feedforward_layernorm_2: RmsNorm,
411    /// Unquantized top-k router.
412    pub router: Gemma4Router,
413    /// Sparse expert ensemble (`Dense`, `LittleBit`, or `PackedLittleBitLinear`).
414    pub experts: Gemma4Experts,
415    /// Post-normalization on the summed sparse expert output.
416    pub post_feedforward_layernorm_2: RmsNorm,
417}
418
419#[derive(Debug, Clone)]
420struct Gemma4Attention {
421    q_proj: LinearLayer,
422    k_proj: LinearLayer,
423    v_proj: Option<LinearLayer>,
424    o_proj: LinearLayer,
425    q_norm: RmsNorm,
426    k_norm: RmsNorm,
427    v_norm: RmsNormNoScale,
428    num_heads: usize,
429    num_kv_heads: usize,
430    head_dim: usize,
431    softcap: Option<f64>,
432    sliding_window: Option<usize>,
433    is_sliding: bool,
434}
435
436impl Gemma4Attention {
437    fn forward(
438        &self,
439        hidden_states: &Tensor,
440        rope: &RotaryEmbedding,
441        layer_idx: usize,
442        cache: Option<&mut KvCache>,
443    ) -> Result<Tensor> {
444        let (batch, seq_len, _) = hidden_states.dims3()?;
445        // Use cumulative sequence position so sliding-window eviction does not stall RoPE positions.
446        let pos_offset = cache.as_ref().map_or(0, |c| c.seq_pos(layer_idx));
447
448        let q_raw = self
449            .q_proj
450            .forward(hidden_states)?
451            .reshape((batch, seq_len, self.num_heads, self.head_dim))?
452            .transpose(1, 2)?
453            .contiguous()?;
454        let k_raw = self
455            .k_proj
456            .forward(hidden_states)?
457            .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
458            .transpose(1, 2)?
459            .contiguous()?;
460
461        // When `attention_k_eq_v` is enabled (`v_proj` is None), clone `k_raw` into `v_raw`
462        // BEFORE `k_norm` and RoPE, matching Hugging Face `Gemma4TextAttention`.
463        let v_raw = match &self.v_proj {
464            Some(vp) => vp
465                .forward(hidden_states)?
466                .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
467                .transpose(1, 2)?
468                .contiguous()?,
469            None => k_raw.clone(),
470        };
471
472        let q = rope.apply(&self.q_norm.forward(&q_raw)?, pos_offset)?;
473        let k = rope.apply(&self.k_norm.forward(&k_raw)?, pos_offset)?;
474        let v = self.v_norm.forward(&v_raw)?;
475
476        let attn_out = compute_attention(
477            &q,
478            &k,
479            &v,
480            layer_idx,
481            cache,
482            AttentionParams {
483                scale: 1.0,
484                softcap: self.softcap,
485                sliding_window: self.sliding_window,
486            },
487        )?;
488        self.o_proj.forward(&attn_out)
489    }
490}
491
492#[derive(Debug, Clone)]
493struct Gemma4Mlp {
494    gate_proj: LinearLayer,
495    up_proj: LinearLayer,
496    down_proj: LinearLayer,
497    hidden_act: String,
498}
499
500impl Gemma4Mlp {
501    fn forward(&self, x: &Tensor) -> Result<Tensor> {
502        let gate = apply_activation(&self.gate_proj.forward(x)?, &self.hidden_act)?;
503        let up = self.up_proj.forward(x)?;
504        self.down_proj.forward(&gate.mul(&up)?)
505    }
506}
507
508#[derive(Debug, Clone)]
509struct Gemma4DecoderLayer {
510    input_layernorm: RmsNorm,
511    self_attn: Gemma4Attention,
512    post_attention_layernorm: RmsNorm,
513    pre_feedforward_layernorm: RmsNorm,
514    mlp: Gemma4Mlp,
515    moe: Option<Gemma4MoE>,
516    post_feedforward_layernorm: RmsNorm,
517    layer_scalar: Param,
518}
519
520/// Causal language model supporting Gemma 4 Dense and Mixture-of-Experts checkpoints.
521#[derive(Debug, Clone)]
522pub struct Gemma4Model {
523    config: ModelConfig,
524    weight_prefix: String,
525    embed_tokens: Embedding,
526    layers: Vec<Gemma4DecoderLayer>,
527    norm: RmsNorm,
528    lm_head: Option<LinearLayer>,
529    rope_full: RotaryEmbedding,
530    rope_sliding: Option<RotaryEmbedding>,
531}
532
533fn detect_weight_prefix(tensors: &HashMap<String, Tensor>) -> String {
534    if tensors.contains_key("model.language_model.embed_tokens.weight") {
535        "model.language_model".to_string()
536    } else if tensors.contains_key("language_model.model.embed_tokens.weight") {
537        "language_model.model".to_string()
538    } else {
539        "model".to_string()
540    }
541}
542
543fn build_rope(
544    head_dim: usize,
545    max_seq: usize,
546    params: ResolvedRopeParams,
547    dtype: DType,
548    device: &Device,
549) -> Result<RotaryEmbedding> {
550    let mode = if params.is_proportional {
551        RopeMode::Proportional
552    } else {
553        RopeMode::Default
554    };
555    RotaryEmbedding::new_with_config(
556        head_dim,
557        max_seq,
558        params.rope_theta,
559        params.partial_rotary_factor,
560        params.scale_factor,
561        mode,
562        dtype,
563        device,
564    )
565}
566
567fn load_layer_scalar(
568    key: &str,
569    tensors: &HashMap<String, Tensor>,
570    dtype: DType,
571    device: &Device,
572) -> Result<Param> {
573    match tensors.get(key) {
574        Some(t) => Ok(Param::Frozen(move_cast(t, dtype, device)?)),
575        None => Ok(Param::Frozen(Tensor::ones((1,), dtype, device)?)),
576    }
577}
578
579fn broadcast_scalar_mul(x: &Tensor, scalar: &Param) -> Result<Tensor> {
580    let s = scalar.as_tensor();
581    let s = if s.dtype() == x.dtype() {
582        s.clone()
583    } else {
584        s.to_dtype(x.dtype())?
585    };
586    Ok(x.broadcast_mul(&s)?)
587}
588
589impl Gemma4Model {
590    /// Load Gemma 4 Dense or MoE weights from a state dictionary.
591    ///
592    /// Automatically supports both standalone (`model.*`) and multimodal (`model.language_model.*` /
593    /// `language_model.model.*`) weight prefixes, as well as fused 3D MoE experts and LittleBit packed experts.
594    ///
595    /// # Errors
596    /// Returns [`ModelError::MissingWeight`] or [`ModelError::UnsupportedArchitecture`] on incompatible configs.
597    pub fn from_tensors(
598        config: ModelConfig,
599        tensors: &HashMap<String, Tensor>,
600        spec: LinearLoadSpec<'_>,
601    ) -> Result<Self> {
602        if config.num_kv_shared_layers.unwrap_or(0) > 0 {
603            return Err(ModelError::UnsupportedArchitecture(
604                "Gemma 4 KV-shared edge layers (num_kv_shared_layers > 0) are not supported"
605                    .to_string(),
606            ));
607        }
608
609        let weight_prefix = detect_weight_prefix(tensors);
610        let embed_scale = Some((config.hidden_size as f64).sqrt());
611        let embed_tokens = Embedding::from_tensors(
612            &format!("{weight_prefix}.embed_tokens"),
613            tensors,
614            embed_scale,
615            spec.dtype,
616            spec.device,
617        )?;
618
619        let num_heads = config.num_attention_heads;
620        let inter = config.effective_intermediate_size();
621        let attn_bias = BiasMode::from(config.attention_bias);
622        let mlp_bias = BiasMode::from(config.mlp_bias);
623        let kv_factor = spec.littlebit.map_or(1.0, |c| c.kv_factor);
624        let hidden_act = config.effective_hidden_act().to_string();
625        let layer_types = config.effective_layer_types();
626        let enable_moe = config.enable_moe_block.unwrap_or(false);
627
628        let kv_spec = LinearLoadSpec {
629            ratio_factor: kv_factor,
630            ..spec
631        };
632        let unit_spec = LinearLoadSpec {
633            ratio_factor: 1.0,
634            ..spec
635        };
636
637        let mut any_sliding = false;
638        let mut layers = Vec::with_capacity(config.num_hidden_layers);
639
640        for (i, layer_type) in layer_types.iter().enumerate() {
641            let p = format!("{weight_prefix}.layers.{i}");
642            let is_sliding = layer_type == "sliding_attention";
643            if is_sliding {
644                any_sliding = true;
645            }
646            let head_dim = if is_sliding {
647                config.effective_head_dim()
648            } else {
649                config.effective_global_head_dim()
650            };
651            let num_kv_heads = if is_sliding {
652                config.effective_kv_heads()
653            } else {
654                config.effective_global_kv_heads()
655            };
656            let sliding_window = if is_sliding {
657                config.sliding_window
658            } else {
659                None
660            };
661            let use_k_eq_v = !is_sliding && config.attention_k_eq_v.unwrap_or(false);
662
663            let input_layernorm = RmsNorm::from_tensors(
664                &format!("{p}.input_layernorm"),
665                tensors,
666                config.rms_norm_eps,
667                NormKind::Standard,
668                spec.dtype,
669                spec.device,
670            )?;
671            let q_proj = LinearLayer::from_tensors(
672                &format!("{p}.self_attn.q_proj"),
673                tensors,
674                LayerShape::new(config.hidden_size, num_heads * head_dim, attn_bias),
675                unit_spec,
676            )?;
677            let k_proj = LinearLayer::from_tensors(
678                &format!("{p}.self_attn.k_proj"),
679                tensors,
680                LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
681                kv_spec,
682            )?;
683            let v_proj = if use_k_eq_v {
684                None
685            } else {
686                Some(LinearLayer::from_tensors(
687                    &format!("{p}.self_attn.v_proj"),
688                    tensors,
689                    LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
690                    kv_spec,
691                )?)
692            };
693            let o_proj = LinearLayer::from_tensors(
694                &format!("{p}.self_attn.o_proj"),
695                tensors,
696                LayerShape::new(num_heads * head_dim, config.hidden_size, attn_bias),
697                unit_spec,
698            )?;
699            let q_norm = RmsNorm::from_tensors(
700                &format!("{p}.self_attn.q_norm"),
701                tensors,
702                config.rms_norm_eps,
703                NormKind::Standard,
704                spec.dtype,
705                spec.device,
706            )?;
707            let k_norm = RmsNorm::from_tensors(
708                &format!("{p}.self_attn.k_norm"),
709                tensors,
710                config.rms_norm_eps,
711                NormKind::Standard,
712                spec.dtype,
713                spec.device,
714            )?;
715            let v_norm = RmsNormNoScale::new(config.rms_norm_eps);
716
717            let post_attention_layernorm = RmsNorm::from_tensors(
718                &format!("{p}.post_attention_layernorm"),
719                tensors,
720                config.rms_norm_eps,
721                NormKind::Standard,
722                spec.dtype,
723                spec.device,
724            )?;
725            let pre_feedforward_layernorm = RmsNorm::from_tensors(
726                &format!("{p}.pre_feedforward_layernorm"),
727                tensors,
728                config.rms_norm_eps,
729                NormKind::Standard,
730                spec.dtype,
731                spec.device,
732            )?;
733            let gate_proj = LinearLayer::from_tensors(
734                &format!("{p}.mlp.gate_proj"),
735                tensors,
736                LayerShape::new(config.hidden_size, inter, mlp_bias),
737                unit_spec,
738            )?;
739            let up_proj = LinearLayer::from_tensors(
740                &format!("{p}.mlp.up_proj"),
741                tensors,
742                LayerShape::new(config.hidden_size, inter, mlp_bias),
743                unit_spec,
744            )?;
745            let down_proj = LinearLayer::from_tensors(
746                &format!("{p}.mlp.down_proj"),
747                tensors,
748                LayerShape::new(inter, config.hidden_size, mlp_bias),
749                unit_spec,
750            )?;
751            let post_feedforward_layernorm = RmsNorm::from_tensors(
752                &format!("{p}.post_feedforward_layernorm"),
753                tensors,
754                config.rms_norm_eps,
755                NormKind::Standard,
756                spec.dtype,
757                spec.device,
758            )?;
759
760            let moe = if enable_moe {
761                Some(Gemma4MoE {
762                    post_feedforward_layernorm_1: RmsNorm::from_tensors(
763                        &format!("{p}.post_feedforward_layernorm_1"),
764                        tensors,
765                        config.rms_norm_eps,
766                        NormKind::Standard,
767                        spec.dtype,
768                        spec.device,
769                    )?,
770                    pre_feedforward_layernorm_2: RmsNorm::from_tensors(
771                        &format!("{p}.pre_feedforward_layernorm_2"),
772                        tensors,
773                        config.rms_norm_eps,
774                        NormKind::Standard,
775                        spec.dtype,
776                        spec.device,
777                    )?,
778                    router: Gemma4Router::from_tensors(
779                        &format!("{p}.router"),
780                        &config,
781                        tensors,
782                        unit_spec,
783                    )?,
784                    experts: Gemma4Experts::from_tensors(
785                        &format!("{p}.experts"),
786                        &config,
787                        tensors,
788                        unit_spec,
789                    )?,
790                    post_feedforward_layernorm_2: RmsNorm::from_tensors(
791                        &format!("{p}.post_feedforward_layernorm_2"),
792                        tensors,
793                        config.rms_norm_eps,
794                        NormKind::Standard,
795                        spec.dtype,
796                        spec.device,
797                    )?,
798                })
799            } else {
800                None
801            };
802
803            let layer_scalar = load_layer_scalar(
804                &format!("{p}.layer_scalar"),
805                tensors,
806                spec.dtype,
807                spec.device,
808            )?;
809
810            layers.push(Gemma4DecoderLayer {
811                input_layernorm,
812                self_attn: Gemma4Attention {
813                    q_proj,
814                    k_proj,
815                    v_proj,
816                    o_proj,
817                    q_norm,
818                    k_norm,
819                    v_norm,
820                    num_heads,
821                    num_kv_heads,
822                    head_dim,
823                    softcap: config.attn_logit_softcapping,
824                    sliding_window,
825                    is_sliding,
826                },
827                post_attention_layernorm,
828                pre_feedforward_layernorm,
829                mlp: Gemma4Mlp {
830                    gate_proj,
831                    up_proj,
832                    down_proj,
833                    hidden_act: hidden_act.clone(),
834                },
835                moe,
836                post_feedforward_layernorm,
837                layer_scalar,
838            });
839        }
840
841        let norm = RmsNorm::from_tensors(
842            &format!("{weight_prefix}.norm"),
843            tensors,
844            config.rms_norm_eps,
845            NormKind::Standard,
846            spec.dtype,
847            spec.device,
848        )?;
849        let lm_head_key = if tensors.contains_key("lm_head.weight") {
850            Some("lm_head")
851        } else if tensors.contains_key("language_model.lm_head.weight") {
852            Some("language_model.lm_head")
853        } else {
854            None
855        };
856        let lm_head = match lm_head_key {
857            Some(prefix) if !config.tie_word_embeddings => Some(LinearLayer::from_tensors(
858                prefix,
859                tensors,
860                LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
861                LinearLoadSpec {
862                    littlebit: None,
863                    ratio_factor: 1.0,
864                    ..spec
865                },
866            )?),
867            _ => None,
868        };
869
870        let max_seq = config.max_position_embeddings.max(256);
871        let rope_full = build_rope(
872            config.effective_global_head_dim(),
873            max_seq,
874            config.gemma4_rope_params("full_attention"),
875            spec.dtype,
876            spec.device,
877        )?;
878        let rope_sliding = if any_sliding {
879            Some(build_rope(
880                config.effective_head_dim(),
881                max_seq,
882                config.gemma4_rope_params("sliding_attention"),
883                spec.dtype,
884                spec.device,
885            )?)
886        } else {
887            None
888        };
889
890        Ok(Self {
891            config,
892            weight_prefix,
893            embed_tokens,
894            layers,
895            norm,
896            lm_head,
897            rope_full,
898            rope_sliding,
899        })
900    }
901
902    /// Initialize a random Gemma 4 Dense or MoE model for testing and benchmarking.
903    ///
904    /// # Errors
905    /// Returns an error if tensor allocation or configuration validation fails.
906    pub fn new_random(
907        config: ModelConfig,
908        dtype: DType,
909        device: &Device,
910        rng: &mut ChaCha8Rng,
911    ) -> Result<Self> {
912        let embed_scale = Some((config.hidden_size as f64).sqrt());
913        let embed_tokens = Embedding::new_random(
914            config.vocab_size,
915            config.hidden_size,
916            embed_scale,
917            dtype,
918            device,
919            rng,
920        )?;
921
922        let num_heads = config.num_attention_heads;
923        let inter = config.effective_intermediate_size();
924        let attn_bias = BiasMode::from(config.attention_bias);
925        let mlp_bias = BiasMode::from(config.mlp_bias);
926        let hidden_act = config.effective_hidden_act().to_string();
927        let layer_types = config.effective_layer_types();
928        let enable_moe = config.enable_moe_block.unwrap_or(false);
929
930        let scalar_dist = Normal::new(1.0f32, 0.05f32).map_err(|err| {
931            ModelError::InvalidArgument(format!("failed to construct scalar normal dist: {err}"))
932        })?;
933
934        let mut any_sliding = false;
935        let mut layers = Vec::with_capacity(config.num_hidden_layers);
936
937        for layer_type in &layer_types {
938            let is_sliding = layer_type == "sliding_attention";
939            if is_sliding {
940                any_sliding = true;
941            }
942            let head_dim = if is_sliding {
943                config.effective_head_dim()
944            } else {
945                config.effective_global_head_dim()
946            };
947            let num_kv_heads = if is_sliding {
948                config.effective_kv_heads()
949            } else {
950                config.effective_global_kv_heads()
951            };
952            let sliding_window = if is_sliding {
953                config.sliding_window
954            } else {
955                None
956            };
957            let use_k_eq_v = !is_sliding && config.attention_k_eq_v.unwrap_or(false);
958
959            let v_proj = if use_k_eq_v {
960                None
961            } else {
962                Some(LinearLayer::new_random(
963                    LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
964                    dtype,
965                    device,
966                    rng,
967                )?)
968            };
969
970            let moe = if enable_moe {
971                Some(Gemma4MoE {
972                    post_feedforward_layernorm_1: RmsNorm::new_init(
973                        config.hidden_size,
974                        config.rms_norm_eps,
975                        NormKind::Standard,
976                        dtype,
977                        device,
978                    )?,
979                    pre_feedforward_layernorm_2: RmsNorm::new_init(
980                        config.hidden_size,
981                        config.rms_norm_eps,
982                        NormKind::Standard,
983                        dtype,
984                        device,
985                    )?,
986                    router: Gemma4Router::new_random(&config, dtype, device, rng)?,
987                    experts: Gemma4Experts::new_random(&config, dtype, device, rng)?,
988                    post_feedforward_layernorm_2: RmsNorm::new_init(
989                        config.hidden_size,
990                        config.rms_norm_eps,
991                        NormKind::Standard,
992                        dtype,
993                        device,
994                    )?,
995                })
996            } else {
997                None
998            };
999
1000            let scalar_val = scalar_dist.sample(rng);
1001            let layer_scalar =
1002                Param::Frozen(Tensor::from_slice(&[scalar_val], (1,), device)?.to_dtype(dtype)?);
1003
1004            layers.push(Gemma4DecoderLayer {
1005                input_layernorm: RmsNorm::new_init(
1006                    config.hidden_size,
1007                    config.rms_norm_eps,
1008                    NormKind::Standard,
1009                    dtype,
1010                    device,
1011                )?,
1012                self_attn: Gemma4Attention {
1013                    q_proj: LinearLayer::new_random(
1014                        LayerShape::new(config.hidden_size, num_heads * head_dim, attn_bias),
1015                        dtype,
1016                        device,
1017                        rng,
1018                    )?,
1019                    k_proj: LinearLayer::new_random(
1020                        LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
1021                        dtype,
1022                        device,
1023                        rng,
1024                    )?,
1025                    v_proj,
1026                    o_proj: LinearLayer::new_random(
1027                        LayerShape::new(num_heads * head_dim, config.hidden_size, attn_bias),
1028                        dtype,
1029                        device,
1030                        rng,
1031                    )?,
1032                    q_norm: RmsNorm::new_init(
1033                        head_dim,
1034                        config.rms_norm_eps,
1035                        NormKind::Standard,
1036                        dtype,
1037                        device,
1038                    )?,
1039                    k_norm: RmsNorm::new_init(
1040                        head_dim,
1041                        config.rms_norm_eps,
1042                        NormKind::Standard,
1043                        dtype,
1044                        device,
1045                    )?,
1046                    v_norm: RmsNormNoScale::new(config.rms_norm_eps),
1047                    num_heads,
1048                    num_kv_heads,
1049                    head_dim,
1050                    softcap: config.attn_logit_softcapping,
1051                    sliding_window,
1052                    is_sliding,
1053                },
1054                post_attention_layernorm: RmsNorm::new_init(
1055                    config.hidden_size,
1056                    config.rms_norm_eps,
1057                    NormKind::Standard,
1058                    dtype,
1059                    device,
1060                )?,
1061                pre_feedforward_layernorm: RmsNorm::new_init(
1062                    config.hidden_size,
1063                    config.rms_norm_eps,
1064                    NormKind::Standard,
1065                    dtype,
1066                    device,
1067                )?,
1068                mlp: Gemma4Mlp {
1069                    gate_proj: LinearLayer::new_random(
1070                        LayerShape::new(config.hidden_size, inter, mlp_bias),
1071                        dtype,
1072                        device,
1073                        rng,
1074                    )?,
1075                    up_proj: LinearLayer::new_random(
1076                        LayerShape::new(config.hidden_size, inter, mlp_bias),
1077                        dtype,
1078                        device,
1079                        rng,
1080                    )?,
1081                    down_proj: LinearLayer::new_random(
1082                        LayerShape::new(inter, config.hidden_size, mlp_bias),
1083                        dtype,
1084                        device,
1085                        rng,
1086                    )?,
1087                    hidden_act: hidden_act.clone(),
1088                },
1089                moe,
1090                post_feedforward_layernorm: RmsNorm::new_init(
1091                    config.hidden_size,
1092                    config.rms_norm_eps,
1093                    NormKind::Standard,
1094                    dtype,
1095                    device,
1096                )?,
1097                layer_scalar,
1098            });
1099        }
1100
1101        let norm = RmsNorm::new_init(
1102            config.hidden_size,
1103            config.rms_norm_eps,
1104            NormKind::Standard,
1105            dtype,
1106            device,
1107        )?;
1108        let lm_head = if config.tie_word_embeddings {
1109            None
1110        } else {
1111            Some(LinearLayer::new_random(
1112                LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
1113                dtype,
1114                device,
1115                rng,
1116            )?)
1117        };
1118
1119        let max_seq = config.max_position_embeddings.max(256);
1120        let rope_full = build_rope(
1121            config.effective_global_head_dim(),
1122            max_seq,
1123            config.gemma4_rope_params("full_attention"),
1124            dtype,
1125            device,
1126        )?;
1127        let rope_sliding = if any_sliding {
1128            Some(build_rope(
1129                config.effective_head_dim(),
1130                max_seq,
1131                config.gemma4_rope_params("sliding_attention"),
1132                dtype,
1133                device,
1134            )?)
1135        } else {
1136            None
1137        };
1138
1139        Ok(Self {
1140            config,
1141            weight_prefix: "model".to_string(),
1142            embed_tokens,
1143            layers,
1144            norm,
1145            lm_head,
1146            rope_full,
1147            rope_sliding,
1148        })
1149    }
1150
1151    /// Borrow a decoder layer's MoE block (useful for testing router / expert quantization states).
1152    #[must_use]
1153    pub fn layer_moe(&self, layer_idx: usize) -> Option<&Gemma4MoE> {
1154        self.layers.get(layer_idx).and_then(|l| l.moe.as_ref())
1155    }
1156
1157    /// Mutably borrow a decoder layer's MoE block (useful in unit tests).
1158    #[must_use]
1159    pub fn layer_moe_mut(&mut self, layer_idx: usize) -> Option<&mut Gemma4MoE> {
1160        self.layers.get_mut(layer_idx).and_then(|l| l.moe.as_mut())
1161    }
1162
1163    /// Execute causal forward pass over `input_ids` `(batch, seq_len)`.
1164    ///
1165    /// # Errors
1166    /// Returns an error if layer computation fails.
1167    pub fn forward(
1168        &self,
1169        input_ids: &Tensor,
1170        mut cache: Option<&mut KvCache>,
1171        hidden_mode: HiddenStatesMode,
1172    ) -> Result<CausalLmOutput> {
1173        let mut hidden = self.embed_tokens.forward(input_ids)?;
1174        let collect = hidden_mode == HiddenStatesMode::All;
1175        let mut all_hidden = if collect {
1176            Vec::with_capacity(self.layers.len() + 1)
1177        } else {
1178            Vec::new()
1179        };
1180
1181        for (idx, layer) in self.layers.iter().enumerate() {
1182            if collect {
1183                all_hidden.push(hidden.clone());
1184            }
1185            let residual = hidden.clone();
1186            let normed = layer.input_layernorm.forward(&hidden)?;
1187            let rope = if layer.self_attn.is_sliding {
1188                self.rope_sliding.as_ref().unwrap_or(&self.rope_full)
1189            } else {
1190                &self.rope_full
1191            };
1192            let attn = layer
1193                .self_attn
1194                .forward(&normed, rope, idx, cache.as_deref_mut())?;
1195            let attn_normed = layer.post_attention_layernorm.forward(&attn)?;
1196            let h_mid = residual.add(&attn_normed)?;
1197
1198            let mut next_hidden = match &layer.moe {
1199                None => {
1200                    let pre_ff = layer.pre_feedforward_layernorm.forward(&h_mid)?;
1201                    let ff_out = layer.mlp.forward(&pre_ff)?;
1202                    let ff_normed = layer.post_feedforward_layernorm.forward(&ff_out)?;
1203                    h_mid.add(&ff_normed)?
1204                }
1205                Some(moe) => {
1206                    let pre_ff = layer.pre_feedforward_layernorm.forward(&h_mid)?;
1207                    let mlp_out = layer.mlp.forward(&pre_ff)?;
1208                    let h1 = moe.post_feedforward_layernorm_1.forward(&mlp_out)?;
1209
1210                    let (b, s, d) = h_mid.dims3()?;
1211                    let flat = h_mid.reshape((b * s, d))?;
1212                    let dispatch = moe.router.forward(&flat)?;
1213                    let h2_in = moe.pre_feedforward_layernorm_2.forward(&flat)?;
1214                    let h2_experts = moe.experts.forward(&h2_in, &dispatch)?.reshape((b, s, d))?;
1215                    let h2 = moe.post_feedforward_layernorm_2.forward(&h2_experts)?;
1216
1217                    let ff_normed = layer.post_feedforward_layernorm.forward(&h1.add(&h2)?)?;
1218                    h_mid.add(&ff_normed)?
1219                }
1220            };
1221
1222            next_hidden = broadcast_scalar_mul(&next_hidden, &layer.layer_scalar)?;
1223            hidden = next_hidden;
1224        }
1225
1226        hidden = self.norm.forward(&hidden)?;
1227        if collect {
1228            all_hidden.push(hidden.clone());
1229        }
1230        let mut logits = match &self.lm_head {
1231            Some(head) => head.forward(&hidden)?,
1232            None => self.embed_tokens.forward_lm_head(&hidden)?,
1233        };
1234        if let Some(cap) = self.config.final_logit_softcapping {
1235            logits = ((logits / cap)?.tanh()? * cap)?;
1236        }
1237        Ok(CausalLmOutput {
1238            logits,
1239            hidden_states: all_hidden,
1240        })
1241    }
1242
1243    pub(crate) fn apply_littlebit_patch(
1244        &mut self,
1245        lb_cfg: &LittleBitConfig,
1246        policy: &PatchPolicy,
1247        train_mode: TrainMode,
1248        base_seed: u64,
1249        summary: &mut PatchSummary,
1250    ) -> Result<()> {
1251        for (i, layer) in self.layers.iter_mut().enumerate() {
1252            let p = format!("{}.layers.{i}", self.weight_prefix);
1253            patch_linear_layer(
1254                &format!("{p}.self_attn.q_proj"),
1255                &mut layer.self_attn.q_proj,
1256                lb_cfg,
1257                policy,
1258                train_mode,
1259                base_seed,
1260                summary,
1261            )?;
1262            patch_linear_layer(
1263                &format!("{p}.self_attn.k_proj"),
1264                &mut layer.self_attn.k_proj,
1265                lb_cfg,
1266                policy,
1267                train_mode,
1268                base_seed,
1269                summary,
1270            )?;
1271            if let Some(vp) = &mut layer.self_attn.v_proj {
1272                patch_linear_layer(
1273                    &format!("{p}.self_attn.v_proj"),
1274                    vp,
1275                    lb_cfg,
1276                    policy,
1277                    train_mode,
1278                    base_seed,
1279                    summary,
1280                )?;
1281            }
1282            patch_linear_layer(
1283                &format!("{p}.self_attn.o_proj"),
1284                &mut layer.self_attn.o_proj,
1285                lb_cfg,
1286                policy,
1287                train_mode,
1288                base_seed,
1289                summary,
1290            )?;
1291            patch_linear_layer(
1292                &format!("{p}.mlp.gate_proj"),
1293                &mut layer.mlp.gate_proj,
1294                lb_cfg,
1295                policy,
1296                train_mode,
1297                base_seed,
1298                summary,
1299            )?;
1300            patch_linear_layer(
1301                &format!("{p}.mlp.up_proj"),
1302                &mut layer.mlp.up_proj,
1303                lb_cfg,
1304                policy,
1305                train_mode,
1306                base_seed,
1307                summary,
1308            )?;
1309            patch_linear_layer(
1310                &format!("{p}.mlp.down_proj"),
1311                &mut layer.mlp.down_proj,
1312                lb_cfg,
1313                policy,
1314                train_mode,
1315                base_seed,
1316                summary,
1317            )?;
1318            if let Some(moe) = &mut layer.moe {
1319                // Passes through policy (`router` is excluded -> stays unquantized dense).
1320                patch_linear_layer(
1321                    &format!("{p}.router.proj"),
1322                    &mut moe.router.proj,
1323                    lb_cfg,
1324                    policy,
1325                    train_mode,
1326                    base_seed,
1327                    summary,
1328                )?;
1329                for (e, expert) in moe.experts.experts.iter_mut().enumerate() {
1330                    patch_linear_layer(
1331                        &format!("{p}.experts.{e}.gate_up_proj"),
1332                        &mut expert.gate_up_proj,
1333                        lb_cfg,
1334                        policy,
1335                        train_mode,
1336                        base_seed,
1337                        summary,
1338                    )?;
1339                    patch_linear_layer(
1340                        &format!("{p}.experts.{e}.down_proj"),
1341                        &mut expert.down_proj,
1342                        lb_cfg,
1343                        policy,
1344                        train_mode,
1345                        base_seed,
1346                        summary,
1347                    )?;
1348                }
1349            }
1350        }
1351        if let Some(head) = &mut self.lm_head {
1352            patch_linear_layer(
1353                "lm_head", head, lb_cfg, policy, train_mode, base_seed, summary,
1354            )?;
1355        }
1356        Ok(())
1357    }
1358
1359    pub(crate) fn convert_to_packed(&mut self) -> Result<()> {
1360        for layer in &mut self.layers {
1361            layer.self_attn.q_proj.convert_to_packed()?;
1362            layer.self_attn.k_proj.convert_to_packed()?;
1363            if let Some(vp) = &mut layer.self_attn.v_proj {
1364                vp.convert_to_packed()?;
1365            }
1366            layer.self_attn.o_proj.convert_to_packed()?;
1367            layer.mlp.gate_proj.convert_to_packed()?;
1368            layer.mlp.up_proj.convert_to_packed()?;
1369            layer.mlp.down_proj.convert_to_packed()?;
1370            if let Some(moe) = &mut layer.moe {
1371                // Keep moe.router.proj unquantized (Dense); pack all LittleBit experts.
1372                for expert in &mut moe.experts.experts {
1373                    expert.gate_up_proj.convert_to_packed()?;
1374                    expert.down_proj.convert_to_packed()?;
1375                }
1376            }
1377        }
1378        if let Some(head) = &mut self.lm_head {
1379            head.convert_to_packed()?;
1380        }
1381        Ok(())
1382    }
1383
1384    pub(crate) fn make_non_quantized_trainable(&mut self) -> Result<()> {
1385        self.embed_tokens.make_trainable()?;
1386        for layer in &mut self.layers {
1387            layer.input_layernorm.make_trainable()?;
1388            layer.self_attn.q_norm.make_trainable()?;
1389            layer.self_attn.k_norm.make_trainable()?;
1390            layer.post_attention_layernorm.make_trainable()?;
1391            layer.pre_feedforward_layernorm.make_trainable()?;
1392            layer.post_feedforward_layernorm.make_trainable()?;
1393            if let Param::Frozen(t) = &layer.layer_scalar {
1394                layer.layer_scalar = Param::Trainable(Var::from_tensor(t)?);
1395            }
1396            if let Some(moe) = &mut layer.moe {
1397                moe.post_feedforward_layernorm_1.make_trainable()?;
1398                moe.pre_feedforward_layernorm_2.make_trainable()?;
1399                moe.post_feedforward_layernorm_2.make_trainable()?;
1400                if let Param::Frozen(t) = &moe.router.scale {
1401                    moe.router.scale = Param::Trainable(Var::from_tensor(t)?);
1402                }
1403                if let Param::Frozen(t) = &moe.router.per_expert_scale {
1404                    moe.router.per_expert_scale = Param::Trainable(Var::from_tensor(t)?);
1405                }
1406                moe.router.proj.make_trainable()?;
1407            }
1408        }
1409        self.norm.make_trainable()?;
1410        if let Some(head) = &mut self.lm_head {
1411            head.make_trainable()?;
1412        }
1413        Ok(())
1414    }
1415
1416    pub(crate) fn freeze_matching(&mut self, exclude: &[&str]) {
1417        let matches = |path: &str| exclude.iter().any(|pat| path.contains(pat));
1418        let wp = &self.weight_prefix;
1419        if matches("embed_tokens") {
1420            self.embed_tokens.freeze();
1421        }
1422        if matches("lm_head") {
1423            if let Some(head) = &mut self.lm_head {
1424                head.freeze();
1425            }
1426        }
1427        if matches("norm") {
1428            self.norm.freeze();
1429        }
1430        for (i, layer) in self.layers.iter_mut().enumerate() {
1431            let p = format!("{wp}.layers.{i}");
1432            if matches(&format!("{p}.input_layernorm")) || matches("layernorm") {
1433                layer.input_layernorm.freeze();
1434            }
1435            if matches(&format!("{p}.post_attention_layernorm")) || matches("layernorm") {
1436                layer.post_attention_layernorm.freeze();
1437            }
1438            if matches(&format!("{p}.pre_feedforward_layernorm")) || matches("layernorm") {
1439                layer.pre_feedforward_layernorm.freeze();
1440            }
1441            if matches(&format!("{p}.post_feedforward_layernorm")) || matches("layernorm") {
1442                layer.post_feedforward_layernorm.freeze();
1443            }
1444            if matches(&format!("{p}.self_attn.q_norm")) || matches("norm") {
1445                layer.self_attn.q_norm.freeze();
1446            }
1447            if matches(&format!("{p}.self_attn.k_norm")) || matches("norm") {
1448                layer.self_attn.k_norm.freeze();
1449            }
1450            if matches(&format!("{p}.layer_scalar")) || matches("layer_scalar") {
1451                layer.layer_scalar.freeze();
1452            }
1453            if matches(&format!("{p}.self_attn")) {
1454                layer.self_attn.q_proj.freeze();
1455                layer.self_attn.k_proj.freeze();
1456                if let Some(vp) = &mut layer.self_attn.v_proj {
1457                    vp.freeze();
1458                }
1459                layer.self_attn.o_proj.freeze();
1460            }
1461            if matches(&format!("{p}.mlp")) {
1462                layer.mlp.gate_proj.freeze();
1463                layer.mlp.up_proj.freeze();
1464                layer.mlp.down_proj.freeze();
1465            }
1466            if let Some(moe) = &mut layer.moe {
1467                if matches(&format!("{p}.post_feedforward_layernorm_1")) || matches("layernorm") {
1468                    moe.post_feedforward_layernorm_1.freeze();
1469                }
1470                if matches(&format!("{p}.pre_feedforward_layernorm_2")) || matches("layernorm") {
1471                    moe.pre_feedforward_layernorm_2.freeze();
1472                }
1473                if matches(&format!("{p}.post_feedforward_layernorm_2")) || matches("layernorm") {
1474                    moe.post_feedforward_layernorm_2.freeze();
1475                }
1476                if matches(&format!("{p}.router")) {
1477                    moe.router.scale.freeze();
1478                    moe.router.per_expert_scale.freeze();
1479                    moe.router.proj.freeze();
1480                }
1481                if matches(&format!("{p}.experts")) {
1482                    for expert in &mut moe.experts.experts {
1483                        expert.gate_up_proj.freeze();
1484                        expert.down_proj.freeze();
1485                    }
1486                }
1487            }
1488        }
1489    }
1490
1491    pub(crate) fn trainable_vars(&self) -> Vec<Var> {
1492        let mut vars = self.embed_tokens.trainable_vars();
1493        for layer in &self.layers {
1494            vars.extend(layer.input_layernorm.trainable_vars());
1495            vars.extend(layer.self_attn.q_proj.trainable_vars());
1496            vars.extend(layer.self_attn.k_proj.trainable_vars());
1497            if let Some(vp) = &layer.self_attn.v_proj {
1498                vars.extend(vp.trainable_vars());
1499            }
1500            vars.extend(layer.self_attn.o_proj.trainable_vars());
1501            vars.extend(layer.self_attn.q_norm.trainable_vars());
1502            vars.extend(layer.self_attn.k_norm.trainable_vars());
1503            vars.extend(layer.post_attention_layernorm.trainable_vars());
1504            vars.extend(layer.pre_feedforward_layernorm.trainable_vars());
1505            vars.extend(layer.mlp.gate_proj.trainable_vars());
1506            vars.extend(layer.mlp.up_proj.trainable_vars());
1507            vars.extend(layer.mlp.down_proj.trainable_vars());
1508            if let Some(moe) = &layer.moe {
1509                vars.extend(moe.post_feedforward_layernorm_1.trainable_vars());
1510                vars.extend(moe.pre_feedforward_layernorm_2.trainable_vars());
1511                if let Some(v) = moe.router.scale.as_var() {
1512                    vars.push(v.clone());
1513                }
1514                if let Some(v) = moe.router.per_expert_scale.as_var() {
1515                    vars.push(v.clone());
1516                }
1517                vars.extend(moe.router.proj.trainable_vars());
1518                for expert in &moe.experts.experts {
1519                    vars.extend(expert.gate_up_proj.trainable_vars());
1520                    vars.extend(expert.down_proj.trainable_vars());
1521                }
1522                vars.extend(moe.post_feedforward_layernorm_2.trainable_vars());
1523            }
1524            vars.extend(layer.post_feedforward_layernorm.trainable_vars());
1525            if let Some(v) = layer.layer_scalar.as_var() {
1526                vars.push(v.clone());
1527            }
1528        }
1529        vars.extend(self.norm.trainable_vars());
1530        if let Some(head) = &self.lm_head {
1531            vars.extend(head.trainable_vars());
1532        }
1533        vars
1534    }
1535
1536    pub(crate) fn total_params(&self) -> usize {
1537        let mut count = self.embed_tokens.weight.elem_count() + self.norm.weight.elem_count();
1538        for layer in &self.layers {
1539            count += layer.input_layernorm.weight.elem_count()
1540                + layer.post_attention_layernorm.weight.elem_count()
1541                + layer.pre_feedforward_layernorm.weight.elem_count()
1542                + layer.post_feedforward_layernorm.weight.elem_count()
1543                + layer.layer_scalar.elem_count()
1544                + layer.self_attn.q_proj.param_count()
1545                + layer.self_attn.k_proj.param_count()
1546                + layer
1547                    .self_attn
1548                    .v_proj
1549                    .as_ref()
1550                    .map_or(0, LinearLayer::param_count)
1551                + layer.self_attn.o_proj.param_count()
1552                + layer.self_attn.q_norm.weight.elem_count()
1553                + layer.self_attn.k_norm.weight.elem_count()
1554                + layer.mlp.gate_proj.param_count()
1555                + layer.mlp.up_proj.param_count()
1556                + layer.mlp.down_proj.param_count();
1557            if let Some(moe) = &layer.moe {
1558                count += moe.post_feedforward_layernorm_1.weight.elem_count()
1559                    + moe.pre_feedforward_layernorm_2.weight.elem_count()
1560                    + moe.post_feedforward_layernorm_2.weight.elem_count()
1561                    + moe.router.scale.elem_count()
1562                    + moe.router.per_expert_scale.elem_count()
1563                    + moe.router.proj.param_count();
1564                for expert in &moe.experts.experts {
1565                    count += expert.gate_up_proj.param_count() + expert.down_proj.param_count();
1566                }
1567            }
1568        }
1569        if let Some(head) = &self.lm_head {
1570            count += head.param_count();
1571        }
1572        count
1573    }
1574
1575    pub(crate) fn pack_state_dict(&self) -> Result<HashMap<String, Tensor>> {
1576        let mut out = HashMap::new();
1577        let wp = &self.weight_prefix;
1578        self.embed_tokens
1579            .pack_state_dict(&format!("{wp}.embed_tokens"), &mut out);
1580        for (i, layer) in self.layers.iter().enumerate() {
1581            let p = format!("{wp}.layers.{i}");
1582            layer
1583                .input_layernorm
1584                .pack_state_dict(&format!("{p}.input_layernorm"), &mut out);
1585            out.extend(
1586                layer
1587                    .self_attn
1588                    .q_proj
1589                    .pack_state_dict(&format!("{p}.self_attn.q_proj"))?,
1590            );
1591            out.extend(
1592                layer
1593                    .self_attn
1594                    .k_proj
1595                    .pack_state_dict(&format!("{p}.self_attn.k_proj"))?,
1596            );
1597            if let Some(vp) = &layer.self_attn.v_proj {
1598                out.extend(vp.pack_state_dict(&format!("{p}.self_attn.v_proj"))?);
1599            }
1600            out.extend(
1601                layer
1602                    .self_attn
1603                    .o_proj
1604                    .pack_state_dict(&format!("{p}.self_attn.o_proj"))?,
1605            );
1606            layer
1607                .self_attn
1608                .q_norm
1609                .pack_state_dict(&format!("{p}.self_attn.q_norm"), &mut out);
1610            layer
1611                .self_attn
1612                .k_norm
1613                .pack_state_dict(&format!("{p}.self_attn.k_norm"), &mut out);
1614            layer
1615                .post_attention_layernorm
1616                .pack_state_dict(&format!("{p}.post_attention_layernorm"), &mut out);
1617            layer
1618                .pre_feedforward_layernorm
1619                .pack_state_dict(&format!("{p}.pre_feedforward_layernorm"), &mut out);
1620            out.extend(
1621                layer
1622                    .mlp
1623                    .gate_proj
1624                    .pack_state_dict(&format!("{p}.mlp.gate_proj"))?,
1625            );
1626            out.extend(
1627                layer
1628                    .mlp
1629                    .up_proj
1630                    .pack_state_dict(&format!("{p}.mlp.up_proj"))?,
1631            );
1632            out.extend(
1633                layer
1634                    .mlp
1635                    .down_proj
1636                    .pack_state_dict(&format!("{p}.mlp.down_proj"))?,
1637            );
1638            if let Some(moe) = &layer.moe {
1639                moe.post_feedforward_layernorm_1
1640                    .pack_state_dict(&format!("{p}.post_feedforward_layernorm_1"), &mut out);
1641                moe.pre_feedforward_layernorm_2
1642                    .pack_state_dict(&format!("{p}.pre_feedforward_layernorm_2"), &mut out);
1643                out.insert(
1644                    format!("{p}.router.scale"),
1645                    moe.router.scale.as_tensor().detach(),
1646                );
1647                out.insert(
1648                    format!("{p}.router.per_expert_scale"),
1649                    moe.router.per_expert_scale.as_tensor().detach(),
1650                );
1651                out.extend(
1652                    moe.router
1653                        .proj
1654                        .pack_state_dict(&format!("{p}.router.proj"))?,
1655                );
1656                for (e, expert) in moe.experts.experts.iter().enumerate() {
1657                    out.extend(
1658                        expert
1659                            .gate_up_proj
1660                            .pack_state_dict(&format!("{p}.experts.{e}.gate_up_proj"))?,
1661                    );
1662                    out.extend(
1663                        expert
1664                            .down_proj
1665                            .pack_state_dict(&format!("{p}.experts.{e}.down_proj"))?,
1666                    );
1667                }
1668                moe.post_feedforward_layernorm_2
1669                    .pack_state_dict(&format!("{p}.post_feedforward_layernorm_2"), &mut out);
1670            }
1671            layer
1672                .post_feedforward_layernorm
1673                .pack_state_dict(&format!("{p}.post_feedforward_layernorm"), &mut out);
1674            out.insert(
1675                format!("{p}.layer_scalar"),
1676                layer.layer_scalar.as_tensor().detach(),
1677            );
1678        }
1679        self.norm.pack_state_dict(&format!("{wp}.norm"), &mut out);
1680        if let Some(head) = &self.lm_head {
1681            out.extend(head.pack_state_dict("lm_head")?);
1682        }
1683        Ok(out)
1684    }
1685
1686    pub(crate) fn config(&self) -> &ModelConfig {
1687        &self.config
1688    }
1689}
1690
1691#[cfg(test)]
1692mod tests {
1693    use super::*;
1694    use crate::config::ModelArchitecture;
1695    use crate::hub::{LittleBitModel, LoadOptions};
1696    use crate::model::CausalLm;
1697    use littlebit_core::{CastPolicy, InferenceMode};
1698    use rand::SeedableRng;
1699    use tempfile::TempDir;
1700
1701    #[test]
1702    fn multimodal_gemma4_config_unwraps_text_config_and_prefixes() -> Result<()> {
1703        let dir = TempDir::new().expect("tempdir");
1704        let cfg_json = serde_json::json!({
1705            "model_type": "gemma4",
1706            "architectures": ["Gemma4ForConditionalGeneration"],
1707            "tie_word_embeddings": true,
1708            "text_config": {
1709                "vocab_size": 128,
1710                "hidden_size": 64,
1711                "intermediate_size": 128,
1712                "num_hidden_layers": 2,
1713                "num_attention_heads": 4,
1714                "num_key_value_heads": 2,
1715                "head_dim": 16,
1716                "global_head_dim": 32,
1717                "num_global_key_value_heads": 1,
1718                "attention_k_eq_v": true,
1719                "sliding_window": 4,
1720                "layer_types": ["sliding_attention", "full_attention"],
1721                "rms_norm_eps": 1e-6,
1722                "hidden_activation": "gelu_pytorch_tanh",
1723                "rope_parameters": {
1724                    "sliding_attention": { "rope_type": "default", "rope_theta": 10000.0 },
1725                    "full_attention": {
1726                        "rope_type": "proportional",
1727                        "partial_rotary_factor": 0.25,
1728                        "rope_theta": 1000000.0
1729                    }
1730                }
1731            }
1732        });
1733        std::fs::write(
1734            dir.path().join("config.json"),
1735            serde_json::to_string_pretty(&cfg_json)?,
1736        )?;
1737        let parsed = ModelConfig::from_file(&dir.path().join("config.json"))?;
1738        assert_eq!(parsed.detect_architecture()?, ModelArchitecture::Gemma4);
1739        assert!(parsed.tie_word_embeddings);
1740        assert_eq!(parsed.effective_global_head_dim(), 32);
1741        assert_eq!(parsed.effective_global_kv_heads(), 1);
1742        assert_eq!(parsed.effective_hidden_act(), "gelu_pytorch_tanh");
1743
1744        let full_rope = parsed.gemma4_rope_params("full_attention");
1745        assert!(full_rope.is_proportional);
1746        assert!((full_rope.partial_rotary_factor - 0.25).abs() < 1e-12);
1747        assert!((full_rope.rope_theta - 1_000_000.0).abs() < 1e-12);
1748
1749        // Verify multimodal weight prefix (`model.language_model.*`) loads cleanly via remapped keys.
1750        let dev = Device::Cpu;
1751        let base_model = CausalLm::new_random(parsed.clone(), DType::F32, &dev, 9)?;
1752        let raw_sd = base_model.pack_state_dict()?;
1753        let mut mm_sd = HashMap::new();
1754        for (k, v) in raw_sd {
1755            let new_k = if let Some(rest) = k.strip_prefix("model.") {
1756                format!("model.language_model.{rest}")
1757            } else {
1758                k
1759            };
1760            mm_sd.insert(new_k, v);
1761        }
1762        let mm_model = CausalLm::from_tensors(
1763            parsed,
1764            &mm_sd,
1765            None,
1766            littlebit_core::WeightState::Raw,
1767            InferenceMode::Unpacked,
1768            DType::F32,
1769            &dev,
1770        )?;
1771        let ids = Tensor::from_slice(&[1u32, 3, 5], (1, 3), &dev)?;
1772        let out_a = base_model.forward(&ids, None, HiddenStatesMode::None)?;
1773        let out_b = mm_model.forward(&ids, None, HiddenStatesMode::None)?;
1774        let diff = out_a
1775            .logits
1776            .sub(&out_b.logits)?
1777            .abs()?
1778            .max_all()?
1779            .to_scalar::<f32>()?;
1780        assert!(diff < 1e-6, "multimodal prefix drift: {diff}");
1781        Ok(())
1782    }
1783
1784    #[test]
1785    fn proportional_rope_preserves_nope_subspace_and_scales_angles_by_full_head_dim() -> Result<()>
1786    {
1787        let dev = Device::Cpu;
1788        let head_dim = 16;
1789        let rope = RotaryEmbedding::new_with_config(
1790            head_dim,
1791            32,
1792            10_000.0,
1793            0.25,
1794            1.0,
1795            RopeMode::Proportional,
1796            DType::F32,
1797            &dev,
1798        )?;
1799        let x = Tensor::ones((1, 2, 5, head_dim), DType::F32, &dev)?;
1800        let out = rope.apply(&x, 3)?;
1801
1802        let half = head_dim / 2;
1803        let rope_angles = 2; // 0.25 * 16 / 2
1804        let vals = out.get(0)?.to_vec3::<f32>()?;
1805        let inv_freq_1 = 1.0f64 / 10_000.0f64.powf(2.0 / 16.0);
1806        for head in &vals {
1807            for (t, token) in head.iter().enumerate() {
1808                // Non-rotated NOPE pairs (`rope_angles..half` and `half+rope_angles..head_dim`) stay 1.0.
1809                for d in rope_angles..half {
1810                    assert!(
1811                        (token[d] - 1.0).abs() < 1e-6,
1812                        "NOPE dim {d} modified: {}",
1813                        token[d]
1814                    );
1815                    assert!(
1816                        (token[half + d] - 1.0).abs() < 1e-6,
1817                        "NOPE pair dim {} modified: {}",
1818                        half + d,
1819                        token[half + d]
1820                    );
1821                }
1822                // Check angle index `i = 1` at `pos = 3 + t` uses `denom = head_dim (16)`, not `rotary_dim (4)`.
1823                let pos = (3 + t) as f64;
1824                let expected_dim1 = (pos * inv_freq_1).cos() - (pos * inv_freq_1).sin();
1825                assert!(
1826                    (f64::from(token[1]) - expected_dim1).abs() < 1e-5,
1827                    "p-RoPE angle mismatch at pos {pos}: got {}, expected {expected_dim1}",
1828                    token[1]
1829                );
1830            }
1831        }
1832        Ok(())
1833    }
1834
1835    #[test]
1836    fn gemma4_dense_k_eq_v_and_sliding_window_kv_cache_parity() -> Result<()> {
1837        let dev = Device::Cpu;
1838        let mut cfg = ModelConfig::tiny_gemma4_dense(96, 64, 2);
1839        // Set small sliding window = 4 and test 7 tokens to exercise sliding eviction + seq_pos RoPE!
1840        cfg.sliding_window = Some(4);
1841
1842        let mut model = CausalLm::new_random(cfg.clone(), DType::F32, &dev, 41)?;
1843        if let CausalLm::Gemma4(ref g4) = model {
1844            assert!(g4.layers[0].self_attn.is_sliding);
1845            assert!(g4.layers[0].self_attn.v_proj.is_some());
1846            assert!(!g4.layers[1].self_attn.is_sliding);
1847            assert!(
1848                g4.layers[1].self_attn.v_proj.is_none(),
1849                "full_attention with attention_k_eq_v must omit v_proj"
1850            );
1851        } else {
1852            panic!("expected Gemma4");
1853        }
1854
1855        let lb_cfg = LittleBitConfig {
1856            eff_bit: Some(1.0),
1857            use_itq: true,
1858            itq_n_iter: 5,
1859            ..Default::default()
1860        };
1861        model.apply_littlebit_patch(&lb_cfg, TrainMode::Eval, 19)?;
1862
1863        let prompt = [2u32, 7, 15, 3, 21, 9, 11];
1864        let full_t = Tensor::from_slice(&prompt, (1, prompt.len()), &dev)?;
1865        let full_logits = model
1866            .forward(&full_t, None, HiddenStatesMode::None)?
1867            .logits
1868            .get(0)?
1869            .get(prompt.len() - 1)?;
1870
1871        let mut cache = KvCache::new(cfg.num_hidden_layers);
1872        let mut step_logits = None;
1873        for &tok in &prompt {
1874            let tok_t = Tensor::from_slice(&[tok], (1, 1), &dev)?;
1875            let out = model.forward(&tok_t, Some(&mut cache), HiddenStatesMode::None)?;
1876            step_logits = Some(out.logits.get(0)?.get(0)?);
1877        }
1878        assert_eq!(cache.seq_pos(0), prompt.len());
1879        assert_eq!(cache.past_len(0), 4);
1880        assert_eq!(cache.past_len(1), prompt.len());
1881
1882        let diff = full_logits
1883            .sub(&step_logits.expect("last step logits"))?
1884            .abs()?
1885            .max_all()?
1886            .to_scalar::<f32>()?;
1887        assert!(
1888            diff < 1e-4,
1889            "Gemma 4 sliding+full KV cache parity failed beyond window: diff={diff}"
1890        );
1891        Ok(())
1892    }
1893
1894    #[test]
1895    fn gemma4_moe_sparse_dispatch_matches_dense_reference_and_keeps_router_dense() -> Result<()> {
1896        let dev = Device::Cpu;
1897        let cfg = ModelConfig::tiny_gemma4_moe(96, 64, 2, 16, 8);
1898        let mut model = CausalLm::new_random(cfg.clone(), DType::F32, &dev, 77)?;
1899
1900        // Verify sparse top-k dispatch mathematically against explicit dense all-expert loop.
1901        if let CausalLm::Gemma4(ref g4) = model {
1902            let moe = g4.layer_moe(0).expect("moe layer 0");
1903            let x = Tensor::randn(0.0f32, 0.5, (5, cfg.hidden_size), &dev)?;
1904            let dispatch = moe.router.forward(&x)?;
1905            assert_eq!(dispatch.top_k_indices.len(), 5);
1906            assert!(dispatch.top_k_indices.iter().all(|row| row.len() == 8));
1907
1908            let sparse_out = moe.experts.forward(&x, &dispatch)?;
1909
1910            let inter = moe.experts.intermediate_size;
1911            let mut ref_rows = Vec::with_capacity(5);
1912            for (t, (eids, weights)) in dispatch
1913                .top_k_indices
1914                .iter()
1915                .zip(dispatch.top_k_weights.iter())
1916                .enumerate()
1917            {
1918                let x_tok = x.narrow(0, t, 1)?;
1919                let mut row_sum = Tensor::zeros_like(&x_tok)?;
1920                for (&eid, &w) in eids.iter().zip(weights.iter()) {
1921                    let exp = &moe.experts.experts[eid];
1922                    let gu = exp.gate_up_proj.forward(&x_tok)?;
1923                    let g = gu.narrow(1, 0, inter)?;
1924                    let u = gu.narrow(1, inter, inter)?;
1925                    let act = apply_activation(&g, &moe.experts.hidden_act)?.mul(&u)?;
1926                    let y = (exp.down_proj.forward(&act)? * f64::from(w))?;
1927                    row_sum = row_sum.add(&y)?;
1928                }
1929                ref_rows.push(row_sum);
1930            }
1931            let dense_ref = Tensor::cat(&ref_rows, 0)?;
1932            let diff = sparse_out
1933                .sub(&dense_ref)?
1934                .abs()?
1935                .max_all()?
1936                .to_scalar::<f32>()?;
1937            assert!(diff < 1e-5, "sparse vs reference expert sum diff: {diff}");
1938        }
1939
1940        // Quantize and verify router stays LinearLayer::Dense while experts become LittleBit / Packed!
1941        let lb_cfg = LittleBitConfig {
1942            eff_bit: Some(1.0),
1943            residual: true,
1944            use_itq: true,
1945            itq_n_iter: 5,
1946            ..Default::default()
1947        };
1948        let summary = model.apply_littlebit_patch(&lb_cfg, TrainMode::Eval, 23)?;
1949        assert!(summary.patched_layers > 0);
1950        assert!(
1951            summary.skipped_layers >= cfg.num_hidden_layers,
1952            "all router projections must be recorded as skipped"
1953        );
1954        if let CausalLm::Gemma4(ref g4) = model {
1955            for layer_idx in 0..cfg.num_hidden_layers {
1956                let moe = g4.layer_moe(layer_idx).expect("moe");
1957                assert!(
1958                    matches!(moe.router.proj, LinearLayer::Dense { .. }),
1959                    "router at layer {layer_idx} must remain Dense"
1960                );
1961                assert!(
1962                    matches!(
1963                        moe.experts.experts[0].gate_up_proj,
1964                        LinearLayer::LittleBit(_)
1965                    ),
1966                    "experts must be quantized to LittleBit"
1967                );
1968            }
1969        }
1970
1971        let input = Tensor::from_slice(&[1u32, 4, 9, 12], (1, 4), &dev)?;
1972        let out_unpacked = model.forward(&input, None, HiddenStatesMode::None)?;
1973
1974        // Save, reload both Unpacked and Packed, verify bit-tight logits match and KV-cache decode parity.
1975        let dir = TempDir::new().expect("tempdir");
1976        let wrapper = LittleBitModel::new(model, lb_cfg, Some("test/gemma4-moe".to_string()), None);
1977        wrapper.save_pretrained(dir.path(), None, CastPolicy::Preserve)?;
1978
1979        let reloaded_packed = LittleBitModel::from_pretrained(
1980            dir.path().to_str().expect("utf8"),
1981            &LoadOptions {
1982                inference_mode: InferenceMode::Packed,
1983                ..Default::default()
1984            },
1985        )?;
1986        if let CausalLm::Gemma4(ref g4) = reloaded_packed.model {
1987            let moe = g4.layer_moe(0).expect("moe");
1988            assert!(matches!(moe.router.proj, LinearLayer::Dense { .. }));
1989            assert!(matches!(
1990                moe.experts.experts[0].gate_up_proj,
1991                LinearLayer::Packed(_)
1992            ));
1993        }
1994
1995        let out_packed = reloaded_packed
1996            .model
1997            .forward(&input, None, HiddenStatesMode::None)?;
1998        let drift = out_unpacked
1999            .logits
2000            .sub(&out_packed.logits)?
2001            .abs()?
2002            .max_all()?
2003            .to_scalar::<f32>()?;
2004        assert!(drift < 1e-4, "Gemma 4 MoE packed reload drift: {drift}");
2005
2006        // KV cache step-by-step parity on Packed Gemma 4 MoE.
2007        let last_full = out_packed.logits.get(0)?.get(3)?;
2008        let mut cache = KvCache::new(cfg.num_hidden_layers);
2009        let mut last_step = None;
2010        for tok in [1u32, 4, 9, 12] {
2011            let t = Tensor::from_slice(&[tok], (1, 1), &dev)?;
2012            last_step = Some(
2013                reloaded_packed
2014                    .model
2015                    .forward(&t, Some(&mut cache), HiddenStatesMode::None)?
2016                    .logits
2017                    .get(0)?
2018                    .get(0)?,
2019            );
2020        }
2021        let kv_drift = last_full
2022            .sub(&last_step.expect("last step"))?
2023            .abs()?
2024            .max_all()?
2025            .to_scalar::<f32>()?;
2026        assert!(kv_drift < 1e-4, "Packed Gemma 4 MoE KV drift: {kv_drift}");
2027
2028        Ok(())
2029    }
2030
2031    #[test]
2032    fn gemma4_moe_loads_hf_fused_3d_expert_tensors_and_skips_unrouted_experts() -> Result<()> {
2033        let dev = Device::Cpu;
2034        let cfg = ModelConfig::tiny_gemma4_moe(64, 32, 1, 6, 2);
2035        let mut rng = ChaCha8Rng::seed_from_u64(99);
2036        let model = Gemma4Model::new_random(cfg.clone(), DType::F32, &dev, &mut rng)?;
2037        let mut sd = model.pack_state_dict()?;
2038
2039        // Collapse per-expert keys into Hugging Face 3D fused `experts.gate_up_proj` & `experts.down_proj`.
2040        let mut gu_slices = Vec::new();
2041        let mut dn_slices = Vec::new();
2042        for e in 0..6 {
2043            let gu = sd
2044                .remove(&format!("model.layers.0.experts.{e}.gate_up_proj.weight"))
2045                .expect("expert gu");
2046            let dn = sd
2047                .remove(&format!("model.layers.0.experts.{e}.down_proj.weight"))
2048                .expect("expert dn");
2049            gu_slices.push(gu);
2050            dn_slices.push(dn);
2051        }
2052        sd.insert(
2053            "model.layers.0.experts.gate_up_proj".to_string(),
2054            Tensor::stack(&gu_slices, 0)?,
2055        );
2056        sd.insert(
2057            "model.layers.0.experts.down_proj".to_string(),
2058            Tensor::stack(&dn_slices, 0)?,
2059        );
2060
2061        let loaded = Gemma4Model::from_tensors(
2062            cfg.clone(),
2063            &sd,
2064            LinearLoadSpec {
2065                littlebit: None,
2066                ratio_factor: 1.0,
2067                weight_state: littlebit_core::WeightState::Raw,
2068                inference_mode: InferenceMode::Unpacked,
2069                dtype: DType::F32,
2070                device: &dev,
2071            },
2072        )?;
2073
2074        let ids = Tensor::from_slice(&[1u32, 3, 5], (1, 3), &dev)?;
2075        let out_orig = model.forward(&ids, None, HiddenStatesMode::None)?;
2076        let out_3d = loaded.forward(&ids, None, HiddenStatesMode::None)?;
2077        let diff = out_orig
2078            .logits
2079            .sub(&out_3d.logits)?
2080            .abs()?
2081            .max_all()?
2082            .to_scalar::<f32>()?;
2083        assert!(diff < 1e-6, "HF 3D fused experts load mismatch: {diff}");
2084
2085        // Also verify unrouted experts are never touched: inject NaNs into an expert that is not
2086        // selected for a single token forward and check finite output.
2087        let mut model_sparse = loaded;
2088        let single_tok =
2089            Tensor::from_vec(vec![0.25f32; cfg.hidden_size], (1, cfg.hidden_size), &dev)?;
2090        let moe = model_sparse.layer_moe(0).expect("moe");
2091        let dispatch = moe.router.forward(&single_tok)?;
2092        let chosen = &dispatch.top_k_indices[0];
2093        let unused = (0..6)
2094            .find(|eid| !chosen.contains(eid))
2095            .expect("at least one unchosen expert");
2096
2097        let moe_mut = model_sparse.layer_moe_mut(0).expect("moe mut");
2098        let nan_shape = (
2099            2 * cfg.moe_intermediate_size.expect("moe inter"),
2100            cfg.hidden_size,
2101        );
2102        let nan_w = Tensor::from_vec(vec![f32::NAN; nan_shape.0 * nan_shape.1], nan_shape, &dev)?;
2103        moe_mut.experts.experts[unused].gate_up_proj = LinearLayer::Dense {
2104            weight: Param::Frozen(nan_w),
2105            bias: None,
2106        };
2107        let sparse_out = moe_mut.experts.forward(&single_tok, &dispatch)?;
2108        let sum_val = sparse_out.sum_all()?.to_scalar::<f32>()?;
2109        assert!(
2110            sum_val.is_finite(),
2111            "sparse dispatch touched unrouted expert {unused} (produced non-finite output)"
2112        );
2113
2114        Ok(())
2115    }
2116}