1use std::collections::{BTreeMap, HashMap};
13
14use candle_core::{DType, Device, Tensor, Var, D};
15use rand_chacha::ChaCha8Rng;
16use rand_distr::{Distribution, Normal};
17
18use littlebit_core::{BiasMode, LayerShape, LittleBitConfig, Param, TrainMode};
19
20use crate::config::{ModelConfig, ResolvedRopeParams};
21use crate::error::{ModelError, Result};
22use crate::layer::{
23 apply_activation, compute_attention, move_cast, AttentionParams, Embedding, KvCache,
24 LinearLayer, LinearLoadSpec, NormKind, RmsNorm, RmsNormNoScale, RopeMode, RotaryEmbedding,
25};
26use crate::model::{CausalLmOutput, HiddenStatesMode};
27use crate::patch::{patch_linear_layer, PatchPolicy, PatchSummary};
28
29#[derive(Debug, Clone)]
31pub struct RouterDispatch {
32 pub top_k_indices: Vec<Vec<usize>>,
34 pub top_k_weights: Vec<Vec<f32>>,
36}
37
38#[derive(Debug, Clone)]
43pub struct Gemma4Router {
44 norm: RmsNormNoScale,
45 scale: Param,
46 pub proj: LinearLayer,
48 per_expert_scale: Param,
49 scalar_root_size: f64,
50 num_experts: usize,
51 top_k: usize,
52}
53
54impl Gemma4Router {
55 pub fn from_tensors(
60 prefix: &str,
61 config: &ModelConfig,
62 tensors: &HashMap<String, Tensor>,
63 spec: LinearLoadSpec<'_>,
64 ) -> Result<Self> {
65 let num_experts = config.num_experts.ok_or_else(|| {
66 ModelError::InvalidArgument("Gemma 4 MoE requires config.num_experts".to_string())
67 })?;
68 let top_k = config.top_k_experts.unwrap_or(8).min(num_experts).max(1);
69
70 let scale_key = format!("{prefix}.scale");
71 let scale_t = tensors
72 .get(&scale_key)
73 .ok_or_else(|| ModelError::MissingWeight(scale_key.clone()))?;
74 let scale = Param::Frozen(move_cast(scale_t, spec.dtype, spec.device)?);
75
76 let per_expert_key = format!("{prefix}.per_expert_scale");
77 let per_expert_t = tensors
78 .get(&per_expert_key)
79 .ok_or_else(|| ModelError::MissingWeight(per_expert_key.clone()))?;
80 let per_expert_scale = Param::Frozen(move_cast(per_expert_t, spec.dtype, spec.device)?);
81
82 let router_spec = LinearLoadSpec {
84 littlebit: None,
85 ratio_factor: 1.0,
86 ..spec
87 };
88 let proj = LinearLayer::from_tensors(
89 &format!("{prefix}.proj"),
90 tensors,
91 LayerShape::new(config.hidden_size, num_experts, BiasMode::NoBias),
92 router_spec,
93 )?;
94
95 Ok(Self {
96 norm: RmsNormNoScale::new(config.rms_norm_eps),
97 scale,
98 proj,
99 per_expert_scale,
100 scalar_root_size: (config.hidden_size as f64).powf(-0.5),
101 num_experts,
102 top_k,
103 })
104 }
105
106 pub fn new_random(
111 config: &ModelConfig,
112 dtype: DType,
113 device: &Device,
114 rng: &mut ChaCha8Rng,
115 ) -> Result<Self> {
116 let num_experts = config.num_experts.unwrap_or(8).max(1);
117 let top_k = config.top_k_experts.unwrap_or(2).min(num_experts).max(1);
118 Ok(Self {
119 norm: RmsNormNoScale::new(config.rms_norm_eps),
120 scale: Param::Frozen(Tensor::ones(config.hidden_size, dtype, device)?),
121 proj: LinearLayer::new_random(
122 LayerShape::new(config.hidden_size, num_experts, BiasMode::NoBias),
123 dtype,
124 device,
125 rng,
126 )?,
127 per_expert_scale: Param::Frozen(Tensor::ones(num_experts, dtype, device)?),
128 scalar_root_size: (config.hidden_size as f64).powf(-0.5),
129 num_experts,
130 top_k,
131 })
132 }
133
134 #[must_use]
136 pub const fn num_experts(&self) -> usize {
137 self.num_experts
138 }
139
140 #[must_use]
142 pub const fn top_k(&self) -> usize {
143 self.top_k
144 }
145
146 pub fn forward(&self, hidden_flat: &Tensor) -> Result<RouterDispatch> {
151 let normed = self.norm.forward(hidden_flat)?;
152 let scale_raw = self.scale.as_tensor();
153 let scale_t = if scale_raw.dtype() == normed.dtype() {
154 scale_raw.clone()
155 } else {
156 scale_raw.to_dtype(normed.dtype())?
157 };
158 let scaled = (normed * self.scalar_root_size)?.broadcast_mul(&scale_t)?;
159 let logits = self.proj.forward(&scaled)?.to_dtype(DType::F32)?;
160 let probs = candle_nn::ops::softmax(&logits, D::Minus1)?;
161
162 let probs_vec = probs.to_vec2::<f32>()?;
163 let per_expert = self
164 .per_expert_scale
165 .as_tensor()
166 .to_dtype(DType::F32)?
167 .to_vec1::<f32>()?;
168
169 let mut top_k_indices = Vec::with_capacity(probs_vec.len());
170 let mut top_k_weights = Vec::with_capacity(probs_vec.len());
171
172 for row in probs_vec {
173 let mut ranked: Vec<(usize, f32)> = row.into_iter().enumerate().collect();
174 ranked.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
175 ranked.truncate(self.top_k);
176
177 let denom: f32 = ranked.iter().map(|&(_, p)| p).sum();
178 let mut idxs = Vec::with_capacity(self.top_k);
179 let mut weights = Vec::with_capacity(self.top_k);
180 for (eid, p) in ranked {
181 let norm_w = if denom > 0.0 {
182 p / denom
183 } else {
184 1.0 / (self.top_k as f32)
185 };
186 let expert_scale = per_expert.get(eid).copied().unwrap_or(1.0);
187 idxs.push(eid);
188 weights.push(norm_w * expert_scale);
189 }
190 top_k_indices.push(idxs);
191 top_k_weights.push(weights);
192 }
193
194 Ok(RouterDispatch {
195 top_k_indices,
196 top_k_weights,
197 })
198 }
199}
200
201#[derive(Debug, Clone)]
203pub struct Gemma4Expert {
204 pub gate_up_proj: LinearLayer,
206 pub down_proj: LinearLayer,
208}
209
210#[derive(Debug, Clone)]
212pub struct Gemma4Experts {
213 pub experts: Vec<Gemma4Expert>,
215 intermediate_size: usize,
216 hidden_act: String,
217}
218
219impl Gemma4Experts {
220 pub fn from_tensors(
226 prefix: &str,
227 config: &ModelConfig,
228 tensors: &HashMap<String, Tensor>,
229 spec: LinearLoadSpec<'_>,
230 ) -> Result<Self> {
231 let num_experts = config.num_experts.ok_or_else(|| {
232 ModelError::InvalidArgument("Gemma 4 MoE requires config.num_experts".to_string())
233 })?;
234 let inter = config
235 .moe_intermediate_size
236 .unwrap_or_else(|| config.effective_intermediate_size());
237
238 let has_per_expert_keys = tensors
239 .contains_key(&format!("{prefix}.0.gate_up_proj.U_packed"))
240 || tensors.contains_key(&format!("{prefix}.0.gate_up_proj.U"))
241 || tensors.contains_key(&format!("{prefix}.0.gate_up_proj.weight"));
242
243 let mut experts = Vec::with_capacity(num_experts);
244 if has_per_expert_keys {
245 for e in 0..num_experts {
246 let gate_up_proj = LinearLayer::from_tensors(
247 &format!("{prefix}.{e}.gate_up_proj"),
248 tensors,
249 LayerShape::new(config.hidden_size, 2 * inter, BiasMode::NoBias),
250 spec,
251 )?;
252 let down_proj = LinearLayer::from_tensors(
253 &format!("{prefix}.{e}.down_proj"),
254 tensors,
255 LayerShape::new(inter, config.hidden_size, BiasMode::NoBias),
256 spec,
257 )?;
258 experts.push(Gemma4Expert {
259 gate_up_proj,
260 down_proj,
261 });
262 }
263 } else {
264 let gu_key = format!("{prefix}.gate_up_proj");
265 let dn_key = format!("{prefix}.down_proj");
266 let gu_3d = tensors
267 .get(&gu_key)
268 .ok_or_else(|| ModelError::MissingWeight(gu_key.clone()))?;
269 let dn_3d = tensors
270 .get(&dn_key)
271 .ok_or_else(|| ModelError::MissingWeight(dn_key.clone()))?;
272
273 let gu_dims = gu_3d.dims();
274 let dn_dims = dn_3d.dims();
275 if gu_dims.len() != 3 || gu_dims[0] != num_experts {
276 return Err(ModelError::InvalidArgument(format!(
277 "expected `{gu_key}` with shape [{num_experts}, 2*{inter}, {}], got {gu_dims:?}",
278 config.hidden_size
279 )));
280 }
281 if dn_dims.len() != 3 || dn_dims[0] != num_experts {
282 return Err(ModelError::InvalidArgument(format!(
283 "expected `{dn_key}` with shape [{num_experts}, {}, {inter}], got {dn_dims:?}",
284 config.hidden_size
285 )));
286 }
287
288 for e in 0..num_experts {
289 let w_gu = move_cast(&gu_3d.get(e)?.contiguous()?, spec.dtype, spec.device)?;
290 let w_dn = move_cast(&dn_3d.get(e)?.contiguous()?, spec.dtype, spec.device)?;
291 experts.push(Gemma4Expert {
292 gate_up_proj: LinearLayer::Dense {
293 weight: Param::Frozen(w_gu),
294 bias: None,
295 },
296 down_proj: LinearLayer::Dense {
297 weight: Param::Frozen(w_dn),
298 bias: None,
299 },
300 });
301 }
302 }
303
304 Ok(Self {
305 experts,
306 intermediate_size: inter,
307 hidden_act: config.effective_hidden_act().to_string(),
308 })
309 }
310
311 pub fn new_random(
316 config: &ModelConfig,
317 dtype: DType,
318 device: &Device,
319 rng: &mut ChaCha8Rng,
320 ) -> Result<Self> {
321 let num_experts = config.num_experts.unwrap_or(8).max(1);
322 let inter = config
323 .moe_intermediate_size
324 .unwrap_or_else(|| config.effective_intermediate_size());
325 let mut experts = Vec::with_capacity(num_experts);
326 for _ in 0..num_experts {
327 experts.push(Gemma4Expert {
328 gate_up_proj: LinearLayer::new_random(
329 LayerShape::new(config.hidden_size, 2 * inter, BiasMode::NoBias),
330 dtype,
331 device,
332 rng,
333 )?,
334 down_proj: LinearLayer::new_random(
335 LayerShape::new(inter, config.hidden_size, BiasMode::NoBias),
336 dtype,
337 device,
338 rng,
339 )?,
340 });
341 }
342 Ok(Self {
343 experts,
344 intermediate_size: inter,
345 hidden_act: config.effective_hidden_act().to_string(),
346 })
347 }
348
349 pub fn forward(&self, hidden_flat: &Tensor, dispatch: &RouterDispatch) -> Result<Tensor> {
354 let (num_tokens, _) = hidden_flat.dims2()?;
355 if num_tokens == 0 {
356 return Ok(Tensor::zeros_like(hidden_flat)?);
357 }
358
359 let mut buckets: BTreeMap<usize, Vec<(u32, f32)>> = BTreeMap::new();
360 for (tok_idx, (eids, weights)) in dispatch
361 .top_k_indices
362 .iter()
363 .zip(dispatch.top_k_weights.iter())
364 .enumerate()
365 {
366 let tok_u32 = tok_idx as u32;
367 for (&eid, &w) in eids.iter().zip(weights.iter()) {
368 buckets.entry(eid).or_default().push((tok_u32, w));
369 }
370 }
371
372 let device = hidden_flat.device();
373 let dtype = hidden_flat.dtype();
374 let mut acc = Tensor::zeros_like(hidden_flat)?;
375
376 for (expert_id, entries) in buckets {
377 let expert = self.experts.get(expert_id).ok_or_else(|| {
378 ModelError::InvalidArgument(format!(
379 "routed expert index {expert_id} out of bounds (num_experts = {})",
380 self.experts.len()
381 ))
382 })?;
383 let n_tok = entries.len();
384 let tok_indices: Vec<u32> = entries.iter().map(|&(t, _)| t).collect();
385 let weights: Vec<f32> = entries.iter().map(|&(_, w)| w).collect();
386
387 let tok_t = Tensor::from_vec(tok_indices, n_tok, device)?;
388 let w_col = Tensor::from_vec(weights, (n_tok, 1), device)?.to_dtype(dtype)?;
389
390 let x_e = hidden_flat.index_select(&tok_t, 0)?;
391 let gate_up = expert.gate_up_proj.forward(&x_e)?;
392 let gate = gate_up.narrow(1, 0, self.intermediate_size)?;
393 let up = gate_up.narrow(1, self.intermediate_size, self.intermediate_size)?;
394 let act = apply_activation(&gate, &self.hidden_act)?.mul(&up)?;
395 let y_e = expert.down_proj.forward(&act)?.broadcast_mul(&w_col)?;
396
397 acc = acc.index_add(&tok_t, &y_e, 0)?;
398 }
399
400 Ok(acc)
401 }
402}
403
404#[derive(Debug, Clone)]
406pub struct Gemma4MoE {
407 pub post_feedforward_layernorm_1: RmsNorm,
409 pub pre_feedforward_layernorm_2: RmsNorm,
411 pub router: Gemma4Router,
413 pub experts: Gemma4Experts,
415 pub post_feedforward_layernorm_2: RmsNorm,
417}
418
419#[derive(Debug, Clone)]
420struct Gemma4Attention {
421 q_proj: LinearLayer,
422 k_proj: LinearLayer,
423 v_proj: Option<LinearLayer>,
424 o_proj: LinearLayer,
425 q_norm: RmsNorm,
426 k_norm: RmsNorm,
427 v_norm: RmsNormNoScale,
428 num_heads: usize,
429 num_kv_heads: usize,
430 head_dim: usize,
431 softcap: Option<f64>,
432 sliding_window: Option<usize>,
433 is_sliding: bool,
434}
435
436impl Gemma4Attention {
437 fn forward(
438 &self,
439 hidden_states: &Tensor,
440 rope: &RotaryEmbedding,
441 layer_idx: usize,
442 cache: Option<&mut KvCache>,
443 ) -> Result<Tensor> {
444 let (batch, seq_len, _) = hidden_states.dims3()?;
445 let pos_offset = cache.as_ref().map_or(0, |c| c.seq_pos(layer_idx));
447
448 let q_raw = self
449 .q_proj
450 .forward(hidden_states)?
451 .reshape((batch, seq_len, self.num_heads, self.head_dim))?
452 .transpose(1, 2)?
453 .contiguous()?;
454 let k_raw = self
455 .k_proj
456 .forward(hidden_states)?
457 .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
458 .transpose(1, 2)?
459 .contiguous()?;
460
461 let v_raw = match &self.v_proj {
464 Some(vp) => vp
465 .forward(hidden_states)?
466 .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
467 .transpose(1, 2)?
468 .contiguous()?,
469 None => k_raw.clone(),
470 };
471
472 let q = rope.apply(&self.q_norm.forward(&q_raw)?, pos_offset)?;
473 let k = rope.apply(&self.k_norm.forward(&k_raw)?, pos_offset)?;
474 let v = self.v_norm.forward(&v_raw)?;
475
476 let attn_out = compute_attention(
477 &q,
478 &k,
479 &v,
480 layer_idx,
481 cache,
482 AttentionParams {
483 scale: 1.0,
484 softcap: self.softcap,
485 sliding_window: self.sliding_window,
486 },
487 )?;
488 self.o_proj.forward(&attn_out)
489 }
490}
491
492#[derive(Debug, Clone)]
493struct Gemma4Mlp {
494 gate_proj: LinearLayer,
495 up_proj: LinearLayer,
496 down_proj: LinearLayer,
497 hidden_act: String,
498}
499
500impl Gemma4Mlp {
501 fn forward(&self, x: &Tensor) -> Result<Tensor> {
502 let gate = apply_activation(&self.gate_proj.forward(x)?, &self.hidden_act)?;
503 let up = self.up_proj.forward(x)?;
504 self.down_proj.forward(&gate.mul(&up)?)
505 }
506}
507
508#[derive(Debug, Clone)]
509struct Gemma4DecoderLayer {
510 input_layernorm: RmsNorm,
511 self_attn: Gemma4Attention,
512 post_attention_layernorm: RmsNorm,
513 pre_feedforward_layernorm: RmsNorm,
514 mlp: Gemma4Mlp,
515 moe: Option<Gemma4MoE>,
516 post_feedforward_layernorm: RmsNorm,
517 layer_scalar: Param,
518}
519
520#[derive(Debug, Clone)]
522pub struct Gemma4Model {
523 config: ModelConfig,
524 weight_prefix: String,
525 embed_tokens: Embedding,
526 layers: Vec<Gemma4DecoderLayer>,
527 norm: RmsNorm,
528 lm_head: Option<LinearLayer>,
529 rope_full: RotaryEmbedding,
530 rope_sliding: Option<RotaryEmbedding>,
531}
532
533fn detect_weight_prefix(tensors: &HashMap<String, Tensor>) -> String {
534 if tensors.contains_key("model.language_model.embed_tokens.weight") {
535 "model.language_model".to_string()
536 } else if tensors.contains_key("language_model.model.embed_tokens.weight") {
537 "language_model.model".to_string()
538 } else {
539 "model".to_string()
540 }
541}
542
543fn build_rope(
544 head_dim: usize,
545 max_seq: usize,
546 params: ResolvedRopeParams,
547 dtype: DType,
548 device: &Device,
549) -> Result<RotaryEmbedding> {
550 let mode = if params.is_proportional {
551 RopeMode::Proportional
552 } else {
553 RopeMode::Default
554 };
555 RotaryEmbedding::new_with_config(
556 head_dim,
557 max_seq,
558 params.rope_theta,
559 params.partial_rotary_factor,
560 params.scale_factor,
561 mode,
562 dtype,
563 device,
564 )
565}
566
567fn load_layer_scalar(
568 key: &str,
569 tensors: &HashMap<String, Tensor>,
570 dtype: DType,
571 device: &Device,
572) -> Result<Param> {
573 match tensors.get(key) {
574 Some(t) => Ok(Param::Frozen(move_cast(t, dtype, device)?)),
575 None => Ok(Param::Frozen(Tensor::ones((1,), dtype, device)?)),
576 }
577}
578
579fn broadcast_scalar_mul(x: &Tensor, scalar: &Param) -> Result<Tensor> {
580 let s = scalar.as_tensor();
581 let s = if s.dtype() == x.dtype() {
582 s.clone()
583 } else {
584 s.to_dtype(x.dtype())?
585 };
586 Ok(x.broadcast_mul(&s)?)
587}
588
589impl Gemma4Model {
590 pub fn from_tensors(
598 config: ModelConfig,
599 tensors: &HashMap<String, Tensor>,
600 spec: LinearLoadSpec<'_>,
601 ) -> Result<Self> {
602 if config.num_kv_shared_layers.unwrap_or(0) > 0 {
603 return Err(ModelError::UnsupportedArchitecture(
604 "Gemma 4 KV-shared edge layers (num_kv_shared_layers > 0) are not supported"
605 .to_string(),
606 ));
607 }
608
609 let weight_prefix = detect_weight_prefix(tensors);
610 let embed_scale = Some((config.hidden_size as f64).sqrt());
611 let embed_tokens = Embedding::from_tensors(
612 &format!("{weight_prefix}.embed_tokens"),
613 tensors,
614 embed_scale,
615 spec.dtype,
616 spec.device,
617 )?;
618
619 let num_heads = config.num_attention_heads;
620 let inter = config.effective_intermediate_size();
621 let attn_bias = BiasMode::from(config.attention_bias);
622 let mlp_bias = BiasMode::from(config.mlp_bias);
623 let kv_factor = spec.littlebit.map_or(1.0, |c| c.kv_factor);
624 let hidden_act = config.effective_hidden_act().to_string();
625 let layer_types = config.effective_layer_types();
626 let enable_moe = config.enable_moe_block.unwrap_or(false);
627
628 let kv_spec = LinearLoadSpec {
629 ratio_factor: kv_factor,
630 ..spec
631 };
632 let unit_spec = LinearLoadSpec {
633 ratio_factor: 1.0,
634 ..spec
635 };
636
637 let mut any_sliding = false;
638 let mut layers = Vec::with_capacity(config.num_hidden_layers);
639
640 for (i, layer_type) in layer_types.iter().enumerate() {
641 let p = format!("{weight_prefix}.layers.{i}");
642 let is_sliding = layer_type == "sliding_attention";
643 if is_sliding {
644 any_sliding = true;
645 }
646 let head_dim = if is_sliding {
647 config.effective_head_dim()
648 } else {
649 config.effective_global_head_dim()
650 };
651 let num_kv_heads = if is_sliding {
652 config.effective_kv_heads()
653 } else {
654 config.effective_global_kv_heads()
655 };
656 let sliding_window = if is_sliding {
657 config.sliding_window
658 } else {
659 None
660 };
661 let use_k_eq_v = !is_sliding && config.attention_k_eq_v.unwrap_or(false);
662
663 let input_layernorm = RmsNorm::from_tensors(
664 &format!("{p}.input_layernorm"),
665 tensors,
666 config.rms_norm_eps,
667 NormKind::Standard,
668 spec.dtype,
669 spec.device,
670 )?;
671 let q_proj = LinearLayer::from_tensors(
672 &format!("{p}.self_attn.q_proj"),
673 tensors,
674 LayerShape::new(config.hidden_size, num_heads * head_dim, attn_bias),
675 unit_spec,
676 )?;
677 let k_proj = LinearLayer::from_tensors(
678 &format!("{p}.self_attn.k_proj"),
679 tensors,
680 LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
681 kv_spec,
682 )?;
683 let v_proj = if use_k_eq_v {
684 None
685 } else {
686 Some(LinearLayer::from_tensors(
687 &format!("{p}.self_attn.v_proj"),
688 tensors,
689 LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
690 kv_spec,
691 )?)
692 };
693 let o_proj = LinearLayer::from_tensors(
694 &format!("{p}.self_attn.o_proj"),
695 tensors,
696 LayerShape::new(num_heads * head_dim, config.hidden_size, attn_bias),
697 unit_spec,
698 )?;
699 let q_norm = RmsNorm::from_tensors(
700 &format!("{p}.self_attn.q_norm"),
701 tensors,
702 config.rms_norm_eps,
703 NormKind::Standard,
704 spec.dtype,
705 spec.device,
706 )?;
707 let k_norm = RmsNorm::from_tensors(
708 &format!("{p}.self_attn.k_norm"),
709 tensors,
710 config.rms_norm_eps,
711 NormKind::Standard,
712 spec.dtype,
713 spec.device,
714 )?;
715 let v_norm = RmsNormNoScale::new(config.rms_norm_eps);
716
717 let post_attention_layernorm = RmsNorm::from_tensors(
718 &format!("{p}.post_attention_layernorm"),
719 tensors,
720 config.rms_norm_eps,
721 NormKind::Standard,
722 spec.dtype,
723 spec.device,
724 )?;
725 let pre_feedforward_layernorm = RmsNorm::from_tensors(
726 &format!("{p}.pre_feedforward_layernorm"),
727 tensors,
728 config.rms_norm_eps,
729 NormKind::Standard,
730 spec.dtype,
731 spec.device,
732 )?;
733 let gate_proj = LinearLayer::from_tensors(
734 &format!("{p}.mlp.gate_proj"),
735 tensors,
736 LayerShape::new(config.hidden_size, inter, mlp_bias),
737 unit_spec,
738 )?;
739 let up_proj = LinearLayer::from_tensors(
740 &format!("{p}.mlp.up_proj"),
741 tensors,
742 LayerShape::new(config.hidden_size, inter, mlp_bias),
743 unit_spec,
744 )?;
745 let down_proj = LinearLayer::from_tensors(
746 &format!("{p}.mlp.down_proj"),
747 tensors,
748 LayerShape::new(inter, config.hidden_size, mlp_bias),
749 unit_spec,
750 )?;
751 let post_feedforward_layernorm = RmsNorm::from_tensors(
752 &format!("{p}.post_feedforward_layernorm"),
753 tensors,
754 config.rms_norm_eps,
755 NormKind::Standard,
756 spec.dtype,
757 spec.device,
758 )?;
759
760 let moe = if enable_moe {
761 Some(Gemma4MoE {
762 post_feedforward_layernorm_1: RmsNorm::from_tensors(
763 &format!("{p}.post_feedforward_layernorm_1"),
764 tensors,
765 config.rms_norm_eps,
766 NormKind::Standard,
767 spec.dtype,
768 spec.device,
769 )?,
770 pre_feedforward_layernorm_2: RmsNorm::from_tensors(
771 &format!("{p}.pre_feedforward_layernorm_2"),
772 tensors,
773 config.rms_norm_eps,
774 NormKind::Standard,
775 spec.dtype,
776 spec.device,
777 )?,
778 router: Gemma4Router::from_tensors(
779 &format!("{p}.router"),
780 &config,
781 tensors,
782 unit_spec,
783 )?,
784 experts: Gemma4Experts::from_tensors(
785 &format!("{p}.experts"),
786 &config,
787 tensors,
788 unit_spec,
789 )?,
790 post_feedforward_layernorm_2: RmsNorm::from_tensors(
791 &format!("{p}.post_feedforward_layernorm_2"),
792 tensors,
793 config.rms_norm_eps,
794 NormKind::Standard,
795 spec.dtype,
796 spec.device,
797 )?,
798 })
799 } else {
800 None
801 };
802
803 let layer_scalar = load_layer_scalar(
804 &format!("{p}.layer_scalar"),
805 tensors,
806 spec.dtype,
807 spec.device,
808 )?;
809
810 layers.push(Gemma4DecoderLayer {
811 input_layernorm,
812 self_attn: Gemma4Attention {
813 q_proj,
814 k_proj,
815 v_proj,
816 o_proj,
817 q_norm,
818 k_norm,
819 v_norm,
820 num_heads,
821 num_kv_heads,
822 head_dim,
823 softcap: config.attn_logit_softcapping,
824 sliding_window,
825 is_sliding,
826 },
827 post_attention_layernorm,
828 pre_feedforward_layernorm,
829 mlp: Gemma4Mlp {
830 gate_proj,
831 up_proj,
832 down_proj,
833 hidden_act: hidden_act.clone(),
834 },
835 moe,
836 post_feedforward_layernorm,
837 layer_scalar,
838 });
839 }
840
841 let norm = RmsNorm::from_tensors(
842 &format!("{weight_prefix}.norm"),
843 tensors,
844 config.rms_norm_eps,
845 NormKind::Standard,
846 spec.dtype,
847 spec.device,
848 )?;
849 let lm_head_key = if tensors.contains_key("lm_head.weight") {
850 Some("lm_head")
851 } else if tensors.contains_key("language_model.lm_head.weight") {
852 Some("language_model.lm_head")
853 } else {
854 None
855 };
856 let lm_head = match lm_head_key {
857 Some(prefix) if !config.tie_word_embeddings => Some(LinearLayer::from_tensors(
858 prefix,
859 tensors,
860 LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
861 LinearLoadSpec {
862 littlebit: None,
863 ratio_factor: 1.0,
864 ..spec
865 },
866 )?),
867 _ => None,
868 };
869
870 let max_seq = config.max_position_embeddings.max(256);
871 let rope_full = build_rope(
872 config.effective_global_head_dim(),
873 max_seq,
874 config.gemma4_rope_params("full_attention"),
875 spec.dtype,
876 spec.device,
877 )?;
878 let rope_sliding = if any_sliding {
879 Some(build_rope(
880 config.effective_head_dim(),
881 max_seq,
882 config.gemma4_rope_params("sliding_attention"),
883 spec.dtype,
884 spec.device,
885 )?)
886 } else {
887 None
888 };
889
890 Ok(Self {
891 config,
892 weight_prefix,
893 embed_tokens,
894 layers,
895 norm,
896 lm_head,
897 rope_full,
898 rope_sliding,
899 })
900 }
901
902 pub fn new_random(
907 config: ModelConfig,
908 dtype: DType,
909 device: &Device,
910 rng: &mut ChaCha8Rng,
911 ) -> Result<Self> {
912 let embed_scale = Some((config.hidden_size as f64).sqrt());
913 let embed_tokens = Embedding::new_random(
914 config.vocab_size,
915 config.hidden_size,
916 embed_scale,
917 dtype,
918 device,
919 rng,
920 )?;
921
922 let num_heads = config.num_attention_heads;
923 let inter = config.effective_intermediate_size();
924 let attn_bias = BiasMode::from(config.attention_bias);
925 let mlp_bias = BiasMode::from(config.mlp_bias);
926 let hidden_act = config.effective_hidden_act().to_string();
927 let layer_types = config.effective_layer_types();
928 let enable_moe = config.enable_moe_block.unwrap_or(false);
929
930 let scalar_dist = Normal::new(1.0f32, 0.05f32).map_err(|err| {
931 ModelError::InvalidArgument(format!("failed to construct scalar normal dist: {err}"))
932 })?;
933
934 let mut any_sliding = false;
935 let mut layers = Vec::with_capacity(config.num_hidden_layers);
936
937 for layer_type in &layer_types {
938 let is_sliding = layer_type == "sliding_attention";
939 if is_sliding {
940 any_sliding = true;
941 }
942 let head_dim = if is_sliding {
943 config.effective_head_dim()
944 } else {
945 config.effective_global_head_dim()
946 };
947 let num_kv_heads = if is_sliding {
948 config.effective_kv_heads()
949 } else {
950 config.effective_global_kv_heads()
951 };
952 let sliding_window = if is_sliding {
953 config.sliding_window
954 } else {
955 None
956 };
957 let use_k_eq_v = !is_sliding && config.attention_k_eq_v.unwrap_or(false);
958
959 let v_proj = if use_k_eq_v {
960 None
961 } else {
962 Some(LinearLayer::new_random(
963 LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
964 dtype,
965 device,
966 rng,
967 )?)
968 };
969
970 let moe = if enable_moe {
971 Some(Gemma4MoE {
972 post_feedforward_layernorm_1: RmsNorm::new_init(
973 config.hidden_size,
974 config.rms_norm_eps,
975 NormKind::Standard,
976 dtype,
977 device,
978 )?,
979 pre_feedforward_layernorm_2: RmsNorm::new_init(
980 config.hidden_size,
981 config.rms_norm_eps,
982 NormKind::Standard,
983 dtype,
984 device,
985 )?,
986 router: Gemma4Router::new_random(&config, dtype, device, rng)?,
987 experts: Gemma4Experts::new_random(&config, dtype, device, rng)?,
988 post_feedforward_layernorm_2: RmsNorm::new_init(
989 config.hidden_size,
990 config.rms_norm_eps,
991 NormKind::Standard,
992 dtype,
993 device,
994 )?,
995 })
996 } else {
997 None
998 };
999
1000 let scalar_val = scalar_dist.sample(rng);
1001 let layer_scalar =
1002 Param::Frozen(Tensor::from_slice(&[scalar_val], (1,), device)?.to_dtype(dtype)?);
1003
1004 layers.push(Gemma4DecoderLayer {
1005 input_layernorm: RmsNorm::new_init(
1006 config.hidden_size,
1007 config.rms_norm_eps,
1008 NormKind::Standard,
1009 dtype,
1010 device,
1011 )?,
1012 self_attn: Gemma4Attention {
1013 q_proj: LinearLayer::new_random(
1014 LayerShape::new(config.hidden_size, num_heads * head_dim, attn_bias),
1015 dtype,
1016 device,
1017 rng,
1018 )?,
1019 k_proj: LinearLayer::new_random(
1020 LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
1021 dtype,
1022 device,
1023 rng,
1024 )?,
1025 v_proj,
1026 o_proj: LinearLayer::new_random(
1027 LayerShape::new(num_heads * head_dim, config.hidden_size, attn_bias),
1028 dtype,
1029 device,
1030 rng,
1031 )?,
1032 q_norm: RmsNorm::new_init(
1033 head_dim,
1034 config.rms_norm_eps,
1035 NormKind::Standard,
1036 dtype,
1037 device,
1038 )?,
1039 k_norm: RmsNorm::new_init(
1040 head_dim,
1041 config.rms_norm_eps,
1042 NormKind::Standard,
1043 dtype,
1044 device,
1045 )?,
1046 v_norm: RmsNormNoScale::new(config.rms_norm_eps),
1047 num_heads,
1048 num_kv_heads,
1049 head_dim,
1050 softcap: config.attn_logit_softcapping,
1051 sliding_window,
1052 is_sliding,
1053 },
1054 post_attention_layernorm: RmsNorm::new_init(
1055 config.hidden_size,
1056 config.rms_norm_eps,
1057 NormKind::Standard,
1058 dtype,
1059 device,
1060 )?,
1061 pre_feedforward_layernorm: RmsNorm::new_init(
1062 config.hidden_size,
1063 config.rms_norm_eps,
1064 NormKind::Standard,
1065 dtype,
1066 device,
1067 )?,
1068 mlp: Gemma4Mlp {
1069 gate_proj: LinearLayer::new_random(
1070 LayerShape::new(config.hidden_size, inter, mlp_bias),
1071 dtype,
1072 device,
1073 rng,
1074 )?,
1075 up_proj: LinearLayer::new_random(
1076 LayerShape::new(config.hidden_size, inter, mlp_bias),
1077 dtype,
1078 device,
1079 rng,
1080 )?,
1081 down_proj: LinearLayer::new_random(
1082 LayerShape::new(inter, config.hidden_size, mlp_bias),
1083 dtype,
1084 device,
1085 rng,
1086 )?,
1087 hidden_act: hidden_act.clone(),
1088 },
1089 moe,
1090 post_feedforward_layernorm: RmsNorm::new_init(
1091 config.hidden_size,
1092 config.rms_norm_eps,
1093 NormKind::Standard,
1094 dtype,
1095 device,
1096 )?,
1097 layer_scalar,
1098 });
1099 }
1100
1101 let norm = RmsNorm::new_init(
1102 config.hidden_size,
1103 config.rms_norm_eps,
1104 NormKind::Standard,
1105 dtype,
1106 device,
1107 )?;
1108 let lm_head = if config.tie_word_embeddings {
1109 None
1110 } else {
1111 Some(LinearLayer::new_random(
1112 LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
1113 dtype,
1114 device,
1115 rng,
1116 )?)
1117 };
1118
1119 let max_seq = config.max_position_embeddings.max(256);
1120 let rope_full = build_rope(
1121 config.effective_global_head_dim(),
1122 max_seq,
1123 config.gemma4_rope_params("full_attention"),
1124 dtype,
1125 device,
1126 )?;
1127 let rope_sliding = if any_sliding {
1128 Some(build_rope(
1129 config.effective_head_dim(),
1130 max_seq,
1131 config.gemma4_rope_params("sliding_attention"),
1132 dtype,
1133 device,
1134 )?)
1135 } else {
1136 None
1137 };
1138
1139 Ok(Self {
1140 config,
1141 weight_prefix: "model".to_string(),
1142 embed_tokens,
1143 layers,
1144 norm,
1145 lm_head,
1146 rope_full,
1147 rope_sliding,
1148 })
1149 }
1150
1151 #[must_use]
1153 pub fn layer_moe(&self, layer_idx: usize) -> Option<&Gemma4MoE> {
1154 self.layers.get(layer_idx).and_then(|l| l.moe.as_ref())
1155 }
1156
1157 #[must_use]
1159 pub fn layer_moe_mut(&mut self, layer_idx: usize) -> Option<&mut Gemma4MoE> {
1160 self.layers.get_mut(layer_idx).and_then(|l| l.moe.as_mut())
1161 }
1162
1163 pub fn forward(
1168 &self,
1169 input_ids: &Tensor,
1170 mut cache: Option<&mut KvCache>,
1171 hidden_mode: HiddenStatesMode,
1172 ) -> Result<CausalLmOutput> {
1173 let mut hidden = self.embed_tokens.forward(input_ids)?;
1174 let collect = hidden_mode == HiddenStatesMode::All;
1175 let mut all_hidden = if collect {
1176 Vec::with_capacity(self.layers.len() + 1)
1177 } else {
1178 Vec::new()
1179 };
1180
1181 for (idx, layer) in self.layers.iter().enumerate() {
1182 if collect {
1183 all_hidden.push(hidden.clone());
1184 }
1185 let residual = hidden.clone();
1186 let normed = layer.input_layernorm.forward(&hidden)?;
1187 let rope = if layer.self_attn.is_sliding {
1188 self.rope_sliding.as_ref().unwrap_or(&self.rope_full)
1189 } else {
1190 &self.rope_full
1191 };
1192 let attn = layer
1193 .self_attn
1194 .forward(&normed, rope, idx, cache.as_deref_mut())?;
1195 let attn_normed = layer.post_attention_layernorm.forward(&attn)?;
1196 let h_mid = residual.add(&attn_normed)?;
1197
1198 let mut next_hidden = match &layer.moe {
1199 None => {
1200 let pre_ff = layer.pre_feedforward_layernorm.forward(&h_mid)?;
1201 let ff_out = layer.mlp.forward(&pre_ff)?;
1202 let ff_normed = layer.post_feedforward_layernorm.forward(&ff_out)?;
1203 h_mid.add(&ff_normed)?
1204 }
1205 Some(moe) => {
1206 let pre_ff = layer.pre_feedforward_layernorm.forward(&h_mid)?;
1207 let mlp_out = layer.mlp.forward(&pre_ff)?;
1208 let h1 = moe.post_feedforward_layernorm_1.forward(&mlp_out)?;
1209
1210 let (b, s, d) = h_mid.dims3()?;
1211 let flat = h_mid.reshape((b * s, d))?;
1212 let dispatch = moe.router.forward(&flat)?;
1213 let h2_in = moe.pre_feedforward_layernorm_2.forward(&flat)?;
1214 let h2_experts = moe.experts.forward(&h2_in, &dispatch)?.reshape((b, s, d))?;
1215 let h2 = moe.post_feedforward_layernorm_2.forward(&h2_experts)?;
1216
1217 let ff_normed = layer.post_feedforward_layernorm.forward(&h1.add(&h2)?)?;
1218 h_mid.add(&ff_normed)?
1219 }
1220 };
1221
1222 next_hidden = broadcast_scalar_mul(&next_hidden, &layer.layer_scalar)?;
1223 hidden = next_hidden;
1224 }
1225
1226 hidden = self.norm.forward(&hidden)?;
1227 if collect {
1228 all_hidden.push(hidden.clone());
1229 }
1230 let mut logits = match &self.lm_head {
1231 Some(head) => head.forward(&hidden)?,
1232 None => self.embed_tokens.forward_lm_head(&hidden)?,
1233 };
1234 if let Some(cap) = self.config.final_logit_softcapping {
1235 logits = ((logits / cap)?.tanh()? * cap)?;
1236 }
1237 Ok(CausalLmOutput {
1238 logits,
1239 hidden_states: all_hidden,
1240 })
1241 }
1242
1243 pub(crate) fn apply_littlebit_patch(
1244 &mut self,
1245 lb_cfg: &LittleBitConfig,
1246 policy: &PatchPolicy,
1247 train_mode: TrainMode,
1248 base_seed: u64,
1249 summary: &mut PatchSummary,
1250 ) -> Result<()> {
1251 for (i, layer) in self.layers.iter_mut().enumerate() {
1252 let p = format!("{}.layers.{i}", self.weight_prefix);
1253 patch_linear_layer(
1254 &format!("{p}.self_attn.q_proj"),
1255 &mut layer.self_attn.q_proj,
1256 lb_cfg,
1257 policy,
1258 train_mode,
1259 base_seed,
1260 summary,
1261 )?;
1262 patch_linear_layer(
1263 &format!("{p}.self_attn.k_proj"),
1264 &mut layer.self_attn.k_proj,
1265 lb_cfg,
1266 policy,
1267 train_mode,
1268 base_seed,
1269 summary,
1270 )?;
1271 if let Some(vp) = &mut layer.self_attn.v_proj {
1272 patch_linear_layer(
1273 &format!("{p}.self_attn.v_proj"),
1274 vp,
1275 lb_cfg,
1276 policy,
1277 train_mode,
1278 base_seed,
1279 summary,
1280 )?;
1281 }
1282 patch_linear_layer(
1283 &format!("{p}.self_attn.o_proj"),
1284 &mut layer.self_attn.o_proj,
1285 lb_cfg,
1286 policy,
1287 train_mode,
1288 base_seed,
1289 summary,
1290 )?;
1291 patch_linear_layer(
1292 &format!("{p}.mlp.gate_proj"),
1293 &mut layer.mlp.gate_proj,
1294 lb_cfg,
1295 policy,
1296 train_mode,
1297 base_seed,
1298 summary,
1299 )?;
1300 patch_linear_layer(
1301 &format!("{p}.mlp.up_proj"),
1302 &mut layer.mlp.up_proj,
1303 lb_cfg,
1304 policy,
1305 train_mode,
1306 base_seed,
1307 summary,
1308 )?;
1309 patch_linear_layer(
1310 &format!("{p}.mlp.down_proj"),
1311 &mut layer.mlp.down_proj,
1312 lb_cfg,
1313 policy,
1314 train_mode,
1315 base_seed,
1316 summary,
1317 )?;
1318 if let Some(moe) = &mut layer.moe {
1319 patch_linear_layer(
1321 &format!("{p}.router.proj"),
1322 &mut moe.router.proj,
1323 lb_cfg,
1324 policy,
1325 train_mode,
1326 base_seed,
1327 summary,
1328 )?;
1329 for (e, expert) in moe.experts.experts.iter_mut().enumerate() {
1330 patch_linear_layer(
1331 &format!("{p}.experts.{e}.gate_up_proj"),
1332 &mut expert.gate_up_proj,
1333 lb_cfg,
1334 policy,
1335 train_mode,
1336 base_seed,
1337 summary,
1338 )?;
1339 patch_linear_layer(
1340 &format!("{p}.experts.{e}.down_proj"),
1341 &mut expert.down_proj,
1342 lb_cfg,
1343 policy,
1344 train_mode,
1345 base_seed,
1346 summary,
1347 )?;
1348 }
1349 }
1350 }
1351 if let Some(head) = &mut self.lm_head {
1352 patch_linear_layer(
1353 "lm_head", head, lb_cfg, policy, train_mode, base_seed, summary,
1354 )?;
1355 }
1356 Ok(())
1357 }
1358
1359 pub(crate) fn convert_to_packed(&mut self) -> Result<()> {
1360 for layer in &mut self.layers {
1361 layer.self_attn.q_proj.convert_to_packed()?;
1362 layer.self_attn.k_proj.convert_to_packed()?;
1363 if let Some(vp) = &mut layer.self_attn.v_proj {
1364 vp.convert_to_packed()?;
1365 }
1366 layer.self_attn.o_proj.convert_to_packed()?;
1367 layer.mlp.gate_proj.convert_to_packed()?;
1368 layer.mlp.up_proj.convert_to_packed()?;
1369 layer.mlp.down_proj.convert_to_packed()?;
1370 if let Some(moe) = &mut layer.moe {
1371 for expert in &mut moe.experts.experts {
1373 expert.gate_up_proj.convert_to_packed()?;
1374 expert.down_proj.convert_to_packed()?;
1375 }
1376 }
1377 }
1378 if let Some(head) = &mut self.lm_head {
1379 head.convert_to_packed()?;
1380 }
1381 Ok(())
1382 }
1383
1384 pub(crate) fn make_non_quantized_trainable(&mut self) -> Result<()> {
1385 self.embed_tokens.make_trainable()?;
1386 for layer in &mut self.layers {
1387 layer.input_layernorm.make_trainable()?;
1388 layer.self_attn.q_norm.make_trainable()?;
1389 layer.self_attn.k_norm.make_trainable()?;
1390 layer.post_attention_layernorm.make_trainable()?;
1391 layer.pre_feedforward_layernorm.make_trainable()?;
1392 layer.post_feedforward_layernorm.make_trainable()?;
1393 if let Param::Frozen(t) = &layer.layer_scalar {
1394 layer.layer_scalar = Param::Trainable(Var::from_tensor(t)?);
1395 }
1396 if let Some(moe) = &mut layer.moe {
1397 moe.post_feedforward_layernorm_1.make_trainable()?;
1398 moe.pre_feedforward_layernorm_2.make_trainable()?;
1399 moe.post_feedforward_layernorm_2.make_trainable()?;
1400 if let Param::Frozen(t) = &moe.router.scale {
1401 moe.router.scale = Param::Trainable(Var::from_tensor(t)?);
1402 }
1403 if let Param::Frozen(t) = &moe.router.per_expert_scale {
1404 moe.router.per_expert_scale = Param::Trainable(Var::from_tensor(t)?);
1405 }
1406 moe.router.proj.make_trainable()?;
1407 }
1408 }
1409 self.norm.make_trainable()?;
1410 if let Some(head) = &mut self.lm_head {
1411 head.make_trainable()?;
1412 }
1413 Ok(())
1414 }
1415
1416 pub(crate) fn freeze_matching(&mut self, exclude: &[&str]) {
1417 let matches = |path: &str| exclude.iter().any(|pat| path.contains(pat));
1418 let wp = &self.weight_prefix;
1419 if matches("embed_tokens") {
1420 self.embed_tokens.freeze();
1421 }
1422 if matches("lm_head") {
1423 if let Some(head) = &mut self.lm_head {
1424 head.freeze();
1425 }
1426 }
1427 if matches("norm") {
1428 self.norm.freeze();
1429 }
1430 for (i, layer) in self.layers.iter_mut().enumerate() {
1431 let p = format!("{wp}.layers.{i}");
1432 if matches(&format!("{p}.input_layernorm")) || matches("layernorm") {
1433 layer.input_layernorm.freeze();
1434 }
1435 if matches(&format!("{p}.post_attention_layernorm")) || matches("layernorm") {
1436 layer.post_attention_layernorm.freeze();
1437 }
1438 if matches(&format!("{p}.pre_feedforward_layernorm")) || matches("layernorm") {
1439 layer.pre_feedforward_layernorm.freeze();
1440 }
1441 if matches(&format!("{p}.post_feedforward_layernorm")) || matches("layernorm") {
1442 layer.post_feedforward_layernorm.freeze();
1443 }
1444 if matches(&format!("{p}.self_attn.q_norm")) || matches("norm") {
1445 layer.self_attn.q_norm.freeze();
1446 }
1447 if matches(&format!("{p}.self_attn.k_norm")) || matches("norm") {
1448 layer.self_attn.k_norm.freeze();
1449 }
1450 if matches(&format!("{p}.layer_scalar")) || matches("layer_scalar") {
1451 layer.layer_scalar.freeze();
1452 }
1453 if matches(&format!("{p}.self_attn")) {
1454 layer.self_attn.q_proj.freeze();
1455 layer.self_attn.k_proj.freeze();
1456 if let Some(vp) = &mut layer.self_attn.v_proj {
1457 vp.freeze();
1458 }
1459 layer.self_attn.o_proj.freeze();
1460 }
1461 if matches(&format!("{p}.mlp")) {
1462 layer.mlp.gate_proj.freeze();
1463 layer.mlp.up_proj.freeze();
1464 layer.mlp.down_proj.freeze();
1465 }
1466 if let Some(moe) = &mut layer.moe {
1467 if matches(&format!("{p}.post_feedforward_layernorm_1")) || matches("layernorm") {
1468 moe.post_feedforward_layernorm_1.freeze();
1469 }
1470 if matches(&format!("{p}.pre_feedforward_layernorm_2")) || matches("layernorm") {
1471 moe.pre_feedforward_layernorm_2.freeze();
1472 }
1473 if matches(&format!("{p}.post_feedforward_layernorm_2")) || matches("layernorm") {
1474 moe.post_feedforward_layernorm_2.freeze();
1475 }
1476 if matches(&format!("{p}.router")) {
1477 moe.router.scale.freeze();
1478 moe.router.per_expert_scale.freeze();
1479 moe.router.proj.freeze();
1480 }
1481 if matches(&format!("{p}.experts")) {
1482 for expert in &mut moe.experts.experts {
1483 expert.gate_up_proj.freeze();
1484 expert.down_proj.freeze();
1485 }
1486 }
1487 }
1488 }
1489 }
1490
1491 pub(crate) fn trainable_vars(&self) -> Vec<Var> {
1492 let mut vars = self.embed_tokens.trainable_vars();
1493 for layer in &self.layers {
1494 vars.extend(layer.input_layernorm.trainable_vars());
1495 vars.extend(layer.self_attn.q_proj.trainable_vars());
1496 vars.extend(layer.self_attn.k_proj.trainable_vars());
1497 if let Some(vp) = &layer.self_attn.v_proj {
1498 vars.extend(vp.trainable_vars());
1499 }
1500 vars.extend(layer.self_attn.o_proj.trainable_vars());
1501 vars.extend(layer.self_attn.q_norm.trainable_vars());
1502 vars.extend(layer.self_attn.k_norm.trainable_vars());
1503 vars.extend(layer.post_attention_layernorm.trainable_vars());
1504 vars.extend(layer.pre_feedforward_layernorm.trainable_vars());
1505 vars.extend(layer.mlp.gate_proj.trainable_vars());
1506 vars.extend(layer.mlp.up_proj.trainable_vars());
1507 vars.extend(layer.mlp.down_proj.trainable_vars());
1508 if let Some(moe) = &layer.moe {
1509 vars.extend(moe.post_feedforward_layernorm_1.trainable_vars());
1510 vars.extend(moe.pre_feedforward_layernorm_2.trainable_vars());
1511 if let Some(v) = moe.router.scale.as_var() {
1512 vars.push(v.clone());
1513 }
1514 if let Some(v) = moe.router.per_expert_scale.as_var() {
1515 vars.push(v.clone());
1516 }
1517 vars.extend(moe.router.proj.trainable_vars());
1518 for expert in &moe.experts.experts {
1519 vars.extend(expert.gate_up_proj.trainable_vars());
1520 vars.extend(expert.down_proj.trainable_vars());
1521 }
1522 vars.extend(moe.post_feedforward_layernorm_2.trainable_vars());
1523 }
1524 vars.extend(layer.post_feedforward_layernorm.trainable_vars());
1525 if let Some(v) = layer.layer_scalar.as_var() {
1526 vars.push(v.clone());
1527 }
1528 }
1529 vars.extend(self.norm.trainable_vars());
1530 if let Some(head) = &self.lm_head {
1531 vars.extend(head.trainable_vars());
1532 }
1533 vars
1534 }
1535
1536 pub(crate) fn total_params(&self) -> usize {
1537 let mut count = self.embed_tokens.weight.elem_count() + self.norm.weight.elem_count();
1538 for layer in &self.layers {
1539 count += layer.input_layernorm.weight.elem_count()
1540 + layer.post_attention_layernorm.weight.elem_count()
1541 + layer.pre_feedforward_layernorm.weight.elem_count()
1542 + layer.post_feedforward_layernorm.weight.elem_count()
1543 + layer.layer_scalar.elem_count()
1544 + layer.self_attn.q_proj.param_count()
1545 + layer.self_attn.k_proj.param_count()
1546 + layer
1547 .self_attn
1548 .v_proj
1549 .as_ref()
1550 .map_or(0, LinearLayer::param_count)
1551 + layer.self_attn.o_proj.param_count()
1552 + layer.self_attn.q_norm.weight.elem_count()
1553 + layer.self_attn.k_norm.weight.elem_count()
1554 + layer.mlp.gate_proj.param_count()
1555 + layer.mlp.up_proj.param_count()
1556 + layer.mlp.down_proj.param_count();
1557 if let Some(moe) = &layer.moe {
1558 count += moe.post_feedforward_layernorm_1.weight.elem_count()
1559 + moe.pre_feedforward_layernorm_2.weight.elem_count()
1560 + moe.post_feedforward_layernorm_2.weight.elem_count()
1561 + moe.router.scale.elem_count()
1562 + moe.router.per_expert_scale.elem_count()
1563 + moe.router.proj.param_count();
1564 for expert in &moe.experts.experts {
1565 count += expert.gate_up_proj.param_count() + expert.down_proj.param_count();
1566 }
1567 }
1568 }
1569 if let Some(head) = &self.lm_head {
1570 count += head.param_count();
1571 }
1572 count
1573 }
1574
1575 pub(crate) fn pack_state_dict(&self) -> Result<HashMap<String, Tensor>> {
1576 let mut out = HashMap::new();
1577 let wp = &self.weight_prefix;
1578 self.embed_tokens
1579 .pack_state_dict(&format!("{wp}.embed_tokens"), &mut out);
1580 for (i, layer) in self.layers.iter().enumerate() {
1581 let p = format!("{wp}.layers.{i}");
1582 layer
1583 .input_layernorm
1584 .pack_state_dict(&format!("{p}.input_layernorm"), &mut out);
1585 out.extend(
1586 layer
1587 .self_attn
1588 .q_proj
1589 .pack_state_dict(&format!("{p}.self_attn.q_proj"))?,
1590 );
1591 out.extend(
1592 layer
1593 .self_attn
1594 .k_proj
1595 .pack_state_dict(&format!("{p}.self_attn.k_proj"))?,
1596 );
1597 if let Some(vp) = &layer.self_attn.v_proj {
1598 out.extend(vp.pack_state_dict(&format!("{p}.self_attn.v_proj"))?);
1599 }
1600 out.extend(
1601 layer
1602 .self_attn
1603 .o_proj
1604 .pack_state_dict(&format!("{p}.self_attn.o_proj"))?,
1605 );
1606 layer
1607 .self_attn
1608 .q_norm
1609 .pack_state_dict(&format!("{p}.self_attn.q_norm"), &mut out);
1610 layer
1611 .self_attn
1612 .k_norm
1613 .pack_state_dict(&format!("{p}.self_attn.k_norm"), &mut out);
1614 layer
1615 .post_attention_layernorm
1616 .pack_state_dict(&format!("{p}.post_attention_layernorm"), &mut out);
1617 layer
1618 .pre_feedforward_layernorm
1619 .pack_state_dict(&format!("{p}.pre_feedforward_layernorm"), &mut out);
1620 out.extend(
1621 layer
1622 .mlp
1623 .gate_proj
1624 .pack_state_dict(&format!("{p}.mlp.gate_proj"))?,
1625 );
1626 out.extend(
1627 layer
1628 .mlp
1629 .up_proj
1630 .pack_state_dict(&format!("{p}.mlp.up_proj"))?,
1631 );
1632 out.extend(
1633 layer
1634 .mlp
1635 .down_proj
1636 .pack_state_dict(&format!("{p}.mlp.down_proj"))?,
1637 );
1638 if let Some(moe) = &layer.moe {
1639 moe.post_feedforward_layernorm_1
1640 .pack_state_dict(&format!("{p}.post_feedforward_layernorm_1"), &mut out);
1641 moe.pre_feedforward_layernorm_2
1642 .pack_state_dict(&format!("{p}.pre_feedforward_layernorm_2"), &mut out);
1643 out.insert(
1644 format!("{p}.router.scale"),
1645 moe.router.scale.as_tensor().detach(),
1646 );
1647 out.insert(
1648 format!("{p}.router.per_expert_scale"),
1649 moe.router.per_expert_scale.as_tensor().detach(),
1650 );
1651 out.extend(
1652 moe.router
1653 .proj
1654 .pack_state_dict(&format!("{p}.router.proj"))?,
1655 );
1656 for (e, expert) in moe.experts.experts.iter().enumerate() {
1657 out.extend(
1658 expert
1659 .gate_up_proj
1660 .pack_state_dict(&format!("{p}.experts.{e}.gate_up_proj"))?,
1661 );
1662 out.extend(
1663 expert
1664 .down_proj
1665 .pack_state_dict(&format!("{p}.experts.{e}.down_proj"))?,
1666 );
1667 }
1668 moe.post_feedforward_layernorm_2
1669 .pack_state_dict(&format!("{p}.post_feedforward_layernorm_2"), &mut out);
1670 }
1671 layer
1672 .post_feedforward_layernorm
1673 .pack_state_dict(&format!("{p}.post_feedforward_layernorm"), &mut out);
1674 out.insert(
1675 format!("{p}.layer_scalar"),
1676 layer.layer_scalar.as_tensor().detach(),
1677 );
1678 }
1679 self.norm.pack_state_dict(&format!("{wp}.norm"), &mut out);
1680 if let Some(head) = &self.lm_head {
1681 out.extend(head.pack_state_dict("lm_head")?);
1682 }
1683 Ok(out)
1684 }
1685
1686 pub(crate) fn config(&self) -> &ModelConfig {
1687 &self.config
1688 }
1689}
1690
1691#[cfg(test)]
1692mod tests {
1693 use super::*;
1694 use crate::config::ModelArchitecture;
1695 use crate::hub::{LittleBitModel, LoadOptions};
1696 use crate::model::CausalLm;
1697 use littlebit_core::{CastPolicy, InferenceMode};
1698 use rand::SeedableRng;
1699 use tempfile::TempDir;
1700
1701 #[test]
1702 fn multimodal_gemma4_config_unwraps_text_config_and_prefixes() -> Result<()> {
1703 let dir = TempDir::new().expect("tempdir");
1704 let cfg_json = serde_json::json!({
1705 "model_type": "gemma4",
1706 "architectures": ["Gemma4ForConditionalGeneration"],
1707 "tie_word_embeddings": true,
1708 "text_config": {
1709 "vocab_size": 128,
1710 "hidden_size": 64,
1711 "intermediate_size": 128,
1712 "num_hidden_layers": 2,
1713 "num_attention_heads": 4,
1714 "num_key_value_heads": 2,
1715 "head_dim": 16,
1716 "global_head_dim": 32,
1717 "num_global_key_value_heads": 1,
1718 "attention_k_eq_v": true,
1719 "sliding_window": 4,
1720 "layer_types": ["sliding_attention", "full_attention"],
1721 "rms_norm_eps": 1e-6,
1722 "hidden_activation": "gelu_pytorch_tanh",
1723 "rope_parameters": {
1724 "sliding_attention": { "rope_type": "default", "rope_theta": 10000.0 },
1725 "full_attention": {
1726 "rope_type": "proportional",
1727 "partial_rotary_factor": 0.25,
1728 "rope_theta": 1000000.0
1729 }
1730 }
1731 }
1732 });
1733 std::fs::write(
1734 dir.path().join("config.json"),
1735 serde_json::to_string_pretty(&cfg_json)?,
1736 )?;
1737 let parsed = ModelConfig::from_file(&dir.path().join("config.json"))?;
1738 assert_eq!(parsed.detect_architecture()?, ModelArchitecture::Gemma4);
1739 assert!(parsed.tie_word_embeddings);
1740 assert_eq!(parsed.effective_global_head_dim(), 32);
1741 assert_eq!(parsed.effective_global_kv_heads(), 1);
1742 assert_eq!(parsed.effective_hidden_act(), "gelu_pytorch_tanh");
1743
1744 let full_rope = parsed.gemma4_rope_params("full_attention");
1745 assert!(full_rope.is_proportional);
1746 assert!((full_rope.partial_rotary_factor - 0.25).abs() < 1e-12);
1747 assert!((full_rope.rope_theta - 1_000_000.0).abs() < 1e-12);
1748
1749 let dev = Device::Cpu;
1751 let base_model = CausalLm::new_random(parsed.clone(), DType::F32, &dev, 9)?;
1752 let raw_sd = base_model.pack_state_dict()?;
1753 let mut mm_sd = HashMap::new();
1754 for (k, v) in raw_sd {
1755 let new_k = if let Some(rest) = k.strip_prefix("model.") {
1756 format!("model.language_model.{rest}")
1757 } else {
1758 k
1759 };
1760 mm_sd.insert(new_k, v);
1761 }
1762 let mm_model = CausalLm::from_tensors(
1763 parsed,
1764 &mm_sd,
1765 None,
1766 littlebit_core::WeightState::Raw,
1767 InferenceMode::Unpacked,
1768 DType::F32,
1769 &dev,
1770 )?;
1771 let ids = Tensor::from_slice(&[1u32, 3, 5], (1, 3), &dev)?;
1772 let out_a = base_model.forward(&ids, None, HiddenStatesMode::None)?;
1773 let out_b = mm_model.forward(&ids, None, HiddenStatesMode::None)?;
1774 let diff = out_a
1775 .logits
1776 .sub(&out_b.logits)?
1777 .abs()?
1778 .max_all()?
1779 .to_scalar::<f32>()?;
1780 assert!(diff < 1e-6, "multimodal prefix drift: {diff}");
1781 Ok(())
1782 }
1783
1784 #[test]
1785 fn proportional_rope_preserves_nope_subspace_and_scales_angles_by_full_head_dim() -> Result<()>
1786 {
1787 let dev = Device::Cpu;
1788 let head_dim = 16;
1789 let rope = RotaryEmbedding::new_with_config(
1790 head_dim,
1791 32,
1792 10_000.0,
1793 0.25,
1794 1.0,
1795 RopeMode::Proportional,
1796 DType::F32,
1797 &dev,
1798 )?;
1799 let x = Tensor::ones((1, 2, 5, head_dim), DType::F32, &dev)?;
1800 let out = rope.apply(&x, 3)?;
1801
1802 let half = head_dim / 2;
1803 let rope_angles = 2; let vals = out.get(0)?.to_vec3::<f32>()?;
1805 let inv_freq_1 = 1.0f64 / 10_000.0f64.powf(2.0 / 16.0);
1806 for head in &vals {
1807 for (t, token) in head.iter().enumerate() {
1808 for d in rope_angles..half {
1810 assert!(
1811 (token[d] - 1.0).abs() < 1e-6,
1812 "NOPE dim {d} modified: {}",
1813 token[d]
1814 );
1815 assert!(
1816 (token[half + d] - 1.0).abs() < 1e-6,
1817 "NOPE pair dim {} modified: {}",
1818 half + d,
1819 token[half + d]
1820 );
1821 }
1822 let pos = (3 + t) as f64;
1824 let expected_dim1 = (pos * inv_freq_1).cos() - (pos * inv_freq_1).sin();
1825 assert!(
1826 (f64::from(token[1]) - expected_dim1).abs() < 1e-5,
1827 "p-RoPE angle mismatch at pos {pos}: got {}, expected {expected_dim1}",
1828 token[1]
1829 );
1830 }
1831 }
1832 Ok(())
1833 }
1834
1835 #[test]
1836 fn gemma4_dense_k_eq_v_and_sliding_window_kv_cache_parity() -> Result<()> {
1837 let dev = Device::Cpu;
1838 let mut cfg = ModelConfig::tiny_gemma4_dense(96, 64, 2);
1839 cfg.sliding_window = Some(4);
1841
1842 let mut model = CausalLm::new_random(cfg.clone(), DType::F32, &dev, 41)?;
1843 if let CausalLm::Gemma4(ref g4) = model {
1844 assert!(g4.layers[0].self_attn.is_sliding);
1845 assert!(g4.layers[0].self_attn.v_proj.is_some());
1846 assert!(!g4.layers[1].self_attn.is_sliding);
1847 assert!(
1848 g4.layers[1].self_attn.v_proj.is_none(),
1849 "full_attention with attention_k_eq_v must omit v_proj"
1850 );
1851 } else {
1852 panic!("expected Gemma4");
1853 }
1854
1855 let lb_cfg = LittleBitConfig {
1856 eff_bit: Some(1.0),
1857 use_itq: true,
1858 itq_n_iter: 5,
1859 ..Default::default()
1860 };
1861 model.apply_littlebit_patch(&lb_cfg, TrainMode::Eval, 19)?;
1862
1863 let prompt = [2u32, 7, 15, 3, 21, 9, 11];
1864 let full_t = Tensor::from_slice(&prompt, (1, prompt.len()), &dev)?;
1865 let full_logits = model
1866 .forward(&full_t, None, HiddenStatesMode::None)?
1867 .logits
1868 .get(0)?
1869 .get(prompt.len() - 1)?;
1870
1871 let mut cache = KvCache::new(cfg.num_hidden_layers);
1872 let mut step_logits = None;
1873 for &tok in &prompt {
1874 let tok_t = Tensor::from_slice(&[tok], (1, 1), &dev)?;
1875 let out = model.forward(&tok_t, Some(&mut cache), HiddenStatesMode::None)?;
1876 step_logits = Some(out.logits.get(0)?.get(0)?);
1877 }
1878 assert_eq!(cache.seq_pos(0), prompt.len());
1879 assert_eq!(cache.past_len(0), 4);
1880 assert_eq!(cache.past_len(1), prompt.len());
1881
1882 let diff = full_logits
1883 .sub(&step_logits.expect("last step logits"))?
1884 .abs()?
1885 .max_all()?
1886 .to_scalar::<f32>()?;
1887 assert!(
1888 diff < 1e-4,
1889 "Gemma 4 sliding+full KV cache parity failed beyond window: diff={diff}"
1890 );
1891 Ok(())
1892 }
1893
1894 #[test]
1895 fn gemma4_moe_sparse_dispatch_matches_dense_reference_and_keeps_router_dense() -> Result<()> {
1896 let dev = Device::Cpu;
1897 let cfg = ModelConfig::tiny_gemma4_moe(96, 64, 2, 16, 8);
1898 let mut model = CausalLm::new_random(cfg.clone(), DType::F32, &dev, 77)?;
1899
1900 if let CausalLm::Gemma4(ref g4) = model {
1902 let moe = g4.layer_moe(0).expect("moe layer 0");
1903 let x = Tensor::randn(0.0f32, 0.5, (5, cfg.hidden_size), &dev)?;
1904 let dispatch = moe.router.forward(&x)?;
1905 assert_eq!(dispatch.top_k_indices.len(), 5);
1906 assert!(dispatch.top_k_indices.iter().all(|row| row.len() == 8));
1907
1908 let sparse_out = moe.experts.forward(&x, &dispatch)?;
1909
1910 let inter = moe.experts.intermediate_size;
1911 let mut ref_rows = Vec::with_capacity(5);
1912 for (t, (eids, weights)) in dispatch
1913 .top_k_indices
1914 .iter()
1915 .zip(dispatch.top_k_weights.iter())
1916 .enumerate()
1917 {
1918 let x_tok = x.narrow(0, t, 1)?;
1919 let mut row_sum = Tensor::zeros_like(&x_tok)?;
1920 for (&eid, &w) in eids.iter().zip(weights.iter()) {
1921 let exp = &moe.experts.experts[eid];
1922 let gu = exp.gate_up_proj.forward(&x_tok)?;
1923 let g = gu.narrow(1, 0, inter)?;
1924 let u = gu.narrow(1, inter, inter)?;
1925 let act = apply_activation(&g, &moe.experts.hidden_act)?.mul(&u)?;
1926 let y = (exp.down_proj.forward(&act)? * f64::from(w))?;
1927 row_sum = row_sum.add(&y)?;
1928 }
1929 ref_rows.push(row_sum);
1930 }
1931 let dense_ref = Tensor::cat(&ref_rows, 0)?;
1932 let diff = sparse_out
1933 .sub(&dense_ref)?
1934 .abs()?
1935 .max_all()?
1936 .to_scalar::<f32>()?;
1937 assert!(diff < 1e-5, "sparse vs reference expert sum diff: {diff}");
1938 }
1939
1940 let lb_cfg = LittleBitConfig {
1942 eff_bit: Some(1.0),
1943 residual: true,
1944 use_itq: true,
1945 itq_n_iter: 5,
1946 ..Default::default()
1947 };
1948 let summary = model.apply_littlebit_patch(&lb_cfg, TrainMode::Eval, 23)?;
1949 assert!(summary.patched_layers > 0);
1950 assert!(
1951 summary.skipped_layers >= cfg.num_hidden_layers,
1952 "all router projections must be recorded as skipped"
1953 );
1954 if let CausalLm::Gemma4(ref g4) = model {
1955 for layer_idx in 0..cfg.num_hidden_layers {
1956 let moe = g4.layer_moe(layer_idx).expect("moe");
1957 assert!(
1958 matches!(moe.router.proj, LinearLayer::Dense { .. }),
1959 "router at layer {layer_idx} must remain Dense"
1960 );
1961 assert!(
1962 matches!(
1963 moe.experts.experts[0].gate_up_proj,
1964 LinearLayer::LittleBit(_)
1965 ),
1966 "experts must be quantized to LittleBit"
1967 );
1968 }
1969 }
1970
1971 let input = Tensor::from_slice(&[1u32, 4, 9, 12], (1, 4), &dev)?;
1972 let out_unpacked = model.forward(&input, None, HiddenStatesMode::None)?;
1973
1974 let dir = TempDir::new().expect("tempdir");
1976 let wrapper = LittleBitModel::new(model, lb_cfg, Some("test/gemma4-moe".to_string()), None);
1977 wrapper.save_pretrained(dir.path(), None, CastPolicy::Preserve)?;
1978
1979 let reloaded_packed = LittleBitModel::from_pretrained(
1980 dir.path().to_str().expect("utf8"),
1981 &LoadOptions {
1982 inference_mode: InferenceMode::Packed,
1983 ..Default::default()
1984 },
1985 )?;
1986 if let CausalLm::Gemma4(ref g4) = reloaded_packed.model {
1987 let moe = g4.layer_moe(0).expect("moe");
1988 assert!(matches!(moe.router.proj, LinearLayer::Dense { .. }));
1989 assert!(matches!(
1990 moe.experts.experts[0].gate_up_proj,
1991 LinearLayer::Packed(_)
1992 ));
1993 }
1994
1995 let out_packed = reloaded_packed
1996 .model
1997 .forward(&input, None, HiddenStatesMode::None)?;
1998 let drift = out_unpacked
1999 .logits
2000 .sub(&out_packed.logits)?
2001 .abs()?
2002 .max_all()?
2003 .to_scalar::<f32>()?;
2004 assert!(drift < 1e-4, "Gemma 4 MoE packed reload drift: {drift}");
2005
2006 let last_full = out_packed.logits.get(0)?.get(3)?;
2008 let mut cache = KvCache::new(cfg.num_hidden_layers);
2009 let mut last_step = None;
2010 for tok in [1u32, 4, 9, 12] {
2011 let t = Tensor::from_slice(&[tok], (1, 1), &dev)?;
2012 last_step = Some(
2013 reloaded_packed
2014 .model
2015 .forward(&t, Some(&mut cache), HiddenStatesMode::None)?
2016 .logits
2017 .get(0)?
2018 .get(0)?,
2019 );
2020 }
2021 let kv_drift = last_full
2022 .sub(&last_step.expect("last step"))?
2023 .abs()?
2024 .max_all()?
2025 .to_scalar::<f32>()?;
2026 assert!(kv_drift < 1e-4, "Packed Gemma 4 MoE KV drift: {kv_drift}");
2027
2028 Ok(())
2029 }
2030
2031 #[test]
2032 fn gemma4_moe_loads_hf_fused_3d_expert_tensors_and_skips_unrouted_experts() -> Result<()> {
2033 let dev = Device::Cpu;
2034 let cfg = ModelConfig::tiny_gemma4_moe(64, 32, 1, 6, 2);
2035 let mut rng = ChaCha8Rng::seed_from_u64(99);
2036 let model = Gemma4Model::new_random(cfg.clone(), DType::F32, &dev, &mut rng)?;
2037 let mut sd = model.pack_state_dict()?;
2038
2039 let mut gu_slices = Vec::new();
2041 let mut dn_slices = Vec::new();
2042 for e in 0..6 {
2043 let gu = sd
2044 .remove(&format!("model.layers.0.experts.{e}.gate_up_proj.weight"))
2045 .expect("expert gu");
2046 let dn = sd
2047 .remove(&format!("model.layers.0.experts.{e}.down_proj.weight"))
2048 .expect("expert dn");
2049 gu_slices.push(gu);
2050 dn_slices.push(dn);
2051 }
2052 sd.insert(
2053 "model.layers.0.experts.gate_up_proj".to_string(),
2054 Tensor::stack(&gu_slices, 0)?,
2055 );
2056 sd.insert(
2057 "model.layers.0.experts.down_proj".to_string(),
2058 Tensor::stack(&dn_slices, 0)?,
2059 );
2060
2061 let loaded = Gemma4Model::from_tensors(
2062 cfg.clone(),
2063 &sd,
2064 LinearLoadSpec {
2065 littlebit: None,
2066 ratio_factor: 1.0,
2067 weight_state: littlebit_core::WeightState::Raw,
2068 inference_mode: InferenceMode::Unpacked,
2069 dtype: DType::F32,
2070 device: &dev,
2071 },
2072 )?;
2073
2074 let ids = Tensor::from_slice(&[1u32, 3, 5], (1, 3), &dev)?;
2075 let out_orig = model.forward(&ids, None, HiddenStatesMode::None)?;
2076 let out_3d = loaded.forward(&ids, None, HiddenStatesMode::None)?;
2077 let diff = out_orig
2078 .logits
2079 .sub(&out_3d.logits)?
2080 .abs()?
2081 .max_all()?
2082 .to_scalar::<f32>()?;
2083 assert!(diff < 1e-6, "HF 3D fused experts load mismatch: {diff}");
2084
2085 let mut model_sparse = loaded;
2088 let single_tok =
2089 Tensor::from_vec(vec![0.25f32; cfg.hidden_size], (1, cfg.hidden_size), &dev)?;
2090 let moe = model_sparse.layer_moe(0).expect("moe");
2091 let dispatch = moe.router.forward(&single_tok)?;
2092 let chosen = &dispatch.top_k_indices[0];
2093 let unused = (0..6)
2094 .find(|eid| !chosen.contains(eid))
2095 .expect("at least one unchosen expert");
2096
2097 let moe_mut = model_sparse.layer_moe_mut(0).expect("moe mut");
2098 let nan_shape = (
2099 2 * cfg.moe_intermediate_size.expect("moe inter"),
2100 cfg.hidden_size,
2101 );
2102 let nan_w = Tensor::from_vec(vec![f32::NAN; nan_shape.0 * nan_shape.1], nan_shape, &dev)?;
2103 moe_mut.experts.experts[unused].gate_up_proj = LinearLayer::Dense {
2104 weight: Param::Frozen(nan_w),
2105 bias: None,
2106 };
2107 let sparse_out = moe_mut.experts.forward(&single_tok, &dispatch)?;
2108 let sum_val = sparse_out.sum_all()?.to_scalar::<f32>()?;
2109 assert!(
2110 sum_val.is_finite(),
2111 "sparse dispatch touched unrouted expert {unused} (produced non-finite output)"
2112 );
2113
2114 Ok(())
2115 }
2116}