Skip to main content

littlebit_models/
gemma.rs

1//! Gemma 2 and Gemma 3 decoder architectures.
2
3use std::collections::HashMap;
4
5use candle_core::{DType, Device, Tensor, Var};
6use rand_chacha::ChaCha8Rng;
7
8use littlebit_core::{BiasMode, LayerShape, LittleBitConfig, TrainMode};
9
10use crate::config::{ModelArchitecture, ModelConfig};
11use crate::error::Result;
12use crate::layer::{
13    apply_activation, compute_attention, AttentionParams, Embedding, KvCache, LinearLayer,
14    LinearLoadSpec, NormKind, RmsNorm, RotaryEmbedding,
15};
16use crate::model::{CausalLmOutput, HiddenStatesMode};
17use crate::patch::{patch_linear_layer, PatchPolicy, PatchSummary};
18
19#[derive(Debug, Clone)]
20struct GemmaAttention {
21    q_proj: LinearLayer,
22    k_proj: LinearLayer,
23    v_proj: LinearLayer,
24    o_proj: LinearLayer,
25    q_norm: Option<RmsNorm>,
26    k_norm: Option<RmsNorm>,
27    num_heads: usize,
28    num_kv_heads: usize,
29    head_dim: usize,
30    scale: f64,
31    softcap: Option<f64>,
32    sliding_window: Option<usize>,
33    is_local: bool,
34}
35
36impl GemmaAttention {
37    #[allow(clippy::too_many_arguments)]
38    fn forward(
39        &self,
40        hidden_states: &Tensor,
41        rope_global: &RotaryEmbedding,
42        rope_local: Option<&RotaryEmbedding>,
43        layer_idx: usize,
44        cache: Option<&mut KvCache>,
45    ) -> Result<Tensor> {
46        let (batch, seq_len, _) = hidden_states.dims3()?;
47        let pos_offset = cache.as_ref().map_or(0, |c| c.past_len(layer_idx));
48
49        let mut q = self
50            .q_proj
51            .forward(hidden_states)?
52            .reshape((batch, seq_len, self.num_heads, self.head_dim))?
53            .transpose(1, 2)?
54            .contiguous()?;
55        let mut k = self
56            .k_proj
57            .forward(hidden_states)?
58            .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
59            .transpose(1, 2)?
60            .contiguous()?;
61        let v = self
62            .v_proj
63            .forward(hidden_states)?
64            .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
65            .transpose(1, 2)?
66            .contiguous()?;
67
68        if let Some(qn) = &self.q_norm {
69            q = qn.forward(&q)?;
70        }
71        if let Some(kn) = &self.k_norm {
72            k = kn.forward(&k)?;
73        }
74
75        let rope = if self.is_local {
76            rope_local.unwrap_or(rope_global)
77        } else {
78            rope_global
79        };
80        let q = rope.apply(&q, pos_offset)?;
81        let k = rope.apply(&k, pos_offset)?;
82
83        let attn_out = compute_attention(
84            &q,
85            &k,
86            &v,
87            layer_idx,
88            cache,
89            AttentionParams {
90                scale: self.scale,
91                softcap: self.softcap,
92                sliding_window: self.sliding_window,
93            },
94        )?;
95        self.o_proj.forward(&attn_out)
96    }
97}
98
99#[derive(Debug, Clone)]
100struct GemmaMlp {
101    gate_proj: LinearLayer,
102    up_proj: LinearLayer,
103    down_proj: LinearLayer,
104    hidden_act: String,
105}
106
107impl GemmaMlp {
108    fn forward(&self, x: &Tensor) -> Result<Tensor> {
109        let gate = apply_activation(&self.gate_proj.forward(x)?, &self.hidden_act)?;
110        let up = self.up_proj.forward(x)?;
111        self.down_proj.forward(&gate.mul(&up)?)
112    }
113}
114
115#[derive(Debug, Clone)]
116struct GemmaDecoderLayer {
117    input_layernorm: RmsNorm,
118    self_attn: GemmaAttention,
119    post_attention_layernorm: RmsNorm,
120    pre_feedforward_layernorm: RmsNorm,
121    mlp: GemmaMlp,
122    post_feedforward_layernorm: RmsNorm,
123}
124
125/// Causal language model supporting Gemma 2 and Gemma 3.
126#[derive(Debug, Clone)]
127pub struct GemmaModel {
128    config: ModelConfig,
129    embed_tokens: Embedding,
130    layers: Vec<GemmaDecoderLayer>,
131    norm: RmsNorm,
132    lm_head: Option<LinearLayer>,
133    rope_global: RotaryEmbedding,
134    rope_local: Option<RotaryEmbedding>,
135}
136
137fn layer_window_and_local(
138    cfg: &ModelConfig,
139    arch: ModelArchitecture,
140    layer_idx: usize,
141) -> (Option<usize>, bool) {
142    match arch {
143        ModelArchitecture::Gemma2 => {
144            let is_sliding = layer_idx.is_multiple_of(2);
145            (if is_sliding { cfg.sliding_window } else { None }, false)
146        }
147        ModelArchitecture::Gemma3 => {
148            let pattern = cfg.sliding_window_pattern.unwrap_or(6).max(1);
149            let is_local = !(layer_idx + 1).is_multiple_of(pattern);
150            (if is_local { cfg.sliding_window } else { None }, is_local)
151        }
152        _ => (cfg.sliding_window, false),
153    }
154}
155
156impl GemmaModel {
157    /// Load Gemma 2 / Gemma 3 weights from a state dictionary.
158    ///
159    /// # Errors
160    /// Returns [`crate::ModelError::MissingWeight`] if expected tensors are absent.
161    pub fn from_tensors(
162        config: ModelConfig,
163        tensors: &HashMap<String, Tensor>,
164        spec: LinearLoadSpec<'_>,
165    ) -> Result<Self> {
166        let arch = config.detect_architecture()?;
167        let embed_scale = Some((config.hidden_size as f64).sqrt());
168        let embed_tokens = Embedding::from_tensors(
169            "model.embed_tokens",
170            tensors,
171            embed_scale,
172            spec.dtype,
173            spec.device,
174        )?;
175        let head_dim = config.effective_head_dim();
176        let num_heads = config.num_attention_heads;
177        let num_kv_heads = config.effective_kv_heads();
178        let inter = config.effective_intermediate_size();
179        let attn_bias = BiasMode::from(config.attention_bias);
180        let mlp_bias = BiasMode::from(config.mlp_bias);
181        let kv_factor = spec.littlebit.map_or(1.0, |c| c.kv_factor);
182        let scale = 1.0
183            / config
184                .query_pre_attn_scalar
185                .unwrap_or(head_dim as f64)
186                .sqrt();
187
188        let kv_spec = LinearLoadSpec {
189            ratio_factor: kv_factor,
190            ..spec
191        };
192        let unit_spec = LinearLoadSpec {
193            ratio_factor: 1.0,
194            ..spec
195        };
196
197        let mut layers = Vec::with_capacity(config.num_hidden_layers);
198        for i in 0..config.num_hidden_layers {
199            let p = format!("model.layers.{i}");
200            let (sliding_window, is_local) = layer_window_and_local(&config, arch, i);
201
202            let input_layernorm = RmsNorm::from_tensors(
203                &format!("{p}.input_layernorm"),
204                tensors,
205                config.rms_norm_eps,
206                NormKind::PlusOne,
207                spec.dtype,
208                spec.device,
209            )?;
210            let q_proj = LinearLayer::from_tensors(
211                &format!("{p}.self_attn.q_proj"),
212                tensors,
213                LayerShape::new(config.hidden_size, num_heads * head_dim, attn_bias),
214                unit_spec,
215            )?;
216            let k_proj = LinearLayer::from_tensors(
217                &format!("{p}.self_attn.k_proj"),
218                tensors,
219                LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
220                kv_spec,
221            )?;
222            let v_proj = LinearLayer::from_tensors(
223                &format!("{p}.self_attn.v_proj"),
224                tensors,
225                LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
226                kv_spec,
227            )?;
228            let o_proj = LinearLayer::from_tensors(
229                &format!("{p}.self_attn.o_proj"),
230                tensors,
231                LayerShape::new(num_heads * head_dim, config.hidden_size, attn_bias),
232                unit_spec,
233            )?;
234            let q_norm = if tensors.contains_key(&format!("{p}.self_attn.q_norm.weight")) {
235                Some(RmsNorm::from_tensors(
236                    &format!("{p}.self_attn.q_norm"),
237                    tensors,
238                    config.rms_norm_eps,
239                    NormKind::PlusOne,
240                    spec.dtype,
241                    spec.device,
242                )?)
243            } else {
244                None
245            };
246            let k_norm = if tensors.contains_key(&format!("{p}.self_attn.k_norm.weight")) {
247                Some(RmsNorm::from_tensors(
248                    &format!("{p}.self_attn.k_norm"),
249                    tensors,
250                    config.rms_norm_eps,
251                    NormKind::PlusOne,
252                    spec.dtype,
253                    spec.device,
254                )?)
255            } else {
256                None
257            };
258            let post_attention_layernorm = RmsNorm::from_tensors(
259                &format!("{p}.post_attention_layernorm"),
260                tensors,
261                config.rms_norm_eps,
262                NormKind::PlusOne,
263                spec.dtype,
264                spec.device,
265            )?;
266            let pre_feedforward_layernorm = RmsNorm::from_tensors(
267                &format!("{p}.pre_feedforward_layernorm"),
268                tensors,
269                config.rms_norm_eps,
270                NormKind::PlusOne,
271                spec.dtype,
272                spec.device,
273            )?;
274            let gate_proj = LinearLayer::from_tensors(
275                &format!("{p}.mlp.gate_proj"),
276                tensors,
277                LayerShape::new(config.hidden_size, inter, mlp_bias),
278                unit_spec,
279            )?;
280            let up_proj = LinearLayer::from_tensors(
281                &format!("{p}.mlp.up_proj"),
282                tensors,
283                LayerShape::new(config.hidden_size, inter, mlp_bias),
284                unit_spec,
285            )?;
286            let down_proj = LinearLayer::from_tensors(
287                &format!("{p}.mlp.down_proj"),
288                tensors,
289                LayerShape::new(inter, config.hidden_size, mlp_bias),
290                unit_spec,
291            )?;
292            let post_feedforward_layernorm = RmsNorm::from_tensors(
293                &format!("{p}.post_feedforward_layernorm"),
294                tensors,
295                config.rms_norm_eps,
296                NormKind::PlusOne,
297                spec.dtype,
298                spec.device,
299            )?;
300
301            layers.push(GemmaDecoderLayer {
302                input_layernorm,
303                self_attn: GemmaAttention {
304                    q_proj,
305                    k_proj,
306                    v_proj,
307                    o_proj,
308                    q_norm,
309                    k_norm,
310                    num_heads,
311                    num_kv_heads,
312                    head_dim,
313                    scale,
314                    softcap: config.attn_logit_softcapping,
315                    sliding_window,
316                    is_local,
317                },
318                post_attention_layernorm,
319                pre_feedforward_layernorm,
320                mlp: GemmaMlp {
321                    gate_proj,
322                    up_proj,
323                    down_proj,
324                    hidden_act: config.hidden_act.clone(),
325                },
326                post_feedforward_layernorm,
327            });
328        }
329
330        let norm = RmsNorm::from_tensors(
331            "model.norm",
332            tensors,
333            config.rms_norm_eps,
334            NormKind::PlusOne,
335            spec.dtype,
336            spec.device,
337        )?;
338        let lm_head = if !config.tie_word_embeddings && tensors.contains_key("lm_head.weight") {
339            Some(LinearLayer::from_tensors(
340                "lm_head",
341                tensors,
342                LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
343                LinearLoadSpec {
344                    littlebit: None,
345                    ratio_factor: 1.0,
346                    ..spec
347                },
348            )?)
349        } else {
350            None
351        };
352
353        let rope_global = RotaryEmbedding::new(
354            head_dim,
355            config.max_position_embeddings.max(256),
356            config.rope_theta,
357            spec.dtype,
358            spec.device,
359        )?;
360        let rope_local = match config.rope_local_base_freq {
361            Some(theta) => Some(RotaryEmbedding::new(
362                head_dim,
363                config.max_position_embeddings.max(256),
364                theta,
365                spec.dtype,
366                spec.device,
367            )?),
368            None => None,
369        };
370
371        Ok(Self {
372            config,
373            embed_tokens,
374            layers,
375            norm,
376            lm_head,
377            rope_global,
378            rope_local,
379        })
380    }
381
382    /// Initialize a random Gemma 2 / Gemma 3 model for testing.
383    ///
384    /// # Errors
385    /// Returns an error if tensor creation fails.
386    pub fn new_random(
387        config: ModelConfig,
388        dtype: DType,
389        device: &Device,
390        rng: &mut ChaCha8Rng,
391    ) -> Result<Self> {
392        let arch = config.detect_architecture()?;
393        let embed_scale = Some((config.hidden_size as f64).sqrt());
394        let embed_tokens = Embedding::new_random(
395            config.vocab_size,
396            config.hidden_size,
397            embed_scale,
398            dtype,
399            device,
400            rng,
401        )?;
402        let head_dim = config.effective_head_dim();
403        let num_heads = config.num_attention_heads;
404        let num_kv_heads = config.effective_kv_heads();
405        let inter = config.effective_intermediate_size();
406        let attn_bias = BiasMode::from(config.attention_bias);
407        let mlp_bias = BiasMode::from(config.mlp_bias);
408        let scale = 1.0
409            / config
410                .query_pre_attn_scalar
411                .unwrap_or(head_dim as f64)
412                .sqrt();
413
414        let mut layers = Vec::with_capacity(config.num_hidden_layers);
415        for i in 0..config.num_hidden_layers {
416            let (sliding_window, is_local) = layer_window_and_local(&config, arch, i);
417            let (q_norm, k_norm) = if arch == ModelArchitecture::Gemma3 {
418                (
419                    Some(RmsNorm::new_init(
420                        head_dim,
421                        config.rms_norm_eps,
422                        NormKind::PlusOne,
423                        dtype,
424                        device,
425                    )?),
426                    Some(RmsNorm::new_init(
427                        head_dim,
428                        config.rms_norm_eps,
429                        NormKind::PlusOne,
430                        dtype,
431                        device,
432                    )?),
433                )
434            } else {
435                (None, None)
436            };
437            layers.push(GemmaDecoderLayer {
438                input_layernorm: RmsNorm::new_init(
439                    config.hidden_size,
440                    config.rms_norm_eps,
441                    NormKind::PlusOne,
442                    dtype,
443                    device,
444                )?,
445                self_attn: GemmaAttention {
446                    q_proj: LinearLayer::new_random(
447                        LayerShape::new(config.hidden_size, num_heads * head_dim, attn_bias),
448                        dtype,
449                        device,
450                        rng,
451                    )?,
452                    k_proj: LinearLayer::new_random(
453                        LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
454                        dtype,
455                        device,
456                        rng,
457                    )?,
458                    v_proj: LinearLayer::new_random(
459                        LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
460                        dtype,
461                        device,
462                        rng,
463                    )?,
464                    o_proj: LinearLayer::new_random(
465                        LayerShape::new(num_heads * head_dim, config.hidden_size, attn_bias),
466                        dtype,
467                        device,
468                        rng,
469                    )?,
470                    q_norm,
471                    k_norm,
472                    num_heads,
473                    num_kv_heads,
474                    head_dim,
475                    scale,
476                    softcap: config.attn_logit_softcapping,
477                    sliding_window,
478                    is_local,
479                },
480                post_attention_layernorm: RmsNorm::new_init(
481                    config.hidden_size,
482                    config.rms_norm_eps,
483                    NormKind::PlusOne,
484                    dtype,
485                    device,
486                )?,
487                pre_feedforward_layernorm: RmsNorm::new_init(
488                    config.hidden_size,
489                    config.rms_norm_eps,
490                    NormKind::PlusOne,
491                    dtype,
492                    device,
493                )?,
494                mlp: GemmaMlp {
495                    gate_proj: LinearLayer::new_random(
496                        LayerShape::new(config.hidden_size, inter, mlp_bias),
497                        dtype,
498                        device,
499                        rng,
500                    )?,
501                    up_proj: LinearLayer::new_random(
502                        LayerShape::new(config.hidden_size, inter, mlp_bias),
503                        dtype,
504                        device,
505                        rng,
506                    )?,
507                    down_proj: LinearLayer::new_random(
508                        LayerShape::new(inter, config.hidden_size, mlp_bias),
509                        dtype,
510                        device,
511                        rng,
512                    )?,
513                    hidden_act: config.hidden_act.clone(),
514                },
515                post_feedforward_layernorm: RmsNorm::new_init(
516                    config.hidden_size,
517                    config.rms_norm_eps,
518                    NormKind::PlusOne,
519                    dtype,
520                    device,
521                )?,
522            });
523        }
524
525        let norm = RmsNorm::new_init(
526            config.hidden_size,
527            config.rms_norm_eps,
528            NormKind::PlusOne,
529            dtype,
530            device,
531        )?;
532        let lm_head = if config.tie_word_embeddings {
533            None
534        } else {
535            Some(LinearLayer::new_random(
536                LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
537                dtype,
538                device,
539                rng,
540            )?)
541        };
542        let rope_global = RotaryEmbedding::new(
543            head_dim,
544            config.max_position_embeddings.max(256),
545            config.rope_theta,
546            dtype,
547            device,
548        )?;
549        let rope_local = match config.rope_local_base_freq {
550            Some(theta) => Some(RotaryEmbedding::new(
551                head_dim,
552                config.max_position_embeddings.max(256),
553                theta,
554                dtype,
555                device,
556            )?),
557            None => None,
558        };
559        Ok(Self {
560            config,
561            embed_tokens,
562            layers,
563            norm,
564            lm_head,
565            rope_global,
566            rope_local,
567        })
568    }
569
570    /// Forward pass returning logits `(batch, seq_len, vocab_size)` and optional hidden states.
571    ///
572    /// # Errors
573    /// Returns an error if layer forward passes fail.
574    pub fn forward(
575        &self,
576        input_ids: &Tensor,
577        mut cache: Option<&mut KvCache>,
578        hidden_mode: HiddenStatesMode,
579    ) -> Result<CausalLmOutput> {
580        let mut hidden = self.embed_tokens.forward(input_ids)?;
581        let collect = hidden_mode == HiddenStatesMode::All;
582        let mut all_hidden = if collect {
583            Vec::with_capacity(self.layers.len() + 1)
584        } else {
585            Vec::new()
586        };
587
588        for (idx, layer) in self.layers.iter().enumerate() {
589            if collect {
590                all_hidden.push(hidden.clone());
591            }
592            let residual = hidden.clone();
593            let normed = layer.input_layernorm.forward(&hidden)?;
594            let attn = layer.self_attn.forward(
595                &normed,
596                &self.rope_global,
597                self.rope_local.as_ref(),
598                idx,
599                cache.as_deref_mut(),
600            )?;
601            let attn_normed = layer.post_attention_layernorm.forward(&attn)?;
602            let h_mid = residual.add(&attn_normed)?;
603
604            let pre_ff = layer.pre_feedforward_layernorm.forward(&h_mid)?;
605            let ff_out = layer.mlp.forward(&pre_ff)?;
606            let ff_normed = layer.post_feedforward_layernorm.forward(&ff_out)?;
607            hidden = h_mid.add(&ff_normed)?;
608        }
609
610        hidden = self.norm.forward(&hidden)?;
611        if collect {
612            all_hidden.push(hidden.clone());
613        }
614        let mut logits = match &self.lm_head {
615            Some(head) => head.forward(&hidden)?,
616            None => self.embed_tokens.forward_lm_head(&hidden)?,
617        };
618        if let Some(cap) = self.config.final_logit_softcapping {
619            logits = ((logits / cap)?.tanh()? * cap)?;
620        }
621        Ok(CausalLmOutput {
622            logits,
623            hidden_states: all_hidden,
624        })
625    }
626
627    pub(crate) fn apply_littlebit_patch(
628        &mut self,
629        lb_cfg: &LittleBitConfig,
630        policy: &PatchPolicy,
631        train_mode: TrainMode,
632        base_seed: u64,
633        summary: &mut PatchSummary,
634    ) -> Result<()> {
635        for (i, layer) in self.layers.iter_mut().enumerate() {
636            let p = format!("model.layers.{i}");
637            patch_linear_layer(
638                &format!("{p}.self_attn.q_proj"),
639                &mut layer.self_attn.q_proj,
640                lb_cfg,
641                policy,
642                train_mode,
643                base_seed,
644                summary,
645            )?;
646            patch_linear_layer(
647                &format!("{p}.self_attn.k_proj"),
648                &mut layer.self_attn.k_proj,
649                lb_cfg,
650                policy,
651                train_mode,
652                base_seed,
653                summary,
654            )?;
655            patch_linear_layer(
656                &format!("{p}.self_attn.v_proj"),
657                &mut layer.self_attn.v_proj,
658                lb_cfg,
659                policy,
660                train_mode,
661                base_seed,
662                summary,
663            )?;
664            patch_linear_layer(
665                &format!("{p}.self_attn.o_proj"),
666                &mut layer.self_attn.o_proj,
667                lb_cfg,
668                policy,
669                train_mode,
670                base_seed,
671                summary,
672            )?;
673            patch_linear_layer(
674                &format!("{p}.mlp.gate_proj"),
675                &mut layer.mlp.gate_proj,
676                lb_cfg,
677                policy,
678                train_mode,
679                base_seed,
680                summary,
681            )?;
682            patch_linear_layer(
683                &format!("{p}.mlp.up_proj"),
684                &mut layer.mlp.up_proj,
685                lb_cfg,
686                policy,
687                train_mode,
688                base_seed,
689                summary,
690            )?;
691            patch_linear_layer(
692                &format!("{p}.mlp.down_proj"),
693                &mut layer.mlp.down_proj,
694                lb_cfg,
695                policy,
696                train_mode,
697                base_seed,
698                summary,
699            )?;
700        }
701        if let Some(head) = &mut self.lm_head {
702            patch_linear_layer(
703                "lm_head", head, lb_cfg, policy, train_mode, base_seed, summary,
704            )?;
705        }
706        Ok(())
707    }
708
709    pub(crate) fn convert_to_packed(&mut self) -> Result<()> {
710        for layer in &mut self.layers {
711            layer.self_attn.q_proj.convert_to_packed()?;
712            layer.self_attn.k_proj.convert_to_packed()?;
713            layer.self_attn.v_proj.convert_to_packed()?;
714            layer.self_attn.o_proj.convert_to_packed()?;
715            layer.mlp.gate_proj.convert_to_packed()?;
716            layer.mlp.up_proj.convert_to_packed()?;
717            layer.mlp.down_proj.convert_to_packed()?;
718        }
719        if let Some(head) = &mut self.lm_head {
720            head.convert_to_packed()?;
721        }
722        Ok(())
723    }
724
725    pub(crate) fn make_non_quantized_trainable(&mut self) -> Result<()> {
726        self.embed_tokens.make_trainable()?;
727        for layer in &mut self.layers {
728            layer.input_layernorm.make_trainable()?;
729            layer.post_attention_layernorm.make_trainable()?;
730            layer.pre_feedforward_layernorm.make_trainable()?;
731            layer.post_feedforward_layernorm.make_trainable()?;
732            layer.self_attn.q_proj.make_trainable()?;
733            layer.self_attn.k_proj.make_trainable()?;
734            layer.self_attn.v_proj.make_trainable()?;
735            layer.self_attn.o_proj.make_trainable()?;
736            if let Some(qn) = &mut layer.self_attn.q_norm {
737                qn.make_trainable()?;
738            }
739            if let Some(kn) = &mut layer.self_attn.k_norm {
740                kn.make_trainable()?;
741            }
742            layer.mlp.gate_proj.make_trainable()?;
743            layer.mlp.up_proj.make_trainable()?;
744            layer.mlp.down_proj.make_trainable()?;
745        }
746        self.norm.make_trainable()?;
747        if let Some(head) = &mut self.lm_head {
748            head.make_trainable()?;
749        }
750        Ok(())
751    }
752
753    pub(crate) fn freeze_matching(&mut self, exclude: &[&str]) {
754        let matches = |name: &str| exclude.iter().any(|k| name.contains(k));
755        if matches("model.embed_tokens") {
756            self.embed_tokens.freeze();
757        }
758        if matches("model.norm") {
759            self.norm.freeze();
760        }
761        if let Some(head) = &mut self.lm_head {
762            if matches("lm_head") {
763                head.freeze();
764            }
765        }
766        for (i, layer) in self.layers.iter_mut().enumerate() {
767            let p = format!("model.layers.{i}");
768            if matches(&format!("{p}.input_layernorm")) {
769                layer.input_layernorm.freeze();
770            }
771            if matches(&format!("{p}.post_attention_layernorm")) {
772                layer.post_attention_layernorm.freeze();
773            }
774            if matches(&format!("{p}.pre_feedforward_layernorm")) {
775                layer.pre_feedforward_layernorm.freeze();
776            }
777            if matches(&format!("{p}.post_feedforward_layernorm")) {
778                layer.post_feedforward_layernorm.freeze();
779            }
780            if matches(&format!("{p}.self_attn")) {
781                layer.self_attn.q_proj.freeze();
782                layer.self_attn.k_proj.freeze();
783                layer.self_attn.v_proj.freeze();
784                layer.self_attn.o_proj.freeze();
785            }
786            if matches(&format!("{p}.mlp")) {
787                layer.mlp.gate_proj.freeze();
788                layer.mlp.up_proj.freeze();
789                layer.mlp.down_proj.freeze();
790            }
791        }
792    }
793
794    pub(crate) fn trainable_vars(&self) -> Vec<Var> {
795        let mut vars = self.embed_tokens.trainable_vars();
796        for layer in &self.layers {
797            vars.extend(layer.input_layernorm.trainable_vars());
798            vars.extend(layer.self_attn.q_proj.trainable_vars());
799            vars.extend(layer.self_attn.k_proj.trainable_vars());
800            vars.extend(layer.self_attn.v_proj.trainable_vars());
801            vars.extend(layer.self_attn.o_proj.trainable_vars());
802            if let Some(qn) = &layer.self_attn.q_norm {
803                vars.extend(qn.trainable_vars());
804            }
805            if let Some(kn) = &layer.self_attn.k_norm {
806                vars.extend(kn.trainable_vars());
807            }
808            vars.extend(layer.post_attention_layernorm.trainable_vars());
809            vars.extend(layer.pre_feedforward_layernorm.trainable_vars());
810            vars.extend(layer.mlp.gate_proj.trainable_vars());
811            vars.extend(layer.mlp.up_proj.trainable_vars());
812            vars.extend(layer.mlp.down_proj.trainable_vars());
813            vars.extend(layer.post_feedforward_layernorm.trainable_vars());
814        }
815        vars.extend(self.norm.trainable_vars());
816        if let Some(head) = &self.lm_head {
817            vars.extend(head.trainable_vars());
818        }
819        vars
820    }
821
822    pub(crate) fn total_params(&self) -> usize {
823        let mut count = self.embed_tokens.weight.elem_count() + self.norm.weight.elem_count();
824        for layer in &self.layers {
825            count += layer.input_layernorm.weight.elem_count()
826                + layer.post_attention_layernorm.weight.elem_count()
827                + layer.pre_feedforward_layernorm.weight.elem_count()
828                + layer.post_feedforward_layernorm.weight.elem_count()
829                + layer.self_attn.q_proj.param_count()
830                + layer.self_attn.k_proj.param_count()
831                + layer.self_attn.v_proj.param_count()
832                + layer.self_attn.o_proj.param_count()
833                + layer
834                    .self_attn
835                    .q_norm
836                    .as_ref()
837                    .map_or(0, |n| n.weight.elem_count())
838                + layer
839                    .self_attn
840                    .k_norm
841                    .as_ref()
842                    .map_or(0, |n| n.weight.elem_count())
843                + layer.mlp.gate_proj.param_count()
844                + layer.mlp.up_proj.param_count()
845                + layer.mlp.down_proj.param_count();
846        }
847        if let Some(head) = &self.lm_head {
848            count += head.param_count();
849        }
850        count
851    }
852
853    pub(crate) fn pack_state_dict(&self) -> Result<HashMap<String, Tensor>> {
854        let mut out = HashMap::new();
855        self.embed_tokens
856            .pack_state_dict("model.embed_tokens", &mut out);
857        for (i, layer) in self.layers.iter().enumerate() {
858            let p = format!("model.layers.{i}");
859            layer
860                .input_layernorm
861                .pack_state_dict(&format!("{p}.input_layernorm"), &mut out);
862            out.extend(
863                layer
864                    .self_attn
865                    .q_proj
866                    .pack_state_dict(&format!("{p}.self_attn.q_proj"))?,
867            );
868            out.extend(
869                layer
870                    .self_attn
871                    .k_proj
872                    .pack_state_dict(&format!("{p}.self_attn.k_proj"))?,
873            );
874            out.extend(
875                layer
876                    .self_attn
877                    .v_proj
878                    .pack_state_dict(&format!("{p}.self_attn.v_proj"))?,
879            );
880            out.extend(
881                layer
882                    .self_attn
883                    .o_proj
884                    .pack_state_dict(&format!("{p}.self_attn.o_proj"))?,
885            );
886            if let Some(qn) = &layer.self_attn.q_norm {
887                qn.pack_state_dict(&format!("{p}.self_attn.q_norm"), &mut out);
888            }
889            if let Some(kn) = &layer.self_attn.k_norm {
890                kn.pack_state_dict(&format!("{p}.self_attn.k_norm"), &mut out);
891            }
892            layer
893                .post_attention_layernorm
894                .pack_state_dict(&format!("{p}.post_attention_layernorm"), &mut out);
895            layer
896                .pre_feedforward_layernorm
897                .pack_state_dict(&format!("{p}.pre_feedforward_layernorm"), &mut out);
898            out.extend(
899                layer
900                    .mlp
901                    .gate_proj
902                    .pack_state_dict(&format!("{p}.mlp.gate_proj"))?,
903            );
904            out.extend(
905                layer
906                    .mlp
907                    .up_proj
908                    .pack_state_dict(&format!("{p}.mlp.up_proj"))?,
909            );
910            out.extend(
911                layer
912                    .mlp
913                    .down_proj
914                    .pack_state_dict(&format!("{p}.mlp.down_proj"))?,
915            );
916            layer
917                .post_feedforward_layernorm
918                .pack_state_dict(&format!("{p}.post_feedforward_layernorm"), &mut out);
919        }
920        self.norm.pack_state_dict("model.norm", &mut out);
921        if let Some(head) = &self.lm_head {
922            out.extend(head.pack_state_dict("lm_head")?);
923        }
924        Ok(out)
925    }
926
927    pub(crate) fn config(&self) -> &ModelConfig {
928        &self.config
929    }
930}