Skip to main content

littlebit_models/
llama.rs

1//! Unified decoder implementation for Llama, Llama 2/3, Qwen2/2.5/QwQ, and Qwen3.
2
3use std::collections::HashMap;
4
5use candle_core::{DType, Device, Tensor, Var};
6use rand_chacha::ChaCha8Rng;
7
8use littlebit_core::{BiasMode, LayerShape, LittleBitConfig, TrainMode, WeightState};
9
10use crate::config::{ModelArchitecture, ModelConfig};
11use crate::error::Result;
12use crate::layer::{
13    apply_activation, compute_attention, AttentionParams, Embedding, KvCache, LinearLayer,
14    LinearLoadSpec, NormKind, RmsNorm, RotaryEmbedding,
15};
16use crate::model::{CausalLmOutput, HiddenStatesMode};
17use crate::patch::{patch_linear_layer, PatchPolicy, PatchSummary};
18
19#[derive(Debug, Clone)]
20struct LlamaAttention {
21    q_proj: LinearLayer,
22    k_proj: LinearLayer,
23    v_proj: LinearLayer,
24    o_proj: LinearLayer,
25    q_norm: Option<RmsNorm>,
26    k_norm: Option<RmsNorm>,
27    num_heads: usize,
28    num_kv_heads: usize,
29    head_dim: usize,
30}
31
32impl LlamaAttention {
33    #[allow(clippy::too_many_arguments)]
34    fn from_tensors(
35        prefix: &str,
36        tensors: &HashMap<String, Tensor>,
37        cfg: &ModelConfig,
38        arch: ModelArchitecture,
39        littlebit: Option<&LittleBitConfig>,
40        weight_state: WeightState,
41        inference_mode: littlebit_core::InferenceMode,
42        dtype: DType,
43        device: &Device,
44    ) -> Result<Self> {
45        let head_dim = cfg.effective_head_dim();
46        let num_heads = cfg.num_attention_heads;
47        let num_kv_heads = cfg.effective_kv_heads();
48        let kv_factor = littlebit.map_or(1.0, |c| c.kv_factor);
49
50        let bias_for = |name: &str, default_bias: bool| -> BiasMode {
51            BiasMode::from(default_bias || tensors.contains_key(&format!("{prefix}.{name}.bias")))
52        };
53
54        let q_bias = bias_for(
55            "q_proj",
56            cfg.attention_bias || arch == ModelArchitecture::Qwen2,
57        );
58        let k_bias = bias_for(
59            "k_proj",
60            cfg.attention_bias || arch == ModelArchitecture::Qwen2,
61        );
62        let v_bias = bias_for(
63            "v_proj",
64            cfg.attention_bias || arch == ModelArchitecture::Qwen2,
65        );
66        let o_bias = bias_for("o_proj", cfg.attention_bias);
67
68        let base_spec = LinearLoadSpec {
69            littlebit,
70            ratio_factor: 1.0,
71            weight_state,
72            inference_mode,
73            dtype,
74            device,
75        };
76        let kv_spec = LinearLoadSpec {
77            ratio_factor: kv_factor,
78            ..base_spec
79        };
80
81        let q_proj = LinearLayer::from_tensors(
82            &format!("{prefix}.q_proj"),
83            tensors,
84            LayerShape::new(cfg.hidden_size, num_heads * head_dim, q_bias),
85            base_spec,
86        )?;
87        let k_proj = LinearLayer::from_tensors(
88            &format!("{prefix}.k_proj"),
89            tensors,
90            LayerShape::new(cfg.hidden_size, num_kv_heads * head_dim, k_bias),
91            kv_spec,
92        )?;
93        let v_proj = LinearLayer::from_tensors(
94            &format!("{prefix}.v_proj"),
95            tensors,
96            LayerShape::new(cfg.hidden_size, num_kv_heads * head_dim, v_bias),
97            kv_spec,
98        )?;
99        let o_proj = LinearLayer::from_tensors(
100            &format!("{prefix}.o_proj"),
101            tensors,
102            LayerShape::new(num_heads * head_dim, cfg.hidden_size, o_bias),
103            base_spec,
104        )?;
105
106        let q_norm = if tensors.contains_key(&format!("{prefix}.q_norm.weight")) {
107            Some(RmsNorm::from_tensors(
108                &format!("{prefix}.q_norm"),
109                tensors,
110                cfg.rms_norm_eps,
111                NormKind::Standard,
112                dtype,
113                device,
114            )?)
115        } else {
116            None
117        };
118        let k_norm = if tensors.contains_key(&format!("{prefix}.k_norm.weight")) {
119            Some(RmsNorm::from_tensors(
120                &format!("{prefix}.k_norm"),
121                tensors,
122                cfg.rms_norm_eps,
123                NormKind::Standard,
124                dtype,
125                device,
126            )?)
127        } else {
128            None
129        };
130
131        Ok(Self {
132            q_proj,
133            k_proj,
134            v_proj,
135            o_proj,
136            q_norm,
137            k_norm,
138            num_heads,
139            num_kv_heads,
140            head_dim,
141        })
142    }
143
144    fn new_random(
145        cfg: &ModelConfig,
146        arch: ModelArchitecture,
147        dtype: DType,
148        device: &Device,
149        rng: &mut ChaCha8Rng,
150    ) -> Result<Self> {
151        let head_dim = cfg.effective_head_dim();
152        let num_heads = cfg.num_attention_heads;
153        let num_kv_heads = cfg.effective_kv_heads();
154        let qkv_bias = BiasMode::from(cfg.attention_bias || arch == ModelArchitecture::Qwen2);
155        let o_bias = BiasMode::from(cfg.attention_bias);
156
157        let q_proj = LinearLayer::new_random(
158            LayerShape::new(cfg.hidden_size, num_heads * head_dim, qkv_bias),
159            dtype,
160            device,
161            rng,
162        )?;
163        let k_proj = LinearLayer::new_random(
164            LayerShape::new(cfg.hidden_size, num_kv_heads * head_dim, qkv_bias),
165            dtype,
166            device,
167            rng,
168        )?;
169        let v_proj = LinearLayer::new_random(
170            LayerShape::new(cfg.hidden_size, num_kv_heads * head_dim, qkv_bias),
171            dtype,
172            device,
173            rng,
174        )?;
175        let o_proj = LinearLayer::new_random(
176            LayerShape::new(num_heads * head_dim, cfg.hidden_size, o_bias),
177            dtype,
178            device,
179            rng,
180        )?;
181
182        let (q_norm, k_norm) = if arch == ModelArchitecture::Qwen3 {
183            (
184                Some(RmsNorm::new_init(
185                    head_dim,
186                    cfg.rms_norm_eps,
187                    NormKind::Standard,
188                    dtype,
189                    device,
190                )?),
191                Some(RmsNorm::new_init(
192                    head_dim,
193                    cfg.rms_norm_eps,
194                    NormKind::Standard,
195                    dtype,
196                    device,
197                )?),
198            )
199        } else {
200            (None, None)
201        };
202
203        Ok(Self {
204            q_proj,
205            k_proj,
206            v_proj,
207            o_proj,
208            q_norm,
209            k_norm,
210            num_heads,
211            num_kv_heads,
212            head_dim,
213        })
214    }
215
216    fn forward(
217        &self,
218        hidden_states: &Tensor,
219        rope: &RotaryEmbedding,
220        layer_idx: usize,
221        cache: Option<&mut KvCache>,
222    ) -> Result<Tensor> {
223        let (batch, seq_len, _) = hidden_states.dims3()?;
224        let pos_offset = cache.as_ref().map_or(0, |c| c.past_len(layer_idx));
225
226        let mut q = self
227            .q_proj
228            .forward(hidden_states)?
229            .reshape((batch, seq_len, self.num_heads, self.head_dim))?
230            .transpose(1, 2)?
231            .contiguous()?;
232        let mut k = self
233            .k_proj
234            .forward(hidden_states)?
235            .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
236            .transpose(1, 2)?
237            .contiguous()?;
238        let v = self
239            .v_proj
240            .forward(hidden_states)?
241            .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
242            .transpose(1, 2)?
243            .contiguous()?;
244
245        if let Some(qn) = &self.q_norm {
246            q = qn.forward(&q)?;
247        }
248        if let Some(kn) = &self.k_norm {
249            k = kn.forward(&k)?;
250        }
251
252        let q = rope.apply(&q, pos_offset)?;
253        let k = rope.apply(&k, pos_offset)?;
254
255        let attn_out = compute_attention(
256            &q,
257            &k,
258            &v,
259            layer_idx,
260            cache,
261            AttentionParams {
262                scale: 1.0 / (self.head_dim as f64).sqrt(),
263                softcap: None,
264                sliding_window: None,
265            },
266        )?;
267        self.o_proj.forward(&attn_out)
268    }
269}
270
271#[derive(Debug, Clone)]
272struct LlamaMlp {
273    gate_proj: LinearLayer,
274    up_proj: LinearLayer,
275    down_proj: LinearLayer,
276    hidden_act: String,
277}
278
279impl LlamaMlp {
280    fn from_tensors(
281        prefix: &str,
282        tensors: &HashMap<String, Tensor>,
283        cfg: &ModelConfig,
284        spec: LinearLoadSpec<'_>,
285    ) -> Result<Self> {
286        let inter = cfg.effective_intermediate_size();
287        let bias = BiasMode::from(cfg.mlp_bias);
288        Ok(Self {
289            gate_proj: LinearLayer::from_tensors(
290                &format!("{prefix}.gate_proj"),
291                tensors,
292                LayerShape::new(cfg.hidden_size, inter, bias),
293                spec,
294            )?,
295            up_proj: LinearLayer::from_tensors(
296                &format!("{prefix}.up_proj"),
297                tensors,
298                LayerShape::new(cfg.hidden_size, inter, bias),
299                spec,
300            )?,
301            down_proj: LinearLayer::from_tensors(
302                &format!("{prefix}.down_proj"),
303                tensors,
304                LayerShape::new(inter, cfg.hidden_size, bias),
305                spec,
306            )?,
307            hidden_act: cfg.hidden_act.clone(),
308        })
309    }
310
311    fn new_random(
312        cfg: &ModelConfig,
313        dtype: DType,
314        device: &Device,
315        rng: &mut ChaCha8Rng,
316    ) -> Result<Self> {
317        let inter = cfg.effective_intermediate_size();
318        let bias = BiasMode::from(cfg.mlp_bias);
319        Ok(Self {
320            gate_proj: LinearLayer::new_random(
321                LayerShape::new(cfg.hidden_size, inter, bias),
322                dtype,
323                device,
324                rng,
325            )?,
326            up_proj: LinearLayer::new_random(
327                LayerShape::new(cfg.hidden_size, inter, bias),
328                dtype,
329                device,
330                rng,
331            )?,
332            down_proj: LinearLayer::new_random(
333                LayerShape::new(inter, cfg.hidden_size, bias),
334                dtype,
335                device,
336                rng,
337            )?,
338            hidden_act: cfg.hidden_act.clone(),
339        })
340    }
341
342    fn forward(&self, x: &Tensor) -> Result<Tensor> {
343        let gate = apply_activation(&self.gate_proj.forward(x)?, &self.hidden_act)?;
344        let up = self.up_proj.forward(x)?;
345        self.down_proj.forward(&gate.mul(&up)?)
346    }
347}
348
349#[derive(Debug, Clone)]
350struct LlamaDecoderLayer {
351    input_layernorm: RmsNorm,
352    self_attn: LlamaAttention,
353    post_attention_layernorm: RmsNorm,
354    mlp: LlamaMlp,
355}
356
357impl LlamaDecoderLayer {
358    fn forward(
359        &self,
360        hidden_states: &Tensor,
361        rope: &RotaryEmbedding,
362        layer_idx: usize,
363        cache: Option<&mut KvCache>,
364    ) -> Result<Tensor> {
365        let residual = hidden_states;
366        let normed = self.input_layernorm.forward(hidden_states)?;
367        let attn = self.self_attn.forward(&normed, rope, layer_idx, cache)?;
368        let hidden = residual.add(&attn)?;
369
370        let residual = &hidden;
371        let normed = self.post_attention_layernorm.forward(&hidden)?;
372        let mlp_out = self.mlp.forward(&normed)?;
373        Ok(residual.add(&mlp_out)?)
374    }
375}
376
377/// Causal language model supporting Llama / Llama 2/3, Qwen2 / Qwen2.5 / QwQ, and Qwen3.
378#[derive(Debug, Clone)]
379pub struct LlamaModel {
380    config: ModelConfig,
381    embed_tokens: Embedding,
382    layers: Vec<LlamaDecoderLayer>,
383    norm: RmsNorm,
384    lm_head: Option<LinearLayer>,
385    rope: RotaryEmbedding,
386}
387
388impl LlamaModel {
389    /// Construct a Llama/Qwen2/Qwen3 model from a loaded state dictionary.
390    ///
391    /// # Errors
392    /// Returns [`crate::ModelError::MissingWeight`] or tensor shape errors if checkpoint tensors are missing.
393    pub fn from_tensors(
394        config: ModelConfig,
395        tensors: &HashMap<String, Tensor>,
396        spec: LinearLoadSpec<'_>,
397    ) -> Result<Self> {
398        let arch = config.detect_architecture()?;
399        let embed_tokens =
400            Embedding::from_tensors("model.embed_tokens", tensors, None, spec.dtype, spec.device)?;
401        let mut layers = Vec::with_capacity(config.num_hidden_layers);
402        for i in 0..config.num_hidden_layers {
403            let prefix = format!("model.layers.{i}");
404            let input_layernorm = RmsNorm::from_tensors(
405                &format!("{prefix}.input_layernorm"),
406                tensors,
407                config.rms_norm_eps,
408                NormKind::Standard,
409                spec.dtype,
410                spec.device,
411            )?;
412            let self_attn = LlamaAttention::from_tensors(
413                &format!("{prefix}.self_attn"),
414                tensors,
415                &config,
416                arch,
417                spec.littlebit,
418                spec.weight_state,
419                spec.inference_mode,
420                spec.dtype,
421                spec.device,
422            )?;
423            let post_attention_layernorm = RmsNorm::from_tensors(
424                &format!("{prefix}.post_attention_layernorm"),
425                tensors,
426                config.rms_norm_eps,
427                NormKind::Standard,
428                spec.dtype,
429                spec.device,
430            )?;
431            let mlp = LlamaMlp::from_tensors(
432                &format!("{prefix}.mlp"),
433                tensors,
434                &config,
435                LinearLoadSpec {
436                    ratio_factor: 1.0,
437                    ..spec
438                },
439            )?;
440            layers.push(LlamaDecoderLayer {
441                input_layernorm,
442                self_attn,
443                post_attention_layernorm,
444                mlp,
445            });
446        }
447        let norm = RmsNorm::from_tensors(
448            "model.norm",
449            tensors,
450            config.rms_norm_eps,
451            NormKind::Standard,
452            spec.dtype,
453            spec.device,
454        )?;
455        let lm_head = if !config.tie_word_embeddings && tensors.contains_key("lm_head.weight") {
456            Some(LinearLayer::from_tensors(
457                "lm_head",
458                tensors,
459                LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
460                LinearLoadSpec {
461                    littlebit: None,
462                    ratio_factor: 1.0,
463                    ..spec
464                },
465            )?)
466        } else {
467            None
468        };
469        let rope = RotaryEmbedding::new(
470            config.effective_head_dim(),
471            config.max_position_embeddings.max(256),
472            config.rope_theta,
473            spec.dtype,
474            spec.device,
475        )?;
476        Ok(Self {
477            config,
478            embed_tokens,
479            layers,
480            norm,
481            lm_head,
482            rope,
483        })
484    }
485
486    /// Initialize a random Llama/Qwen2/Qwen3 model (for testing and synthetic benchmarks).
487    ///
488    /// # Errors
489    /// Returns an error if tensor creation fails.
490    pub fn new_random(
491        config: ModelConfig,
492        dtype: DType,
493        device: &Device,
494        rng: &mut ChaCha8Rng,
495    ) -> Result<Self> {
496        let arch = config.detect_architecture()?;
497        let embed_tokens = Embedding::new_random(
498            config.vocab_size,
499            config.hidden_size,
500            None,
501            dtype,
502            device,
503            rng,
504        )?;
505        let mut layers = Vec::with_capacity(config.num_hidden_layers);
506        for _ in 0..config.num_hidden_layers {
507            layers.push(LlamaDecoderLayer {
508                input_layernorm: RmsNorm::new_init(
509                    config.hidden_size,
510                    config.rms_norm_eps,
511                    NormKind::Standard,
512                    dtype,
513                    device,
514                )?,
515                self_attn: LlamaAttention::new_random(&config, arch, dtype, device, rng)?,
516                post_attention_layernorm: RmsNorm::new_init(
517                    config.hidden_size,
518                    config.rms_norm_eps,
519                    NormKind::Standard,
520                    dtype,
521                    device,
522                )?,
523                mlp: LlamaMlp::new_random(&config, dtype, device, rng)?,
524            });
525        }
526        let norm = RmsNorm::new_init(
527            config.hidden_size,
528            config.rms_norm_eps,
529            NormKind::Standard,
530            dtype,
531            device,
532        )?;
533        let lm_head = if config.tie_word_embeddings {
534            None
535        } else {
536            Some(LinearLayer::new_random(
537                LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
538                dtype,
539                device,
540                rng,
541            )?)
542        };
543        let rope = RotaryEmbedding::new(
544            config.effective_head_dim(),
545            config.max_position_embeddings.max(256),
546            config.rope_theta,
547            dtype,
548            device,
549        )?;
550        Ok(Self {
551            config,
552            embed_tokens,
553            layers,
554            norm,
555            lm_head,
556            rope,
557        })
558    }
559
560    /// Forward pass returning logits `(batch, seq_len, vocab_size)` and optional hidden states.
561    ///
562    /// # Errors
563    /// Returns an error if tensor shapes are invalid or layer forward passes fail.
564    pub fn forward(
565        &self,
566        input_ids: &Tensor,
567        mut cache: Option<&mut KvCache>,
568        hidden_mode: HiddenStatesMode,
569    ) -> Result<CausalLmOutput> {
570        let mut hidden = self.embed_tokens.forward(input_ids)?;
571        let collect = hidden_mode == HiddenStatesMode::All;
572        let mut all_hidden = if collect {
573            Vec::with_capacity(self.layers.len() + 1)
574        } else {
575            Vec::new()
576        };
577
578        let last_idx = self.layers.len().saturating_sub(1);
579        for (idx, layer) in self.layers.iter().enumerate() {
580            if collect {
581                all_hidden.push(hidden.clone());
582            }
583            let c = cache.as_deref_mut();
584            hidden = layer.forward(&hidden, &self.rope, idx, c)?;
585            if collect && idx == last_idx {
586                // Final entry after `self.norm` is appended below to preserve `L + 1` count.
587            }
588        }
589        hidden = self.norm.forward(&hidden)?;
590        if collect {
591            all_hidden.push(hidden.clone());
592        }
593
594        let logits = match &self.lm_head {
595            Some(head) => head.forward(&hidden)?,
596            None => self.embed_tokens.forward_lm_head(&hidden)?,
597        };
598        Ok(CausalLmOutput {
599            logits,
600            hidden_states: all_hidden,
601        })
602    }
603
604    pub(crate) fn apply_littlebit_patch(
605        &mut self,
606        lb_cfg: &LittleBitConfig,
607        policy: &PatchPolicy,
608        train_mode: TrainMode,
609        base_seed: u64,
610        summary: &mut PatchSummary,
611    ) -> Result<()> {
612        for (i, layer) in self.layers.iter_mut().enumerate() {
613            let p = format!("model.layers.{i}");
614            patch_linear_layer(
615                &format!("{p}.self_attn.q_proj"),
616                &mut layer.self_attn.q_proj,
617                lb_cfg,
618                policy,
619                train_mode,
620                base_seed,
621                summary,
622            )?;
623            patch_linear_layer(
624                &format!("{p}.self_attn.k_proj"),
625                &mut layer.self_attn.k_proj,
626                lb_cfg,
627                policy,
628                train_mode,
629                base_seed,
630                summary,
631            )?;
632            patch_linear_layer(
633                &format!("{p}.self_attn.v_proj"),
634                &mut layer.self_attn.v_proj,
635                lb_cfg,
636                policy,
637                train_mode,
638                base_seed,
639                summary,
640            )?;
641            patch_linear_layer(
642                &format!("{p}.self_attn.o_proj"),
643                &mut layer.self_attn.o_proj,
644                lb_cfg,
645                policy,
646                train_mode,
647                base_seed,
648                summary,
649            )?;
650            patch_linear_layer(
651                &format!("{p}.mlp.gate_proj"),
652                &mut layer.mlp.gate_proj,
653                lb_cfg,
654                policy,
655                train_mode,
656                base_seed,
657                summary,
658            )?;
659            patch_linear_layer(
660                &format!("{p}.mlp.up_proj"),
661                &mut layer.mlp.up_proj,
662                lb_cfg,
663                policy,
664                train_mode,
665                base_seed,
666                summary,
667            )?;
668            patch_linear_layer(
669                &format!("{p}.mlp.down_proj"),
670                &mut layer.mlp.down_proj,
671                lb_cfg,
672                policy,
673                train_mode,
674                base_seed,
675                summary,
676            )?;
677        }
678        if let Some(head) = &mut self.lm_head {
679            patch_linear_layer(
680                "lm_head", head, lb_cfg, policy, train_mode, base_seed, summary,
681            )?;
682        }
683        Ok(())
684    }
685
686    pub(crate) fn convert_to_packed(&mut self) -> Result<()> {
687        for layer in &mut self.layers {
688            layer.self_attn.q_proj.convert_to_packed()?;
689            layer.self_attn.k_proj.convert_to_packed()?;
690            layer.self_attn.v_proj.convert_to_packed()?;
691            layer.self_attn.o_proj.convert_to_packed()?;
692            layer.mlp.gate_proj.convert_to_packed()?;
693            layer.mlp.up_proj.convert_to_packed()?;
694            layer.mlp.down_proj.convert_to_packed()?;
695        }
696        if let Some(head) = &mut self.lm_head {
697            head.convert_to_packed()?;
698        }
699        Ok(())
700    }
701
702    pub(crate) fn make_non_quantized_trainable(&mut self) -> Result<()> {
703        self.embed_tokens.make_trainable()?;
704        for layer in &mut self.layers {
705            layer.input_layernorm.make_trainable()?;
706            layer.post_attention_layernorm.make_trainable()?;
707            layer.self_attn.q_proj.make_trainable()?;
708            layer.self_attn.k_proj.make_trainable()?;
709            layer.self_attn.v_proj.make_trainable()?;
710            layer.self_attn.o_proj.make_trainable()?;
711            if let Some(qn) = &mut layer.self_attn.q_norm {
712                qn.make_trainable()?;
713            }
714            if let Some(kn) = &mut layer.self_attn.k_norm {
715                kn.make_trainable()?;
716            }
717            layer.mlp.gate_proj.make_trainable()?;
718            layer.mlp.up_proj.make_trainable()?;
719            layer.mlp.down_proj.make_trainable()?;
720        }
721        self.norm.make_trainable()?;
722        if let Some(head) = &mut self.lm_head {
723            head.make_trainable()?;
724        }
725        Ok(())
726    }
727
728    pub(crate) fn freeze_matching(&mut self, exclude: &[&str]) {
729        let matches = |name: &str| exclude.iter().any(|k| name.contains(k));
730        if matches("model.embed_tokens") {
731            self.embed_tokens.freeze();
732        }
733        if matches("model.norm") {
734            self.norm.freeze();
735        }
736        if let Some(head) = &mut self.lm_head {
737            if matches("lm_head") {
738                head.freeze();
739            }
740        }
741        for (i, layer) in self.layers.iter_mut().enumerate() {
742            let p = format!("model.layers.{i}");
743            if matches(&format!("{p}.input_layernorm")) {
744                layer.input_layernorm.freeze();
745            }
746            if matches(&format!("{p}.post_attention_layernorm")) {
747                layer.post_attention_layernorm.freeze();
748            }
749            if matches(&format!("{p}.self_attn.q_proj")) {
750                layer.self_attn.q_proj.freeze();
751            }
752            if matches(&format!("{p}.self_attn.k_proj")) {
753                layer.self_attn.k_proj.freeze();
754            }
755            if matches(&format!("{p}.self_attn.v_proj")) {
756                layer.self_attn.v_proj.freeze();
757            }
758            if matches(&format!("{p}.self_attn.o_proj")) {
759                layer.self_attn.o_proj.freeze();
760            }
761            if matches(&format!("{p}.mlp.gate_proj")) {
762                layer.mlp.gate_proj.freeze();
763            }
764            if matches(&format!("{p}.mlp.up_proj")) {
765                layer.mlp.up_proj.freeze();
766            }
767            if matches(&format!("{p}.mlp.down_proj")) {
768                layer.mlp.down_proj.freeze();
769            }
770        }
771    }
772
773    pub(crate) fn trainable_vars(&self) -> Vec<Var> {
774        let mut vars = self.embed_tokens.trainable_vars();
775        for layer in &self.layers {
776            vars.extend(layer.input_layernorm.trainable_vars());
777            vars.extend(layer.self_attn.q_proj.trainable_vars());
778            vars.extend(layer.self_attn.k_proj.trainable_vars());
779            vars.extend(layer.self_attn.v_proj.trainable_vars());
780            vars.extend(layer.self_attn.o_proj.trainable_vars());
781            if let Some(qn) = &layer.self_attn.q_norm {
782                vars.extend(qn.trainable_vars());
783            }
784            if let Some(kn) = &layer.self_attn.k_norm {
785                vars.extend(kn.trainable_vars());
786            }
787            vars.extend(layer.post_attention_layernorm.trainable_vars());
788            vars.extend(layer.mlp.gate_proj.trainable_vars());
789            vars.extend(layer.mlp.up_proj.trainable_vars());
790            vars.extend(layer.mlp.down_proj.trainable_vars());
791        }
792        vars.extend(self.norm.trainable_vars());
793        if let Some(head) = &self.lm_head {
794            vars.extend(head.trainable_vars());
795        }
796        vars
797    }
798
799    pub(crate) fn total_params(&self) -> usize {
800        let mut count = self.embed_tokens.weight.elem_count() + self.norm.weight.elem_count();
801        for layer in &self.layers {
802            count += layer.input_layernorm.weight.elem_count()
803                + layer.post_attention_layernorm.weight.elem_count()
804                + layer.self_attn.q_proj.param_count()
805                + layer.self_attn.k_proj.param_count()
806                + layer.self_attn.v_proj.param_count()
807                + layer.self_attn.o_proj.param_count()
808                + layer
809                    .self_attn
810                    .q_norm
811                    .as_ref()
812                    .map_or(0, |n| n.weight.elem_count())
813                + layer
814                    .self_attn
815                    .k_norm
816                    .as_ref()
817                    .map_or(0, |n| n.weight.elem_count())
818                + layer.mlp.gate_proj.param_count()
819                + layer.mlp.up_proj.param_count()
820                + layer.mlp.down_proj.param_count();
821        }
822        if let Some(head) = &self.lm_head {
823            count += head.param_count();
824        }
825        count
826    }
827
828    pub(crate) fn pack_state_dict(&self) -> Result<HashMap<String, Tensor>> {
829        let mut out = HashMap::new();
830        self.embed_tokens
831            .pack_state_dict("model.embed_tokens", &mut out);
832        for (i, layer) in self.layers.iter().enumerate() {
833            let p = format!("model.layers.{i}");
834            layer
835                .input_layernorm
836                .pack_state_dict(&format!("{p}.input_layernorm"), &mut out);
837            out.extend(
838                layer
839                    .self_attn
840                    .q_proj
841                    .pack_state_dict(&format!("{p}.self_attn.q_proj"))?,
842            );
843            out.extend(
844                layer
845                    .self_attn
846                    .k_proj
847                    .pack_state_dict(&format!("{p}.self_attn.k_proj"))?,
848            );
849            out.extend(
850                layer
851                    .self_attn
852                    .v_proj
853                    .pack_state_dict(&format!("{p}.self_attn.v_proj"))?,
854            );
855            out.extend(
856                layer
857                    .self_attn
858                    .o_proj
859                    .pack_state_dict(&format!("{p}.self_attn.o_proj"))?,
860            );
861            if let Some(qn) = &layer.self_attn.q_norm {
862                qn.pack_state_dict(&format!("{p}.self_attn.q_norm"), &mut out);
863            }
864            if let Some(kn) = &layer.self_attn.k_norm {
865                kn.pack_state_dict(&format!("{p}.self_attn.k_norm"), &mut out);
866            }
867            layer
868                .post_attention_layernorm
869                .pack_state_dict(&format!("{p}.post_attention_layernorm"), &mut out);
870            out.extend(
871                layer
872                    .mlp
873                    .gate_proj
874                    .pack_state_dict(&format!("{p}.mlp.gate_proj"))?,
875            );
876            out.extend(
877                layer
878                    .mlp
879                    .up_proj
880                    .pack_state_dict(&format!("{p}.mlp.up_proj"))?,
881            );
882            out.extend(
883                layer
884                    .mlp
885                    .down_proj
886                    .pack_state_dict(&format!("{p}.mlp.down_proj"))?,
887            );
888        }
889        self.norm.pack_state_dict("model.norm", &mut out);
890        if let Some(head) = &self.lm_head {
891            out.extend(head.pack_state_dict("lm_head")?);
892        }
893        Ok(out)
894    }
895
896    pub(crate) fn config(&self) -> &ModelConfig {
897        &self.config
898    }
899}