Skip to main content

littlebit_models/
opt.rs

1//! Meta OPT causal language model architecture.
2
3use std::collections::HashMap;
4
5use candle_core::{DType, Device, Tensor, Var};
6use rand_chacha::ChaCha8Rng;
7
8use littlebit_core::{BiasMode, LayerShape, LittleBitConfig, TrainMode};
9
10use crate::config::ModelConfig;
11use crate::error::Result;
12use crate::layer::{
13    compute_attention, AttentionParams, Embedding, KvCache, LayerNorm, LinearLayer, LinearLoadSpec,
14};
15use crate::model::{CausalLmOutput, HiddenStatesMode};
16use crate::patch::{patch_linear_layer, PatchPolicy, PatchSummary};
17
18const OPT_POSITION_OFFSET: usize = 2;
19
20#[derive(Debug, Clone)]
21struct OptAttention {
22    q_proj: LinearLayer,
23    k_proj: LinearLayer,
24    v_proj: LinearLayer,
25    out_proj: LinearLayer,
26    num_heads: usize,
27    head_dim: usize,
28}
29
30impl OptAttention {
31    fn forward(
32        &self,
33        hidden_states: &Tensor,
34        layer_idx: usize,
35        cache: Option<&mut KvCache>,
36    ) -> Result<Tensor> {
37        let (batch, seq_len, _) = hidden_states.dims3()?;
38        let q = self
39            .q_proj
40            .forward(hidden_states)?
41            .reshape((batch, seq_len, self.num_heads, self.head_dim))?
42            .transpose(1, 2)?
43            .contiguous()?;
44        let k = self
45            .k_proj
46            .forward(hidden_states)?
47            .reshape((batch, seq_len, self.num_heads, self.head_dim))?
48            .transpose(1, 2)?
49            .contiguous()?;
50        let v = self
51            .v_proj
52            .forward(hidden_states)?
53            .reshape((batch, seq_len, self.num_heads, self.head_dim))?
54            .transpose(1, 2)?
55            .contiguous()?;
56
57        let attn_out = compute_attention(
58            &q,
59            &k,
60            &v,
61            layer_idx,
62            cache,
63            AttentionParams {
64                scale: 1.0 / (self.head_dim as f64).sqrt(),
65                softcap: None,
66                sliding_window: None,
67            },
68        )?;
69        self.out_proj.forward(&attn_out)
70    }
71}
72
73#[derive(Debug, Clone)]
74struct OptDecoderLayer {
75    self_attn: OptAttention,
76    self_attn_layer_norm: LayerNorm,
77    fc1: LinearLayer,
78    fc2: LinearLayer,
79    final_layer_norm: LayerNorm,
80    do_layer_norm_before: bool,
81}
82
83impl OptDecoderLayer {
84    fn forward(
85        &self,
86        hidden_states: &Tensor,
87        layer_idx: usize,
88        cache: Option<&mut KvCache>,
89    ) -> Result<Tensor> {
90        let residual = hidden_states.clone();
91        let x_attn_in = if self.do_layer_norm_before {
92            self.self_attn_layer_norm.forward(hidden_states)?
93        } else {
94            hidden_states.clone()
95        };
96        let attn = self.self_attn.forward(&x_attn_in, layer_idx, cache)?;
97        let mut hidden = residual.add(&attn)?;
98        if !self.do_layer_norm_before {
99            hidden = self.self_attn_layer_norm.forward(&hidden)?;
100        }
101
102        let residual = hidden.clone();
103        let x_ff_in = if self.do_layer_norm_before {
104            self.final_layer_norm.forward(&hidden)?
105        } else {
106            hidden
107        };
108        let ff = self.fc2.forward(&self.fc1.forward(&x_ff_in)?.relu()?)?;
109        let mut out = residual.add(&ff)?;
110        if !self.do_layer_norm_before {
111            out = self.final_layer_norm.forward(&out)?;
112        }
113        Ok(out)
114    }
115}
116
117/// Meta OPT causal language model (`model_type = "opt"`).
118#[derive(Debug, Clone)]
119pub struct OptModel {
120    config: ModelConfig,
121    embed_tokens: Embedding,
122    embed_positions: Embedding,
123    project_in: Option<LinearLayer>,
124    project_out: Option<LinearLayer>,
125    layers: Vec<OptDecoderLayer>,
126    final_layer_norm: Option<LayerNorm>,
127    lm_head: Option<LinearLayer>,
128}
129
130impl OptModel {
131    /// Load OPT weights from a Hugging Face or LittleBit state dictionary.
132    ///
133    /// # Errors
134    /// Returns [`crate::ModelError::MissingWeight`] if expected checkpoint keys are missing.
135    pub fn from_tensors(
136        config: ModelConfig,
137        tensors: &HashMap<String, Tensor>,
138        spec: LinearLoadSpec<'_>,
139    ) -> Result<Self> {
140        let embed_dim = config.word_embed_proj_dim.unwrap_or(config.hidden_size);
141        let embed_tokens = Embedding::from_tensors(
142            "model.decoder.embed_tokens",
143            tensors,
144            None,
145            spec.dtype,
146            spec.device,
147        )?;
148        let embed_positions = Embedding::from_tensors(
149            "model.decoder.embed_positions",
150            tensors,
151            None,
152            spec.dtype,
153            spec.device,
154        )?;
155
156        let unit_spec = LinearLoadSpec {
157            ratio_factor: 1.0,
158            ..spec
159        };
160        let kv_factor = spec.littlebit.map_or(1.0, |c| c.kv_factor);
161        let kv_spec = LinearLoadSpec {
162            ratio_factor: kv_factor,
163            ..spec
164        };
165
166        let project_in = if embed_dim != config.hidden_size
167            || tensors.contains_key("model.decoder.project_in.weight")
168            || tensors.contains_key("model.decoder.project_in.U")
169            || tensors.contains_key("model.decoder.project_in.U_packed")
170        {
171            Some(LinearLayer::from_tensors(
172                "model.decoder.project_in",
173                tensors,
174                LayerShape::new(embed_dim, config.hidden_size, BiasMode::NoBias),
175                unit_spec,
176            )?)
177        } else {
178            None
179        };
180
181        let project_out = if embed_dim != config.hidden_size
182            || tensors.contains_key("model.decoder.project_out.weight")
183            || tensors.contains_key("model.decoder.project_out.U")
184            || tensors.contains_key("model.decoder.project_out.U_packed")
185        {
186            Some(LinearLayer::from_tensors(
187                "model.decoder.project_out",
188                tensors,
189                LayerShape::new(config.hidden_size, embed_dim, BiasMode::NoBias),
190                unit_spec,
191            )?)
192        } else {
193            None
194        };
195
196        let bias = BiasMode::from(config.enable_bias.unwrap_or(true));
197        let affine = config.layer_norm_elementwise_affine.unwrap_or(true);
198        let do_ln_before = config.do_layer_norm_before.unwrap_or(true);
199        let ffn_dim = config.effective_intermediate_size();
200        let num_heads = config.num_attention_heads;
201        let head_dim = config.hidden_size / num_heads.max(1);
202
203        let mut layers = Vec::with_capacity(config.num_hidden_layers);
204        for i in 0..config.num_hidden_layers {
205            let p = format!("model.decoder.layers.{i}");
206            let self_attn = OptAttention {
207                q_proj: LinearLayer::from_tensors(
208                    &format!("{p}.self_attn.q_proj"),
209                    tensors,
210                    LayerShape::new(config.hidden_size, config.hidden_size, bias),
211                    unit_spec,
212                )?,
213                k_proj: LinearLayer::from_tensors(
214                    &format!("{p}.self_attn.k_proj"),
215                    tensors,
216                    LayerShape::new(config.hidden_size, config.hidden_size, bias),
217                    kv_spec,
218                )?,
219                v_proj: LinearLayer::from_tensors(
220                    &format!("{p}.self_attn.v_proj"),
221                    tensors,
222                    LayerShape::new(config.hidden_size, config.hidden_size, bias),
223                    kv_spec,
224                )?,
225                out_proj: LinearLayer::from_tensors(
226                    &format!("{p}.self_attn.out_proj"),
227                    tensors,
228                    LayerShape::new(config.hidden_size, config.hidden_size, bias),
229                    unit_spec,
230                )?,
231                num_heads,
232                head_dim,
233            };
234            let self_attn_layer_norm = LayerNorm::from_tensors(
235                &format!("{p}.self_attn_layer_norm"),
236                tensors,
237                config.hidden_size,
238                1e-5,
239                affine,
240                spec.dtype,
241                spec.device,
242            )?;
243            let fc1 = LinearLayer::from_tensors(
244                &format!("{p}.fc1"),
245                tensors,
246                LayerShape::new(config.hidden_size, ffn_dim, bias),
247                unit_spec,
248            )?;
249            let fc2 = LinearLayer::from_tensors(
250                &format!("{p}.fc2"),
251                tensors,
252                LayerShape::new(ffn_dim, config.hidden_size, bias),
253                unit_spec,
254            )?;
255            let final_layer_norm = LayerNorm::from_tensors(
256                &format!("{p}.final_layer_norm"),
257                tensors,
258                config.hidden_size,
259                1e-5,
260                affine,
261                spec.dtype,
262                spec.device,
263            )?;
264            layers.push(OptDecoderLayer {
265                self_attn,
266                self_attn_layer_norm,
267                fc1,
268                fc2,
269                final_layer_norm,
270                do_layer_norm_before: do_ln_before,
271            });
272        }
273
274        let final_layer_norm = if tensors.contains_key("model.decoder.final_layer_norm.weight") {
275            Some(LayerNorm::from_tensors(
276                "model.decoder.final_layer_norm",
277                tensors,
278                config.hidden_size,
279                1e-5,
280                affine,
281                spec.dtype,
282                spec.device,
283            )?)
284        } else {
285            None
286        };
287
288        let lm_head = if !config.tie_word_embeddings && tensors.contains_key("lm_head.weight") {
289            Some(LinearLayer::from_tensors(
290                "lm_head",
291                tensors,
292                LayerShape::new(embed_dim, config.vocab_size, BiasMode::NoBias),
293                LinearLoadSpec {
294                    littlebit: None,
295                    ratio_factor: 1.0,
296                    ..spec
297                },
298            )?)
299        } else {
300            None
301        };
302
303        Ok(Self {
304            config,
305            embed_tokens,
306            embed_positions,
307            project_in,
308            project_out,
309            layers,
310            final_layer_norm,
311            lm_head,
312        })
313    }
314
315    /// Initialize a random OPT model for testing.
316    ///
317    /// # Errors
318    /// Returns an error if tensor creation fails.
319    pub fn new_random(
320        config: ModelConfig,
321        dtype: DType,
322        device: &Device,
323        rng: &mut ChaCha8Rng,
324    ) -> Result<Self> {
325        let embed_dim = config.word_embed_proj_dim.unwrap_or(config.hidden_size);
326        let embed_tokens =
327            Embedding::new_random(config.vocab_size, embed_dim, None, dtype, device, rng)?;
328        let embed_positions = Embedding::new_random(
329            config.max_position_embeddings + OPT_POSITION_OFFSET,
330            config.hidden_size,
331            None,
332            dtype,
333            device,
334            rng,
335        )?;
336        let bias = BiasMode::from(config.enable_bias.unwrap_or(true));
337        let affine = config.layer_norm_elementwise_affine.unwrap_or(true);
338        let do_ln_before = config.do_layer_norm_before.unwrap_or(true);
339        let ffn_dim = config.effective_intermediate_size();
340        let num_heads = config.num_attention_heads;
341        let head_dim = config.hidden_size / num_heads.max(1);
342
343        let mut layers = Vec::with_capacity(config.num_hidden_layers);
344        for _ in 0..config.num_hidden_layers {
345            layers.push(OptDecoderLayer {
346                self_attn: OptAttention {
347                    q_proj: LinearLayer::new_random(
348                        LayerShape::new(config.hidden_size, config.hidden_size, bias),
349                        dtype,
350                        device,
351                        rng,
352                    )?,
353                    k_proj: LinearLayer::new_random(
354                        LayerShape::new(config.hidden_size, config.hidden_size, bias),
355                        dtype,
356                        device,
357                        rng,
358                    )?,
359                    v_proj: LinearLayer::new_random(
360                        LayerShape::new(config.hidden_size, config.hidden_size, bias),
361                        dtype,
362                        device,
363                        rng,
364                    )?,
365                    out_proj: LinearLayer::new_random(
366                        LayerShape::new(config.hidden_size, config.hidden_size, bias),
367                        dtype,
368                        device,
369                        rng,
370                    )?,
371                    num_heads,
372                    head_dim,
373                },
374                self_attn_layer_norm: LayerNorm::new_init(
375                    config.hidden_size,
376                    1e-5,
377                    affine,
378                    dtype,
379                    device,
380                )?,
381                fc1: LinearLayer::new_random(
382                    LayerShape::new(config.hidden_size, ffn_dim, bias),
383                    dtype,
384                    device,
385                    rng,
386                )?,
387                fc2: LinearLayer::new_random(
388                    LayerShape::new(ffn_dim, config.hidden_size, bias),
389                    dtype,
390                    device,
391                    rng,
392                )?,
393                final_layer_norm: LayerNorm::new_init(
394                    config.hidden_size,
395                    1e-5,
396                    affine,
397                    dtype,
398                    device,
399                )?,
400                do_layer_norm_before: do_ln_before,
401            });
402        }
403        let final_layer_norm = if do_ln_before {
404            Some(LayerNorm::new_init(
405                config.hidden_size,
406                1e-5,
407                affine,
408                dtype,
409                device,
410            )?)
411        } else {
412            None
413        };
414        let lm_head = if config.tie_word_embeddings {
415            None
416        } else {
417            Some(LinearLayer::new_random(
418                LayerShape::new(embed_dim, config.vocab_size, BiasMode::NoBias),
419                dtype,
420                device,
421                rng,
422            )?)
423        };
424
425        Ok(Self {
426            config,
427            embed_tokens,
428            embed_positions,
429            project_in: None,
430            project_out: None,
431            layers,
432            final_layer_norm,
433            lm_head,
434        })
435    }
436
437    /// Forward pass returning logits `(batch, seq_len, vocab_size)` and optional hidden states.
438    ///
439    /// # Errors
440    /// Returns an error if layer forward passes fail.
441    pub fn forward(
442        &self,
443        input_ids: &Tensor,
444        mut cache: Option<&mut KvCache>,
445        hidden_mode: HiddenStatesMode,
446    ) -> Result<CausalLmOutput> {
447        let (batch, seq_len) = input_ids.dims2()?;
448        let past_len = cache.as_ref().map_or(0, |c| c.past_len(0));
449
450        let mut tok_emb = self.embed_tokens.forward(input_ids)?;
451        if let Some(proj_in) = &self.project_in {
452            tok_emb = proj_in.forward(&tok_emb)?;
453        }
454
455        let pos_ids: Vec<u32> = (0..seq_len)
456            .map(|p| (past_len + p + OPT_POSITION_OFFSET) as u32)
457            .collect();
458        let pos_t = Tensor::from_vec(pos_ids, (1, seq_len), input_ids.device())?
459            .broadcast_as((batch, seq_len))?;
460        let pos_emb = self.embed_positions.forward(&pos_t)?;
461        let mut hidden = tok_emb.add(&pos_emb)?;
462
463        let collect = hidden_mode == HiddenStatesMode::All;
464        let mut all_hidden = if collect {
465            Vec::with_capacity(self.layers.len() + 1)
466        } else {
467            Vec::new()
468        };
469
470        for (idx, layer) in self.layers.iter().enumerate() {
471            if collect {
472                all_hidden.push(hidden.clone());
473            }
474            hidden = layer.forward(&hidden, idx, cache.as_deref_mut())?;
475        }
476
477        if let Some(norm) = &self.final_layer_norm {
478            hidden = norm.forward(&hidden)?;
479        }
480        if let Some(proj_out) = &self.project_out {
481            hidden = proj_out.forward(&hidden)?;
482        }
483        if collect {
484            all_hidden.push(hidden.clone());
485        }
486
487        let logits = match &self.lm_head {
488            Some(head) => head.forward(&hidden)?,
489            None => self.embed_tokens.forward_lm_head(&hidden)?,
490        };
491        Ok(CausalLmOutput {
492            logits,
493            hidden_states: all_hidden,
494        })
495    }
496
497    pub(crate) fn apply_littlebit_patch(
498        &mut self,
499        lb_cfg: &LittleBitConfig,
500        policy: &PatchPolicy,
501        train_mode: TrainMode,
502        base_seed: u64,
503        summary: &mut PatchSummary,
504    ) -> Result<()> {
505        if let Some(pin) = &mut self.project_in {
506            patch_linear_layer(
507                "model.decoder.project_in",
508                pin,
509                lb_cfg,
510                policy,
511                train_mode,
512                base_seed,
513                summary,
514            )?;
515        }
516        if let Some(pout) = &mut self.project_out {
517            patch_linear_layer(
518                "model.decoder.project_out",
519                pout,
520                lb_cfg,
521                policy,
522                train_mode,
523                base_seed,
524                summary,
525            )?;
526        }
527        for (i, layer) in self.layers.iter_mut().enumerate() {
528            let p = format!("model.decoder.layers.{i}");
529            patch_linear_layer(
530                &format!("{p}.self_attn.q_proj"),
531                &mut layer.self_attn.q_proj,
532                lb_cfg,
533                policy,
534                train_mode,
535                base_seed,
536                summary,
537            )?;
538            patch_linear_layer(
539                &format!("{p}.self_attn.k_proj"),
540                &mut layer.self_attn.k_proj,
541                lb_cfg,
542                policy,
543                train_mode,
544                base_seed,
545                summary,
546            )?;
547            patch_linear_layer(
548                &format!("{p}.self_attn.v_proj"),
549                &mut layer.self_attn.v_proj,
550                lb_cfg,
551                policy,
552                train_mode,
553                base_seed,
554                summary,
555            )?;
556            patch_linear_layer(
557                &format!("{p}.self_attn.out_proj"),
558                &mut layer.self_attn.out_proj,
559                lb_cfg,
560                policy,
561                train_mode,
562                base_seed,
563                summary,
564            )?;
565            patch_linear_layer(
566                &format!("{p}.fc1"),
567                &mut layer.fc1,
568                lb_cfg,
569                policy,
570                train_mode,
571                base_seed,
572                summary,
573            )?;
574            patch_linear_layer(
575                &format!("{p}.fc2"),
576                &mut layer.fc2,
577                lb_cfg,
578                policy,
579                train_mode,
580                base_seed,
581                summary,
582            )?;
583        }
584        if let Some(head) = &mut self.lm_head {
585            patch_linear_layer(
586                "lm_head", head, lb_cfg, policy, train_mode, base_seed, summary,
587            )?;
588        }
589        Ok(())
590    }
591
592    pub(crate) fn convert_to_packed(&mut self) -> Result<()> {
593        if let Some(pin) = &mut self.project_in {
594            pin.convert_to_packed()?;
595        }
596        if let Some(pout) = &mut self.project_out {
597            pout.convert_to_packed()?;
598        }
599        for layer in &mut self.layers {
600            layer.self_attn.q_proj.convert_to_packed()?;
601            layer.self_attn.k_proj.convert_to_packed()?;
602            layer.self_attn.v_proj.convert_to_packed()?;
603            layer.self_attn.out_proj.convert_to_packed()?;
604            layer.fc1.convert_to_packed()?;
605            layer.fc2.convert_to_packed()?;
606        }
607        if let Some(head) = &mut self.lm_head {
608            head.convert_to_packed()?;
609        }
610        Ok(())
611    }
612
613    pub(crate) fn make_non_quantized_trainable(&mut self) -> Result<()> {
614        self.embed_tokens.make_trainable()?;
615        self.embed_positions.make_trainable()?;
616        if let Some(pin) = &mut self.project_in {
617            pin.make_trainable()?;
618        }
619        if let Some(pout) = &mut self.project_out {
620            pout.make_trainable()?;
621        }
622        for layer in &mut self.layers {
623            layer.self_attn_layer_norm.make_trainable()?;
624            layer.final_layer_norm.make_trainable()?;
625            layer.self_attn.q_proj.make_trainable()?;
626            layer.self_attn.k_proj.make_trainable()?;
627            layer.self_attn.v_proj.make_trainable()?;
628            layer.self_attn.out_proj.make_trainable()?;
629            layer.fc1.make_trainable()?;
630            layer.fc2.make_trainable()?;
631        }
632        if let Some(fln) = &mut self.final_layer_norm {
633            fln.make_trainable()?;
634        }
635        if let Some(head) = &mut self.lm_head {
636            head.make_trainable()?;
637        }
638        Ok(())
639    }
640
641    pub(crate) fn freeze_matching(&mut self, exclude: &[&str]) {
642        let matches = |name: &str| exclude.iter().any(|k| name.contains(k));
643        if matches("model.decoder.embed_tokens") {
644            self.embed_tokens.freeze();
645        }
646        if matches("model.decoder.embed_positions") {
647            self.embed_positions.freeze();
648        }
649        if let Some(fln) = &mut self.final_layer_norm {
650            if matches("model.decoder.final_layer_norm") {
651                fln.freeze();
652            }
653        }
654        if let Some(head) = &mut self.lm_head {
655            if matches("lm_head") {
656                head.freeze();
657            }
658        }
659        for (i, layer) in self.layers.iter_mut().enumerate() {
660            let p = format!("model.decoder.layers.{i}");
661            if matches(&format!("{p}.self_attn_layer_norm")) {
662                layer.self_attn_layer_norm.freeze();
663            }
664            if matches(&format!("{p}.final_layer_norm")) {
665                layer.final_layer_norm.freeze();
666            }
667            if matches(&format!("{p}.self_attn")) {
668                layer.self_attn.q_proj.freeze();
669                layer.self_attn.k_proj.freeze();
670                layer.self_attn.v_proj.freeze();
671                layer.self_attn.out_proj.freeze();
672            }
673            if matches(&format!("{p}.fc1")) {
674                layer.fc1.freeze();
675            }
676            if matches(&format!("{p}.fc2")) {
677                layer.fc2.freeze();
678            }
679        }
680    }
681
682    pub(crate) fn trainable_vars(&self) -> Vec<Var> {
683        let mut vars = self.embed_tokens.trainable_vars();
684        vars.extend(self.embed_positions.trainable_vars());
685        if let Some(pin) = &self.project_in {
686            vars.extend(pin.trainable_vars());
687        }
688        if let Some(pout) = &self.project_out {
689            vars.extend(pout.trainable_vars());
690        }
691        for layer in &self.layers {
692            vars.extend(layer.self_attn.q_proj.trainable_vars());
693            vars.extend(layer.self_attn.k_proj.trainable_vars());
694            vars.extend(layer.self_attn.v_proj.trainable_vars());
695            vars.extend(layer.self_attn.out_proj.trainable_vars());
696            vars.extend(layer.self_attn_layer_norm.trainable_vars());
697            vars.extend(layer.fc1.trainable_vars());
698            vars.extend(layer.fc2.trainable_vars());
699            vars.extend(layer.final_layer_norm.trainable_vars());
700        }
701        if let Some(fln) = &self.final_layer_norm {
702            vars.extend(fln.trainable_vars());
703        }
704        if let Some(head) = &self.lm_head {
705            vars.extend(head.trainable_vars());
706        }
707        vars
708    }
709
710    pub(crate) fn total_params(&self) -> usize {
711        let mut count = self.embed_tokens.weight.elem_count()
712            + self.embed_positions.weight.elem_count()
713            + self.project_in.as_ref().map_or(0, LinearLayer::param_count)
714            + self
715                .project_out
716                .as_ref()
717                .map_or(0, LinearLayer::param_count)
718            + self
719                .final_layer_norm
720                .as_ref()
721                .map_or(0, LayerNorm::param_count);
722        for layer in &self.layers {
723            count += layer.self_attn.q_proj.param_count()
724                + layer.self_attn.k_proj.param_count()
725                + layer.self_attn.v_proj.param_count()
726                + layer.self_attn.out_proj.param_count()
727                + layer.self_attn_layer_norm.param_count()
728                + layer.fc1.param_count()
729                + layer.fc2.param_count()
730                + layer.final_layer_norm.param_count();
731        }
732        if let Some(head) = &self.lm_head {
733            count += head.param_count();
734        }
735        count
736    }
737
738    pub(crate) fn pack_state_dict(&self) -> Result<HashMap<String, Tensor>> {
739        let mut out = HashMap::new();
740        self.embed_tokens
741            .pack_state_dict("model.decoder.embed_tokens", &mut out);
742        self.embed_positions
743            .pack_state_dict("model.decoder.embed_positions", &mut out);
744        if let Some(pin) = &self.project_in {
745            out.extend(pin.pack_state_dict("model.decoder.project_in")?);
746        }
747        if let Some(pout) = &self.project_out {
748            out.extend(pout.pack_state_dict("model.decoder.project_out")?);
749        }
750        for (i, layer) in self.layers.iter().enumerate() {
751            let p = format!("model.decoder.layers.{i}");
752            out.extend(
753                layer
754                    .self_attn
755                    .q_proj
756                    .pack_state_dict(&format!("{p}.self_attn.q_proj"))?,
757            );
758            out.extend(
759                layer
760                    .self_attn
761                    .k_proj
762                    .pack_state_dict(&format!("{p}.self_attn.k_proj"))?,
763            );
764            out.extend(
765                layer
766                    .self_attn
767                    .v_proj
768                    .pack_state_dict(&format!("{p}.self_attn.v_proj"))?,
769            );
770            out.extend(
771                layer
772                    .self_attn
773                    .out_proj
774                    .pack_state_dict(&format!("{p}.self_attn.out_proj"))?,
775            );
776            layer
777                .self_attn_layer_norm
778                .pack_state_dict(&format!("{p}.self_attn_layer_norm"), &mut out);
779            out.extend(layer.fc1.pack_state_dict(&format!("{p}.fc1"))?);
780            out.extend(layer.fc2.pack_state_dict(&format!("{p}.fc2"))?);
781            layer
782                .final_layer_norm
783                .pack_state_dict(&format!("{p}.final_layer_norm"), &mut out);
784        }
785        if let Some(fln) = &self.final_layer_norm {
786            fln.pack_state_dict("model.decoder.final_layer_norm", &mut out);
787        }
788        if let Some(head) = &self.lm_head {
789            out.extend(head.pack_state_dict("lm_head")?);
790        }
791        Ok(out)
792    }
793
794    pub(crate) fn config(&self) -> &ModelConfig {
795        &self.config
796    }
797}