1use std::collections::HashMap;
4
5use candle_core::{DType, Device, Tensor, Var};
6use rand_chacha::ChaCha8Rng;
7
8use littlebit_core::{BiasMode, LayerShape, LittleBitConfig, TrainMode};
9
10use crate::config::ModelConfig;
11use crate::error::Result;
12use crate::layer::{
13 compute_attention, AttentionParams, Embedding, KvCache, LayerNorm, LinearLayer, LinearLoadSpec,
14};
15use crate::model::{CausalLmOutput, HiddenStatesMode};
16use crate::patch::{patch_linear_layer, PatchPolicy, PatchSummary};
17
18const OPT_POSITION_OFFSET: usize = 2;
19
20#[derive(Debug, Clone)]
21struct OptAttention {
22 q_proj: LinearLayer,
23 k_proj: LinearLayer,
24 v_proj: LinearLayer,
25 out_proj: LinearLayer,
26 num_heads: usize,
27 head_dim: usize,
28}
29
30impl OptAttention {
31 fn forward(
32 &self,
33 hidden_states: &Tensor,
34 layer_idx: usize,
35 cache: Option<&mut KvCache>,
36 ) -> Result<Tensor> {
37 let (batch, seq_len, _) = hidden_states.dims3()?;
38 let q = self
39 .q_proj
40 .forward(hidden_states)?
41 .reshape((batch, seq_len, self.num_heads, self.head_dim))?
42 .transpose(1, 2)?
43 .contiguous()?;
44 let k = self
45 .k_proj
46 .forward(hidden_states)?
47 .reshape((batch, seq_len, self.num_heads, self.head_dim))?
48 .transpose(1, 2)?
49 .contiguous()?;
50 let v = self
51 .v_proj
52 .forward(hidden_states)?
53 .reshape((batch, seq_len, self.num_heads, self.head_dim))?
54 .transpose(1, 2)?
55 .contiguous()?;
56
57 let attn_out = compute_attention(
58 &q,
59 &k,
60 &v,
61 layer_idx,
62 cache,
63 AttentionParams {
64 scale: 1.0 / (self.head_dim as f64).sqrt(),
65 softcap: None,
66 sliding_window: None,
67 },
68 )?;
69 self.out_proj.forward(&attn_out)
70 }
71}
72
73#[derive(Debug, Clone)]
74struct OptDecoderLayer {
75 self_attn: OptAttention,
76 self_attn_layer_norm: LayerNorm,
77 fc1: LinearLayer,
78 fc2: LinearLayer,
79 final_layer_norm: LayerNorm,
80 do_layer_norm_before: bool,
81}
82
83impl OptDecoderLayer {
84 fn forward(
85 &self,
86 hidden_states: &Tensor,
87 layer_idx: usize,
88 cache: Option<&mut KvCache>,
89 ) -> Result<Tensor> {
90 let residual = hidden_states.clone();
91 let x_attn_in = if self.do_layer_norm_before {
92 self.self_attn_layer_norm.forward(hidden_states)?
93 } else {
94 hidden_states.clone()
95 };
96 let attn = self.self_attn.forward(&x_attn_in, layer_idx, cache)?;
97 let mut hidden = residual.add(&attn)?;
98 if !self.do_layer_norm_before {
99 hidden = self.self_attn_layer_norm.forward(&hidden)?;
100 }
101
102 let residual = hidden.clone();
103 let x_ff_in = if self.do_layer_norm_before {
104 self.final_layer_norm.forward(&hidden)?
105 } else {
106 hidden
107 };
108 let ff = self.fc2.forward(&self.fc1.forward(&x_ff_in)?.relu()?)?;
109 let mut out = residual.add(&ff)?;
110 if !self.do_layer_norm_before {
111 out = self.final_layer_norm.forward(&out)?;
112 }
113 Ok(out)
114 }
115}
116
117#[derive(Debug, Clone)]
119pub struct OptModel {
120 config: ModelConfig,
121 embed_tokens: Embedding,
122 embed_positions: Embedding,
123 project_in: Option<LinearLayer>,
124 project_out: Option<LinearLayer>,
125 layers: Vec<OptDecoderLayer>,
126 final_layer_norm: Option<LayerNorm>,
127 lm_head: Option<LinearLayer>,
128}
129
130impl OptModel {
131 pub fn from_tensors(
136 config: ModelConfig,
137 tensors: &HashMap<String, Tensor>,
138 spec: LinearLoadSpec<'_>,
139 ) -> Result<Self> {
140 let embed_dim = config.word_embed_proj_dim.unwrap_or(config.hidden_size);
141 let embed_tokens = Embedding::from_tensors(
142 "model.decoder.embed_tokens",
143 tensors,
144 None,
145 spec.dtype,
146 spec.device,
147 )?;
148 let embed_positions = Embedding::from_tensors(
149 "model.decoder.embed_positions",
150 tensors,
151 None,
152 spec.dtype,
153 spec.device,
154 )?;
155
156 let unit_spec = LinearLoadSpec {
157 ratio_factor: 1.0,
158 ..spec
159 };
160 let kv_factor = spec.littlebit.map_or(1.0, |c| c.kv_factor);
161 let kv_spec = LinearLoadSpec {
162 ratio_factor: kv_factor,
163 ..spec
164 };
165
166 let project_in = if embed_dim != config.hidden_size
167 || tensors.contains_key("model.decoder.project_in.weight")
168 || tensors.contains_key("model.decoder.project_in.U")
169 || tensors.contains_key("model.decoder.project_in.U_packed")
170 {
171 Some(LinearLayer::from_tensors(
172 "model.decoder.project_in",
173 tensors,
174 LayerShape::new(embed_dim, config.hidden_size, BiasMode::NoBias),
175 unit_spec,
176 )?)
177 } else {
178 None
179 };
180
181 let project_out = if embed_dim != config.hidden_size
182 || tensors.contains_key("model.decoder.project_out.weight")
183 || tensors.contains_key("model.decoder.project_out.U")
184 || tensors.contains_key("model.decoder.project_out.U_packed")
185 {
186 Some(LinearLayer::from_tensors(
187 "model.decoder.project_out",
188 tensors,
189 LayerShape::new(config.hidden_size, embed_dim, BiasMode::NoBias),
190 unit_spec,
191 )?)
192 } else {
193 None
194 };
195
196 let bias = BiasMode::from(config.enable_bias.unwrap_or(true));
197 let affine = config.layer_norm_elementwise_affine.unwrap_or(true);
198 let do_ln_before = config.do_layer_norm_before.unwrap_or(true);
199 let ffn_dim = config.effective_intermediate_size();
200 let num_heads = config.num_attention_heads;
201 let head_dim = config.hidden_size / num_heads.max(1);
202
203 let mut layers = Vec::with_capacity(config.num_hidden_layers);
204 for i in 0..config.num_hidden_layers {
205 let p = format!("model.decoder.layers.{i}");
206 let self_attn = OptAttention {
207 q_proj: LinearLayer::from_tensors(
208 &format!("{p}.self_attn.q_proj"),
209 tensors,
210 LayerShape::new(config.hidden_size, config.hidden_size, bias),
211 unit_spec,
212 )?,
213 k_proj: LinearLayer::from_tensors(
214 &format!("{p}.self_attn.k_proj"),
215 tensors,
216 LayerShape::new(config.hidden_size, config.hidden_size, bias),
217 kv_spec,
218 )?,
219 v_proj: LinearLayer::from_tensors(
220 &format!("{p}.self_attn.v_proj"),
221 tensors,
222 LayerShape::new(config.hidden_size, config.hidden_size, bias),
223 kv_spec,
224 )?,
225 out_proj: LinearLayer::from_tensors(
226 &format!("{p}.self_attn.out_proj"),
227 tensors,
228 LayerShape::new(config.hidden_size, config.hidden_size, bias),
229 unit_spec,
230 )?,
231 num_heads,
232 head_dim,
233 };
234 let self_attn_layer_norm = LayerNorm::from_tensors(
235 &format!("{p}.self_attn_layer_norm"),
236 tensors,
237 config.hidden_size,
238 1e-5,
239 affine,
240 spec.dtype,
241 spec.device,
242 )?;
243 let fc1 = LinearLayer::from_tensors(
244 &format!("{p}.fc1"),
245 tensors,
246 LayerShape::new(config.hidden_size, ffn_dim, bias),
247 unit_spec,
248 )?;
249 let fc2 = LinearLayer::from_tensors(
250 &format!("{p}.fc2"),
251 tensors,
252 LayerShape::new(ffn_dim, config.hidden_size, bias),
253 unit_spec,
254 )?;
255 let final_layer_norm = LayerNorm::from_tensors(
256 &format!("{p}.final_layer_norm"),
257 tensors,
258 config.hidden_size,
259 1e-5,
260 affine,
261 spec.dtype,
262 spec.device,
263 )?;
264 layers.push(OptDecoderLayer {
265 self_attn,
266 self_attn_layer_norm,
267 fc1,
268 fc2,
269 final_layer_norm,
270 do_layer_norm_before: do_ln_before,
271 });
272 }
273
274 let final_layer_norm = if tensors.contains_key("model.decoder.final_layer_norm.weight") {
275 Some(LayerNorm::from_tensors(
276 "model.decoder.final_layer_norm",
277 tensors,
278 config.hidden_size,
279 1e-5,
280 affine,
281 spec.dtype,
282 spec.device,
283 )?)
284 } else {
285 None
286 };
287
288 let lm_head = if !config.tie_word_embeddings && tensors.contains_key("lm_head.weight") {
289 Some(LinearLayer::from_tensors(
290 "lm_head",
291 tensors,
292 LayerShape::new(embed_dim, config.vocab_size, BiasMode::NoBias),
293 LinearLoadSpec {
294 littlebit: None,
295 ratio_factor: 1.0,
296 ..spec
297 },
298 )?)
299 } else {
300 None
301 };
302
303 Ok(Self {
304 config,
305 embed_tokens,
306 embed_positions,
307 project_in,
308 project_out,
309 layers,
310 final_layer_norm,
311 lm_head,
312 })
313 }
314
315 pub fn new_random(
320 config: ModelConfig,
321 dtype: DType,
322 device: &Device,
323 rng: &mut ChaCha8Rng,
324 ) -> Result<Self> {
325 let embed_dim = config.word_embed_proj_dim.unwrap_or(config.hidden_size);
326 let embed_tokens =
327 Embedding::new_random(config.vocab_size, embed_dim, None, dtype, device, rng)?;
328 let embed_positions = Embedding::new_random(
329 config.max_position_embeddings + OPT_POSITION_OFFSET,
330 config.hidden_size,
331 None,
332 dtype,
333 device,
334 rng,
335 )?;
336 let bias = BiasMode::from(config.enable_bias.unwrap_or(true));
337 let affine = config.layer_norm_elementwise_affine.unwrap_or(true);
338 let do_ln_before = config.do_layer_norm_before.unwrap_or(true);
339 let ffn_dim = config.effective_intermediate_size();
340 let num_heads = config.num_attention_heads;
341 let head_dim = config.hidden_size / num_heads.max(1);
342
343 let mut layers = Vec::with_capacity(config.num_hidden_layers);
344 for _ in 0..config.num_hidden_layers {
345 layers.push(OptDecoderLayer {
346 self_attn: OptAttention {
347 q_proj: LinearLayer::new_random(
348 LayerShape::new(config.hidden_size, config.hidden_size, bias),
349 dtype,
350 device,
351 rng,
352 )?,
353 k_proj: LinearLayer::new_random(
354 LayerShape::new(config.hidden_size, config.hidden_size, bias),
355 dtype,
356 device,
357 rng,
358 )?,
359 v_proj: LinearLayer::new_random(
360 LayerShape::new(config.hidden_size, config.hidden_size, bias),
361 dtype,
362 device,
363 rng,
364 )?,
365 out_proj: LinearLayer::new_random(
366 LayerShape::new(config.hidden_size, config.hidden_size, bias),
367 dtype,
368 device,
369 rng,
370 )?,
371 num_heads,
372 head_dim,
373 },
374 self_attn_layer_norm: LayerNorm::new_init(
375 config.hidden_size,
376 1e-5,
377 affine,
378 dtype,
379 device,
380 )?,
381 fc1: LinearLayer::new_random(
382 LayerShape::new(config.hidden_size, ffn_dim, bias),
383 dtype,
384 device,
385 rng,
386 )?,
387 fc2: LinearLayer::new_random(
388 LayerShape::new(ffn_dim, config.hidden_size, bias),
389 dtype,
390 device,
391 rng,
392 )?,
393 final_layer_norm: LayerNorm::new_init(
394 config.hidden_size,
395 1e-5,
396 affine,
397 dtype,
398 device,
399 )?,
400 do_layer_norm_before: do_ln_before,
401 });
402 }
403 let final_layer_norm = if do_ln_before {
404 Some(LayerNorm::new_init(
405 config.hidden_size,
406 1e-5,
407 affine,
408 dtype,
409 device,
410 )?)
411 } else {
412 None
413 };
414 let lm_head = if config.tie_word_embeddings {
415 None
416 } else {
417 Some(LinearLayer::new_random(
418 LayerShape::new(embed_dim, config.vocab_size, BiasMode::NoBias),
419 dtype,
420 device,
421 rng,
422 )?)
423 };
424
425 Ok(Self {
426 config,
427 embed_tokens,
428 embed_positions,
429 project_in: None,
430 project_out: None,
431 layers,
432 final_layer_norm,
433 lm_head,
434 })
435 }
436
437 pub fn forward(
442 &self,
443 input_ids: &Tensor,
444 mut cache: Option<&mut KvCache>,
445 hidden_mode: HiddenStatesMode,
446 ) -> Result<CausalLmOutput> {
447 let (batch, seq_len) = input_ids.dims2()?;
448 let past_len = cache.as_ref().map_or(0, |c| c.past_len(0));
449
450 let mut tok_emb = self.embed_tokens.forward(input_ids)?;
451 if let Some(proj_in) = &self.project_in {
452 tok_emb = proj_in.forward(&tok_emb)?;
453 }
454
455 let pos_ids: Vec<u32> = (0..seq_len)
456 .map(|p| (past_len + p + OPT_POSITION_OFFSET) as u32)
457 .collect();
458 let pos_t = Tensor::from_vec(pos_ids, (1, seq_len), input_ids.device())?
459 .broadcast_as((batch, seq_len))?;
460 let pos_emb = self.embed_positions.forward(&pos_t)?;
461 let mut hidden = tok_emb.add(&pos_emb)?;
462
463 let collect = hidden_mode == HiddenStatesMode::All;
464 let mut all_hidden = if collect {
465 Vec::with_capacity(self.layers.len() + 1)
466 } else {
467 Vec::new()
468 };
469
470 for (idx, layer) in self.layers.iter().enumerate() {
471 if collect {
472 all_hidden.push(hidden.clone());
473 }
474 hidden = layer.forward(&hidden, idx, cache.as_deref_mut())?;
475 }
476
477 if let Some(norm) = &self.final_layer_norm {
478 hidden = norm.forward(&hidden)?;
479 }
480 if let Some(proj_out) = &self.project_out {
481 hidden = proj_out.forward(&hidden)?;
482 }
483 if collect {
484 all_hidden.push(hidden.clone());
485 }
486
487 let logits = match &self.lm_head {
488 Some(head) => head.forward(&hidden)?,
489 None => self.embed_tokens.forward_lm_head(&hidden)?,
490 };
491 Ok(CausalLmOutput {
492 logits,
493 hidden_states: all_hidden,
494 })
495 }
496
497 pub(crate) fn apply_littlebit_patch(
498 &mut self,
499 lb_cfg: &LittleBitConfig,
500 policy: &PatchPolicy,
501 train_mode: TrainMode,
502 base_seed: u64,
503 summary: &mut PatchSummary,
504 ) -> Result<()> {
505 if let Some(pin) = &mut self.project_in {
506 patch_linear_layer(
507 "model.decoder.project_in",
508 pin,
509 lb_cfg,
510 policy,
511 train_mode,
512 base_seed,
513 summary,
514 )?;
515 }
516 if let Some(pout) = &mut self.project_out {
517 patch_linear_layer(
518 "model.decoder.project_out",
519 pout,
520 lb_cfg,
521 policy,
522 train_mode,
523 base_seed,
524 summary,
525 )?;
526 }
527 for (i, layer) in self.layers.iter_mut().enumerate() {
528 let p = format!("model.decoder.layers.{i}");
529 patch_linear_layer(
530 &format!("{p}.self_attn.q_proj"),
531 &mut layer.self_attn.q_proj,
532 lb_cfg,
533 policy,
534 train_mode,
535 base_seed,
536 summary,
537 )?;
538 patch_linear_layer(
539 &format!("{p}.self_attn.k_proj"),
540 &mut layer.self_attn.k_proj,
541 lb_cfg,
542 policy,
543 train_mode,
544 base_seed,
545 summary,
546 )?;
547 patch_linear_layer(
548 &format!("{p}.self_attn.v_proj"),
549 &mut layer.self_attn.v_proj,
550 lb_cfg,
551 policy,
552 train_mode,
553 base_seed,
554 summary,
555 )?;
556 patch_linear_layer(
557 &format!("{p}.self_attn.out_proj"),
558 &mut layer.self_attn.out_proj,
559 lb_cfg,
560 policy,
561 train_mode,
562 base_seed,
563 summary,
564 )?;
565 patch_linear_layer(
566 &format!("{p}.fc1"),
567 &mut layer.fc1,
568 lb_cfg,
569 policy,
570 train_mode,
571 base_seed,
572 summary,
573 )?;
574 patch_linear_layer(
575 &format!("{p}.fc2"),
576 &mut layer.fc2,
577 lb_cfg,
578 policy,
579 train_mode,
580 base_seed,
581 summary,
582 )?;
583 }
584 if let Some(head) = &mut self.lm_head {
585 patch_linear_layer(
586 "lm_head", head, lb_cfg, policy, train_mode, base_seed, summary,
587 )?;
588 }
589 Ok(())
590 }
591
592 pub(crate) fn convert_to_packed(&mut self) -> Result<()> {
593 if let Some(pin) = &mut self.project_in {
594 pin.convert_to_packed()?;
595 }
596 if let Some(pout) = &mut self.project_out {
597 pout.convert_to_packed()?;
598 }
599 for layer in &mut self.layers {
600 layer.self_attn.q_proj.convert_to_packed()?;
601 layer.self_attn.k_proj.convert_to_packed()?;
602 layer.self_attn.v_proj.convert_to_packed()?;
603 layer.self_attn.out_proj.convert_to_packed()?;
604 layer.fc1.convert_to_packed()?;
605 layer.fc2.convert_to_packed()?;
606 }
607 if let Some(head) = &mut self.lm_head {
608 head.convert_to_packed()?;
609 }
610 Ok(())
611 }
612
613 pub(crate) fn make_non_quantized_trainable(&mut self) -> Result<()> {
614 self.embed_tokens.make_trainable()?;
615 self.embed_positions.make_trainable()?;
616 if let Some(pin) = &mut self.project_in {
617 pin.make_trainable()?;
618 }
619 if let Some(pout) = &mut self.project_out {
620 pout.make_trainable()?;
621 }
622 for layer in &mut self.layers {
623 layer.self_attn_layer_norm.make_trainable()?;
624 layer.final_layer_norm.make_trainable()?;
625 layer.self_attn.q_proj.make_trainable()?;
626 layer.self_attn.k_proj.make_trainable()?;
627 layer.self_attn.v_proj.make_trainable()?;
628 layer.self_attn.out_proj.make_trainable()?;
629 layer.fc1.make_trainable()?;
630 layer.fc2.make_trainable()?;
631 }
632 if let Some(fln) = &mut self.final_layer_norm {
633 fln.make_trainable()?;
634 }
635 if let Some(head) = &mut self.lm_head {
636 head.make_trainable()?;
637 }
638 Ok(())
639 }
640
641 pub(crate) fn freeze_matching(&mut self, exclude: &[&str]) {
642 let matches = |name: &str| exclude.iter().any(|k| name.contains(k));
643 if matches("model.decoder.embed_tokens") {
644 self.embed_tokens.freeze();
645 }
646 if matches("model.decoder.embed_positions") {
647 self.embed_positions.freeze();
648 }
649 if let Some(fln) = &mut self.final_layer_norm {
650 if matches("model.decoder.final_layer_norm") {
651 fln.freeze();
652 }
653 }
654 if let Some(head) = &mut self.lm_head {
655 if matches("lm_head") {
656 head.freeze();
657 }
658 }
659 for (i, layer) in self.layers.iter_mut().enumerate() {
660 let p = format!("model.decoder.layers.{i}");
661 if matches(&format!("{p}.self_attn_layer_norm")) {
662 layer.self_attn_layer_norm.freeze();
663 }
664 if matches(&format!("{p}.final_layer_norm")) {
665 layer.final_layer_norm.freeze();
666 }
667 if matches(&format!("{p}.self_attn")) {
668 layer.self_attn.q_proj.freeze();
669 layer.self_attn.k_proj.freeze();
670 layer.self_attn.v_proj.freeze();
671 layer.self_attn.out_proj.freeze();
672 }
673 if matches(&format!("{p}.fc1")) {
674 layer.fc1.freeze();
675 }
676 if matches(&format!("{p}.fc2")) {
677 layer.fc2.freeze();
678 }
679 }
680 }
681
682 pub(crate) fn trainable_vars(&self) -> Vec<Var> {
683 let mut vars = self.embed_tokens.trainable_vars();
684 vars.extend(self.embed_positions.trainable_vars());
685 if let Some(pin) = &self.project_in {
686 vars.extend(pin.trainable_vars());
687 }
688 if let Some(pout) = &self.project_out {
689 vars.extend(pout.trainable_vars());
690 }
691 for layer in &self.layers {
692 vars.extend(layer.self_attn.q_proj.trainable_vars());
693 vars.extend(layer.self_attn.k_proj.trainable_vars());
694 vars.extend(layer.self_attn.v_proj.trainable_vars());
695 vars.extend(layer.self_attn.out_proj.trainable_vars());
696 vars.extend(layer.self_attn_layer_norm.trainable_vars());
697 vars.extend(layer.fc1.trainable_vars());
698 vars.extend(layer.fc2.trainable_vars());
699 vars.extend(layer.final_layer_norm.trainable_vars());
700 }
701 if let Some(fln) = &self.final_layer_norm {
702 vars.extend(fln.trainable_vars());
703 }
704 if let Some(head) = &self.lm_head {
705 vars.extend(head.trainable_vars());
706 }
707 vars
708 }
709
710 pub(crate) fn total_params(&self) -> usize {
711 let mut count = self.embed_tokens.weight.elem_count()
712 + self.embed_positions.weight.elem_count()
713 + self.project_in.as_ref().map_or(0, LinearLayer::param_count)
714 + self
715 .project_out
716 .as_ref()
717 .map_or(0, LinearLayer::param_count)
718 + self
719 .final_layer_norm
720 .as_ref()
721 .map_or(0, LayerNorm::param_count);
722 for layer in &self.layers {
723 count += layer.self_attn.q_proj.param_count()
724 + layer.self_attn.k_proj.param_count()
725 + layer.self_attn.v_proj.param_count()
726 + layer.self_attn.out_proj.param_count()
727 + layer.self_attn_layer_norm.param_count()
728 + layer.fc1.param_count()
729 + layer.fc2.param_count()
730 + layer.final_layer_norm.param_count();
731 }
732 if let Some(head) = &self.lm_head {
733 count += head.param_count();
734 }
735 count
736 }
737
738 pub(crate) fn pack_state_dict(&self) -> Result<HashMap<String, Tensor>> {
739 let mut out = HashMap::new();
740 self.embed_tokens
741 .pack_state_dict("model.decoder.embed_tokens", &mut out);
742 self.embed_positions
743 .pack_state_dict("model.decoder.embed_positions", &mut out);
744 if let Some(pin) = &self.project_in {
745 out.extend(pin.pack_state_dict("model.decoder.project_in")?);
746 }
747 if let Some(pout) = &self.project_out {
748 out.extend(pout.pack_state_dict("model.decoder.project_out")?);
749 }
750 for (i, layer) in self.layers.iter().enumerate() {
751 let p = format!("model.decoder.layers.{i}");
752 out.extend(
753 layer
754 .self_attn
755 .q_proj
756 .pack_state_dict(&format!("{p}.self_attn.q_proj"))?,
757 );
758 out.extend(
759 layer
760 .self_attn
761 .k_proj
762 .pack_state_dict(&format!("{p}.self_attn.k_proj"))?,
763 );
764 out.extend(
765 layer
766 .self_attn
767 .v_proj
768 .pack_state_dict(&format!("{p}.self_attn.v_proj"))?,
769 );
770 out.extend(
771 layer
772 .self_attn
773 .out_proj
774 .pack_state_dict(&format!("{p}.self_attn.out_proj"))?,
775 );
776 layer
777 .self_attn_layer_norm
778 .pack_state_dict(&format!("{p}.self_attn_layer_norm"), &mut out);
779 out.extend(layer.fc1.pack_state_dict(&format!("{p}.fc1"))?);
780 out.extend(layer.fc2.pack_state_dict(&format!("{p}.fc2"))?);
781 layer
782 .final_layer_norm
783 .pack_state_dict(&format!("{p}.final_layer_norm"), &mut out);
784 }
785 if let Some(fln) = &self.final_layer_norm {
786 fln.pack_state_dict("model.decoder.final_layer_norm", &mut out);
787 }
788 if let Some(head) = &self.lm_head {
789 out.extend(head.pack_state_dict("lm_head")?);
790 }
791 Ok(out)
792 }
793
794 pub(crate) fn config(&self) -> &ModelConfig {
795 &self.config
796 }
797}