1use std::collections::HashMap;
4
5use candle_core::{DType, Device, Tensor, Var};
6use rand_chacha::ChaCha8Rng;
7
8use littlebit_core::{BiasMode, LayerShape, LittleBitConfig, TrainMode, WeightState};
9
10use crate::config::{ModelArchitecture, ModelConfig};
11use crate::error::Result;
12use crate::layer::{
13 apply_activation, compute_attention, AttentionParams, Embedding, KvCache, LinearLayer,
14 LinearLoadSpec, NormKind, RmsNorm, RotaryEmbedding,
15};
16use crate::model::{CausalLmOutput, HiddenStatesMode};
17use crate::patch::{patch_linear_layer, PatchPolicy, PatchSummary};
18
19#[derive(Debug, Clone)]
20struct LlamaAttention {
21 q_proj: LinearLayer,
22 k_proj: LinearLayer,
23 v_proj: LinearLayer,
24 o_proj: LinearLayer,
25 q_norm: Option<RmsNorm>,
26 k_norm: Option<RmsNorm>,
27 num_heads: usize,
28 num_kv_heads: usize,
29 head_dim: usize,
30}
31
32impl LlamaAttention {
33 #[allow(clippy::too_many_arguments)]
34 fn from_tensors(
35 prefix: &str,
36 tensors: &HashMap<String, Tensor>,
37 cfg: &ModelConfig,
38 arch: ModelArchitecture,
39 littlebit: Option<&LittleBitConfig>,
40 weight_state: WeightState,
41 inference_mode: littlebit_core::InferenceMode,
42 dtype: DType,
43 device: &Device,
44 ) -> Result<Self> {
45 let head_dim = cfg.effective_head_dim();
46 let num_heads = cfg.num_attention_heads;
47 let num_kv_heads = cfg.effective_kv_heads();
48 let kv_factor = littlebit.map_or(1.0, |c| c.kv_factor);
49
50 let bias_for = |name: &str, default_bias: bool| -> BiasMode {
51 BiasMode::from(default_bias || tensors.contains_key(&format!("{prefix}.{name}.bias")))
52 };
53
54 let q_bias = bias_for(
55 "q_proj",
56 cfg.attention_bias || arch == ModelArchitecture::Qwen2,
57 );
58 let k_bias = bias_for(
59 "k_proj",
60 cfg.attention_bias || arch == ModelArchitecture::Qwen2,
61 );
62 let v_bias = bias_for(
63 "v_proj",
64 cfg.attention_bias || arch == ModelArchitecture::Qwen2,
65 );
66 let o_bias = bias_for("o_proj", cfg.attention_bias);
67
68 let base_spec = LinearLoadSpec {
69 littlebit,
70 ratio_factor: 1.0,
71 weight_state,
72 inference_mode,
73 dtype,
74 device,
75 };
76 let kv_spec = LinearLoadSpec {
77 ratio_factor: kv_factor,
78 ..base_spec
79 };
80
81 let q_proj = LinearLayer::from_tensors(
82 &format!("{prefix}.q_proj"),
83 tensors,
84 LayerShape::new(cfg.hidden_size, num_heads * head_dim, q_bias),
85 base_spec,
86 )?;
87 let k_proj = LinearLayer::from_tensors(
88 &format!("{prefix}.k_proj"),
89 tensors,
90 LayerShape::new(cfg.hidden_size, num_kv_heads * head_dim, k_bias),
91 kv_spec,
92 )?;
93 let v_proj = LinearLayer::from_tensors(
94 &format!("{prefix}.v_proj"),
95 tensors,
96 LayerShape::new(cfg.hidden_size, num_kv_heads * head_dim, v_bias),
97 kv_spec,
98 )?;
99 let o_proj = LinearLayer::from_tensors(
100 &format!("{prefix}.o_proj"),
101 tensors,
102 LayerShape::new(num_heads * head_dim, cfg.hidden_size, o_bias),
103 base_spec,
104 )?;
105
106 let q_norm = if tensors.contains_key(&format!("{prefix}.q_norm.weight")) {
107 Some(RmsNorm::from_tensors(
108 &format!("{prefix}.q_norm"),
109 tensors,
110 cfg.rms_norm_eps,
111 NormKind::Standard,
112 dtype,
113 device,
114 )?)
115 } else {
116 None
117 };
118 let k_norm = if tensors.contains_key(&format!("{prefix}.k_norm.weight")) {
119 Some(RmsNorm::from_tensors(
120 &format!("{prefix}.k_norm"),
121 tensors,
122 cfg.rms_norm_eps,
123 NormKind::Standard,
124 dtype,
125 device,
126 )?)
127 } else {
128 None
129 };
130
131 Ok(Self {
132 q_proj,
133 k_proj,
134 v_proj,
135 o_proj,
136 q_norm,
137 k_norm,
138 num_heads,
139 num_kv_heads,
140 head_dim,
141 })
142 }
143
144 fn new_random(
145 cfg: &ModelConfig,
146 arch: ModelArchitecture,
147 dtype: DType,
148 device: &Device,
149 rng: &mut ChaCha8Rng,
150 ) -> Result<Self> {
151 let head_dim = cfg.effective_head_dim();
152 let num_heads = cfg.num_attention_heads;
153 let num_kv_heads = cfg.effective_kv_heads();
154 let qkv_bias = BiasMode::from(cfg.attention_bias || arch == ModelArchitecture::Qwen2);
155 let o_bias = BiasMode::from(cfg.attention_bias);
156
157 let q_proj = LinearLayer::new_random(
158 LayerShape::new(cfg.hidden_size, num_heads * head_dim, qkv_bias),
159 dtype,
160 device,
161 rng,
162 )?;
163 let k_proj = LinearLayer::new_random(
164 LayerShape::new(cfg.hidden_size, num_kv_heads * head_dim, qkv_bias),
165 dtype,
166 device,
167 rng,
168 )?;
169 let v_proj = LinearLayer::new_random(
170 LayerShape::new(cfg.hidden_size, num_kv_heads * head_dim, qkv_bias),
171 dtype,
172 device,
173 rng,
174 )?;
175 let o_proj = LinearLayer::new_random(
176 LayerShape::new(num_heads * head_dim, cfg.hidden_size, o_bias),
177 dtype,
178 device,
179 rng,
180 )?;
181
182 let (q_norm, k_norm) = if arch == ModelArchitecture::Qwen3 {
183 (
184 Some(RmsNorm::new_init(
185 head_dim,
186 cfg.rms_norm_eps,
187 NormKind::Standard,
188 dtype,
189 device,
190 )?),
191 Some(RmsNorm::new_init(
192 head_dim,
193 cfg.rms_norm_eps,
194 NormKind::Standard,
195 dtype,
196 device,
197 )?),
198 )
199 } else {
200 (None, None)
201 };
202
203 Ok(Self {
204 q_proj,
205 k_proj,
206 v_proj,
207 o_proj,
208 q_norm,
209 k_norm,
210 num_heads,
211 num_kv_heads,
212 head_dim,
213 })
214 }
215
216 fn forward(
217 &self,
218 hidden_states: &Tensor,
219 rope: &RotaryEmbedding,
220 layer_idx: usize,
221 cache: Option<&mut KvCache>,
222 ) -> Result<Tensor> {
223 let (batch, seq_len, _) = hidden_states.dims3()?;
224 let pos_offset = cache.as_ref().map_or(0, |c| c.past_len(layer_idx));
225
226 let mut q = self
227 .q_proj
228 .forward(hidden_states)?
229 .reshape((batch, seq_len, self.num_heads, self.head_dim))?
230 .transpose(1, 2)?
231 .contiguous()?;
232 let mut k = self
233 .k_proj
234 .forward(hidden_states)?
235 .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
236 .transpose(1, 2)?
237 .contiguous()?;
238 let v = self
239 .v_proj
240 .forward(hidden_states)?
241 .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
242 .transpose(1, 2)?
243 .contiguous()?;
244
245 if let Some(qn) = &self.q_norm {
246 q = qn.forward(&q)?;
247 }
248 if let Some(kn) = &self.k_norm {
249 k = kn.forward(&k)?;
250 }
251
252 let q = rope.apply(&q, pos_offset)?;
253 let k = rope.apply(&k, pos_offset)?;
254
255 let attn_out = compute_attention(
256 &q,
257 &k,
258 &v,
259 layer_idx,
260 cache,
261 AttentionParams {
262 scale: 1.0 / (self.head_dim as f64).sqrt(),
263 softcap: None,
264 sliding_window: None,
265 },
266 )?;
267 self.o_proj.forward(&attn_out)
268 }
269}
270
271#[derive(Debug, Clone)]
272struct LlamaMlp {
273 gate_proj: LinearLayer,
274 up_proj: LinearLayer,
275 down_proj: LinearLayer,
276 hidden_act: String,
277}
278
279impl LlamaMlp {
280 fn from_tensors(
281 prefix: &str,
282 tensors: &HashMap<String, Tensor>,
283 cfg: &ModelConfig,
284 spec: LinearLoadSpec<'_>,
285 ) -> Result<Self> {
286 let inter = cfg.effective_intermediate_size();
287 let bias = BiasMode::from(cfg.mlp_bias);
288 Ok(Self {
289 gate_proj: LinearLayer::from_tensors(
290 &format!("{prefix}.gate_proj"),
291 tensors,
292 LayerShape::new(cfg.hidden_size, inter, bias),
293 spec,
294 )?,
295 up_proj: LinearLayer::from_tensors(
296 &format!("{prefix}.up_proj"),
297 tensors,
298 LayerShape::new(cfg.hidden_size, inter, bias),
299 spec,
300 )?,
301 down_proj: LinearLayer::from_tensors(
302 &format!("{prefix}.down_proj"),
303 tensors,
304 LayerShape::new(inter, cfg.hidden_size, bias),
305 spec,
306 )?,
307 hidden_act: cfg.hidden_act.clone(),
308 })
309 }
310
311 fn new_random(
312 cfg: &ModelConfig,
313 dtype: DType,
314 device: &Device,
315 rng: &mut ChaCha8Rng,
316 ) -> Result<Self> {
317 let inter = cfg.effective_intermediate_size();
318 let bias = BiasMode::from(cfg.mlp_bias);
319 Ok(Self {
320 gate_proj: LinearLayer::new_random(
321 LayerShape::new(cfg.hidden_size, inter, bias),
322 dtype,
323 device,
324 rng,
325 )?,
326 up_proj: LinearLayer::new_random(
327 LayerShape::new(cfg.hidden_size, inter, bias),
328 dtype,
329 device,
330 rng,
331 )?,
332 down_proj: LinearLayer::new_random(
333 LayerShape::new(inter, cfg.hidden_size, bias),
334 dtype,
335 device,
336 rng,
337 )?,
338 hidden_act: cfg.hidden_act.clone(),
339 })
340 }
341
342 fn forward(&self, x: &Tensor) -> Result<Tensor> {
343 let gate = apply_activation(&self.gate_proj.forward(x)?, &self.hidden_act)?;
344 let up = self.up_proj.forward(x)?;
345 self.down_proj.forward(&gate.mul(&up)?)
346 }
347}
348
349#[derive(Debug, Clone)]
350struct LlamaDecoderLayer {
351 input_layernorm: RmsNorm,
352 self_attn: LlamaAttention,
353 post_attention_layernorm: RmsNorm,
354 mlp: LlamaMlp,
355}
356
357impl LlamaDecoderLayer {
358 fn forward(
359 &self,
360 hidden_states: &Tensor,
361 rope: &RotaryEmbedding,
362 layer_idx: usize,
363 cache: Option<&mut KvCache>,
364 ) -> Result<Tensor> {
365 let residual = hidden_states;
366 let normed = self.input_layernorm.forward(hidden_states)?;
367 let attn = self.self_attn.forward(&normed, rope, layer_idx, cache)?;
368 let hidden = residual.add(&attn)?;
369
370 let residual = &hidden;
371 let normed = self.post_attention_layernorm.forward(&hidden)?;
372 let mlp_out = self.mlp.forward(&normed)?;
373 Ok(residual.add(&mlp_out)?)
374 }
375}
376
377#[derive(Debug, Clone)]
379pub struct LlamaModel {
380 config: ModelConfig,
381 embed_tokens: Embedding,
382 layers: Vec<LlamaDecoderLayer>,
383 norm: RmsNorm,
384 lm_head: Option<LinearLayer>,
385 rope: RotaryEmbedding,
386}
387
388impl LlamaModel {
389 pub fn from_tensors(
394 config: ModelConfig,
395 tensors: &HashMap<String, Tensor>,
396 spec: LinearLoadSpec<'_>,
397 ) -> Result<Self> {
398 let arch = config.detect_architecture()?;
399 let embed_tokens =
400 Embedding::from_tensors("model.embed_tokens", tensors, None, spec.dtype, spec.device)?;
401 let mut layers = Vec::with_capacity(config.num_hidden_layers);
402 for i in 0..config.num_hidden_layers {
403 let prefix = format!("model.layers.{i}");
404 let input_layernorm = RmsNorm::from_tensors(
405 &format!("{prefix}.input_layernorm"),
406 tensors,
407 config.rms_norm_eps,
408 NormKind::Standard,
409 spec.dtype,
410 spec.device,
411 )?;
412 let self_attn = LlamaAttention::from_tensors(
413 &format!("{prefix}.self_attn"),
414 tensors,
415 &config,
416 arch,
417 spec.littlebit,
418 spec.weight_state,
419 spec.inference_mode,
420 spec.dtype,
421 spec.device,
422 )?;
423 let post_attention_layernorm = RmsNorm::from_tensors(
424 &format!("{prefix}.post_attention_layernorm"),
425 tensors,
426 config.rms_norm_eps,
427 NormKind::Standard,
428 spec.dtype,
429 spec.device,
430 )?;
431 let mlp = LlamaMlp::from_tensors(
432 &format!("{prefix}.mlp"),
433 tensors,
434 &config,
435 LinearLoadSpec {
436 ratio_factor: 1.0,
437 ..spec
438 },
439 )?;
440 layers.push(LlamaDecoderLayer {
441 input_layernorm,
442 self_attn,
443 post_attention_layernorm,
444 mlp,
445 });
446 }
447 let norm = RmsNorm::from_tensors(
448 "model.norm",
449 tensors,
450 config.rms_norm_eps,
451 NormKind::Standard,
452 spec.dtype,
453 spec.device,
454 )?;
455 let lm_head = if !config.tie_word_embeddings && tensors.contains_key("lm_head.weight") {
456 Some(LinearLayer::from_tensors(
457 "lm_head",
458 tensors,
459 LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
460 LinearLoadSpec {
461 littlebit: None,
462 ratio_factor: 1.0,
463 ..spec
464 },
465 )?)
466 } else {
467 None
468 };
469 let rope = RotaryEmbedding::new(
470 config.effective_head_dim(),
471 config.max_position_embeddings.max(256),
472 config.rope_theta,
473 spec.dtype,
474 spec.device,
475 )?;
476 Ok(Self {
477 config,
478 embed_tokens,
479 layers,
480 norm,
481 lm_head,
482 rope,
483 })
484 }
485
486 pub fn new_random(
491 config: ModelConfig,
492 dtype: DType,
493 device: &Device,
494 rng: &mut ChaCha8Rng,
495 ) -> Result<Self> {
496 let arch = config.detect_architecture()?;
497 let embed_tokens = Embedding::new_random(
498 config.vocab_size,
499 config.hidden_size,
500 None,
501 dtype,
502 device,
503 rng,
504 )?;
505 let mut layers = Vec::with_capacity(config.num_hidden_layers);
506 for _ in 0..config.num_hidden_layers {
507 layers.push(LlamaDecoderLayer {
508 input_layernorm: RmsNorm::new_init(
509 config.hidden_size,
510 config.rms_norm_eps,
511 NormKind::Standard,
512 dtype,
513 device,
514 )?,
515 self_attn: LlamaAttention::new_random(&config, arch, dtype, device, rng)?,
516 post_attention_layernorm: RmsNorm::new_init(
517 config.hidden_size,
518 config.rms_norm_eps,
519 NormKind::Standard,
520 dtype,
521 device,
522 )?,
523 mlp: LlamaMlp::new_random(&config, dtype, device, rng)?,
524 });
525 }
526 let norm = RmsNorm::new_init(
527 config.hidden_size,
528 config.rms_norm_eps,
529 NormKind::Standard,
530 dtype,
531 device,
532 )?;
533 let lm_head = if config.tie_word_embeddings {
534 None
535 } else {
536 Some(LinearLayer::new_random(
537 LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
538 dtype,
539 device,
540 rng,
541 )?)
542 };
543 let rope = RotaryEmbedding::new(
544 config.effective_head_dim(),
545 config.max_position_embeddings.max(256),
546 config.rope_theta,
547 dtype,
548 device,
549 )?;
550 Ok(Self {
551 config,
552 embed_tokens,
553 layers,
554 norm,
555 lm_head,
556 rope,
557 })
558 }
559
560 pub fn forward(
565 &self,
566 input_ids: &Tensor,
567 mut cache: Option<&mut KvCache>,
568 hidden_mode: HiddenStatesMode,
569 ) -> Result<CausalLmOutput> {
570 let mut hidden = self.embed_tokens.forward(input_ids)?;
571 let collect = hidden_mode == HiddenStatesMode::All;
572 let mut all_hidden = if collect {
573 Vec::with_capacity(self.layers.len() + 1)
574 } else {
575 Vec::new()
576 };
577
578 let last_idx = self.layers.len().saturating_sub(1);
579 for (idx, layer) in self.layers.iter().enumerate() {
580 if collect {
581 all_hidden.push(hidden.clone());
582 }
583 let c = cache.as_deref_mut();
584 hidden = layer.forward(&hidden, &self.rope, idx, c)?;
585 if collect && idx == last_idx {
586 }
588 }
589 hidden = self.norm.forward(&hidden)?;
590 if collect {
591 all_hidden.push(hidden.clone());
592 }
593
594 let logits = match &self.lm_head {
595 Some(head) => head.forward(&hidden)?,
596 None => self.embed_tokens.forward_lm_head(&hidden)?,
597 };
598 Ok(CausalLmOutput {
599 logits,
600 hidden_states: all_hidden,
601 })
602 }
603
604 pub(crate) fn apply_littlebit_patch(
605 &mut self,
606 lb_cfg: &LittleBitConfig,
607 policy: &PatchPolicy,
608 train_mode: TrainMode,
609 base_seed: u64,
610 summary: &mut PatchSummary,
611 ) -> Result<()> {
612 for (i, layer) in self.layers.iter_mut().enumerate() {
613 let p = format!("model.layers.{i}");
614 patch_linear_layer(
615 &format!("{p}.self_attn.q_proj"),
616 &mut layer.self_attn.q_proj,
617 lb_cfg,
618 policy,
619 train_mode,
620 base_seed,
621 summary,
622 )?;
623 patch_linear_layer(
624 &format!("{p}.self_attn.k_proj"),
625 &mut layer.self_attn.k_proj,
626 lb_cfg,
627 policy,
628 train_mode,
629 base_seed,
630 summary,
631 )?;
632 patch_linear_layer(
633 &format!("{p}.self_attn.v_proj"),
634 &mut layer.self_attn.v_proj,
635 lb_cfg,
636 policy,
637 train_mode,
638 base_seed,
639 summary,
640 )?;
641 patch_linear_layer(
642 &format!("{p}.self_attn.o_proj"),
643 &mut layer.self_attn.o_proj,
644 lb_cfg,
645 policy,
646 train_mode,
647 base_seed,
648 summary,
649 )?;
650 patch_linear_layer(
651 &format!("{p}.mlp.gate_proj"),
652 &mut layer.mlp.gate_proj,
653 lb_cfg,
654 policy,
655 train_mode,
656 base_seed,
657 summary,
658 )?;
659 patch_linear_layer(
660 &format!("{p}.mlp.up_proj"),
661 &mut layer.mlp.up_proj,
662 lb_cfg,
663 policy,
664 train_mode,
665 base_seed,
666 summary,
667 )?;
668 patch_linear_layer(
669 &format!("{p}.mlp.down_proj"),
670 &mut layer.mlp.down_proj,
671 lb_cfg,
672 policy,
673 train_mode,
674 base_seed,
675 summary,
676 )?;
677 }
678 if let Some(head) = &mut self.lm_head {
679 patch_linear_layer(
680 "lm_head", head, lb_cfg, policy, train_mode, base_seed, summary,
681 )?;
682 }
683 Ok(())
684 }
685
686 pub(crate) fn convert_to_packed(&mut self) -> Result<()> {
687 for layer in &mut self.layers {
688 layer.self_attn.q_proj.convert_to_packed()?;
689 layer.self_attn.k_proj.convert_to_packed()?;
690 layer.self_attn.v_proj.convert_to_packed()?;
691 layer.self_attn.o_proj.convert_to_packed()?;
692 layer.mlp.gate_proj.convert_to_packed()?;
693 layer.mlp.up_proj.convert_to_packed()?;
694 layer.mlp.down_proj.convert_to_packed()?;
695 }
696 if let Some(head) = &mut self.lm_head {
697 head.convert_to_packed()?;
698 }
699 Ok(())
700 }
701
702 pub(crate) fn make_non_quantized_trainable(&mut self) -> Result<()> {
703 self.embed_tokens.make_trainable()?;
704 for layer in &mut self.layers {
705 layer.input_layernorm.make_trainable()?;
706 layer.post_attention_layernorm.make_trainable()?;
707 layer.self_attn.q_proj.make_trainable()?;
708 layer.self_attn.k_proj.make_trainable()?;
709 layer.self_attn.v_proj.make_trainable()?;
710 layer.self_attn.o_proj.make_trainable()?;
711 if let Some(qn) = &mut layer.self_attn.q_norm {
712 qn.make_trainable()?;
713 }
714 if let Some(kn) = &mut layer.self_attn.k_norm {
715 kn.make_trainable()?;
716 }
717 layer.mlp.gate_proj.make_trainable()?;
718 layer.mlp.up_proj.make_trainable()?;
719 layer.mlp.down_proj.make_trainable()?;
720 }
721 self.norm.make_trainable()?;
722 if let Some(head) = &mut self.lm_head {
723 head.make_trainable()?;
724 }
725 Ok(())
726 }
727
728 pub(crate) fn freeze_matching(&mut self, exclude: &[&str]) {
729 let matches = |name: &str| exclude.iter().any(|k| name.contains(k));
730 if matches("model.embed_tokens") {
731 self.embed_tokens.freeze();
732 }
733 if matches("model.norm") {
734 self.norm.freeze();
735 }
736 if let Some(head) = &mut self.lm_head {
737 if matches("lm_head") {
738 head.freeze();
739 }
740 }
741 for (i, layer) in self.layers.iter_mut().enumerate() {
742 let p = format!("model.layers.{i}");
743 if matches(&format!("{p}.input_layernorm")) {
744 layer.input_layernorm.freeze();
745 }
746 if matches(&format!("{p}.post_attention_layernorm")) {
747 layer.post_attention_layernorm.freeze();
748 }
749 if matches(&format!("{p}.self_attn.q_proj")) {
750 layer.self_attn.q_proj.freeze();
751 }
752 if matches(&format!("{p}.self_attn.k_proj")) {
753 layer.self_attn.k_proj.freeze();
754 }
755 if matches(&format!("{p}.self_attn.v_proj")) {
756 layer.self_attn.v_proj.freeze();
757 }
758 if matches(&format!("{p}.self_attn.o_proj")) {
759 layer.self_attn.o_proj.freeze();
760 }
761 if matches(&format!("{p}.mlp.gate_proj")) {
762 layer.mlp.gate_proj.freeze();
763 }
764 if matches(&format!("{p}.mlp.up_proj")) {
765 layer.mlp.up_proj.freeze();
766 }
767 if matches(&format!("{p}.mlp.down_proj")) {
768 layer.mlp.down_proj.freeze();
769 }
770 }
771 }
772
773 pub(crate) fn trainable_vars(&self) -> Vec<Var> {
774 let mut vars = self.embed_tokens.trainable_vars();
775 for layer in &self.layers {
776 vars.extend(layer.input_layernorm.trainable_vars());
777 vars.extend(layer.self_attn.q_proj.trainable_vars());
778 vars.extend(layer.self_attn.k_proj.trainable_vars());
779 vars.extend(layer.self_attn.v_proj.trainable_vars());
780 vars.extend(layer.self_attn.o_proj.trainable_vars());
781 if let Some(qn) = &layer.self_attn.q_norm {
782 vars.extend(qn.trainable_vars());
783 }
784 if let Some(kn) = &layer.self_attn.k_norm {
785 vars.extend(kn.trainable_vars());
786 }
787 vars.extend(layer.post_attention_layernorm.trainable_vars());
788 vars.extend(layer.mlp.gate_proj.trainable_vars());
789 vars.extend(layer.mlp.up_proj.trainable_vars());
790 vars.extend(layer.mlp.down_proj.trainable_vars());
791 }
792 vars.extend(self.norm.trainable_vars());
793 if let Some(head) = &self.lm_head {
794 vars.extend(head.trainable_vars());
795 }
796 vars
797 }
798
799 pub(crate) fn total_params(&self) -> usize {
800 let mut count = self.embed_tokens.weight.elem_count() + self.norm.weight.elem_count();
801 for layer in &self.layers {
802 count += layer.input_layernorm.weight.elem_count()
803 + layer.post_attention_layernorm.weight.elem_count()
804 + layer.self_attn.q_proj.param_count()
805 + layer.self_attn.k_proj.param_count()
806 + layer.self_attn.v_proj.param_count()
807 + layer.self_attn.o_proj.param_count()
808 + layer
809 .self_attn
810 .q_norm
811 .as_ref()
812 .map_or(0, |n| n.weight.elem_count())
813 + layer
814 .self_attn
815 .k_norm
816 .as_ref()
817 .map_or(0, |n| n.weight.elem_count())
818 + layer.mlp.gate_proj.param_count()
819 + layer.mlp.up_proj.param_count()
820 + layer.mlp.down_proj.param_count();
821 }
822 if let Some(head) = &self.lm_head {
823 count += head.param_count();
824 }
825 count
826 }
827
828 pub(crate) fn pack_state_dict(&self) -> Result<HashMap<String, Tensor>> {
829 let mut out = HashMap::new();
830 self.embed_tokens
831 .pack_state_dict("model.embed_tokens", &mut out);
832 for (i, layer) in self.layers.iter().enumerate() {
833 let p = format!("model.layers.{i}");
834 layer
835 .input_layernorm
836 .pack_state_dict(&format!("{p}.input_layernorm"), &mut out);
837 out.extend(
838 layer
839 .self_attn
840 .q_proj
841 .pack_state_dict(&format!("{p}.self_attn.q_proj"))?,
842 );
843 out.extend(
844 layer
845 .self_attn
846 .k_proj
847 .pack_state_dict(&format!("{p}.self_attn.k_proj"))?,
848 );
849 out.extend(
850 layer
851 .self_attn
852 .v_proj
853 .pack_state_dict(&format!("{p}.self_attn.v_proj"))?,
854 );
855 out.extend(
856 layer
857 .self_attn
858 .o_proj
859 .pack_state_dict(&format!("{p}.self_attn.o_proj"))?,
860 );
861 if let Some(qn) = &layer.self_attn.q_norm {
862 qn.pack_state_dict(&format!("{p}.self_attn.q_norm"), &mut out);
863 }
864 if let Some(kn) = &layer.self_attn.k_norm {
865 kn.pack_state_dict(&format!("{p}.self_attn.k_norm"), &mut out);
866 }
867 layer
868 .post_attention_layernorm
869 .pack_state_dict(&format!("{p}.post_attention_layernorm"), &mut out);
870 out.extend(
871 layer
872 .mlp
873 .gate_proj
874 .pack_state_dict(&format!("{p}.mlp.gate_proj"))?,
875 );
876 out.extend(
877 layer
878 .mlp
879 .up_proj
880 .pack_state_dict(&format!("{p}.mlp.up_proj"))?,
881 );
882 out.extend(
883 layer
884 .mlp
885 .down_proj
886 .pack_state_dict(&format!("{p}.mlp.down_proj"))?,
887 );
888 }
889 self.norm.pack_state_dict("model.norm", &mut out);
890 if let Some(head) = &self.lm_head {
891 out.extend(head.pack_state_dict("lm_head")?);
892 }
893 Ok(out)
894 }
895
896 pub(crate) fn config(&self) -> &ModelConfig {
897 &self.config
898 }
899}