1use std::collections::HashMap;
4
5use candle_core::{DType, Device, Tensor, Var};
6use rand_chacha::ChaCha8Rng;
7
8use littlebit_core::{BiasMode, LayerShape, LittleBitConfig, TrainMode};
9
10use crate::config::{ModelArchitecture, ModelConfig};
11use crate::error::Result;
12use crate::layer::{
13 apply_activation, compute_attention, AttentionParams, Embedding, KvCache, LinearLayer,
14 LinearLoadSpec, NormKind, RmsNorm, RotaryEmbedding,
15};
16use crate::model::{CausalLmOutput, HiddenStatesMode};
17use crate::patch::{patch_linear_layer, PatchPolicy, PatchSummary};
18
19#[derive(Debug, Clone)]
20struct GemmaAttention {
21 q_proj: LinearLayer,
22 k_proj: LinearLayer,
23 v_proj: LinearLayer,
24 o_proj: LinearLayer,
25 q_norm: Option<RmsNorm>,
26 k_norm: Option<RmsNorm>,
27 num_heads: usize,
28 num_kv_heads: usize,
29 head_dim: usize,
30 scale: f64,
31 softcap: Option<f64>,
32 sliding_window: Option<usize>,
33 is_local: bool,
34}
35
36impl GemmaAttention {
37 #[allow(clippy::too_many_arguments)]
38 fn forward(
39 &self,
40 hidden_states: &Tensor,
41 rope_global: &RotaryEmbedding,
42 rope_local: Option<&RotaryEmbedding>,
43 layer_idx: usize,
44 cache: Option<&mut KvCache>,
45 ) -> Result<Tensor> {
46 let (batch, seq_len, _) = hidden_states.dims3()?;
47 let pos_offset = cache.as_ref().map_or(0, |c| c.past_len(layer_idx));
48
49 let mut q = self
50 .q_proj
51 .forward(hidden_states)?
52 .reshape((batch, seq_len, self.num_heads, self.head_dim))?
53 .transpose(1, 2)?
54 .contiguous()?;
55 let mut k = self
56 .k_proj
57 .forward(hidden_states)?
58 .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
59 .transpose(1, 2)?
60 .contiguous()?;
61 let v = self
62 .v_proj
63 .forward(hidden_states)?
64 .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
65 .transpose(1, 2)?
66 .contiguous()?;
67
68 if let Some(qn) = &self.q_norm {
69 q = qn.forward(&q)?;
70 }
71 if let Some(kn) = &self.k_norm {
72 k = kn.forward(&k)?;
73 }
74
75 let rope = if self.is_local {
76 rope_local.unwrap_or(rope_global)
77 } else {
78 rope_global
79 };
80 let q = rope.apply(&q, pos_offset)?;
81 let k = rope.apply(&k, pos_offset)?;
82
83 let attn_out = compute_attention(
84 &q,
85 &k,
86 &v,
87 layer_idx,
88 cache,
89 AttentionParams {
90 scale: self.scale,
91 softcap: self.softcap,
92 sliding_window: self.sliding_window,
93 },
94 )?;
95 self.o_proj.forward(&attn_out)
96 }
97}
98
99#[derive(Debug, Clone)]
100struct GemmaMlp {
101 gate_proj: LinearLayer,
102 up_proj: LinearLayer,
103 down_proj: LinearLayer,
104 hidden_act: String,
105}
106
107impl GemmaMlp {
108 fn forward(&self, x: &Tensor) -> Result<Tensor> {
109 let gate = apply_activation(&self.gate_proj.forward(x)?, &self.hidden_act)?;
110 let up = self.up_proj.forward(x)?;
111 self.down_proj.forward(&gate.mul(&up)?)
112 }
113}
114
115#[derive(Debug, Clone)]
116struct GemmaDecoderLayer {
117 input_layernorm: RmsNorm,
118 self_attn: GemmaAttention,
119 post_attention_layernorm: RmsNorm,
120 pre_feedforward_layernorm: RmsNorm,
121 mlp: GemmaMlp,
122 post_feedforward_layernorm: RmsNorm,
123}
124
125#[derive(Debug, Clone)]
127pub struct GemmaModel {
128 config: ModelConfig,
129 embed_tokens: Embedding,
130 layers: Vec<GemmaDecoderLayer>,
131 norm: RmsNorm,
132 lm_head: Option<LinearLayer>,
133 rope_global: RotaryEmbedding,
134 rope_local: Option<RotaryEmbedding>,
135}
136
137fn layer_window_and_local(
138 cfg: &ModelConfig,
139 arch: ModelArchitecture,
140 layer_idx: usize,
141) -> (Option<usize>, bool) {
142 match arch {
143 ModelArchitecture::Gemma2 => {
144 let is_sliding = layer_idx.is_multiple_of(2);
145 (if is_sliding { cfg.sliding_window } else { None }, false)
146 }
147 ModelArchitecture::Gemma3 => {
148 let pattern = cfg.sliding_window_pattern.unwrap_or(6).max(1);
149 let is_local = !(layer_idx + 1).is_multiple_of(pattern);
150 (if is_local { cfg.sliding_window } else { None }, is_local)
151 }
152 _ => (cfg.sliding_window, false),
153 }
154}
155
156impl GemmaModel {
157 pub fn from_tensors(
162 config: ModelConfig,
163 tensors: &HashMap<String, Tensor>,
164 spec: LinearLoadSpec<'_>,
165 ) -> Result<Self> {
166 let arch = config.detect_architecture()?;
167 let embed_scale = Some((config.hidden_size as f64).sqrt());
168 let embed_tokens = Embedding::from_tensors(
169 "model.embed_tokens",
170 tensors,
171 embed_scale,
172 spec.dtype,
173 spec.device,
174 )?;
175 let head_dim = config.effective_head_dim();
176 let num_heads = config.num_attention_heads;
177 let num_kv_heads = config.effective_kv_heads();
178 let inter = config.effective_intermediate_size();
179 let attn_bias = BiasMode::from(config.attention_bias);
180 let mlp_bias = BiasMode::from(config.mlp_bias);
181 let kv_factor = spec.littlebit.map_or(1.0, |c| c.kv_factor);
182 let scale = 1.0
183 / config
184 .query_pre_attn_scalar
185 .unwrap_or(head_dim as f64)
186 .sqrt();
187
188 let kv_spec = LinearLoadSpec {
189 ratio_factor: kv_factor,
190 ..spec
191 };
192 let unit_spec = LinearLoadSpec {
193 ratio_factor: 1.0,
194 ..spec
195 };
196
197 let mut layers = Vec::with_capacity(config.num_hidden_layers);
198 for i in 0..config.num_hidden_layers {
199 let p = format!("model.layers.{i}");
200 let (sliding_window, is_local) = layer_window_and_local(&config, arch, i);
201
202 let input_layernorm = RmsNorm::from_tensors(
203 &format!("{p}.input_layernorm"),
204 tensors,
205 config.rms_norm_eps,
206 NormKind::PlusOne,
207 spec.dtype,
208 spec.device,
209 )?;
210 let q_proj = LinearLayer::from_tensors(
211 &format!("{p}.self_attn.q_proj"),
212 tensors,
213 LayerShape::new(config.hidden_size, num_heads * head_dim, attn_bias),
214 unit_spec,
215 )?;
216 let k_proj = LinearLayer::from_tensors(
217 &format!("{p}.self_attn.k_proj"),
218 tensors,
219 LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
220 kv_spec,
221 )?;
222 let v_proj = LinearLayer::from_tensors(
223 &format!("{p}.self_attn.v_proj"),
224 tensors,
225 LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
226 kv_spec,
227 )?;
228 let o_proj = LinearLayer::from_tensors(
229 &format!("{p}.self_attn.o_proj"),
230 tensors,
231 LayerShape::new(num_heads * head_dim, config.hidden_size, attn_bias),
232 unit_spec,
233 )?;
234 let q_norm = if tensors.contains_key(&format!("{p}.self_attn.q_norm.weight")) {
235 Some(RmsNorm::from_tensors(
236 &format!("{p}.self_attn.q_norm"),
237 tensors,
238 config.rms_norm_eps,
239 NormKind::PlusOne,
240 spec.dtype,
241 spec.device,
242 )?)
243 } else {
244 None
245 };
246 let k_norm = if tensors.contains_key(&format!("{p}.self_attn.k_norm.weight")) {
247 Some(RmsNorm::from_tensors(
248 &format!("{p}.self_attn.k_norm"),
249 tensors,
250 config.rms_norm_eps,
251 NormKind::PlusOne,
252 spec.dtype,
253 spec.device,
254 )?)
255 } else {
256 None
257 };
258 let post_attention_layernorm = RmsNorm::from_tensors(
259 &format!("{p}.post_attention_layernorm"),
260 tensors,
261 config.rms_norm_eps,
262 NormKind::PlusOne,
263 spec.dtype,
264 spec.device,
265 )?;
266 let pre_feedforward_layernorm = RmsNorm::from_tensors(
267 &format!("{p}.pre_feedforward_layernorm"),
268 tensors,
269 config.rms_norm_eps,
270 NormKind::PlusOne,
271 spec.dtype,
272 spec.device,
273 )?;
274 let gate_proj = LinearLayer::from_tensors(
275 &format!("{p}.mlp.gate_proj"),
276 tensors,
277 LayerShape::new(config.hidden_size, inter, mlp_bias),
278 unit_spec,
279 )?;
280 let up_proj = LinearLayer::from_tensors(
281 &format!("{p}.mlp.up_proj"),
282 tensors,
283 LayerShape::new(config.hidden_size, inter, mlp_bias),
284 unit_spec,
285 )?;
286 let down_proj = LinearLayer::from_tensors(
287 &format!("{p}.mlp.down_proj"),
288 tensors,
289 LayerShape::new(inter, config.hidden_size, mlp_bias),
290 unit_spec,
291 )?;
292 let post_feedforward_layernorm = RmsNorm::from_tensors(
293 &format!("{p}.post_feedforward_layernorm"),
294 tensors,
295 config.rms_norm_eps,
296 NormKind::PlusOne,
297 spec.dtype,
298 spec.device,
299 )?;
300
301 layers.push(GemmaDecoderLayer {
302 input_layernorm,
303 self_attn: GemmaAttention {
304 q_proj,
305 k_proj,
306 v_proj,
307 o_proj,
308 q_norm,
309 k_norm,
310 num_heads,
311 num_kv_heads,
312 head_dim,
313 scale,
314 softcap: config.attn_logit_softcapping,
315 sliding_window,
316 is_local,
317 },
318 post_attention_layernorm,
319 pre_feedforward_layernorm,
320 mlp: GemmaMlp {
321 gate_proj,
322 up_proj,
323 down_proj,
324 hidden_act: config.hidden_act.clone(),
325 },
326 post_feedforward_layernorm,
327 });
328 }
329
330 let norm = RmsNorm::from_tensors(
331 "model.norm",
332 tensors,
333 config.rms_norm_eps,
334 NormKind::PlusOne,
335 spec.dtype,
336 spec.device,
337 )?;
338 let lm_head = if !config.tie_word_embeddings && tensors.contains_key("lm_head.weight") {
339 Some(LinearLayer::from_tensors(
340 "lm_head",
341 tensors,
342 LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
343 LinearLoadSpec {
344 littlebit: None,
345 ratio_factor: 1.0,
346 ..spec
347 },
348 )?)
349 } else {
350 None
351 };
352
353 let rope_global = RotaryEmbedding::new(
354 head_dim,
355 config.max_position_embeddings.max(256),
356 config.rope_theta,
357 spec.dtype,
358 spec.device,
359 )?;
360 let rope_local = match config.rope_local_base_freq {
361 Some(theta) => Some(RotaryEmbedding::new(
362 head_dim,
363 config.max_position_embeddings.max(256),
364 theta,
365 spec.dtype,
366 spec.device,
367 )?),
368 None => None,
369 };
370
371 Ok(Self {
372 config,
373 embed_tokens,
374 layers,
375 norm,
376 lm_head,
377 rope_global,
378 rope_local,
379 })
380 }
381
382 pub fn new_random(
387 config: ModelConfig,
388 dtype: DType,
389 device: &Device,
390 rng: &mut ChaCha8Rng,
391 ) -> Result<Self> {
392 let arch = config.detect_architecture()?;
393 let embed_scale = Some((config.hidden_size as f64).sqrt());
394 let embed_tokens = Embedding::new_random(
395 config.vocab_size,
396 config.hidden_size,
397 embed_scale,
398 dtype,
399 device,
400 rng,
401 )?;
402 let head_dim = config.effective_head_dim();
403 let num_heads = config.num_attention_heads;
404 let num_kv_heads = config.effective_kv_heads();
405 let inter = config.effective_intermediate_size();
406 let attn_bias = BiasMode::from(config.attention_bias);
407 let mlp_bias = BiasMode::from(config.mlp_bias);
408 let scale = 1.0
409 / config
410 .query_pre_attn_scalar
411 .unwrap_or(head_dim as f64)
412 .sqrt();
413
414 let mut layers = Vec::with_capacity(config.num_hidden_layers);
415 for i in 0..config.num_hidden_layers {
416 let (sliding_window, is_local) = layer_window_and_local(&config, arch, i);
417 let (q_norm, k_norm) = if arch == ModelArchitecture::Gemma3 {
418 (
419 Some(RmsNorm::new_init(
420 head_dim,
421 config.rms_norm_eps,
422 NormKind::PlusOne,
423 dtype,
424 device,
425 )?),
426 Some(RmsNorm::new_init(
427 head_dim,
428 config.rms_norm_eps,
429 NormKind::PlusOne,
430 dtype,
431 device,
432 )?),
433 )
434 } else {
435 (None, None)
436 };
437 layers.push(GemmaDecoderLayer {
438 input_layernorm: RmsNorm::new_init(
439 config.hidden_size,
440 config.rms_norm_eps,
441 NormKind::PlusOne,
442 dtype,
443 device,
444 )?,
445 self_attn: GemmaAttention {
446 q_proj: LinearLayer::new_random(
447 LayerShape::new(config.hidden_size, num_heads * head_dim, attn_bias),
448 dtype,
449 device,
450 rng,
451 )?,
452 k_proj: LinearLayer::new_random(
453 LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
454 dtype,
455 device,
456 rng,
457 )?,
458 v_proj: LinearLayer::new_random(
459 LayerShape::new(config.hidden_size, num_kv_heads * head_dim, attn_bias),
460 dtype,
461 device,
462 rng,
463 )?,
464 o_proj: LinearLayer::new_random(
465 LayerShape::new(num_heads * head_dim, config.hidden_size, attn_bias),
466 dtype,
467 device,
468 rng,
469 )?,
470 q_norm,
471 k_norm,
472 num_heads,
473 num_kv_heads,
474 head_dim,
475 scale,
476 softcap: config.attn_logit_softcapping,
477 sliding_window,
478 is_local,
479 },
480 post_attention_layernorm: RmsNorm::new_init(
481 config.hidden_size,
482 config.rms_norm_eps,
483 NormKind::PlusOne,
484 dtype,
485 device,
486 )?,
487 pre_feedforward_layernorm: RmsNorm::new_init(
488 config.hidden_size,
489 config.rms_norm_eps,
490 NormKind::PlusOne,
491 dtype,
492 device,
493 )?,
494 mlp: GemmaMlp {
495 gate_proj: LinearLayer::new_random(
496 LayerShape::new(config.hidden_size, inter, mlp_bias),
497 dtype,
498 device,
499 rng,
500 )?,
501 up_proj: LinearLayer::new_random(
502 LayerShape::new(config.hidden_size, inter, mlp_bias),
503 dtype,
504 device,
505 rng,
506 )?,
507 down_proj: LinearLayer::new_random(
508 LayerShape::new(inter, config.hidden_size, mlp_bias),
509 dtype,
510 device,
511 rng,
512 )?,
513 hidden_act: config.hidden_act.clone(),
514 },
515 post_feedforward_layernorm: RmsNorm::new_init(
516 config.hidden_size,
517 config.rms_norm_eps,
518 NormKind::PlusOne,
519 dtype,
520 device,
521 )?,
522 });
523 }
524
525 let norm = RmsNorm::new_init(
526 config.hidden_size,
527 config.rms_norm_eps,
528 NormKind::PlusOne,
529 dtype,
530 device,
531 )?;
532 let lm_head = if config.tie_word_embeddings {
533 None
534 } else {
535 Some(LinearLayer::new_random(
536 LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
537 dtype,
538 device,
539 rng,
540 )?)
541 };
542 let rope_global = RotaryEmbedding::new(
543 head_dim,
544 config.max_position_embeddings.max(256),
545 config.rope_theta,
546 dtype,
547 device,
548 )?;
549 let rope_local = match config.rope_local_base_freq {
550 Some(theta) => Some(RotaryEmbedding::new(
551 head_dim,
552 config.max_position_embeddings.max(256),
553 theta,
554 dtype,
555 device,
556 )?),
557 None => None,
558 };
559 Ok(Self {
560 config,
561 embed_tokens,
562 layers,
563 norm,
564 lm_head,
565 rope_global,
566 rope_local,
567 })
568 }
569
570 pub fn forward(
575 &self,
576 input_ids: &Tensor,
577 mut cache: Option<&mut KvCache>,
578 hidden_mode: HiddenStatesMode,
579 ) -> Result<CausalLmOutput> {
580 let mut hidden = self.embed_tokens.forward(input_ids)?;
581 let collect = hidden_mode == HiddenStatesMode::All;
582 let mut all_hidden = if collect {
583 Vec::with_capacity(self.layers.len() + 1)
584 } else {
585 Vec::new()
586 };
587
588 for (idx, layer) in self.layers.iter().enumerate() {
589 if collect {
590 all_hidden.push(hidden.clone());
591 }
592 let residual = hidden.clone();
593 let normed = layer.input_layernorm.forward(&hidden)?;
594 let attn = layer.self_attn.forward(
595 &normed,
596 &self.rope_global,
597 self.rope_local.as_ref(),
598 idx,
599 cache.as_deref_mut(),
600 )?;
601 let attn_normed = layer.post_attention_layernorm.forward(&attn)?;
602 let h_mid = residual.add(&attn_normed)?;
603
604 let pre_ff = layer.pre_feedforward_layernorm.forward(&h_mid)?;
605 let ff_out = layer.mlp.forward(&pre_ff)?;
606 let ff_normed = layer.post_feedforward_layernorm.forward(&ff_out)?;
607 hidden = h_mid.add(&ff_normed)?;
608 }
609
610 hidden = self.norm.forward(&hidden)?;
611 if collect {
612 all_hidden.push(hidden.clone());
613 }
614 let mut logits = match &self.lm_head {
615 Some(head) => head.forward(&hidden)?,
616 None => self.embed_tokens.forward_lm_head(&hidden)?,
617 };
618 if let Some(cap) = self.config.final_logit_softcapping {
619 logits = ((logits / cap)?.tanh()? * cap)?;
620 }
621 Ok(CausalLmOutput {
622 logits,
623 hidden_states: all_hidden,
624 })
625 }
626
627 pub(crate) fn apply_littlebit_patch(
628 &mut self,
629 lb_cfg: &LittleBitConfig,
630 policy: &PatchPolicy,
631 train_mode: TrainMode,
632 base_seed: u64,
633 summary: &mut PatchSummary,
634 ) -> Result<()> {
635 for (i, layer) in self.layers.iter_mut().enumerate() {
636 let p = format!("model.layers.{i}");
637 patch_linear_layer(
638 &format!("{p}.self_attn.q_proj"),
639 &mut layer.self_attn.q_proj,
640 lb_cfg,
641 policy,
642 train_mode,
643 base_seed,
644 summary,
645 )?;
646 patch_linear_layer(
647 &format!("{p}.self_attn.k_proj"),
648 &mut layer.self_attn.k_proj,
649 lb_cfg,
650 policy,
651 train_mode,
652 base_seed,
653 summary,
654 )?;
655 patch_linear_layer(
656 &format!("{p}.self_attn.v_proj"),
657 &mut layer.self_attn.v_proj,
658 lb_cfg,
659 policy,
660 train_mode,
661 base_seed,
662 summary,
663 )?;
664 patch_linear_layer(
665 &format!("{p}.self_attn.o_proj"),
666 &mut layer.self_attn.o_proj,
667 lb_cfg,
668 policy,
669 train_mode,
670 base_seed,
671 summary,
672 )?;
673 patch_linear_layer(
674 &format!("{p}.mlp.gate_proj"),
675 &mut layer.mlp.gate_proj,
676 lb_cfg,
677 policy,
678 train_mode,
679 base_seed,
680 summary,
681 )?;
682 patch_linear_layer(
683 &format!("{p}.mlp.up_proj"),
684 &mut layer.mlp.up_proj,
685 lb_cfg,
686 policy,
687 train_mode,
688 base_seed,
689 summary,
690 )?;
691 patch_linear_layer(
692 &format!("{p}.mlp.down_proj"),
693 &mut layer.mlp.down_proj,
694 lb_cfg,
695 policy,
696 train_mode,
697 base_seed,
698 summary,
699 )?;
700 }
701 if let Some(head) = &mut self.lm_head {
702 patch_linear_layer(
703 "lm_head", head, lb_cfg, policy, train_mode, base_seed, summary,
704 )?;
705 }
706 Ok(())
707 }
708
709 pub(crate) fn convert_to_packed(&mut self) -> Result<()> {
710 for layer in &mut self.layers {
711 layer.self_attn.q_proj.convert_to_packed()?;
712 layer.self_attn.k_proj.convert_to_packed()?;
713 layer.self_attn.v_proj.convert_to_packed()?;
714 layer.self_attn.o_proj.convert_to_packed()?;
715 layer.mlp.gate_proj.convert_to_packed()?;
716 layer.mlp.up_proj.convert_to_packed()?;
717 layer.mlp.down_proj.convert_to_packed()?;
718 }
719 if let Some(head) = &mut self.lm_head {
720 head.convert_to_packed()?;
721 }
722 Ok(())
723 }
724
725 pub(crate) fn make_non_quantized_trainable(&mut self) -> Result<()> {
726 self.embed_tokens.make_trainable()?;
727 for layer in &mut self.layers {
728 layer.input_layernorm.make_trainable()?;
729 layer.post_attention_layernorm.make_trainable()?;
730 layer.pre_feedforward_layernorm.make_trainable()?;
731 layer.post_feedforward_layernorm.make_trainable()?;
732 layer.self_attn.q_proj.make_trainable()?;
733 layer.self_attn.k_proj.make_trainable()?;
734 layer.self_attn.v_proj.make_trainable()?;
735 layer.self_attn.o_proj.make_trainable()?;
736 if let Some(qn) = &mut layer.self_attn.q_norm {
737 qn.make_trainable()?;
738 }
739 if let Some(kn) = &mut layer.self_attn.k_norm {
740 kn.make_trainable()?;
741 }
742 layer.mlp.gate_proj.make_trainable()?;
743 layer.mlp.up_proj.make_trainable()?;
744 layer.mlp.down_proj.make_trainable()?;
745 }
746 self.norm.make_trainable()?;
747 if let Some(head) = &mut self.lm_head {
748 head.make_trainable()?;
749 }
750 Ok(())
751 }
752
753 pub(crate) fn freeze_matching(&mut self, exclude: &[&str]) {
754 let matches = |name: &str| exclude.iter().any(|k| name.contains(k));
755 if matches("model.embed_tokens") {
756 self.embed_tokens.freeze();
757 }
758 if matches("model.norm") {
759 self.norm.freeze();
760 }
761 if let Some(head) = &mut self.lm_head {
762 if matches("lm_head") {
763 head.freeze();
764 }
765 }
766 for (i, layer) in self.layers.iter_mut().enumerate() {
767 let p = format!("model.layers.{i}");
768 if matches(&format!("{p}.input_layernorm")) {
769 layer.input_layernorm.freeze();
770 }
771 if matches(&format!("{p}.post_attention_layernorm")) {
772 layer.post_attention_layernorm.freeze();
773 }
774 if matches(&format!("{p}.pre_feedforward_layernorm")) {
775 layer.pre_feedforward_layernorm.freeze();
776 }
777 if matches(&format!("{p}.post_feedforward_layernorm")) {
778 layer.post_feedforward_layernorm.freeze();
779 }
780 if matches(&format!("{p}.self_attn")) {
781 layer.self_attn.q_proj.freeze();
782 layer.self_attn.k_proj.freeze();
783 layer.self_attn.v_proj.freeze();
784 layer.self_attn.o_proj.freeze();
785 }
786 if matches(&format!("{p}.mlp")) {
787 layer.mlp.gate_proj.freeze();
788 layer.mlp.up_proj.freeze();
789 layer.mlp.down_proj.freeze();
790 }
791 }
792 }
793
794 pub(crate) fn trainable_vars(&self) -> Vec<Var> {
795 let mut vars = self.embed_tokens.trainable_vars();
796 for layer in &self.layers {
797 vars.extend(layer.input_layernorm.trainable_vars());
798 vars.extend(layer.self_attn.q_proj.trainable_vars());
799 vars.extend(layer.self_attn.k_proj.trainable_vars());
800 vars.extend(layer.self_attn.v_proj.trainable_vars());
801 vars.extend(layer.self_attn.o_proj.trainable_vars());
802 if let Some(qn) = &layer.self_attn.q_norm {
803 vars.extend(qn.trainable_vars());
804 }
805 if let Some(kn) = &layer.self_attn.k_norm {
806 vars.extend(kn.trainable_vars());
807 }
808 vars.extend(layer.post_attention_layernorm.trainable_vars());
809 vars.extend(layer.pre_feedforward_layernorm.trainable_vars());
810 vars.extend(layer.mlp.gate_proj.trainable_vars());
811 vars.extend(layer.mlp.up_proj.trainable_vars());
812 vars.extend(layer.mlp.down_proj.trainable_vars());
813 vars.extend(layer.post_feedforward_layernorm.trainable_vars());
814 }
815 vars.extend(self.norm.trainable_vars());
816 if let Some(head) = &self.lm_head {
817 vars.extend(head.trainable_vars());
818 }
819 vars
820 }
821
822 pub(crate) fn total_params(&self) -> usize {
823 let mut count = self.embed_tokens.weight.elem_count() + self.norm.weight.elem_count();
824 for layer in &self.layers {
825 count += layer.input_layernorm.weight.elem_count()
826 + layer.post_attention_layernorm.weight.elem_count()
827 + layer.pre_feedforward_layernorm.weight.elem_count()
828 + layer.post_feedforward_layernorm.weight.elem_count()
829 + layer.self_attn.q_proj.param_count()
830 + layer.self_attn.k_proj.param_count()
831 + layer.self_attn.v_proj.param_count()
832 + layer.self_attn.o_proj.param_count()
833 + layer
834 .self_attn
835 .q_norm
836 .as_ref()
837 .map_or(0, |n| n.weight.elem_count())
838 + layer
839 .self_attn
840 .k_norm
841 .as_ref()
842 .map_or(0, |n| n.weight.elem_count())
843 + layer.mlp.gate_proj.param_count()
844 + layer.mlp.up_proj.param_count()
845 + layer.mlp.down_proj.param_count();
846 }
847 if let Some(head) = &self.lm_head {
848 count += head.param_count();
849 }
850 count
851 }
852
853 pub(crate) fn pack_state_dict(&self) -> Result<HashMap<String, Tensor>> {
854 let mut out = HashMap::new();
855 self.embed_tokens
856 .pack_state_dict("model.embed_tokens", &mut out);
857 for (i, layer) in self.layers.iter().enumerate() {
858 let p = format!("model.layers.{i}");
859 layer
860 .input_layernorm
861 .pack_state_dict(&format!("{p}.input_layernorm"), &mut out);
862 out.extend(
863 layer
864 .self_attn
865 .q_proj
866 .pack_state_dict(&format!("{p}.self_attn.q_proj"))?,
867 );
868 out.extend(
869 layer
870 .self_attn
871 .k_proj
872 .pack_state_dict(&format!("{p}.self_attn.k_proj"))?,
873 );
874 out.extend(
875 layer
876 .self_attn
877 .v_proj
878 .pack_state_dict(&format!("{p}.self_attn.v_proj"))?,
879 );
880 out.extend(
881 layer
882 .self_attn
883 .o_proj
884 .pack_state_dict(&format!("{p}.self_attn.o_proj"))?,
885 );
886 if let Some(qn) = &layer.self_attn.q_norm {
887 qn.pack_state_dict(&format!("{p}.self_attn.q_norm"), &mut out);
888 }
889 if let Some(kn) = &layer.self_attn.k_norm {
890 kn.pack_state_dict(&format!("{p}.self_attn.k_norm"), &mut out);
891 }
892 layer
893 .post_attention_layernorm
894 .pack_state_dict(&format!("{p}.post_attention_layernorm"), &mut out);
895 layer
896 .pre_feedforward_layernorm
897 .pack_state_dict(&format!("{p}.pre_feedforward_layernorm"), &mut out);
898 out.extend(
899 layer
900 .mlp
901 .gate_proj
902 .pack_state_dict(&format!("{p}.mlp.gate_proj"))?,
903 );
904 out.extend(
905 layer
906 .mlp
907 .up_proj
908 .pack_state_dict(&format!("{p}.mlp.up_proj"))?,
909 );
910 out.extend(
911 layer
912 .mlp
913 .down_proj
914 .pack_state_dict(&format!("{p}.mlp.down_proj"))?,
915 );
916 layer
917 .post_feedforward_layernorm
918 .pack_state_dict(&format!("{p}.post_feedforward_layernorm"), &mut out);
919 }
920 self.norm.pack_state_dict("model.norm", &mut out);
921 if let Some(head) = &self.lm_head {
922 out.extend(head.pack_state_dict("lm_head")?);
923 }
924 Ok(out)
925 }
926
927 pub(crate) fn config(&self) -> &ModelConfig {
928 &self.config
929 }
930}