1use std::collections::HashMap;
4
5use candle_core::{DType, Device, Tensor, Var};
6use rand_chacha::ChaCha8Rng;
7
8use littlebit_core::{BiasMode, LayerShape, LittleBitConfig, TrainMode};
9
10use crate::config::ModelConfig;
11use crate::error::Result;
12use crate::layer::{
13 apply_activation, compute_attention, AttentionParams, Embedding, KvCache, LinearLayer,
14 LinearLoadSpec, NormKind, RmsNorm, RotaryEmbedding,
15};
16use crate::model::{CausalLmOutput, HiddenStatesMode};
17use crate::patch::{patch_linear_layer, PatchPolicy, PatchSummary};
18use crate::phi_attn::PhiAttentionProjection;
19
20#[derive(Debug, Clone)]
21struct PhiAttention {
22 qkv: PhiAttentionProjection,
23 o_proj: LinearLayer,
24 num_heads: usize,
25 num_kv_heads: usize,
26 head_dim: usize,
27}
28
29impl PhiAttention {
30 fn forward(
31 &self,
32 hidden_states: &Tensor,
33 cfg: &ModelConfig,
34 rope: &RotaryEmbedding,
35 layer_idx: usize,
36 cache: Option<&mut KvCache>,
37 ) -> Result<Tensor> {
38 let (batch, seq_len, _) = hidden_states.dims3()?;
39 let pos_offset = cache.as_ref().map_or(0, |c| c.past_len(layer_idx));
40
41 let (q_raw, k_raw, v_raw) = self.qkv.project(hidden_states, cfg)?;
42 let q = q_raw
43 .reshape((batch, seq_len, self.num_heads, self.head_dim))?
44 .transpose(1, 2)?
45 .contiguous()?;
46 let k = k_raw
47 .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
48 .transpose(1, 2)?
49 .contiguous()?;
50 let v = v_raw
51 .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
52 .transpose(1, 2)?
53 .contiguous()?;
54
55 let q = rope.apply(&q, pos_offset)?;
56 let k = rope.apply(&k, pos_offset)?;
57
58 let attn_out = compute_attention(
59 &q,
60 &k,
61 &v,
62 layer_idx,
63 cache,
64 AttentionParams {
65 scale: 1.0 / (self.head_dim as f64).sqrt(),
66 softcap: None,
67 sliding_window: None,
68 },
69 )?;
70 self.o_proj.forward(&attn_out)
71 }
72}
73
74#[derive(Debug, Clone)]
75struct PhiMlp {
76 gate_up_proj: LinearLayer,
77 down_proj: LinearLayer,
78 intermediate_size: usize,
79 hidden_act: String,
80}
81
82impl PhiMlp {
83 fn forward(&self, x: &Tensor) -> Result<Tensor> {
84 let up_states = self.gate_up_proj.forward(x)?;
85 let last = up_states.rank() - 1;
86 let gate = up_states.narrow(last, 0, self.intermediate_size)?;
87 let up = up_states.narrow(last, self.intermediate_size, self.intermediate_size)?;
88 let act = apply_activation(&gate, &self.hidden_act)?;
89 self.down_proj.forward(&act.mul(&up)?)
90 }
91}
92
93#[derive(Debug, Clone)]
94struct PhiDecoderLayer {
95 input_layernorm: RmsNorm,
96 self_attn: PhiAttention,
97 post_attention_layernorm: RmsNorm,
98 mlp: PhiMlp,
99}
100
101#[derive(Debug, Clone)]
103pub struct PhiModel {
104 config: ModelConfig,
105 embed_tokens: Embedding,
106 layers: Vec<PhiDecoderLayer>,
107 norm: RmsNorm,
108 lm_head: Option<LinearLayer>,
109 rope: RotaryEmbedding,
110}
111
112impl PhiModel {
113 pub fn from_tensors(
118 config: ModelConfig,
119 tensors: &HashMap<String, Tensor>,
120 spec: LinearLoadSpec<'_>,
121 ) -> Result<Self> {
122 let embed_tokens =
123 Embedding::from_tensors("model.embed_tokens", tensors, None, spec.dtype, spec.device)?;
124 let head_dim = config.effective_head_dim();
125 let num_heads = config.num_attention_heads;
126 let num_kv_heads = config.effective_kv_heads();
127 let inter = config.effective_intermediate_size();
128 let attn_bias = BiasMode::from(config.attention_bias);
129 let mlp_bias = BiasMode::from(config.mlp_bias);
130
131 let mut layers = Vec::with_capacity(config.num_hidden_layers);
132 for i in 0..config.num_hidden_layers {
133 let p = format!("model.layers.{i}");
134 let input_layernorm = RmsNorm::from_tensors(
135 &format!("{p}.input_layernorm"),
136 tensors,
137 config.rms_norm_eps,
138 NormKind::Standard,
139 spec.dtype,
140 spec.device,
141 )?;
142 let qkv = PhiAttentionProjection::from_tensors(
143 &format!("{p}.self_attn"),
144 tensors,
145 &config,
146 spec.littlebit,
147 spec,
148 )?;
149 let o_proj = LinearLayer::from_tensors(
150 &format!("{p}.self_attn.o_proj"),
151 tensors,
152 LayerShape::new(num_heads * head_dim, config.hidden_size, attn_bias),
153 LinearLoadSpec {
154 ratio_factor: 1.0,
155 ..spec
156 },
157 )?;
158 let post_attention_layernorm = RmsNorm::from_tensors(
159 &format!("{p}.post_attention_layernorm"),
160 tensors,
161 config.rms_norm_eps,
162 NormKind::Standard,
163 spec.dtype,
164 spec.device,
165 )?;
166 let gate_up_proj = LinearLayer::from_tensors(
167 &format!("{p}.mlp.gate_up_proj"),
168 tensors,
169 LayerShape::new(config.hidden_size, 2 * inter, mlp_bias),
170 LinearLoadSpec {
171 ratio_factor: 1.0,
172 ..spec
173 },
174 )?;
175 let down_proj = LinearLayer::from_tensors(
176 &format!("{p}.mlp.down_proj"),
177 tensors,
178 LayerShape::new(inter, config.hidden_size, mlp_bias),
179 LinearLoadSpec {
180 ratio_factor: 1.0,
181 ..spec
182 },
183 )?;
184 layers.push(PhiDecoderLayer {
185 input_layernorm,
186 self_attn: PhiAttention {
187 qkv,
188 o_proj,
189 num_heads,
190 num_kv_heads,
191 head_dim,
192 },
193 post_attention_layernorm,
194 mlp: PhiMlp {
195 gate_up_proj,
196 down_proj,
197 intermediate_size: inter,
198 hidden_act: config.hidden_act.clone(),
199 },
200 });
201 }
202
203 let norm = RmsNorm::from_tensors(
204 "model.norm",
205 tensors,
206 config.rms_norm_eps,
207 NormKind::Standard,
208 spec.dtype,
209 spec.device,
210 )?;
211 let lm_head = if !config.tie_word_embeddings && tensors.contains_key("lm_head.weight") {
212 Some(LinearLayer::from_tensors(
213 "lm_head",
214 tensors,
215 LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
216 LinearLoadSpec {
217 littlebit: None,
218 ratio_factor: 1.0,
219 ..spec
220 },
221 )?)
222 } else {
223 None
224 };
225 let rope = RotaryEmbedding::new(
226 head_dim,
227 config.max_position_embeddings.max(256),
228 config.rope_theta,
229 spec.dtype,
230 spec.device,
231 )?;
232 Ok(Self {
233 config,
234 embed_tokens,
235 layers,
236 norm,
237 lm_head,
238 rope,
239 })
240 }
241
242 pub fn new_random(
247 config: ModelConfig,
248 dtype: DType,
249 device: &Device,
250 rng: &mut ChaCha8Rng,
251 ) -> Result<Self> {
252 let embed_tokens = Embedding::new_random(
253 config.vocab_size,
254 config.hidden_size,
255 None,
256 dtype,
257 device,
258 rng,
259 )?;
260 let head_dim = config.effective_head_dim();
261 let num_heads = config.num_attention_heads;
262 let num_kv_heads = config.effective_kv_heads();
263 let inter = config.effective_intermediate_size();
264 let attn_bias = BiasMode::from(config.attention_bias);
265 let mlp_bias = BiasMode::from(config.mlp_bias);
266
267 let mut layers = Vec::with_capacity(config.num_hidden_layers);
268 for _ in 0..config.num_hidden_layers {
269 let qkv = PhiAttentionProjection::new_random_fused(&config, dtype, device, rng)?;
270 let o_proj = LinearLayer::new_random(
271 LayerShape::new(num_heads * head_dim, config.hidden_size, attn_bias),
272 dtype,
273 device,
274 rng,
275 )?;
276 let gate_up_proj = LinearLayer::new_random(
277 LayerShape::new(config.hidden_size, 2 * inter, mlp_bias),
278 dtype,
279 device,
280 rng,
281 )?;
282 let down_proj = LinearLayer::new_random(
283 LayerShape::new(inter, config.hidden_size, mlp_bias),
284 dtype,
285 device,
286 rng,
287 )?;
288 layers.push(PhiDecoderLayer {
289 input_layernorm: RmsNorm::new_init(
290 config.hidden_size,
291 config.rms_norm_eps,
292 NormKind::Standard,
293 dtype,
294 device,
295 )?,
296 self_attn: PhiAttention {
297 qkv,
298 o_proj,
299 num_heads,
300 num_kv_heads,
301 head_dim,
302 },
303 post_attention_layernorm: RmsNorm::new_init(
304 config.hidden_size,
305 config.rms_norm_eps,
306 NormKind::Standard,
307 dtype,
308 device,
309 )?,
310 mlp: PhiMlp {
311 gate_up_proj,
312 down_proj,
313 intermediate_size: inter,
314 hidden_act: config.hidden_act.clone(),
315 },
316 });
317 }
318 let norm = RmsNorm::new_init(
319 config.hidden_size,
320 config.rms_norm_eps,
321 NormKind::Standard,
322 dtype,
323 device,
324 )?;
325 let lm_head = if config.tie_word_embeddings {
326 None
327 } else {
328 Some(LinearLayer::new_random(
329 LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
330 dtype,
331 device,
332 rng,
333 )?)
334 };
335 let rope = RotaryEmbedding::new(
336 head_dim,
337 config.max_position_embeddings.max(256),
338 config.rope_theta,
339 dtype,
340 device,
341 )?;
342 Ok(Self {
343 config,
344 embed_tokens,
345 layers,
346 norm,
347 lm_head,
348 rope,
349 })
350 }
351
352 pub fn forward(
357 &self,
358 input_ids: &Tensor,
359 mut cache: Option<&mut KvCache>,
360 hidden_mode: HiddenStatesMode,
361 ) -> Result<CausalLmOutput> {
362 let mut hidden = self.embed_tokens.forward(input_ids)?;
363 let collect = hidden_mode == HiddenStatesMode::All;
364 let mut all_hidden = if collect {
365 Vec::with_capacity(self.layers.len() + 1)
366 } else {
367 Vec::new()
368 };
369
370 for (idx, layer) in self.layers.iter().enumerate() {
371 if collect {
372 all_hidden.push(hidden.clone());
373 }
374 let residual = hidden.clone();
375 let normed = layer.input_layernorm.forward(&hidden)?;
376 let attn = layer.self_attn.forward(
377 &normed,
378 &self.config,
379 &self.rope,
380 idx,
381 cache.as_deref_mut(),
382 )?;
383 let hidden_mid = residual.add(&attn)?;
384 let normed_mid = layer.post_attention_layernorm.forward(&hidden_mid)?;
385 let mlp_out = layer.mlp.forward(&normed_mid)?;
386 hidden = hidden_mid.add(&mlp_out)?;
387 }
388
389 hidden = self.norm.forward(&hidden)?;
390 if collect {
391 all_hidden.push(hidden.clone());
392 }
393 let logits = match &self.lm_head {
394 Some(head) => head.forward(&hidden)?,
395 None => self.embed_tokens.forward_lm_head(&hidden)?,
396 };
397 Ok(CausalLmOutput {
398 logits,
399 hidden_states: all_hidden,
400 })
401 }
402
403 #[must_use]
405 pub fn layer_qkv_projection(&self, layer_idx: usize) -> Option<&PhiAttentionProjection> {
406 self.layers.get(layer_idx).map(|l| &l.self_attn.qkv)
407 }
408
409 pub fn split_qkv(&mut self) -> Result<()> {
415 for layer in &mut self.layers {
416 layer.self_attn.qkv.split_qkv(&self.config)?;
417 }
418 Ok(())
419 }
420
421 pub(crate) fn apply_littlebit_patch(
422 &mut self,
423 lb_cfg: &LittleBitConfig,
424 policy: &PatchPolicy,
425 train_mode: TrainMode,
426 base_seed: u64,
427 summary: &mut PatchSummary,
428 ) -> Result<()> {
429 for (i, layer) in self.layers.iter_mut().enumerate() {
430 let p = format!("model.layers.{i}");
431 layer.self_attn.qkv.apply_littlebit_patch(
432 &format!("{p}.self_attn"),
433 &self.config,
434 lb_cfg,
435 policy,
436 train_mode,
437 base_seed,
438 summary,
439 )?;
440 patch_linear_layer(
441 &format!("{p}.self_attn.o_proj"),
442 &mut layer.self_attn.o_proj,
443 lb_cfg,
444 policy,
445 train_mode,
446 base_seed,
447 summary,
448 )?;
449 patch_linear_layer(
450 &format!("{p}.mlp.gate_up_proj"),
451 &mut layer.mlp.gate_up_proj,
452 lb_cfg,
453 policy,
454 train_mode,
455 base_seed,
456 summary,
457 )?;
458 patch_linear_layer(
459 &format!("{p}.mlp.down_proj"),
460 &mut layer.mlp.down_proj,
461 lb_cfg,
462 policy,
463 train_mode,
464 base_seed,
465 summary,
466 )?;
467 }
468 if let Some(head) = &mut self.lm_head {
469 patch_linear_layer(
470 "lm_head", head, lb_cfg, policy, train_mode, base_seed, summary,
471 )?;
472 }
473 Ok(())
474 }
475
476 pub(crate) fn convert_to_packed(&mut self) -> Result<()> {
477 for layer in &mut self.layers {
478 layer.self_attn.qkv.convert_to_packed()?;
479 layer.self_attn.o_proj.convert_to_packed()?;
480 layer.mlp.gate_up_proj.convert_to_packed()?;
481 layer.mlp.down_proj.convert_to_packed()?;
482 }
483 if let Some(head) = &mut self.lm_head {
484 head.convert_to_packed()?;
485 }
486 Ok(())
487 }
488
489 pub(crate) fn make_non_quantized_trainable(&mut self) -> Result<()> {
490 self.embed_tokens.make_trainable()?;
491 for layer in &mut self.layers {
492 layer.input_layernorm.make_trainable()?;
493 layer.post_attention_layernorm.make_trainable()?;
494 layer.self_attn.o_proj.make_trainable()?;
495 layer.mlp.gate_up_proj.make_trainable()?;
496 layer.mlp.down_proj.make_trainable()?;
497 }
498 self.norm.make_trainable()?;
499 if let Some(head) = &mut self.lm_head {
500 head.make_trainable()?;
501 }
502 Ok(())
503 }
504
505 pub(crate) fn freeze_matching(&mut self, exclude: &[&str]) {
506 let matches = |name: &str| exclude.iter().any(|k| name.contains(k));
507 if matches("model.embed_tokens") {
508 self.embed_tokens.freeze();
509 }
510 if matches("model.norm") {
511 self.norm.freeze();
512 }
513 if let Some(head) = &mut self.lm_head {
514 if matches("lm_head") {
515 head.freeze();
516 }
517 }
518 for (i, layer) in self.layers.iter_mut().enumerate() {
519 let p = format!("model.layers.{i}");
520 if matches(&format!("{p}.input_layernorm")) {
521 layer.input_layernorm.freeze();
522 }
523 if matches(&format!("{p}.post_attention_layernorm")) {
524 layer.post_attention_layernorm.freeze();
525 }
526 if matches(&format!("{p}.self_attn")) {
527 layer.self_attn.qkv.freeze();
528 layer.self_attn.o_proj.freeze();
529 }
530 if matches(&format!("{p}.mlp")) {
531 layer.mlp.gate_up_proj.freeze();
532 layer.mlp.down_proj.freeze();
533 }
534 }
535 }
536
537 pub(crate) fn trainable_vars(&self) -> Vec<Var> {
538 let mut vars = self.embed_tokens.trainable_vars();
539 for layer in &self.layers {
540 vars.extend(layer.input_layernorm.trainable_vars());
541 vars.extend(layer.self_attn.qkv.trainable_vars());
542 vars.extend(layer.self_attn.o_proj.trainable_vars());
543 vars.extend(layer.post_attention_layernorm.trainable_vars());
544 vars.extend(layer.mlp.gate_up_proj.trainable_vars());
545 vars.extend(layer.mlp.down_proj.trainable_vars());
546 }
547 vars.extend(self.norm.trainable_vars());
548 if let Some(head) = &self.lm_head {
549 vars.extend(head.trainable_vars());
550 }
551 vars
552 }
553
554 pub(crate) fn total_params(&self) -> usize {
555 let mut count = self.embed_tokens.weight.elem_count() + self.norm.weight.elem_count();
556 for layer in &self.layers {
557 count += layer.input_layernorm.weight.elem_count()
558 + layer.post_attention_layernorm.weight.elem_count()
559 + layer.self_attn.qkv.param_count()
560 + layer.self_attn.o_proj.param_count()
561 + layer.mlp.gate_up_proj.param_count()
562 + layer.mlp.down_proj.param_count();
563 }
564 if let Some(head) = &self.lm_head {
565 count += head.param_count();
566 }
567 count
568 }
569
570 pub(crate) fn pack_state_dict(&self) -> Result<HashMap<String, Tensor>> {
571 let mut out = HashMap::new();
572 self.embed_tokens
573 .pack_state_dict("model.embed_tokens", &mut out);
574 for (i, layer) in self.layers.iter().enumerate() {
575 let p = format!("model.layers.{i}");
576 layer
577 .input_layernorm
578 .pack_state_dict(&format!("{p}.input_layernorm"), &mut out);
579 layer
580 .self_attn
581 .qkv
582 .pack_state_dict(&format!("{p}.self_attn"), &mut out)?;
583 out.extend(
584 layer
585 .self_attn
586 .o_proj
587 .pack_state_dict(&format!("{p}.self_attn.o_proj"))?,
588 );
589 layer
590 .post_attention_layernorm
591 .pack_state_dict(&format!("{p}.post_attention_layernorm"), &mut out);
592 out.extend(
593 layer
594 .mlp
595 .gate_up_proj
596 .pack_state_dict(&format!("{p}.mlp.gate_up_proj"))?,
597 );
598 out.extend(
599 layer
600 .mlp
601 .down_proj
602 .pack_state_dict(&format!("{p}.mlp.down_proj"))?,
603 );
604 }
605 self.norm.pack_state_dict("model.norm", &mut out);
606 if let Some(head) = &self.lm_head {
607 out.extend(head.pack_state_dict("lm_head")?);
608 }
609 Ok(out)
610 }
611
612 pub(crate) fn config(&self) -> &ModelConfig {
613 &self.config
614 }
615}