Skip to main content

littlebit_models/
phi.rs

1//! Phi-3 and Phi-4 decoder model using [`PhiAttentionProjection`] (`PhiQKVSplitAttention`).
2
3use std::collections::HashMap;
4
5use candle_core::{DType, Device, Tensor, Var};
6use rand_chacha::ChaCha8Rng;
7
8use littlebit_core::{BiasMode, LayerShape, LittleBitConfig, TrainMode};
9
10use crate::config::ModelConfig;
11use crate::error::Result;
12use crate::layer::{
13    apply_activation, compute_attention, AttentionParams, Embedding, KvCache, LinearLayer,
14    LinearLoadSpec, NormKind, RmsNorm, RotaryEmbedding,
15};
16use crate::model::{CausalLmOutput, HiddenStatesMode};
17use crate::patch::{patch_linear_layer, PatchPolicy, PatchSummary};
18use crate::phi_attn::PhiAttentionProjection;
19
20#[derive(Debug, Clone)]
21struct PhiAttention {
22    qkv: PhiAttentionProjection,
23    o_proj: LinearLayer,
24    num_heads: usize,
25    num_kv_heads: usize,
26    head_dim: usize,
27}
28
29impl PhiAttention {
30    fn forward(
31        &self,
32        hidden_states: &Tensor,
33        cfg: &ModelConfig,
34        rope: &RotaryEmbedding,
35        layer_idx: usize,
36        cache: Option<&mut KvCache>,
37    ) -> Result<Tensor> {
38        let (batch, seq_len, _) = hidden_states.dims3()?;
39        let pos_offset = cache.as_ref().map_or(0, |c| c.past_len(layer_idx));
40
41        let (q_raw, k_raw, v_raw) = self.qkv.project(hidden_states, cfg)?;
42        let q = q_raw
43            .reshape((batch, seq_len, self.num_heads, self.head_dim))?
44            .transpose(1, 2)?
45            .contiguous()?;
46        let k = k_raw
47            .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
48            .transpose(1, 2)?
49            .contiguous()?;
50        let v = v_raw
51            .reshape((batch, seq_len, self.num_kv_heads, self.head_dim))?
52            .transpose(1, 2)?
53            .contiguous()?;
54
55        let q = rope.apply(&q, pos_offset)?;
56        let k = rope.apply(&k, pos_offset)?;
57
58        let attn_out = compute_attention(
59            &q,
60            &k,
61            &v,
62            layer_idx,
63            cache,
64            AttentionParams {
65                scale: 1.0 / (self.head_dim as f64).sqrt(),
66                softcap: None,
67                sliding_window: None,
68            },
69        )?;
70        self.o_proj.forward(&attn_out)
71    }
72}
73
74#[derive(Debug, Clone)]
75struct PhiMlp {
76    gate_up_proj: LinearLayer,
77    down_proj: LinearLayer,
78    intermediate_size: usize,
79    hidden_act: String,
80}
81
82impl PhiMlp {
83    fn forward(&self, x: &Tensor) -> Result<Tensor> {
84        let up_states = self.gate_up_proj.forward(x)?;
85        let last = up_states.rank() - 1;
86        let gate = up_states.narrow(last, 0, self.intermediate_size)?;
87        let up = up_states.narrow(last, self.intermediate_size, self.intermediate_size)?;
88        let act = apply_activation(&gate, &self.hidden_act)?;
89        self.down_proj.forward(&act.mul(&up)?)
90    }
91}
92
93#[derive(Debug, Clone)]
94struct PhiDecoderLayer {
95    input_layernorm: RmsNorm,
96    self_attn: PhiAttention,
97    post_attention_layernorm: RmsNorm,
98    mlp: PhiMlp,
99}
100
101/// Causal language model supporting Phi-3 and Phi-4 (`model_type = "phi3"`).
102#[derive(Debug, Clone)]
103pub struct PhiModel {
104    config: ModelConfig,
105    embed_tokens: Embedding,
106    layers: Vec<PhiDecoderLayer>,
107    norm: RmsNorm,
108    lm_head: Option<LinearLayer>,
109    rope: RotaryEmbedding,
110}
111
112impl PhiModel {
113    /// Load a Phi-3 / Phi-4 model from tensors (accepts either fused `qkv_proj` or split `q_proj/k_proj/v_proj`).
114    ///
115    /// # Errors
116    /// Returns [`crate::ModelError::MissingWeight`] if required tensors are absent.
117    pub fn from_tensors(
118        config: ModelConfig,
119        tensors: &HashMap<String, Tensor>,
120        spec: LinearLoadSpec<'_>,
121    ) -> Result<Self> {
122        let embed_tokens =
123            Embedding::from_tensors("model.embed_tokens", tensors, None, spec.dtype, spec.device)?;
124        let head_dim = config.effective_head_dim();
125        let num_heads = config.num_attention_heads;
126        let num_kv_heads = config.effective_kv_heads();
127        let inter = config.effective_intermediate_size();
128        let attn_bias = BiasMode::from(config.attention_bias);
129        let mlp_bias = BiasMode::from(config.mlp_bias);
130
131        let mut layers = Vec::with_capacity(config.num_hidden_layers);
132        for i in 0..config.num_hidden_layers {
133            let p = format!("model.layers.{i}");
134            let input_layernorm = RmsNorm::from_tensors(
135                &format!("{p}.input_layernorm"),
136                tensors,
137                config.rms_norm_eps,
138                NormKind::Standard,
139                spec.dtype,
140                spec.device,
141            )?;
142            let qkv = PhiAttentionProjection::from_tensors(
143                &format!("{p}.self_attn"),
144                tensors,
145                &config,
146                spec.littlebit,
147                spec,
148            )?;
149            let o_proj = LinearLayer::from_tensors(
150                &format!("{p}.self_attn.o_proj"),
151                tensors,
152                LayerShape::new(num_heads * head_dim, config.hidden_size, attn_bias),
153                LinearLoadSpec {
154                    ratio_factor: 1.0,
155                    ..spec
156                },
157            )?;
158            let post_attention_layernorm = RmsNorm::from_tensors(
159                &format!("{p}.post_attention_layernorm"),
160                tensors,
161                config.rms_norm_eps,
162                NormKind::Standard,
163                spec.dtype,
164                spec.device,
165            )?;
166            let gate_up_proj = LinearLayer::from_tensors(
167                &format!("{p}.mlp.gate_up_proj"),
168                tensors,
169                LayerShape::new(config.hidden_size, 2 * inter, mlp_bias),
170                LinearLoadSpec {
171                    ratio_factor: 1.0,
172                    ..spec
173                },
174            )?;
175            let down_proj = LinearLayer::from_tensors(
176                &format!("{p}.mlp.down_proj"),
177                tensors,
178                LayerShape::new(inter, config.hidden_size, mlp_bias),
179                LinearLoadSpec {
180                    ratio_factor: 1.0,
181                    ..spec
182                },
183            )?;
184            layers.push(PhiDecoderLayer {
185                input_layernorm,
186                self_attn: PhiAttention {
187                    qkv,
188                    o_proj,
189                    num_heads,
190                    num_kv_heads,
191                    head_dim,
192                },
193                post_attention_layernorm,
194                mlp: PhiMlp {
195                    gate_up_proj,
196                    down_proj,
197                    intermediate_size: inter,
198                    hidden_act: config.hidden_act.clone(),
199                },
200            });
201        }
202
203        let norm = RmsNorm::from_tensors(
204            "model.norm",
205            tensors,
206            config.rms_norm_eps,
207            NormKind::Standard,
208            spec.dtype,
209            spec.device,
210        )?;
211        let lm_head = if !config.tie_word_embeddings && tensors.contains_key("lm_head.weight") {
212            Some(LinearLayer::from_tensors(
213                "lm_head",
214                tensors,
215                LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
216                LinearLoadSpec {
217                    littlebit: None,
218                    ratio_factor: 1.0,
219                    ..spec
220                },
221            )?)
222        } else {
223            None
224        };
225        let rope = RotaryEmbedding::new(
226            head_dim,
227            config.max_position_embeddings.max(256),
228            config.rope_theta,
229            spec.dtype,
230            spec.device,
231        )?;
232        Ok(Self {
233            config,
234            embed_tokens,
235            layers,
236            norm,
237            lm_head,
238            rope,
239        })
240    }
241
242    /// Initialize a random Phi-3 / Phi-4 model with fused `qkv_proj` for testing.
243    ///
244    /// # Errors
245    /// Returns an error if tensor creation fails.
246    pub fn new_random(
247        config: ModelConfig,
248        dtype: DType,
249        device: &Device,
250        rng: &mut ChaCha8Rng,
251    ) -> Result<Self> {
252        let embed_tokens = Embedding::new_random(
253            config.vocab_size,
254            config.hidden_size,
255            None,
256            dtype,
257            device,
258            rng,
259        )?;
260        let head_dim = config.effective_head_dim();
261        let num_heads = config.num_attention_heads;
262        let num_kv_heads = config.effective_kv_heads();
263        let inter = config.effective_intermediate_size();
264        let attn_bias = BiasMode::from(config.attention_bias);
265        let mlp_bias = BiasMode::from(config.mlp_bias);
266
267        let mut layers = Vec::with_capacity(config.num_hidden_layers);
268        for _ in 0..config.num_hidden_layers {
269            let qkv = PhiAttentionProjection::new_random_fused(&config, dtype, device, rng)?;
270            let o_proj = LinearLayer::new_random(
271                LayerShape::new(num_heads * head_dim, config.hidden_size, attn_bias),
272                dtype,
273                device,
274                rng,
275            )?;
276            let gate_up_proj = LinearLayer::new_random(
277                LayerShape::new(config.hidden_size, 2 * inter, mlp_bias),
278                dtype,
279                device,
280                rng,
281            )?;
282            let down_proj = LinearLayer::new_random(
283                LayerShape::new(inter, config.hidden_size, mlp_bias),
284                dtype,
285                device,
286                rng,
287            )?;
288            layers.push(PhiDecoderLayer {
289                input_layernorm: RmsNorm::new_init(
290                    config.hidden_size,
291                    config.rms_norm_eps,
292                    NormKind::Standard,
293                    dtype,
294                    device,
295                )?,
296                self_attn: PhiAttention {
297                    qkv,
298                    o_proj,
299                    num_heads,
300                    num_kv_heads,
301                    head_dim,
302                },
303                post_attention_layernorm: RmsNorm::new_init(
304                    config.hidden_size,
305                    config.rms_norm_eps,
306                    NormKind::Standard,
307                    dtype,
308                    device,
309                )?,
310                mlp: PhiMlp {
311                    gate_up_proj,
312                    down_proj,
313                    intermediate_size: inter,
314                    hidden_act: config.hidden_act.clone(),
315                },
316            });
317        }
318        let norm = RmsNorm::new_init(
319            config.hidden_size,
320            config.rms_norm_eps,
321            NormKind::Standard,
322            dtype,
323            device,
324        )?;
325        let lm_head = if config.tie_word_embeddings {
326            None
327        } else {
328            Some(LinearLayer::new_random(
329                LayerShape::new(config.hidden_size, config.vocab_size, BiasMode::NoBias),
330                dtype,
331                device,
332                rng,
333            )?)
334        };
335        let rope = RotaryEmbedding::new(
336            head_dim,
337            config.max_position_embeddings.max(256),
338            config.rope_theta,
339            dtype,
340            device,
341        )?;
342        Ok(Self {
343            config,
344            embed_tokens,
345            layers,
346            norm,
347            lm_head,
348            rope,
349        })
350    }
351
352    /// Forward pass returning logits `(batch, seq_len, vocab_size)` and optional hidden states.
353    ///
354    /// # Errors
355    /// Returns an error if layer forward passes fail.
356    pub fn forward(
357        &self,
358        input_ids: &Tensor,
359        mut cache: Option<&mut KvCache>,
360        hidden_mode: HiddenStatesMode,
361    ) -> Result<CausalLmOutput> {
362        let mut hidden = self.embed_tokens.forward(input_ids)?;
363        let collect = hidden_mode == HiddenStatesMode::All;
364        let mut all_hidden = if collect {
365            Vec::with_capacity(self.layers.len() + 1)
366        } else {
367            Vec::new()
368        };
369
370        for (idx, layer) in self.layers.iter().enumerate() {
371            if collect {
372                all_hidden.push(hidden.clone());
373            }
374            let residual = hidden.clone();
375            let normed = layer.input_layernorm.forward(&hidden)?;
376            let attn = layer.self_attn.forward(
377                &normed,
378                &self.config,
379                &self.rope,
380                idx,
381                cache.as_deref_mut(),
382            )?;
383            let hidden_mid = residual.add(&attn)?;
384            let normed_mid = layer.post_attention_layernorm.forward(&hidden_mid)?;
385            let mlp_out = layer.mlp.forward(&normed_mid)?;
386            hidden = hidden_mid.add(&mlp_out)?;
387        }
388
389        hidden = self.norm.forward(&hidden)?;
390        if collect {
391            all_hidden.push(hidden.clone());
392        }
393        let logits = match &self.lm_head {
394            Some(head) => head.forward(&hidden)?,
395            None => self.embed_tokens.forward_lm_head(&hidden)?,
396        };
397        Ok(CausalLmOutput {
398            logits,
399            hidden_states: all_hidden,
400        })
401    }
402
403    /// Borrow attention projections for verification (e.g., verifying split QKV).
404    #[must_use]
405    pub fn layer_qkv_projection(&self, layer_idx: usize) -> Option<&PhiAttentionProjection> {
406        self.layers.get(layer_idx).map(|l| &l.self_attn.qkv)
407    }
408
409    /// Split all fused `qkv_proj` attention projections into separate `q_proj`, `k_proj`, and `v_proj`
410    /// dense layers (`PhiQKVSplitAttention` initialization step).
411    ///
412    /// # Errors
413    /// Returns an error if splitting any layer fails.
414    pub fn split_qkv(&mut self) -> Result<()> {
415        for layer in &mut self.layers {
416            layer.self_attn.qkv.split_qkv(&self.config)?;
417        }
418        Ok(())
419    }
420
421    pub(crate) fn apply_littlebit_patch(
422        &mut self,
423        lb_cfg: &LittleBitConfig,
424        policy: &PatchPolicy,
425        train_mode: TrainMode,
426        base_seed: u64,
427        summary: &mut PatchSummary,
428    ) -> Result<()> {
429        for (i, layer) in self.layers.iter_mut().enumerate() {
430            let p = format!("model.layers.{i}");
431            layer.self_attn.qkv.apply_littlebit_patch(
432                &format!("{p}.self_attn"),
433                &self.config,
434                lb_cfg,
435                policy,
436                train_mode,
437                base_seed,
438                summary,
439            )?;
440            patch_linear_layer(
441                &format!("{p}.self_attn.o_proj"),
442                &mut layer.self_attn.o_proj,
443                lb_cfg,
444                policy,
445                train_mode,
446                base_seed,
447                summary,
448            )?;
449            patch_linear_layer(
450                &format!("{p}.mlp.gate_up_proj"),
451                &mut layer.mlp.gate_up_proj,
452                lb_cfg,
453                policy,
454                train_mode,
455                base_seed,
456                summary,
457            )?;
458            patch_linear_layer(
459                &format!("{p}.mlp.down_proj"),
460                &mut layer.mlp.down_proj,
461                lb_cfg,
462                policy,
463                train_mode,
464                base_seed,
465                summary,
466            )?;
467        }
468        if let Some(head) = &mut self.lm_head {
469            patch_linear_layer(
470                "lm_head", head, lb_cfg, policy, train_mode, base_seed, summary,
471            )?;
472        }
473        Ok(())
474    }
475
476    pub(crate) fn convert_to_packed(&mut self) -> Result<()> {
477        for layer in &mut self.layers {
478            layer.self_attn.qkv.convert_to_packed()?;
479            layer.self_attn.o_proj.convert_to_packed()?;
480            layer.mlp.gate_up_proj.convert_to_packed()?;
481            layer.mlp.down_proj.convert_to_packed()?;
482        }
483        if let Some(head) = &mut self.lm_head {
484            head.convert_to_packed()?;
485        }
486        Ok(())
487    }
488
489    pub(crate) fn make_non_quantized_trainable(&mut self) -> Result<()> {
490        self.embed_tokens.make_trainable()?;
491        for layer in &mut self.layers {
492            layer.input_layernorm.make_trainable()?;
493            layer.post_attention_layernorm.make_trainable()?;
494            layer.self_attn.o_proj.make_trainable()?;
495            layer.mlp.gate_up_proj.make_trainable()?;
496            layer.mlp.down_proj.make_trainable()?;
497        }
498        self.norm.make_trainable()?;
499        if let Some(head) = &mut self.lm_head {
500            head.make_trainable()?;
501        }
502        Ok(())
503    }
504
505    pub(crate) fn freeze_matching(&mut self, exclude: &[&str]) {
506        let matches = |name: &str| exclude.iter().any(|k| name.contains(k));
507        if matches("model.embed_tokens") {
508            self.embed_tokens.freeze();
509        }
510        if matches("model.norm") {
511            self.norm.freeze();
512        }
513        if let Some(head) = &mut self.lm_head {
514            if matches("lm_head") {
515                head.freeze();
516            }
517        }
518        for (i, layer) in self.layers.iter_mut().enumerate() {
519            let p = format!("model.layers.{i}");
520            if matches(&format!("{p}.input_layernorm")) {
521                layer.input_layernorm.freeze();
522            }
523            if matches(&format!("{p}.post_attention_layernorm")) {
524                layer.post_attention_layernorm.freeze();
525            }
526            if matches(&format!("{p}.self_attn")) {
527                layer.self_attn.qkv.freeze();
528                layer.self_attn.o_proj.freeze();
529            }
530            if matches(&format!("{p}.mlp")) {
531                layer.mlp.gate_up_proj.freeze();
532                layer.mlp.down_proj.freeze();
533            }
534        }
535    }
536
537    pub(crate) fn trainable_vars(&self) -> Vec<Var> {
538        let mut vars = self.embed_tokens.trainable_vars();
539        for layer in &self.layers {
540            vars.extend(layer.input_layernorm.trainable_vars());
541            vars.extend(layer.self_attn.qkv.trainable_vars());
542            vars.extend(layer.self_attn.o_proj.trainable_vars());
543            vars.extend(layer.post_attention_layernorm.trainable_vars());
544            vars.extend(layer.mlp.gate_up_proj.trainable_vars());
545            vars.extend(layer.mlp.down_proj.trainable_vars());
546        }
547        vars.extend(self.norm.trainable_vars());
548        if let Some(head) = &self.lm_head {
549            vars.extend(head.trainable_vars());
550        }
551        vars
552    }
553
554    pub(crate) fn total_params(&self) -> usize {
555        let mut count = self.embed_tokens.weight.elem_count() + self.norm.weight.elem_count();
556        for layer in &self.layers {
557            count += layer.input_layernorm.weight.elem_count()
558                + layer.post_attention_layernorm.weight.elem_count()
559                + layer.self_attn.qkv.param_count()
560                + layer.self_attn.o_proj.param_count()
561                + layer.mlp.gate_up_proj.param_count()
562                + layer.mlp.down_proj.param_count();
563        }
564        if let Some(head) = &self.lm_head {
565            count += head.param_count();
566        }
567        count
568    }
569
570    pub(crate) fn pack_state_dict(&self) -> Result<HashMap<String, Tensor>> {
571        let mut out = HashMap::new();
572        self.embed_tokens
573            .pack_state_dict("model.embed_tokens", &mut out);
574        for (i, layer) in self.layers.iter().enumerate() {
575            let p = format!("model.layers.{i}");
576            layer
577                .input_layernorm
578                .pack_state_dict(&format!("{p}.input_layernorm"), &mut out);
579            layer
580                .self_attn
581                .qkv
582                .pack_state_dict(&format!("{p}.self_attn"), &mut out)?;
583            out.extend(
584                layer
585                    .self_attn
586                    .o_proj
587                    .pack_state_dict(&format!("{p}.self_attn.o_proj"))?,
588            );
589            layer
590                .post_attention_layernorm
591                .pack_state_dict(&format!("{p}.post_attention_layernorm"), &mut out);
592            out.extend(
593                layer
594                    .mlp
595                    .gate_up_proj
596                    .pack_state_dict(&format!("{p}.mlp.gate_up_proj"))?,
597            );
598            out.extend(
599                layer
600                    .mlp
601                    .down_proj
602                    .pack_state_dict(&format!("{p}.mlp.down_proj"))?,
603            );
604        }
605        self.norm.pack_state_dict("model.norm", &mut out);
606        if let Some(head) = &self.lm_head {
607            out.extend(head.pack_state_dict("lm_head")?);
608        }
609        Ok(out)
610    }
611
612    pub(crate) fn config(&self) -> &ModelConfig {
613        &self.config
614    }
615}