Skip to main content

littlebit_models/
phi_attn.rs

1// -----------------------------------------------------------------------------
2// Derived from quantization/modules/attention.py in SamsungLabs/LittleBit,
3// originally adapted under the Apache-2.0 License:
4// Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
5// -----------------------------------------------------------------------------
6
7//! Phi-3 / Phi-4 QKV splitting projection (`PhiQKVSplitAttention` parity).
8
9use std::collections::HashMap;
10
11use candle_core::{DType, Device, IndexOp, Tensor, Var};
12
13use littlebit_core::{BiasMode, LayerShape, LittleBitConfig, Param, TrainMode};
14
15use crate::config::ModelConfig;
16use crate::error::{ModelError, Result};
17use crate::layer::{LinearLayer, LinearLoadSpec};
18use crate::patch::{patch_linear_layer, PatchPolicy, PatchSummary};
19
20/// Attention QKV projection for Phi-3 / Phi-4 supporting both fused dense checkpoints
21/// (`qkv_proj`) and split LittleBit checkpoints (`q_proj`, `k_proj`, `v_proj`).
22#[derive(Debug, Clone)]
23pub enum PhiAttentionProjection {
24    /// Original fused Hugging Face Phi-3/Phi-4 `qkv_proj` dense layer.
25    Fused {
26        /// Combined `(q_dim + 2 * kv_dim, hidden_size)` linear projection.
27        qkv_proj: Box<LinearLayer>,
28    },
29    /// Split Q/K/V projections produced by `PhiQKVSplitAttention`, allowing separate
30    /// `kv_factor` scaling on `k_proj` and `v_proj`.
31    Split {
32        /// Query projection `(num_heads * head_dim, hidden_size)`.
33        q_proj: Box<LinearLayer>,
34        /// Key projection `(num_kv_heads * head_dim, hidden_size)`.
35        k_proj: Box<LinearLayer>,
36        /// Value projection `(num_kv_heads * head_dim, hidden_size)`.
37        v_proj: Box<LinearLayer>,
38    },
39}
40
41impl PhiAttentionProjection {
42    /// Load either split Q/K/V weights (when present in a quantized checkpoint) or fused `qkv_proj`.
43    ///
44    /// # Errors
45    /// Returns [`ModelError::MissingWeight`] if neither split nor fused projections exist.
46    pub fn from_tensors(
47        attn_prefix: &str,
48        tensors: &HashMap<String, Tensor>,
49        cfg: &ModelConfig,
50        littlebit_cfg: Option<&LittleBitConfig>,
51        spec_base: LinearLoadSpec<'_>,
52    ) -> Result<Self> {
53        let head_dim = cfg.effective_head_dim();
54        let q_dim = cfg.num_attention_heads * head_dim;
55        let kv_dim = cfg.effective_kv_heads() * head_dim;
56        let bias_mode = BiasMode::from(cfg.attention_bias);
57
58        let has_split = tensors.contains_key(&format!("{attn_prefix}.q_proj.U_packed"))
59            || tensors.contains_key(&format!("{attn_prefix}.q_proj.U"))
60            || tensors.contains_key(&format!("{attn_prefix}.q_proj.weight"));
61
62        if has_split {
63            let kv_factor = littlebit_cfg.map_or(1.0, |c| c.kv_factor);
64            let q_spec = LinearLoadSpec {
65                ratio_factor: 1.0,
66                ..spec_base
67            };
68            let kv_spec = LinearLoadSpec {
69                ratio_factor: kv_factor,
70                ..spec_base
71            };
72            let q_proj = LinearLayer::from_tensors(
73                &format!("{attn_prefix}.q_proj"),
74                tensors,
75                LayerShape::new(cfg.hidden_size, q_dim, bias_mode),
76                q_spec,
77            )?;
78            let k_proj = LinearLayer::from_tensors(
79                &format!("{attn_prefix}.k_proj"),
80                tensors,
81                LayerShape::new(cfg.hidden_size, kv_dim, bias_mode),
82                kv_spec,
83            )?;
84            let v_proj = LinearLayer::from_tensors(
85                &format!("{attn_prefix}.v_proj"),
86                tensors,
87                LayerShape::new(cfg.hidden_size, kv_dim, bias_mode),
88                kv_spec,
89            )?;
90            Ok(Self::Split {
91                q_proj: Box::new(q_proj),
92                k_proj: Box::new(k_proj),
93                v_proj: Box::new(v_proj),
94            })
95        } else {
96            let qkv_proj = LinearLayer::from_tensors(
97                &format!("{attn_prefix}.qkv_proj"),
98                tensors,
99                LayerShape::new(cfg.hidden_size, q_dim + 2 * kv_dim, bias_mode),
100                LinearLoadSpec {
101                    littlebit: None,
102                    ratio_factor: 1.0,
103                    ..spec_base
104                },
105            )?;
106            Ok(Self::Fused {
107                qkv_proj: Box::new(qkv_proj),
108            })
109        }
110    }
111
112    /// Project `hidden_states` into `(query, key, value)` tensors of shape `[..., dim]`.
113    ///
114    /// # Errors
115    /// Returns an error if underlying linear projections or slicing fail.
116    pub fn project(
117        &self,
118        hidden_states: &Tensor,
119        cfg: &ModelConfig,
120    ) -> Result<(Tensor, Tensor, Tensor)> {
121        let head_dim = cfg.effective_head_dim();
122        let q_dim = cfg.num_attention_heads * head_dim;
123        let kv_dim = cfg.effective_kv_heads() * head_dim;
124
125        match self {
126            Self::Split {
127                q_proj,
128                k_proj,
129                v_proj,
130            } => Ok((
131                q_proj.forward(hidden_states)?,
132                k_proj.forward(hidden_states)?,
133                v_proj.forward(hidden_states)?,
134            )),
135            Self::Fused { qkv_proj } => {
136                let qkv = qkv_proj.forward(hidden_states)?;
137                let last = qkv.rank() - 1;
138                let q = qkv.narrow(last, 0, q_dim)?;
139                let k = qkv.narrow(last, q_dim, kv_dim)?;
140                let v = qkv.narrow(last, q_dim + kv_dim, kv_dim)?;
141                Ok((q, k, v))
142            }
143        }
144    }
145
146    /// Convert fused `qkv_proj` into separate `q_proj`, `k_proj`, and `v_proj` layers
147    /// matching upstream `PhiQKVSplitAttention._initialize_parameters`.
148    ///
149    /// # Errors
150    /// Returns an error if slicing weights/biases fails or if `qkv_proj` is already quantized.
151    pub fn split_qkv(&mut self, cfg: &ModelConfig) -> Result<()> {
152        let Self::Fused { qkv_proj } = self else {
153            return Ok(());
154        };
155        let (weight, bias) = match qkv_proj.as_ref() {
156            LinearLayer::Dense { weight, bias } => (weight.as_tensor().clone(), bias.clone()),
157            _ => {
158                return Err(ModelError::InvalidArgument(
159                    "cannot split non-dense qkv_proj".to_string(),
160                ));
161            }
162        };
163
164        let head_dim = cfg.effective_head_dim();
165        let q_dim = cfg.num_attention_heads * head_dim;
166        let kv_dim = cfg.effective_kv_heads() * head_dim;
167
168        let qw = weight.i(0..q_dim)?.contiguous()?;
169        let kw = weight.i(q_dim..q_dim + kv_dim)?.contiguous()?;
170        let vw = weight.i(q_dim + kv_dim..q_dim + 2 * kv_dim)?.contiguous()?;
171
172        let (qb, kb, vb) = if let Some(b) = bias {
173            let bt = b.as_tensor();
174            (
175                Some(Param::Frozen(bt.i(0..q_dim)?.contiguous()?)),
176                Some(Param::Frozen(bt.i(q_dim..q_dim + kv_dim)?.contiguous()?)),
177                Some(Param::Frozen(
178                    bt.i(q_dim + kv_dim..q_dim + 2 * kv_dim)?.contiguous()?,
179                )),
180            )
181        } else {
182            (None, None, None)
183        };
184
185        *self = Self::Split {
186            q_proj: Box::new(LinearLayer::Dense {
187                weight: Param::Frozen(qw),
188                bias: qb,
189            }),
190            k_proj: Box::new(LinearLayer::Dense {
191                weight: Param::Frozen(kw),
192                bias: kb,
193            }),
194            v_proj: Box::new(LinearLayer::Dense {
195                weight: Param::Frozen(vw),
196                bias: vb,
197            }),
198        };
199        Ok(())
200    }
201
202    /// Split fused `qkv_proj` if needed and patch `q_proj`, `k_proj`, `v_proj` with LittleBit.
203    ///
204    /// # Errors
205    /// Returns an error if splitting or LittleBit factorization fails.
206    pub fn apply_littlebit_patch(
207        &mut self,
208        attn_prefix: &str,
209        cfg: &ModelConfig,
210        lb_cfg: &LittleBitConfig,
211        policy: &PatchPolicy,
212        train_mode: TrainMode,
213        base_seed: u64,
214        summary: &mut PatchSummary,
215    ) -> Result<()> {
216        self.split_qkv(cfg)?;
217        if let Self::Split {
218            q_proj,
219            k_proj,
220            v_proj,
221        } = self
222        {
223            patch_linear_layer(
224                &format!("{attn_prefix}.q_proj"),
225                q_proj,
226                lb_cfg,
227                policy,
228                train_mode,
229                base_seed,
230                summary,
231            )?;
232            patch_linear_layer(
233                &format!("{attn_prefix}.k_proj"),
234                k_proj,
235                lb_cfg,
236                policy,
237                train_mode,
238                base_seed,
239                summary,
240            )?;
241            patch_linear_layer(
242                &format!("{attn_prefix}.v_proj"),
243                v_proj,
244                lb_cfg,
245                policy,
246                train_mode,
247                base_seed,
248                summary,
249            )?;
250        }
251        Ok(())
252    }
253
254    /// Convert any `LittleBit` projections inside to compact 1-bit `Packed` representation.
255    ///
256    /// # Errors
257    /// Returns an error if packing fails.
258    pub fn convert_to_packed(&mut self) -> Result<()> {
259        match self {
260            Self::Fused { qkv_proj } => qkv_proj.convert_to_packed(),
261            Self::Split {
262                q_proj,
263                k_proj,
264                v_proj,
265            } => {
266                q_proj.convert_to_packed()?;
267                k_proj.convert_to_packed()?;
268                v_proj.convert_to_packed()
269            }
270        }
271    }
272
273    /// Export weights under `{attn_prefix}.{qkv_proj|q_proj,k_proj,v_proj}.*`.
274    ///
275    /// # Errors
276    /// Returns an error if tensor export fails.
277    pub fn pack_state_dict(
278        &self,
279        attn_prefix: &str,
280        out: &mut HashMap<String, Tensor>,
281    ) -> Result<()> {
282        match self {
283            Self::Fused { qkv_proj } => {
284                out.extend(qkv_proj.pack_state_dict(&format!("{attn_prefix}.qkv_proj"))?);
285            }
286            Self::Split {
287                q_proj,
288                k_proj,
289                v_proj,
290            } => {
291                out.extend(q_proj.pack_state_dict(&format!("{attn_prefix}.q_proj"))?);
292                out.extend(k_proj.pack_state_dict(&format!("{attn_prefix}.k_proj"))?);
293                out.extend(v_proj.pack_state_dict(&format!("{attn_prefix}.v_proj"))?);
294            }
295        }
296        Ok(())
297    }
298
299    /// Collect trainable variables.
300    #[must_use]
301    pub fn trainable_vars(&self) -> Vec<Var> {
302        match self {
303            Self::Fused { qkv_proj } => qkv_proj.trainable_vars(),
304            Self::Split {
305                q_proj,
306                k_proj,
307                v_proj,
308            } => {
309                let mut vars = q_proj.trainable_vars();
310                vars.extend(k_proj.trainable_vars());
311                vars.extend(v_proj.trainable_vars());
312                vars
313            }
314        }
315    }
316
317    /// Freeze all trainable parameters.
318    pub fn freeze(&mut self) {
319        match self {
320            Self::Fused { qkv_proj } => qkv_proj.freeze(),
321            Self::Split {
322                q_proj,
323                k_proj,
324                v_proj,
325            } => {
326                q_proj.freeze();
327                k_proj.freeze();
328                v_proj.freeze();
329            }
330        }
331    }
332
333    /// Total scalar parameters across Q/K/V projections.
334    #[must_use]
335    pub fn param_count(&self) -> usize {
336        match self {
337            Self::Fused { qkv_proj } => qkv_proj.param_count(),
338            Self::Split {
339                q_proj,
340                k_proj,
341                v_proj,
342            } => q_proj.param_count() + k_proj.param_count() + v_proj.param_count(),
343        }
344    }
345
346    /// Ensure dtype and device helper for tests.
347    #[allow(dead_code)]
348    pub(crate) fn new_random_fused(
349        cfg: &ModelConfig,
350        dtype: DType,
351        device: &Device,
352        rng: &mut rand_chacha::ChaCha8Rng,
353    ) -> Result<Self> {
354        let head_dim = cfg.effective_head_dim();
355        let q_dim = cfg.num_attention_heads * head_dim;
356        let kv_dim = cfg.effective_kv_heads() * head_dim;
357        let bias = BiasMode::from(cfg.attention_bias);
358        Ok(Self::Fused {
359            qkv_proj: Box::new(LinearLayer::new_random(
360                LayerShape::new(cfg.hidden_size, q_dim + 2 * kv_dim, bias),
361                dtype,
362                device,
363                rng,
364            )?),
365        })
366    }
367}