1use std::collections::HashMap;
10
11use candle_core::{DType, Device, IndexOp, Tensor, Var};
12
13use littlebit_core::{BiasMode, LayerShape, LittleBitConfig, Param, TrainMode};
14
15use crate::config::ModelConfig;
16use crate::error::{ModelError, Result};
17use crate::layer::{LinearLayer, LinearLoadSpec};
18use crate::patch::{patch_linear_layer, PatchPolicy, PatchSummary};
19
20#[derive(Debug, Clone)]
23pub enum PhiAttentionProjection {
24 Fused {
26 qkv_proj: Box<LinearLayer>,
28 },
29 Split {
32 q_proj: Box<LinearLayer>,
34 k_proj: Box<LinearLayer>,
36 v_proj: Box<LinearLayer>,
38 },
39}
40
41impl PhiAttentionProjection {
42 pub fn from_tensors(
47 attn_prefix: &str,
48 tensors: &HashMap<String, Tensor>,
49 cfg: &ModelConfig,
50 littlebit_cfg: Option<&LittleBitConfig>,
51 spec_base: LinearLoadSpec<'_>,
52 ) -> Result<Self> {
53 let head_dim = cfg.effective_head_dim();
54 let q_dim = cfg.num_attention_heads * head_dim;
55 let kv_dim = cfg.effective_kv_heads() * head_dim;
56 let bias_mode = BiasMode::from(cfg.attention_bias);
57
58 let has_split = tensors.contains_key(&format!("{attn_prefix}.q_proj.U_packed"))
59 || tensors.contains_key(&format!("{attn_prefix}.q_proj.U"))
60 || tensors.contains_key(&format!("{attn_prefix}.q_proj.weight"));
61
62 if has_split {
63 let kv_factor = littlebit_cfg.map_or(1.0, |c| c.kv_factor);
64 let q_spec = LinearLoadSpec {
65 ratio_factor: 1.0,
66 ..spec_base
67 };
68 let kv_spec = LinearLoadSpec {
69 ratio_factor: kv_factor,
70 ..spec_base
71 };
72 let q_proj = LinearLayer::from_tensors(
73 &format!("{attn_prefix}.q_proj"),
74 tensors,
75 LayerShape::new(cfg.hidden_size, q_dim, bias_mode),
76 q_spec,
77 )?;
78 let k_proj = LinearLayer::from_tensors(
79 &format!("{attn_prefix}.k_proj"),
80 tensors,
81 LayerShape::new(cfg.hidden_size, kv_dim, bias_mode),
82 kv_spec,
83 )?;
84 let v_proj = LinearLayer::from_tensors(
85 &format!("{attn_prefix}.v_proj"),
86 tensors,
87 LayerShape::new(cfg.hidden_size, kv_dim, bias_mode),
88 kv_spec,
89 )?;
90 Ok(Self::Split {
91 q_proj: Box::new(q_proj),
92 k_proj: Box::new(k_proj),
93 v_proj: Box::new(v_proj),
94 })
95 } else {
96 let qkv_proj = LinearLayer::from_tensors(
97 &format!("{attn_prefix}.qkv_proj"),
98 tensors,
99 LayerShape::new(cfg.hidden_size, q_dim + 2 * kv_dim, bias_mode),
100 LinearLoadSpec {
101 littlebit: None,
102 ratio_factor: 1.0,
103 ..spec_base
104 },
105 )?;
106 Ok(Self::Fused {
107 qkv_proj: Box::new(qkv_proj),
108 })
109 }
110 }
111
112 pub fn project(
117 &self,
118 hidden_states: &Tensor,
119 cfg: &ModelConfig,
120 ) -> Result<(Tensor, Tensor, Tensor)> {
121 let head_dim = cfg.effective_head_dim();
122 let q_dim = cfg.num_attention_heads * head_dim;
123 let kv_dim = cfg.effective_kv_heads() * head_dim;
124
125 match self {
126 Self::Split {
127 q_proj,
128 k_proj,
129 v_proj,
130 } => Ok((
131 q_proj.forward(hidden_states)?,
132 k_proj.forward(hidden_states)?,
133 v_proj.forward(hidden_states)?,
134 )),
135 Self::Fused { qkv_proj } => {
136 let qkv = qkv_proj.forward(hidden_states)?;
137 let last = qkv.rank() - 1;
138 let q = qkv.narrow(last, 0, q_dim)?;
139 let k = qkv.narrow(last, q_dim, kv_dim)?;
140 let v = qkv.narrow(last, q_dim + kv_dim, kv_dim)?;
141 Ok((q, k, v))
142 }
143 }
144 }
145
146 pub fn split_qkv(&mut self, cfg: &ModelConfig) -> Result<()> {
152 let Self::Fused { qkv_proj } = self else {
153 return Ok(());
154 };
155 let (weight, bias) = match qkv_proj.as_ref() {
156 LinearLayer::Dense { weight, bias } => (weight.as_tensor().clone(), bias.clone()),
157 _ => {
158 return Err(ModelError::InvalidArgument(
159 "cannot split non-dense qkv_proj".to_string(),
160 ));
161 }
162 };
163
164 let head_dim = cfg.effective_head_dim();
165 let q_dim = cfg.num_attention_heads * head_dim;
166 let kv_dim = cfg.effective_kv_heads() * head_dim;
167
168 let qw = weight.i(0..q_dim)?.contiguous()?;
169 let kw = weight.i(q_dim..q_dim + kv_dim)?.contiguous()?;
170 let vw = weight.i(q_dim + kv_dim..q_dim + 2 * kv_dim)?.contiguous()?;
171
172 let (qb, kb, vb) = if let Some(b) = bias {
173 let bt = b.as_tensor();
174 (
175 Some(Param::Frozen(bt.i(0..q_dim)?.contiguous()?)),
176 Some(Param::Frozen(bt.i(q_dim..q_dim + kv_dim)?.contiguous()?)),
177 Some(Param::Frozen(
178 bt.i(q_dim + kv_dim..q_dim + 2 * kv_dim)?.contiguous()?,
179 )),
180 )
181 } else {
182 (None, None, None)
183 };
184
185 *self = Self::Split {
186 q_proj: Box::new(LinearLayer::Dense {
187 weight: Param::Frozen(qw),
188 bias: qb,
189 }),
190 k_proj: Box::new(LinearLayer::Dense {
191 weight: Param::Frozen(kw),
192 bias: kb,
193 }),
194 v_proj: Box::new(LinearLayer::Dense {
195 weight: Param::Frozen(vw),
196 bias: vb,
197 }),
198 };
199 Ok(())
200 }
201
202 pub fn apply_littlebit_patch(
207 &mut self,
208 attn_prefix: &str,
209 cfg: &ModelConfig,
210 lb_cfg: &LittleBitConfig,
211 policy: &PatchPolicy,
212 train_mode: TrainMode,
213 base_seed: u64,
214 summary: &mut PatchSummary,
215 ) -> Result<()> {
216 self.split_qkv(cfg)?;
217 if let Self::Split {
218 q_proj,
219 k_proj,
220 v_proj,
221 } = self
222 {
223 patch_linear_layer(
224 &format!("{attn_prefix}.q_proj"),
225 q_proj,
226 lb_cfg,
227 policy,
228 train_mode,
229 base_seed,
230 summary,
231 )?;
232 patch_linear_layer(
233 &format!("{attn_prefix}.k_proj"),
234 k_proj,
235 lb_cfg,
236 policy,
237 train_mode,
238 base_seed,
239 summary,
240 )?;
241 patch_linear_layer(
242 &format!("{attn_prefix}.v_proj"),
243 v_proj,
244 lb_cfg,
245 policy,
246 train_mode,
247 base_seed,
248 summary,
249 )?;
250 }
251 Ok(())
252 }
253
254 pub fn convert_to_packed(&mut self) -> Result<()> {
259 match self {
260 Self::Fused { qkv_proj } => qkv_proj.convert_to_packed(),
261 Self::Split {
262 q_proj,
263 k_proj,
264 v_proj,
265 } => {
266 q_proj.convert_to_packed()?;
267 k_proj.convert_to_packed()?;
268 v_proj.convert_to_packed()
269 }
270 }
271 }
272
273 pub fn pack_state_dict(
278 &self,
279 attn_prefix: &str,
280 out: &mut HashMap<String, Tensor>,
281 ) -> Result<()> {
282 match self {
283 Self::Fused { qkv_proj } => {
284 out.extend(qkv_proj.pack_state_dict(&format!("{attn_prefix}.qkv_proj"))?);
285 }
286 Self::Split {
287 q_proj,
288 k_proj,
289 v_proj,
290 } => {
291 out.extend(q_proj.pack_state_dict(&format!("{attn_prefix}.q_proj"))?);
292 out.extend(k_proj.pack_state_dict(&format!("{attn_prefix}.k_proj"))?);
293 out.extend(v_proj.pack_state_dict(&format!("{attn_prefix}.v_proj"))?);
294 }
295 }
296 Ok(())
297 }
298
299 #[must_use]
301 pub fn trainable_vars(&self) -> Vec<Var> {
302 match self {
303 Self::Fused { qkv_proj } => qkv_proj.trainable_vars(),
304 Self::Split {
305 q_proj,
306 k_proj,
307 v_proj,
308 } => {
309 let mut vars = q_proj.trainable_vars();
310 vars.extend(k_proj.trainable_vars());
311 vars.extend(v_proj.trainable_vars());
312 vars
313 }
314 }
315 }
316
317 pub fn freeze(&mut self) {
319 match self {
320 Self::Fused { qkv_proj } => qkv_proj.freeze(),
321 Self::Split {
322 q_proj,
323 k_proj,
324 v_proj,
325 } => {
326 q_proj.freeze();
327 k_proj.freeze();
328 v_proj.freeze();
329 }
330 }
331 }
332
333 #[must_use]
335 pub fn param_count(&self) -> usize {
336 match self {
337 Self::Fused { qkv_proj } => qkv_proj.param_count(),
338 Self::Split {
339 q_proj,
340 k_proj,
341 v_proj,
342 } => q_proj.param_count() + k_proj.param_count() + v_proj.param_count(),
343 }
344 }
345
346 #[allow(dead_code)]
348 pub(crate) fn new_random_fused(
349 cfg: &ModelConfig,
350 dtype: DType,
351 device: &Device,
352 rng: &mut rand_chacha::ChaCha8Rng,
353 ) -> Result<Self> {
354 let head_dim = cfg.effective_head_dim();
355 let q_dim = cfg.num_attention_heads * head_dim;
356 let kv_dim = cfg.effective_kv_heads() * head_dim;
357 let bias = BiasMode::from(cfg.attention_bias);
358 Ok(Self::Fused {
359 qkv_proj: Box::new(LinearLayer::new_random(
360 LayerShape::new(cfg.hidden_size, q_dim + 2 * kv_dim, bias),
361 dtype,
362 device,
363 rng,
364 )?),
365 })
366 }
367}