1use std::collections::HashMap;
4
5use candle_core::{DType, Device, Tensor, Var, D};
6use rand_chacha::ChaCha8Rng;
7use rand_distr::{Distribution, Normal};
8
9use littlebit_core::{
10 BiasMode, InferenceMode, LayerShape, LittleBitConfig, LittleBitLinear, PackedLittleBitLinear,
11 Param, TrainMode, WeightState,
12};
13
14use crate::error::{ModelError, Result};
15
16#[derive(Debug, Clone, Copy)]
18pub struct LinearLoadSpec<'a> {
19 pub littlebit: Option<&'a LittleBitConfig>,
21 pub ratio_factor: f64,
23 pub weight_state: WeightState,
25 pub inference_mode: InferenceMode,
27 pub dtype: DType,
29 pub device: &'a Device,
31}
32
33#[derive(Debug, Clone)]
36pub enum LinearLayer {
37 Dense {
39 weight: Param,
41 bias: Option<Param>,
43 },
44 LittleBit(Box<LittleBitLinear>),
46 Packed(Box<PackedLittleBitLinear>),
48}
49
50impl LinearLayer {
51 pub fn new_random(
56 shape: LayerShape,
57 dtype: DType,
58 device: &Device,
59 rng: &mut ChaCha8Rng,
60 ) -> Result<Self> {
61 let dist = Normal::new(0.0f32, 0.02f32).map_err(|err| {
62 ModelError::InvalidArgument(format!("failed to construct normal dist: {err}"))
63 })?;
64 let n = shape.out_features * shape.in_features;
65 let data: Vec<f32> = (0..n).map(|_| dist.sample(rng)).collect();
66 let w = Tensor::from_vec(data, (shape.out_features, shape.in_features), device)?
67 .to_dtype(dtype)?;
68 let bias = if shape.bias == BiasMode::WithBias {
69 Some(Param::Frozen(Tensor::zeros(
70 shape.out_features,
71 dtype,
72 device,
73 )?))
74 } else {
75 None
76 };
77 Ok(Self::Dense {
78 weight: Param::Frozen(w),
79 bias,
80 })
81 }
82
83 pub fn from_tensors(
89 prefix: &str,
90 tensors: &HashMap<String, Tensor>,
91 shape: LayerShape,
92 spec: LinearLoadSpec<'_>,
93 ) -> Result<Self> {
94 let packed_key = format!("{prefix}.U_packed");
95 let unpacked_key = format!("{prefix}.U");
96
97 if let Some(lb_cfg) = spec.littlebit {
98 let effective_cfg = lb_cfg.effective_for_module(prefix)?;
99 if spec.inference_mode == InferenceMode::Packed && tensors.contains_key(&packed_key) {
100 let packed = PackedLittleBitLinear::from_packed_tensors(
101 prefix,
102 tensors,
103 shape,
104 &effective_cfg,
105 spec.ratio_factor,
106 )?;
107 return Ok(Self::Packed(Box::new(packed)));
108 }
109 if tensors.contains_key(&unpacked_key) {
110 let lb = LittleBitLinear::from_state_dict(
111 prefix,
112 tensors,
113 shape,
114 &effective_cfg,
115 spec.ratio_factor,
116 spec.weight_state,
117 spec.dtype,
118 spec.device,
119 )?;
120 return Ok(Self::LittleBit(Box::new(lb)));
121 }
122 }
123
124 let w_key = format!("{prefix}.weight");
125 let w = tensors
126 .get(&w_key)
127 .ok_or_else(|| ModelError::MissingWeight(w_key.clone()))?;
128 let w = move_cast(w, spec.dtype, spec.device)?;
129
130 let bias = if shape.bias == BiasMode::WithBias {
131 let b_key = format!("{prefix}.bias");
132 match tensors.get(&b_key) {
133 Some(b) => Some(Param::Frozen(move_cast(b, spec.dtype, spec.device)?)),
134 None => None,
135 }
136 } else {
137 None
138 };
139
140 Ok(Self::Dense {
141 weight: Param::Frozen(w),
142 bias,
143 })
144 }
145
146 #[must_use]
148 pub fn in_features(&self) -> usize {
149 match self {
150 Self::Dense { weight, .. } => weight.as_tensor().dims().get(1).copied().unwrap_or(0),
151 Self::LittleBit(lb) => lb.in_features(),
152 Self::Packed(p) => p.budget().in_features,
153 }
154 }
155
156 #[must_use]
158 pub fn out_features(&self) -> usize {
159 match self {
160 Self::Dense { weight, .. } => weight.as_tensor().dims().first().copied().unwrap_or(0),
161 Self::LittleBit(lb) => lb.out_features(),
162 Self::Packed(p) => p.budget().out_features,
163 }
164 }
165
166 #[must_use]
168 pub fn bias_mode(&self) -> BiasMode {
169 match self {
170 Self::Dense { bias, .. } => BiasMode::from(bias.is_some()),
171 Self::LittleBit(lb) => BiasMode::from(lb.bias().is_some()),
172 Self::Packed(_) => BiasMode::NoBias,
173 }
174 }
175
176 pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
181 match self {
182 Self::Dense { weight, bias } => {
183 let dims = x.dims();
184 let in_f = self.in_features();
185 let out_f = self.out_features();
186 if dims.is_empty() || dims[dims.len() - 1] != in_f {
187 return Err(ModelError::Core(
188 littlebit_core::LittleBitError::InvalidShape {
189 expected: format!("[..., {in_f}]"),
190 actual: format!("{dims:?}"),
191 },
192 ));
193 }
194 let mut out_dims = dims[..dims.len() - 1].to_vec();
195 let batch_rows: usize = out_dims.iter().product::<usize>().max(1);
196 out_dims.push(out_f);
197
198 let wt = weight.as_tensor();
199 let wt = if wt.dtype() == x.dtype() {
200 wt.clone()
201 } else {
202 wt.to_dtype(x.dtype())?
203 };
204 let wt_t = wt.t()?.contiguous()?;
205 let mut y = x.reshape((batch_rows, in_f))?.matmul(&wt_t)?;
206 if let Some(b) = bias {
207 let bt = b.as_tensor();
208 let bt = if bt.dtype() == y.dtype() {
209 bt.clone()
210 } else {
211 bt.to_dtype(y.dtype())?
212 };
213 y = y.broadcast_add(&bt)?;
214 }
215 Ok(y.reshape(out_dims)?)
216 }
217 Self::LittleBit(lb) => Ok(lb.forward(x)?),
218 Self::Packed(p) => Ok(p.forward(x)?),
219 }
220 }
221
222 pub fn convert_to_littlebit(
227 &mut self,
228 config: &LittleBitConfig,
229 ratio_factor: f64,
230 train_mode: TrainMode,
231 seed: u64,
232 ) -> Result<()> {
233 match self {
234 Self::Dense { weight, bias } => {
235 let b_ref = bias.as_ref().map(Param::as_tensor);
236 let lb = LittleBitLinear::from_dense_weight(
237 weight.as_tensor(),
238 b_ref,
239 config,
240 ratio_factor,
241 train_mode,
242 seed,
243 )?;
244 *self = Self::LittleBit(Box::new(lb));
245 Ok(())
246 }
247 Self::LittleBit(_) => Ok(()),
248 Self::Packed(_) => Err(ModelError::InvalidArgument(
249 "cannot re-factorize an already packed linear layer".to_string(),
250 )),
251 }
252 }
253
254 pub fn convert_to_packed(&mut self) -> Result<()> {
259 if let Self::LittleBit(lb) = self {
260 *self = Self::Packed(Box::new(lb.to_packed()?));
261 }
262 Ok(())
263 }
264
265 pub fn pack_state_dict(&self, prefix: &str) -> Result<HashMap<String, Tensor>> {
270 match self {
271 Self::Dense { weight, bias } => {
272 let mut map = HashMap::new();
273 map.insert(format!("{prefix}.weight"), weight.as_tensor().detach());
274 if let Some(b) = bias {
275 map.insert(format!("{prefix}.bias"), b.as_tensor().detach());
276 }
277 Ok(map)
278 }
279 Self::LittleBit(lb) => Ok(lb.pack_state_dict(prefix)?),
280 Self::Packed(p) => Ok(p.pack_state_dict(prefix, &Device::Cpu)?),
281 }
282 }
283
284 #[must_use]
286 pub fn trainable_vars(&self) -> Vec<Var> {
287 match self {
288 Self::Dense { weight, bias } => {
289 let mut out = Vec::new();
290 if let Some(v) = weight.as_var() {
291 out.push(v.clone());
292 }
293 if let Some(b) = bias {
294 if let Some(v) = b.as_var() {
295 out.push(v.clone());
296 }
297 }
298 out
299 }
300 Self::LittleBit(lb) => lb.trainable_vars(),
301 Self::Packed(_) => Vec::new(),
302 }
303 }
304
305 pub fn make_trainable(&mut self) -> Result<()> {
310 if let Self::Dense { weight, bias } = self {
311 *weight = Param::Trainable(Var::from_tensor(weight.as_tensor())?);
312 if let Some(b) = bias {
313 *b = Param::Trainable(Var::from_tensor(b.as_tensor())?);
314 }
315 }
316 Ok(())
317 }
318
319 pub fn freeze(&mut self) {
321 match self {
322 Self::Dense { weight, bias } => {
323 weight.freeze();
324 if let Some(b) = bias {
325 b.freeze();
326 }
327 }
328 Self::LittleBit(lb) => lb.freeze(),
329 Self::Packed(_) => {}
330 }
331 }
332
333 #[must_use]
335 pub fn param_count(&self) -> usize {
336 match self {
337 Self::Dense { weight, bias } => {
338 weight.elem_count() + bias.as_ref().map_or(0, Param::elem_count)
339 }
340 Self::LittleBit(lb) => lb.param_count(),
341 Self::Packed(p) => {
342 let b = p.budget();
343 let mult = if b.residual == littlebit_core::ResidualMode::Dual {
344 2
345 } else {
346 1
347 };
348 mult * (b.out_features * b.split_dim
349 + b.split_dim * b.in_features
350 + b.out_features
351 + b.in_features
352 + 2 * b.split_dim)
353 }
354 }
355 }
356}
357
358#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
360pub enum NormKind {
361 Standard,
363 PlusOne,
365}
366
367#[derive(Debug, Clone)]
369pub struct RmsNorm {
370 pub weight: Param,
372 pub eps: f32,
374 pub kind: NormKind,
376}
377
378impl RmsNorm {
379 pub fn from_tensors(
384 prefix: &str,
385 tensors: &HashMap<String, Tensor>,
386 eps: f64,
387 kind: NormKind,
388 dtype: DType,
389 device: &Device,
390 ) -> Result<Self> {
391 let key = format!("{prefix}.weight");
392 let w = tensors
393 .get(&key)
394 .ok_or_else(|| ModelError::MissingWeight(key.clone()))?;
395 Ok(Self {
396 weight: Param::Frozen(move_cast(w, dtype, device)?),
397 eps: eps as f32,
398 kind,
399 })
400 }
401
402 pub fn new_init(
407 dim: usize,
408 eps: f64,
409 kind: NormKind,
410 dtype: DType,
411 device: &Device,
412 ) -> Result<Self> {
413 let weight = match kind {
414 NormKind::Standard => Tensor::ones(dim, dtype, device)?,
415 NormKind::PlusOne => Tensor::zeros(dim, dtype, device)?,
416 };
417 Ok(Self {
418 weight: Param::Frozen(weight),
419 eps: eps as f32,
420 kind,
421 })
422 }
423
424 pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
429 let w = self.weight.as_tensor();
430 let w = if w.dtype() == x.dtype() {
431 w.clone()
432 } else {
433 w.to_dtype(x.dtype())?
434 };
435 let alpha = match self.kind {
436 NormKind::Standard => w,
437 NormKind::PlusOne => (w + 1.0)?,
438 };
439 Ok(candle_nn::ops::rms_norm_slow(x, &alpha, self.eps)?)
440 }
441
442 #[must_use]
444 pub fn trainable_vars(&self) -> Vec<Var> {
445 self.weight.as_var().into_iter().cloned().collect()
446 }
447
448 pub fn make_trainable(&mut self) -> Result<()> {
453 if let Param::Frozen(t) = &self.weight {
454 self.weight = Param::Trainable(Var::from_tensor(t)?);
455 }
456 Ok(())
457 }
458
459 pub fn freeze(&mut self) {
461 self.weight.freeze();
462 }
463
464 pub fn pack_state_dict(&self, prefix: &str, out: &mut HashMap<String, Tensor>) {
466 out.insert(format!("{prefix}.weight"), self.weight.as_tensor().detach());
467 }
468}
469
470#[derive(Debug, Clone, Copy)]
472pub struct RmsNormNoScale {
473 pub eps: f32,
475}
476
477impl RmsNormNoScale {
478 #[must_use]
480 pub fn new(eps: f64) -> Self {
481 Self { eps: eps as f32 }
482 }
483
484 pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
489 let orig_dtype = x.dtype();
490 let x_f32 = if orig_dtype == DType::F32 {
491 x.clone()
492 } else {
493 x.to_dtype(DType::F32)?
494 };
495 let mean_sq = x_f32.sqr()?.mean_keepdim(D::Minus1)?;
496 let inv_rms = (mean_sq + f64::from(self.eps))?.powf(-0.5)?;
497 let out = x_f32.broadcast_mul(&inv_rms)?;
498 if orig_dtype == DType::F32 {
499 Ok(out)
500 } else {
501 Ok(out.to_dtype(orig_dtype)?)
502 }
503 }
504}
505
506#[derive(Debug, Clone)]
508pub struct LayerNorm {
509 pub weight: Option<Param>,
511 pub bias: Option<Param>,
513 pub dim: usize,
515 pub eps: f32,
517}
518
519impl LayerNorm {
520 pub fn from_tensors(
525 prefix: &str,
526 tensors: &HashMap<String, Tensor>,
527 dim: usize,
528 eps: f64,
529 affine: bool,
530 dtype: DType,
531 device: &Device,
532 ) -> Result<Self> {
533 if !affine {
534 return Ok(Self {
535 weight: None,
536 bias: None,
537 dim,
538 eps: eps as f32,
539 });
540 }
541 let w_key = format!("{prefix}.weight");
542 let b_key = format!("{prefix}.bias");
543 let w = tensors
544 .get(&w_key)
545 .ok_or_else(|| ModelError::MissingWeight(w_key.clone()))?;
546 let b = tensors
547 .get(&b_key)
548 .ok_or_else(|| ModelError::MissingWeight(b_key.clone()))?;
549 Ok(Self {
550 weight: Some(Param::Frozen(move_cast(w, dtype, device)?)),
551 bias: Some(Param::Frozen(move_cast(b, dtype, device)?)),
552 dim,
553 eps: eps as f32,
554 })
555 }
556
557 pub fn new_init(
562 dim: usize,
563 eps: f64,
564 affine: bool,
565 dtype: DType,
566 device: &Device,
567 ) -> Result<Self> {
568 let (weight, bias) = if affine {
569 (
570 Some(Param::Frozen(Tensor::ones(dim, dtype, device)?)),
571 Some(Param::Frozen(Tensor::zeros(dim, dtype, device)?)),
572 )
573 } else {
574 (None, None)
575 };
576 Ok(Self {
577 weight,
578 bias,
579 dim,
580 eps: eps as f32,
581 })
582 }
583
584 pub fn forward(&self, x: &Tensor) -> Result<Tensor> {
589 let alpha = match &self.weight {
590 Some(w) => {
591 let wt = w.as_tensor();
592 if wt.dtype() == x.dtype() {
593 wt.clone()
594 } else {
595 wt.to_dtype(x.dtype())?
596 }
597 }
598 None => Tensor::ones(self.dim, x.dtype(), x.device())?,
599 };
600 let beta = match &self.bias {
601 Some(b) => {
602 let bt = b.as_tensor();
603 if bt.dtype() == x.dtype() {
604 bt.clone()
605 } else {
606 bt.to_dtype(x.dtype())?
607 }
608 }
609 None => Tensor::zeros(self.dim, x.dtype(), x.device())?,
610 };
611 Ok(candle_nn::ops::layer_norm_slow(x, &alpha, &beta, self.eps)?)
612 }
613
614 #[must_use]
616 pub fn trainable_vars(&self) -> Vec<Var> {
617 let mut vars = Vec::new();
618 if let Some(w) = &self.weight {
619 if let Some(v) = w.as_var() {
620 vars.push(v.clone());
621 }
622 }
623 if let Some(b) = &self.bias {
624 if let Some(v) = b.as_var() {
625 vars.push(v.clone());
626 }
627 }
628 vars
629 }
630
631 pub fn make_trainable(&mut self) -> Result<()> {
636 if let Some(Param::Frozen(w)) = &self.weight {
637 self.weight = Some(Param::Trainable(Var::from_tensor(w)?));
638 }
639 if let Some(Param::Frozen(b)) = &self.bias {
640 self.bias = Some(Param::Trainable(Var::from_tensor(b)?));
641 }
642 Ok(())
643 }
644
645 pub fn freeze(&mut self) {
647 if let Some(w) = &mut self.weight {
648 w.freeze();
649 }
650 if let Some(b) = &mut self.bias {
651 b.freeze();
652 }
653 }
654
655 pub fn pack_state_dict(&self, prefix: &str, out: &mut HashMap<String, Tensor>) {
657 if let Some(w) = &self.weight {
658 out.insert(format!("{prefix}.weight"), w.as_tensor().detach());
659 }
660 if let Some(b) = &self.bias {
661 out.insert(format!("{prefix}.bias"), b.as_tensor().detach());
662 }
663 }
664
665 #[must_use]
667 pub fn param_count(&self) -> usize {
668 self.weight.as_ref().map_or(0, Param::elem_count)
669 + self.bias.as_ref().map_or(0, Param::elem_count)
670 }
671}
672
673#[derive(Debug, Clone)]
675pub struct Embedding {
676 pub weight: Param,
678 pub scale: Option<f64>,
680}
681
682impl Embedding {
683 pub fn from_tensors(
688 prefix: &str,
689 tensors: &HashMap<String, Tensor>,
690 scale: Option<f64>,
691 dtype: DType,
692 device: &Device,
693 ) -> Result<Self> {
694 let key = format!("{prefix}.weight");
695 let w = tensors
696 .get(&key)
697 .ok_or_else(|| ModelError::MissingWeight(key.clone()))?;
698 Ok(Self {
699 weight: Param::Frozen(move_cast(w, dtype, device)?),
700 scale,
701 })
702 }
703
704 pub fn new_random(
709 num_embeddings: usize,
710 embed_dim: usize,
711 scale: Option<f64>,
712 dtype: DType,
713 device: &Device,
714 rng: &mut ChaCha8Rng,
715 ) -> Result<Self> {
716 let dist = Normal::new(0.0f32, 0.02f32).map_err(|err| {
717 ModelError::InvalidArgument(format!("failed to construct normal dist: {err}"))
718 })?;
719 let data: Vec<f32> = (0..num_embeddings * embed_dim)
720 .map(|_| dist.sample(rng))
721 .collect();
722 let w = Tensor::from_vec(data, (num_embeddings, embed_dim), device)?.to_dtype(dtype)?;
723 Ok(Self {
724 weight: Param::Frozen(w),
725 scale,
726 })
727 }
728
729 pub fn forward(&self, ids: &Tensor) -> Result<Tensor> {
734 let mut out_dims = ids.dims().to_vec();
735 let embed_dim = self.weight.as_tensor().dims().get(1).copied().unwrap_or(0);
736 out_dims.push(embed_dim);
737
738 let flat_ids = ids.flatten_all()?.to_dtype(DType::U32)?;
739 let selected = self
740 .weight
741 .as_tensor()
742 .index_select(&flat_ids, 0)?
743 .reshape(out_dims)?;
744 match self.scale {
745 Some(s) => Ok((selected * s)?),
746 None => Ok(selected),
747 }
748 }
749
750 pub fn forward_lm_head(&self, hidden: &Tensor) -> Result<Tensor> {
755 let dims = hidden.dims();
756 let embed_dim = self.weight.as_tensor().dims().get(1).copied().unwrap_or(0);
757 let vocab_size = self.weight.as_tensor().dims().first().copied().unwrap_or(0);
758 let mut out_dims = dims[..dims.len() - 1].to_vec();
759 let batch_rows: usize = out_dims.iter().product::<usize>().max(1);
760 out_dims.push(vocab_size);
761
762 let w = self.weight.as_tensor();
763 let w = if w.dtype() == hidden.dtype() {
764 w.clone()
765 } else {
766 w.to_dtype(hidden.dtype())?
767 };
768 let wt = w.t()?.contiguous()?;
769 let out = hidden.reshape((batch_rows, embed_dim))?.matmul(&wt)?;
770 Ok(out.reshape(out_dims)?)
771 }
772
773 pub fn freeze(&mut self) {
775 self.weight.freeze();
776 }
777
778 #[must_use]
780 pub fn trainable_vars(&self) -> Vec<Var> {
781 self.weight.as_var().into_iter().cloned().collect()
782 }
783
784 pub fn make_trainable(&mut self) -> Result<()> {
789 if let Param::Frozen(w) = &self.weight {
790 self.weight = Param::Trainable(Var::from_tensor(w)?);
791 }
792 Ok(())
793 }
794
795 pub fn pack_state_dict(&self, prefix: &str, out: &mut HashMap<String, Tensor>) {
797 out.insert(format!("{prefix}.weight"), self.weight.as_tensor().detach());
798 }
799}
800
801#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
803pub enum RopeMode {
804 Default,
806 Proportional,
809}
810
811#[derive(Debug, Clone)]
814pub struct RotaryEmbedding {
815 head_dim: usize,
816 rope_theta: f64,
817 partial_rotary_factor: f64,
818 scale_factor: f64,
819 mode: RopeMode,
820 cos: Tensor,
821 sin: Tensor,
822}
823
824impl RotaryEmbedding {
825 pub fn new(
830 head_dim: usize,
831 max_seq: usize,
832 rope_theta: f64,
833 dtype: DType,
834 device: &Device,
835 ) -> Result<Self> {
836 Self::new_with_config(
837 head_dim,
838 max_seq,
839 rope_theta,
840 1.0,
841 1.0,
842 RopeMode::Default,
843 dtype,
844 device,
845 )
846 }
847
848 #[allow(clippy::too_many_arguments)]
854 pub fn new_with_config(
855 head_dim: usize,
856 max_seq: usize,
857 rope_theta: f64,
858 partial_rotary_factor: f64,
859 scale_factor: f64,
860 mode: RopeMode,
861 dtype: DType,
862 device: &Device,
863 ) -> Result<Self> {
864 if !head_dim.is_multiple_of(2) {
865 return Err(ModelError::InvalidArgument(format!(
866 "rotary embedding head_dim must be even, got {head_dim}"
867 )));
868 }
869 if !partial_rotary_factor.is_finite()
870 || partial_rotary_factor <= 0.0
871 || partial_rotary_factor > 1.0
872 {
873 return Err(ModelError::InvalidArgument(format!(
874 "partial_rotary_factor must be in (0.0, 1.0], got {partial_rotary_factor}"
875 )));
876 }
877 if !scale_factor.is_finite() || scale_factor <= 0.0 {
878 return Err(ModelError::InvalidArgument(format!(
879 "rope scale_factor must be finite and positive, got {scale_factor}"
880 )));
881 }
882 let (cos, sin) = compute_rope_tables_with_config(
883 head_dim,
884 max_seq,
885 rope_theta,
886 partial_rotary_factor,
887 scale_factor,
888 mode,
889 dtype,
890 device,
891 )?;
892 Ok(Self {
893 head_dim,
894 rope_theta,
895 partial_rotary_factor,
896 scale_factor,
897 mode,
898 cos,
899 sin,
900 })
901 }
902
903 pub fn apply(&self, x: &Tensor, pos_offset: usize) -> Result<Tensor> {
908 let (_, _, seq_len, _) = x.dims4()?;
909 let needed = pos_offset + seq_len;
910 let (cos_full, sin_full) = if needed <= self.cos.dims()[0] {
911 (self.cos.clone(), self.sin.clone())
912 } else {
913 compute_rope_tables_with_config(
914 self.head_dim,
915 needed,
916 self.rope_theta,
917 self.partial_rotary_factor,
918 self.scale_factor,
919 self.mode,
920 x.dtype(),
921 x.device(),
922 )?
923 };
924 let cos = cos_full
925 .narrow(0, pos_offset, seq_len)?
926 .to_dtype(x.dtype())?;
927 let sin = sin_full
928 .narrow(0, pos_offset, seq_len)?
929 .to_dtype(x.dtype())?;
930 Ok(candle_nn::rotary_emb::rope_slow(x, &cos, &sin)?)
931 }
932}
933
934#[allow(clippy::too_many_arguments)]
935fn compute_rope_tables_with_config(
936 head_dim: usize,
937 max_seq: usize,
938 rope_theta: f64,
939 partial_rotary_factor: f64,
940 scale_factor: f64,
941 mode: RopeMode,
942 dtype: DType,
943 device: &Device,
944) -> Result<(Tensor, Tensor)> {
945 let half_dim = head_dim / 2;
946 let rope_angles =
947 (((partial_rotary_factor * head_dim as f64) / 2.0).floor() as usize).min(half_dim);
948 let denom = match mode {
949 RopeMode::Proportional => head_dim as f64,
950 RopeMode::Default => (2 * rope_angles).max(2) as f64,
951 };
952 let mut inv_freq = vec![0.0f32; half_dim];
953 for (i, slot) in inv_freq.iter_mut().take(rope_angles).enumerate() {
954 *slot = (1.0f64 / rope_theta.powf((2 * i) as f64 / denom) / scale_factor) as f32;
955 }
956 let inv_freq_t = Tensor::from_vec(inv_freq, (1, half_dim), device)?;
957 let positions: Vec<f32> = (0..max_seq).map(|p| p as f32).collect();
958 let pos_t = Tensor::from_vec(positions, (max_seq, 1), device)?;
959 let freqs = pos_t.matmul(&inv_freq_t)?;
960 let cos = freqs.cos()?.to_dtype(dtype)?;
961 let sin = freqs.sin()?.to_dtype(dtype)?;
962 Ok((cos, sin))
963}
964
965#[derive(Debug, Clone)]
967pub struct KvCache {
968 layers: Vec<Option<(Tensor, Tensor)>>,
969 seen_tokens: Vec<usize>,
970}
971
972impl KvCache {
973 #[must_use]
975 pub fn new(num_layers: usize) -> Self {
976 Self {
977 layers: vec![None; num_layers],
978 seen_tokens: vec![0; num_layers],
979 }
980 }
981
982 #[must_use]
984 pub fn past_len(&self, layer_idx: usize) -> usize {
985 self.layers
986 .get(layer_idx)
987 .and_then(|entry| entry.as_ref())
988 .and_then(|(k, _)| k.dims().get(2).copied())
989 .unwrap_or(0)
990 }
991
992 #[must_use]
994 pub fn seq_pos(&self, layer_idx: usize) -> usize {
995 self.seen_tokens.get(layer_idx).copied().unwrap_or(0)
996 }
997
998 pub fn append(
1003 &mut self,
1004 layer_idx: usize,
1005 k: &Tensor,
1006 v: &Tensor,
1007 sliding_window: Option<usize>,
1008 ) -> Result<(Tensor, Tensor)> {
1009 let added = k.dims()[2];
1010 let slot = self.layers.get_mut(layer_idx).ok_or_else(|| {
1011 ModelError::InvalidArgument(format!("kv cache layer index {layer_idx} out of bounds"))
1012 })?;
1013 let (mut next_k, mut next_v) = match slot.take() {
1014 Some((prev_k, prev_v)) => (
1015 Tensor::cat(&[&prev_k, k], 2)?,
1016 Tensor::cat(&[&prev_v, v], 2)?,
1017 ),
1018 None => (k.clone(), v.clone()),
1019 };
1020
1021 if let Some(seen) = self.seen_tokens.get_mut(layer_idx) {
1022 *seen += added;
1023 }
1024
1025 if let Some(win) = sliding_window {
1026 let cur_len = next_k.dims()[2];
1027 if cur_len > win {
1028 next_k = next_k.narrow(2, cur_len - win, win)?;
1029 next_v = next_v.narrow(2, cur_len - win, win)?;
1030 }
1031 }
1032
1033 *slot = Some((next_k.clone(), next_v.clone()));
1034 Ok((next_k, next_v))
1035 }
1036
1037 pub fn clear(&mut self) {
1039 for slot in &mut self.layers {
1040 *slot = None;
1041 }
1042 self.seen_tokens.fill(0);
1043 }
1044}
1045
1046pub fn repeat_kv(x: &Tensor, n_rep: usize) -> Result<Tensor> {
1051 if n_rep == 1 {
1052 return Ok(x.clone());
1053 }
1054 let (batch, kv_heads, seq_len, head_dim) = x.dims4()?;
1055 let expanded = x
1056 .unsqueeze(2)?
1057 .expand((batch, kv_heads, n_rep, seq_len, head_dim))?
1058 .reshape((batch, kv_heads * n_rep, seq_len, head_dim))?;
1059 Ok(expanded)
1060}
1061
1062pub fn causal_mask(
1067 seq_len: usize,
1068 past_len: usize,
1069 sliding_window: Option<usize>,
1070 dtype: DType,
1071 device: &Device,
1072) -> Result<Tensor> {
1073 let total_len = past_len + seq_len;
1074 let mut mask = vec![0.0f32; seq_len * total_len];
1075 for q in 0..seq_len {
1076 let q_pos = past_len + q;
1077 for k in 0..total_len {
1078 let masked_future = k > q_pos;
1079 let masked_window = match sliding_window {
1080 Some(win) if win > 0 => q_pos >= win && k <= q_pos - win,
1081 _ => false,
1082 };
1083 if masked_future || masked_window {
1084 mask[q * total_len + k] = -1.0e9;
1085 }
1086 }
1087 }
1088 let t = Tensor::from_vec(mask, (1, 1, seq_len, total_len), device)?;
1089 if dtype == DType::F32 {
1090 Ok(t)
1091 } else {
1092 Ok(t.to_dtype(dtype)?)
1093 }
1094}
1095
1096pub fn apply_activation(x: &Tensor, name: &str) -> Result<Tensor> {
1101 match name.to_ascii_lowercase().as_str() {
1102 "silu" | "swish" => Ok(candle_nn::ops::silu(x)?),
1103 "gelu" => Ok(x.gelu_erf()?),
1104 "gelu_pytorch_tanh" | "gelu_new" | "gelu_fast" => Ok(x.gelu()?),
1105 "relu" => Ok(x.relu()?),
1106 other => Err(ModelError::UnsupportedArchitecture(format!(
1107 "unsupported activation function `{other}`"
1108 ))),
1109 }
1110}
1111
1112#[derive(Debug, Clone, Copy)]
1114pub struct AttentionParams {
1115 pub scale: f64,
1117 pub softcap: Option<f64>,
1119 pub sliding_window: Option<usize>,
1121}
1122
1123pub(crate) fn compute_attention(
1131 q: &Tensor,
1132 k: &Tensor,
1133 v: &Tensor,
1134 layer_idx: usize,
1135 cache: Option<&mut KvCache>,
1136 params: AttentionParams,
1137) -> Result<Tensor> {
1138 let (batch, num_heads, seq_len, head_dim) = q.dims4()?;
1139 let (_, num_kv_heads, _, _) = k.dims4()?;
1140 if num_kv_heads == 0 || num_heads % num_kv_heads != 0 {
1141 return Err(ModelError::InvalidArgument(format!(
1142 "num_attention_heads ({num_heads}) must be divisible by num_key_value_heads ({num_kv_heads})"
1143 )));
1144 }
1145 let n_rep = num_heads / num_kv_heads;
1146
1147 let (k_all, v_all) = match cache {
1148 Some(c) => c.append(layer_idx, k, v, params.sliding_window)?,
1149 None => (k.clone(), v.clone()),
1150 };
1151 let effective_past_len = k_all.dims()[2].saturating_sub(seq_len);
1152
1153 let k_rep = repeat_kv(&k_all, n_rep)?;
1154 let v_rep = repeat_kv(&v_all, n_rep)?;
1155
1156 let mut scores = (q.matmul(&k_rep.t()?.contiguous()?)? * params.scale)?;
1157 if let Some(cap) = params.softcap {
1158 scores = ((scores / cap)?.tanh()? * cap)?;
1159 }
1160 if seq_len > 1 {
1161 let mask = causal_mask(
1162 seq_len,
1163 effective_past_len,
1164 params.sliding_window,
1165 scores.dtype(),
1166 scores.device(),
1167 )?;
1168 scores = scores.broadcast_add(&mask)?;
1169 }
1170
1171 let probs = candle_nn::ops::softmax(&scores, D::Minus1)?;
1173 let ctx = probs.matmul(&v_rep.contiguous()?)?;
1174 Ok(ctx
1175 .transpose(1, 2)?
1176 .contiguous()?
1177 .reshape((batch, seq_len, num_heads * head_dim))?)
1178}
1179
1180pub(crate) fn move_cast(t: &Tensor, dtype: DType, device: &Device) -> Result<Tensor> {
1181 let moved = if t.device().same_device(device) {
1182 t.clone()
1183 } else {
1184 t.to_device(device)?
1185 };
1186 if moved.dtype() == dtype {
1187 Ok(moved)
1188 } else {
1189 Ok(moved.to_dtype(dtype)?)
1190 }
1191}