Skip to main content

littlebit_models/
streaming.rs

1//! Streaming layer-by-layer quantization for large checkpoints under constrained RAM.
2//!
3//! Reads one transformer block (or one MoE expert slice) at a time directly from `.safetensors`
4//! shards via [`SafetensorsReader`], runs SVD + Joint-ITQ factorization and `i32` binary packing,
5//! streams the packed outputs to sharded safetensors via [`ShardedSafetensorsWriter`], and
6//! immediately drops dense weights between blocks.
7
8use std::collections::{BTreeMap, HashSet};
9use std::path::Path;
10
11use candle_core::{DType, Device, IndexOp, Tensor};
12use regex::Regex;
13use tracing::info;
14
15use littlebit_core::{
16    CastPolicy, LittleBitConfig, LittleBitLinear, SafetensorsReader, ShardedSafetensorsWriter,
17    TrainMode,
18};
19
20use crate::config::{ModelArchitecture, ModelConfig};
21use crate::error::{ModelError, Result};
22use crate::hub::write_quantized_metadata;
23use crate::patch::{derive_layer_seed, PatchPolicy, PatchSummary};
24
25const DEFAULT_MAX_SHARD_BYTES: usize = 2 * 1024 * 1024 * 1024;
26
27/// Configuration options for [`stream_quantize_checkpoint`].
28#[derive(Debug, Clone)]
29pub struct StreamingQuantizeOptions {
30    /// Deterministic RNG seed for randomized SVD and Joint-ITQ (`42` default).
31    pub seed: u64,
32    /// Serialization cast policy for scale/norm tensors (`ToBf16` or `Preserve`).
33    pub cast_policy: CastPolicy,
34    /// Maximum byte size per output shard before flushing to disk (`2 GiB` default).
35    pub max_shard_bytes: usize,
36    /// Compute device used during SVD/ITQ decomposition (`Device::Cpu` default).
37    pub device: Device,
38    /// Floating-point compute dtype during factorization (`DType::F32` default).
39    pub dtype: DType,
40    /// Skip multimodal vision/audio tower weights when quantizing multimodal checkpoints.
41    pub text_only: bool,
42    /// Optional base model ID recorded in `base_model.json`.
43    pub base_model_id: Option<String>,
44}
45
46impl Default for StreamingQuantizeOptions {
47    fn default() -> Self {
48        Self {
49            seed: 42,
50            cast_policy: CastPolicy::ToBf16,
51            max_shard_bytes: DEFAULT_MAX_SHARD_BYTES,
52            device: Device::Cpu,
53            dtype: DType::F32,
54            text_only: false,
55            base_model_id: None,
56        }
57    }
58}
59
60/// Summary report returned by [`stream_quantize_checkpoint`].
61#[derive(Debug, Clone)]
62pub struct StreamingQuantizeReport {
63    /// Quantization layer & bit-budget statistics across all processed blocks.
64    pub patch_summary: PatchSummary,
65    /// Number of `.safetensors` output shards written.
66    pub shards_written: usize,
67    /// Number of decoder transformer blocks processed.
68    pub blocks_processed: usize,
69    /// Number of non-quantized passthrough tensors copied directly (embeddings, norms, router, head).
70    pub passthrough_tensors: usize,
71}
72
73fn is_multimodal_tensor(name: &str) -> bool {
74    [
75        "vision_tower",
76        "audio_tower",
77        "multi_modal_projector",
78        "embed_vision",
79        "embed_audio",
80    ]
81    .iter()
82    .any(|pat| name.contains(pat))
83}
84
85fn quantize_and_write_linear(
86    module_name: &str,
87    weight: &Tensor,
88    bias: Option<&Tensor>,
89    lb_cfg: &LittleBitConfig,
90    policy: &PatchPolicy,
91    seed: u64,
92    writer: &mut ShardedSafetensorsWriter,
93    summary: &mut PatchSummary,
94) -> Result<()> {
95    let effective_cfg = lb_cfg.effective_for_module(module_name)?;
96    let ratio = policy.ratio_factor(module_name, &effective_cfg);
97    let layer_seed = derive_layer_seed(seed, module_name);
98    let lb = LittleBitLinear::from_dense_weight(
99        weight,
100        bias,
101        &effective_cfg,
102        ratio,
103        TrainMode::Eval,
104        layer_seed,
105    )?;
106    summary.record_budget(lb.budget());
107    let packed_sd = lb.pack_state_dict(module_name)?;
108    for (k, v) in packed_sd {
109        writer.add_tensor(k, v)?;
110    }
111    Ok(())
112}
113
114/// Quantize a dense local checkpoint in streaming layer-by-layer fashion without loading the
115/// full model weights into RAM.
116///
117/// Handles single-file and multi-shard input checkpoints across all supported architectures,
118/// including Phi-3/4 `qkv_proj` splitting, OPT `project_in`/`project_out`, and Gemma 4
119/// fused 3D MoE expert tensors (`gate_up_proj` / `down_proj`, loaded one expert slice at a time).
120///
121/// # Errors
122/// Returns an error if configs are invalid, required weights are malformed, or disk I/O fails.
123#[allow(clippy::too_many_lines)]
124pub fn stream_quantize_checkpoint(
125    src_dir: &Path,
126    dst_dir: &Path,
127    lb_cfg: &LittleBitConfig,
128    opts: &StreamingQuantizeOptions,
129) -> Result<StreamingQuantizeReport> {
130    lb_cfg.validate()?;
131    let model_cfg = ModelConfig::from_file(&src_dir.join("config.json"))?;
132    let arch = model_cfg.detect_architecture()?;
133    let mut reader = SafetensorsReader::open(src_dir)?;
134    let mut writer =
135        ShardedSafetensorsWriter::new(dst_dir, opts.max_shard_bytes, opts.cast_policy)?;
136    let policy = PatchPolicy::new();
137    let mut summary = PatchSummary::default();
138    let mut passthrough_tensors = 0usize;
139
140    let layer_re = Regex::new(r"(?:^|\.)layers\.(\d+)\.").map_err(|err| {
141        ModelError::InvalidArgument(format!("failed to compile layer index regex: {err}"))
142    })?;
143
144    let all_names: Vec<String> = reader.tensor_names().map(str::to_string).collect();
145    let mut blocks: BTreeMap<usize, Vec<String>> = BTreeMap::new();
146    let mut global_keys: Vec<String> = Vec::new();
147
148    for name in all_names {
149        let multimodal = is_multimodal_tensor(&name);
150        if opts.text_only && multimodal {
151            continue;
152        }
153        if !multimodal {
154            if let Some(caps) = layer_re.captures(&name) {
155                if let Ok(idx) = caps[1].parse::<usize>() {
156                    blocks.entry(idx).or_default().push(name);
157                    continue;
158                }
159            }
160        }
161        global_keys.push(name);
162    }
163
164    // 1) Process non-layer (global) tensors.
165    let global_set: HashSet<&str> = global_keys.iter().map(String::as_str).collect();
166    let mut global_consumed: HashSet<String> = HashSet::new();
167
168    for key in &global_keys {
169        if global_consumed.contains(key) {
170            continue;
171        }
172        if arch == ModelArchitecture::Opt
173            && (key == "model.decoder.project_in.weight"
174                || key == "model.decoder.project_out.weight")
175        {
176            let module_prefix = key.trim_end_matches(".weight");
177            let bias_key = format!("{module_prefix}.bias");
178            let w = reader
179                .load_tensor(key, &opts.device)?
180                .to_dtype(opts.dtype)?;
181            let b = if global_set.contains(bias_key.as_str()) {
182                Some(
183                    reader
184                        .load_tensor(&bias_key, &opts.device)?
185                        .to_dtype(opts.dtype)?,
186                )
187            } else {
188                None
189            };
190            quantize_and_write_linear(
191                module_prefix,
192                &w,
193                b.as_ref(),
194                lb_cfg,
195                &policy,
196                opts.seed,
197                &mut writer,
198                &mut summary,
199            )?;
200            global_consumed.insert(key.clone());
201            if b.is_some() {
202                global_consumed.insert(bias_key);
203            }
204            writer.flush_if_needed()?;
205            continue;
206        }
207
208        if let Some(prefix) = key.strip_suffix(".weight") {
209            if policy.is_excluded(prefix) {
210                summary.record_skipped();
211            }
212        }
213        let tensor = reader.load_tensor(key, &Device::Cpu)?;
214        writer.add_tensor(key.clone(), tensor)?;
215        passthrough_tensors += 1;
216        global_consumed.insert(key.clone());
217        writer.flush_if_needed()?;
218    }
219
220    // 2) Process decoder transformer blocks one at a time.
221    let mut blocks_processed = 0usize;
222    let total_blocks = blocks.len();
223
224    for (block_idx, keys) in blocks {
225        let key_set: HashSet<&str> = keys.iter().map(String::as_str).collect();
226        let mut consumed: HashSet<String> = HashSet::new();
227
228        // 2a) Phi-3 / Phi-4 fused `qkv_proj` split.
229        if arch == ModelArchitecture::Phi3 {
230            for key in &keys {
231                if let Some(module_prefix) = key.strip_suffix(".weight") {
232                    if let Some(attn_prefix) = module_prefix.strip_suffix(".qkv_proj") {
233                        let bias_key = format!("{module_prefix}.bias");
234                        let w = reader
235                            .load_tensor(key, &opts.device)?
236                            .to_dtype(opts.dtype)?;
237                        let b = if key_set.contains(bias_key.as_str()) {
238                            Some(
239                                reader
240                                    .load_tensor(&bias_key, &opts.device)?
241                                    .to_dtype(opts.dtype)?,
242                            )
243                        } else {
244                            None
245                        };
246
247                        let head_dim = model_cfg.effective_head_dim();
248                        let q_dim = model_cfg.num_attention_heads * head_dim;
249                        let kv_dim = model_cfg.effective_kv_heads() * head_dim;
250
251                        let qw = w.i(0..q_dim)?.contiguous()?;
252                        let kw = w.i(q_dim..q_dim + kv_dim)?.contiguous()?;
253                        let vw = w.i(q_dim + kv_dim..q_dim + 2 * kv_dim)?.contiguous()?;
254
255                        let (qb, kb, vb) = if let Some(ref bt) = b {
256                            (
257                                Some(bt.i(0..q_dim)?.contiguous()?),
258                                Some(bt.i(q_dim..q_dim + kv_dim)?.contiguous()?),
259                                Some(bt.i(q_dim + kv_dim..q_dim + 2 * kv_dim)?.contiguous()?),
260                            )
261                        } else {
262                            (None, None, None)
263                        };
264                        drop(w);
265                        drop(b);
266
267                        for (suffix, slice_w, slice_b) in
268                            [("q_proj", qw, qb), ("k_proj", kw, kb), ("v_proj", vw, vb)]
269                        {
270                            let split_name = format!("{attn_prefix}.{suffix}");
271                            quantize_and_write_linear(
272                                &split_name,
273                                &slice_w,
274                                slice_b.as_ref(),
275                                lb_cfg,
276                                &policy,
277                                opts.seed,
278                                &mut writer,
279                                &mut summary,
280                            )?;
281                            drop(slice_w);
282                            drop(slice_b);
283                            writer.flush_if_needed()?;
284                        }
285
286                        consumed.insert(key.clone());
287                        if key_set.contains(bias_key.as_str()) {
288                            consumed.insert(bias_key);
289                        }
290                    }
291                }
292            }
293        }
294
295        // 2b) Gemma 4 fused 3D MoE experts (`experts.gate_up_proj`, `experts.down_proj`).
296        for key in &keys {
297            if consumed.contains(key) {
298                continue;
299            }
300            let is_fused_expert =
301                key.ends_with(".experts.gate_up_proj") || key.ends_with(".experts.down_proj");
302            if !is_fused_expert {
303                continue;
304            }
305            let Some(loc) = reader.tensor_info(key) else {
306                continue;
307            };
308            if loc.shape.len() != 3 {
309                continue;
310            }
311            let num_experts = loc.shape[0];
312            let Some((experts_prefix, proj_name)) = key.rsplit_once('.') else {
313                continue;
314            };
315            let experts_prefix = experts_prefix.to_string();
316            let proj_name = proj_name.to_string();
317
318            for e in 0..num_experts {
319                let w_2d = reader
320                    .load_tensor_slice_dim0(key, e, &opts.device)?
321                    .to_dtype(opts.dtype)?;
322                let mod_name = format!("{experts_prefix}.{e}.{proj_name}");
323                quantize_and_write_linear(
324                    &mod_name,
325                    &w_2d,
326                    None,
327                    lb_cfg,
328                    &policy,
329                    opts.seed,
330                    &mut writer,
331                    &mut summary,
332                )?;
333                drop(w_2d);
334                writer.flush_if_needed()?;
335            }
336            consumed.insert(key.clone());
337        }
338
339        // 2c) Standard 2D linear weights (plus excluded router projections).
340        for key in &keys {
341            if consumed.contains(key) {
342                continue;
343            }
344            let Some(module_prefix) = key.strip_suffix(".weight") else {
345                continue;
346            };
347            let Some(loc) = reader.tensor_info(key) else {
348                continue;
349            };
350            if loc.shape.len() != 2 {
351                continue;
352            }
353
354            let bias_key = format!("{module_prefix}.bias");
355            if policy.is_excluded(module_prefix) {
356                summary.record_skipped();
357                let w = reader.load_tensor(key, &Device::Cpu)?;
358                writer.add_tensor(key.clone(), w)?;
359                passthrough_tensors += 1;
360                consumed.insert(key.clone());
361                if key_set.contains(bias_key.as_str()) {
362                    let b = reader.load_tensor(&bias_key, &Device::Cpu)?;
363                    writer.add_tensor(bias_key.clone(), b)?;
364                    passthrough_tensors += 1;
365                    consumed.insert(bias_key);
366                }
367                writer.flush_if_needed()?;
368                continue;
369            }
370
371            let w = reader
372                .load_tensor(key, &opts.device)?
373                .to_dtype(opts.dtype)?;
374            let b = if key_set.contains(bias_key.as_str()) {
375                Some(
376                    reader
377                        .load_tensor(&bias_key, &opts.device)?
378                        .to_dtype(opts.dtype)?,
379                )
380            } else {
381                None
382            };
383            quantize_and_write_linear(
384                module_prefix,
385                &w,
386                b.as_ref(),
387                lb_cfg,
388                &policy,
389                opts.seed,
390                &mut writer,
391                &mut summary,
392            )?;
393            drop(w);
394            drop(b);
395            consumed.insert(key.clone());
396            if key_set.contains(bias_key.as_str()) {
397                consumed.insert(bias_key);
398            }
399            writer.flush_if_needed()?;
400        }
401
402        // 2d) Remaining unconsumed non-linear tensors inside the block (norms, scales, buffers).
403        for key in &keys {
404            if consumed.contains(key) {
405                continue;
406            }
407            let tensor = reader.load_tensor(key, &Device::Cpu)?;
408            writer.add_tensor(key.clone(), tensor)?;
409            passthrough_tensors += 1;
410            writer.flush_if_needed()?;
411        }
412
413        blocks_processed += 1;
414        info!(
415            block = block_idx + 1,
416            total = total_blocks,
417            patched = summary.patched_layers,
418            avg_eff_bits = summary.avg_eff_bits,
419            "stream-quantized transformer block"
420        );
421    }
422
423    let shards_written = writer.finish()?;
424    write_quantized_metadata(
425        dst_dir,
426        &model_cfg,
427        lb_cfg,
428        opts.base_model_id.as_deref(),
429        Some(src_dir),
430    )?;
431
432    Ok(StreamingQuantizeReport {
433        patch_summary: summary,
434        shards_written,
435        blocks_processed,
436        passthrough_tensors,
437    })
438}
439
440#[cfg(test)]
441mod tests {
442    use super::*;
443    use crate::hub::{LittleBitModel, LoadOptions};
444    use crate::model::{CausalLm, HiddenStatesMode};
445    use littlebit_core::{inspect_checkpoint, InferenceMode, QuantFunc};
446    use std::collections::HashMap;
447    use std::fs;
448    use tempfile::TempDir;
449
450    fn save_dense_checkpoint(
451        dir: &Path,
452        cfg: &ModelConfig,
453        state_dict: &HashMap<String, Tensor>,
454    ) -> Result<()> {
455        fs::create_dir_all(dir)?;
456        fs::write(dir.join("config.json"), serde_json::to_string_pretty(cfg)?)?;
457        candle_core::safetensors::save(state_dict, dir.join("model.safetensors"))?;
458        Ok(())
459    }
460
461    #[test]
462    fn streaming_quantize_gemma4_dense_matches_in_memory_and_reloads_packed() -> Result<()> {
463        let dev = Device::Cpu;
464        let cfg = ModelConfig::tiny_gemma4_dense(96, 64, 2);
465        let src_dir = TempDir::new().expect("src");
466        let dst_dir = TempDir::new().expect("dst");
467
468        let mut ref_model = CausalLm::new_random(cfg.clone(), DType::F32, &dev, 51)?;
469        let dense_sd = ref_model.pack_state_dict()?;
470        save_dense_checkpoint(src_dir.path(), &cfg, &dense_sd)?;
471
472        let lb_cfg = LittleBitConfig {
473            quant_func: QuantFunc::SmoothSign { alpha: 100.0 },
474            eff_bit: Some(0.75),
475            residual: true,
476            kv_factor: 1.5,
477            min_split_dim: 8,
478            use_itq: true,
479            itq_n_iter: 10,
480            ..Default::default()
481        };
482
483        let ref_summary = ref_model.apply_littlebit_patch(&lb_cfg, TrainMode::Eval, 101)?;
484        let report = stream_quantize_checkpoint(
485            src_dir.path(),
486            dst_dir.path(),
487            &lb_cfg,
488            &StreamingQuantizeOptions {
489                seed: 101,
490                cast_policy: CastPolicy::Preserve,
491                max_shard_bytes: 16 * 1024,
492                ..Default::default()
493            },
494        )?;
495
496        assert_eq!(report.blocks_processed, cfg.num_hidden_layers);
497        assert_eq!(
498            report.patch_summary.patched_layers,
499            ref_summary.patched_layers
500        );
501        assert!(report.shards_written >= 2, "expected sharded output");
502
503        let input = Tensor::from_slice(&[1u32, 4, 7, 15], (1, 4), &dev)?;
504        let ref_logits = ref_model
505            .forward(&input, None, HiddenStatesMode::None)?
506            .logits;
507
508        for mode in [InferenceMode::Unpacked, InferenceMode::Packed] {
509            let reloaded = LittleBitModel::from_pretrained(
510                dst_dir.path().to_str().expect("utf8"),
511                &LoadOptions {
512                    inference_mode: mode,
513                    ..Default::default()
514                },
515            )?;
516            let got_logits = reloaded
517                .model
518                .forward(&input, None, HiddenStatesMode::None)?
519                .logits;
520            let diff = ref_logits
521                .sub(&got_logits)?
522                .abs()?
523                .max_all()?
524                .to_scalar::<f32>()?;
525            assert!(
526                diff < 1e-4,
527                "streaming vs in-memory Gemma 4 Dense drift ({mode:?}): {diff}"
528            );
529        }
530        Ok(())
531    }
532
533    #[test]
534    fn streaming_quantize_gemma4_moe_from_fused_3d_experts_multi_shard() -> Result<()> {
535        let dev = Device::Cpu;
536        let cfg = ModelConfig::tiny_gemma4_moe(80, 32, 2, 6, 2);
537        let src_dir = TempDir::new().expect("src");
538        let dst_dir = TempDir::new().expect("dst");
539
540        let mut ref_model = CausalLm::new_random(cfg.clone(), DType::F32, &dev, 88)?;
541        let mut hf_sd = ref_model.pack_state_dict()?;
542
543        // Convert per-expert keys to Hugging Face fused 3D `experts.gate_up_proj` / `experts.down_proj`.
544        for l in 0..cfg.num_hidden_layers {
545            let mut gu_slices = Vec::new();
546            let mut dn_slices = Vec::new();
547            for e in 0..6 {
548                gu_slices.push(
549                    hf_sd
550                        .remove(&format!("model.layers.{l}.experts.{e}.gate_up_proj.weight"))
551                        .expect("gu"),
552                );
553                dn_slices.push(
554                    hf_sd
555                        .remove(&format!("model.layers.{l}.experts.{e}.down_proj.weight"))
556                        .expect("dn"),
557                );
558            }
559            hf_sd.insert(
560                format!("model.layers.{l}.experts.gate_up_proj"),
561                Tensor::stack(&gu_slices, 0)?,
562            );
563            hf_sd.insert(
564                format!("model.layers.{l}.experts.down_proj"),
565                Tensor::stack(&dn_slices, 0)?,
566            );
567        }
568        save_dense_checkpoint(src_dir.path(), &cfg, &hf_sd)?;
569
570        let lb_cfg = LittleBitConfig {
571            eff_bit: Some(0.8),
572            residual: false,
573            use_itq: true,
574            itq_n_iter: 6,
575            ..Default::default()
576        };
577        let ref_summary = ref_model.apply_littlebit_patch(&lb_cfg, TrainMode::Eval, 55)?;
578
579        let report = stream_quantize_checkpoint(
580            src_dir.path(),
581            dst_dir.path(),
582            &lb_cfg,
583            &StreamingQuantizeOptions {
584                seed: 55,
585                cast_policy: CastPolicy::Preserve,
586                max_shard_bytes: 8 * 1024,
587                ..Default::default()
588            },
589        )?;
590        assert_eq!(
591            report.patch_summary.patched_layers,
592            ref_summary.patched_layers
593        );
594        assert_eq!(
595            report.patch_summary.skipped_layers,
596            ref_summary.skipped_layers
597        );
598        assert!(report.shards_written >= 2);
599
600        let stats = inspect_checkpoint(dst_dir.path())?;
601        assert!(stats.is_packed);
602        assert_eq!(stats.quantized_layer_count, ref_summary.patched_layers);
603
604        let loaded = LittleBitModel::from_pretrained(
605            dst_dir.path().to_str().expect("utf8"),
606            &LoadOptions {
607                inference_mode: InferenceMode::Packed,
608                ..Default::default()
609            },
610        )?;
611        if let CausalLm::Gemma4(ref g4) = loaded.model {
612            let moe = g4.layer_moe(0).expect("moe 0");
613            assert!(matches!(
614                moe.router.proj,
615                crate::layer::LinearLayer::Dense { .. }
616            ));
617            assert!(matches!(
618                moe.experts.experts[0].gate_up_proj,
619                crate::layer::LinearLayer::Packed(_)
620            ));
621        } else {
622            panic!("expected Gemma4");
623        }
624
625        let input = Tensor::from_slice(&[2u32, 9, 13], (1, 3), &dev)?;
626        let ref_logits = ref_model
627            .forward(&input, None, HiddenStatesMode::None)?
628            .logits;
629        let got_logits = loaded
630            .model
631            .forward(&input, None, HiddenStatesMode::None)?
632            .logits;
633        let diff = ref_logits
634            .sub(&got_logits)?
635            .abs()?
636            .max_all()?
637            .to_scalar::<f32>()?;
638        assert!(diff < 1e-4, "streaming MoE vs in-memory drift: {diff}");
639        Ok(())
640    }
641
642    #[test]
643    fn streaming_quantize_phi3_and_text_only_stripping() -> Result<()> {
644        let dev = Device::Cpu;
645        let cfg = ModelConfig::tiny_phi3(64, 32, 2);
646        let src_dir = TempDir::new().expect("src");
647        let dst_dir = TempDir::new().expect("dst");
648
649        let mut ref_model = CausalLm::new_random(cfg.clone(), DType::F32, &dev, 17)?;
650        let mut raw_sd = ref_model.pack_state_dict()?;
651        raw_sd.insert(
652            "vision_tower.layers.0.proj.weight".to_string(),
653            Tensor::ones((16, 16), DType::F32, &dev)?,
654        );
655        save_dense_checkpoint(src_dir.path(), &cfg, &raw_sd)?;
656
657        let lb_cfg = LittleBitConfig {
658            eff_bit: Some(1.0),
659            use_itq: false,
660            ..Default::default()
661        };
662        let ref_summary = ref_model.apply_littlebit_patch(&lb_cfg, TrainMode::Eval, 29)?;
663
664        let report = stream_quantize_checkpoint(
665            src_dir.path(),
666            dst_dir.path(),
667            &lb_cfg,
668            &StreamingQuantizeOptions {
669                seed: 29,
670                cast_policy: CastPolicy::Preserve,
671                text_only: true,
672                ..Default::default()
673            },
674        )?;
675        assert_eq!(report.blocks_processed, cfg.num_hidden_layers);
676        assert_eq!(
677            report.patch_summary.patched_layers,
678            ref_summary.patched_layers
679        );
680
681        let raw_out = littlebit_core::load_raw_safetensors_dir(dst_dir.path(), &dev)?;
682        assert!(!raw_out.keys().any(|k| k.contains("vision_tower")));
683
684        let reloaded = LittleBitModel::from_pretrained(
685            dst_dir.path().to_str().expect("utf8"),
686            &LoadOptions {
687                inference_mode: InferenceMode::Packed,
688                ..Default::default()
689            },
690        )?;
691        let input = Tensor::from_slice(&[1u32, 5, 8], (1, 3), &dev)?;
692        let diff = ref_model
693            .forward(&input, None, HiddenStatesMode::None)?
694            .logits
695            .sub(
696                &reloaded
697                    .model
698                    .forward(&input, None, HiddenStatesMode::None)?
699                    .logits,
700            )?
701            .abs()?
702            .max_all()?
703            .to_scalar::<f32>()?;
704        assert!(diff < 1e-4, "streaming Phi-3 drift: {diff}");
705        Ok(())
706    }
707}