1use std::collections::{BTreeMap, HashSet};
9use std::path::Path;
10
11use candle_core::{DType, Device, IndexOp, Tensor};
12use regex::Regex;
13use tracing::info;
14
15use littlebit_core::{
16 CastPolicy, LittleBitConfig, LittleBitLinear, SafetensorsReader, ShardedSafetensorsWriter,
17 TrainMode,
18};
19
20use crate::config::{ModelArchitecture, ModelConfig};
21use crate::error::{ModelError, Result};
22use crate::hub::write_quantized_metadata;
23use crate::patch::{derive_layer_seed, PatchPolicy, PatchSummary};
24
25const DEFAULT_MAX_SHARD_BYTES: usize = 2 * 1024 * 1024 * 1024;
26
27#[derive(Debug, Clone)]
29pub struct StreamingQuantizeOptions {
30 pub seed: u64,
32 pub cast_policy: CastPolicy,
34 pub max_shard_bytes: usize,
36 pub device: Device,
38 pub dtype: DType,
40 pub text_only: bool,
42 pub base_model_id: Option<String>,
44}
45
46impl Default for StreamingQuantizeOptions {
47 fn default() -> Self {
48 Self {
49 seed: 42,
50 cast_policy: CastPolicy::ToBf16,
51 max_shard_bytes: DEFAULT_MAX_SHARD_BYTES,
52 device: Device::Cpu,
53 dtype: DType::F32,
54 text_only: false,
55 base_model_id: None,
56 }
57 }
58}
59
60#[derive(Debug, Clone)]
62pub struct StreamingQuantizeReport {
63 pub patch_summary: PatchSummary,
65 pub shards_written: usize,
67 pub blocks_processed: usize,
69 pub passthrough_tensors: usize,
71}
72
73fn is_multimodal_tensor(name: &str) -> bool {
74 [
75 "vision_tower",
76 "audio_tower",
77 "multi_modal_projector",
78 "embed_vision",
79 "embed_audio",
80 ]
81 .iter()
82 .any(|pat| name.contains(pat))
83}
84
85fn quantize_and_write_linear(
86 module_name: &str,
87 weight: &Tensor,
88 bias: Option<&Tensor>,
89 lb_cfg: &LittleBitConfig,
90 policy: &PatchPolicy,
91 seed: u64,
92 writer: &mut ShardedSafetensorsWriter,
93 summary: &mut PatchSummary,
94) -> Result<()> {
95 let effective_cfg = lb_cfg.effective_for_module(module_name)?;
96 let ratio = policy.ratio_factor(module_name, &effective_cfg);
97 let layer_seed = derive_layer_seed(seed, module_name);
98 let lb = LittleBitLinear::from_dense_weight(
99 weight,
100 bias,
101 &effective_cfg,
102 ratio,
103 TrainMode::Eval,
104 layer_seed,
105 )?;
106 summary.record_budget(lb.budget());
107 let packed_sd = lb.pack_state_dict(module_name)?;
108 for (k, v) in packed_sd {
109 writer.add_tensor(k, v)?;
110 }
111 Ok(())
112}
113
114#[allow(clippy::too_many_lines)]
124pub fn stream_quantize_checkpoint(
125 src_dir: &Path,
126 dst_dir: &Path,
127 lb_cfg: &LittleBitConfig,
128 opts: &StreamingQuantizeOptions,
129) -> Result<StreamingQuantizeReport> {
130 lb_cfg.validate()?;
131 let model_cfg = ModelConfig::from_file(&src_dir.join("config.json"))?;
132 let arch = model_cfg.detect_architecture()?;
133 let mut reader = SafetensorsReader::open(src_dir)?;
134 let mut writer =
135 ShardedSafetensorsWriter::new(dst_dir, opts.max_shard_bytes, opts.cast_policy)?;
136 let policy = PatchPolicy::new();
137 let mut summary = PatchSummary::default();
138 let mut passthrough_tensors = 0usize;
139
140 let layer_re = Regex::new(r"(?:^|\.)layers\.(\d+)\.").map_err(|err| {
141 ModelError::InvalidArgument(format!("failed to compile layer index regex: {err}"))
142 })?;
143
144 let all_names: Vec<String> = reader.tensor_names().map(str::to_string).collect();
145 let mut blocks: BTreeMap<usize, Vec<String>> = BTreeMap::new();
146 let mut global_keys: Vec<String> = Vec::new();
147
148 for name in all_names {
149 let multimodal = is_multimodal_tensor(&name);
150 if opts.text_only && multimodal {
151 continue;
152 }
153 if !multimodal {
154 if let Some(caps) = layer_re.captures(&name) {
155 if let Ok(idx) = caps[1].parse::<usize>() {
156 blocks.entry(idx).or_default().push(name);
157 continue;
158 }
159 }
160 }
161 global_keys.push(name);
162 }
163
164 let global_set: HashSet<&str> = global_keys.iter().map(String::as_str).collect();
166 let mut global_consumed: HashSet<String> = HashSet::new();
167
168 for key in &global_keys {
169 if global_consumed.contains(key) {
170 continue;
171 }
172 if arch == ModelArchitecture::Opt
173 && (key == "model.decoder.project_in.weight"
174 || key == "model.decoder.project_out.weight")
175 {
176 let module_prefix = key.trim_end_matches(".weight");
177 let bias_key = format!("{module_prefix}.bias");
178 let w = reader
179 .load_tensor(key, &opts.device)?
180 .to_dtype(opts.dtype)?;
181 let b = if global_set.contains(bias_key.as_str()) {
182 Some(
183 reader
184 .load_tensor(&bias_key, &opts.device)?
185 .to_dtype(opts.dtype)?,
186 )
187 } else {
188 None
189 };
190 quantize_and_write_linear(
191 module_prefix,
192 &w,
193 b.as_ref(),
194 lb_cfg,
195 &policy,
196 opts.seed,
197 &mut writer,
198 &mut summary,
199 )?;
200 global_consumed.insert(key.clone());
201 if b.is_some() {
202 global_consumed.insert(bias_key);
203 }
204 writer.flush_if_needed()?;
205 continue;
206 }
207
208 if let Some(prefix) = key.strip_suffix(".weight") {
209 if policy.is_excluded(prefix) {
210 summary.record_skipped();
211 }
212 }
213 let tensor = reader.load_tensor(key, &Device::Cpu)?;
214 writer.add_tensor(key.clone(), tensor)?;
215 passthrough_tensors += 1;
216 global_consumed.insert(key.clone());
217 writer.flush_if_needed()?;
218 }
219
220 let mut blocks_processed = 0usize;
222 let total_blocks = blocks.len();
223
224 for (block_idx, keys) in blocks {
225 let key_set: HashSet<&str> = keys.iter().map(String::as_str).collect();
226 let mut consumed: HashSet<String> = HashSet::new();
227
228 if arch == ModelArchitecture::Phi3 {
230 for key in &keys {
231 if let Some(module_prefix) = key.strip_suffix(".weight") {
232 if let Some(attn_prefix) = module_prefix.strip_suffix(".qkv_proj") {
233 let bias_key = format!("{module_prefix}.bias");
234 let w = reader
235 .load_tensor(key, &opts.device)?
236 .to_dtype(opts.dtype)?;
237 let b = if key_set.contains(bias_key.as_str()) {
238 Some(
239 reader
240 .load_tensor(&bias_key, &opts.device)?
241 .to_dtype(opts.dtype)?,
242 )
243 } else {
244 None
245 };
246
247 let head_dim = model_cfg.effective_head_dim();
248 let q_dim = model_cfg.num_attention_heads * head_dim;
249 let kv_dim = model_cfg.effective_kv_heads() * head_dim;
250
251 let qw = w.i(0..q_dim)?.contiguous()?;
252 let kw = w.i(q_dim..q_dim + kv_dim)?.contiguous()?;
253 let vw = w.i(q_dim + kv_dim..q_dim + 2 * kv_dim)?.contiguous()?;
254
255 let (qb, kb, vb) = if let Some(ref bt) = b {
256 (
257 Some(bt.i(0..q_dim)?.contiguous()?),
258 Some(bt.i(q_dim..q_dim + kv_dim)?.contiguous()?),
259 Some(bt.i(q_dim + kv_dim..q_dim + 2 * kv_dim)?.contiguous()?),
260 )
261 } else {
262 (None, None, None)
263 };
264 drop(w);
265 drop(b);
266
267 for (suffix, slice_w, slice_b) in
268 [("q_proj", qw, qb), ("k_proj", kw, kb), ("v_proj", vw, vb)]
269 {
270 let split_name = format!("{attn_prefix}.{suffix}");
271 quantize_and_write_linear(
272 &split_name,
273 &slice_w,
274 slice_b.as_ref(),
275 lb_cfg,
276 &policy,
277 opts.seed,
278 &mut writer,
279 &mut summary,
280 )?;
281 drop(slice_w);
282 drop(slice_b);
283 writer.flush_if_needed()?;
284 }
285
286 consumed.insert(key.clone());
287 if key_set.contains(bias_key.as_str()) {
288 consumed.insert(bias_key);
289 }
290 }
291 }
292 }
293 }
294
295 for key in &keys {
297 if consumed.contains(key) {
298 continue;
299 }
300 let is_fused_expert =
301 key.ends_with(".experts.gate_up_proj") || key.ends_with(".experts.down_proj");
302 if !is_fused_expert {
303 continue;
304 }
305 let Some(loc) = reader.tensor_info(key) else {
306 continue;
307 };
308 if loc.shape.len() != 3 {
309 continue;
310 }
311 let num_experts = loc.shape[0];
312 let Some((experts_prefix, proj_name)) = key.rsplit_once('.') else {
313 continue;
314 };
315 let experts_prefix = experts_prefix.to_string();
316 let proj_name = proj_name.to_string();
317
318 for e in 0..num_experts {
319 let w_2d = reader
320 .load_tensor_slice_dim0(key, e, &opts.device)?
321 .to_dtype(opts.dtype)?;
322 let mod_name = format!("{experts_prefix}.{e}.{proj_name}");
323 quantize_and_write_linear(
324 &mod_name,
325 &w_2d,
326 None,
327 lb_cfg,
328 &policy,
329 opts.seed,
330 &mut writer,
331 &mut summary,
332 )?;
333 drop(w_2d);
334 writer.flush_if_needed()?;
335 }
336 consumed.insert(key.clone());
337 }
338
339 for key in &keys {
341 if consumed.contains(key) {
342 continue;
343 }
344 let Some(module_prefix) = key.strip_suffix(".weight") else {
345 continue;
346 };
347 let Some(loc) = reader.tensor_info(key) else {
348 continue;
349 };
350 if loc.shape.len() != 2 {
351 continue;
352 }
353
354 let bias_key = format!("{module_prefix}.bias");
355 if policy.is_excluded(module_prefix) {
356 summary.record_skipped();
357 let w = reader.load_tensor(key, &Device::Cpu)?;
358 writer.add_tensor(key.clone(), w)?;
359 passthrough_tensors += 1;
360 consumed.insert(key.clone());
361 if key_set.contains(bias_key.as_str()) {
362 let b = reader.load_tensor(&bias_key, &Device::Cpu)?;
363 writer.add_tensor(bias_key.clone(), b)?;
364 passthrough_tensors += 1;
365 consumed.insert(bias_key);
366 }
367 writer.flush_if_needed()?;
368 continue;
369 }
370
371 let w = reader
372 .load_tensor(key, &opts.device)?
373 .to_dtype(opts.dtype)?;
374 let b = if key_set.contains(bias_key.as_str()) {
375 Some(
376 reader
377 .load_tensor(&bias_key, &opts.device)?
378 .to_dtype(opts.dtype)?,
379 )
380 } else {
381 None
382 };
383 quantize_and_write_linear(
384 module_prefix,
385 &w,
386 b.as_ref(),
387 lb_cfg,
388 &policy,
389 opts.seed,
390 &mut writer,
391 &mut summary,
392 )?;
393 drop(w);
394 drop(b);
395 consumed.insert(key.clone());
396 if key_set.contains(bias_key.as_str()) {
397 consumed.insert(bias_key);
398 }
399 writer.flush_if_needed()?;
400 }
401
402 for key in &keys {
404 if consumed.contains(key) {
405 continue;
406 }
407 let tensor = reader.load_tensor(key, &Device::Cpu)?;
408 writer.add_tensor(key.clone(), tensor)?;
409 passthrough_tensors += 1;
410 writer.flush_if_needed()?;
411 }
412
413 blocks_processed += 1;
414 info!(
415 block = block_idx + 1,
416 total = total_blocks,
417 patched = summary.patched_layers,
418 avg_eff_bits = summary.avg_eff_bits,
419 "stream-quantized transformer block"
420 );
421 }
422
423 let shards_written = writer.finish()?;
424 write_quantized_metadata(
425 dst_dir,
426 &model_cfg,
427 lb_cfg,
428 opts.base_model_id.as_deref(),
429 Some(src_dir),
430 )?;
431
432 Ok(StreamingQuantizeReport {
433 patch_summary: summary,
434 shards_written,
435 blocks_processed,
436 passthrough_tensors,
437 })
438}
439
440#[cfg(test)]
441mod tests {
442 use super::*;
443 use crate::hub::{LittleBitModel, LoadOptions};
444 use crate::model::{CausalLm, HiddenStatesMode};
445 use littlebit_core::{inspect_checkpoint, InferenceMode, QuantFunc};
446 use std::collections::HashMap;
447 use std::fs;
448 use tempfile::TempDir;
449
450 fn save_dense_checkpoint(
451 dir: &Path,
452 cfg: &ModelConfig,
453 state_dict: &HashMap<String, Tensor>,
454 ) -> Result<()> {
455 fs::create_dir_all(dir)?;
456 fs::write(dir.join("config.json"), serde_json::to_string_pretty(cfg)?)?;
457 candle_core::safetensors::save(state_dict, dir.join("model.safetensors"))?;
458 Ok(())
459 }
460
461 #[test]
462 fn streaming_quantize_gemma4_dense_matches_in_memory_and_reloads_packed() -> Result<()> {
463 let dev = Device::Cpu;
464 let cfg = ModelConfig::tiny_gemma4_dense(96, 64, 2);
465 let src_dir = TempDir::new().expect("src");
466 let dst_dir = TempDir::new().expect("dst");
467
468 let mut ref_model = CausalLm::new_random(cfg.clone(), DType::F32, &dev, 51)?;
469 let dense_sd = ref_model.pack_state_dict()?;
470 save_dense_checkpoint(src_dir.path(), &cfg, &dense_sd)?;
471
472 let lb_cfg = LittleBitConfig {
473 quant_func: QuantFunc::SmoothSign { alpha: 100.0 },
474 eff_bit: Some(0.75),
475 residual: true,
476 kv_factor: 1.5,
477 min_split_dim: 8,
478 use_itq: true,
479 itq_n_iter: 10,
480 ..Default::default()
481 };
482
483 let ref_summary = ref_model.apply_littlebit_patch(&lb_cfg, TrainMode::Eval, 101)?;
484 let report = stream_quantize_checkpoint(
485 src_dir.path(),
486 dst_dir.path(),
487 &lb_cfg,
488 &StreamingQuantizeOptions {
489 seed: 101,
490 cast_policy: CastPolicy::Preserve,
491 max_shard_bytes: 16 * 1024,
492 ..Default::default()
493 },
494 )?;
495
496 assert_eq!(report.blocks_processed, cfg.num_hidden_layers);
497 assert_eq!(
498 report.patch_summary.patched_layers,
499 ref_summary.patched_layers
500 );
501 assert!(report.shards_written >= 2, "expected sharded output");
502
503 let input = Tensor::from_slice(&[1u32, 4, 7, 15], (1, 4), &dev)?;
504 let ref_logits = ref_model
505 .forward(&input, None, HiddenStatesMode::None)?
506 .logits;
507
508 for mode in [InferenceMode::Unpacked, InferenceMode::Packed] {
509 let reloaded = LittleBitModel::from_pretrained(
510 dst_dir.path().to_str().expect("utf8"),
511 &LoadOptions {
512 inference_mode: mode,
513 ..Default::default()
514 },
515 )?;
516 let got_logits = reloaded
517 .model
518 .forward(&input, None, HiddenStatesMode::None)?
519 .logits;
520 let diff = ref_logits
521 .sub(&got_logits)?
522 .abs()?
523 .max_all()?
524 .to_scalar::<f32>()?;
525 assert!(
526 diff < 1e-4,
527 "streaming vs in-memory Gemma 4 Dense drift ({mode:?}): {diff}"
528 );
529 }
530 Ok(())
531 }
532
533 #[test]
534 fn streaming_quantize_gemma4_moe_from_fused_3d_experts_multi_shard() -> Result<()> {
535 let dev = Device::Cpu;
536 let cfg = ModelConfig::tiny_gemma4_moe(80, 32, 2, 6, 2);
537 let src_dir = TempDir::new().expect("src");
538 let dst_dir = TempDir::new().expect("dst");
539
540 let mut ref_model = CausalLm::new_random(cfg.clone(), DType::F32, &dev, 88)?;
541 let mut hf_sd = ref_model.pack_state_dict()?;
542
543 for l in 0..cfg.num_hidden_layers {
545 let mut gu_slices = Vec::new();
546 let mut dn_slices = Vec::new();
547 for e in 0..6 {
548 gu_slices.push(
549 hf_sd
550 .remove(&format!("model.layers.{l}.experts.{e}.gate_up_proj.weight"))
551 .expect("gu"),
552 );
553 dn_slices.push(
554 hf_sd
555 .remove(&format!("model.layers.{l}.experts.{e}.down_proj.weight"))
556 .expect("dn"),
557 );
558 }
559 hf_sd.insert(
560 format!("model.layers.{l}.experts.gate_up_proj"),
561 Tensor::stack(&gu_slices, 0)?,
562 );
563 hf_sd.insert(
564 format!("model.layers.{l}.experts.down_proj"),
565 Tensor::stack(&dn_slices, 0)?,
566 );
567 }
568 save_dense_checkpoint(src_dir.path(), &cfg, &hf_sd)?;
569
570 let lb_cfg = LittleBitConfig {
571 eff_bit: Some(0.8),
572 residual: false,
573 use_itq: true,
574 itq_n_iter: 6,
575 ..Default::default()
576 };
577 let ref_summary = ref_model.apply_littlebit_patch(&lb_cfg, TrainMode::Eval, 55)?;
578
579 let report = stream_quantize_checkpoint(
580 src_dir.path(),
581 dst_dir.path(),
582 &lb_cfg,
583 &StreamingQuantizeOptions {
584 seed: 55,
585 cast_policy: CastPolicy::Preserve,
586 max_shard_bytes: 8 * 1024,
587 ..Default::default()
588 },
589 )?;
590 assert_eq!(
591 report.patch_summary.patched_layers,
592 ref_summary.patched_layers
593 );
594 assert_eq!(
595 report.patch_summary.skipped_layers,
596 ref_summary.skipped_layers
597 );
598 assert!(report.shards_written >= 2);
599
600 let stats = inspect_checkpoint(dst_dir.path())?;
601 assert!(stats.is_packed);
602 assert_eq!(stats.quantized_layer_count, ref_summary.patched_layers);
603
604 let loaded = LittleBitModel::from_pretrained(
605 dst_dir.path().to_str().expect("utf8"),
606 &LoadOptions {
607 inference_mode: InferenceMode::Packed,
608 ..Default::default()
609 },
610 )?;
611 if let CausalLm::Gemma4(ref g4) = loaded.model {
612 let moe = g4.layer_moe(0).expect("moe 0");
613 assert!(matches!(
614 moe.router.proj,
615 crate::layer::LinearLayer::Dense { .. }
616 ));
617 assert!(matches!(
618 moe.experts.experts[0].gate_up_proj,
619 crate::layer::LinearLayer::Packed(_)
620 ));
621 } else {
622 panic!("expected Gemma4");
623 }
624
625 let input = Tensor::from_slice(&[2u32, 9, 13], (1, 3), &dev)?;
626 let ref_logits = ref_model
627 .forward(&input, None, HiddenStatesMode::None)?
628 .logits;
629 let got_logits = loaded
630 .model
631 .forward(&input, None, HiddenStatesMode::None)?
632 .logits;
633 let diff = ref_logits
634 .sub(&got_logits)?
635 .abs()?
636 .max_all()?
637 .to_scalar::<f32>()?;
638 assert!(diff < 1e-4, "streaming MoE vs in-memory drift: {diff}");
639 Ok(())
640 }
641
642 #[test]
643 fn streaming_quantize_phi3_and_text_only_stripping() -> Result<()> {
644 let dev = Device::Cpu;
645 let cfg = ModelConfig::tiny_phi3(64, 32, 2);
646 let src_dir = TempDir::new().expect("src");
647 let dst_dir = TempDir::new().expect("dst");
648
649 let mut ref_model = CausalLm::new_random(cfg.clone(), DType::F32, &dev, 17)?;
650 let mut raw_sd = ref_model.pack_state_dict()?;
651 raw_sd.insert(
652 "vision_tower.layers.0.proj.weight".to_string(),
653 Tensor::ones((16, 16), DType::F32, &dev)?,
654 );
655 save_dense_checkpoint(src_dir.path(), &cfg, &raw_sd)?;
656
657 let lb_cfg = LittleBitConfig {
658 eff_bit: Some(1.0),
659 use_itq: false,
660 ..Default::default()
661 };
662 let ref_summary = ref_model.apply_littlebit_patch(&lb_cfg, TrainMode::Eval, 29)?;
663
664 let report = stream_quantize_checkpoint(
665 src_dir.path(),
666 dst_dir.path(),
667 &lb_cfg,
668 &StreamingQuantizeOptions {
669 seed: 29,
670 cast_policy: CastPolicy::Preserve,
671 text_only: true,
672 ..Default::default()
673 },
674 )?;
675 assert_eq!(report.blocks_processed, cfg.num_hidden_layers);
676 assert_eq!(
677 report.patch_summary.patched_layers,
678 ref_summary.patched_layers
679 );
680
681 let raw_out = littlebit_core::load_raw_safetensors_dir(dst_dir.path(), &dev)?;
682 assert!(!raw_out.keys().any(|k| k.contains("vision_tower")));
683
684 let reloaded = LittleBitModel::from_pretrained(
685 dst_dir.path().to_str().expect("utf8"),
686 &LoadOptions {
687 inference_mode: InferenceMode::Packed,
688 ..Default::default()
689 },
690 )?;
691 let input = Tensor::from_slice(&[1u32, 5, 8], (1, 3), &dev)?;
692 let diff = ref_model
693 .forward(&input, None, HiddenStatesMode::None)?
694 .logits
695 .sub(
696 &reloaded
697 .model
698 .forward(&input, None, HiddenStatesMode::None)?
699 .logits,
700 )?
701 .abs()?
702 .max_all()?
703 .to_scalar::<f32>()?;
704 assert!(diff < 1e-4, "streaming Phi-3 drift: {diff}");
705 Ok(())
706 }
707}