1use std::collections::{BTreeMap, BTreeSet, HashMap};
4use std::fs::{self, File};
5use std::io::{Read, Seek, SeekFrom};
6use std::path::{Path, PathBuf};
7
8use candle_core::{DType, Device, Tensor};
9use regex::Regex;
10use safetensors::tensor::TensorInfo;
11use serde::{Deserialize, Serialize};
12
13use crate::config::{
14 compute_eff_bits, CastPolicy, LittleBitConfig, LittleBitConfigOverrides, ResidualMode,
15};
16use crate::error::{LittleBitError, Result};
17use crate::packer::binary_unpacker;
18
19#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
21pub enum CheckpointFormat {
22 Packed,
24 LegacyUnpacked,
26}
27
28#[derive(Debug)]
30pub struct LoadedStateDict {
31 pub tensors: HashMap<String, Tensor>,
33 pub format: CheckpointFormat,
35}
36
37#[derive(Debug, Deserialize, Serialize)]
38struct SafetensorsIndex {
39 #[serde(default)]
40 metadata: IndexMetadata,
41 weight_map: BTreeMap<String, String>,
42}
43
44#[derive(Debug, Default, Deserialize, Serialize)]
45struct IndexMetadata {
46 #[serde(default)]
47 total_size: usize,
48}
49
50fn resolve_safetensor_files(dir: &Path) -> Result<Vec<PathBuf>> {
51 let index_path = dir.join("model.safetensors.index.json");
52 if index_path.exists() {
53 let raw = fs::read_to_string(&index_path)?;
54 let index: SafetensorsIndex = serde_json::from_str(&raw)?;
55 let shards: BTreeSet<String> = index.weight_map.into_values().collect();
56 if shards.is_empty() {
57 return Err(LittleBitError::CheckpointNotFound(
58 dir.display().to_string(),
59 ));
60 }
61 return Ok(shards.into_iter().map(|s| dir.join(s)).collect());
62 }
63
64 let single = dir.join("model.safetensors");
65 if single.exists() {
66 return Ok(vec![single]);
67 }
68
69 Err(LittleBitError::CheckpointNotFound(
70 dir.display().to_string(),
71 ))
72}
73
74pub fn load_raw_safetensors_dir(dir: &Path, device: &Device) -> Result<HashMap<String, Tensor>> {
80 let shard_paths = resolve_safetensor_files(dir)?;
81 let mut state_dict = HashMap::new();
82 for path in shard_paths {
83 let loaded = candle_core::safetensors::load(&path, device)?;
84 state_dict.extend(loaded);
85 }
86 Ok(state_dict)
87}
88
89pub fn load_and_unpack_state_dict(
95 dir: &Path,
96 target_dtype: DType,
97 device: &Device,
98) -> Result<LoadedStateDict> {
99 let raw = load_raw_safetensors_dir(dir, &Device::Cpu)?;
100 let has_packed = raw.keys().any(|k| k.ends_with("_packed"));
101
102 if !has_packed {
103 let mut moved = HashMap::with_capacity(raw.len());
104 for (k, v) in raw {
105 let t = if v.device().same_device(device) {
106 v
107 } else {
108 v.to_device(device)?
109 };
110 moved.insert(k, t);
111 }
112 return Ok(LoadedStateDict {
113 tensors: moved,
114 format: CheckpointFormat::LegacyUnpacked,
115 });
116 }
117
118 let re = Regex::new(r"^(.*)\.([^.]+?)_(packed|shape)$").map_err(|err| {
119 LittleBitError::InvalidConfig(format!("failed to compile packed tensor regex: {err}"))
120 })?;
121
122 let mut packed_groups: HashMap<String, HashMap<String, Tensor>> = HashMap::new();
123 let mut final_dict = HashMap::new();
124
125 for (key, value) in raw {
126 if let Some(caps) = re.captures(&key) {
127 let prefix = caps[1].to_string();
128 let param_name = caps[2].to_string();
129 let suffix_type = caps[3].to_string();
130 packed_groups
131 .entry(prefix)
132 .or_default()
133 .insert(format!("{param_name}_{suffix_type}"), value);
134 } else {
135 let t = if value.device().same_device(device) {
136 value
137 } else {
138 value.to_device(device)?
139 };
140 final_dict.insert(key, t);
141 }
142 }
143
144 for (prefix, components) in packed_groups {
145 let param_names: Vec<String> = components
146 .keys()
147 .filter_map(|k| k.strip_suffix("_packed").map(str::to_string))
148 .collect();
149
150 for name in param_names {
151 let packed_t = components
152 .get(&format!("{name}_packed"))
153 .ok_or_else(|| LittleBitError::MissingTensor(format!("{prefix}.{name}_packed")))?;
154 let shape_t = components
155 .get(&format!("{name}_shape"))
156 .ok_or_else(|| LittleBitError::MissingTensor(format!("{prefix}.{name}_shape")))?;
157 let shape_vec = shape_t
158 .to_dtype(DType::I64)?
159 .flatten_all()?
160 .to_vec1::<i64>()?;
161 if shape_vec.len() != 2 || shape_vec[0] < 0 || shape_vec[1] < 0 {
162 return Err(LittleBitError::InvalidShape {
163 expected: "2 non-negative shape elements".to_string(),
164 actual: format!("{shape_vec:?}"),
165 });
166 }
167 let shape = (shape_vec[0] as usize, shape_vec[1] as usize);
168 let unpacked = binary_unpacker(packed_t, shape, target_dtype, device)?;
169 final_dict.insert(format!("{prefix}.{name}"), unpacked);
170 }
171 }
172
173 Ok(LoadedStateDict {
174 tensors: final_dict,
175 format: CheckpointFormat::Packed,
176 })
177}
178
179fn prepare_tensor_for_save(
180 key: &str,
181 tensor: &Tensor,
182 cast_policy: CastPolicy,
183) -> Result<(Tensor, usize)> {
184 let is_metadata_tensor = key.contains("packed")
185 || key.contains("shape")
186 || key.ends_with("._eff_bit_target")
187 || key.ends_with("._split_dim_final")
188 || key.ends_with("._eff_bit_actual");
189
190 let cpu_t = if tensor.device().is_cpu() {
191 tensor.clone()
192 } else {
193 tensor.to_device(&Device::Cpu)?
194 };
195
196 let final_t = match cast_policy {
197 CastPolicy::ToBf16 if !is_metadata_tensor && cpu_t.dtype() == DType::F32 => {
198 cpu_t.to_dtype(DType::BF16)?
199 }
200 _ => cpu_t,
201 };
202 let bytes = final_t.elem_count() * final_t.dtype().size_in_bytes();
203 Ok((final_t, bytes))
204}
205
206fn safetensors_dtype_to_candle(dtype: safetensors::Dtype) -> Result<DType> {
207 match dtype {
208 safetensors::Dtype::U8 => Ok(DType::U8),
209 safetensors::Dtype::U32 => Ok(DType::U32),
210 safetensors::Dtype::I16 => Ok(DType::I16),
211 safetensors::Dtype::I32 => Ok(DType::I32),
212 safetensors::Dtype::I64 => Ok(DType::I64),
213 safetensors::Dtype::BF16 => Ok(DType::BF16),
214 safetensors::Dtype::F16 => Ok(DType::F16),
215 safetensors::Dtype::F32 => Ok(DType::F32),
216 safetensors::Dtype::F64 => Ok(DType::F64),
217 other => Err(LittleBitError::InvalidConfig(format!(
218 "unsupported safetensors dtype {other:?} in streaming reader"
219 ))),
220 }
221}
222
223fn read_safetensors_header(file: &mut File) -> Result<(SafetensorsHeader, u64)> {
224 let mut len_buf = [0u8; 8];
225 file.read_exact(&mut len_buf)?;
226 let header_len = u64::from_le_bytes(len_buf) as usize;
227 if header_len > 100_000_000 {
228 return Err(LittleBitError::InvalidConfig(
229 "safetensors header exceeds 100 MB limit".to_string(),
230 ));
231 }
232
233 let mut header_bytes = vec![0u8; header_len];
234 file.read_exact(&mut header_bytes)?;
235 let header: SafetensorsHeader = serde_json::from_slice(&header_bytes)?;
236 let data_start = 8 + header_len as u64;
237 Ok((header, data_start))
238}
239
240#[derive(Debug, Clone)]
242pub struct TensorLocation {
243 pub shard_path: PathBuf,
245 pub data_start: u64,
247 pub dtype: safetensors::Dtype,
249 pub shape: Vec<usize>,
251 pub data_offsets: (usize, usize),
253}
254
255#[derive(Debug)]
258pub struct SafetensorsReader {
259 locations: BTreeMap<String, TensorLocation>,
260 open_files: HashMap<PathBuf, File>,
261}
262
263impl SafetensorsReader {
264 pub fn open(dir: &Path) -> Result<Self> {
270 let shard_paths = resolve_safetensor_files(dir)?;
271 let mut locations = BTreeMap::new();
272 let mut open_files = HashMap::with_capacity(shard_paths.len());
273
274 for path in shard_paths {
275 let mut file = File::open(&path)?;
276 let (header, data_start) = read_safetensors_header(&mut file)?;
277 for (name, info) in header.tensors {
278 locations.insert(
279 name,
280 TensorLocation {
281 shard_path: path.clone(),
282 data_start,
283 dtype: info.dtype,
284 shape: info.shape,
285 data_offsets: info.data_offsets,
286 },
287 );
288 }
289 open_files.insert(path, file);
290 }
291
292 Ok(Self {
293 locations,
294 open_files,
295 })
296 }
297
298 pub fn tensor_names(&self) -> impl Iterator<Item = &str> {
300 self.locations.keys().map(String::as_str)
301 }
302
303 #[must_use]
305 pub fn contains(&self, name: &str) -> bool {
306 self.locations.contains_key(name)
307 }
308
309 #[must_use]
311 pub fn tensor_info(&self, name: &str) -> Option<&TensorLocation> {
312 self.locations.get(name)
313 }
314
315 #[must_use]
317 pub fn len(&self) -> usize {
318 self.locations.len()
319 }
320
321 #[must_use]
323 pub fn is_empty(&self) -> bool {
324 self.locations.is_empty()
325 }
326
327 fn read_slice(
328 &mut self,
329 shard_path: &Path,
330 abs_offset: u64,
331 byte_len: usize,
332 ) -> Result<Vec<u8>> {
333 if !self.open_files.contains_key(shard_path) {
334 let file = File::open(shard_path)?;
335 self.open_files.insert(shard_path.to_path_buf(), file);
336 }
337 let file = self
338 .open_files
339 .get_mut(shard_path)
340 .ok_or_else(|| LittleBitError::CheckpointNotFound(shard_path.display().to_string()))?;
341 let mut buf = vec![0u8; byte_len];
342 file.seek(SeekFrom::Start(abs_offset))?;
343 file.read_exact(&mut buf)?;
344 Ok(buf)
345 }
346
347 pub fn load_tensor(&mut self, name: &str, device: &Device) -> Result<Tensor> {
352 let loc = self
353 .locations
354 .get(name)
355 .cloned()
356 .ok_or_else(|| LittleBitError::MissingTensor(name.to_string()))?;
357 let candle_dtype = safetensors_dtype_to_candle(loc.dtype)?;
358 let elem_count: usize = loc.shape.iter().product();
359 let expected_bytes = elem_count * candle_dtype.size_in_bytes();
360 let actual_bytes = loc.data_offsets.1.saturating_sub(loc.data_offsets.0);
361 if actual_bytes != expected_bytes {
362 return Err(LittleBitError::InvalidShape {
363 expected: format!("{expected_bytes} payload bytes for `{name}`"),
364 actual: format!("{actual_bytes} payload bytes"),
365 });
366 }
367 let buf = self.read_slice(
368 &loc.shard_path,
369 loc.data_start + loc.data_offsets.0 as u64,
370 actual_bytes,
371 )?;
372 Ok(Tensor::from_raw_buffer(
373 &buf,
374 candle_dtype,
375 &loc.shape,
376 device,
377 )?)
378 }
379
380 pub fn load_tensor_slice_dim0(
386 &mut self,
387 name: &str,
388 index: usize,
389 device: &Device,
390 ) -> Result<Tensor> {
391 let loc = self
392 .locations
393 .get(name)
394 .cloned()
395 .ok_or_else(|| LittleBitError::MissingTensor(name.to_string()))?;
396 if loc.shape.len() < 2 {
397 return Err(LittleBitError::InvalidShape {
398 expected: "tensor with rank >= 2".to_string(),
399 actual: format!("{:?}", loc.shape),
400 });
401 }
402 if index >= loc.shape[0] {
403 return Err(LittleBitError::InvalidShape {
404 expected: format!("dim0 index < {}", loc.shape[0]),
405 actual: format!("{index}"),
406 });
407 }
408 let candle_dtype = safetensors_dtype_to_candle(loc.dtype)?;
409 let slice_shape = &loc.shape[1..];
410 let slice_elems: usize = slice_shape.iter().product();
411 let slice_bytes = slice_elems * candle_dtype.size_in_bytes();
412 let start_rel = loc.data_offsets.0 + index * slice_bytes;
413 if start_rel + slice_bytes > loc.data_offsets.1 {
414 return Err(LittleBitError::InvalidShape {
415 expected: format!("slice ending at byte {}", start_rel + slice_bytes),
416 actual: format!("tensor payload ends at byte {}", loc.data_offsets.1),
417 });
418 }
419 let buf = self.read_slice(
420 &loc.shard_path,
421 loc.data_start + start_rel as u64,
422 slice_bytes,
423 )?;
424 Ok(Tensor::from_raw_buffer(
425 &buf,
426 candle_dtype,
427 slice_shape,
428 device,
429 )?)
430 }
431}
432
433#[derive(Debug)]
436pub struct ShardedSafetensorsWriter {
437 output_dir: PathBuf,
438 max_shard_bytes: usize,
439 cast_policy: CastPolicy,
440 pending: HashMap<String, Tensor>,
441 pending_bytes: usize,
442 completed_shards: Vec<(PathBuf, Vec<String>)>,
443 total_size: usize,
444}
445
446impl ShardedSafetensorsWriter {
447 pub fn new(
452 output_dir: impl AsRef<Path>,
453 max_shard_bytes: usize,
454 cast_policy: CastPolicy,
455 ) -> Result<Self> {
456 if max_shard_bytes == 0 {
457 return Err(LittleBitError::InvalidConfig(
458 "max_shard_bytes must be >= 1".to_string(),
459 ));
460 }
461 let dir = output_dir.as_ref().to_path_buf();
462 fs::create_dir_all(&dir)?;
463 Ok(Self {
464 output_dir: dir,
465 max_shard_bytes,
466 cast_policy,
467 pending: HashMap::new(),
468 pending_bytes: 0,
469 completed_shards: Vec::new(),
470 total_size: 0,
471 })
472 }
473
474 pub fn add_tensor(&mut self, key: String, tensor: Tensor) -> Result<()> {
479 let (final_t, bytes) = prepare_tensor_for_save(&key, &tensor, self.cast_policy)?;
480 if let Some(prev) = self.pending.insert(key, final_t) {
481 let prev_bytes = prev.elem_count() * prev.dtype().size_in_bytes();
482 self.pending_bytes = self.pending_bytes.saturating_sub(prev_bytes);
483 self.total_size = self.total_size.saturating_sub(prev_bytes);
484 }
485 self.pending_bytes += bytes;
486 self.total_size += bytes;
487 Ok(())
488 }
489
490 pub fn flush_if_needed(&mut self) -> Result<()> {
495 if self.pending_bytes >= self.max_shard_bytes {
496 self.flush_shard()?;
497 }
498 Ok(())
499 }
500
501 pub fn flush_shard(&mut self) -> Result<()> {
506 if self.pending.is_empty() {
507 return Ok(());
508 }
509 let shard_idx = self.completed_shards.len();
510 let tmp_path = self
511 .output_dir
512 .join(format!("._tmp_shard_{shard_idx:05}.safetensors"));
513 let keys: Vec<String> = self.pending.keys().cloned().collect();
514 candle_core::safetensors::save(&self.pending, &tmp_path)?;
515 self.pending.clear();
516 self.pending_bytes = 0;
517 self.completed_shards.push((tmp_path, keys));
518 Ok(())
519 }
520
521 pub fn finish(mut self) -> Result<usize> {
527 self.flush_shard()?;
528 let n = self.completed_shards.len();
529 let mut weight_map = BTreeMap::new();
530
531 let shards_written = if n == 0 {
532 let empty: HashMap<String, Tensor> = HashMap::new();
533 candle_core::safetensors::save(&empty, self.output_dir.join("model.safetensors"))?;
534 1
535 } else if n == 1 {
536 let (tmp_path, keys) = self.completed_shards.remove(0);
537 fs::rename(&tmp_path, self.output_dir.join("model.safetensors"))?;
538 for k in keys {
539 weight_map.insert(k, "model.safetensors".to_string());
540 }
541 1
542 } else {
543 for (idx, (tmp_path, keys)) in self.completed_shards.into_iter().enumerate() {
544 let shard_name = format!("model-{:05}-of-{:05}.safetensors", idx + 1, n);
545 fs::rename(&tmp_path, self.output_dir.join(&shard_name))?;
546 for k in keys {
547 weight_map.insert(k, shard_name.clone());
548 }
549 }
550 n
551 };
552
553 let index = SafetensorsIndex {
554 metadata: IndexMetadata {
555 total_size: self.total_size,
556 },
557 weight_map,
558 };
559 let index_json = serde_json::to_string_pretty(&index)?;
560 fs::write(
561 self.output_dir.join("model.safetensors.index.json"),
562 index_json,
563 )?;
564 Ok(shards_written)
565 }
566}
567
568pub fn save_packed_safetensors(
575 dir: &Path,
576 state_dict: &HashMap<String, Tensor>,
577 cast_policy: CastPolicy,
578) -> Result<()> {
579 fs::create_dir_all(dir)?;
580
581 let mut prepared: HashMap<String, Tensor> = HashMap::with_capacity(state_dict.len());
582 let mut weight_map = BTreeMap::new();
583 let mut total_size = 0usize;
584
585 for (key, tensor) in state_dict {
586 let (final_t, bytes) = prepare_tensor_for_save(key, tensor, cast_policy)?;
587 total_size += bytes;
588 weight_map.insert(key.clone(), "model.safetensors".to_string());
589 prepared.insert(key.clone(), final_t);
590 }
591
592 let safetensors_path = dir.join("model.safetensors");
593 candle_core::safetensors::save(&prepared, &safetensors_path)?;
594
595 let index = SafetensorsIndex {
596 metadata: IndexMetadata { total_size },
597 weight_map,
598 };
599 let index_json = serde_json::to_string_pretty(&index)?;
600 fs::write(dir.join("model.safetensors.index.json"), index_json)?;
601 Ok(())
602}
603
604#[derive(Debug, Clone, Serialize, Deserialize)]
606pub struct LayerInspection {
607 pub name: String,
609 pub in_features: usize,
611 pub out_features: usize,
613 pub split_dim: usize,
615 pub has_residual: bool,
617 pub eff_bit_actual: f64,
619 pub serialized_bytes: usize,
621 pub dense_fp16_bytes: usize,
623}
624
625#[derive(Debug, Clone, Serialize, Deserialize)]
627pub struct CheckpointStats {
628 pub config: LittleBitConfig,
630 pub is_packed: bool,
632 pub tensor_count: usize,
634 pub quantized_layer_count: usize,
636 pub total_serialized_bytes: usize,
638 pub quantized_layers_bytes: usize,
640 pub dense_fp16_bytes: usize,
642 pub avg_eff_bits: f64,
644 pub compression_ratio_vs_fp16: f64,
646 pub layers: Vec<LayerInspection>,
648}
649
650fn read_header_and_slice(
651 file: &mut File,
652 data_start: u64,
653 offsets: (usize, usize),
654) -> Result<Vec<u8>> {
655 let len = offsets.1.saturating_sub(offsets.0);
656 let mut buf = vec![0u8; len];
657 file.seek(SeekFrom::Start(data_start + offsets.0 as u64))?;
658 file.read_exact(&mut buf)?;
659 Ok(buf)
660}
661
662fn decode_shape_tensor_bytes(bytes: &[u8], dtype: safetensors::Dtype) -> Option<(usize, usize)> {
663 match dtype {
664 safetensors::Dtype::I64 if bytes.len() == 16 => {
665 let r = i64::from_le_bytes(bytes[0..8].try_into().ok()?);
666 let c = i64::from_le_bytes(bytes[8..16].try_into().ok()?);
667 if r >= 0 && c >= 0 {
668 Some((r as usize, c as usize))
669 } else {
670 None
671 }
672 }
673 safetensors::Dtype::I32 if bytes.len() == 8 => {
674 let r = i32::from_le_bytes(bytes[0..4].try_into().ok()?);
675 let c = i32::from_le_bytes(bytes[4..8].try_into().ok()?);
676 if r >= 0 && c >= 0 {
677 Some((r as usize, c as usize))
678 } else {
679 None
680 }
681 }
682 _ => None,
683 }
684}
685
686#[derive(Debug, Deserialize)]
687struct SafetensorsHeader {
688 #[serde(default, rename = "__metadata__")]
689 _metadata: Option<HashMap<String, String>>,
690 #[serde(flatten)]
691 tensors: HashMap<String, TensorInfo>,
692}
693
694pub fn inspect_checkpoint(dir: &Path) -> Result<CheckpointStats> {
699 let config = LittleBitConfig::load_with_priority(dir, &LittleBitConfigOverrides::default())?;
700 let shard_paths = resolve_safetensor_files(dir)?;
701
702 let mut tensor_shapes: HashMap<String, (Vec<usize>, usize)> = HashMap::new();
703 let mut decoded_shapes: HashMap<String, (usize, usize)> = HashMap::new();
704 let mut is_packed = false;
705 let mut total_serialized_bytes = 0usize;
706
707 for path in shard_paths {
708 let mut file = File::open(&path)?;
709 let (header, data_start) = read_safetensors_header(&mut file)?;
710
711 for (name, info) in header.tensors {
712 let byte_len = info.data_offsets.1.saturating_sub(info.data_offsets.0);
713 total_serialized_bytes += byte_len;
714 if name.ends_with("_packed") {
715 is_packed = true;
716 }
717 if name.ends_with(".U_shape") || name.ends_with(".V_shape") {
718 let raw = read_header_and_slice(&mut file, data_start, info.data_offsets)?;
719 if let Some(dim) = decode_shape_tensor_bytes(&raw, info.dtype) {
720 decoded_shapes.insert(name.clone(), dim);
721 }
722 }
723 tensor_shapes.insert(name, (info.shape, byte_len));
724 }
725 }
726
727 let mut layer_prefixes: BTreeSet<String> = BTreeSet::new();
728 for key in tensor_shapes.keys() {
729 if let Some(prefix) = key
730 .strip_suffix(".U_packed")
731 .or_else(|| key.strip_suffix(".U"))
732 {
733 layer_prefixes.insert(prefix.to_string());
734 }
735 }
736
737 let mut layers = Vec::with_capacity(layer_prefixes.len());
738 let mut quantized_layers_bytes = 0usize;
739 let mut dense_fp16_bytes = 0usize;
740 let mut total_bits = 0.0_f64;
741 let mut total_weights = 0usize;
742
743 for prefix in layer_prefixes {
744 let u_shape = decoded_shapes
745 .get(&format!("{prefix}.U_shape"))
746 .copied()
747 .or_else(|| {
748 let (dims, _) = tensor_shapes.get(&format!("{prefix}.U"))?;
749 (dims.len() == 2).then_some((dims[0], dims[1]))
750 });
751 let v_shape = decoded_shapes
752 .get(&format!("{prefix}.V_shape"))
753 .copied()
754 .or_else(|| {
755 let (dims, _) = tensor_shapes.get(&format!("{prefix}.V"))?;
756 (dims.len() == 2).then_some((dims[0], dims[1]))
757 });
758
759 let (Some((out_features, split_dim)), Some((_, in_features))) = (u_shape, v_shape) else {
760 continue;
761 };
762
763 let has_residual = tensor_shapes.contains_key(&format!("{prefix}.U_R_packed"))
764 || tensor_shapes.contains_key(&format!("{prefix}.U_R"));
765 let residual_mode = ResidualMode::from(has_residual);
766 let eff_bit_actual = compute_eff_bits(in_features, out_features, split_dim, residual_mode);
767
768 let prefix_dot = format!("{prefix}.");
769 let layer_bytes: usize = tensor_shapes
770 .iter()
771 .filter(|(k, _)| k.starts_with(&prefix_dot))
772 .map(|(_, (_, bytes))| *bytes)
773 .sum();
774
775 let fp16_bytes = out_features * in_features * 2;
776 quantized_layers_bytes += layer_bytes;
777 dense_fp16_bytes += fp16_bytes;
778 total_bits += eff_bit_actual * (out_features * in_features) as f64;
779 total_weights += out_features * in_features;
780
781 layers.push(LayerInspection {
782 name: prefix,
783 in_features,
784 out_features,
785 split_dim,
786 has_residual,
787 eff_bit_actual,
788 serialized_bytes: layer_bytes,
789 dense_fp16_bytes: fp16_bytes,
790 });
791 }
792
793 let avg_eff_bits = if total_weights > 0 {
794 total_bits / total_weights as f64
795 } else {
796 0.0
797 };
798 let compression_ratio_vs_fp16 = if quantized_layers_bytes > 0 {
799 dense_fp16_bytes as f64 / quantized_layers_bytes as f64
800 } else {
801 0.0
802 };
803
804 Ok(CheckpointStats {
805 config,
806 is_packed,
807 tensor_count: tensor_shapes.len(),
808 quantized_layer_count: layers.len(),
809 total_serialized_bytes,
810 quantized_layers_bytes,
811 dense_fp16_bytes,
812 avg_eff_bits,
813 compression_ratio_vs_fp16,
814 layers,
815 })
816}
817
818#[cfg(test)]
819mod tests {
820 use super::*;
821 use crate::config::TrainMode;
822 use crate::linear::LittleBitLinear;
823 use tempfile::TempDir;
824
825 #[test]
826 fn packed_save_unpack_load_and_inspect_round_trip() -> Result<()> {
827 let dir = TempDir::new().expect("tempdir");
828 let dev = Device::Cpu;
829 let w = Tensor::randn(0.0f32, 0.5, (64, 96), &dev)?;
830 let cfg = LittleBitConfig {
831 eff_bit: Some(0.5),
832 residual: true,
833 use_itq: true,
834 itq_n_iter: 15,
835 ..Default::default()
836 };
837 let layer = LittleBitLinear::from_dense_weight(&w, None, &cfg, 1.0, TrainMode::Eval, 5)?;
838 let state = layer.pack_state_dict("model.layers.0.self_attn.q_proj")?;
839
840 fs::write(
841 dir.path().join("littlebit_config.json"),
842 serde_json::to_string_pretty(&cfg)?,
843 )?;
844 save_packed_safetensors(dir.path(), &state, CastPolicy::ToBf16)?;
845
846 let stats = inspect_checkpoint(dir.path())?;
847 assert!(stats.is_packed);
848 assert_eq!(stats.quantized_layer_count, 1);
849 assert_eq!(stats.layers[0].in_features, 96);
850 assert_eq!(stats.layers[0].out_features, 64);
851 assert!(stats.layers[0].has_residual);
852
853 let loaded = load_and_unpack_state_dict(dir.path(), DType::F32, &dev)?;
854 assert_eq!(loaded.format, CheckpointFormat::Packed);
855 assert!(loaded
856 .tensors
857 .contains_key("model.layers.0.self_attn.q_proj.U"));
858 assert!(loaded
859 .tensors
860 .contains_key("model.layers.0.self_attn.q_proj.V_R"));
861 assert_eq!(
862 loaded
863 .tensors
864 .get("model.layers.0.self_attn.q_proj.U")
865 .expect("U present")
866 .dtype(),
867 DType::F32
868 );
869 assert_eq!(
870 loaded
871 .tensors
872 .get("model.layers.0.self_attn.q_proj.u1")
873 .expect("u1 present")
874 .dtype(),
875 DType::BF16
876 );
877 Ok(())
878 }
879
880 #[test]
881 fn loads_sharded_safetensors_via_index_json() -> Result<()> {
882 let dir = TempDir::new().expect("tempdir");
883 let dev = Device::Cpu;
884
885 let mut s1 = HashMap::new();
886 s1.insert(
887 "a.weight".to_string(),
888 Tensor::ones((4, 4), DType::F32, &dev)?,
889 );
890 candle_core::safetensors::save(&s1, dir.path().join("shard-00001.safetensors"))?;
891
892 let mut s2 = HashMap::new();
893 s2.insert(
894 "b.weight".to_string(),
895 Tensor::zeros((2, 3), DType::F32, &dev)?,
896 );
897 candle_core::safetensors::save(&s2, dir.path().join("shard-00002.safetensors"))?;
898
899 let index = serde_json::json!({
900 "metadata": { "total_size": 88 },
901 "weight_map": {
902 "a.weight": "shard-00001.safetensors",
903 "b.weight": "shard-00002.safetensors"
904 }
905 });
906 fs::write(
907 dir.path().join("model.safetensors.index.json"),
908 serde_json::to_string_pretty(&index)?,
909 )?;
910
911 let loaded = load_and_unpack_state_dict(dir.path(), DType::F32, &dev)?;
912 assert_eq!(loaded.format, CheckpointFormat::LegacyUnpacked);
913 assert_eq!(loaded.tensors.len(), 2);
914 assert_eq!(loaded.tensors["a.weight"].dims(), &[4, 4]);
915 assert_eq!(loaded.tensors["b.weight"].dims(), &[2, 3]);
916 Ok(())
917 }
918
919 #[test]
920 fn streaming_reader_loads_single_sharded_and_3d_slices_exact() -> Result<()> {
921 let dir = TempDir::new().expect("tempdir");
922 let dev = Device::Cpu;
923
924 let t3d = Tensor::randn(0.0f32, 1.0, (4, 6, 8), &dev)?.to_dtype(DType::BF16)?;
925 let t_f32 = Tensor::randn(0.0f32, 0.5, (5, 7), &dev)?;
926 let t_f16 = Tensor::randn(0.0f32, 0.5, (3, 4), &dev)?.to_dtype(DType::F16)?;
927 let t_i32 = Tensor::from_slice(&[1i32, -2, 3, 4], (2, 2), &dev)?;
928 let t_i64 = Tensor::from_slice(&[10i64, 20], (2,), &dev)?;
929
930 let mut s1 = HashMap::new();
931 s1.insert("experts.gate_up_proj".to_string(), t3d.clone());
932 s1.insert("w.f32".to_string(), t_f32.clone());
933 candle_core::safetensors::save(&s1, dir.path().join("model-00001-of-00002.safetensors"))?;
934
935 let mut s2 = HashMap::new();
936 s2.insert("w.f16".to_string(), t_f16.clone());
937 s2.insert("idx.i32".to_string(), t_i32.clone());
938 s2.insert("shape.i64".to_string(), t_i64.clone());
939 candle_core::safetensors::save(&s2, dir.path().join("model-00002-of-00002.safetensors"))?;
940
941 let index = serde_json::json!({
942 "metadata": { "total_size": 0 },
943 "weight_map": {
944 "experts.gate_up_proj": "model-00001-of-00002.safetensors",
945 "w.f32": "model-00001-of-00002.safetensors",
946 "w.f16": "model-00002-of-00002.safetensors",
947 "idx.i32": "model-00002-of-00002.safetensors",
948 "shape.i64": "model-00002-of-00002.safetensors"
949 }
950 });
951 fs::write(
952 dir.path().join("model.safetensors.index.json"),
953 serde_json::to_string_pretty(&index)?,
954 )?;
955
956 let mut reader = SafetensorsReader::open(dir.path())?;
957 assert_eq!(reader.len(), 5);
958 assert!(reader.contains("experts.gate_up_proj"));
959
960 let baseline = load_raw_safetensors_dir(dir.path(), &dev)?;
961 for name in ["w.f32", "w.f16", "idx.i32", "shape.i64"] {
962 let got = reader.load_tensor(name, &dev)?;
963 let exp = &baseline[name];
964 assert_eq!(got.dtype(), exp.dtype());
965 assert_eq!(got.dims(), exp.dims());
966 let diff = got
967 .to_dtype(DType::F32)?
968 .sub(&exp.to_dtype(DType::F32)?)?
969 .abs()?
970 .max_all()?
971 .to_scalar::<f32>()?;
972 assert_eq!(diff, 0.0, "tensor `{name}` mismatch");
973 }
974
975 for e in 0..4 {
976 let slice = reader.load_tensor_slice_dim0("experts.gate_up_proj", e, &dev)?;
977 let expected = t3d.get(e)?;
978 assert_eq!(slice.dims(), &[6, 8]);
979 assert_eq!(slice.dtype(), DType::BF16);
980 let diff = slice
981 .to_dtype(DType::F32)?
982 .sub(&expected.to_dtype(DType::F32)?)?
983 .abs()?
984 .max_all()?
985 .to_scalar::<f32>()?;
986 assert_eq!(diff, 0.0, "3D slice {e} mismatch");
987 }
988 Ok(())
989 }
990
991 #[test]
992 fn sharded_writer_splits_shards_and_roundtrips_with_inspect() -> Result<()> {
993 let dir = TempDir::new().expect("tempdir");
994 let dev = Device::Cpu;
995
996 let cfg = LittleBitConfig {
997 eff_bit: Some(0.5),
998 residual: false,
999 use_itq: true,
1000 itq_n_iter: 5,
1001 ..Default::default()
1002 };
1003 fs::write(
1004 dir.path().join("littlebit_config.json"),
1005 serde_json::to_string_pretty(&cfg)?,
1006 )?;
1007
1008 let mut writer = ShardedSafetensorsWriter::new(dir.path(), 512, CastPolicy::ToBf16)?;
1010 for idx in 0..2 {
1011 let w = Tensor::randn(0.0f32, 0.5, (64, 64), &dev)?;
1012 let layer = LittleBitLinear::from_dense_weight(
1013 &w,
1014 None,
1015 &cfg,
1016 1.0,
1017 TrainMode::Eval,
1018 idx as u64,
1019 )?;
1020 let sd = layer.pack_state_dict(&format!("model.layers.{idx}.self_attn.q_proj"))?;
1021 for (k, v) in sd {
1022 writer.add_tensor(k, v)?;
1023 }
1024 writer.flush_if_needed()?;
1025 }
1026 let shards_written = writer.finish()?;
1027 assert!(
1028 shards_written >= 2,
1029 "expected multiple shards, got {shards_written}"
1030 );
1031 assert!(dir.path().join("model.safetensors.index.json").is_file());
1032 assert!(dir
1033 .path()
1034 .join(format!("model-00001-of-{shards_written:05}.safetensors"))
1035 .is_file());
1036
1037 let stats = inspect_checkpoint(dir.path())?;
1038 assert!(stats.is_packed);
1039 assert_eq!(stats.quantized_layer_count, 2);
1040
1041 let unpacked = load_and_unpack_state_dict(dir.path(), DType::F32, &dev)?;
1042 assert_eq!(unpacked.format, CheckpointFormat::Packed);
1043 assert!(unpacked
1044 .tensors
1045 .contains_key("model.layers.0.self_attn.q_proj.U"));
1046 assert!(unpacked
1047 .tensors
1048 .contains_key("model.layers.1.self_attn.q_proj.U"));
1049 Ok(())
1050 }
1051}