Skip to main content

littlebit_core/
safetensors_io.rs

1//! Safetensors checkpoint loading, binary unpacking, packed saving, and header inspection.
2
3use std::collections::{BTreeMap, BTreeSet, HashMap};
4use std::fs::{self, File};
5use std::io::{Read, Seek, SeekFrom};
6use std::path::{Path, PathBuf};
7
8use candle_core::{DType, Device, Tensor};
9use regex::Regex;
10use safetensors::tensor::TensorInfo;
11use serde::{Deserialize, Serialize};
12
13use crate::config::{
14    compute_eff_bits, CastPolicy, LittleBitConfig, LittleBitConfigOverrides, ResidualMode,
15};
16use crate::error::{LittleBitError, Result};
17use crate::packer::binary_unpacker;
18
19/// Detected checkpoint storage format.
20#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
21pub enum CheckpointFormat {
22    /// Binary factors are stored bit-packed as `*_packed` (`I32`) and `*_shape` (`I64`).
23    Packed,
24    /// Binary factors are stored unpacked as float `U` / `V` tensors (legacy format).
25    LegacyUnpacked,
26}
27
28/// Unpacked checkpoint tensors alongside detected format metadata.
29#[derive(Debug)]
30pub struct LoadedStateDict {
31    /// Tensors loaded onto the requested device, with `*_packed`/`*_shape` unpacked to `U`/`V`.
32    pub tensors: HashMap<String, Tensor>,
33    /// Original format detected on disk (`Packed` vs. `LegacyUnpacked`).
34    pub format: CheckpointFormat,
35}
36
37#[derive(Debug, Deserialize, Serialize)]
38struct SafetensorsIndex {
39    #[serde(default)]
40    metadata: IndexMetadata,
41    weight_map: BTreeMap<String, String>,
42}
43
44#[derive(Debug, Default, Deserialize, Serialize)]
45struct IndexMetadata {
46    #[serde(default)]
47    total_size: usize,
48}
49
50fn resolve_safetensor_files(dir: &Path) -> Result<Vec<PathBuf>> {
51    let index_path = dir.join("model.safetensors.index.json");
52    if index_path.exists() {
53        let raw = fs::read_to_string(&index_path)?;
54        let index: SafetensorsIndex = serde_json::from_str(&raw)?;
55        let shards: BTreeSet<String> = index.weight_map.into_values().collect();
56        if shards.is_empty() {
57            return Err(LittleBitError::CheckpointNotFound(
58                dir.display().to_string(),
59            ));
60        }
61        return Ok(shards.into_iter().map(|s| dir.join(s)).collect());
62    }
63
64    let single = dir.join("model.safetensors");
65    if single.exists() {
66        return Ok(vec![single]);
67    }
68
69    Err(LittleBitError::CheckpointNotFound(
70        dir.display().to_string(),
71    ))
72}
73
74/// Load all raw safetensors tensors from a checkpoint directory (single file or sharded index).
75///
76/// # Errors
77/// Returns [`LittleBitError::CheckpointNotFound`] when neither `model.safetensors` nor
78/// `model.safetensors.index.json` exists, or I/O/safetensors decoding errors.
79pub fn load_raw_safetensors_dir(dir: &Path, device: &Device) -> Result<HashMap<String, Tensor>> {
80    let shard_paths = resolve_safetensor_files(dir)?;
81    let mut state_dict = HashMap::new();
82    for path in shard_paths {
83        let loaded = candle_core::safetensors::load(&path, device)?;
84        state_dict.extend(loaded);
85    }
86    Ok(state_dict)
87}
88
89/// Load a checkpoint directory and automatically unpack `*_packed`/`*_shape` tensors into
90/// `target_dtype` float tensors matching upstream `_load_and_process_state_dict`.
91///
92/// # Errors
93/// Returns an error if files are missing, corrupted, or packed shapes are malformed.
94pub fn load_and_unpack_state_dict(
95    dir: &Path,
96    target_dtype: DType,
97    device: &Device,
98) -> Result<LoadedStateDict> {
99    let raw = load_raw_safetensors_dir(dir, &Device::Cpu)?;
100    let has_packed = raw.keys().any(|k| k.ends_with("_packed"));
101
102    if !has_packed {
103        let mut moved = HashMap::with_capacity(raw.len());
104        for (k, v) in raw {
105            let t = if v.device().same_device(device) {
106                v
107            } else {
108                v.to_device(device)?
109            };
110            moved.insert(k, t);
111        }
112        return Ok(LoadedStateDict {
113            tensors: moved,
114            format: CheckpointFormat::LegacyUnpacked,
115        });
116    }
117
118    let re = Regex::new(r"^(.*)\.([^.]+?)_(packed|shape)$").map_err(|err| {
119        LittleBitError::InvalidConfig(format!("failed to compile packed tensor regex: {err}"))
120    })?;
121
122    let mut packed_groups: HashMap<String, HashMap<String, Tensor>> = HashMap::new();
123    let mut final_dict = HashMap::new();
124
125    for (key, value) in raw {
126        if let Some(caps) = re.captures(&key) {
127            let prefix = caps[1].to_string();
128            let param_name = caps[2].to_string();
129            let suffix_type = caps[3].to_string();
130            packed_groups
131                .entry(prefix)
132                .or_default()
133                .insert(format!("{param_name}_{suffix_type}"), value);
134        } else {
135            let t = if value.device().same_device(device) {
136                value
137            } else {
138                value.to_device(device)?
139            };
140            final_dict.insert(key, t);
141        }
142    }
143
144    for (prefix, components) in packed_groups {
145        let param_names: Vec<String> = components
146            .keys()
147            .filter_map(|k| k.strip_suffix("_packed").map(str::to_string))
148            .collect();
149
150        for name in param_names {
151            let packed_t = components
152                .get(&format!("{name}_packed"))
153                .ok_or_else(|| LittleBitError::MissingTensor(format!("{prefix}.{name}_packed")))?;
154            let shape_t = components
155                .get(&format!("{name}_shape"))
156                .ok_or_else(|| LittleBitError::MissingTensor(format!("{prefix}.{name}_shape")))?;
157            let shape_vec = shape_t
158                .to_dtype(DType::I64)?
159                .flatten_all()?
160                .to_vec1::<i64>()?;
161            if shape_vec.len() != 2 || shape_vec[0] < 0 || shape_vec[1] < 0 {
162                return Err(LittleBitError::InvalidShape {
163                    expected: "2 non-negative shape elements".to_string(),
164                    actual: format!("{shape_vec:?}"),
165                });
166            }
167            let shape = (shape_vec[0] as usize, shape_vec[1] as usize);
168            let unpacked = binary_unpacker(packed_t, shape, target_dtype, device)?;
169            final_dict.insert(format!("{prefix}.{name}"), unpacked);
170        }
171    }
172
173    Ok(LoadedStateDict {
174        tensors: final_dict,
175        format: CheckpointFormat::Packed,
176    })
177}
178
179fn prepare_tensor_for_save(
180    key: &str,
181    tensor: &Tensor,
182    cast_policy: CastPolicy,
183) -> Result<(Tensor, usize)> {
184    let is_metadata_tensor = key.contains("packed")
185        || key.contains("shape")
186        || key.ends_with("._eff_bit_target")
187        || key.ends_with("._split_dim_final")
188        || key.ends_with("._eff_bit_actual");
189
190    let cpu_t = if tensor.device().is_cpu() {
191        tensor.clone()
192    } else {
193        tensor.to_device(&Device::Cpu)?
194    };
195
196    let final_t = match cast_policy {
197        CastPolicy::ToBf16 if !is_metadata_tensor && cpu_t.dtype() == DType::F32 => {
198            cpu_t.to_dtype(DType::BF16)?
199        }
200        _ => cpu_t,
201    };
202    let bytes = final_t.elem_count() * final_t.dtype().size_in_bytes();
203    Ok((final_t, bytes))
204}
205
206fn safetensors_dtype_to_candle(dtype: safetensors::Dtype) -> Result<DType> {
207    match dtype {
208        safetensors::Dtype::U8 => Ok(DType::U8),
209        safetensors::Dtype::U32 => Ok(DType::U32),
210        safetensors::Dtype::I16 => Ok(DType::I16),
211        safetensors::Dtype::I32 => Ok(DType::I32),
212        safetensors::Dtype::I64 => Ok(DType::I64),
213        safetensors::Dtype::BF16 => Ok(DType::BF16),
214        safetensors::Dtype::F16 => Ok(DType::F16),
215        safetensors::Dtype::F32 => Ok(DType::F32),
216        safetensors::Dtype::F64 => Ok(DType::F64),
217        other => Err(LittleBitError::InvalidConfig(format!(
218            "unsupported safetensors dtype {other:?} in streaming reader"
219        ))),
220    }
221}
222
223fn read_safetensors_header(file: &mut File) -> Result<(SafetensorsHeader, u64)> {
224    let mut len_buf = [0u8; 8];
225    file.read_exact(&mut len_buf)?;
226    let header_len = u64::from_le_bytes(len_buf) as usize;
227    if header_len > 100_000_000 {
228        return Err(LittleBitError::InvalidConfig(
229            "safetensors header exceeds 100 MB limit".to_string(),
230        ));
231    }
232
233    let mut header_bytes = vec![0u8; header_len];
234    file.read_exact(&mut header_bytes)?;
235    let header: SafetensorsHeader = serde_json::from_slice(&header_bytes)?;
236    let data_start = 8 + header_len as u64;
237    Ok((header, data_start))
238}
239
240/// Location metadata for a single tensor inside a safetensors shard on disk.
241#[derive(Debug, Clone)]
242pub struct TensorLocation {
243    /// Path to the `.safetensors` file holding this tensor.
244    pub shard_path: PathBuf,
245    /// Byte offset where the binary payload section starts (`8 + header_len`).
246    pub data_start: u64,
247    /// Safetensors element data type.
248    pub dtype: safetensors::Dtype,
249    /// Tensor dimensions.
250    pub shape: Vec<usize>,
251    /// Byte range `(start, end)` relative to `data_start`.
252    pub data_offsets: (usize, usize),
253}
254
255/// Safe streaming safetensors reader that parses headers once and seeks to load
256/// individual tensors or 3D dimension-0 slices on demand using zero `unsafe` code.
257#[derive(Debug)]
258pub struct SafetensorsReader {
259    locations: BTreeMap<String, TensorLocation>,
260    open_files: HashMap<PathBuf, File>,
261}
262
263impl SafetensorsReader {
264    /// Open a local single-file or sharded safetensors checkpoint directory.
265    ///
266    /// # Errors
267    /// Returns [`LittleBitError::CheckpointNotFound`] if no safetensors checkpoint exists in `dir`,
268    /// or I/O/header parsing errors.
269    pub fn open(dir: &Path) -> Result<Self> {
270        let shard_paths = resolve_safetensor_files(dir)?;
271        let mut locations = BTreeMap::new();
272        let mut open_files = HashMap::with_capacity(shard_paths.len());
273
274        for path in shard_paths {
275            let mut file = File::open(&path)?;
276            let (header, data_start) = read_safetensors_header(&mut file)?;
277            for (name, info) in header.tensors {
278                locations.insert(
279                    name,
280                    TensorLocation {
281                        shard_path: path.clone(),
282                        data_start,
283                        dtype: info.dtype,
284                        shape: info.shape,
285                        data_offsets: info.data_offsets,
286                    },
287                );
288            }
289            open_files.insert(path, file);
290        }
291
292        Ok(Self {
293            locations,
294            open_files,
295        })
296    }
297
298    /// Iterate over all tensor keys across all shards in deterministic lexicographical order.
299    pub fn tensor_names(&self) -> impl Iterator<Item = &str> {
300        self.locations.keys().map(String::as_str)
301    }
302
303    /// Return true if `name` is present in any shard.
304    #[must_use]
305    pub fn contains(&self, name: &str) -> bool {
306        self.locations.contains_key(name)
307    }
308
309    /// Return location & shape metadata for `name` without reading tensor bytes from disk.
310    #[must_use]
311    pub fn tensor_info(&self, name: &str) -> Option<&TensorLocation> {
312        self.locations.get(name)
313    }
314
315    /// Total number of tensors indexed across all shards.
316    #[must_use]
317    pub fn len(&self) -> usize {
318        self.locations.len()
319    }
320
321    /// Return true if zero tensors were indexed.
322    #[must_use]
323    pub fn is_empty(&self) -> bool {
324        self.locations.is_empty()
325    }
326
327    fn read_slice(
328        &mut self,
329        shard_path: &Path,
330        abs_offset: u64,
331        byte_len: usize,
332    ) -> Result<Vec<u8>> {
333        if !self.open_files.contains_key(shard_path) {
334            let file = File::open(shard_path)?;
335            self.open_files.insert(shard_path.to_path_buf(), file);
336        }
337        let file = self
338            .open_files
339            .get_mut(shard_path)
340            .ok_or_else(|| LittleBitError::CheckpointNotFound(shard_path.display().to_string()))?;
341        let mut buf = vec![0u8; byte_len];
342        file.seek(SeekFrom::Start(abs_offset))?;
343        file.read_exact(&mut buf)?;
344        Ok(buf)
345    }
346
347    /// Read and materialize a single tensor `name` onto `device`.
348    ///
349    /// # Errors
350    /// Returns [`LittleBitError::MissingTensor`] if absent or I/O/decoding errors.
351    pub fn load_tensor(&mut self, name: &str, device: &Device) -> Result<Tensor> {
352        let loc = self
353            .locations
354            .get(name)
355            .cloned()
356            .ok_or_else(|| LittleBitError::MissingTensor(name.to_string()))?;
357        let candle_dtype = safetensors_dtype_to_candle(loc.dtype)?;
358        let elem_count: usize = loc.shape.iter().product();
359        let expected_bytes = elem_count * candle_dtype.size_in_bytes();
360        let actual_bytes = loc.data_offsets.1.saturating_sub(loc.data_offsets.0);
361        if actual_bytes != expected_bytes {
362            return Err(LittleBitError::InvalidShape {
363                expected: format!("{expected_bytes} payload bytes for `{name}`"),
364                actual: format!("{actual_bytes} payload bytes"),
365            });
366        }
367        let buf = self.read_slice(
368            &loc.shard_path,
369            loc.data_start + loc.data_offsets.0 as u64,
370            actual_bytes,
371        )?;
372        Ok(Tensor::from_raw_buffer(
373            &buf,
374            candle_dtype,
375            &loc.shape,
376            device,
377        )?)
378    }
379
380    /// Stream only `tensor[index]` along dimension 0 without reading the rest of the tensor
381    /// into memory (enabling single-expert loading from fused 3D MoE weights).
382    ///
383    /// # Errors
384    /// Returns an error if `name` is missing, rank `< 2`, `index` is out of bounds, or reading fails.
385    pub fn load_tensor_slice_dim0(
386        &mut self,
387        name: &str,
388        index: usize,
389        device: &Device,
390    ) -> Result<Tensor> {
391        let loc = self
392            .locations
393            .get(name)
394            .cloned()
395            .ok_or_else(|| LittleBitError::MissingTensor(name.to_string()))?;
396        if loc.shape.len() < 2 {
397            return Err(LittleBitError::InvalidShape {
398                expected: "tensor with rank >= 2".to_string(),
399                actual: format!("{:?}", loc.shape),
400            });
401        }
402        if index >= loc.shape[0] {
403            return Err(LittleBitError::InvalidShape {
404                expected: format!("dim0 index < {}", loc.shape[0]),
405                actual: format!("{index}"),
406            });
407        }
408        let candle_dtype = safetensors_dtype_to_candle(loc.dtype)?;
409        let slice_shape = &loc.shape[1..];
410        let slice_elems: usize = slice_shape.iter().product();
411        let slice_bytes = slice_elems * candle_dtype.size_in_bytes();
412        let start_rel = loc.data_offsets.0 + index * slice_bytes;
413        if start_rel + slice_bytes > loc.data_offsets.1 {
414            return Err(LittleBitError::InvalidShape {
415                expected: format!("slice ending at byte {}", start_rel + slice_bytes),
416                actual: format!("tensor payload ends at byte {}", loc.data_offsets.1),
417            });
418        }
419        let buf = self.read_slice(
420            &loc.shard_path,
421            loc.data_start + start_rel as u64,
422            slice_bytes,
423        )?;
424        Ok(Tensor::from_raw_buffer(
425            &buf,
426            candle_dtype,
427            slice_shape,
428            device,
429        )?)
430    }
431}
432
433/// Incremental sharded safetensors writer that flushes completed shards to disk
434/// and frees their in-memory buffers as quantization progresses.
435#[derive(Debug)]
436pub struct ShardedSafetensorsWriter {
437    output_dir: PathBuf,
438    max_shard_bytes: usize,
439    cast_policy: CastPolicy,
440    pending: HashMap<String, Tensor>,
441    pending_bytes: usize,
442    completed_shards: Vec<(PathBuf, Vec<String>)>,
443    total_size: usize,
444}
445
446impl ShardedSafetensorsWriter {
447    /// Initialize a streaming sharded safetensors writer under `output_dir`.
448    ///
449    /// # Errors
450    /// Returns [`LittleBitError::InvalidConfig`] if `max_shard_bytes == 0` or creating `output_dir` fails.
451    pub fn new(
452        output_dir: impl AsRef<Path>,
453        max_shard_bytes: usize,
454        cast_policy: CastPolicy,
455    ) -> Result<Self> {
456        if max_shard_bytes == 0 {
457            return Err(LittleBitError::InvalidConfig(
458                "max_shard_bytes must be >= 1".to_string(),
459            ));
460        }
461        let dir = output_dir.as_ref().to_path_buf();
462        fs::create_dir_all(&dir)?;
463        Ok(Self {
464            output_dir: dir,
465            max_shard_bytes,
466            cast_policy,
467            pending: HashMap::new(),
468            pending_bytes: 0,
469            completed_shards: Vec::new(),
470            total_size: 0,
471        })
472    }
473
474    /// Stage a tensor for writing, applying [`CastPolicy`] and recording its serialized byte size.
475    ///
476    /// # Errors
477    /// Returns an error if moving or casting the tensor fails.
478    pub fn add_tensor(&mut self, key: String, tensor: Tensor) -> Result<()> {
479        let (final_t, bytes) = prepare_tensor_for_save(&key, &tensor, self.cast_policy)?;
480        if let Some(prev) = self.pending.insert(key, final_t) {
481            let prev_bytes = prev.elem_count() * prev.dtype().size_in_bytes();
482            self.pending_bytes = self.pending_bytes.saturating_sub(prev_bytes);
483            self.total_size = self.total_size.saturating_sub(prev_bytes);
484        }
485        self.pending_bytes += bytes;
486        self.total_size += bytes;
487        Ok(())
488    }
489
490    /// Flush the current pending shard to disk if `pending_bytes >= max_shard_bytes`.
491    ///
492    /// # Errors
493    /// Returns an error if writing the shard fails.
494    pub fn flush_if_needed(&mut self) -> Result<()> {
495        if self.pending_bytes >= self.max_shard_bytes {
496            self.flush_shard()?;
497        }
498        Ok(())
499    }
500
501    /// Force-flush any buffered tensors into a numbered temporary shard and clear memory.
502    ///
503    /// # Errors
504    /// Returns an error if writing the shard file fails.
505    pub fn flush_shard(&mut self) -> Result<()> {
506        if self.pending.is_empty() {
507            return Ok(());
508        }
509        let shard_idx = self.completed_shards.len();
510        let tmp_path = self
511            .output_dir
512            .join(format!("._tmp_shard_{shard_idx:05}.safetensors"));
513        let keys: Vec<String> = self.pending.keys().cloned().collect();
514        candle_core::safetensors::save(&self.pending, &tmp_path)?;
515        self.pending.clear();
516        self.pending_bytes = 0;
517        self.completed_shards.push((tmp_path, keys));
518        Ok(())
519    }
520
521    /// Finalize shard filenames (`model.safetensors` or `model-00001-of-NNNNN.safetensors`),
522    /// write `model.safetensors.index.json`, and return the number of output shards written.
523    ///
524    /// # Errors
525    /// Returns an error if flushing, renaming, or writing `model.safetensors.index.json` fails.
526    pub fn finish(mut self) -> Result<usize> {
527        self.flush_shard()?;
528        let n = self.completed_shards.len();
529        let mut weight_map = BTreeMap::new();
530
531        let shards_written = if n == 0 {
532            let empty: HashMap<String, Tensor> = HashMap::new();
533            candle_core::safetensors::save(&empty, self.output_dir.join("model.safetensors"))?;
534            1
535        } else if n == 1 {
536            let (tmp_path, keys) = self.completed_shards.remove(0);
537            fs::rename(&tmp_path, self.output_dir.join("model.safetensors"))?;
538            for k in keys {
539                weight_map.insert(k, "model.safetensors".to_string());
540            }
541            1
542        } else {
543            for (idx, (tmp_path, keys)) in self.completed_shards.into_iter().enumerate() {
544                let shard_name = format!("model-{:05}-of-{:05}.safetensors", idx + 1, n);
545                fs::rename(&tmp_path, self.output_dir.join(&shard_name))?;
546                for k in keys {
547                    weight_map.insert(k, shard_name.clone());
548                }
549            }
550            n
551        };
552
553        let index = SafetensorsIndex {
554            metadata: IndexMetadata {
555                total_size: self.total_size,
556            },
557            weight_map,
558        };
559        let index_json = serde_json::to_string_pretty(&index)?;
560        fs::write(
561            self.output_dir.join("model.safetensors.index.json"),
562            index_json,
563        )?;
564        Ok(shards_written)
565    }
566}
567
568/// Save a state dictionary to `dir/model.safetensors` and write `dir/model.safetensors.index.json`.
569///
570/// Matches upstream `main.py` / `hub.py` checkpoint layout.
571///
572/// # Errors
573/// Returns an error if creating `dir`, converting tensors, or writing safetensors/index fails.
574pub fn save_packed_safetensors(
575    dir: &Path,
576    state_dict: &HashMap<String, Tensor>,
577    cast_policy: CastPolicy,
578) -> Result<()> {
579    fs::create_dir_all(dir)?;
580
581    let mut prepared: HashMap<String, Tensor> = HashMap::with_capacity(state_dict.len());
582    let mut weight_map = BTreeMap::new();
583    let mut total_size = 0usize;
584
585    for (key, tensor) in state_dict {
586        let (final_t, bytes) = prepare_tensor_for_save(key, tensor, cast_policy)?;
587        total_size += bytes;
588        weight_map.insert(key.clone(), "model.safetensors".to_string());
589        prepared.insert(key.clone(), final_t);
590    }
591
592    let safetensors_path = dir.join("model.safetensors");
593    candle_core::safetensors::save(&prepared, &safetensors_path)?;
594
595    let index = SafetensorsIndex {
596        metadata: IndexMetadata { total_size },
597        weight_map,
598    };
599    let index_json = serde_json::to_string_pretty(&index)?;
600    fs::write(dir.join("model.safetensors.index.json"), index_json)?;
601    Ok(())
602}
603
604/// Per-layer metadata extracted from a LittleBit checkpoint.
605#[derive(Debug, Clone, Serialize, Deserialize)]
606pub struct LayerInspection {
607    /// Module path prefix (e.g., `model.layers.0.self_attn.q_proj`).
608    pub name: String,
609    /// Input feature dimension.
610    pub in_features: usize,
611    /// Output feature dimension.
612    pub out_features: usize,
613    /// Latent rank `split_dim`.
614    pub split_dim: usize,
615    /// Whether this layer contains a residual factorization branch (`U_R`, `V_R`).
616    pub has_residual: bool,
617    /// Actual effective bits per weight.
618    pub eff_bit_actual: f64,
619    /// Total serialized bytes for this layer in the checkpoint.
620    pub serialized_bytes: usize,
621    /// Equivalent dense FP16 weight bytes (`out_features * in_features * 2`).
622    pub dense_fp16_bytes: usize,
623}
624
625/// Summary statistics for a LittleBit checkpoint directory.
626#[derive(Debug, Clone, Serialize, Deserialize)]
627pub struct CheckpointStats {
628    /// Resolved quantization config.
629    pub config: LittleBitConfig,
630    /// Whether any tensor uses `*_packed` binary packing.
631    pub is_packed: bool,
632    /// Number of tensors in checkpoint.
633    pub tensor_count: usize,
634    /// Number of factorized `LittleBitLinear` layers found.
635    pub quantized_layer_count: usize,
636    /// Total serialized tensor payload bytes across all shards.
637    pub total_serialized_bytes: usize,
638    /// Total serialized bytes across all quantized layers.
639    pub quantized_layers_bytes: usize,
640    /// Equivalent dense FP16 bytes across quantized layers.
641    pub dense_fp16_bytes: usize,
642    /// Weighted average effective bits per weight across all quantized layers.
643    pub avg_eff_bits: f64,
644    /// Compression ratio (`dense_fp16_bytes / quantized_layers_bytes`).
645    pub compression_ratio_vs_fp16: f64,
646    /// Sorted per-layer inspection entries.
647    pub layers: Vec<LayerInspection>,
648}
649
650fn read_header_and_slice(
651    file: &mut File,
652    data_start: u64,
653    offsets: (usize, usize),
654) -> Result<Vec<u8>> {
655    let len = offsets.1.saturating_sub(offsets.0);
656    let mut buf = vec![0u8; len];
657    file.seek(SeekFrom::Start(data_start + offsets.0 as u64))?;
658    file.read_exact(&mut buf)?;
659    Ok(buf)
660}
661
662fn decode_shape_tensor_bytes(bytes: &[u8], dtype: safetensors::Dtype) -> Option<(usize, usize)> {
663    match dtype {
664        safetensors::Dtype::I64 if bytes.len() == 16 => {
665            let r = i64::from_le_bytes(bytes[0..8].try_into().ok()?);
666            let c = i64::from_le_bytes(bytes[8..16].try_into().ok()?);
667            if r >= 0 && c >= 0 {
668                Some((r as usize, c as usize))
669            } else {
670                None
671            }
672        }
673        safetensors::Dtype::I32 if bytes.len() == 8 => {
674            let r = i32::from_le_bytes(bytes[0..4].try_into().ok()?);
675            let c = i32::from_le_bytes(bytes[4..8].try_into().ok()?);
676            if r >= 0 && c >= 0 {
677                Some((r as usize, c as usize))
678            } else {
679                None
680            }
681        }
682        _ => None,
683    }
684}
685
686#[derive(Debug, Deserialize)]
687struct SafetensorsHeader {
688    #[serde(default, rename = "__metadata__")]
689    _metadata: Option<HashMap<String, String>>,
690    #[serde(flatten)]
691    tensors: HashMap<String, TensorInfo>,
692}
693
694/// Inspect a local LittleBit checkpoint without loading full weight tensors into memory.
695///
696/// # Errors
697/// Returns an error if the directory has no safetensors weights or headers are corrupted.
698pub fn inspect_checkpoint(dir: &Path) -> Result<CheckpointStats> {
699    let config = LittleBitConfig::load_with_priority(dir, &LittleBitConfigOverrides::default())?;
700    let shard_paths = resolve_safetensor_files(dir)?;
701
702    let mut tensor_shapes: HashMap<String, (Vec<usize>, usize)> = HashMap::new();
703    let mut decoded_shapes: HashMap<String, (usize, usize)> = HashMap::new();
704    let mut is_packed = false;
705    let mut total_serialized_bytes = 0usize;
706
707    for path in shard_paths {
708        let mut file = File::open(&path)?;
709        let (header, data_start) = read_safetensors_header(&mut file)?;
710
711        for (name, info) in header.tensors {
712            let byte_len = info.data_offsets.1.saturating_sub(info.data_offsets.0);
713            total_serialized_bytes += byte_len;
714            if name.ends_with("_packed") {
715                is_packed = true;
716            }
717            if name.ends_with(".U_shape") || name.ends_with(".V_shape") {
718                let raw = read_header_and_slice(&mut file, data_start, info.data_offsets)?;
719                if let Some(dim) = decode_shape_tensor_bytes(&raw, info.dtype) {
720                    decoded_shapes.insert(name.clone(), dim);
721                }
722            }
723            tensor_shapes.insert(name, (info.shape, byte_len));
724        }
725    }
726
727    let mut layer_prefixes: BTreeSet<String> = BTreeSet::new();
728    for key in tensor_shapes.keys() {
729        if let Some(prefix) = key
730            .strip_suffix(".U_packed")
731            .or_else(|| key.strip_suffix(".U"))
732        {
733            layer_prefixes.insert(prefix.to_string());
734        }
735    }
736
737    let mut layers = Vec::with_capacity(layer_prefixes.len());
738    let mut quantized_layers_bytes = 0usize;
739    let mut dense_fp16_bytes = 0usize;
740    let mut total_bits = 0.0_f64;
741    let mut total_weights = 0usize;
742
743    for prefix in layer_prefixes {
744        let u_shape = decoded_shapes
745            .get(&format!("{prefix}.U_shape"))
746            .copied()
747            .or_else(|| {
748                let (dims, _) = tensor_shapes.get(&format!("{prefix}.U"))?;
749                (dims.len() == 2).then_some((dims[0], dims[1]))
750            });
751        let v_shape = decoded_shapes
752            .get(&format!("{prefix}.V_shape"))
753            .copied()
754            .or_else(|| {
755                let (dims, _) = tensor_shapes.get(&format!("{prefix}.V"))?;
756                (dims.len() == 2).then_some((dims[0], dims[1]))
757            });
758
759        let (Some((out_features, split_dim)), Some((_, in_features))) = (u_shape, v_shape) else {
760            continue;
761        };
762
763        let has_residual = tensor_shapes.contains_key(&format!("{prefix}.U_R_packed"))
764            || tensor_shapes.contains_key(&format!("{prefix}.U_R"));
765        let residual_mode = ResidualMode::from(has_residual);
766        let eff_bit_actual = compute_eff_bits(in_features, out_features, split_dim, residual_mode);
767
768        let prefix_dot = format!("{prefix}.");
769        let layer_bytes: usize = tensor_shapes
770            .iter()
771            .filter(|(k, _)| k.starts_with(&prefix_dot))
772            .map(|(_, (_, bytes))| *bytes)
773            .sum();
774
775        let fp16_bytes = out_features * in_features * 2;
776        quantized_layers_bytes += layer_bytes;
777        dense_fp16_bytes += fp16_bytes;
778        total_bits += eff_bit_actual * (out_features * in_features) as f64;
779        total_weights += out_features * in_features;
780
781        layers.push(LayerInspection {
782            name: prefix,
783            in_features,
784            out_features,
785            split_dim,
786            has_residual,
787            eff_bit_actual,
788            serialized_bytes: layer_bytes,
789            dense_fp16_bytes: fp16_bytes,
790        });
791    }
792
793    let avg_eff_bits = if total_weights > 0 {
794        total_bits / total_weights as f64
795    } else {
796        0.0
797    };
798    let compression_ratio_vs_fp16 = if quantized_layers_bytes > 0 {
799        dense_fp16_bytes as f64 / quantized_layers_bytes as f64
800    } else {
801        0.0
802    };
803
804    Ok(CheckpointStats {
805        config,
806        is_packed,
807        tensor_count: tensor_shapes.len(),
808        quantized_layer_count: layers.len(),
809        total_serialized_bytes,
810        quantized_layers_bytes,
811        dense_fp16_bytes,
812        avg_eff_bits,
813        compression_ratio_vs_fp16,
814        layers,
815    })
816}
817
818#[cfg(test)]
819mod tests {
820    use super::*;
821    use crate::config::TrainMode;
822    use crate::linear::LittleBitLinear;
823    use tempfile::TempDir;
824
825    #[test]
826    fn packed_save_unpack_load_and_inspect_round_trip() -> Result<()> {
827        let dir = TempDir::new().expect("tempdir");
828        let dev = Device::Cpu;
829        let w = Tensor::randn(0.0f32, 0.5, (64, 96), &dev)?;
830        let cfg = LittleBitConfig {
831            eff_bit: Some(0.5),
832            residual: true,
833            use_itq: true,
834            itq_n_iter: 15,
835            ..Default::default()
836        };
837        let layer = LittleBitLinear::from_dense_weight(&w, None, &cfg, 1.0, TrainMode::Eval, 5)?;
838        let state = layer.pack_state_dict("model.layers.0.self_attn.q_proj")?;
839
840        fs::write(
841            dir.path().join("littlebit_config.json"),
842            serde_json::to_string_pretty(&cfg)?,
843        )?;
844        save_packed_safetensors(dir.path(), &state, CastPolicy::ToBf16)?;
845
846        let stats = inspect_checkpoint(dir.path())?;
847        assert!(stats.is_packed);
848        assert_eq!(stats.quantized_layer_count, 1);
849        assert_eq!(stats.layers[0].in_features, 96);
850        assert_eq!(stats.layers[0].out_features, 64);
851        assert!(stats.layers[0].has_residual);
852
853        let loaded = load_and_unpack_state_dict(dir.path(), DType::F32, &dev)?;
854        assert_eq!(loaded.format, CheckpointFormat::Packed);
855        assert!(loaded
856            .tensors
857            .contains_key("model.layers.0.self_attn.q_proj.U"));
858        assert!(loaded
859            .tensors
860            .contains_key("model.layers.0.self_attn.q_proj.V_R"));
861        assert_eq!(
862            loaded
863                .tensors
864                .get("model.layers.0.self_attn.q_proj.U")
865                .expect("U present")
866                .dtype(),
867            DType::F32
868        );
869        assert_eq!(
870            loaded
871                .tensors
872                .get("model.layers.0.self_attn.q_proj.u1")
873                .expect("u1 present")
874                .dtype(),
875            DType::BF16
876        );
877        Ok(())
878    }
879
880    #[test]
881    fn loads_sharded_safetensors_via_index_json() -> Result<()> {
882        let dir = TempDir::new().expect("tempdir");
883        let dev = Device::Cpu;
884
885        let mut s1 = HashMap::new();
886        s1.insert(
887            "a.weight".to_string(),
888            Tensor::ones((4, 4), DType::F32, &dev)?,
889        );
890        candle_core::safetensors::save(&s1, dir.path().join("shard-00001.safetensors"))?;
891
892        let mut s2 = HashMap::new();
893        s2.insert(
894            "b.weight".to_string(),
895            Tensor::zeros((2, 3), DType::F32, &dev)?,
896        );
897        candle_core::safetensors::save(&s2, dir.path().join("shard-00002.safetensors"))?;
898
899        let index = serde_json::json!({
900            "metadata": { "total_size": 88 },
901            "weight_map": {
902                "a.weight": "shard-00001.safetensors",
903                "b.weight": "shard-00002.safetensors"
904            }
905        });
906        fs::write(
907            dir.path().join("model.safetensors.index.json"),
908            serde_json::to_string_pretty(&index)?,
909        )?;
910
911        let loaded = load_and_unpack_state_dict(dir.path(), DType::F32, &dev)?;
912        assert_eq!(loaded.format, CheckpointFormat::LegacyUnpacked);
913        assert_eq!(loaded.tensors.len(), 2);
914        assert_eq!(loaded.tensors["a.weight"].dims(), &[4, 4]);
915        assert_eq!(loaded.tensors["b.weight"].dims(), &[2, 3]);
916        Ok(())
917    }
918
919    #[test]
920    fn streaming_reader_loads_single_sharded_and_3d_slices_exact() -> Result<()> {
921        let dir = TempDir::new().expect("tempdir");
922        let dev = Device::Cpu;
923
924        let t3d = Tensor::randn(0.0f32, 1.0, (4, 6, 8), &dev)?.to_dtype(DType::BF16)?;
925        let t_f32 = Tensor::randn(0.0f32, 0.5, (5, 7), &dev)?;
926        let t_f16 = Tensor::randn(0.0f32, 0.5, (3, 4), &dev)?.to_dtype(DType::F16)?;
927        let t_i32 = Tensor::from_slice(&[1i32, -2, 3, 4], (2, 2), &dev)?;
928        let t_i64 = Tensor::from_slice(&[10i64, 20], (2,), &dev)?;
929
930        let mut s1 = HashMap::new();
931        s1.insert("experts.gate_up_proj".to_string(), t3d.clone());
932        s1.insert("w.f32".to_string(), t_f32.clone());
933        candle_core::safetensors::save(&s1, dir.path().join("model-00001-of-00002.safetensors"))?;
934
935        let mut s2 = HashMap::new();
936        s2.insert("w.f16".to_string(), t_f16.clone());
937        s2.insert("idx.i32".to_string(), t_i32.clone());
938        s2.insert("shape.i64".to_string(), t_i64.clone());
939        candle_core::safetensors::save(&s2, dir.path().join("model-00002-of-00002.safetensors"))?;
940
941        let index = serde_json::json!({
942            "metadata": { "total_size": 0 },
943            "weight_map": {
944                "experts.gate_up_proj": "model-00001-of-00002.safetensors",
945                "w.f32": "model-00001-of-00002.safetensors",
946                "w.f16": "model-00002-of-00002.safetensors",
947                "idx.i32": "model-00002-of-00002.safetensors",
948                "shape.i64": "model-00002-of-00002.safetensors"
949            }
950        });
951        fs::write(
952            dir.path().join("model.safetensors.index.json"),
953            serde_json::to_string_pretty(&index)?,
954        )?;
955
956        let mut reader = SafetensorsReader::open(dir.path())?;
957        assert_eq!(reader.len(), 5);
958        assert!(reader.contains("experts.gate_up_proj"));
959
960        let baseline = load_raw_safetensors_dir(dir.path(), &dev)?;
961        for name in ["w.f32", "w.f16", "idx.i32", "shape.i64"] {
962            let got = reader.load_tensor(name, &dev)?;
963            let exp = &baseline[name];
964            assert_eq!(got.dtype(), exp.dtype());
965            assert_eq!(got.dims(), exp.dims());
966            let diff = got
967                .to_dtype(DType::F32)?
968                .sub(&exp.to_dtype(DType::F32)?)?
969                .abs()?
970                .max_all()?
971                .to_scalar::<f32>()?;
972            assert_eq!(diff, 0.0, "tensor `{name}` mismatch");
973        }
974
975        for e in 0..4 {
976            let slice = reader.load_tensor_slice_dim0("experts.gate_up_proj", e, &dev)?;
977            let expected = t3d.get(e)?;
978            assert_eq!(slice.dims(), &[6, 8]);
979            assert_eq!(slice.dtype(), DType::BF16);
980            let diff = slice
981                .to_dtype(DType::F32)?
982                .sub(&expected.to_dtype(DType::F32)?)?
983                .abs()?
984                .max_all()?
985                .to_scalar::<f32>()?;
986            assert_eq!(diff, 0.0, "3D slice {e} mismatch");
987        }
988        Ok(())
989    }
990
991    #[test]
992    fn sharded_writer_splits_shards_and_roundtrips_with_inspect() -> Result<()> {
993        let dir = TempDir::new().expect("tempdir");
994        let dev = Device::Cpu;
995
996        let cfg = LittleBitConfig {
997            eff_bit: Some(0.5),
998            residual: false,
999            use_itq: true,
1000            itq_n_iter: 5,
1001            ..Default::default()
1002        };
1003        fs::write(
1004            dir.path().join("littlebit_config.json"),
1005            serde_json::to_string_pretty(&cfg)?,
1006        )?;
1007
1008        // Small max_shard_bytes forces at least 2 output shards across two LittleBit layers.
1009        let mut writer = ShardedSafetensorsWriter::new(dir.path(), 512, CastPolicy::ToBf16)?;
1010        for idx in 0..2 {
1011            let w = Tensor::randn(0.0f32, 0.5, (64, 64), &dev)?;
1012            let layer = LittleBitLinear::from_dense_weight(
1013                &w,
1014                None,
1015                &cfg,
1016                1.0,
1017                TrainMode::Eval,
1018                idx as u64,
1019            )?;
1020            let sd = layer.pack_state_dict(&format!("model.layers.{idx}.self_attn.q_proj"))?;
1021            for (k, v) in sd {
1022                writer.add_tensor(k, v)?;
1023            }
1024            writer.flush_if_needed()?;
1025        }
1026        let shards_written = writer.finish()?;
1027        assert!(
1028            shards_written >= 2,
1029            "expected multiple shards, got {shards_written}"
1030        );
1031        assert!(dir.path().join("model.safetensors.index.json").is_file());
1032        assert!(dir
1033            .path()
1034            .join(format!("model-00001-of-{shards_written:05}.safetensors"))
1035            .is_file());
1036
1037        let stats = inspect_checkpoint(dir.path())?;
1038        assert!(stats.is_packed);
1039        assert_eq!(stats.quantized_layer_count, 2);
1040
1041        let unpacked = load_and_unpack_state_dict(dir.path(), DType::F32, &dev)?;
1042        assert_eq!(unpacked.format, CheckpointFormat::Packed);
1043        assert!(unpacked
1044            .tensors
1045            .contains_key("model.layers.0.self_attn.q_proj.U"));
1046        assert!(unpacked
1047            .tensors
1048            .contains_key("model.layers.1.self_attn.q_proj.U"));
1049        Ok(())
1050    }
1051}