Skip to main content

littlebit_models/
hub.rs

1//! Hugging Face Hub integration and checkpoint packaging (`quantization/hub.py` parity).
2
3use std::fs;
4use std::path::{Path, PathBuf};
5
6use candle_core::{DType, Device};
7use hf_hub::{split_id, HFClient, RepoTypeModel};
8use serde::{Deserialize, Serialize};
9use tempfile::TempDir;
10
11use littlebit_core::{
12    load_and_unpack_state_dict, load_raw_safetensors_dir, save_packed_safetensors, CastPolicy,
13    CheckpointFormat, InferenceMode, LittleBitConfig, LittleBitConfigOverrides, WeightState,
14};
15
16use crate::config::ModelConfig;
17use crate::error::{ModelError, Result};
18use crate::model::CausalLm;
19
20const TOKENIZER_ARTIFACT_FILES: [&str; 6] = [
21    "tokenizer.json",
22    "tokenizer_config.json",
23    "special_tokens_map.json",
24    "vocab.json",
25    "merges.txt",
26    "tokenizer.model",
27];
28
29#[derive(Debug, Serialize, Deserialize)]
30struct BaseModelMeta {
31    model_id: String,
32}
33
34/// Options controlling how a checkpoint or dense model is loaded from disk or Hub.
35#[derive(Debug, Clone)]
36pub struct LoadOptions {
37    /// Target compute dtype (`DType::BF16` or `DType::F32`).
38    pub dtype: DType,
39    /// Target compute device (`Device::Cpu`, Metal, or CUDA).
40    pub device: Device,
41    /// Unpack binary weights into float matrices (`Unpacked`) or keep 1-bit packed (`Packed`).
42    pub inference_mode: InferenceMode,
43    /// Explicit CLI/API overrides applied with highest priority over config files.
44    pub overrides: LittleBitConfigOverrides,
45    /// Optional custom Hugging Face cache directory.
46    pub cache_dir: Option<PathBuf>,
47    /// Optional Hugging Face API token.
48    pub token: Option<String>,
49    /// Use only local cached files when resolving a Hub repo ID.
50    pub local_files_only: bool,
51}
52
53impl Default for LoadOptions {
54    fn default() -> Self {
55        Self {
56            dtype: DType::F32,
57            device: Device::Cpu,
58            inference_mode: InferenceMode::Unpacked,
59            overrides: LittleBitConfigOverrides::default(),
60            cache_dir: None,
61            token: None,
62            local_files_only: false,
63        }
64    }
65}
66
67/// Options controlling Hub publication via [`LittleBitModel::push_to_hub`].
68#[derive(Debug, Clone)]
69pub struct PushOptions {
70    /// Commit message for the Hub upload.
71    pub commit_message: String,
72    /// Whether to create the repository as private.
73    pub private: bool,
74    /// Optional Hugging Face API token (defaults to `HF_TOKEN` / cached login).
75    pub token: Option<String>,
76    /// Optional local staging directory (if omitted, uses a temporary directory).
77    pub local_dir: Option<PathBuf>,
78    /// Optional directory containing tokenizer artifacts to bundle before uploading.
79    pub tokenizer_dir: Option<PathBuf>,
80}
81
82impl Default for PushOptions {
83    fn default() -> Self {
84        Self {
85            commit_message: "Push LittleBit quantized model to Hub".to_string(),
86            private: false,
87            token: None,
88            local_dir: None,
89            tokenizer_dir: None,
90        }
91    }
92}
93
94/// LittleBit quantized model wrapper supporting local persistence and Hugging Face Hub upload/download.
95#[derive(Debug, Clone)]
96pub struct LittleBitModel {
97    /// Underlying causal language model.
98    pub model: CausalLm,
99    /// Resolved LittleBit quantization hyperparameters.
100    pub littlebit_config: LittleBitConfig,
101    /// Optional source base model identifier (e.g., `meta-llama/Llama-2-7b-hf`).
102    pub base_model_id: Option<String>,
103    /// Local directory from which the model or tokenizer was resolved.
104    pub source_dir: Option<PathBuf>,
105}
106
107/// Resolve a local directory or download a model repository snapshot from Hugging Face Hub.
108///
109/// # Errors
110/// Returns an error if the Hub client cannot be built or snapshot download fails.
111pub fn resolve_model_dir(
112    model_id_or_path: &str,
113    cache_dir: Option<&Path>,
114    token: Option<&str>,
115    local_files_only: bool,
116) -> Result<PathBuf> {
117    let as_path = Path::new(model_id_or_path);
118    if as_path.is_dir() {
119        return Ok(as_path.to_path_buf());
120    }
121
122    let mut builder = HFClient::builder();
123    if let Some(tok) = token {
124        builder = builder.token(tok);
125    }
126    if let Some(cache) = cache_dir {
127        builder = builder.cache_dir(cache);
128    }
129    let client = builder.build_sync()?;
130    let (owner, name) = split_id(model_id_or_path);
131    let path = client
132        .model(owner, name)
133        .snapshot_download()
134        .local_files_only(local_files_only)
135        .send()?;
136    Ok(path)
137}
138
139/// Load a dense (unquantized) Hugging Face causal LM from a local directory or Hub repo ID.
140///
141/// # Errors
142/// Returns an error if config or weights cannot be resolved or loaded.
143pub fn load_dense_model(
144    model_id_or_path: &str,
145    options: &LoadOptions,
146) -> Result<(CausalLm, PathBuf)> {
147    let dir = resolve_model_dir(
148        model_id_or_path,
149        options.cache_dir.as_deref(),
150        options.token.as_deref(),
151        options.local_files_only,
152    )?;
153    let cfg = ModelConfig::from_file(&dir.join("config.json"))?;
154    let tensors = load_raw_safetensors_dir(&dir, &options.device)?;
155    let model = CausalLm::from_tensors(
156        cfg,
157        &tensors,
158        None,
159        WeightState::Raw,
160        InferenceMode::Unpacked,
161        options.dtype,
162        &options.device,
163    )?;
164    Ok((model, dir))
165}
166
167pub(crate) fn write_quantized_metadata(
168    save_dir: &Path,
169    model_config: &ModelConfig,
170    littlebit_config: &LittleBitConfig,
171    base_model_id: Option<&str>,
172    tokenizer_src: Option<&Path>,
173) -> Result<()> {
174    fs::create_dir_all(save_dir)?;
175
176    let lb_json = serde_json::to_string_pretty(littlebit_config)?;
177    fs::write(save_dir.join("littlebit_config.json"), lb_json)?;
178
179    if let Some(base_id) = base_model_id {
180        let meta = serde_json::to_string_pretty(&BaseModelMeta {
181            model_id: base_id.to_string(),
182        })?;
183        fs::write(save_dir.join("base_model.json"), meta)?;
184    }
185
186    let mut model_cfg_val = serde_json::to_value(model_config)?;
187    if let Some(obj) = model_cfg_val.as_object_mut() {
188        obj.insert(
189            "auto_map".to_string(),
190            serde_json::json!({ "AutoModelForCausalLM": "quantization.hub.LittleBitModel" }),
191        );
192        obj.insert(
193            "quant_func".to_string(),
194            serde_json::json!(littlebit_config.quant_func.as_wire_name()),
195        );
196        obj.insert(
197            "eff_bit".to_string(),
198            serde_json::json!(littlebit_config.eff_bit),
199        );
200        obj.insert(
201            "split_dim".to_string(),
202            serde_json::json!(littlebit_config.split_dim),
203        );
204        obj.insert(
205            "residual".to_string(),
206            serde_json::json!(littlebit_config.residual),
207        );
208        obj.insert(
209            "kv_factor".to_string(),
210            serde_json::json!(littlebit_config.kv_factor),
211        );
212        obj.insert(
213            "min_split_dim".to_string(),
214            serde_json::json!(littlebit_config.min_split_dim),
215        );
216        obj.insert(
217            "quant_mod".to_string(),
218            serde_json::json!(littlebit_config.quant_mod),
219        );
220        obj.insert(
221            "use_itq".to_string(),
222            serde_json::json!(littlebit_config.use_itq),
223        );
224        obj.insert(
225            "itq_n_iter".to_string(),
226            serde_json::json!(littlebit_config.itq_n_iter),
227        );
228        if let Some(ref layers) = littlebit_config.eff_bit_layers {
229            obj.insert("eff_bit_layers".to_string(), serde_json::json!(layers));
230        }
231    }
232    fs::write(
233        save_dir.join("config.json"),
234        serde_json::to_string_pretty(&model_cfg_val)?,
235    )?;
236
237    if let Some(tok_dir) = tokenizer_src {
238        for filename in TOKENIZER_ARTIFACT_FILES {
239            let src = tok_dir.join(filename);
240            if src.is_file() {
241                let dst = save_dir.join(filename);
242                if src != dst {
243                    fs::copy(&src, &dst)?;
244                }
245            }
246        }
247    }
248    Ok(())
249}
250
251impl LittleBitModel {
252    /// Wrap a quantized [`CausalLm`] alongside its [`LittleBitConfig`] and optional base model ID.
253    #[must_use]
254    pub fn new(
255        model: CausalLm,
256        littlebit_config: LittleBitConfig,
257        base_model_id: Option<String>,
258        source_dir: Option<PathBuf>,
259    ) -> Self {
260        Self {
261            model,
262            littlebit_config,
263            base_model_id,
264            source_dir,
265        }
266    }
267
268    /// Load a pre-quantized LittleBit model from a local path or Hugging Face Hub repository ID.
269    ///
270    /// Implements upstream priority:
271    /// 1. `options.overrides`
272    /// 2. `littlebit_config.json`
273    /// 3. `config.json` fallback
274    /// 4. Defaults
275    ///
276    /// # Errors
277    /// Returns an error if downloading, config resolution, or weight loading fails.
278    pub fn from_pretrained(model_id_or_path: &str, options: &LoadOptions) -> Result<Self> {
279        let dir = resolve_model_dir(
280            model_id_or_path,
281            options.cache_dir.as_deref(),
282            options.token.as_deref(),
283            options.local_files_only,
284        )?;
285        let lb_cfg = LittleBitConfig::load_with_priority(&dir, &options.overrides)?;
286        let model_cfg = ModelConfig::from_file(&dir.join("config.json"))?;
287
288        let (tensors, weight_state) = match options.inference_mode {
289            InferenceMode::Packed => {
290                let raw = load_raw_safetensors_dir(&dir, &Device::Cpu)?;
291                if raw.keys().any(|k| k.ends_with("_packed")) {
292                    (raw, WeightState::PreBinarized)
293                } else {
294                    let unpacked =
295                        load_and_unpack_state_dict(&dir, options.dtype, &options.device)?;
296                    let ws = match unpacked.format {
297                        CheckpointFormat::Packed => WeightState::PreBinarized,
298                        CheckpointFormat::LegacyUnpacked => WeightState::Raw,
299                    };
300                    (unpacked.tensors, ws)
301                }
302            }
303            InferenceMode::Unpacked => {
304                let unpacked = load_and_unpack_state_dict(&dir, options.dtype, &options.device)?;
305                let ws = match unpacked.format {
306                    CheckpointFormat::Packed => WeightState::PreBinarized,
307                    CheckpointFormat::LegacyUnpacked => WeightState::Raw,
308                };
309                (unpacked.tensors, ws)
310            }
311        };
312
313        let model = CausalLm::from_tensors(
314            model_cfg,
315            &tensors,
316            Some(&lb_cfg),
317            weight_state,
318            options.inference_mode,
319            options.dtype,
320            &options.device,
321        )?;
322
323        let base_model_path = dir.join("base_model.json");
324        let base_model_id = if base_model_path.exists() {
325            let raw = fs::read_to_string(&base_model_path)?;
326            serde_json::from_str::<BaseModelMeta>(&raw)
327                .ok()
328                .map(|m| m.model_id)
329        } else {
330            None
331        };
332
333        Ok(Self {
334            model,
335            littlebit_config: lb_cfg,
336            base_model_id,
337            source_dir: Some(dir),
338        })
339    }
340
341    /// Save `littlebit_config.json`, `base_model.json`, `config.json`, packed `model.safetensors`,
342    /// `model.safetensors.index.json`, and tokenizer files (when found) to `save_dir`.
343    ///
344    /// # Errors
345    /// Returns an error if packing weights or writing files fails.
346    pub fn save_pretrained(
347        &self,
348        save_dir: &Path,
349        tokenizer_src: Option<&Path>,
350        cast_policy: CastPolicy,
351    ) -> Result<()> {
352        write_quantized_metadata(
353            save_dir,
354            self.model.config(),
355            &self.littlebit_config,
356            self.base_model_id.as_deref(),
357            tokenizer_src.or(self.source_dir.as_deref()),
358        )?;
359        let state_dict = self.model.pack_state_dict()?;
360        save_packed_safetensors(save_dir, &state_dict, cast_policy)?;
361        Ok(())
362    }
363
364    /// Generate a Hugging Face Hub model card matching upstream `LittleBitModel._generate_readme`.
365    #[must_use]
366    pub fn generate_readme(&self, repo_id: &str) -> String {
367        let model_name = repo_id.rsplit('/').next().unwrap_or(repo_id);
368        let cfg = self.model.config();
369        let lb = &self.littlebit_config;
370        let eff_bit_str = lb
371            .eff_bit
372            .map_or_else(|| "None".to_string(), |v| format!("{v}"));
373        let original_section = match &self.base_model_id {
374            Some(base) => {
375                format!("## Original Model\n\nBase model: [{base}](https://huggingface.co/{base})")
376            }
377            None => "## Original Model\n\n- Not specified".to_string(),
378        };
379
380        [
381            format!(
382                "---\nlicense: other\ntags:\n- quantization\n- littlebit\n- {}\n---",
383                cfg.model_type
384            ),
385            format!("# {model_name}\n\nA LittleBit quantized model."),
386            format!(
387                "## Quantization Config\n\n\
388                - **Quantization Function**: {}\n\
389                - **Effective Bits**: {eff_bit_str}\n\
390                - **Split Dim**: {}\n\
391                - **Residual**: {}\n\
392                - **KV Factor**: {}\n\
393                - **Use ITQ (LittleBit-2)**: {}",
394                lb.quant_func, lb.split_dim, lb.residual, lb.kv_factor, lb.use_itq
395            ),
396            format!(
397                "## Usage\n\nLoad with `littlebit` CLI:\n\n```bash\nlittlebit eval --model-id \"{repo_id}\"\n```"
398            ),
399            original_section,
400            format!(
401                "## Model Details\n\n\
402                - **Model Type**: {}\n\
403                - **Hidden Size**: {}\n\
404                - **Num Attention Heads**: {}\n\
405                - **Num Hidden Layers**: {}",
406                cfg.model_type, cfg.hidden_size, cfg.num_attention_heads, cfg.num_hidden_layers
407            ),
408            "## Citation\n\nIf you use this model, please cite:\n\n```bibtex\n\
409@inproceedings{lee2026littlebit2,\n\
410  title={LittleBit-2: Maximizing the Spectral Energy Gain in Sub-1-Bit LLMs via Latent Geometry Alignment},\n\
411  author={Lee, Banseok and Kim, Youngmin},\n\
412  booktitle={ICML},\n\
413  year={2026}\n\
414}\n\n\
415@inproceedings{lee2025littlebit,\n\
416  title={LittleBit: Ultra Low-Bit Quantization via Latent Factorization},\n\
417  author={Lee, Banseok and Kim, Dongkyu and You, Youngcheon and Kim, Youngmin},\n\
418  booktitle={NeurIPS},\n\
419  year={2025}\n\
420}\n\
421```"
422                .to_string(),
423        ]
424        .join("\n\n")
425    }
426
427    /// Save the quantized checkpoint and upload it to Hugging Face Hub.
428    ///
429    /// # Errors
430    /// Returns [`ModelError::Hub`] or I/O errors if staging or uploading fails.
431    pub fn push_to_hub(&self, repo_id: &str, options: &PushOptions) -> Result<String> {
432        let _temp_guard: Option<TempDir>;
433        let stage_dir: PathBuf = match &options.local_dir {
434            Some(dir) => {
435                _temp_guard = None;
436                fs::create_dir_all(dir)?;
437                dir.clone()
438            }
439            None => {
440                let tmp = TempDir::new()?;
441                let p = tmp.path().to_path_buf();
442                _temp_guard = Some(tmp);
443                p
444            }
445        };
446
447        self.save_pretrained(
448            &stage_dir,
449            options.tokenizer_dir.as_deref(),
450            CastPolicy::ToBf16,
451        )?;
452        fs::write(stage_dir.join("README.md"), self.generate_readme(repo_id))?;
453
454        let mut builder = HFClient::builder();
455        if let Some(tok) = &options.token {
456            builder = builder.token(tok);
457        }
458        let client = builder.build_sync()?;
459        let _ = client
460            .create_repository()
461            .repo_id(repo_id)
462            .repo_type(RepoTypeModel)
463            .private(options.private)
464            .exist_ok(true)
465            .send();
466
467        let (owner, name) = split_id(repo_id);
468        if owner.is_empty() {
469            return Err(ModelError::InvalidArgument(format!(
470                "repo_id must be in `owner/name` format, got `{repo_id}`"
471            )));
472        }
473        let commit = client
474            .model(owner, name)
475            .upload_folder()
476            .folder_path(&stage_dir)
477            .commit_message(&options.commit_message)
478            .send()?;
479
480        Ok(commit
481            .commit_url
482            .unwrap_or_else(|| format!("https://huggingface.co/{repo_id}")))
483    }
484}