1use std::fs;
4use std::path::{Path, PathBuf};
5
6use candle_core::{DType, Device};
7use hf_hub::{split_id, HFClient, RepoTypeModel};
8use serde::{Deserialize, Serialize};
9use tempfile::TempDir;
10
11use littlebit_core::{
12 load_and_unpack_state_dict, load_raw_safetensors_dir, save_packed_safetensors, CastPolicy,
13 CheckpointFormat, InferenceMode, LittleBitConfig, LittleBitConfigOverrides, WeightState,
14};
15
16use crate::config::ModelConfig;
17use crate::error::{ModelError, Result};
18use crate::model::CausalLm;
19
20const TOKENIZER_ARTIFACT_FILES: [&str; 6] = [
21 "tokenizer.json",
22 "tokenizer_config.json",
23 "special_tokens_map.json",
24 "vocab.json",
25 "merges.txt",
26 "tokenizer.model",
27];
28
29#[derive(Debug, Serialize, Deserialize)]
30struct BaseModelMeta {
31 model_id: String,
32}
33
34#[derive(Debug, Clone)]
36pub struct LoadOptions {
37 pub dtype: DType,
39 pub device: Device,
41 pub inference_mode: InferenceMode,
43 pub overrides: LittleBitConfigOverrides,
45 pub cache_dir: Option<PathBuf>,
47 pub token: Option<String>,
49 pub local_files_only: bool,
51}
52
53impl Default for LoadOptions {
54 fn default() -> Self {
55 Self {
56 dtype: DType::F32,
57 device: Device::Cpu,
58 inference_mode: InferenceMode::Unpacked,
59 overrides: LittleBitConfigOverrides::default(),
60 cache_dir: None,
61 token: None,
62 local_files_only: false,
63 }
64 }
65}
66
67#[derive(Debug, Clone)]
69pub struct PushOptions {
70 pub commit_message: String,
72 pub private: bool,
74 pub token: Option<String>,
76 pub local_dir: Option<PathBuf>,
78 pub tokenizer_dir: Option<PathBuf>,
80}
81
82impl Default for PushOptions {
83 fn default() -> Self {
84 Self {
85 commit_message: "Push LittleBit quantized model to Hub".to_string(),
86 private: false,
87 token: None,
88 local_dir: None,
89 tokenizer_dir: None,
90 }
91 }
92}
93
94#[derive(Debug, Clone)]
96pub struct LittleBitModel {
97 pub model: CausalLm,
99 pub littlebit_config: LittleBitConfig,
101 pub base_model_id: Option<String>,
103 pub source_dir: Option<PathBuf>,
105}
106
107pub fn resolve_model_dir(
112 model_id_or_path: &str,
113 cache_dir: Option<&Path>,
114 token: Option<&str>,
115 local_files_only: bool,
116) -> Result<PathBuf> {
117 let as_path = Path::new(model_id_or_path);
118 if as_path.is_dir() {
119 return Ok(as_path.to_path_buf());
120 }
121
122 let mut builder = HFClient::builder();
123 if let Some(tok) = token {
124 builder = builder.token(tok);
125 }
126 if let Some(cache) = cache_dir {
127 builder = builder.cache_dir(cache);
128 }
129 let client = builder.build_sync()?;
130 let (owner, name) = split_id(model_id_or_path);
131 let path = client
132 .model(owner, name)
133 .snapshot_download()
134 .local_files_only(local_files_only)
135 .send()?;
136 Ok(path)
137}
138
139pub fn load_dense_model(
144 model_id_or_path: &str,
145 options: &LoadOptions,
146) -> Result<(CausalLm, PathBuf)> {
147 let dir = resolve_model_dir(
148 model_id_or_path,
149 options.cache_dir.as_deref(),
150 options.token.as_deref(),
151 options.local_files_only,
152 )?;
153 let cfg = ModelConfig::from_file(&dir.join("config.json"))?;
154 let tensors = load_raw_safetensors_dir(&dir, &options.device)?;
155 let model = CausalLm::from_tensors(
156 cfg,
157 &tensors,
158 None,
159 WeightState::Raw,
160 InferenceMode::Unpacked,
161 options.dtype,
162 &options.device,
163 )?;
164 Ok((model, dir))
165}
166
167pub(crate) fn write_quantized_metadata(
168 save_dir: &Path,
169 model_config: &ModelConfig,
170 littlebit_config: &LittleBitConfig,
171 base_model_id: Option<&str>,
172 tokenizer_src: Option<&Path>,
173) -> Result<()> {
174 fs::create_dir_all(save_dir)?;
175
176 let lb_json = serde_json::to_string_pretty(littlebit_config)?;
177 fs::write(save_dir.join("littlebit_config.json"), lb_json)?;
178
179 if let Some(base_id) = base_model_id {
180 let meta = serde_json::to_string_pretty(&BaseModelMeta {
181 model_id: base_id.to_string(),
182 })?;
183 fs::write(save_dir.join("base_model.json"), meta)?;
184 }
185
186 let mut model_cfg_val = serde_json::to_value(model_config)?;
187 if let Some(obj) = model_cfg_val.as_object_mut() {
188 obj.insert(
189 "auto_map".to_string(),
190 serde_json::json!({ "AutoModelForCausalLM": "quantization.hub.LittleBitModel" }),
191 );
192 obj.insert(
193 "quant_func".to_string(),
194 serde_json::json!(littlebit_config.quant_func.as_wire_name()),
195 );
196 obj.insert(
197 "eff_bit".to_string(),
198 serde_json::json!(littlebit_config.eff_bit),
199 );
200 obj.insert(
201 "split_dim".to_string(),
202 serde_json::json!(littlebit_config.split_dim),
203 );
204 obj.insert(
205 "residual".to_string(),
206 serde_json::json!(littlebit_config.residual),
207 );
208 obj.insert(
209 "kv_factor".to_string(),
210 serde_json::json!(littlebit_config.kv_factor),
211 );
212 obj.insert(
213 "min_split_dim".to_string(),
214 serde_json::json!(littlebit_config.min_split_dim),
215 );
216 obj.insert(
217 "quant_mod".to_string(),
218 serde_json::json!(littlebit_config.quant_mod),
219 );
220 obj.insert(
221 "use_itq".to_string(),
222 serde_json::json!(littlebit_config.use_itq),
223 );
224 obj.insert(
225 "itq_n_iter".to_string(),
226 serde_json::json!(littlebit_config.itq_n_iter),
227 );
228 if let Some(ref layers) = littlebit_config.eff_bit_layers {
229 obj.insert("eff_bit_layers".to_string(), serde_json::json!(layers));
230 }
231 }
232 fs::write(
233 save_dir.join("config.json"),
234 serde_json::to_string_pretty(&model_cfg_val)?,
235 )?;
236
237 if let Some(tok_dir) = tokenizer_src {
238 for filename in TOKENIZER_ARTIFACT_FILES {
239 let src = tok_dir.join(filename);
240 if src.is_file() {
241 let dst = save_dir.join(filename);
242 if src != dst {
243 fs::copy(&src, &dst)?;
244 }
245 }
246 }
247 }
248 Ok(())
249}
250
251impl LittleBitModel {
252 #[must_use]
254 pub fn new(
255 model: CausalLm,
256 littlebit_config: LittleBitConfig,
257 base_model_id: Option<String>,
258 source_dir: Option<PathBuf>,
259 ) -> Self {
260 Self {
261 model,
262 littlebit_config,
263 base_model_id,
264 source_dir,
265 }
266 }
267
268 pub fn from_pretrained(model_id_or_path: &str, options: &LoadOptions) -> Result<Self> {
279 let dir = resolve_model_dir(
280 model_id_or_path,
281 options.cache_dir.as_deref(),
282 options.token.as_deref(),
283 options.local_files_only,
284 )?;
285 let lb_cfg = LittleBitConfig::load_with_priority(&dir, &options.overrides)?;
286 let model_cfg = ModelConfig::from_file(&dir.join("config.json"))?;
287
288 let (tensors, weight_state) = match options.inference_mode {
289 InferenceMode::Packed => {
290 let raw = load_raw_safetensors_dir(&dir, &Device::Cpu)?;
291 if raw.keys().any(|k| k.ends_with("_packed")) {
292 (raw, WeightState::PreBinarized)
293 } else {
294 let unpacked =
295 load_and_unpack_state_dict(&dir, options.dtype, &options.device)?;
296 let ws = match unpacked.format {
297 CheckpointFormat::Packed => WeightState::PreBinarized,
298 CheckpointFormat::LegacyUnpacked => WeightState::Raw,
299 };
300 (unpacked.tensors, ws)
301 }
302 }
303 InferenceMode::Unpacked => {
304 let unpacked = load_and_unpack_state_dict(&dir, options.dtype, &options.device)?;
305 let ws = match unpacked.format {
306 CheckpointFormat::Packed => WeightState::PreBinarized,
307 CheckpointFormat::LegacyUnpacked => WeightState::Raw,
308 };
309 (unpacked.tensors, ws)
310 }
311 };
312
313 let model = CausalLm::from_tensors(
314 model_cfg,
315 &tensors,
316 Some(&lb_cfg),
317 weight_state,
318 options.inference_mode,
319 options.dtype,
320 &options.device,
321 )?;
322
323 let base_model_path = dir.join("base_model.json");
324 let base_model_id = if base_model_path.exists() {
325 let raw = fs::read_to_string(&base_model_path)?;
326 serde_json::from_str::<BaseModelMeta>(&raw)
327 .ok()
328 .map(|m| m.model_id)
329 } else {
330 None
331 };
332
333 Ok(Self {
334 model,
335 littlebit_config: lb_cfg,
336 base_model_id,
337 source_dir: Some(dir),
338 })
339 }
340
341 pub fn save_pretrained(
347 &self,
348 save_dir: &Path,
349 tokenizer_src: Option<&Path>,
350 cast_policy: CastPolicy,
351 ) -> Result<()> {
352 write_quantized_metadata(
353 save_dir,
354 self.model.config(),
355 &self.littlebit_config,
356 self.base_model_id.as_deref(),
357 tokenizer_src.or(self.source_dir.as_deref()),
358 )?;
359 let state_dict = self.model.pack_state_dict()?;
360 save_packed_safetensors(save_dir, &state_dict, cast_policy)?;
361 Ok(())
362 }
363
364 #[must_use]
366 pub fn generate_readme(&self, repo_id: &str) -> String {
367 let model_name = repo_id.rsplit('/').next().unwrap_or(repo_id);
368 let cfg = self.model.config();
369 let lb = &self.littlebit_config;
370 let eff_bit_str = lb
371 .eff_bit
372 .map_or_else(|| "None".to_string(), |v| format!("{v}"));
373 let original_section = match &self.base_model_id {
374 Some(base) => {
375 format!("## Original Model\n\nBase model: [{base}](https://huggingface.co/{base})")
376 }
377 None => "## Original Model\n\n- Not specified".to_string(),
378 };
379
380 [
381 format!(
382 "---\nlicense: other\ntags:\n- quantization\n- littlebit\n- {}\n---",
383 cfg.model_type
384 ),
385 format!("# {model_name}\n\nA LittleBit quantized model."),
386 format!(
387 "## Quantization Config\n\n\
388 - **Quantization Function**: {}\n\
389 - **Effective Bits**: {eff_bit_str}\n\
390 - **Split Dim**: {}\n\
391 - **Residual**: {}\n\
392 - **KV Factor**: {}\n\
393 - **Use ITQ (LittleBit-2)**: {}",
394 lb.quant_func, lb.split_dim, lb.residual, lb.kv_factor, lb.use_itq
395 ),
396 format!(
397 "## Usage\n\nLoad with `littlebit` CLI:\n\n```bash\nlittlebit eval --model-id \"{repo_id}\"\n```"
398 ),
399 original_section,
400 format!(
401 "## Model Details\n\n\
402 - **Model Type**: {}\n\
403 - **Hidden Size**: {}\n\
404 - **Num Attention Heads**: {}\n\
405 - **Num Hidden Layers**: {}",
406 cfg.model_type, cfg.hidden_size, cfg.num_attention_heads, cfg.num_hidden_layers
407 ),
408 "## Citation\n\nIf you use this model, please cite:\n\n```bibtex\n\
409@inproceedings{lee2026littlebit2,\n\
410 title={LittleBit-2: Maximizing the Spectral Energy Gain in Sub-1-Bit LLMs via Latent Geometry Alignment},\n\
411 author={Lee, Banseok and Kim, Youngmin},\n\
412 booktitle={ICML},\n\
413 year={2026}\n\
414}\n\n\
415@inproceedings{lee2025littlebit,\n\
416 title={LittleBit: Ultra Low-Bit Quantization via Latent Factorization},\n\
417 author={Lee, Banseok and Kim, Dongkyu and You, Youngcheon and Kim, Youngmin},\n\
418 booktitle={NeurIPS},\n\
419 year={2025}\n\
420}\n\
421```"
422 .to_string(),
423 ]
424 .join("\n\n")
425 }
426
427 pub fn push_to_hub(&self, repo_id: &str, options: &PushOptions) -> Result<String> {
432 let _temp_guard: Option<TempDir>;
433 let stage_dir: PathBuf = match &options.local_dir {
434 Some(dir) => {
435 _temp_guard = None;
436 fs::create_dir_all(dir)?;
437 dir.clone()
438 }
439 None => {
440 let tmp = TempDir::new()?;
441 let p = tmp.path().to_path_buf();
442 _temp_guard = Some(tmp);
443 p
444 }
445 };
446
447 self.save_pretrained(
448 &stage_dir,
449 options.tokenizer_dir.as_deref(),
450 CastPolicy::ToBf16,
451 )?;
452 fs::write(stage_dir.join("README.md"), self.generate_readme(repo_id))?;
453
454 let mut builder = HFClient::builder();
455 if let Some(tok) = &options.token {
456 builder = builder.token(tok);
457 }
458 let client = builder.build_sync()?;
459 let _ = client
460 .create_repository()
461 .repo_id(repo_id)
462 .repo_type(RepoTypeModel)
463 .private(options.private)
464 .exist_ok(true)
465 .send();
466
467 let (owner, name) = split_id(repo_id);
468 if owner.is_empty() {
469 return Err(ModelError::InvalidArgument(format!(
470 "repo_id must be in `owner/name` format, got `{repo_id}`"
471 )));
472 }
473 let commit = client
474 .model(owner, name)
475 .upload_folder()
476 .folder_path(&stage_dir)
477 .commit_message(&options.commit_message)
478 .send()?;
479
480 Ok(commit
481 .commit_url
482 .unwrap_or_else(|| format!("https://huggingface.co/{repo_id}")))
483 }
484}