Skip to main content

littlebit_train/
data.rs

1// -----------------------------------------------------------------------------
2// Derived from utils/datautils.py in SamsungLabs/LittleBit under the MIT License:
3// Copyright (c) 2024 Dongwon Jo
4// Modified by Dongkyu Kim in 2026 (Samsung Electronics Co., Ltd.)
5// -----------------------------------------------------------------------------
6
7//! Dataset loading, tokenization, SHA-256 caching, and block chunking (`utils/datautils.py`).
8
9use std::fmt;
10use std::fs::{self, File};
11use std::io::{BufRead, BufReader, Read, Write};
12use std::path::{Path, PathBuf};
13use std::str::FromStr;
14
15use flate2::read::GzDecoder;
16use hf_hub::{split_id, HFClient};
17use rand::Rng;
18use rand::SeedableRng;
19use rand_chacha::ChaCha8Rng;
20use serde::{Deserialize, Serialize};
21use sha2::{Digest, Sha256};
22use tokenizers::Tokenizer;
23use tracing::info;
24
25use littlebit_models::resolve_model_dir;
26
27use crate::error::{Result, TrainError};
28
29/// Default sequence block length matching `utils/datautils.py`.
30pub const DEFAULT_BLOCK_SIZE: usize = 2048;
31
32/// Supported training and perplexity evaluation datasets.
33#[derive(Debug, Clone, PartialEq, Eq)]
34pub enum DatasetKind {
35    /// WikiText-2 raw (`Salesforce/wikitext` / `wikitext-2-raw-v1`).
36    WikiText2,
37    /// AllenAI C4 English (`allenai/c4`).
38    C4,
39    /// Combined C4 + WikiText-2 training corpus (`c4_wiki`).
40    C4Wiki,
41    /// Penn Treebank validation split (`ptb`).
42    Ptb,
43    /// Penn Treebank test split (`ptb_new`).
44    PtbNew,
45    /// AllenAI C4 contiguous slice (`c4_new`).
46    C4New,
47    /// Local `.txt`, `.jsonl`, or `.jsonl.gz` corpus file.
48    LocalFile(PathBuf),
49}
50
51impl DatasetKind {
52    /// Canonical cache directory name matching upstream `utils/datautils.py`.
53    #[must_use]
54    pub fn cache_name(&self) -> &str {
55        match self {
56            Self::WikiText2 => "wikitext2",
57            Self::C4 => "c4",
58            Self::C4Wiki => "c4_wiki",
59            Self::Ptb => "ptb",
60            Self::PtbNew => "ptb_new",
61            Self::C4New => "c4_new",
62            Self::LocalFile(_) => "local",
63        }
64    }
65}
66
67impl fmt::Display for DatasetKind {
68    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
69        match self {
70            Self::LocalFile(p) => write!(f, "{}", p.display()),
71            _ => f.write_str(self.cache_name()),
72        }
73    }
74}
75
76impl FromStr for DatasetKind {
77    type Err = TrainError;
78
79    fn from_str(s: &str) -> Result<Self> {
80        match s.trim().to_ascii_lowercase().as_str() {
81            "wikitext2" => Ok(Self::WikiText2),
82            "c4" => Ok(Self::C4),
83            "c4_wiki" => Ok(Self::C4Wiki),
84            "ptb" => Ok(Self::Ptb),
85            "ptb_new" | "ptb-new" => Ok(Self::PtbNew),
86            "c4_new" | "c4-new" => Ok(Self::C4New),
87            _ => {
88                let p = PathBuf::from(s);
89                if p.exists() {
90                    Ok(Self::LocalFile(p))
91                } else {
92                    Err(TrainError::InvalidArgument(format!(
93                        "unknown dataset `{s}` (expected wikitext2, c4, c4_wiki, ptb, ptb_new, c4_new, or an existing file path)"
94                    )))
95                }
96            }
97        }
98    }
99}
100
101/// Compute 7-character SHA-256 hex cache key from tokenizer config metadata,
102/// matching `hashlib.sha256(hash_key.encode()).hexdigest()[:7]`.
103#[must_use]
104pub fn tokenizer_cache_key(tokenizer_descriptor: &str) -> String {
105    let digest = Sha256::digest(tokenizer_descriptor.as_bytes());
106    let full = hex::encode(digest);
107    full[..7].to_string()
108}
109
110/// Concatenate tokenized streams into contiguous `block_size` chunks, dropping any short tail
111/// remainder (exact match to `group_texts` in `utils/datautils.py`).
112#[must_use]
113pub fn group_token_blocks(streams: &[Vec<u32>], block_size: usize) -> Vec<Vec<u32>> {
114    if block_size == 0 {
115        return Vec::new();
116    }
117    let total_tokens: usize = streams.iter().map(Vec::len).sum();
118    let usable = (total_tokens / block_size) * block_size;
119    if usable == 0 {
120        return Vec::new();
121    }
122
123    let mut flat = Vec::with_capacity(usable);
124    for stream in streams {
125        let remaining = usable - flat.len();
126        if remaining == 0 {
127            break;
128        }
129        let take = stream.len().min(remaining);
130        flat.extend_from_slice(&stream[..take]);
131    }
132    flat.chunks_exact(block_size).map(Vec::from).collect()
133}
134
135/// Load a Hugging Face `tokenizer.json` from a local checkpoint directory or Hub model repo ID.
136///
137/// # Errors
138/// Returns [`TrainError::Tokenizer`] or [`TrainError::Hub`] if `tokenizer.json` cannot be resolved.
139pub fn load_tokenizer(
140    model_id_or_path: &str,
141    cache_dir: Option<&Path>,
142    token: Option<&str>,
143    local_files_only: bool,
144) -> Result<Tokenizer> {
145    let local_tok = Path::new(model_id_or_path).join("tokenizer.json");
146    let tok_path = if local_tok.is_file() {
147        local_tok
148    } else if Path::new(model_id_or_path).is_dir() {
149        return Err(TrainError::Tokenizer(format!(
150            "directory `{model_id_or_path}` does not contain `tokenizer.json`"
151        )));
152    } else {
153        let dir = resolve_model_dir(model_id_or_path, cache_dir, token, local_files_only)?;
154        let cached = dir.join("tokenizer.json");
155        if cached.is_file() {
156            cached
157        } else {
158            let mut builder = HFClient::builder();
159            if let Some(t) = token {
160                builder = builder.token(t);
161            }
162            if let Some(c) = cache_dir {
163                builder = builder.cache_dir(c);
164            }
165            let client = builder.build_sync()?;
166            let (owner, name) = split_id(model_id_or_path);
167            client
168                .model(owner, name)
169                .download_file()
170                .filename("tokenizer.json")
171                .local_files_only(local_files_only)
172                .send()?
173        }
174    };
175
176    Tokenizer::from_file(&tok_path)
177        .map_err(|err| TrainError::Tokenizer(format!("failed to load tokenizer: {err}")))
178}
179
180#[derive(Debug, Serialize, Deserialize)]
181struct DatasetCacheMeta {
182    dataset: String,
183    block_size: usize,
184    num_blocks: usize,
185    tokenizer_hash: String,
186}
187
188/// Persisted token-block dataset with fast binary cache round-trip (`blocks.bin` + `metadata.json`).
189#[derive(Debug, Clone)]
190pub struct TokenBlockDataset {
191    /// Fixed-length token sequences `(num_blocks, block_size)`.
192    pub blocks: Vec<Vec<u32>>,
193    /// Block sequence length.
194    pub block_size: usize,
195    /// Cache directory if persisted.
196    pub cache_dir: Option<PathBuf>,
197}
198
199impl TokenBlockDataset {
200    /// Construct in-memory from raw token blocks.
201    #[must_use]
202    pub fn from_blocks(blocks: Vec<Vec<u32>>, block_size: usize) -> Self {
203        Self {
204            blocks,
205            block_size,
206            cache_dir: None,
207        }
208    }
209
210    /// Save token blocks as little-endian `u32` chunks alongside `metadata.json` and `tokenizer_info`.
211    ///
212    /// # Errors
213    /// Returns an I/O or serialization error if writing fails.
214    pub fn save_to_disk(&self, dir: &Path, dataset_name: &str, tokenizer_info: &str) -> Result<()> {
215        fs::create_dir_all(dir)?;
216        let hash = tokenizer_cache_key(tokenizer_info);
217        let meta = DatasetCacheMeta {
218            dataset: dataset_name.to_string(),
219            block_size: self.block_size,
220            num_blocks: self.blocks.len(),
221            tokenizer_hash: hash,
222        };
223        fs::write(
224            dir.join("metadata.json"),
225            serde_json::to_string_pretty(&meta)?,
226        )?;
227        fs::write(dir.join("tokenizer_info"), tokenizer_info)?;
228
229        let mut file = File::create(dir.join("blocks.bin"))?;
230        for block in &self.blocks {
231            for &id in block {
232                file.write_all(&id.to_le_bytes())?;
233            }
234        }
235        Ok(())
236    }
237
238    /// Load previously cached token blocks from `dir`.
239    ///
240    /// # Errors
241    /// Returns an error if metadata or `blocks.bin` is missing or truncated.
242    pub fn load_from_disk(dir: &Path) -> Result<Self> {
243        let meta_raw = fs::read_to_string(dir.join("metadata.json"))?;
244        let meta: DatasetCacheMeta = serde_json::from_str(&meta_raw)?;
245        let mut file = File::open(dir.join("blocks.bin"))?;
246        let mut raw = Vec::new();
247        file.read_to_end(&mut raw)?;
248
249        let expected_bytes = meta.num_blocks * meta.block_size * std::mem::size_of::<u32>();
250        if raw.len() != expected_bytes {
251            return Err(TrainError::InvalidArgument(format!(
252                "corrupted dataset cache `{}`: expected {expected_bytes} bytes, got {}",
253                dir.display(),
254                raw.len()
255            )));
256        }
257
258        let mut blocks = Vec::with_capacity(meta.num_blocks);
259        for chunk in raw.chunks_exact(meta.block_size * 4) {
260            let mut block = Vec::with_capacity(meta.block_size);
261            for bytes in chunk.chunks_exact(4) {
262                block.push(u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]));
263            }
264            blocks.push(block);
265        }
266
267        Ok(Self {
268            blocks,
269            block_size: meta.block_size,
270            cache_dir: Some(dir.to_path_buf()),
271        })
272    }
273}
274
275/// Extract text documents from a local `.txt`, `.jsonl`, or `.jsonl.gz` file.
276///
277/// # Errors
278/// Returns an error if reading or gzip decoding fails.
279pub fn load_local_texts(path: &Path) -> Result<Vec<String>> {
280    let ext_gz = path
281        .file_name()
282        .and_then(|n| n.to_str())
283        .is_some_and(|s| s.ends_with(".gz"));
284
285    if ext_gz {
286        let file = File::open(path)?;
287        let decoder = GzDecoder::new(file);
288        read_lines_or_jsonl(BufReader::new(decoder), true)
289    } else {
290        let is_jsonl = path
291            .extension()
292            .and_then(|e| e.to_str())
293            .is_some_and(|e| e.eq_ignore_ascii_case("jsonl"));
294        if is_jsonl {
295            let file = File::open(path)?;
296            read_lines_or_jsonl(BufReader::new(file), true)
297        } else {
298            let text = fs::read_to_string(path)?;
299            Ok(vec![text])
300        }
301    }
302}
303
304fn read_lines_or_jsonl<R: BufRead>(reader: R, parse_json: bool) -> Result<Vec<String>> {
305    let mut texts = Vec::new();
306    for line in reader.lines() {
307        let line = line?;
308        let trimmed = line.trim();
309        if trimmed.is_empty() {
310            continue;
311        }
312        if parse_json {
313            if let Ok(val) = serde_json::from_str::<serde_json::Value>(trimmed) {
314                if let Some(txt) = val
315                    .get("text")
316                    .or_else(|| val.get("sentence"))
317                    .or_else(|| val.get("content"))
318                    .and_then(serde_json::Value::as_str)
319                {
320                    if !txt.is_empty() {
321                        texts.push(txt.to_string());
322                    }
323                    continue;
324                }
325            }
326        }
327        texts.push(trimmed.to_string());
328    }
329    Ok(texts)
330}
331
332fn tokenize_texts(tokenizer: &Tokenizer, texts: &[String]) -> Result<Vec<Vec<u32>>> {
333    let mut streams = Vec::with_capacity(texts.len());
334    for text in texts {
335        let enc = tokenizer
336            .encode(text.as_str(), false)
337            .map_err(|err| TrainError::Tokenizer(format!("tokenization failed: {err}")))?;
338        streams.push(enc.get_ids().to_vec());
339    }
340    Ok(streams)
341}
342
343fn fetch_hf_rows(dataset: &str, config: &str, split: &str, max_rows: usize) -> Result<Vec<String>> {
344    let mut texts = Vec::new();
345    let mut offset = 0usize;
346    let page_size = 100usize;
347
348    while texts.len() < max_rows {
349        let length = page_size.min(max_rows - texts.len());
350        let url = format!(
351            "https://datasets-server.huggingface.co/rows?dataset={dataset}&config={config}&split={split}&offset={offset}&length={length}"
352        );
353        let body: serde_json::Value = ureq::get(&url)
354            .call()
355            .map_err(|err| TrainError::Http(format!("GET {url} failed: {err}")))?
356            .body_mut()
357            .read_json()
358            .map_err(|err| TrainError::Http(format!("JSON decode from {url} failed: {err}")))?;
359
360        let Some(rows) = body.get("rows").and_then(serde_json::Value::as_array) else {
361            break;
362        };
363        if rows.is_empty() {
364            break;
365        }
366        for row_entry in rows {
367            if let Some(txt) = row_entry
368                .get("row")
369                .and_then(|r| r.get("text").or_else(|| r.get("sentence")))
370                .and_then(serde_json::Value::as_str)
371            {
372                texts.push(txt.to_string());
373            }
374        }
375        offset += rows.len();
376        if rows.len() < length {
377            break;
378        }
379    }
380    Ok(texts)
381}
382
383fn fetch_c4_shard_texts(
384    filename: &str,
385    max_docs: usize,
386    cache_dir: Option<&Path>,
387    token: Option<&str>,
388) -> Result<Vec<String>> {
389    let mut builder = HFClient::builder();
390    if let Some(t) = token {
391        builder = builder.token(t);
392    }
393    if let Some(c) = cache_dir {
394        builder = builder.cache_dir(c);
395    }
396    let client = builder.build_sync()?;
397    let path = client
398        .dataset("allenai", "c4")
399        .download_file()
400        .filename(filename)
401        .send()?;
402
403    let file = File::open(&path)?;
404    let decoder = GzDecoder::new(file);
405    let mut texts = Vec::new();
406    for line in BufReader::new(decoder).lines() {
407        let line = line?;
408        if let Ok(val) = serde_json::from_str::<serde_json::Value>(&line) {
409            if let Some(t) = val.get("text").and_then(serde_json::Value::as_str) {
410                texts.push(t.to_string());
411                if texts.len() >= max_docs {
412                    break;
413                }
414            }
415        }
416    }
417    Ok(texts)
418}
419
420/// Prepare or load cached training token blocks matching `utils/datautils.py::prepare_dataset`.
421///
422/// # Errors
423/// Returns an error if downloading, local reading, tokenization, or disk caching fails.
424#[allow(clippy::too_many_arguments)]
425pub fn prepare_dataset(
426    kind: &DatasetKind,
427    data_root: &Path,
428    tokenizer: &Tokenizer,
429    block_size: usize,
430    max_remote_docs: Option<usize>,
431    cache_dir: Option<&Path>,
432    token: Option<&str>,
433) -> Result<TokenBlockDataset> {
434    let tok_desc = tokenizer
435        .to_string(false)
436        .map_err(|err| TrainError::Tokenizer(format!("serialize tokenizer failed: {err}")))?;
437    let hash_value = tokenizer_cache_key(&tok_desc);
438    let ds_dir = data_root.join(kind.cache_name()).join(&hash_value);
439
440    if ds_dir.join("blocks.bin").is_file() && ds_dir.join("metadata.json").is_file() {
441        if let Ok(ds) = TokenBlockDataset::load_from_disk(&ds_dir) {
442            if ds.block_size == block_size {
443                info!("Loaded cached dataset from {}", ds_dir.display());
444                return Ok(ds);
445            }
446        }
447    }
448
449    let doc_limit = max_remote_docs.unwrap_or(usize::MAX);
450    let texts = match kind {
451        DatasetKind::LocalFile(p) => load_local_texts(p)?,
452        DatasetKind::WikiText2 => {
453            let rows = fetch_hf_rows(
454                "Salesforce/wikitext",
455                "wikitext-2-raw-v1",
456                "train",
457                doc_limit,
458            )?;
459            vec![rows.join("\n\n")]
460        }
461        DatasetKind::C4 => fetch_c4_shard_texts(
462            "en/c4-train.00000-of-01024.json.gz",
463            doc_limit,
464            cache_dir,
465            token,
466        )?,
467        DatasetKind::C4Wiki => {
468            let mut c4 = fetch_c4_shard_texts(
469                "en/c4-train.00000-of-01024.json.gz",
470                doc_limit,
471                cache_dir,
472                token,
473            )?;
474            let wt = fetch_hf_rows(
475                "Salesforce/wikitext",
476                "wikitext-2-raw-v1",
477                "train",
478                doc_limit.min(50_000),
479            )?;
480            c4.push(wt.join("\n\n"));
481            c4
482        }
483        DatasetKind::Ptb => {
484            let rows = fetch_hf_rows("ptb_text_only", "penn_treebank", "validation", doc_limit)?;
485            vec![rows.join("\n\n")]
486        }
487        DatasetKind::PtbNew => {
488            let rows = fetch_hf_rows("ptb_text_only", "penn_treebank", "test", doc_limit)?;
489            vec![rows.join(" ")]
490        }
491        DatasetKind::C4New => {
492            let rows = fetch_c4_shard_texts(
493                "en/c4-validation.00000-of-00008.json.gz",
494                1100.min(doc_limit),
495                cache_dir,
496                token,
497            )?;
498            vec![rows.join(" ")]
499        }
500    };
501
502    let streams = tokenize_texts(tokenizer, &texts)?;
503    let blocks = group_token_blocks(&streams, block_size);
504    let ds = TokenBlockDataset {
505        blocks,
506        block_size,
507        cache_dir: Some(ds_dir.clone()),
508    };
509    ds.save_to_disk(&ds_dir, kind.cache_name(), &tok_desc)?;
510    info!("Saved dataset cache to {}", ds_dir.display());
511    Ok(ds)
512}
513
514/// Build a single contiguous evaluation token vector for perplexity measurement (`get_eval_loaders`).
515///
516/// # Errors
517/// Returns an error if loading or tokenizing fails.
518pub fn load_eval_tokens(
519    kind: &DatasetKind,
520    tokenizer: &Tokenizer,
521    seqlen: usize,
522    cache_dir: Option<&Path>,
523    token: Option<&str>,
524) -> Result<Vec<u32>> {
525    match kind {
526        DatasetKind::LocalFile(p) => {
527            let texts = load_local_texts(p)?;
528            let joined = texts.join("\n\n");
529            let enc = tokenizer
530                .encode(joined.as_str(), false)
531                .map_err(|err| TrainError::Tokenizer(format!("tokenization failed: {err}")))?;
532            Ok(enc.get_ids().to_vec())
533        }
534        DatasetKind::WikiText2 => {
535            let rows = fetch_hf_rows("Salesforce/wikitext", "wikitext-2-raw-v1", "test", 10_000)?;
536            let joined = rows.join("\n\n");
537            let enc = tokenizer
538                .encode(joined.as_str(), false)
539                .map_err(|err| TrainError::Tokenizer(format!("tokenization failed: {err}")))?;
540            Ok(enc.get_ids().to_vec())
541        }
542        DatasetKind::Ptb => {
543            let rows = fetch_hf_rows("ptb_text_only", "penn_treebank", "validation", 10_000)?;
544            let joined = rows.join("\n\n");
545            let enc = tokenizer
546                .encode(joined.as_str(), false)
547                .map_err(|err| TrainError::Tokenizer(format!("tokenization failed: {err}")))?;
548            Ok(enc.get_ids().to_vec())
549        }
550        DatasetKind::PtbNew => {
551            let rows = fetch_hf_rows("ptb_text_only", "penn_treebank", "test", 10_000)?;
552            let joined = rows.join(" ");
553            let enc = tokenizer
554                .encode(joined.as_str(), false)
555                .map_err(|err| TrainError::Tokenizer(format!("tokenization failed: {err}")))?;
556            Ok(enc.get_ids().to_vec())
557        }
558        DatasetKind::C4 => {
559            let val_docs = fetch_c4_shard_texts(
560                "en/c4-validation.00000-of-00008.json.gz",
561                10_000,
562                cache_dir,
563                token,
564            )?;
565            let mut rng = ChaCha8Rng::seed_from_u64(0);
566            let mut out = Vec::with_capacity(256 * seqlen);
567            let mut attempts = 0usize;
568            for _ in 0..256 {
569                loop {
570                    attempts += 1;
571                    if attempts > 10_000 || val_docs.is_empty() {
572                        break;
573                    }
574                    let doc_idx = rng.random_range(0..val_docs.len());
575                    let enc = tokenizer
576                        .encode(val_docs[doc_idx].as_str(), false)
577                        .map_err(|err| {
578                            TrainError::Tokenizer(format!("tokenization failed: {err}"))
579                        })?;
580                    let ids = enc.get_ids();
581                    if ids.len() > seqlen {
582                        let start = rng.random_range(0..(ids.len() - seqlen));
583                        out.extend_from_slice(&ids[start..start + seqlen]);
584                        break;
585                    }
586                }
587            }
588            Ok(out)
589        }
590        DatasetKind::C4New => {
591            let val_docs = fetch_c4_shard_texts(
592                "en/c4-validation.00000-of-00008.json.gz",
593                1100,
594                cache_dir,
595                token,
596            )?;
597            let joined = val_docs.join(" ");
598            let enc = tokenizer
599                .encode(joined.as_str(), false)
600                .map_err(|err| TrainError::Tokenizer(format!("tokenization failed: {err}")))?;
601            let mut ids = enc.get_ids().to_vec();
602            ids.truncate(256 * seqlen);
603            Ok(ids)
604        }
605        DatasetKind::C4Wiki => {
606            let rows = fetch_hf_rows("Salesforce/wikitext", "wikitext-2-raw-v1", "test", 10_000)?;
607            let joined = rows.join("\n\n");
608            let enc = tokenizer
609                .encode(joined.as_str(), false)
610                .map_err(|err| TrainError::Tokenizer(format!("tokenization failed: {err}")))?;
611            Ok(enc.get_ids().to_vec())
612        }
613    }
614}
615
616#[cfg(test)]
617mod tests {
618    use super::*;
619    use tempfile::TempDir;
620
621    #[test]
622    fn group_token_blocks_drops_short_remainder() {
623        let streams = vec![vec![1, 2, 3], vec![4, 5, 6, 7], vec![8, 9, 10]];
624        let blocks = group_token_blocks(&streams, 4);
625        assert_eq!(blocks, vec![vec![1, 2, 3, 4], vec![5, 6, 7, 8]]);
626    }
627
628    #[test]
629    fn tokenizer_cache_key_is_deterministic_7_hex_chars() {
630        let k1 = tokenizer_cache_key("LlamaTokenizer(vocab_size=32000)");
631        let k2 = tokenizer_cache_key("LlamaTokenizer(vocab_size=32000)");
632        let k3 = tokenizer_cache_key("LlamaTokenizer(vocab_size=128256)");
633        assert_eq!(k1.len(), 7);
634        assert_eq!(k1, k2);
635        assert_ne!(k1, k3);
636        assert!(k1.chars().all(|c| c.is_ascii_hexdigit()));
637    }
638
639    #[test]
640    fn token_block_dataset_disk_cache_round_trip() -> Result<()> {
641        let dir = TempDir::new().expect("tempdir");
642        let blocks = vec![vec![10, 20, 30, 40], vec![50, 60, 70, 80]];
643        let ds = TokenBlockDataset::from_blocks(blocks.clone(), 4);
644        ds.save_to_disk(dir.path(), "wikitext2", "dummy_tok")?;
645        assert!(dir.path().join("tokenizer_info").exists());
646
647        let loaded = TokenBlockDataset::load_from_disk(dir.path())?;
648        assert_eq!(loaded.block_size, 4);
649        assert_eq!(loaded.blocks, blocks);
650        Ok(())
651    }
652}