1use std::fmt;
10use std::fs::{self, File};
11use std::io::{BufRead, BufReader, Read, Write};
12use std::path::{Path, PathBuf};
13use std::str::FromStr;
14
15use flate2::read::GzDecoder;
16use hf_hub::{split_id, HFClient};
17use rand::Rng;
18use rand::SeedableRng;
19use rand_chacha::ChaCha8Rng;
20use serde::{Deserialize, Serialize};
21use sha2::{Digest, Sha256};
22use tokenizers::Tokenizer;
23use tracing::info;
24
25use littlebit_models::resolve_model_dir;
26
27use crate::error::{Result, TrainError};
28
29pub const DEFAULT_BLOCK_SIZE: usize = 2048;
31
32#[derive(Debug, Clone, PartialEq, Eq)]
34pub enum DatasetKind {
35 WikiText2,
37 C4,
39 C4Wiki,
41 Ptb,
43 PtbNew,
45 C4New,
47 LocalFile(PathBuf),
49}
50
51impl DatasetKind {
52 #[must_use]
54 pub fn cache_name(&self) -> &str {
55 match self {
56 Self::WikiText2 => "wikitext2",
57 Self::C4 => "c4",
58 Self::C4Wiki => "c4_wiki",
59 Self::Ptb => "ptb",
60 Self::PtbNew => "ptb_new",
61 Self::C4New => "c4_new",
62 Self::LocalFile(_) => "local",
63 }
64 }
65}
66
67impl fmt::Display for DatasetKind {
68 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
69 match self {
70 Self::LocalFile(p) => write!(f, "{}", p.display()),
71 _ => f.write_str(self.cache_name()),
72 }
73 }
74}
75
76impl FromStr for DatasetKind {
77 type Err = TrainError;
78
79 fn from_str(s: &str) -> Result<Self> {
80 match s.trim().to_ascii_lowercase().as_str() {
81 "wikitext2" => Ok(Self::WikiText2),
82 "c4" => Ok(Self::C4),
83 "c4_wiki" => Ok(Self::C4Wiki),
84 "ptb" => Ok(Self::Ptb),
85 "ptb_new" | "ptb-new" => Ok(Self::PtbNew),
86 "c4_new" | "c4-new" => Ok(Self::C4New),
87 _ => {
88 let p = PathBuf::from(s);
89 if p.exists() {
90 Ok(Self::LocalFile(p))
91 } else {
92 Err(TrainError::InvalidArgument(format!(
93 "unknown dataset `{s}` (expected wikitext2, c4, c4_wiki, ptb, ptb_new, c4_new, or an existing file path)"
94 )))
95 }
96 }
97 }
98 }
99}
100
101#[must_use]
104pub fn tokenizer_cache_key(tokenizer_descriptor: &str) -> String {
105 let digest = Sha256::digest(tokenizer_descriptor.as_bytes());
106 let full = hex::encode(digest);
107 full[..7].to_string()
108}
109
110#[must_use]
113pub fn group_token_blocks(streams: &[Vec<u32>], block_size: usize) -> Vec<Vec<u32>> {
114 if block_size == 0 {
115 return Vec::new();
116 }
117 let total_tokens: usize = streams.iter().map(Vec::len).sum();
118 let usable = (total_tokens / block_size) * block_size;
119 if usable == 0 {
120 return Vec::new();
121 }
122
123 let mut flat = Vec::with_capacity(usable);
124 for stream in streams {
125 let remaining = usable - flat.len();
126 if remaining == 0 {
127 break;
128 }
129 let take = stream.len().min(remaining);
130 flat.extend_from_slice(&stream[..take]);
131 }
132 flat.chunks_exact(block_size).map(Vec::from).collect()
133}
134
135pub fn load_tokenizer(
140 model_id_or_path: &str,
141 cache_dir: Option<&Path>,
142 token: Option<&str>,
143 local_files_only: bool,
144) -> Result<Tokenizer> {
145 let local_tok = Path::new(model_id_or_path).join("tokenizer.json");
146 let tok_path = if local_tok.is_file() {
147 local_tok
148 } else if Path::new(model_id_or_path).is_dir() {
149 return Err(TrainError::Tokenizer(format!(
150 "directory `{model_id_or_path}` does not contain `tokenizer.json`"
151 )));
152 } else {
153 let dir = resolve_model_dir(model_id_or_path, cache_dir, token, local_files_only)?;
154 let cached = dir.join("tokenizer.json");
155 if cached.is_file() {
156 cached
157 } else {
158 let mut builder = HFClient::builder();
159 if let Some(t) = token {
160 builder = builder.token(t);
161 }
162 if let Some(c) = cache_dir {
163 builder = builder.cache_dir(c);
164 }
165 let client = builder.build_sync()?;
166 let (owner, name) = split_id(model_id_or_path);
167 client
168 .model(owner, name)
169 .download_file()
170 .filename("tokenizer.json")
171 .local_files_only(local_files_only)
172 .send()?
173 }
174 };
175
176 Tokenizer::from_file(&tok_path)
177 .map_err(|err| TrainError::Tokenizer(format!("failed to load tokenizer: {err}")))
178}
179
180#[derive(Debug, Serialize, Deserialize)]
181struct DatasetCacheMeta {
182 dataset: String,
183 block_size: usize,
184 num_blocks: usize,
185 tokenizer_hash: String,
186}
187
188#[derive(Debug, Clone)]
190pub struct TokenBlockDataset {
191 pub blocks: Vec<Vec<u32>>,
193 pub block_size: usize,
195 pub cache_dir: Option<PathBuf>,
197}
198
199impl TokenBlockDataset {
200 #[must_use]
202 pub fn from_blocks(blocks: Vec<Vec<u32>>, block_size: usize) -> Self {
203 Self {
204 blocks,
205 block_size,
206 cache_dir: None,
207 }
208 }
209
210 pub fn save_to_disk(&self, dir: &Path, dataset_name: &str, tokenizer_info: &str) -> Result<()> {
215 fs::create_dir_all(dir)?;
216 let hash = tokenizer_cache_key(tokenizer_info);
217 let meta = DatasetCacheMeta {
218 dataset: dataset_name.to_string(),
219 block_size: self.block_size,
220 num_blocks: self.blocks.len(),
221 tokenizer_hash: hash,
222 };
223 fs::write(
224 dir.join("metadata.json"),
225 serde_json::to_string_pretty(&meta)?,
226 )?;
227 fs::write(dir.join("tokenizer_info"), tokenizer_info)?;
228
229 let mut file = File::create(dir.join("blocks.bin"))?;
230 for block in &self.blocks {
231 for &id in block {
232 file.write_all(&id.to_le_bytes())?;
233 }
234 }
235 Ok(())
236 }
237
238 pub fn load_from_disk(dir: &Path) -> Result<Self> {
243 let meta_raw = fs::read_to_string(dir.join("metadata.json"))?;
244 let meta: DatasetCacheMeta = serde_json::from_str(&meta_raw)?;
245 let mut file = File::open(dir.join("blocks.bin"))?;
246 let mut raw = Vec::new();
247 file.read_to_end(&mut raw)?;
248
249 let expected_bytes = meta.num_blocks * meta.block_size * std::mem::size_of::<u32>();
250 if raw.len() != expected_bytes {
251 return Err(TrainError::InvalidArgument(format!(
252 "corrupted dataset cache `{}`: expected {expected_bytes} bytes, got {}",
253 dir.display(),
254 raw.len()
255 )));
256 }
257
258 let mut blocks = Vec::with_capacity(meta.num_blocks);
259 for chunk in raw.chunks_exact(meta.block_size * 4) {
260 let mut block = Vec::with_capacity(meta.block_size);
261 for bytes in chunk.chunks_exact(4) {
262 block.push(u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]));
263 }
264 blocks.push(block);
265 }
266
267 Ok(Self {
268 blocks,
269 block_size: meta.block_size,
270 cache_dir: Some(dir.to_path_buf()),
271 })
272 }
273}
274
275pub fn load_local_texts(path: &Path) -> Result<Vec<String>> {
280 let ext_gz = path
281 .file_name()
282 .and_then(|n| n.to_str())
283 .is_some_and(|s| s.ends_with(".gz"));
284
285 if ext_gz {
286 let file = File::open(path)?;
287 let decoder = GzDecoder::new(file);
288 read_lines_or_jsonl(BufReader::new(decoder), true)
289 } else {
290 let is_jsonl = path
291 .extension()
292 .and_then(|e| e.to_str())
293 .is_some_and(|e| e.eq_ignore_ascii_case("jsonl"));
294 if is_jsonl {
295 let file = File::open(path)?;
296 read_lines_or_jsonl(BufReader::new(file), true)
297 } else {
298 let text = fs::read_to_string(path)?;
299 Ok(vec![text])
300 }
301 }
302}
303
304fn read_lines_or_jsonl<R: BufRead>(reader: R, parse_json: bool) -> Result<Vec<String>> {
305 let mut texts = Vec::new();
306 for line in reader.lines() {
307 let line = line?;
308 let trimmed = line.trim();
309 if trimmed.is_empty() {
310 continue;
311 }
312 if parse_json {
313 if let Ok(val) = serde_json::from_str::<serde_json::Value>(trimmed) {
314 if let Some(txt) = val
315 .get("text")
316 .or_else(|| val.get("sentence"))
317 .or_else(|| val.get("content"))
318 .and_then(serde_json::Value::as_str)
319 {
320 if !txt.is_empty() {
321 texts.push(txt.to_string());
322 }
323 continue;
324 }
325 }
326 }
327 texts.push(trimmed.to_string());
328 }
329 Ok(texts)
330}
331
332fn tokenize_texts(tokenizer: &Tokenizer, texts: &[String]) -> Result<Vec<Vec<u32>>> {
333 let mut streams = Vec::with_capacity(texts.len());
334 for text in texts {
335 let enc = tokenizer
336 .encode(text.as_str(), false)
337 .map_err(|err| TrainError::Tokenizer(format!("tokenization failed: {err}")))?;
338 streams.push(enc.get_ids().to_vec());
339 }
340 Ok(streams)
341}
342
343fn fetch_hf_rows(dataset: &str, config: &str, split: &str, max_rows: usize) -> Result<Vec<String>> {
344 let mut texts = Vec::new();
345 let mut offset = 0usize;
346 let page_size = 100usize;
347
348 while texts.len() < max_rows {
349 let length = page_size.min(max_rows - texts.len());
350 let url = format!(
351 "https://datasets-server.huggingface.co/rows?dataset={dataset}&config={config}&split={split}&offset={offset}&length={length}"
352 );
353 let body: serde_json::Value = ureq::get(&url)
354 .call()
355 .map_err(|err| TrainError::Http(format!("GET {url} failed: {err}")))?
356 .body_mut()
357 .read_json()
358 .map_err(|err| TrainError::Http(format!("JSON decode from {url} failed: {err}")))?;
359
360 let Some(rows) = body.get("rows").and_then(serde_json::Value::as_array) else {
361 break;
362 };
363 if rows.is_empty() {
364 break;
365 }
366 for row_entry in rows {
367 if let Some(txt) = row_entry
368 .get("row")
369 .and_then(|r| r.get("text").or_else(|| r.get("sentence")))
370 .and_then(serde_json::Value::as_str)
371 {
372 texts.push(txt.to_string());
373 }
374 }
375 offset += rows.len();
376 if rows.len() < length {
377 break;
378 }
379 }
380 Ok(texts)
381}
382
383fn fetch_c4_shard_texts(
384 filename: &str,
385 max_docs: usize,
386 cache_dir: Option<&Path>,
387 token: Option<&str>,
388) -> Result<Vec<String>> {
389 let mut builder = HFClient::builder();
390 if let Some(t) = token {
391 builder = builder.token(t);
392 }
393 if let Some(c) = cache_dir {
394 builder = builder.cache_dir(c);
395 }
396 let client = builder.build_sync()?;
397 let path = client
398 .dataset("allenai", "c4")
399 .download_file()
400 .filename(filename)
401 .send()?;
402
403 let file = File::open(&path)?;
404 let decoder = GzDecoder::new(file);
405 let mut texts = Vec::new();
406 for line in BufReader::new(decoder).lines() {
407 let line = line?;
408 if let Ok(val) = serde_json::from_str::<serde_json::Value>(&line) {
409 if let Some(t) = val.get("text").and_then(serde_json::Value::as_str) {
410 texts.push(t.to_string());
411 if texts.len() >= max_docs {
412 break;
413 }
414 }
415 }
416 }
417 Ok(texts)
418}
419
420#[allow(clippy::too_many_arguments)]
425pub fn prepare_dataset(
426 kind: &DatasetKind,
427 data_root: &Path,
428 tokenizer: &Tokenizer,
429 block_size: usize,
430 max_remote_docs: Option<usize>,
431 cache_dir: Option<&Path>,
432 token: Option<&str>,
433) -> Result<TokenBlockDataset> {
434 let tok_desc = tokenizer
435 .to_string(false)
436 .map_err(|err| TrainError::Tokenizer(format!("serialize tokenizer failed: {err}")))?;
437 let hash_value = tokenizer_cache_key(&tok_desc);
438 let ds_dir = data_root.join(kind.cache_name()).join(&hash_value);
439
440 if ds_dir.join("blocks.bin").is_file() && ds_dir.join("metadata.json").is_file() {
441 if let Ok(ds) = TokenBlockDataset::load_from_disk(&ds_dir) {
442 if ds.block_size == block_size {
443 info!("Loaded cached dataset from {}", ds_dir.display());
444 return Ok(ds);
445 }
446 }
447 }
448
449 let doc_limit = max_remote_docs.unwrap_or(usize::MAX);
450 let texts = match kind {
451 DatasetKind::LocalFile(p) => load_local_texts(p)?,
452 DatasetKind::WikiText2 => {
453 let rows = fetch_hf_rows(
454 "Salesforce/wikitext",
455 "wikitext-2-raw-v1",
456 "train",
457 doc_limit,
458 )?;
459 vec![rows.join("\n\n")]
460 }
461 DatasetKind::C4 => fetch_c4_shard_texts(
462 "en/c4-train.00000-of-01024.json.gz",
463 doc_limit,
464 cache_dir,
465 token,
466 )?,
467 DatasetKind::C4Wiki => {
468 let mut c4 = fetch_c4_shard_texts(
469 "en/c4-train.00000-of-01024.json.gz",
470 doc_limit,
471 cache_dir,
472 token,
473 )?;
474 let wt = fetch_hf_rows(
475 "Salesforce/wikitext",
476 "wikitext-2-raw-v1",
477 "train",
478 doc_limit.min(50_000),
479 )?;
480 c4.push(wt.join("\n\n"));
481 c4
482 }
483 DatasetKind::Ptb => {
484 let rows = fetch_hf_rows("ptb_text_only", "penn_treebank", "validation", doc_limit)?;
485 vec![rows.join("\n\n")]
486 }
487 DatasetKind::PtbNew => {
488 let rows = fetch_hf_rows("ptb_text_only", "penn_treebank", "test", doc_limit)?;
489 vec![rows.join(" ")]
490 }
491 DatasetKind::C4New => {
492 let rows = fetch_c4_shard_texts(
493 "en/c4-validation.00000-of-00008.json.gz",
494 1100.min(doc_limit),
495 cache_dir,
496 token,
497 )?;
498 vec![rows.join(" ")]
499 }
500 };
501
502 let streams = tokenize_texts(tokenizer, &texts)?;
503 let blocks = group_token_blocks(&streams, block_size);
504 let ds = TokenBlockDataset {
505 blocks,
506 block_size,
507 cache_dir: Some(ds_dir.clone()),
508 };
509 ds.save_to_disk(&ds_dir, kind.cache_name(), &tok_desc)?;
510 info!("Saved dataset cache to {}", ds_dir.display());
511 Ok(ds)
512}
513
514pub fn load_eval_tokens(
519 kind: &DatasetKind,
520 tokenizer: &Tokenizer,
521 seqlen: usize,
522 cache_dir: Option<&Path>,
523 token: Option<&str>,
524) -> Result<Vec<u32>> {
525 match kind {
526 DatasetKind::LocalFile(p) => {
527 let texts = load_local_texts(p)?;
528 let joined = texts.join("\n\n");
529 let enc = tokenizer
530 .encode(joined.as_str(), false)
531 .map_err(|err| TrainError::Tokenizer(format!("tokenization failed: {err}")))?;
532 Ok(enc.get_ids().to_vec())
533 }
534 DatasetKind::WikiText2 => {
535 let rows = fetch_hf_rows("Salesforce/wikitext", "wikitext-2-raw-v1", "test", 10_000)?;
536 let joined = rows.join("\n\n");
537 let enc = tokenizer
538 .encode(joined.as_str(), false)
539 .map_err(|err| TrainError::Tokenizer(format!("tokenization failed: {err}")))?;
540 Ok(enc.get_ids().to_vec())
541 }
542 DatasetKind::Ptb => {
543 let rows = fetch_hf_rows("ptb_text_only", "penn_treebank", "validation", 10_000)?;
544 let joined = rows.join("\n\n");
545 let enc = tokenizer
546 .encode(joined.as_str(), false)
547 .map_err(|err| TrainError::Tokenizer(format!("tokenization failed: {err}")))?;
548 Ok(enc.get_ids().to_vec())
549 }
550 DatasetKind::PtbNew => {
551 let rows = fetch_hf_rows("ptb_text_only", "penn_treebank", "test", 10_000)?;
552 let joined = rows.join(" ");
553 let enc = tokenizer
554 .encode(joined.as_str(), false)
555 .map_err(|err| TrainError::Tokenizer(format!("tokenization failed: {err}")))?;
556 Ok(enc.get_ids().to_vec())
557 }
558 DatasetKind::C4 => {
559 let val_docs = fetch_c4_shard_texts(
560 "en/c4-validation.00000-of-00008.json.gz",
561 10_000,
562 cache_dir,
563 token,
564 )?;
565 let mut rng = ChaCha8Rng::seed_from_u64(0);
566 let mut out = Vec::with_capacity(256 * seqlen);
567 let mut attempts = 0usize;
568 for _ in 0..256 {
569 loop {
570 attempts += 1;
571 if attempts > 10_000 || val_docs.is_empty() {
572 break;
573 }
574 let doc_idx = rng.random_range(0..val_docs.len());
575 let enc = tokenizer
576 .encode(val_docs[doc_idx].as_str(), false)
577 .map_err(|err| {
578 TrainError::Tokenizer(format!("tokenization failed: {err}"))
579 })?;
580 let ids = enc.get_ids();
581 if ids.len() > seqlen {
582 let start = rng.random_range(0..(ids.len() - seqlen));
583 out.extend_from_slice(&ids[start..start + seqlen]);
584 break;
585 }
586 }
587 }
588 Ok(out)
589 }
590 DatasetKind::C4New => {
591 let val_docs = fetch_c4_shard_texts(
592 "en/c4-validation.00000-of-00008.json.gz",
593 1100,
594 cache_dir,
595 token,
596 )?;
597 let joined = val_docs.join(" ");
598 let enc = tokenizer
599 .encode(joined.as_str(), false)
600 .map_err(|err| TrainError::Tokenizer(format!("tokenization failed: {err}")))?;
601 let mut ids = enc.get_ids().to_vec();
602 ids.truncate(256 * seqlen);
603 Ok(ids)
604 }
605 DatasetKind::C4Wiki => {
606 let rows = fetch_hf_rows("Salesforce/wikitext", "wikitext-2-raw-v1", "test", 10_000)?;
607 let joined = rows.join("\n\n");
608 let enc = tokenizer
609 .encode(joined.as_str(), false)
610 .map_err(|err| TrainError::Tokenizer(format!("tokenization failed: {err}")))?;
611 Ok(enc.get_ids().to_vec())
612 }
613 }
614}
615
616#[cfg(test)]
617mod tests {
618 use super::*;
619 use tempfile::TempDir;
620
621 #[test]
622 fn group_token_blocks_drops_short_remainder() {
623 let streams = vec![vec![1, 2, 3], vec![4, 5, 6, 7], vec![8, 9, 10]];
624 let blocks = group_token_blocks(&streams, 4);
625 assert_eq!(blocks, vec![vec![1, 2, 3, 4], vec![5, 6, 7, 8]]);
626 }
627
628 #[test]
629 fn tokenizer_cache_key_is_deterministic_7_hex_chars() {
630 let k1 = tokenizer_cache_key("LlamaTokenizer(vocab_size=32000)");
631 let k2 = tokenizer_cache_key("LlamaTokenizer(vocab_size=32000)");
632 let k3 = tokenizer_cache_key("LlamaTokenizer(vocab_size=128256)");
633 assert_eq!(k1.len(), 7);
634 assert_eq!(k1, k2);
635 assert_ne!(k1, k3);
636 assert!(k1.chars().all(|c| c.is_ascii_hexdigit()));
637 }
638
639 #[test]
640 fn token_block_dataset_disk_cache_round_trip() -> Result<()> {
641 let dir = TempDir::new().expect("tempdir");
642 let blocks = vec![vec![10, 20, 30, 40], vec![50, 60, 70, 80]];
643 let ds = TokenBlockDataset::from_blocks(blocks.clone(), 4);
644 ds.save_to_disk(dir.path(), "wikitext2", "dummy_tok")?;
645 assert!(dir.path().join("tokenizer_info").exists());
646
647 let loaded = TokenBlockDataset::load_from_disk(dir.path())?;
648 assert_eq!(loaded.block_size, 4);
649 assert_eq!(loaded.blocks, blocks);
650 Ok(())
651 }
652}