pub fn prepare_dataset(
kind: &DatasetKind,
data_root: &Path,
tokenizer: &Tokenizer,
block_size: usize,
max_remote_docs: Option<usize>,
cache_dir: Option<&Path>,
token: Option<&str>,
) -> Result<TokenBlockDataset>Expand description
Prepare or load cached training token blocks matching utils/datautils.py::prepare_dataset.
ยงErrors
Returns an error if downloading, local reading, tokenization, or disk caching fails.