Expand description
QAT Knowledge Distillation trainer, dataset tokenizer/cache, and perplexity/zero-shot evaluator.
Re-exports§
pub use data::group_token_blocks;pub use data::load_eval_tokens;pub use data::load_local_texts;pub use data::load_tokenizer;pub use data::prepare_dataset;pub use data::tokenizer_cache_key;pub use data::DatasetKind;pub use data::TokenBlockDataset;pub use data::DEFAULT_BLOCK_SIZE;pub use error::Result;pub use error::TrainError;pub use eval::evaluate_multiple_choice;pub use eval::evaluate_perplexity;pub use eval::load_multiple_choice_jsonl;pub use eval::MultipleChoiceExample;pub use eval::PplResult;pub use eval::ZeroShotTaskResult;pub use loss::compute_kd_loss;pub use loss::kl_divergence_batchmean;pub use loss::l2l_mse_loss;pub use loss::KdLossBreakdown;pub use scheduler::CosineWarmupScheduler;pub use trainer::KdTrainer;pub use trainer::StepMetrics;pub use trainer::TrainingConfig;pub use trainer::TrainingSummary;pub use zero_config::ZeroConfig;
Modules§
- data
- Dataset loading, tokenization, SHA-256 caching, and block chunking (
utils/datautils.py). - error
- Typed errors for
littlebit-train. - eval
- Perplexity and zero-shot multiple-choice evaluation (
eval.pyparity). - loss
- Knowledge distillation KL and layer-to-layer hidden-state MSE losses (
utils/kd_utils.py). - scheduler
- Cosine learning rate schedule with linear warmup (
WarmupCosineLR). - trainer
- QAT Knowledge Distillation trainer (
main.py&utils/kd_utils.pyparity). - zero_
config - Parser for upstream DeepSpeed configuration files (
configs/zero*.json).