littlebit_train/
zero_config.rs1use std::fs;
4use std::path::Path;
5
6use serde::{Deserialize, Serialize};
7
8use crate::error::Result;
9
10#[derive(Debug, Clone, Default, Serialize, Deserialize)]
12pub struct ZeroBf16Config {
13 #[serde(default)]
15 pub enabled: bool,
16}
17
18#[derive(Debug, Clone, Default, Serialize, Deserialize)]
20pub struct ZeroSchedulerParams {
21 pub cos_min_ratio: Option<f64>,
23 pub warmup_min_ratio: Option<f64>,
25 pub warmup_type: Option<String>,
27}
28
29#[derive(Debug, Clone, Default, Serialize, Deserialize)]
31pub struct ZeroSchedulerConfig {
32 #[serde(rename = "type")]
34 pub kind: Option<String>,
35 #[serde(default)]
37 pub params: ZeroSchedulerParams,
38}
39
40#[derive(Debug, Clone, Default, Serialize, Deserialize)]
42pub struct ZeroOptimizationConfig {
43 #[serde(default)]
45 pub stage: u8,
46}
47
48#[derive(Debug, Clone, Default, Serialize, Deserialize)]
50pub struct ZeroConfig {
51 #[serde(default)]
53 pub bf16: ZeroBf16Config,
54 pub scheduler: Option<ZeroSchedulerConfig>,
56 pub zero_optimization: Option<ZeroOptimizationConfig>,
58 #[serde(default)]
60 pub gradient_clipping: serde_json::Value,
61 #[serde(default)]
63 pub gradient_accumulation_steps: serde_json::Value,
64}
65
66impl ZeroConfig {
67 pub fn from_file(path: &Path) -> Result<Self> {
72 let raw = fs::read_to_string(path)?;
73 let cfg: Self = serde_json::from_str(&raw)?;
74 Ok(cfg)
75 }
76
77 #[must_use]
79 pub fn cos_min_ratio(&self) -> f64 {
80 self.scheduler
81 .as_ref()
82 .and_then(|s| s.params.cos_min_ratio)
83 .unwrap_or(0.0001)
84 }
85
86 #[must_use]
88 pub fn max_grad_norm(&self) -> Option<f64> {
89 self.gradient_clipping.as_f64()
90 }
91
92 #[must_use]
94 pub fn grad_accum_steps(&self) -> Option<usize> {
95 self.gradient_accumulation_steps
96 .as_u64()
97 .map(|n| n.max(1) as usize)
98 }
99}
100
101#[cfg(test)]
102mod tests {
103 use super::*;
104 use std::path::PathBuf;
105
106 #[test]
107 fn parses_all_reference_zero_configs() -> Result<()> {
108 let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
109 .join("../..")
110 .join("configs");
111 for name in ["zero.json", "zero3.json", "zero3_inference.json"] {
112 let cfg = ZeroConfig::from_file(&root.join(name))?;
113 assert!(cfg.bf16.enabled, "{name} should enable bf16");
114 }
115 let z3 = ZeroConfig::from_file(&root.join("zero3.json"))?;
116 assert!((z3.cos_min_ratio() - 0.0001).abs() < 1e-9);
117 Ok(())
118 }
119}