Skip to main content

littlebit_train/
zero_config.rs

1//! Parser for upstream DeepSpeed configuration files (`configs/zero*.json`).
2
3use std::fs;
4use std::path::Path;
5
6use serde::{Deserialize, Serialize};
7
8use crate::error::Result;
9
10/// BF16 section inside a DeepSpeed config JSON.
11#[derive(Debug, Clone, Default, Serialize, Deserialize)]
12pub struct ZeroBf16Config {
13    /// Whether bfloat16 training/inference is enabled.
14    #[serde(default)]
15    pub enabled: bool,
16}
17
18/// Scheduler parameters inside a DeepSpeed config JSON.
19#[derive(Debug, Clone, Default, Serialize, Deserialize)]
20pub struct ZeroSchedulerParams {
21    /// Minimum cosine decay ratio (`cos_min_ratio`).
22    pub cos_min_ratio: Option<f64>,
23    /// Linear warmup min ratio.
24    pub warmup_min_ratio: Option<f64>,
25    /// Warmup schedule type (`linear`, etc.).
26    pub warmup_type: Option<String>,
27}
28
29/// Scheduler block inside a DeepSpeed config JSON.
30#[derive(Debug, Clone, Default, Serialize, Deserialize)]
31pub struct ZeroSchedulerConfig {
32    /// Scheduler class name (e.g. `WarmupCosineLR`).
33    #[serde(rename = "type")]
34    pub kind: Option<String>,
35    /// Scheduler parameters.
36    #[serde(default)]
37    pub params: ZeroSchedulerParams,
38}
39
40/// ZeRO optimization block inside a DeepSpeed config JSON.
41#[derive(Debug, Clone, Default, Serialize, Deserialize)]
42pub struct ZeroOptimizationConfig {
43    /// ZeRO stage (`0`, `1`, `2`, or `3`).
44    #[serde(default)]
45    pub stage: u8,
46}
47
48/// Single-node compatible representation of `configs/zero*.json`.
49#[derive(Debug, Clone, Default, Serialize, Deserialize)]
50pub struct ZeroConfig {
51    /// Mixed-precision bfloat16 toggle.
52    #[serde(default)]
53    pub bf16: ZeroBf16Config,
54    /// Scheduler configuration.
55    pub scheduler: Option<ZeroSchedulerConfig>,
56    /// ZeRO stage settings (logged for single-node Rust compatibility).
57    pub zero_optimization: Option<ZeroOptimizationConfig>,
58    /// Optional numeric gradient clipping threshold (`"auto"` ignored).
59    #[serde(default)]
60    pub gradient_clipping: serde_json::Value,
61    /// Optional numeric gradient accumulation steps (`"auto"` ignored).
62    #[serde(default)]
63    pub gradient_accumulation_steps: serde_json::Value,
64}
65
66impl ZeroConfig {
67    /// Load and parse a `zero*.json` configuration file.
68    ///
69    /// # Errors
70    /// Returns an error if the file cannot be read or fails JSON parsing.
71    pub fn from_file(path: &Path) -> Result<Self> {
72        let raw = fs::read_to_string(path)?;
73        let cfg: Self = serde_json::from_str(&raw)?;
74        Ok(cfg)
75    }
76
77    /// Return `cos_min_ratio` if specified, defaulting to `0.0001`.
78    #[must_use]
79    pub fn cos_min_ratio(&self) -> f64 {
80        self.scheduler
81            .as_ref()
82            .and_then(|s| s.params.cos_min_ratio)
83            .unwrap_or(0.0001)
84    }
85
86    /// Return numeric `gradient_clipping` if configured (ignoring `"auto"`).
87    #[must_use]
88    pub fn max_grad_norm(&self) -> Option<f64> {
89        self.gradient_clipping.as_f64()
90    }
91
92    /// Return numeric `gradient_accumulation_steps` if configured (ignoring `"auto"`).
93    #[must_use]
94    pub fn grad_accum_steps(&self) -> Option<usize> {
95        self.gradient_accumulation_steps
96            .as_u64()
97            .map(|n| n.max(1) as usize)
98    }
99}
100
101#[cfg(test)]
102mod tests {
103    use super::*;
104    use std::path::PathBuf;
105
106    #[test]
107    fn parses_all_reference_zero_configs() -> Result<()> {
108        let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
109            .join("../..")
110            .join("configs");
111        for name in ["zero.json", "zero3.json", "zero3_inference.json"] {
112            let cfg = ZeroConfig::from_file(&root.join(name))?;
113            assert!(cfg.bf16.enabled, "{name} should enable bf16");
114        }
115        let z3 = ZeroConfig::from_file(&root.join("zero3.json"))?;
116        assert!((z3.cos_min_ratio() - 0.0001).abs() < 1e-9);
117        Ok(())
118    }
119}