Skip to main content

tufazip_codec/
checkpoint.rs

1use std::collections::{BTreeMap, HashMap};
2use std::fs::{self, File, OpenOptions};
3use std::io::Write;
4use std::path::{Path, PathBuf};
5
6use candle_core::{Device, Tensor};
7use serde::{Deserialize, Serialize};
8use tufazip_arith::{BitInputState, BitOutputState, DecoderState, EncoderState};
9use tufazip_prior::sha256_file;
10
11use crate::error::CodecError;
12use crate::rng::RngState;
13
14/// Checkpoint schema version.
15pub const CHECKPOINT_VERSION: u32 = 1;
16
17/// Exit code returned when stopping cleanly at a block boundary for requeue/resume.
18pub const REQUEUE_EXIT_CODE: i32 = 75;
19
20/// Sentinel filename marking successful codec completion.
21pub const DONE_NAME: &str = "DONE";
22
23/// Sentinel filename requesting clean checkpoint-and-stop at the next block boundary.
24pub const STOP_NAME: &str = "STOP_REQUESTED";
25
26const LATEST_META_NAME: &str = "latest.json";
27const LATEST_TENSORS_NAME: &str = "latest.safetensors";
28
29/// Serializable metadata written alongside `latest.safetensors` for resumable execution.
30#[derive(Debug, Clone, Serialize, Deserialize)]
31pub struct CheckpointMetadata {
32    /// Checkpoint format version (`1`).
33    pub version: u32,
34    /// Stable JSON signature of architectural and coding parameters.
35    pub config_signature: String,
36    /// Whether only target positions were projected through `head`.
37    pub project_only_targets: bool,
38    /// Early replay epoch override setting.
39    pub retrain_early_epochs: usize,
40    /// Early replay prefix threshold setting.
41    pub retrain_early_until: usize,
42    /// Input file size in bytes at start of run.
43    pub input_size: u64,
44    /// Unpadded original token count.
45    pub original_len: usize,
46    /// Padded token buffer length (multiple of `batch_size`).
47    pub file_len: usize,
48    /// SHA-256 digest of `--initial_prior` if used.
49    pub initial_prior_sha256: Option<String>,
50    /// SHA-256 digest of `--vocab_file` if used with `--initial_prior`.
51    pub initial_prior_vocab_sha256: Option<String>,
52    /// SHA-256 digest of the paired `latest.safetensors` file.
53    pub tensors_sha256: String,
54    /// Per-parameter Adam step counters for the online optimizer.
55    pub online_adam_steps: BTreeMap<String, u64>,
56    /// Per-parameter Adam step counters for the replay optimizer.
57    pub retrain_adam_steps: BTreeMap<String, u64>,
58    /// Exact PRNG state.
59    pub rng_state: RngState,
60    /// Encoder state (`Some` when compressing with coding enabled).
61    pub encoder_state: Option<EncoderState>,
62    /// Decoder state (`Some` when decompressing).
63    pub decoder_state: Option<DecoderState>,
64    /// Output bit stream state (`Some` when compressing with coding enabled).
65    pub bit_out_state: Option<BitOutputState>,
66    /// Input bit stream state (`Some` when decompressing).
67    pub bit_in_state: Option<BitInputState>,
68    /// Byte offset of the decoded token output file (`Some` when decompressing).
69    pub output_position: Option<u64>,
70    /// Cumulative coded symbol count.
71    pub n_symbols: usize,
72    /// Cumulative modeled cross-entropy bits.
73    pub n_bits: f64,
74    /// Cumulative online optimizer steps.
75    pub train_step: usize,
76    /// Cumulative prefix-replay optimizer steps.
77    pub retrain_train_step: usize,
78    /// Token index at the start of the next block.
79    pub block_start: usize,
80    /// Token index where prefix retraining last ran.
81    pub last_retrain_pos: usize,
82    /// Elapsed active compute seconds before this checkpoint.
83    pub elapsed_seconds: f64,
84    /// Reason this checkpoint was saved (`"periodic"`, `"stop-request"`, `"test-stop"`).
85    pub reason: String,
86}
87
88fn sync_parent_dir(path: &Path) -> Result<(), CodecError> {
89    if let Some(parent) = path.parent() {
90        #[cfg(unix)]
91        if let Ok(dir) = File::open(parent) {
92            let _ = dir.sync_all();
93        }
94        #[cfg(not(unix))]
95        let _ = parent;
96    }
97    Ok(())
98}
99
100/// Atomically writes `bytes` to `path` using a PID-scoped temporary file, fsync, and rename.
101///
102/// # Errors
103///
104/// Returns [`CodecError::Io`] on filesystem failure.
105pub fn write_atomic_bytes(path: &Path, bytes: &[u8]) -> Result<(), CodecError> {
106    if let Some(parent) = path.parent() {
107        fs::create_dir_all(parent)?;
108    }
109    let tmp_path = tmp_path_for(path);
110    let write_res = (|| -> Result<(), CodecError> {
111        let mut file = OpenOptions::new()
112            .create(true)
113            .truncate(true)
114            .write(true)
115            .open(&tmp_path)?;
116        file.write_all(bytes)?;
117        file.flush()?;
118        file.sync_all()?;
119        drop(file);
120        fs::rename(&tmp_path, path)?;
121        sync_parent_dir(path)?;
122        Ok(())
123    })();
124    if write_res.is_err() {
125        let _ = fs::remove_file(&tmp_path);
126    }
127    write_res
128}
129
130fn tmp_path_for(path: &Path) -> PathBuf {
131    let name = path
132        .file_name()
133        .and_then(|n| n.to_str())
134        .unwrap_or("checkpoint");
135    path.with_file_name(format!("{name}.tmp.{}", std::process::id()))
136}
137
138/// Atomically saves a tensor map in `safetensors` format to `path`.
139///
140/// # Errors
141///
142/// Returns [`CodecError`] on serialization or filesystem failure.
143pub fn write_atomic_safetensors(
144    path: &Path,
145    tensors: &HashMap<String, Tensor>,
146) -> Result<(), CodecError> {
147    if let Some(parent) = path.parent() {
148        fs::create_dir_all(parent)?;
149    }
150    let tmp_path = tmp_path_for(path);
151    let save_res = (|| -> Result<(), CodecError> {
152        candle_core::safetensors::save(tensors, &tmp_path)?;
153        if let Ok(file) = File::open(&tmp_path) {
154            let _ = file.sync_all();
155        }
156        fs::rename(&tmp_path, path)?;
157        sync_parent_dir(path)?;
158        Ok(())
159    })();
160    if save_res.is_err() {
161        let _ = fs::remove_file(&tmp_path);
162    }
163    save_res
164}
165
166/// Saves a resumable checkpoint (`latest.safetensors` + `latest.json`) atomically under `dir`.
167///
168/// # Errors
169///
170/// Returns [`CodecError`] on serialization or filesystem failure.
171pub fn save_checkpoint(
172    dir: &Path,
173    meta: &mut CheckpointMetadata,
174    tensors: &HashMap<String, Tensor>,
175) -> Result<(), CodecError> {
176    fs::create_dir_all(dir)?;
177    let tensors_path = dir.join(LATEST_TENSORS_NAME);
178    write_atomic_safetensors(&tensors_path, tensors)?;
179    meta.tensors_sha256 = sha256_file(&tensors_path)?;
180    let json = serde_json::to_vec_pretty(meta)?;
181    write_atomic_bytes(&dir.join(LATEST_META_NAME), &json)?;
182    Ok(())
183}
184
185/// Loaded checkpoint metadata and tensor map.
186#[derive(Debug)]
187pub struct LoadedCheckpoint {
188    /// Parsed checkpoint metadata.
189    pub metadata: CheckpointMetadata,
190    /// Loaded model and optimizer tensors on the target device.
191    pub tensors: HashMap<String, Tensor>,
192}
193
194/// Loads an existing checkpoint from `dir`, verifying `latest.safetensors` against `tensors_sha256`.
195///
196/// Returns `Ok(None)` if no checkpoint exists yet.
197///
198/// # Errors
199///
200/// Returns [`CodecError`] if files are malformed, incomplete, or fail integrity checks.
201pub fn load_checkpoint(
202    dir: &Path,
203    device: &Device,
204) -> Result<Option<LoadedCheckpoint>, CodecError> {
205    let meta_path = dir.join(LATEST_META_NAME);
206    if !meta_path.exists() {
207        return Ok(None);
208    }
209    let tensors_path = dir.join(LATEST_TENSORS_NAME);
210    if !tensors_path.exists() {
211        return Err(CodecError::CheckpointMismatch(format!(
212            "found {} without paired {}",
213            meta_path.display(),
214            tensors_path.display()
215        )));
216    }
217    let meta_bytes = fs::read(&meta_path)?;
218    let meta: CheckpointMetadata = serde_json::from_slice(&meta_bytes)?;
219    let actual_sha = sha256_file(&tensors_path)?;
220    if actual_sha != meta.tensors_sha256 {
221        return Err(CodecError::CheckpointMismatch(format!(
222            "checkpoint tensor digest mismatch: expected {}, found {actual_sha}",
223            meta.tensors_sha256
224        )));
225    }
226    let tensors = candle_core::safetensors::load(&tensors_path, device)?;
227    Ok(Some(LoadedCheckpoint {
228        metadata: meta,
229        tensors,
230    }))
231}
232
233/// Atomically writes the `DONE` completion marker in `dir`.
234///
235/// # Errors
236///
237/// Returns [`CodecError::Io`] on filesystem failure.
238pub fn mark_done(dir: &Path) -> Result<(), CodecError> {
239    write_atomic_bytes(&dir.join(DONE_NAME), b"complete\n")
240}
241
242/// Returns `true` if `dir/DONE` exists.
243#[must_use]
244pub fn is_done(dir: &Path) -> bool {
245    dir.join(DONE_NAME).exists()
246}
247
248/// Writes `STOP_REQUESTED` in `dir`.
249///
250/// # Errors
251///
252/// Returns [`CodecError::Io`] on filesystem failure.
253pub fn request_stop(dir: &Path) -> Result<(), CodecError> {
254    write_atomic_bytes(&dir.join(STOP_NAME), b"stop\n")
255}
256
257/// Removes `STOP_REQUESTED` in `dir` if present.
258///
259/// # Errors
260///
261/// Returns [`CodecError::Io`] on filesystem errors other than `NotFound`.
262pub fn clear_stop(dir: &Path) -> Result<(), CodecError> {
263    match fs::remove_file(dir.join(STOP_NAME)) {
264        Ok(()) => Ok(()),
265        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
266        Err(e) => Err(e.into()),
267    }
268}
269
270/// Returns `true` if `dir/STOP_REQUESTED` exists.
271#[must_use]
272pub fn stop_requested(dir: &Path) -> bool {
273    dir.join(STOP_NAME).exists()
274}