tufazip_codec/
checkpoint.rs1use std::collections::{BTreeMap, HashMap};
2use std::fs::{self, File, OpenOptions};
3use std::io::Write;
4use std::path::{Path, PathBuf};
5
6use candle_core::{Device, Tensor};
7use serde::{Deserialize, Serialize};
8use tufazip_arith::{BitInputState, BitOutputState, DecoderState, EncoderState};
9use tufazip_prior::sha256_file;
10
11use crate::error::CodecError;
12use crate::rng::RngState;
13
14pub const CHECKPOINT_VERSION: u32 = 1;
16
17pub const REQUEUE_EXIT_CODE: i32 = 75;
19
20pub const DONE_NAME: &str = "DONE";
22
23pub const STOP_NAME: &str = "STOP_REQUESTED";
25
26const LATEST_META_NAME: &str = "latest.json";
27const LATEST_TENSORS_NAME: &str = "latest.safetensors";
28
29#[derive(Debug, Clone, Serialize, Deserialize)]
31pub struct CheckpointMetadata {
32 pub version: u32,
34 pub config_signature: String,
36 pub project_only_targets: bool,
38 pub retrain_early_epochs: usize,
40 pub retrain_early_until: usize,
42 pub input_size: u64,
44 pub original_len: usize,
46 pub file_len: usize,
48 pub initial_prior_sha256: Option<String>,
50 pub initial_prior_vocab_sha256: Option<String>,
52 pub tensors_sha256: String,
54 pub online_adam_steps: BTreeMap<String, u64>,
56 pub retrain_adam_steps: BTreeMap<String, u64>,
58 pub rng_state: RngState,
60 pub encoder_state: Option<EncoderState>,
62 pub decoder_state: Option<DecoderState>,
64 pub bit_out_state: Option<BitOutputState>,
66 pub bit_in_state: Option<BitInputState>,
68 pub output_position: Option<u64>,
70 pub n_symbols: usize,
72 pub n_bits: f64,
74 pub train_step: usize,
76 pub retrain_train_step: usize,
78 pub block_start: usize,
80 pub last_retrain_pos: usize,
82 pub elapsed_seconds: f64,
84 pub reason: String,
86}
87
88fn sync_parent_dir(path: &Path) -> Result<(), CodecError> {
89 if let Some(parent) = path.parent() {
90 #[cfg(unix)]
91 if let Ok(dir) = File::open(parent) {
92 let _ = dir.sync_all();
93 }
94 #[cfg(not(unix))]
95 let _ = parent;
96 }
97 Ok(())
98}
99
100pub fn write_atomic_bytes(path: &Path, bytes: &[u8]) -> Result<(), CodecError> {
106 if let Some(parent) = path.parent() {
107 fs::create_dir_all(parent)?;
108 }
109 let tmp_path = tmp_path_for(path);
110 let write_res = (|| -> Result<(), CodecError> {
111 let mut file = OpenOptions::new()
112 .create(true)
113 .truncate(true)
114 .write(true)
115 .open(&tmp_path)?;
116 file.write_all(bytes)?;
117 file.flush()?;
118 file.sync_all()?;
119 drop(file);
120 fs::rename(&tmp_path, path)?;
121 sync_parent_dir(path)?;
122 Ok(())
123 })();
124 if write_res.is_err() {
125 let _ = fs::remove_file(&tmp_path);
126 }
127 write_res
128}
129
130fn tmp_path_for(path: &Path) -> PathBuf {
131 let name = path
132 .file_name()
133 .and_then(|n| n.to_str())
134 .unwrap_or("checkpoint");
135 path.with_file_name(format!("{name}.tmp.{}", std::process::id()))
136}
137
138pub fn write_atomic_safetensors(
144 path: &Path,
145 tensors: &HashMap<String, Tensor>,
146) -> Result<(), CodecError> {
147 if let Some(parent) = path.parent() {
148 fs::create_dir_all(parent)?;
149 }
150 let tmp_path = tmp_path_for(path);
151 let save_res = (|| -> Result<(), CodecError> {
152 candle_core::safetensors::save(tensors, &tmp_path)?;
153 if let Ok(file) = File::open(&tmp_path) {
154 let _ = file.sync_all();
155 }
156 fs::rename(&tmp_path, path)?;
157 sync_parent_dir(path)?;
158 Ok(())
159 })();
160 if save_res.is_err() {
161 let _ = fs::remove_file(&tmp_path);
162 }
163 save_res
164}
165
166pub fn save_checkpoint(
172 dir: &Path,
173 meta: &mut CheckpointMetadata,
174 tensors: &HashMap<String, Tensor>,
175) -> Result<(), CodecError> {
176 fs::create_dir_all(dir)?;
177 let tensors_path = dir.join(LATEST_TENSORS_NAME);
178 write_atomic_safetensors(&tensors_path, tensors)?;
179 meta.tensors_sha256 = sha256_file(&tensors_path)?;
180 let json = serde_json::to_vec_pretty(meta)?;
181 write_atomic_bytes(&dir.join(LATEST_META_NAME), &json)?;
182 Ok(())
183}
184
185#[derive(Debug)]
187pub struct LoadedCheckpoint {
188 pub metadata: CheckpointMetadata,
190 pub tensors: HashMap<String, Tensor>,
192}
193
194pub fn load_checkpoint(
202 dir: &Path,
203 device: &Device,
204) -> Result<Option<LoadedCheckpoint>, CodecError> {
205 let meta_path = dir.join(LATEST_META_NAME);
206 if !meta_path.exists() {
207 return Ok(None);
208 }
209 let tensors_path = dir.join(LATEST_TENSORS_NAME);
210 if !tensors_path.exists() {
211 return Err(CodecError::CheckpointMismatch(format!(
212 "found {} without paired {}",
213 meta_path.display(),
214 tensors_path.display()
215 )));
216 }
217 let meta_bytes = fs::read(&meta_path)?;
218 let meta: CheckpointMetadata = serde_json::from_slice(&meta_bytes)?;
219 let actual_sha = sha256_file(&tensors_path)?;
220 if actual_sha != meta.tensors_sha256 {
221 return Err(CodecError::CheckpointMismatch(format!(
222 "checkpoint tensor digest mismatch: expected {}, found {actual_sha}",
223 meta.tensors_sha256
224 )));
225 }
226 let tensors = candle_core::safetensors::load(&tensors_path, device)?;
227 Ok(Some(LoadedCheckpoint {
228 metadata: meta,
229 tensors,
230 }))
231}
232
233pub fn mark_done(dir: &Path) -> Result<(), CodecError> {
239 write_atomic_bytes(&dir.join(DONE_NAME), b"complete\n")
240}
241
242#[must_use]
244pub fn is_done(dir: &Path) -> bool {
245 dir.join(DONE_NAME).exists()
246}
247
248pub fn request_stop(dir: &Path) -> Result<(), CodecError> {
254 write_atomic_bytes(&dir.join(STOP_NAME), b"stop\n")
255}
256
257pub fn clear_stop(dir: &Path) -> Result<(), CodecError> {
263 match fs::remove_file(dir.join(STOP_NAME)) {
264 Ok(()) => Ok(()),
265 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
266 Err(e) => Err(e.into()),
267 }
268}
269
270#[must_use]
272pub fn stop_requested(dir: &Path) -> bool {
273 dir.join(STOP_NAME).exists()
274}