1use std::collections::HashMap;
2use std::fs::{self, File, OpenOptions};
3use std::io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write};
4use std::path::Path;
5use std::sync::atomic::{AtomicBool, Ordering};
6use std::sync::Arc;
7use std::time::Instant;
8
9use candle_core::{DType, Device, Tensor};
10use serde::{Deserialize, Serialize};
11use tufazip_arith::{
12 make_freq_table_into, ArithmeticDecoder, ArithmeticEncoder, BitInputStream, BitOutputStream,
13 DEFAULT_STATE_BITS,
14};
15
16use crate::checkpoint::{
17 load_checkpoint, mark_done, save_checkpoint, stop_requested, CheckpointMetadata,
18 CHECKPOINT_VERSION,
19};
20use crate::config::{lr_at, resolve_device, CodecConfig, CodecMode, OnlineLoss};
21use crate::error::CodecError;
22use crate::model::{ForwardMode, OutputSlice, PriorLoadInfo, Transformer};
23use crate::optim::Adam;
24use crate::rng::DeterministicRng;
25
26const LOG2: f64 = std::f64::consts::LN_2;
27
28#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
30pub struct RunSummary {
31 pub symbols: usize,
33 pub original_tokens: usize,
35 pub modeled_bits: f64,
37 pub modeled_payload_bytes: f64,
39 pub modeled_bits_per_token: f64,
41 pub modeled_bits_per_byte: Option<f64>,
43 pub actual_compressed_bytes: Option<u64>,
45 pub actual_bits_per_byte: Option<f64>,
47 pub vocab_bytes: u64,
49 pub prior_archive_bytes: u64,
51 pub initial_prior_sha256: Option<String>,
53 pub prior_source_depth: usize,
55 pub prior_initialization_kind: Option<String>,
57 pub prior_block_mapping: Option<Vec<usize>>,
59 pub elapsed_seconds: f64,
61}
62
63#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
65pub enum RunOutcome {
66 Completed(RunSummary),
68 RequeueRequested {
70 block_start: usize,
72 n_symbols: usize,
74 },
75}
76
77struct RunLogger {
78 file: Option<BufWriter<File>>,
79}
80
81impl RunLogger {
82 fn open(run_dir: Option<&Path>) -> Result<Self, CodecError> {
83 let file = match run_dir {
84 Some(dir) => {
85 fs::create_dir_all(dir)?;
86 let f = OpenOptions::new()
87 .create(true)
88 .append(true)
89 .open(dir.join("log.txt"))?;
90 Some(BufWriter::new(f))
91 }
92 None => None,
93 };
94 Ok(Self { file })
95 }
96
97 fn line(&mut self, msg: &str) {
98 tracing::info!("{msg}");
99 if let Some(f) = &mut self.file {
100 let _ = writeln!(f, "{msg}");
101 let _ = f.flush();
102 }
103 }
104}
105
106fn read_be16_tokens(path: &Path) -> Result<Vec<u16>, CodecError> {
107 let bytes = fs::read(path)?;
108 if bytes.len() % 2 != 0 {
109 return Err(CodecError::InvalidData(format!(
110 "token file {} has odd byte count ({})",
111 path.display(),
112 bytes.len()
113 )));
114 }
115 let mut out = Vec::with_capacity(bytes.len() / 2);
116 for chunk in bytes.chunks_exact(2) {
117 out.push(u16::from_be_bytes([chunk[0], chunk[1]]));
118 }
119 Ok(out)
120}
121
122fn validate_resume_state(
123 meta: &CheckpointMetadata,
124 config: &CodecConfig,
125 checkpoint_sig: &str,
126 input_size: u64,
127 prior_info: Option<&PriorLoadInfo>,
128) -> Result<(), CodecError> {
129 if meta.version != CHECKPOINT_VERSION {
130 return Err(CodecError::CheckpointMismatch(format!(
131 "unsupported checkpoint version {}",
132 meta.version
133 )));
134 }
135 if meta.config_signature != checkpoint_sig {
136 return Err(CodecError::CheckpointMismatch(
137 "checkpoint configuration differs from this invocation".to_owned(),
138 ));
139 }
140 if meta.project_only_targets != config.project_only_targets {
141 return Err(CodecError::CheckpointMismatch(
142 "checkpoint project_only_targets differs from this invocation".to_owned(),
143 ));
144 }
145 if meta.retrain_early_epochs != config.retrain.early_epochs
146 || meta.retrain_early_until != config.retrain.early_until
147 {
148 return Err(CodecError::CheckpointMismatch(
149 "checkpoint early replay policy differs from this invocation".to_owned(),
150 ));
151 }
152 if meta.input_size != input_size {
153 return Err(CodecError::CheckpointMismatch(format!(
154 "checkpoint input size {} differs from current input {input_size}",
155 meta.input_size
156 )));
157 }
158 let expected_prior_sha = prior_info.map(|p| p.archive_sha256.as_str());
159 if meta.initial_prior_sha256.as_deref() != expected_prior_sha {
160 return Err(CodecError::CheckpointMismatch(
161 "checkpoint initial_prior SHA-256 differs from this invocation".to_owned(),
162 ));
163 }
164 let expected_vocab_sha = prior_info.map(|p| p.vocab_sha256.as_str());
165 if meta.initial_prior_vocab_sha256.as_deref() != expected_vocab_sha {
166 return Err(CodecError::CheckpointMismatch(
167 "checkpoint prior vocab SHA-256 differs from this invocation".to_owned(),
168 ));
169 }
170 Ok(())
171}
172
173fn cross_entropy_sum_cpu(rows: &[Vec<f32>], targets: &[u32]) -> f64 {
174 let mut total_nats = 0.0f64;
175 for (row, &tgt) in rows.iter().zip(targets.iter()) {
176 let max_logit = row.iter().copied().fold(f32::NEG_INFINITY, f32::max);
177 let mut sum_exp = 0.0f64;
178 for &v in row {
179 sum_exp += f64::from(v - max_logit).exp();
180 }
181 let target_val = f64::from(row[tgt as usize] - max_logit);
182 total_nats += sum_exp.ln() - target_val;
183 }
184 total_nats
185}
186
187fn cross_entropy_sum_detached(logits: &Tensor, targets: &Tensor) -> Result<f64, CodecError> {
188 let n = targets.elem_count();
189 if n == 0 {
190 return Ok(0.0);
191 }
192 let mean = candle_nn::loss::cross_entropy(&logits.detach().to_dtype(DType::F32)?, targets)?;
193 Ok(f64::from(mean.to_scalar::<f32>()?) * (n as f64))
194}
195
196struct RetrainState<'a> {
197 model: &'a Transformer,
198 optimizer: &'a mut Adam,
199 rng: &'a mut DeterministicRng,
200 config: &'a CodecConfig,
201 device: &'a Device,
202 compute_dtype: DType,
203 retrain_train_step: &'a mut usize,
204 logger: &'a mut RunLogger,
205}
206
207fn run_prefix_retrain(
208 state: &mut RetrainState<'_>,
209 file_data: &[u16],
210 file_pos: usize,
211) -> Result<(), CodecError> {
212 let retrain_batch = if state.config.retrain.batch_size == 0 {
213 state.config.batch_size
214 } else {
215 state.config.retrain.batch_size
216 };
217 if retrain_batch == 0 {
218 return Err(CodecError::InvalidConfig(
219 "retrain_batch_size must be positive".to_owned(),
220 ));
221 }
222 let lo = file_pos.saturating_sub(state.config.retrain.len);
223 let stride = (file_pos - lo) / retrain_batch;
224 if stride < 2 {
225 return Ok(());
226 }
227
228 let n_targets = stride - 1;
229 let ctx_len = state.config.model.ctx_len;
230 let update_len = if state.config.retrain.update_len == 0 {
231 ctx_len
232 } else {
233 state.config.retrain.update_len
234 };
235 let retrain_epochs =
236 if state.config.retrain.early_until > 0 && file_pos <= state.config.retrain.early_until {
237 state.config.retrain.early_epochs
238 } else {
239 state.config.retrain.epochs
240 };
241
242 let vocab_size = state.config.model.vocab_size;
243 let mut window_bits = 0.0f64;
244 let mut current_lr = state.config.retrain.lr_schedule.initial_lr();
245 let stream_bases: Vec<usize> = (0..retrain_batch).map(|b| lo + b * stride).collect();
246
247 for _epoch in 0..retrain_epochs {
248 let mut epoch_nats = 0.0f64;
249 let mut start = 0usize;
250 while start < n_targets {
251 let length = update_len.min(n_targets - start);
252 let context_start = if state.config.retrain.update_len > 0 {
253 start.saturating_sub(ctx_len - length)
254 } else {
255 start
256 };
257 let prefix_len = start - context_start;
258 let seq_len = prefix_len + length;
259
260 let mut inp_vec = Vec::with_capacity(retrain_batch * seq_len);
261 let mut tgt_vec = Vec::with_capacity(retrain_batch * length);
262 for &base in &stream_bases {
263 for col in context_start..(context_start + seq_len) {
264 inp_vec.push(u32::from(file_data[base + col]));
265 }
266 for col in start..(start + length) {
267 tgt_vec.push(u32::from(file_data[base + col + 1]));
268 }
269 }
270
271 let inp_t = Tensor::from_vec(inp_vec, (retrain_batch, seq_len), state.device)?;
272 let tgt_t = Tensor::from_vec(tgt_vec, retrain_batch * length, state.device)?;
273
274 let target_logits = {
275 let mut fwd_mode = ForwardMode::Train(state.rng);
276 if state.config.project_only_targets {
277 state.model.forward(
278 &inp_t,
279 OutputSlice::Range {
280 start: prefix_len,
281 len: length,
282 },
283 &mut fwd_mode,
284 state.compute_dtype,
285 )?
286 } else {
287 state
288 .model
289 .forward(&inp_t, OutputSlice::All, &mut fwd_mode, state.compute_dtype)?
290 .narrow(1, prefix_len, length)?
291 }
292 };
293
294 let flat_logits = target_logits
295 .to_dtype(DType::F32)?
296 .reshape((retrain_batch * length, vocab_size))?;
297 let loss = candle_nn::loss::cross_entropy(&flat_logits, &tgt_t)?;
298 let grads = loss.backward()?;
299
300 current_lr = state
301 .config
302 .retrain
303 .lr_schedule
304 .lr_at(*state.retrain_train_step);
305 state.optimizer.set_learning_rate(current_lr);
306 state
307 .optimizer
308 .step_clipped(&grads, state.config.optim.clip)?;
309 *state.retrain_train_step += 1;
310
311 let loss_val = f64::from(loss.to_scalar::<f32>()?);
312 epoch_nats += loss_val * (length as f64);
313 start += update_len;
314 }
315 window_bits = epoch_nats / (n_targets as f64) / LOG2;
316 }
317
318 state.logger.line(&format!(
319 "R {:8} window={:8} epochs={:2} bits/tok={:6.3} step={:7} lr={:.3e}",
320 file_pos,
321 file_pos - lo,
322 retrain_epochs,
323 window_bits,
324 *state.retrain_train_step,
325 current_lr
326 ));
327
328 Ok(())
329}
330
331pub fn run_codec(
341 input_path: impl AsRef<Path>,
342 output_path: impl AsRef<Path>,
343 config: &CodecConfig,
344 mode: CodecMode,
345 stop_flag: Arc<AtomicBool>,
346) -> Result<RunOutcome, CodecError> {
347 config.validate(mode)?;
348 let input_path = input_path.as_ref();
349 let output_path = output_path.as_ref();
350 let device = resolve_device(config.device)?;
351 let compute_dtype = config.effective_compute_dtype(&device);
352
353 let mut logger = RunLogger::open(config.run_dir.as_deref())?;
354 let mut rng = DeterministicRng::from_seed_u64(config.seed);
355
356 let (mut model, prior_info) = match (&config.initial_prior, &config.vocab_file) {
357 (Some(prior_path), Some(vocab_path)) => {
358 let (m, info) = Transformer::from_prior(
359 &config.model,
360 prior_path,
361 vocab_path,
362 config.seed,
363 &device,
364 &mut rng,
365 )?;
366 (m, Some(info))
367 }
368 (Some(_), None) => {
369 return Err(CodecError::InvalidConfig(
370 "vocab_file is required when initial_prior is set".to_owned(),
371 ));
372 }
373 (None, _) => (Transformer::new(&config.model, &device, &mut rng)?, None),
374 };
375
376 let mut optimizer = Adam::new(
377 model.named_vars(),
378 config.optim.lr,
379 config.optim.beta1,
380 config.optim.beta2,
381 config.optim.adam_eps,
382 )?;
383 let mut retrain_optimizer = Adam::new(
384 model.named_vars(),
385 config.retrain.lr_schedule.initial_lr(),
386 config.optim.beta1,
387 config.optim.beta2,
388 config.optim.adam_eps,
389 )?;
390
391 let (n_params, n_params_nonemb) = model.param_counts();
392 logger.line(&format!(
393 "tufazip-codec mode={mode:?} device={device:?} dtype={compute_dtype:?} params={n_params} (non-emb={n_params_nonemb})"
394 ));
395
396 let checkpoint_sig = config.checkpoint_signature();
397 let input_size = fs::metadata(input_path)?.len();
398 let loaded_checkpoint = match &config.checkpoint_dir {
399 Some(dir) => load_checkpoint(dir, &device)?,
400 None => None,
401 };
402
403 if let Some(ref ckpt) = loaded_checkpoint {
404 let meta = &ckpt.metadata;
405 let tensors = &ckpt.tensors;
406 validate_resume_state(
407 meta,
408 config,
409 &checkpoint_sig,
410 input_size,
411 prior_info.as_ref(),
412 )?;
413 model.import_tensors(tensors)?;
414 optimizer.import_state("online_adam", &meta.online_adam_steps, tensors)?;
415 retrain_optimizer.import_state("retrain_adam", &meta.retrain_adam_steps, tensors)?;
416 rng.restore_state(&meta.rng_state);
417 logger.line(&format!(
418 "resuming checkpoint at block_start={} symbols={}",
419 meta.block_start, meta.n_symbols
420 ));
421 }
422
423 let batch = config.batch_size;
424 let ctx_len = config.model.ctx_len;
425 let vocab_size = config.model.vocab_size;
426
427 let original_len: usize;
428 let mut file_data: Vec<u16>;
429 let mut encoder: Option<ArithmeticEncoder<BufWriter<File>>> = None;
430 let mut decoder: Option<ArithmeticDecoder<BufReader<File>>> = None;
431 let mut decode_out_file: Option<BufWriter<File>> = None;
432
433 match mode {
434 CodecMode::Compress | CodecMode::EstimateOnly => {
435 file_data = read_be16_tokens(input_path)?;
436 if let Some(max_tok) = config.max_tokens {
437 file_data.truncate(max_tok);
438 }
439 original_len = file_data.len();
440
441 if mode == CodecMode::Compress {
442 if let Some(parent) = output_path.parent() {
443 if !parent.as_os_str().is_empty() {
444 fs::create_dir_all(parent)?;
445 }
446 }
447 if let Some(ref ckpt) = loaded_checkpoint {
448 let meta = &ckpt.metadata;
449 let file = OpenOptions::new()
450 .read(true)
451 .write(true)
452 .open(output_path)?;
453 let mut bit_out = BitOutputStream::new(BufWriter::new(file));
454 let bit_state = meta.bit_out_state.ok_or_else(|| {
455 CodecError::CheckpointMismatch("missing bit_out_state".to_owned())
456 })?;
457 let enc_state = meta.encoder_state.ok_or_else(|| {
458 CodecError::CheckpointMismatch("missing encoder_state".to_owned())
459 })?;
460 bit_out.restore_file_state(bit_state)?;
461 let mut enc = ArithmeticEncoder::new(DEFAULT_STATE_BITS, bit_out)?;
462 enc.restore_state(enc_state)?;
463 encoder = Some(enc);
464 } else {
465 let mut file = File::create(output_path)?;
466 let len_u32 = u32::try_from(original_len).map_err(|_| {
467 CodecError::InvalidData(format!(
468 "token count {original_len} exceeds u32 header limit"
469 ))
470 })?;
471 file.write_all(&len_u32.to_be_bytes())?;
472 file.flush()?;
473 let bit_out = BitOutputStream::new(BufWriter::new(file));
474 encoder = Some(ArithmeticEncoder::new(DEFAULT_STATE_BITS, bit_out)?);
475 }
476 }
477 }
478 CodecMode::Decompress => {
479 if let Some(parent) = output_path.parent() {
480 if !parent.as_os_str().is_empty() {
481 fs::create_dir_all(parent)?;
482 }
483 }
484 let mut in_file = File::open(input_path)?;
485 let mut len_buf = [0u8; 4];
486 in_file.read_exact(&mut len_buf)?;
487 original_len = u32::from_be_bytes(len_buf) as usize;
488 file_data = vec![0u16; original_len];
489
490 let bit_in = BitInputStream::new(BufReader::new(in_file));
491 let mut dec = ArithmeticDecoder::new(DEFAULT_STATE_BITS, bit_in)?;
492
493 if let Some(ref ckpt) = loaded_checkpoint {
494 let meta = &ckpt.metadata;
495 let bit_state = meta.bit_in_state.ok_or_else(|| {
496 CodecError::CheckpointMismatch("missing bit_in_state".to_owned())
497 })?;
498 let dec_state = meta.decoder_state.ok_or_else(|| {
499 CodecError::CheckpointMismatch("missing decoder_state".to_owned())
500 })?;
501 let output_pos = meta.output_position.ok_or_else(|| {
502 CodecError::CheckpointMismatch("missing output_position".to_owned())
503 })?;
504 dec.bit_stream_mut().restore_state(bit_state)?;
505 dec.restore_state(dec_state)?;
506
507 let mut out_file = OpenOptions::new()
508 .read(true)
509 .write(true)
510 .open(output_path)?;
511 out_file.seek(SeekFrom::Start(0))?;
512 let mut prefix_bytes = vec![0u8; output_pos as usize];
513 out_file.read_exact(&mut prefix_bytes)?;
514 if prefix_bytes.len() % 2 != 0 {
515 return Err(CodecError::CheckpointMismatch(
516 "decoded checkpoint prefix has odd byte count".to_owned(),
517 ));
518 }
519 for (dst, pair) in file_data.iter_mut().zip(prefix_bytes.chunks_exact(2)) {
520 *dst = u16::from_be_bytes([pair[0], pair[1]]);
521 }
522 out_file.set_len(output_pos)?;
523 out_file.seek(SeekFrom::Start(output_pos))?;
524 decode_out_file = Some(BufWriter::new(out_file));
525 } else {
526 let out_file = File::create(output_path)?;
527 decode_out_file = Some(BufWriter::new(out_file));
528 }
529 decoder = Some(dec);
530 }
531 }
532
533 if let Some(ref ckpt) = loaded_checkpoint {
534 let meta = &ckpt.metadata;
535 if meta.original_len != original_len {
536 return Err(CodecError::CheckpointMismatch(format!(
537 "checkpoint token count {} differs from input ({original_len})",
538 meta.original_len
539 )));
540 }
541 }
542
543 let file_len = original_len.div_ceil(batch) * batch;
544 if file_data.len() < file_len {
545 file_data.resize(file_len, 0);
546 }
547 let block_len = batch.max((config.block_len / batch) * batch);
548 let total_steps = file_len / batch;
549 let min_lr = config.optim.lr * config.optim.min_lr_ratio;
550 let bytes_per_token = config
551 .raw_bytes
552 .filter(|_| original_len > 0)
553 .map(|raw| (raw as f64) / (original_len as f64));
554
555 let (
556 mut n_symbols,
557 mut n_bits,
558 mut train_step,
559 mut retrain_train_step,
560 mut block_start,
561 mut last_retrain_pos,
562 elapsed_before,
563 ) = match loaded_checkpoint {
564 Some(ref ckpt) => {
565 let meta = &ckpt.metadata;
566 (
567 meta.n_symbols,
568 meta.n_bits,
569 meta.train_step,
570 meta.retrain_train_step,
571 meta.block_start,
572 meta.last_retrain_pos,
573 meta.elapsed_seconds,
574 )
575 }
576 None => (0, 0.0, 0, 0, 0, 0, 0.0),
577 };
578
579 let mut completed_blocks = block_start / block_len;
580 let session_start = Instant::now();
581 let mut last_symbols = n_symbols;
582 let mut last_bits = n_bits;
583 let mut last_time = Instant::now();
584 let mut freq_buf = vec![0u32; vocab_size];
585
586 logger.line(&format!(
587 "{:>8} {:>11} {:>11} {:>7} {:>7} {:>8} {:>8} {:>7} {:>9}",
588 "step", "symbols", "comp_bytes", "bpt_loc", "bpt_cum", "bpb_loc", "bpb_cum", "kS/s", "lr"
589 ));
590
591 while block_start < file_len {
592 if config.retrain.period > 0
593 && block_start.saturating_sub(last_retrain_pos) >= config.retrain.period
594 {
595 let mut rt_state = RetrainState {
596 model: &model,
597 optimizer: &mut retrain_optimizer,
598 rng: &mut rng,
599 config,
600 device: &device,
601 compute_dtype,
602 retrain_train_step: &mut retrain_train_step,
603 logger: &mut logger,
604 };
605 run_prefix_retrain(&mut rt_state, &file_data, block_start)?;
606 last_retrain_pos = block_start;
607 }
608
609 let cur_block_len = (file_len - block_start).min(block_len);
610 let block_stride = cur_block_len / batch;
611 let stream_base: Vec<usize> = (0..batch).map(|b| block_start + b * block_stride).collect();
612
613 for stream_pos in 0..block_stride {
614 let window = stream_pos.min(ctx_len);
615 let seq_len = window.max(1);
616 let mut inp_vec = Vec::with_capacity(batch * seq_len);
617 if window == 0 {
618 inp_vec.resize(batch, 0u32);
619 } else {
620 let start_off = stream_pos - window;
621 for &base in &stream_base {
622 for off in start_off..(start_off + window) {
623 inp_vec.push(u32::from(file_data[base + off]));
624 }
625 }
626 }
627 let inp_t = Tensor::from_vec(inp_vec, (batch, seq_len), &device)?;
628
629 let lr = lr_at(
630 train_step,
631 config.optim.lr,
632 min_lr,
633 config.optim.warmup,
634 total_steps,
635 );
636 optimizer.set_learning_rate(lr);
637
638 let logits = {
639 let mut eval_mode = ForwardMode::Eval;
640 let slice = if config.project_only_targets {
641 OutputSlice::Last
642 } else {
643 OutputSlice::All
644 };
645 model.forward(&inp_t, slice, &mut eval_mode, compute_dtype)?
646 };
647
648 let code_logits = logits
649 .narrow(1, logits.dim(1)? - 1, 1)?
650 .squeeze(1)?
651 .to_dtype(DType::F32)?;
652
653 let cached_cpu_logits = if encoder.is_some() || decoder.is_some() {
654 let cpu_logits = code_logits
655 .detach()
656 .to_device(&Device::Cpu)?
657 .contiguous()?
658 .to_vec2::<f32>()?;
659 for (i, row_logits) in cpu_logits.iter().enumerate() {
660 make_freq_table_into(row_logits, &mut freq_buf);
661 let pos = stream_base[i] + stream_pos;
662 if let Some(dec) = &mut decoder {
663 let sym = dec.read(&freq_buf)?;
664 file_data[pos] = sym as u16;
665 } else if let Some(enc) = &mut encoder {
666 enc.write(&freq_buf, usize::from(file_data[pos]))?;
667 }
668 }
669 Some(cpu_logits)
670 } else {
671 None
672 };
673
674 let target_vec: Vec<u32> = stream_base
675 .iter()
676 .map(|&base| u32::from(file_data[base + stream_pos]))
677 .collect();
678 let code_targets = Tensor::from_vec(target_vec.clone(), batch, &device)?;
679
680 let (loss, code_nats) = match config.online_loss {
681 OnlineLoss::Code => {
682 let loss = candle_nn::loss::cross_entropy(&code_logits, &code_targets)?;
683 let nats = if let Some(rows) = cached_cpu_logits.as_ref() {
684 cross_entropy_sum_cpu(rows, &target_vec)
685 } else {
686 f64::from(loss.to_scalar::<f32>()?) * (batch as f64)
687 };
688 (loss, nats)
689 }
690 OnlineLoss::Window => {
691 let nats = if let Some(rows) = cached_cpu_logits.as_ref() {
692 cross_entropy_sum_cpu(rows, &target_vec)
693 } else {
694 cross_entropy_sum_detached(&code_logits, &code_targets)?
695 };
696 let mut win_tgts = Vec::with_capacity(batch * seq_len);
697 if window == 0 {
698 for &base in &stream_base {
699 win_tgts.push(u32::from(file_data[base + stream_pos]));
700 }
701 } else {
702 let start_off = stream_pos - window + 1;
703 for &base in &stream_base {
704 for off in start_off..(start_off + window) {
705 win_tgts.push(u32::from(file_data[base + off]));
706 }
707 }
708 }
709 let tgt_t = Tensor::from_vec(win_tgts, batch * seq_len, &device)?;
710 let flat_logits = logits
711 .to_dtype(DType::F32)?
712 .reshape((batch * seq_len, vocab_size))?;
713 let loss = candle_nn::loss::cross_entropy(&flat_logits, &tgt_t)?;
714 (loss, nats)
715 }
716 };
717
718 let grads = loss.backward()?;
719 let _grad_norm = optimizer.step_clipped(&grads, config.optim.clip)?;
720
721 n_bits += code_nats / LOG2;
722 n_symbols += batch;
723 train_step += 1;
724
725 let is_last = block_start + cur_block_len >= file_len && stream_pos + 1 == block_stride;
726 if (config.log_interval > 0 && n_symbols - last_symbols >= config.log_interval)
727 || is_last
728 {
729 let now = Instant::now();
730 let dt = now.duration_since(last_time).as_secs_f64().max(1e-6);
731 let delta_syms = (n_symbols - last_symbols) as f64;
732 let local_bpt = (n_bits - last_bits) / delta_syms;
733 let cum_bpt = n_bits / (n_symbols as f64);
734 let local_bpb = bytes_per_token.map_or(f64::NAN, |b| local_bpt / b);
735 let cum_bpb = bytes_per_token.map_or(f64::NAN, |b| cum_bpt / b);
736 let ks_per_s = delta_syms / (dt * 1000.0);
737 logger.line(&format!(
738 "{train_step:8} {n_symbols:11} {:11.0} {local_bpt:7.3} {cum_bpt:7.3} {local_bpb:8.4} {cum_bpb:8.4} {ks_per_s:7.2} {lr:9.3e}",
739 n_bits / 8.0
740 ));
741 last_symbols = n_symbols;
742 last_bits = n_bits;
743 last_time = now;
744 }
745 }
746
747 if mode == CodecMode::Decompress {
748 if let Some(out_file) = &mut decode_out_file {
749 let end = (block_start + cur_block_len).min(original_len);
750 if block_start < end {
751 for &tok in &file_data[block_start..end] {
752 out_file.write_all(&tok.to_be_bytes())?;
753 }
754 out_file.flush()?;
755 }
756 }
757 }
758
759 block_start += cur_block_len;
760 completed_blocks += 1;
761
762 let requested_stop = stop_flag.load(Ordering::Relaxed)
763 || config.checkpoint_dir.as_deref().is_some_and(stop_requested);
764 let test_stop = config
765 .stop_after_blocks
766 .is_some_and(|limit| completed_blocks >= limit);
767 let periodic = config.checkpoint_dir.is_some()
768 && completed_blocks % config.checkpoint_every_blocks == 0;
769
770 if block_start < file_len && (periodic || requested_stop || test_stop) {
771 let reason = if requested_stop {
772 "stop-request"
773 } else if test_stop {
774 "test-stop"
775 } else {
776 "periodic"
777 };
778 if let Some(dir) = &config.checkpoint_dir {
779 let mut tensors = HashMap::new();
780 model.export_tensors(&mut tensors)?;
781 let online_adam_steps = optimizer.export_state("online_adam", &mut tensors)?;
782 let retrain_adam_steps =
783 retrain_optimizer.export_state("retrain_adam", &mut tensors)?;
784
785 let (encoder_state, bit_out_state) = match &mut encoder {
786 Some(enc) => (Some(enc.state()), Some(enc.bit_stream_mut().state()?)),
787 None => (None, None),
788 };
789 let (decoder_state, bit_in_state) = match &mut decoder {
790 Some(dec) => (Some(dec.state()), Some(dec.bit_stream_mut().state()?)),
791 None => (None, None),
792 };
793 let output_position = match &mut decode_out_file {
794 Some(out) => {
795 out.flush()?;
796 out.get_ref().sync_all()?;
797 Some(out.stream_position()?)
798 }
799 None => None,
800 };
801
802 let mut meta = CheckpointMetadata {
803 version: CHECKPOINT_VERSION,
804 config_signature: checkpoint_sig.clone(),
805 project_only_targets: config.project_only_targets,
806 retrain_early_epochs: config.retrain.early_epochs,
807 retrain_early_until: config.retrain.early_until,
808 input_size,
809 original_len,
810 file_len,
811 initial_prior_sha256: prior_info.as_ref().map(|p| p.archive_sha256.clone()),
812 initial_prior_vocab_sha256: prior_info.as_ref().map(|p| p.vocab_sha256.clone()),
813 tensors_sha256: String::new(),
814 online_adam_steps,
815 retrain_adam_steps,
816 rng_state: rng.state(),
817 encoder_state,
818 decoder_state,
819 bit_out_state,
820 bit_in_state,
821 output_position,
822 n_symbols,
823 n_bits,
824 train_step,
825 retrain_train_step,
826 block_start,
827 last_retrain_pos,
828 elapsed_seconds: elapsed_before + session_start.elapsed().as_secs_f64(),
829 reason: reason.to_owned(),
830 };
831 save_checkpoint(dir, &mut meta, &tensors)?;
832 logger.line(&format!(
833 "checkpoint saved at block_start={block_start} ({reason})"
834 ));
835 }
836 if requested_stop || test_stop {
837 if let Some(out) = &mut decode_out_file {
838 out.flush()?;
839 out.get_ref().sync_all()?;
840 }
841 if let Some(enc) = &mut encoder {
842 enc.bit_stream_mut().flush()?;
843 enc.bit_stream_mut().get_ref().get_ref().sync_all()?;
844 }
845 return Ok(RunOutcome::RequeueRequested {
846 block_start,
847 n_symbols,
848 });
849 }
850 }
851 }
852
853 if let Some(enc) = encoder.take() {
854 let mut out_writer = enc.into_inner()?;
855 out_writer.flush()?;
856 out_writer.get_ref().sync_all()?;
857 }
858 if let Some(mut out_file) = decode_out_file.take() {
859 out_file.flush()?;
860 out_file.get_ref().sync_all()?;
861 }
862
863 let elapsed_seconds = elapsed_before + session_start.elapsed().as_secs_f64();
864 let actual_compressed_bytes = match mode {
865 CodecMode::Compress => Some(fs::metadata(output_path)?.len()),
866 CodecMode::Decompress => Some(fs::metadata(input_path)?.len()),
867 CodecMode::EstimateOnly => None,
868 };
869 let actual_bits_per_byte = match (actual_compressed_bytes, config.raw_bytes) {
870 (Some(bytes), Some(raw)) if raw > 0 => Some((bytes as f64) * 8.0 / (raw as f64)),
871 _ => None,
872 };
873 let modeled_bits_per_token = if n_symbols > 0 {
874 n_bits / (n_symbols as f64)
875 } else {
876 0.0
877 };
878 let modeled_bits_per_byte = bytes_per_token.map(|b| modeled_bits_per_token / b);
879
880 let summary = RunSummary {
881 symbols: n_symbols,
882 original_tokens: original_len,
883 modeled_bits: n_bits,
884 modeled_payload_bytes: n_bits / 8.0,
885 modeled_bits_per_token,
886 modeled_bits_per_byte,
887 actual_compressed_bytes,
888 actual_bits_per_byte,
889 vocab_bytes: config.vocab_bytes,
890 prior_archive_bytes: prior_info.as_ref().map_or(0, |p| p.archive_bytes),
891 initial_prior_sha256: prior_info.as_ref().map(|p| p.archive_sha256.clone()),
892 prior_source_depth: prior_info.as_ref().map_or(0, |p| p.source_depth),
893 prior_initialization_kind: prior_info.as_ref().map(|p| {
894 match p.initialization_kind {
895 tufazip_prior::InitializationKind::DepthGrowth => "depth-growth",
896 tufazip_prior::InitializationKind::LoopUntying => "loop-untying",
897 }
898 .to_owned()
899 }),
900 prior_block_mapping: prior_info.as_ref().map(|p| p.block_mapping.clone()),
901 elapsed_seconds,
902 };
903
904 if let Ok(json) = serde_json::to_string(&summary) {
905 logger.line(&format!("FINAL_RESULT {json}"));
906 }
907
908 if let Some(dir) = &config.checkpoint_dir {
909 mark_done(dir)?;
910 }
911
912 Ok(RunOutcome::Completed(summary))
913}
914
915#[cfg(test)]
916mod tests {
917 use std::path::PathBuf;
918
919 use tufazip_preprocess::write_tokens_be16;
920
921 use super::*;
922 use crate::clip_grad_norm;
923 use crate::config::{
924 AutocastPolicy, ComputeDType, DeviceChoice, ModelConfig, NormStyle, PositionType,
925 };
926
927 fn tiny_tokens(len: usize, vocab_size: usize) -> Vec<u16> {
928 (0..len)
929 .map(|i| ((i * 7 + (i / 5) * 3) % vocab_size) as u16)
930 .collect()
931 }
932
933 #[test]
934 fn forward_backward_gradient_variants_and_bf16() {
935 let variants = [
936 (
937 PositionType::Rope,
938 NormStyle::Peri,
939 false,
940 true,
941 OutputSlice::Last,
942 DType::F32,
943 ),
944 (
945 PositionType::LearnedRelative,
946 NormStyle::Post,
947 true,
948 false,
949 OutputSlice::All,
950 DType::F32,
951 ),
952 (
953 PositionType::Rope,
954 NormStyle::Peri,
955 false,
956 true,
957 OutputSlice::Last,
958 DType::BF16,
959 ),
960 ];
961
962 for (pos, norm, tie, qk_norm, slice, dt) in variants {
963 let mut rng = DeterministicRng::from_seed_u64(42);
964 let cfg = ModelConfig {
965 vocab_size: 32,
966 n_layer: 2,
967 d_model: 16,
968 n_head: 2,
969 d_inner: 32,
970 ctx_len: 8,
971 dropout: 0.25,
972 rope_base: 10_000.0,
973 tie_embeddings: tie,
974 position_type: pos,
975 qk_norm,
976 norm_style: norm,
977 block_schedule: None,
978 };
979 let model = Transformer::new(&cfg, &Device::Cpu, &mut rng).unwrap();
980 let mut adam = Adam::new(model.named_vars(), 1e-3, 0.0, 0.999, 1e-8).unwrap();
981
982 let inp =
983 Tensor::from_vec(vec![1u32, 2, 3, 4, 5, 6, 7, 8], (2, 4), &Device::Cpu).unwrap();
984 let logits = {
985 let mut mode = ForwardMode::Train(&mut rng);
986 model.forward(&inp, slice, &mut mode, dt).unwrap()
987 };
988 let out_seq = logits.dim(1).unwrap();
989 let tgt_len = 2 * out_seq;
990 let tgts: Vec<u32> = (0..tgt_len).map(|i| (i % 32) as u32).collect();
991 let tgt_t = Tensor::from_vec(tgts, tgt_len, &Device::Cpu).unwrap();
992 let flat = logits
993 .to_dtype(DType::F32)
994 .unwrap()
995 .reshape((tgt_len, 32))
996 .unwrap();
997 let loss = candle_nn::loss::cross_entropy(&flat, &tgt_t).unwrap();
998 let mut grads = loss.backward().unwrap();
999 let norm_val = clip_grad_norm(model.named_vars(), &mut grads, 0.25).unwrap();
1000 assert!(
1001 norm_val.is_finite(),
1002 "non-finite grad norm for {pos:?} {dt:?}"
1003 );
1004 adam.step(&grads).unwrap();
1005 assert!(model.weight_norm().unwrap().is_finite());
1006 }
1007 }
1008
1009 #[test]
1010 fn lossless_compress_decompress_roundtrip_code_and_window() {
1011 for (online_loss, project_only) in [(OnlineLoss::Code, true), (OnlineLoss::Window, false)] {
1012 let dir = tempfile::tempdir().unwrap();
1013 let in_pp = dir.path().join("input.pp");
1014 let comp_bin = dir.path().join("comp.bin");
1015 let out_pp = dir.path().join("recovered.pp");
1016
1017 let tokens = tiny_tokens(77, 64);
1018 write_tokens_be16(&in_pp, &tokens).unwrap();
1019
1020 let mut cfg = CodecConfig::tiny(64);
1021 cfg.model.n_layer = 2;
1022 cfg.model.d_model = 32;
1023 cfg.model.n_head = 2;
1024 cfg.model.d_inner = 64;
1025 cfg.model.ctx_len = 16;
1026 cfg.model.dropout = 0.25;
1027 cfg.batch_size = 4;
1028 cfg.block_len = 32;
1029 cfg.retrain.period = 32;
1030 cfg.retrain.len = 64;
1031 cfg.retrain.batch_size = 4;
1032 cfg.retrain.update_len = 8;
1033 cfg.online_loss = online_loss;
1034 cfg.project_only_targets = project_only;
1035 cfg.autocast_policy = AutocastPolicy::Auto;
1036 cfg.dtype = ComputeDType::Fp32;
1037 cfg.device = DeviceChoice::Cpu;
1038
1039 let stop = Arc::new(AtomicBool::new(false));
1040 let res_enc =
1041 run_codec(&in_pp, &comp_bin, &cfg, CodecMode::Compress, stop.clone()).unwrap();
1042 assert!(matches!(res_enc, RunOutcome::Completed(_)));
1043
1044 let res_dec = run_codec(&comp_bin, &out_pp, &cfg, CodecMode::Decompress, stop).unwrap();
1045 assert!(matches!(res_dec, RunOutcome::Completed(_)));
1046
1047 assert_eq!(
1048 std::fs::read(&out_pp).unwrap(),
1049 std::fs::read(&in_pp).unwrap(),
1050 "roundtrip mismatch for {online_loss:?}"
1051 );
1052 }
1053 }
1054
1055 #[test]
1056 fn checkpoint_resume_bit_exact_compress_and_decompress() {
1057 let dir = tempfile::tempdir().unwrap();
1058 let in_pp = dir.path().join("input.pp");
1059 let full_bin = dir.path().join("full.bin");
1060 let resumed_bin = dir.path().join("resumed.bin");
1061 let resumed_pp = dir.path().join("resumed.pp");
1062 let enc_ckpt = dir.path().join("enc_ckpt");
1063 let dec_ckpt = dir.path().join("dec_ckpt");
1064
1065 let tokens = tiny_tokens(113, 48);
1066 write_tokens_be16(&in_pp, &tokens).unwrap();
1067
1068 let mut base_cfg = CodecConfig::tiny(48);
1069 base_cfg.model.n_layer = 2;
1070 base_cfg.model.d_model = 32;
1071 base_cfg.model.n_head = 2;
1072 base_cfg.model.d_inner = 64;
1073 base_cfg.model.ctx_len = 12;
1074 base_cfg.model.dropout = 0.25;
1075 base_cfg.batch_size = 4;
1076 base_cfg.block_len = 32;
1077 base_cfg.retrain.period = 32;
1078 base_cfg.retrain.len = 96;
1079 base_cfg.retrain.batch_size = 4;
1080 base_cfg.retrain.update_len = 8;
1081
1082 let stop = Arc::new(AtomicBool::new(false));
1083 let full_out = run_codec(
1084 &in_pp,
1085 &full_bin,
1086 &base_cfg,
1087 CodecMode::Compress,
1088 stop.clone(),
1089 )
1090 .unwrap();
1091 assert!(matches!(full_out, RunOutcome::Completed(_)));
1092
1093 let mut stop_cfg = base_cfg.clone();
1095 stop_cfg.checkpoint_dir = Some(enc_ckpt.clone());
1096 stop_cfg.stop_after_blocks = Some(1);
1097 let paused = run_codec(
1098 &in_pp,
1099 &resumed_bin,
1100 &stop_cfg,
1101 CodecMode::Compress,
1102 stop.clone(),
1103 )
1104 .unwrap();
1105 assert!(matches!(paused, RunOutcome::RequeueRequested { .. }));
1106
1107 stop_cfg.stop_after_blocks = None;
1108 let finished = run_codec(
1109 &in_pp,
1110 &resumed_bin,
1111 &stop_cfg,
1112 CodecMode::Compress,
1113 stop.clone(),
1114 )
1115 .unwrap();
1116 assert!(matches!(finished, RunOutcome::Completed(_)));
1117 assert!(crate::checkpoint::is_done(&enc_ckpt));
1118 assert_eq!(
1119 std::fs::read(&resumed_bin).unwrap(),
1120 std::fs::read(&full_bin).unwrap()
1121 );
1122
1123 let mut dec_cfg = base_cfg;
1125 dec_cfg.checkpoint_dir = Some(dec_ckpt.clone());
1126 dec_cfg.stop_after_blocks = Some(1);
1127 let dec_paused = run_codec(
1128 &resumed_bin,
1129 &resumed_pp,
1130 &dec_cfg,
1131 CodecMode::Decompress,
1132 stop.clone(),
1133 )
1134 .unwrap();
1135 assert!(matches!(dec_paused, RunOutcome::RequeueRequested { .. }));
1136
1137 dec_cfg.stop_after_blocks = None;
1138 let dec_done = run_codec(
1139 &resumed_bin,
1140 &resumed_pp,
1141 &dec_cfg,
1142 CodecMode::Decompress,
1143 stop,
1144 )
1145 .unwrap();
1146 assert!(matches!(dec_done, RunOutcome::Completed(_)));
1147 assert_eq!(
1148 std::fs::read(&resumed_pp).unwrap(),
1149 std::fs::read(&in_pp).unwrap()
1150 );
1151 }
1152
1153 #[test]
1154 fn v2_model_loads_real_weights_bin_and_runs_forward() {
1155 let assets = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../assets");
1156 let weights_path = assets.join("weights.bin");
1157 let voc_path = assets.join("enwik9.voc");
1158
1159 let cfg = ModelConfig::v2();
1160 let mut rng = DeterministicRng::from_seed_u64(1111);
1161 let (model, info) =
1162 Transformer::from_prior(&cfg, &weights_path, &voc_path, 1111, &Device::Cpu, &mut rng)
1163 .unwrap();
1164 let (total, non_emb) = model.param_counts();
1165 assert_eq!(total, 162_585_600);
1166 assert_eq!(non_emb, 154_194_944);
1167 assert_eq!(info.archive_bytes, 4_334_741);
1168 assert!(!info.initial_state_sha256.is_empty());
1169
1170 let idx = Tensor::from_vec(vec![0u32, 32, 100, 200], (1, 4), &Device::Cpu).unwrap();
1171 let mut eval = ForwardMode::Eval;
1172 let logits = model
1173 .forward(&idx, OutputSlice::Last, &mut eval, DType::F32)
1174 .unwrap();
1175 assert_eq!(logits.dims(), &[1, 1, 4097]);
1176 let vals = logits.flatten_all().unwrap().to_vec1::<f32>().unwrap();
1177 assert!(vals.iter().all(|v| v.is_finite()));
1178 }
1179}