Skip to main content

tufazip_codec/
engine.rs

1use std::collections::HashMap;
2use std::fs::{self, File, OpenOptions};
3use std::io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write};
4use std::path::Path;
5use std::sync::atomic::{AtomicBool, Ordering};
6use std::sync::Arc;
7use std::time::Instant;
8
9use candle_core::{DType, Device, Tensor};
10use serde::{Deserialize, Serialize};
11use tufazip_arith::{
12    make_freq_table_into, ArithmeticDecoder, ArithmeticEncoder, BitInputStream, BitOutputStream,
13    DEFAULT_STATE_BITS,
14};
15
16use crate::checkpoint::{
17    load_checkpoint, mark_done, save_checkpoint, stop_requested, CheckpointMetadata,
18    CHECKPOINT_VERSION,
19};
20use crate::config::{lr_at, resolve_device, CodecConfig, CodecMode, OnlineLoss};
21use crate::error::CodecError;
22use crate::model::{ForwardMode, OutputSlice, PriorLoadInfo, Transformer};
23use crate::optim::Adam;
24use crate::rng::DeterministicRng;
25
26const LOG2: f64 = std::f64::consts::LN_2;
27
28/// Metrics reported upon completion of a neural coding or estimation run.
29#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
30pub struct RunSummary {
31    /// Total tokens processed (including batch padding).
32    pub symbols: usize,
33    /// Unpadded original token count.
34    pub original_tokens: usize,
35    /// Cumulative modeled cross-entropy bits.
36    pub modeled_bits: f64,
37    /// Theoretical payload bytes (`modeled_bits / 8.0`).
38    pub modeled_payload_bytes: f64,
39    /// Average modeled bits per token.
40    pub modeled_bits_per_token: f64,
41    /// Average modeled bits per original source byte (if `raw_bytes` was supplied).
42    pub modeled_bits_per_byte: Option<f64>,
43    /// Actual compressed artifact size in bytes (`None` in `EstimateOnly` mode).
44    pub actual_compressed_bytes: Option<u64>,
45    /// Actual compressed bits per original source byte (`None` unless coding and `raw_bytes` set).
46    pub actual_bits_per_byte: Option<f64>,
47    /// Configured vocabulary size in bytes.
48    pub vocab_bytes: u64,
49    /// Prior archive file size in bytes (`0` if uninitialized).
50    pub prior_archive_bytes: u64,
51    /// Prior archive SHA-256 hex digest if loaded.
52    pub initial_prior_sha256: Option<String>,
53    /// Prior source layer count (`0` if uninitialized).
54    pub prior_source_depth: usize,
55    /// Prior initialization kind string (`"depth-growth"` / `"loop-untying"`).
56    pub prior_initialization_kind: Option<String>,
57    /// Prior block mapping if loaded.
58    pub prior_block_mapping: Option<Vec<usize>>,
59    /// Total wall-clock seconds elapsed across all resumed segments.
60    pub elapsed_seconds: f64,
61}
62
63/// Outcome returned by [`run_codec`].
64#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
65pub enum RunOutcome {
66    /// Stream finished compressing or decompressing completely.
67    Completed(RunSummary),
68    /// Stopped cleanly at a block boundary after persisting a resumable checkpoint.
69    RequeueRequested {
70        /// Token index of the next block to execute upon resume.
71        block_start: usize,
72        /// Cumulative coded symbol count so far.
73        n_symbols: usize,
74    },
75}
76
77struct RunLogger {
78    file: Option<BufWriter<File>>,
79}
80
81impl RunLogger {
82    fn open(run_dir: Option<&Path>) -> Result<Self, CodecError> {
83        let file = match run_dir {
84            Some(dir) => {
85                fs::create_dir_all(dir)?;
86                let f = OpenOptions::new()
87                    .create(true)
88                    .append(true)
89                    .open(dir.join("log.txt"))?;
90                Some(BufWriter::new(f))
91            }
92            None => None,
93        };
94        Ok(Self { file })
95    }
96
97    fn line(&mut self, msg: &str) {
98        tracing::info!("{msg}");
99        if let Some(f) = &mut self.file {
100            let _ = writeln!(f, "{msg}");
101            let _ = f.flush();
102        }
103    }
104}
105
106fn read_be16_tokens(path: &Path) -> Result<Vec<u16>, CodecError> {
107    let bytes = fs::read(path)?;
108    if bytes.len() % 2 != 0 {
109        return Err(CodecError::InvalidData(format!(
110            "token file {} has odd byte count ({})",
111            path.display(),
112            bytes.len()
113        )));
114    }
115    let mut out = Vec::with_capacity(bytes.len() / 2);
116    for chunk in bytes.chunks_exact(2) {
117        out.push(u16::from_be_bytes([chunk[0], chunk[1]]));
118    }
119    Ok(out)
120}
121
122fn validate_resume_state(
123    meta: &CheckpointMetadata,
124    config: &CodecConfig,
125    checkpoint_sig: &str,
126    input_size: u64,
127    prior_info: Option<&PriorLoadInfo>,
128) -> Result<(), CodecError> {
129    if meta.version != CHECKPOINT_VERSION {
130        return Err(CodecError::CheckpointMismatch(format!(
131            "unsupported checkpoint version {}",
132            meta.version
133        )));
134    }
135    if meta.config_signature != checkpoint_sig {
136        return Err(CodecError::CheckpointMismatch(
137            "checkpoint configuration differs from this invocation".to_owned(),
138        ));
139    }
140    if meta.project_only_targets != config.project_only_targets {
141        return Err(CodecError::CheckpointMismatch(
142            "checkpoint project_only_targets differs from this invocation".to_owned(),
143        ));
144    }
145    if meta.retrain_early_epochs != config.retrain.early_epochs
146        || meta.retrain_early_until != config.retrain.early_until
147    {
148        return Err(CodecError::CheckpointMismatch(
149            "checkpoint early replay policy differs from this invocation".to_owned(),
150        ));
151    }
152    if meta.input_size != input_size {
153        return Err(CodecError::CheckpointMismatch(format!(
154            "checkpoint input size {} differs from current input {input_size}",
155            meta.input_size
156        )));
157    }
158    let expected_prior_sha = prior_info.map(|p| p.archive_sha256.as_str());
159    if meta.initial_prior_sha256.as_deref() != expected_prior_sha {
160        return Err(CodecError::CheckpointMismatch(
161            "checkpoint initial_prior SHA-256 differs from this invocation".to_owned(),
162        ));
163    }
164    let expected_vocab_sha = prior_info.map(|p| p.vocab_sha256.as_str());
165    if meta.initial_prior_vocab_sha256.as_deref() != expected_vocab_sha {
166        return Err(CodecError::CheckpointMismatch(
167            "checkpoint prior vocab SHA-256 differs from this invocation".to_owned(),
168        ));
169    }
170    Ok(())
171}
172
173fn cross_entropy_sum_cpu(rows: &[Vec<f32>], targets: &[u32]) -> f64 {
174    let mut total_nats = 0.0f64;
175    for (row, &tgt) in rows.iter().zip(targets.iter()) {
176        let max_logit = row.iter().copied().fold(f32::NEG_INFINITY, f32::max);
177        let mut sum_exp = 0.0f64;
178        for &v in row {
179            sum_exp += f64::from(v - max_logit).exp();
180        }
181        let target_val = f64::from(row[tgt as usize] - max_logit);
182        total_nats += sum_exp.ln() - target_val;
183    }
184    total_nats
185}
186
187fn cross_entropy_sum_detached(logits: &Tensor, targets: &Tensor) -> Result<f64, CodecError> {
188    let n = targets.elem_count();
189    if n == 0 {
190        return Ok(0.0);
191    }
192    let mean = candle_nn::loss::cross_entropy(&logits.detach().to_dtype(DType::F32)?, targets)?;
193    Ok(f64::from(mean.to_scalar::<f32>()?) * (n as f64))
194}
195
196struct RetrainState<'a> {
197    model: &'a Transformer,
198    optimizer: &'a mut Adam,
199    rng: &'a mut DeterministicRng,
200    config: &'a CodecConfig,
201    device: &'a Device,
202    compute_dtype: DType,
203    retrain_train_step: &'a mut usize,
204    logger: &'a mut RunLogger,
205}
206
207fn run_prefix_retrain(
208    state: &mut RetrainState<'_>,
209    file_data: &[u16],
210    file_pos: usize,
211) -> Result<(), CodecError> {
212    let retrain_batch = if state.config.retrain.batch_size == 0 {
213        state.config.batch_size
214    } else {
215        state.config.retrain.batch_size
216    };
217    if retrain_batch == 0 {
218        return Err(CodecError::InvalidConfig(
219            "retrain_batch_size must be positive".to_owned(),
220        ));
221    }
222    let lo = file_pos.saturating_sub(state.config.retrain.len);
223    let stride = (file_pos - lo) / retrain_batch;
224    if stride < 2 {
225        return Ok(());
226    }
227
228    let n_targets = stride - 1;
229    let ctx_len = state.config.model.ctx_len;
230    let update_len = if state.config.retrain.update_len == 0 {
231        ctx_len
232    } else {
233        state.config.retrain.update_len
234    };
235    let retrain_epochs =
236        if state.config.retrain.early_until > 0 && file_pos <= state.config.retrain.early_until {
237            state.config.retrain.early_epochs
238        } else {
239            state.config.retrain.epochs
240        };
241
242    let vocab_size = state.config.model.vocab_size;
243    let mut window_bits = 0.0f64;
244    let mut current_lr = state.config.retrain.lr_schedule.initial_lr();
245    let stream_bases: Vec<usize> = (0..retrain_batch).map(|b| lo + b * stride).collect();
246
247    for _epoch in 0..retrain_epochs {
248        let mut epoch_nats = 0.0f64;
249        let mut start = 0usize;
250        while start < n_targets {
251            let length = update_len.min(n_targets - start);
252            let context_start = if state.config.retrain.update_len > 0 {
253                start.saturating_sub(ctx_len - length)
254            } else {
255                start
256            };
257            let prefix_len = start - context_start;
258            let seq_len = prefix_len + length;
259
260            let mut inp_vec = Vec::with_capacity(retrain_batch * seq_len);
261            let mut tgt_vec = Vec::with_capacity(retrain_batch * length);
262            for &base in &stream_bases {
263                for col in context_start..(context_start + seq_len) {
264                    inp_vec.push(u32::from(file_data[base + col]));
265                }
266                for col in start..(start + length) {
267                    tgt_vec.push(u32::from(file_data[base + col + 1]));
268                }
269            }
270
271            let inp_t = Tensor::from_vec(inp_vec, (retrain_batch, seq_len), state.device)?;
272            let tgt_t = Tensor::from_vec(tgt_vec, retrain_batch * length, state.device)?;
273
274            let target_logits = {
275                let mut fwd_mode = ForwardMode::Train(state.rng);
276                if state.config.project_only_targets {
277                    state.model.forward(
278                        &inp_t,
279                        OutputSlice::Range {
280                            start: prefix_len,
281                            len: length,
282                        },
283                        &mut fwd_mode,
284                        state.compute_dtype,
285                    )?
286                } else {
287                    state
288                        .model
289                        .forward(&inp_t, OutputSlice::All, &mut fwd_mode, state.compute_dtype)?
290                        .narrow(1, prefix_len, length)?
291                }
292            };
293
294            let flat_logits = target_logits
295                .to_dtype(DType::F32)?
296                .reshape((retrain_batch * length, vocab_size))?;
297            let loss = candle_nn::loss::cross_entropy(&flat_logits, &tgt_t)?;
298            let grads = loss.backward()?;
299
300            current_lr = state
301                .config
302                .retrain
303                .lr_schedule
304                .lr_at(*state.retrain_train_step);
305            state.optimizer.set_learning_rate(current_lr);
306            state
307                .optimizer
308                .step_clipped(&grads, state.config.optim.clip)?;
309            *state.retrain_train_step += 1;
310
311            let loss_val = f64::from(loss.to_scalar::<f32>()?);
312            epoch_nats += loss_val * (length as f64);
313            start += update_len;
314        }
315        window_bits = epoch_nats / (n_targets as f64) / LOG2;
316    }
317
318    state.logger.line(&format!(
319        "R {:8}  window={:8}  epochs={:2}  bits/tok={:6.3}  step={:7}  lr={:.3e}",
320        file_pos,
321        file_pos - lo,
322        retrain_epochs,
323        window_bits,
324        *state.retrain_train_step,
325        current_lr
326    ));
327
328    Ok(())
329}
330
331/// Runs online neural compression, decompression, or entropy estimation.
332///
333/// Supports deterministic checkpoint-resume via `config.checkpoint_dir` and graceful
334/// stop-at-next-block-boundary via either `stop_flag` or `<checkpoint_dir>/STOP_REQUESTED`.
335///
336/// # Errors
337///
338/// Returns [`CodecError`] on invalid configuration, checkpoint mismatch, prior validation
339/// failure, arithmetic coder errors, or tensor/I/O failures.
340pub fn run_codec(
341    input_path: impl AsRef<Path>,
342    output_path: impl AsRef<Path>,
343    config: &CodecConfig,
344    mode: CodecMode,
345    stop_flag: Arc<AtomicBool>,
346) -> Result<RunOutcome, CodecError> {
347    config.validate(mode)?;
348    let input_path = input_path.as_ref();
349    let output_path = output_path.as_ref();
350    let device = resolve_device(config.device)?;
351    let compute_dtype = config.effective_compute_dtype(&device);
352
353    let mut logger = RunLogger::open(config.run_dir.as_deref())?;
354    let mut rng = DeterministicRng::from_seed_u64(config.seed);
355
356    let (mut model, prior_info) = match (&config.initial_prior, &config.vocab_file) {
357        (Some(prior_path), Some(vocab_path)) => {
358            let (m, info) = Transformer::from_prior(
359                &config.model,
360                prior_path,
361                vocab_path,
362                config.seed,
363                &device,
364                &mut rng,
365            )?;
366            (m, Some(info))
367        }
368        (Some(_), None) => {
369            return Err(CodecError::InvalidConfig(
370                "vocab_file is required when initial_prior is set".to_owned(),
371            ));
372        }
373        (None, _) => (Transformer::new(&config.model, &device, &mut rng)?, None),
374    };
375
376    let mut optimizer = Adam::new(
377        model.named_vars(),
378        config.optim.lr,
379        config.optim.beta1,
380        config.optim.beta2,
381        config.optim.adam_eps,
382    )?;
383    let mut retrain_optimizer = Adam::new(
384        model.named_vars(),
385        config.retrain.lr_schedule.initial_lr(),
386        config.optim.beta1,
387        config.optim.beta2,
388        config.optim.adam_eps,
389    )?;
390
391    let (n_params, n_params_nonemb) = model.param_counts();
392    logger.line(&format!(
393        "tufazip-codec mode={mode:?} device={device:?} dtype={compute_dtype:?} params={n_params} (non-emb={n_params_nonemb})"
394    ));
395
396    let checkpoint_sig = config.checkpoint_signature();
397    let input_size = fs::metadata(input_path)?.len();
398    let loaded_checkpoint = match &config.checkpoint_dir {
399        Some(dir) => load_checkpoint(dir, &device)?,
400        None => None,
401    };
402
403    if let Some(ref ckpt) = loaded_checkpoint {
404        let meta = &ckpt.metadata;
405        let tensors = &ckpt.tensors;
406        validate_resume_state(
407            meta,
408            config,
409            &checkpoint_sig,
410            input_size,
411            prior_info.as_ref(),
412        )?;
413        model.import_tensors(tensors)?;
414        optimizer.import_state("online_adam", &meta.online_adam_steps, tensors)?;
415        retrain_optimizer.import_state("retrain_adam", &meta.retrain_adam_steps, tensors)?;
416        rng.restore_state(&meta.rng_state);
417        logger.line(&format!(
418            "resuming checkpoint at block_start={} symbols={}",
419            meta.block_start, meta.n_symbols
420        ));
421    }
422
423    let batch = config.batch_size;
424    let ctx_len = config.model.ctx_len;
425    let vocab_size = config.model.vocab_size;
426
427    let original_len: usize;
428    let mut file_data: Vec<u16>;
429    let mut encoder: Option<ArithmeticEncoder<BufWriter<File>>> = None;
430    let mut decoder: Option<ArithmeticDecoder<BufReader<File>>> = None;
431    let mut decode_out_file: Option<BufWriter<File>> = None;
432
433    match mode {
434        CodecMode::Compress | CodecMode::EstimateOnly => {
435            file_data = read_be16_tokens(input_path)?;
436            if let Some(max_tok) = config.max_tokens {
437                file_data.truncate(max_tok);
438            }
439            original_len = file_data.len();
440
441            if mode == CodecMode::Compress {
442                if let Some(parent) = output_path.parent() {
443                    if !parent.as_os_str().is_empty() {
444                        fs::create_dir_all(parent)?;
445                    }
446                }
447                if let Some(ref ckpt) = loaded_checkpoint {
448                    let meta = &ckpt.metadata;
449                    let file = OpenOptions::new()
450                        .read(true)
451                        .write(true)
452                        .open(output_path)?;
453                    let mut bit_out = BitOutputStream::new(BufWriter::new(file));
454                    let bit_state = meta.bit_out_state.ok_or_else(|| {
455                        CodecError::CheckpointMismatch("missing bit_out_state".to_owned())
456                    })?;
457                    let enc_state = meta.encoder_state.ok_or_else(|| {
458                        CodecError::CheckpointMismatch("missing encoder_state".to_owned())
459                    })?;
460                    bit_out.restore_file_state(bit_state)?;
461                    let mut enc = ArithmeticEncoder::new(DEFAULT_STATE_BITS, bit_out)?;
462                    enc.restore_state(enc_state)?;
463                    encoder = Some(enc);
464                } else {
465                    let mut file = File::create(output_path)?;
466                    let len_u32 = u32::try_from(original_len).map_err(|_| {
467                        CodecError::InvalidData(format!(
468                            "token count {original_len} exceeds u32 header limit"
469                        ))
470                    })?;
471                    file.write_all(&len_u32.to_be_bytes())?;
472                    file.flush()?;
473                    let bit_out = BitOutputStream::new(BufWriter::new(file));
474                    encoder = Some(ArithmeticEncoder::new(DEFAULT_STATE_BITS, bit_out)?);
475                }
476            }
477        }
478        CodecMode::Decompress => {
479            if let Some(parent) = output_path.parent() {
480                if !parent.as_os_str().is_empty() {
481                    fs::create_dir_all(parent)?;
482                }
483            }
484            let mut in_file = File::open(input_path)?;
485            let mut len_buf = [0u8; 4];
486            in_file.read_exact(&mut len_buf)?;
487            original_len = u32::from_be_bytes(len_buf) as usize;
488            file_data = vec![0u16; original_len];
489
490            let bit_in = BitInputStream::new(BufReader::new(in_file));
491            let mut dec = ArithmeticDecoder::new(DEFAULT_STATE_BITS, bit_in)?;
492
493            if let Some(ref ckpt) = loaded_checkpoint {
494                let meta = &ckpt.metadata;
495                let bit_state = meta.bit_in_state.ok_or_else(|| {
496                    CodecError::CheckpointMismatch("missing bit_in_state".to_owned())
497                })?;
498                let dec_state = meta.decoder_state.ok_or_else(|| {
499                    CodecError::CheckpointMismatch("missing decoder_state".to_owned())
500                })?;
501                let output_pos = meta.output_position.ok_or_else(|| {
502                    CodecError::CheckpointMismatch("missing output_position".to_owned())
503                })?;
504                dec.bit_stream_mut().restore_state(bit_state)?;
505                dec.restore_state(dec_state)?;
506
507                let mut out_file = OpenOptions::new()
508                    .read(true)
509                    .write(true)
510                    .open(output_path)?;
511                out_file.seek(SeekFrom::Start(0))?;
512                let mut prefix_bytes = vec![0u8; output_pos as usize];
513                out_file.read_exact(&mut prefix_bytes)?;
514                if prefix_bytes.len() % 2 != 0 {
515                    return Err(CodecError::CheckpointMismatch(
516                        "decoded checkpoint prefix has odd byte count".to_owned(),
517                    ));
518                }
519                for (dst, pair) in file_data.iter_mut().zip(prefix_bytes.chunks_exact(2)) {
520                    *dst = u16::from_be_bytes([pair[0], pair[1]]);
521                }
522                out_file.set_len(output_pos)?;
523                out_file.seek(SeekFrom::Start(output_pos))?;
524                decode_out_file = Some(BufWriter::new(out_file));
525            } else {
526                let out_file = File::create(output_path)?;
527                decode_out_file = Some(BufWriter::new(out_file));
528            }
529            decoder = Some(dec);
530        }
531    }
532
533    if let Some(ref ckpt) = loaded_checkpoint {
534        let meta = &ckpt.metadata;
535        if meta.original_len != original_len {
536            return Err(CodecError::CheckpointMismatch(format!(
537                "checkpoint token count {} differs from input ({original_len})",
538                meta.original_len
539            )));
540        }
541    }
542
543    let file_len = original_len.div_ceil(batch) * batch;
544    if file_data.len() < file_len {
545        file_data.resize(file_len, 0);
546    }
547    let block_len = batch.max((config.block_len / batch) * batch);
548    let total_steps = file_len / batch;
549    let min_lr = config.optim.lr * config.optim.min_lr_ratio;
550    let bytes_per_token = config
551        .raw_bytes
552        .filter(|_| original_len > 0)
553        .map(|raw| (raw as f64) / (original_len as f64));
554
555    let (
556        mut n_symbols,
557        mut n_bits,
558        mut train_step,
559        mut retrain_train_step,
560        mut block_start,
561        mut last_retrain_pos,
562        elapsed_before,
563    ) = match loaded_checkpoint {
564        Some(ref ckpt) => {
565            let meta = &ckpt.metadata;
566            (
567                meta.n_symbols,
568                meta.n_bits,
569                meta.train_step,
570                meta.retrain_train_step,
571                meta.block_start,
572                meta.last_retrain_pos,
573                meta.elapsed_seconds,
574            )
575        }
576        None => (0, 0.0, 0, 0, 0, 0, 0.0),
577    };
578
579    let mut completed_blocks = block_start / block_len;
580    let session_start = Instant::now();
581    let mut last_symbols = n_symbols;
582    let mut last_bits = n_bits;
583    let mut last_time = Instant::now();
584    let mut freq_buf = vec![0u32; vocab_size];
585
586    logger.line(&format!(
587        "{:>8} {:>11} {:>11} {:>7} {:>7} {:>8} {:>8} {:>7} {:>9}",
588        "step", "symbols", "comp_bytes", "bpt_loc", "bpt_cum", "bpb_loc", "bpb_cum", "kS/s", "lr"
589    ));
590
591    while block_start < file_len {
592        if config.retrain.period > 0
593            && block_start.saturating_sub(last_retrain_pos) >= config.retrain.period
594        {
595            let mut rt_state = RetrainState {
596                model: &model,
597                optimizer: &mut retrain_optimizer,
598                rng: &mut rng,
599                config,
600                device: &device,
601                compute_dtype,
602                retrain_train_step: &mut retrain_train_step,
603                logger: &mut logger,
604            };
605            run_prefix_retrain(&mut rt_state, &file_data, block_start)?;
606            last_retrain_pos = block_start;
607        }
608
609        let cur_block_len = (file_len - block_start).min(block_len);
610        let block_stride = cur_block_len / batch;
611        let stream_base: Vec<usize> = (0..batch).map(|b| block_start + b * block_stride).collect();
612
613        for stream_pos in 0..block_stride {
614            let window = stream_pos.min(ctx_len);
615            let seq_len = window.max(1);
616            let mut inp_vec = Vec::with_capacity(batch * seq_len);
617            if window == 0 {
618                inp_vec.resize(batch, 0u32);
619            } else {
620                let start_off = stream_pos - window;
621                for &base in &stream_base {
622                    for off in start_off..(start_off + window) {
623                        inp_vec.push(u32::from(file_data[base + off]));
624                    }
625                }
626            }
627            let inp_t = Tensor::from_vec(inp_vec, (batch, seq_len), &device)?;
628
629            let lr = lr_at(
630                train_step,
631                config.optim.lr,
632                min_lr,
633                config.optim.warmup,
634                total_steps,
635            );
636            optimizer.set_learning_rate(lr);
637
638            let logits = {
639                let mut eval_mode = ForwardMode::Eval;
640                let slice = if config.project_only_targets {
641                    OutputSlice::Last
642                } else {
643                    OutputSlice::All
644                };
645                model.forward(&inp_t, slice, &mut eval_mode, compute_dtype)?
646            };
647
648            let code_logits = logits
649                .narrow(1, logits.dim(1)? - 1, 1)?
650                .squeeze(1)?
651                .to_dtype(DType::F32)?;
652
653            let cached_cpu_logits = if encoder.is_some() || decoder.is_some() {
654                let cpu_logits = code_logits
655                    .detach()
656                    .to_device(&Device::Cpu)?
657                    .contiguous()?
658                    .to_vec2::<f32>()?;
659                for (i, row_logits) in cpu_logits.iter().enumerate() {
660                    make_freq_table_into(row_logits, &mut freq_buf);
661                    let pos = stream_base[i] + stream_pos;
662                    if let Some(dec) = &mut decoder {
663                        let sym = dec.read(&freq_buf)?;
664                        file_data[pos] = sym as u16;
665                    } else if let Some(enc) = &mut encoder {
666                        enc.write(&freq_buf, usize::from(file_data[pos]))?;
667                    }
668                }
669                Some(cpu_logits)
670            } else {
671                None
672            };
673
674            let target_vec: Vec<u32> = stream_base
675                .iter()
676                .map(|&base| u32::from(file_data[base + stream_pos]))
677                .collect();
678            let code_targets = Tensor::from_vec(target_vec.clone(), batch, &device)?;
679
680            let (loss, code_nats) = match config.online_loss {
681                OnlineLoss::Code => {
682                    let loss = candle_nn::loss::cross_entropy(&code_logits, &code_targets)?;
683                    let nats = if let Some(rows) = cached_cpu_logits.as_ref() {
684                        cross_entropy_sum_cpu(rows, &target_vec)
685                    } else {
686                        f64::from(loss.to_scalar::<f32>()?) * (batch as f64)
687                    };
688                    (loss, nats)
689                }
690                OnlineLoss::Window => {
691                    let nats = if let Some(rows) = cached_cpu_logits.as_ref() {
692                        cross_entropy_sum_cpu(rows, &target_vec)
693                    } else {
694                        cross_entropy_sum_detached(&code_logits, &code_targets)?
695                    };
696                    let mut win_tgts = Vec::with_capacity(batch * seq_len);
697                    if window == 0 {
698                        for &base in &stream_base {
699                            win_tgts.push(u32::from(file_data[base + stream_pos]));
700                        }
701                    } else {
702                        let start_off = stream_pos - window + 1;
703                        for &base in &stream_base {
704                            for off in start_off..(start_off + window) {
705                                win_tgts.push(u32::from(file_data[base + off]));
706                            }
707                        }
708                    }
709                    let tgt_t = Tensor::from_vec(win_tgts, batch * seq_len, &device)?;
710                    let flat_logits = logits
711                        .to_dtype(DType::F32)?
712                        .reshape((batch * seq_len, vocab_size))?;
713                    let loss = candle_nn::loss::cross_entropy(&flat_logits, &tgt_t)?;
714                    (loss, nats)
715                }
716            };
717
718            let grads = loss.backward()?;
719            let _grad_norm = optimizer.step_clipped(&grads, config.optim.clip)?;
720
721            n_bits += code_nats / LOG2;
722            n_symbols += batch;
723            train_step += 1;
724
725            let is_last = block_start + cur_block_len >= file_len && stream_pos + 1 == block_stride;
726            if (config.log_interval > 0 && n_symbols - last_symbols >= config.log_interval)
727                || is_last
728            {
729                let now = Instant::now();
730                let dt = now.duration_since(last_time).as_secs_f64().max(1e-6);
731                let delta_syms = (n_symbols - last_symbols) as f64;
732                let local_bpt = (n_bits - last_bits) / delta_syms;
733                let cum_bpt = n_bits / (n_symbols as f64);
734                let local_bpb = bytes_per_token.map_or(f64::NAN, |b| local_bpt / b);
735                let cum_bpb = bytes_per_token.map_or(f64::NAN, |b| cum_bpt / b);
736                let ks_per_s = delta_syms / (dt * 1000.0);
737                logger.line(&format!(
738                    "{train_step:8} {n_symbols:11} {:11.0} {local_bpt:7.3} {cum_bpt:7.3} {local_bpb:8.4} {cum_bpb:8.4} {ks_per_s:7.2} {lr:9.3e}",
739                    n_bits / 8.0
740                ));
741                last_symbols = n_symbols;
742                last_bits = n_bits;
743                last_time = now;
744            }
745        }
746
747        if mode == CodecMode::Decompress {
748            if let Some(out_file) = &mut decode_out_file {
749                let end = (block_start + cur_block_len).min(original_len);
750                if block_start < end {
751                    for &tok in &file_data[block_start..end] {
752                        out_file.write_all(&tok.to_be_bytes())?;
753                    }
754                    out_file.flush()?;
755                }
756            }
757        }
758
759        block_start += cur_block_len;
760        completed_blocks += 1;
761
762        let requested_stop = stop_flag.load(Ordering::Relaxed)
763            || config.checkpoint_dir.as_deref().is_some_and(stop_requested);
764        let test_stop = config
765            .stop_after_blocks
766            .is_some_and(|limit| completed_blocks >= limit);
767        let periodic = config.checkpoint_dir.is_some()
768            && completed_blocks % config.checkpoint_every_blocks == 0;
769
770        if block_start < file_len && (periodic || requested_stop || test_stop) {
771            let reason = if requested_stop {
772                "stop-request"
773            } else if test_stop {
774                "test-stop"
775            } else {
776                "periodic"
777            };
778            if let Some(dir) = &config.checkpoint_dir {
779                let mut tensors = HashMap::new();
780                model.export_tensors(&mut tensors)?;
781                let online_adam_steps = optimizer.export_state("online_adam", &mut tensors)?;
782                let retrain_adam_steps =
783                    retrain_optimizer.export_state("retrain_adam", &mut tensors)?;
784
785                let (encoder_state, bit_out_state) = match &mut encoder {
786                    Some(enc) => (Some(enc.state()), Some(enc.bit_stream_mut().state()?)),
787                    None => (None, None),
788                };
789                let (decoder_state, bit_in_state) = match &mut decoder {
790                    Some(dec) => (Some(dec.state()), Some(dec.bit_stream_mut().state()?)),
791                    None => (None, None),
792                };
793                let output_position = match &mut decode_out_file {
794                    Some(out) => {
795                        out.flush()?;
796                        out.get_ref().sync_all()?;
797                        Some(out.stream_position()?)
798                    }
799                    None => None,
800                };
801
802                let mut meta = CheckpointMetadata {
803                    version: CHECKPOINT_VERSION,
804                    config_signature: checkpoint_sig.clone(),
805                    project_only_targets: config.project_only_targets,
806                    retrain_early_epochs: config.retrain.early_epochs,
807                    retrain_early_until: config.retrain.early_until,
808                    input_size,
809                    original_len,
810                    file_len,
811                    initial_prior_sha256: prior_info.as_ref().map(|p| p.archive_sha256.clone()),
812                    initial_prior_vocab_sha256: prior_info.as_ref().map(|p| p.vocab_sha256.clone()),
813                    tensors_sha256: String::new(),
814                    online_adam_steps,
815                    retrain_adam_steps,
816                    rng_state: rng.state(),
817                    encoder_state,
818                    decoder_state,
819                    bit_out_state,
820                    bit_in_state,
821                    output_position,
822                    n_symbols,
823                    n_bits,
824                    train_step,
825                    retrain_train_step,
826                    block_start,
827                    last_retrain_pos,
828                    elapsed_seconds: elapsed_before + session_start.elapsed().as_secs_f64(),
829                    reason: reason.to_owned(),
830                };
831                save_checkpoint(dir, &mut meta, &tensors)?;
832                logger.line(&format!(
833                    "checkpoint saved at block_start={block_start} ({reason})"
834                ));
835            }
836            if requested_stop || test_stop {
837                if let Some(out) = &mut decode_out_file {
838                    out.flush()?;
839                    out.get_ref().sync_all()?;
840                }
841                if let Some(enc) = &mut encoder {
842                    enc.bit_stream_mut().flush()?;
843                    enc.bit_stream_mut().get_ref().get_ref().sync_all()?;
844                }
845                return Ok(RunOutcome::RequeueRequested {
846                    block_start,
847                    n_symbols,
848                });
849            }
850        }
851    }
852
853    if let Some(enc) = encoder.take() {
854        let mut out_writer = enc.into_inner()?;
855        out_writer.flush()?;
856        out_writer.get_ref().sync_all()?;
857    }
858    if let Some(mut out_file) = decode_out_file.take() {
859        out_file.flush()?;
860        out_file.get_ref().sync_all()?;
861    }
862
863    let elapsed_seconds = elapsed_before + session_start.elapsed().as_secs_f64();
864    let actual_compressed_bytes = match mode {
865        CodecMode::Compress => Some(fs::metadata(output_path)?.len()),
866        CodecMode::Decompress => Some(fs::metadata(input_path)?.len()),
867        CodecMode::EstimateOnly => None,
868    };
869    let actual_bits_per_byte = match (actual_compressed_bytes, config.raw_bytes) {
870        (Some(bytes), Some(raw)) if raw > 0 => Some((bytes as f64) * 8.0 / (raw as f64)),
871        _ => None,
872    };
873    let modeled_bits_per_token = if n_symbols > 0 {
874        n_bits / (n_symbols as f64)
875    } else {
876        0.0
877    };
878    let modeled_bits_per_byte = bytes_per_token.map(|b| modeled_bits_per_token / b);
879
880    let summary = RunSummary {
881        symbols: n_symbols,
882        original_tokens: original_len,
883        modeled_bits: n_bits,
884        modeled_payload_bytes: n_bits / 8.0,
885        modeled_bits_per_token,
886        modeled_bits_per_byte,
887        actual_compressed_bytes,
888        actual_bits_per_byte,
889        vocab_bytes: config.vocab_bytes,
890        prior_archive_bytes: prior_info.as_ref().map_or(0, |p| p.archive_bytes),
891        initial_prior_sha256: prior_info.as_ref().map(|p| p.archive_sha256.clone()),
892        prior_source_depth: prior_info.as_ref().map_or(0, |p| p.source_depth),
893        prior_initialization_kind: prior_info.as_ref().map(|p| {
894            match p.initialization_kind {
895                tufazip_prior::InitializationKind::DepthGrowth => "depth-growth",
896                tufazip_prior::InitializationKind::LoopUntying => "loop-untying",
897            }
898            .to_owned()
899        }),
900        prior_block_mapping: prior_info.as_ref().map(|p| p.block_mapping.clone()),
901        elapsed_seconds,
902    };
903
904    if let Ok(json) = serde_json::to_string(&summary) {
905        logger.line(&format!("FINAL_RESULT {json}"));
906    }
907
908    if let Some(dir) = &config.checkpoint_dir {
909        mark_done(dir)?;
910    }
911
912    Ok(RunOutcome::Completed(summary))
913}
914
915#[cfg(test)]
916mod tests {
917    use std::path::PathBuf;
918
919    use tufazip_preprocess::write_tokens_be16;
920
921    use super::*;
922    use crate::clip_grad_norm;
923    use crate::config::{
924        AutocastPolicy, ComputeDType, DeviceChoice, ModelConfig, NormStyle, PositionType,
925    };
926
927    fn tiny_tokens(len: usize, vocab_size: usize) -> Vec<u16> {
928        (0..len)
929            .map(|i| ((i * 7 + (i / 5) * 3) % vocab_size) as u16)
930            .collect()
931    }
932
933    #[test]
934    fn forward_backward_gradient_variants_and_bf16() {
935        let variants = [
936            (
937                PositionType::Rope,
938                NormStyle::Peri,
939                false,
940                true,
941                OutputSlice::Last,
942                DType::F32,
943            ),
944            (
945                PositionType::LearnedRelative,
946                NormStyle::Post,
947                true,
948                false,
949                OutputSlice::All,
950                DType::F32,
951            ),
952            (
953                PositionType::Rope,
954                NormStyle::Peri,
955                false,
956                true,
957                OutputSlice::Last,
958                DType::BF16,
959            ),
960        ];
961
962        for (pos, norm, tie, qk_norm, slice, dt) in variants {
963            let mut rng = DeterministicRng::from_seed_u64(42);
964            let cfg = ModelConfig {
965                vocab_size: 32,
966                n_layer: 2,
967                d_model: 16,
968                n_head: 2,
969                d_inner: 32,
970                ctx_len: 8,
971                dropout: 0.25,
972                rope_base: 10_000.0,
973                tie_embeddings: tie,
974                position_type: pos,
975                qk_norm,
976                norm_style: norm,
977                block_schedule: None,
978            };
979            let model = Transformer::new(&cfg, &Device::Cpu, &mut rng).unwrap();
980            let mut adam = Adam::new(model.named_vars(), 1e-3, 0.0, 0.999, 1e-8).unwrap();
981
982            let inp =
983                Tensor::from_vec(vec![1u32, 2, 3, 4, 5, 6, 7, 8], (2, 4), &Device::Cpu).unwrap();
984            let logits = {
985                let mut mode = ForwardMode::Train(&mut rng);
986                model.forward(&inp, slice, &mut mode, dt).unwrap()
987            };
988            let out_seq = logits.dim(1).unwrap();
989            let tgt_len = 2 * out_seq;
990            let tgts: Vec<u32> = (0..tgt_len).map(|i| (i % 32) as u32).collect();
991            let tgt_t = Tensor::from_vec(tgts, tgt_len, &Device::Cpu).unwrap();
992            let flat = logits
993                .to_dtype(DType::F32)
994                .unwrap()
995                .reshape((tgt_len, 32))
996                .unwrap();
997            let loss = candle_nn::loss::cross_entropy(&flat, &tgt_t).unwrap();
998            let mut grads = loss.backward().unwrap();
999            let norm_val = clip_grad_norm(model.named_vars(), &mut grads, 0.25).unwrap();
1000            assert!(
1001                norm_val.is_finite(),
1002                "non-finite grad norm for {pos:?} {dt:?}"
1003            );
1004            adam.step(&grads).unwrap();
1005            assert!(model.weight_norm().unwrap().is_finite());
1006        }
1007    }
1008
1009    #[test]
1010    fn lossless_compress_decompress_roundtrip_code_and_window() {
1011        for (online_loss, project_only) in [(OnlineLoss::Code, true), (OnlineLoss::Window, false)] {
1012            let dir = tempfile::tempdir().unwrap();
1013            let in_pp = dir.path().join("input.pp");
1014            let comp_bin = dir.path().join("comp.bin");
1015            let out_pp = dir.path().join("recovered.pp");
1016
1017            let tokens = tiny_tokens(77, 64);
1018            write_tokens_be16(&in_pp, &tokens).unwrap();
1019
1020            let mut cfg = CodecConfig::tiny(64);
1021            cfg.model.n_layer = 2;
1022            cfg.model.d_model = 32;
1023            cfg.model.n_head = 2;
1024            cfg.model.d_inner = 64;
1025            cfg.model.ctx_len = 16;
1026            cfg.model.dropout = 0.25;
1027            cfg.batch_size = 4;
1028            cfg.block_len = 32;
1029            cfg.retrain.period = 32;
1030            cfg.retrain.len = 64;
1031            cfg.retrain.batch_size = 4;
1032            cfg.retrain.update_len = 8;
1033            cfg.online_loss = online_loss;
1034            cfg.project_only_targets = project_only;
1035            cfg.autocast_policy = AutocastPolicy::Auto;
1036            cfg.dtype = ComputeDType::Fp32;
1037            cfg.device = DeviceChoice::Cpu;
1038
1039            let stop = Arc::new(AtomicBool::new(false));
1040            let res_enc =
1041                run_codec(&in_pp, &comp_bin, &cfg, CodecMode::Compress, stop.clone()).unwrap();
1042            assert!(matches!(res_enc, RunOutcome::Completed(_)));
1043
1044            let res_dec = run_codec(&comp_bin, &out_pp, &cfg, CodecMode::Decompress, stop).unwrap();
1045            assert!(matches!(res_dec, RunOutcome::Completed(_)));
1046
1047            assert_eq!(
1048                std::fs::read(&out_pp).unwrap(),
1049                std::fs::read(&in_pp).unwrap(),
1050                "roundtrip mismatch for {online_loss:?}"
1051            );
1052        }
1053    }
1054
1055    #[test]
1056    fn checkpoint_resume_bit_exact_compress_and_decompress() {
1057        let dir = tempfile::tempdir().unwrap();
1058        let in_pp = dir.path().join("input.pp");
1059        let full_bin = dir.path().join("full.bin");
1060        let resumed_bin = dir.path().join("resumed.bin");
1061        let resumed_pp = dir.path().join("resumed.pp");
1062        let enc_ckpt = dir.path().join("enc_ckpt");
1063        let dec_ckpt = dir.path().join("dec_ckpt");
1064
1065        let tokens = tiny_tokens(113, 48);
1066        write_tokens_be16(&in_pp, &tokens).unwrap();
1067
1068        let mut base_cfg = CodecConfig::tiny(48);
1069        base_cfg.model.n_layer = 2;
1070        base_cfg.model.d_model = 32;
1071        base_cfg.model.n_head = 2;
1072        base_cfg.model.d_inner = 64;
1073        base_cfg.model.ctx_len = 12;
1074        base_cfg.model.dropout = 0.25;
1075        base_cfg.batch_size = 4;
1076        base_cfg.block_len = 32;
1077        base_cfg.retrain.period = 32;
1078        base_cfg.retrain.len = 96;
1079        base_cfg.retrain.batch_size = 4;
1080        base_cfg.retrain.update_len = 8;
1081
1082        let stop = Arc::new(AtomicBool::new(false));
1083        let full_out = run_codec(
1084            &in_pp,
1085            &full_bin,
1086            &base_cfg,
1087            CodecMode::Compress,
1088            stop.clone(),
1089        )
1090        .unwrap();
1091        assert!(matches!(full_out, RunOutcome::Completed(_)));
1092
1093        // Stop after 1 block, then resume compression
1094        let mut stop_cfg = base_cfg.clone();
1095        stop_cfg.checkpoint_dir = Some(enc_ckpt.clone());
1096        stop_cfg.stop_after_blocks = Some(1);
1097        let paused = run_codec(
1098            &in_pp,
1099            &resumed_bin,
1100            &stop_cfg,
1101            CodecMode::Compress,
1102            stop.clone(),
1103        )
1104        .unwrap();
1105        assert!(matches!(paused, RunOutcome::RequeueRequested { .. }));
1106
1107        stop_cfg.stop_after_blocks = None;
1108        let finished = run_codec(
1109            &in_pp,
1110            &resumed_bin,
1111            &stop_cfg,
1112            CodecMode::Compress,
1113            stop.clone(),
1114        )
1115        .unwrap();
1116        assert!(matches!(finished, RunOutcome::Completed(_)));
1117        assert!(crate::checkpoint::is_done(&enc_ckpt));
1118        assert_eq!(
1119            std::fs::read(&resumed_bin).unwrap(),
1120            std::fs::read(&full_bin).unwrap()
1121        );
1122
1123        // Stop after 1 block, then resume decompression
1124        let mut dec_cfg = base_cfg;
1125        dec_cfg.checkpoint_dir = Some(dec_ckpt.clone());
1126        dec_cfg.stop_after_blocks = Some(1);
1127        let dec_paused = run_codec(
1128            &resumed_bin,
1129            &resumed_pp,
1130            &dec_cfg,
1131            CodecMode::Decompress,
1132            stop.clone(),
1133        )
1134        .unwrap();
1135        assert!(matches!(dec_paused, RunOutcome::RequeueRequested { .. }));
1136
1137        dec_cfg.stop_after_blocks = None;
1138        let dec_done = run_codec(
1139            &resumed_bin,
1140            &resumed_pp,
1141            &dec_cfg,
1142            CodecMode::Decompress,
1143            stop,
1144        )
1145        .unwrap();
1146        assert!(matches!(dec_done, RunOutcome::Completed(_)));
1147        assert_eq!(
1148            std::fs::read(&resumed_pp).unwrap(),
1149            std::fs::read(&in_pp).unwrap()
1150        );
1151    }
1152
1153    #[test]
1154    fn v2_model_loads_real_weights_bin_and_runs_forward() {
1155        let assets = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../assets");
1156        let weights_path = assets.join("weights.bin");
1157        let voc_path = assets.join("enwik9.voc");
1158
1159        let cfg = ModelConfig::v2();
1160        let mut rng = DeterministicRng::from_seed_u64(1111);
1161        let (model, info) =
1162            Transformer::from_prior(&cfg, &weights_path, &voc_path, 1111, &Device::Cpu, &mut rng)
1163                .unwrap();
1164        let (total, non_emb) = model.param_counts();
1165        assert_eq!(total, 162_585_600);
1166        assert_eq!(non_emb, 154_194_944);
1167        assert_eq!(info.archive_bytes, 4_334_741);
1168        assert!(!info.initial_state_sha256.is_empty());
1169
1170        let idx = Tensor::from_vec(vec![0u32, 32, 100, 200], (1, 4), &Device::Cpu).unwrap();
1171        let mut eval = ForwardMode::Eval;
1172        let logits = model
1173            .forward(&idx, OutputSlice::Last, &mut eval, DType::F32)
1174            .unwrap();
1175        assert_eq!(logits.dims(), &[1, 1, 4097]);
1176        let vals = logits.flatten_all().unwrap().to_vec1::<f32>().unwrap();
1177        assert!(vals.iter().all(|v| v.is_finite()));
1178    }
1179}