Skip to main content

tufazip/
main.rs

1//! Pure-Rust CLI for `tufazip` neural compression, decompression, preprocessing, and prior inspection.
2
3use std::collections::BTreeMap;
4use std::fs;
5use std::path::{Path, PathBuf};
6use std::process::ExitCode;
7use std::sync::atomic::AtomicBool;
8use std::sync::Arc;
9
10use anyhow::{bail, Context, Result};
11use clap::{Args, Parser, Subcommand, ValueEnum};
12use tracing_subscriber::EnvFilter;
13use tufazip_codec::{
14    clear_stop, is_done, run_codec, AutocastPolicy, CodecConfig, CodecMode, ComputeDType,
15    DeviceChoice, NormStyle, OnlineLoss, PiecewiseLrSchedule, PositionType, RunOutcome,
16    REQUEUE_EXIT_CODE,
17};
18use tufazip_preprocess::{
19    decode_file, encode_bytes, encode_file, encode_with_dictionary, write_tokens_be16, Dictionary,
20    EncodeConfig, SortOrder,
21};
22use tufazip_prior::{read_archive, sha256_file, InitializationKind};
23
24const PAYLOAD_SHA256: &str = "198451a21f64386b487b8ea97e141562db355eec9cc9af34e7bbb6a022bcce77";
25const PAYLOAD_BYTES: u64 = 100_958_324;
26const VOCAB_SHA256: &str = "a63e517783e84a778f3f08c1f2321f3e2035e64d34c8e9206f03591460e44629";
27const VOCAB_BYTES: u64 = 34_421;
28const WEIGHTS_SHA256: &str = "e94c089f8eb985a87307dca3a7702385fcaccd14d95743811f8788f7da24448d";
29const WEIGHTS_BYTES: u64 = 4_334_741;
30const TOKENS_SHA256: &str = "102fa82b3f8932b37a1ec0d9262e555d23d17852685fe5fb33680dbee19dd694";
31const TOKENS_BYTES: u64 = 570_405_660;
32const RAW_SHA256: &str = "159b85351e5f76e60cbe32e04c677847a9ecba3adc79addab6f4c6c7aa3744bc";
33const RAW_BYTES: u64 = 1_000_000_000;
34
35#[derive(Debug, Parser)]
36#[command(
37    name = "tufazip",
38    version = "0.2.0",
39    about = "Pure-Rust port of tufazip v2 neural lossless compressor and decompressor"
40)]
41struct Cli {
42    #[command(subcommand)]
43    command: Command,
44}
45
46#[derive(Debug, Subcommand)]
47enum Command {
48    /// End-to-end compression of a raw file (preprocess + neural arithmetic coding).
49    Compress {
50        /// Raw input file path.
51        input: PathBuf,
52        /// Compressed output artifact path.
53        output: PathBuf,
54        /// Path to write the induced vocabulary (or read from with --use-existing-vocab).
55        #[arg(long)]
56        vocab: PathBuf,
57        /// Reuse an existing vocabulary file (e.g. assets/enwik9.voc) instead of inducing a new one.
58        #[arg(long)]
59        use_existing_vocab: bool,
60        /// Maximum dictionary size including the 256 base byte symbols.
61        #[arg(long, default_value_t = 4097)]
62        max_words: usize,
63        /// Minimum bigram frequency for dictionary induction.
64        #[arg(long, default_value_t = 5)]
65        min_freq: u32,
66        /// Sort learned vocabulary by decreasing frequency.
67        #[arg(short = 's', long)]
68        sort_by_freq: bool,
69        /// Optional path to keep the intermediate .pp big-endian token file.
70        #[arg(long)]
71        keep_tokens: Option<PathBuf>,
72        /// Neural model and training overrides.
73        #[command(flatten)]
74        codec: CodecArgs,
75    },
76    /// End-to-end decompression of a compressed tufazip artifact back to raw bytes.
77    Decompress {
78        /// Compressed input artifact path.
79        input: PathBuf,
80        /// Decoded raw output file path.
81        output: PathBuf,
82        /// Vocabulary (.voc) file used during compression.
83        #[arg(long)]
84        vocab: PathBuf,
85        /// Optional path to keep the intermediate .pp big-endian token file.
86        #[arg(long)]
87        keep_tokens: Option<PathBuf>,
88        /// Neural model and training overrides.
89        #[command(flatten)]
90        codec: CodecArgs,
91    },
92    /// Low-level neural codec operating directly on .pp token files (mirrors nn.py).
93    Codec {
94        /// Input .pp token stream (when compressing) or compressed .bin stream (when decompressing).
95        #[arg(long)]
96        input: PathBuf,
97        /// Output file path.
98        #[arg(long)]
99        output: Option<PathBuf>,
100        /// Decompress instead of compressing.
101        #[arg(long)]
102        decompress: bool,
103        /// Skip arithmetic coding and only train/report modeled bits.
104        #[arg(long)]
105        no_coding: bool,
106        /// Neural model and training overrides.
107        #[command(flatten)]
108        codec: CodecArgs,
109    },
110    /// Standalone NNCP dictionary preprocessor (mirrors preprocess.c).
111    Preprocess {
112        #[command(subcommand)]
113        action: PreprocessAction,
114    },
115    /// Deterministic verifier/decoder wrapper for the tufazip v2 enwik9 artifact.
116    V2Decode {
117        /// Path to enwik9-v2.bin payload.
118        payload: PathBuf,
119        /// Output path for decoded enwik9.
120        output: PathBuf,
121        /// Resumable state directory.
122        #[arg(default_value = "tufazip-v2-state")]
123        state: PathBuf,
124        /// Path to enwik9.voc (defaults to ./assets/enwik9.voc).
125        #[arg(long)]
126        vocab: Option<PathBuf>,
127        /// Path to weights.bin (defaults to ./assets/weights.bin).
128        #[arg(long)]
129        weights: Option<PathBuf>,
130        /// Execution device backend.
131        #[arg(long, value_enum, default_value_t = CliDevice::Auto)]
132        device: CliDevice,
133        /// Skip strict v2 SHA-256 verification checks (for testing on non-B200 hardware).
134        #[arg(long)]
135        skip_verify: bool,
136    },
137    /// Inspect and optionally verify a QPI1/QPR1/LRA1/QLA1 prior archive.
138    InspectPrior {
139        /// Path to packed prior file (e.g. assets/weights.bin).
140        archive: PathBuf,
141        /// Optional vocabulary file to verify against the prior's vocab_sha256.
142        #[arg(long)]
143        vocab: Option<PathBuf>,
144        /// Print machine-readable JSON instead of human-readable summary.
145        #[arg(long)]
146        json: bool,
147    },
148}
149
150#[derive(Debug, Subcommand)]
151enum PreprocessAction {
152    /// Induce dictionary from raw file and output big-endian u16 tokens (`preprocess c`).
153    Encode {
154        /// Output dictionary path (.voc).
155        dict: PathBuf,
156        /// Raw input file path.
157        input: PathBuf,
158        /// Output big-endian u16 token stream (.pp).
159        output: PathBuf,
160        /// Approximate maximum vocabulary size (including 256 base bytes).
161        n_words: usize,
162        /// Minimum bigram frequency.
163        min_freq: u32,
164        /// Sort dictionary entries by decreasing frequency (`-s`).
165        #[arg(short = 's', long)]
166        sort_by_freq: bool,
167        /// Write debug frequency table (`-D`).
168        #[arg(short = 'D', long)]
169        debug_dict: Option<PathBuf>,
170    },
171    /// Decode big-endian u16 tokens back into original raw bytes (`preprocess d`).
172    Decode {
173        /// Dictionary path (.voc).
174        dict: PathBuf,
175        /// Input big-endian u16 token stream (.pp).
176        input: PathBuf,
177        /// Decoded raw output file path.
178        output: PathBuf,
179    },
180}
181
182#[derive(Debug, Clone, Copy, ValueEnum)]
183enum Preset {
184    Tiny,
185    V2,
186}
187
188#[derive(Debug, Clone, Copy, ValueEnum)]
189enum CliPositionType {
190    Rope,
191    LearnedRelative,
192}
193
194#[derive(Debug, Clone, Copy, ValueEnum)]
195enum CliNormStyle {
196    Peri,
197    Post,
198}
199
200#[derive(Debug, Clone, Copy, ValueEnum)]
201enum CliOnlineLoss {
202    Code,
203    Window,
204}
205
206#[derive(Debug, Clone, Copy, ValueEnum)]
207enum CliDType {
208    Fp32,
209    Bf16,
210}
211
212#[derive(Debug, Clone, Copy, ValueEnum)]
213enum CliDevice {
214    Auto,
215    Cpu,
216    Metal,
217    Cuda,
218}
219
220#[derive(Debug, Args)]
221struct CodecArgs {
222    /// Configuration preset (`tiny` for fast local runs, `v2` for full enwik9 settings).
223    #[arg(long, value_enum, default_value_t = Preset::Tiny)]
224    preset: Preset,
225    #[arg(long)]
226    vocab_size: Option<usize>,
227    #[arg(long)]
228    n_layer: Option<usize>,
229    #[arg(long)]
230    d_model: Option<usize>,
231    #[arg(long)]
232    n_head: Option<usize>,
233    #[arg(long)]
234    d_inner: Option<usize>,
235    #[arg(long)]
236    ctx_len: Option<usize>,
237    #[arg(long)]
238    dropout: Option<f32>,
239    #[arg(long)]
240    tie_embeddings: Option<bool>,
241    #[arg(long, value_enum)]
242    position_type: Option<CliPositionType>,
243    #[arg(long)]
244    no_qk_norm: bool,
245    #[arg(long, value_enum)]
246    norm_style: Option<CliNormStyle>,
247    #[arg(long)]
248    batch_size: Option<usize>,
249    #[arg(long)]
250    block_len: Option<usize>,
251    #[arg(long, value_enum)]
252    online_loss: Option<CliOnlineLoss>,
253    #[arg(long)]
254    project_only_targets: Option<bool>,
255    #[arg(long)]
256    lr: Option<f64>,
257    #[arg(long)]
258    min_lr_ratio: Option<f64>,
259    #[arg(long)]
260    warmup: Option<usize>,
261    #[arg(long)]
262    clip: Option<f64>,
263    #[arg(long)]
264    adam_eps: Option<f64>,
265    #[arg(long)]
266    beta1: Option<f64>,
267    #[arg(long)]
268    beta2: Option<f64>,
269    #[arg(long)]
270    retrain_period: Option<usize>,
271    #[arg(long)]
272    retrain_len: Option<usize>,
273    #[arg(long)]
274    retrain_epochs: Option<usize>,
275    #[arg(long)]
276    retrain_early_epochs: Option<usize>,
277    #[arg(long)]
278    retrain_early_until: Option<usize>,
279    #[arg(long)]
280    retrain_lr: Option<String>,
281    #[arg(long)]
282    retrain_update_len: Option<usize>,
283    #[arg(long)]
284    retrain_batch_size: Option<usize>,
285    #[arg(long)]
286    seed: Option<u64>,
287    #[arg(long, value_enum)]
288    dtype: Option<CliDType>,
289    #[arg(long)]
290    force_bf16: bool,
291    #[arg(long, value_enum)]
292    device: Option<CliDevice>,
293    #[arg(long)]
294    max_tokens: Option<usize>,
295    #[arg(long)]
296    raw_bytes: Option<u64>,
297    #[arg(long)]
298    vocab_bytes: Option<u64>,
299    #[arg(long)]
300    vocab_file: Option<PathBuf>,
301    #[arg(long)]
302    initial_prior: Option<PathBuf>,
303    #[arg(long)]
304    log_interval: Option<usize>,
305    #[arg(long)]
306    checkpoint_dir: Option<PathBuf>,
307    #[arg(long)]
308    checkpoint_every_blocks: Option<usize>,
309    #[arg(long)]
310    stop_after_blocks: Option<usize>,
311    #[arg(long)]
312    run_dir: Option<PathBuf>,
313}
314
315impl CodecArgs {
316    fn build_config(&self, inferred_vocab_size: Option<usize>) -> Result<CodecConfig> {
317        let mut cfg = match self.preset {
318            Preset::V2 => CodecConfig::v2(),
319            Preset::Tiny => CodecConfig::tiny(inferred_vocab_size.unwrap_or(4097)),
320        };
321        if let Some(v) = self.vocab_size {
322            cfg.model.vocab_size = v;
323        } else if let Some(v) = inferred_vocab_size {
324            if matches!(self.preset, Preset::Tiny) {
325                cfg.model.vocab_size = v;
326            }
327        }
328        if let Some(v) = self.n_layer {
329            cfg.model.n_layer = v;
330        }
331        if let Some(v) = self.d_model {
332            cfg.model.d_model = v;
333        }
334        if let Some(v) = self.n_head {
335            cfg.model.n_head = v;
336        }
337        if let Some(v) = self.d_inner {
338            cfg.model.d_inner = v;
339        }
340        if let Some(v) = self.ctx_len {
341            cfg.model.ctx_len = v;
342        }
343        if let Some(v) = self.dropout {
344            cfg.model.dropout = v;
345        }
346        if let Some(v) = self.tie_embeddings {
347            cfg.model.tie_embeddings = v;
348        }
349        if let Some(v) = self.position_type {
350            cfg.model.position_type = match v {
351                CliPositionType::Rope => PositionType::Rope,
352                CliPositionType::LearnedRelative => PositionType::LearnedRelative,
353            };
354        }
355        if self.no_qk_norm {
356            cfg.model.qk_norm = false;
357        }
358        if let Some(v) = self.norm_style {
359            cfg.model.norm_style = match v {
360                CliNormStyle::Peri => NormStyle::Peri,
361                CliNormStyle::Post => NormStyle::Post,
362            };
363        }
364        if let Some(v) = self.batch_size {
365            cfg.batch_size = v;
366        }
367        if let Some(v) = self.block_len {
368            cfg.block_len = v;
369        }
370        if let Some(v) = self.online_loss {
371            cfg.online_loss = match v {
372                CliOnlineLoss::Code => OnlineLoss::Code,
373                CliOnlineLoss::Window => OnlineLoss::Window,
374            };
375        }
376        if let Some(v) = self.project_only_targets {
377            cfg.project_only_targets = v;
378        }
379        if let Some(v) = self.lr {
380            cfg.optim.lr = v;
381        }
382        if let Some(v) = self.min_lr_ratio {
383            cfg.optim.min_lr_ratio = v;
384        }
385        if let Some(v) = self.warmup {
386            cfg.optim.warmup = v;
387        }
388        if let Some(v) = self.clip {
389            cfg.optim.clip = v;
390        }
391        if let Some(v) = self.adam_eps {
392            cfg.optim.adam_eps = v;
393        }
394        if let Some(v) = self.beta1 {
395            cfg.optim.beta1 = v;
396        }
397        if let Some(v) = self.beta2 {
398            cfg.optim.beta2 = v;
399        }
400        if let Some(v) = self.retrain_period {
401            cfg.retrain.period = v;
402        }
403        if let Some(v) = self.retrain_len {
404            cfg.retrain.len = v;
405        }
406        if let Some(v) = self.retrain_epochs {
407            cfg.retrain.epochs = v;
408        }
409        if let Some(v) = self.retrain_early_epochs {
410            cfg.retrain.early_epochs = v;
411        }
412        if let Some(v) = self.retrain_early_until {
413            cfg.retrain.early_until = v;
414        }
415        if let Some(ref spec) = self.retrain_lr {
416            cfg.retrain.lr_schedule = PiecewiseLrSchedule::parse(spec)?;
417        }
418        if let Some(v) = self.retrain_update_len {
419            cfg.retrain.update_len = v;
420        }
421        if let Some(v) = self.retrain_batch_size {
422            cfg.retrain.batch_size = v;
423        }
424        if let Some(v) = self.seed {
425            cfg.seed = v;
426        }
427        if let Some(v) = self.dtype {
428            cfg.dtype = match v {
429                CliDType::Fp32 => ComputeDType::Fp32,
430                CliDType::Bf16 => ComputeDType::Bf16,
431            };
432        }
433        if self.force_bf16 {
434            cfg.autocast_policy = AutocastPolicy::Force;
435        }
436        if let Some(v) = self.device {
437            cfg.device = map_device(v);
438        }
439        if let Some(v) = self.max_tokens {
440            cfg.max_tokens = Some(v);
441        }
442        if let Some(v) = self.raw_bytes {
443            cfg.raw_bytes = Some(v);
444        }
445        if let Some(v) = self.vocab_bytes {
446            cfg.vocab_bytes = v;
447        }
448        if let Some(ref p) = self.vocab_file {
449            cfg.vocab_file = Some(p.clone());
450        }
451        if let Some(ref p) = self.initial_prior {
452            cfg.initial_prior = Some(p.clone());
453        }
454        if let Some(v) = self.log_interval {
455            cfg.log_interval = v;
456        }
457        if let Some(ref p) = self.checkpoint_dir {
458            cfg.checkpoint_dir = Some(p.clone());
459        }
460        if let Some(v) = self.checkpoint_every_blocks {
461            cfg.checkpoint_every_blocks = v;
462        }
463        if let Some(v) = self.stop_after_blocks {
464            cfg.stop_after_blocks = Some(v);
465        }
466        if let Some(ref p) = self.run_dir {
467            cfg.run_dir = Some(p.clone());
468        }
469        Ok(cfg)
470    }
471}
472
473fn map_device(d: CliDevice) -> DeviceChoice {
474    match d {
475        CliDevice::Auto => DeviceChoice::Auto,
476        CliDevice::Cpu => DeviceChoice::Cpu,
477        CliDevice::Metal => DeviceChoice::Metal,
478        CliDevice::Cuda => DeviceChoice::Cuda,
479    }
480}
481
482fn check_file(path: &Path, expected_size: u64, expected_sha256: &str) -> Result<()> {
483    if !path.is_file() {
484        bail!("missing file: {}", path.display());
485    }
486    let size = fs::metadata(path)?.len();
487    if size != expected_size {
488        bail!(
489            "wrong size for {}: expected {expected_size} bytes, found {size}",
490            path.display()
491        );
492    }
493    let digest = sha256_file(path)?;
494    if digest != expected_sha256 {
495        bail!(
496            "wrong SHA-256 for {}: expected {expected_sha256}, found {digest}",
497            path.display()
498        );
499    }
500    Ok(())
501}
502
503fn resolve_default_asset(explicit: Option<PathBuf>, filename: &str) -> PathBuf {
504    if let Some(p) = explicit {
505        return p;
506    }
507    let cwd_asset = PathBuf::from("assets").join(filename);
508    if cwd_asset.is_file() {
509        return cwd_asset;
510    }
511    if let Ok(exe) = std::env::current_exe() {
512        if let Some(dir) = exe.parent() {
513            let sibling = dir.join(filename);
514            if sibling.is_file() {
515                return sibling;
516            }
517            let sibling_assets = dir.join("assets").join(filename);
518            if sibling_assets.is_file() {
519                return sibling_assets;
520            }
521        }
522    }
523    PathBuf::from(filename)
524}
525
526fn install_stop_handlers(stop_flag: &Arc<AtomicBool>) {
527    let _ = signal_hook::flag::register(signal_hook::consts::SIGINT, Arc::clone(stop_flag));
528    #[cfg(unix)]
529    {
530        let _ = signal_hook::flag::register(signal_hook::consts::SIGTERM, Arc::clone(stop_flag));
531        let _ = signal_hook::flag::register(signal_hook::consts::SIGUSR1, Arc::clone(stop_flag));
532    }
533}
534
535fn run(cli: Cli, stop_flag: Arc<AtomicBool>) -> Result<ExitCode> {
536    match cli.command {
537        Command::Compress {
538            input,
539            output,
540            vocab,
541            use_existing_vocab,
542            max_words,
543            min_freq,
544            sort_by_freq,
545            keep_tokens,
546            codec,
547        } => {
548            let raw_bytes = fs::metadata(&input)
549                .with_context(|| format!("reading {}", input.display()))?
550                .len();
551            let raw = fs::read(&input)?;
552            let tokens = if use_existing_vocab {
553                let dict = Dictionary::load(&vocab)?;
554                encode_with_dictionary(&raw, &dict)?
555            } else {
556                let order = if sort_by_freq {
557                    SortOrder::ByFrequency
558                } else {
559                    SortOrder::Discovery
560                };
561                let (dict, tokens, _report) = encode_bytes(
562                    &raw,
563                    &EncodeConfig {
564                        max_words,
565                        min_freq,
566                        sort_order: order,
567                        debug_dict_path: None,
568                    },
569                )?;
570                if let Some(parent) = vocab.parent() {
571                    if !parent.as_os_str().is_empty() {
572                        fs::create_dir_all(parent)?;
573                    }
574                }
575                dict.save(&vocab)?;
576                tokens
577            };
578
579            let dict = Dictionary::load(&vocab)?;
580            let vocab_bytes = fs::metadata(&vocab)?.len();
581            let mut cfg = codec.build_config(Some(dict.len()))?;
582            if cfg.raw_bytes.is_none() {
583                cfg.raw_bytes = Some(raw_bytes);
584            }
585            if cfg.vocab_bytes == 0 {
586                cfg.vocab_bytes = vocab_bytes;
587            }
588
589            let tmp_dir;
590            let pp_path = match keep_tokens {
591                Some(p) => {
592                    write_tokens_be16(&p, &tokens)?;
593                    p
594                }
595                None => {
596                    tmp_dir = tempfile::tempdir()?;
597                    let p = tmp_dir.path().join("tokens.pp");
598                    write_tokens_be16(&p, &tokens)?;
599                    p
600                }
601            };
602
603            match run_codec(&pp_path, &output, &cfg, CodecMode::Compress, stop_flag)? {
604                RunOutcome::Completed(summary) => {
605                    println!(
606                        "Compressed {} ({} bytes) -> {} ({} bytes, vocab {} bytes, {:.4} bpb)",
607                        input.display(),
608                        raw_bytes,
609                        output.display(),
610                        summary.actual_compressed_bytes.unwrap_or(0),
611                        vocab_bytes,
612                        summary.actual_bits_per_byte.unwrap_or(0.0),
613                    );
614                    Ok(ExitCode::SUCCESS)
615                }
616                RunOutcome::RequeueRequested {
617                    block_start,
618                    n_symbols,
619                } => {
620                    eprintln!(
621                        "Stopped at block_start={block_start} (symbols={n_symbols}); repeat command to resume."
622                    );
623                    Ok(ExitCode::from(REQUEUE_EXIT_CODE as u8))
624                }
625            }
626        }
627        Command::Decompress {
628            input,
629            output,
630            vocab,
631            keep_tokens,
632            codec,
633        } => {
634            let dict = Dictionary::load(&vocab)?;
635            let cfg = codec.build_config(Some(dict.len()))?;
636
637            let tmp_dir;
638            let pp_path = match keep_tokens {
639                Some(p) => p,
640                None => {
641                    tmp_dir = tempfile::tempdir()?;
642                    tmp_dir.path().join("tokens.pp")
643                }
644            };
645
646            match run_codec(&input, &pp_path, &cfg, CodecMode::Decompress, stop_flag)? {
647                RunOutcome::Completed(_) => {
648                    if let Some(parent) = output.parent() {
649                        if !parent.as_os_str().is_empty() {
650                            fs::create_dir_all(parent)?;
651                        }
652                    }
653                    let token_count = decode_file(&pp_path, &output, &vocab)?;
654                    let out_bytes = fs::metadata(&output)?.len();
655                    println!(
656                        "Decompressed {} ({token_count} tokens) -> {} ({out_bytes} bytes)",
657                        input.display(),
658                        output.display()
659                    );
660                    Ok(ExitCode::SUCCESS)
661                }
662                RunOutcome::RequeueRequested {
663                    block_start,
664                    n_symbols,
665                } => {
666                    eprintln!(
667                        "Stopped at block_start={block_start} (symbols={n_symbols}); repeat command to resume."
668                    );
669                    Ok(ExitCode::from(REQUEUE_EXIT_CODE as u8))
670                }
671            }
672        }
673        Command::Codec {
674            input,
675            output,
676            decompress,
677            no_coding,
678            codec,
679        } => {
680            if no_coding && decompress {
681                bail!("--no-coding cannot be combined with --decompress");
682            }
683            let mode = if decompress {
684                CodecMode::Decompress
685            } else if no_coding {
686                CodecMode::EstimateOnly
687            } else {
688                CodecMode::Compress
689            };
690            let cfg = codec.build_config(None)?;
691            let out_path = output.unwrap_or_else(|| {
692                cfg.run_dir
693                    .clone()
694                    .unwrap_or_else(|| PathBuf::from("."))
695                    .join("out.bin")
696            });
697            match run_codec(&input, &out_path, &cfg, mode, stop_flag)? {
698                RunOutcome::Completed(summary) => {
699                    println!("{}", serde_json::to_string_pretty(&summary)?);
700                    Ok(ExitCode::SUCCESS)
701                }
702                RunOutcome::RequeueRequested {
703                    block_start,
704                    n_symbols,
705                } => {
706                    eprintln!(
707                        "Stopped at block_start={block_start} (symbols={n_symbols}); repeat command to resume."
708                    );
709                    Ok(ExitCode::from(REQUEUE_EXIT_CODE as u8))
710                }
711            }
712        }
713        Command::Preprocess { action } => match action {
714            PreprocessAction::Encode {
715                dict,
716                input,
717                output,
718                n_words,
719                min_freq,
720                sort_by_freq,
721                debug_dict,
722            } => {
723                let order = if sort_by_freq {
724                    SortOrder::ByFrequency
725                } else {
726                    SortOrder::Discovery
727                };
728                let report = encode_file(
729                    &input,
730                    &output,
731                    &dict,
732                    &EncodeConfig {
733                        max_words: n_words,
734                        min_freq,
735                        sort_order: order,
736                        debug_dict_path: debug_dict,
737                    },
738                )?;
739                println!(
740                    "Number of words={} Final length={} (entropy={:.0} bytes)",
741                    report.vocab_size,
742                    report.token_count,
743                    report.entropy_bits / 8.0
744                );
745                Ok(ExitCode::SUCCESS)
746            }
747            PreprocessAction::Decode {
748                dict,
749                input,
750                output,
751            } => {
752                let tokens = decode_file(&input, &output, &dict)?;
753                println!("Decoded {tokens} tokens -> {}", output.display());
754                Ok(ExitCode::SUCCESS)
755            }
756        },
757        Command::V2Decode {
758            payload,
759            output,
760            state,
761            vocab,
762            weights,
763            device,
764            skip_verify,
765        } => {
766            let vocab_path = resolve_default_asset(vocab, "enwik9.voc");
767            let weights_path = resolve_default_asset(weights, "weights.bin");
768            if !skip_verify {
769                check_file(&payload, PAYLOAD_BYTES, PAYLOAD_SHA256)?;
770                check_file(&vocab_path, VOCAB_BYTES, VOCAB_SHA256)?;
771                check_file(&weights_path, WEIGHTS_BYTES, WEIGHTS_SHA256)?;
772            }
773
774            let ckpt_dir = state.join("checkpoint");
775            let run_dir = state.join("run");
776            let pp_path = state.join("enwik9.pp");
777            fs::create_dir_all(&ckpt_dir)?;
778            fs::create_dir_all(&run_dir)?;
779            if let Some(parent) = output.parent() {
780                if !parent.as_os_str().is_empty() {
781                    fs::create_dir_all(parent)?;
782                }
783            }
784            clear_stop(&ckpt_dir)?;
785
786            if !is_done(&ckpt_dir) {
787                let mut cfg = CodecConfig::v2();
788                cfg.device = map_device(device);
789                cfg.vocab_file = Some(vocab_path.clone());
790                cfg.initial_prior = Some(weights_path);
791                cfg.checkpoint_dir = Some(ckpt_dir.clone());
792                cfg.run_dir = Some(run_dir);
793                match run_codec(&payload, &pp_path, &cfg, CodecMode::Decompress, stop_flag)? {
794                    RunOutcome::Completed(_) => {}
795                    RunOutcome::RequeueRequested { .. } => {
796                        eprintln!(
797                            "Stopping at checkpoint; repeat the identical command to resume."
798                        );
799                        return Ok(ExitCode::from(REQUEUE_EXIT_CODE as u8));
800                    }
801                }
802            }
803
804            if !skip_verify {
805                check_file(&pp_path, TOKENS_BYTES, TOKENS_SHA256)?;
806            }
807            decode_file(&pp_path, &output, &vocab_path)?;
808            if !skip_verify {
809                check_file(&output, RAW_BYTES, RAW_SHA256)?;
810            }
811            println!("Decoded and verified: {}", output.display());
812            Ok(ExitCode::SUCCESS)
813        }
814        Command::InspectPrior {
815            archive,
816            vocab,
817            json,
818        } => {
819            let parsed = read_archive(&archive)?;
820            let mut bit_counts: BTreeMap<u8, usize> = BTreeMap::new();
821            for entry in &parsed.metadata.entries {
822                *bit_counts.entry(entry.bits).or_default() += 1;
823            }
824            let vocab_match = match (&parsed.metadata.initialization, &vocab) {
825                (Some(init), Some(voc)) => Some(sha256_file(voc)? == init.vocab_sha256),
826                _ => None,
827            };
828
829            if json {
830                let report = serde_json::json!({
831                    "magic": parsed.magic.as_str(),
832                    "archive_bytes": parsed.archive_bytes,
833                    "archive_sha256": parsed.archive_sha256,
834                    "version": parsed.metadata.version,
835                    "policy": parsed.metadata.policy,
836                    "initialization": parsed.metadata.initialization,
837                    "tensor_count": parsed.tensors.len(),
838                    "bit_width_counts": bit_counts,
839                    "vocab_sha256_matches": vocab_match,
840                });
841                println!("{}", serde_json::to_string_pretty(&report)?);
842            } else {
843                println!("Magic:        {}", parsed.magic.as_str());
844                println!("Size:         {} bytes", parsed.archive_bytes);
845                println!("SHA-256:      {}", parsed.archive_sha256);
846                println!("Policy:       {:?}", parsed.metadata.policy);
847                println!(
848                    "Tensors:      {} (bits: {bit_counts:?})",
849                    parsed.tensors.len()
850                );
851                if let Some(init) = &parsed.metadata.initialization {
852                    let kind = match init.kind {
853                        InitializationKind::DepthGrowth => "depth-growth",
854                        InitializationKind::LoopUntying => "loop-untying",
855                    };
856                    println!("Init kind:    {kind}");
857                    println!("Seed:         {}", init.initialization_seed);
858                    println!(
859                        "Source layers: {} -> Target layers: {} (d_model={}, n_head={}, d_inner={}, ctx_len={}, vocab={})",
860                        init.source_geometry.n_layer,
861                        init.target_geometry.n_layer,
862                        init.target_geometry.d_model,
863                        init.target_geometry.n_head,
864                        init.target_geometry.d_inner,
865                        init.target_geometry.ctx_len,
866                        init.target_geometry.vocab_size
867                    );
868                    println!("Vocab SHA256: {}", init.vocab_sha256);
869                }
870                if let Some(matched) = vocab_match {
871                    println!(
872                        "Vocab check:  {}",
873                        if matched { "MATCH" } else { "MISMATCH" }
874                    );
875                }
876            }
877            Ok(ExitCode::SUCCESS)
878        }
879    }
880}
881
882fn main() -> ExitCode {
883    let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
884    tracing_subscriber::fmt()
885        .with_env_filter(filter)
886        .with_target(false)
887        .without_time()
888        .with_writer(std::io::stderr)
889        .init();
890
891    let cli = Cli::parse();
892    let stop_flag = Arc::new(AtomicBool::new(false));
893    install_stop_handlers(&stop_flag);
894
895    match run(cli, stop_flag) {
896        Ok(code) => code,
897        Err(err) => {
898            eprintln!("tufazip: {err:#}");
899            ExitCode::FAILURE
900        }
901    }
902}