1use std::collections::BTreeMap;
4use std::fs;
5use std::path::{Path, PathBuf};
6use std::process::ExitCode;
7use std::sync::atomic::AtomicBool;
8use std::sync::Arc;
9
10use anyhow::{bail, Context, Result};
11use clap::{Args, Parser, Subcommand, ValueEnum};
12use tracing_subscriber::EnvFilter;
13use tufazip_codec::{
14 clear_stop, is_done, run_codec, AutocastPolicy, CodecConfig, CodecMode, ComputeDType,
15 DeviceChoice, NormStyle, OnlineLoss, PiecewiseLrSchedule, PositionType, RunOutcome,
16 REQUEUE_EXIT_CODE,
17};
18use tufazip_preprocess::{
19 decode_file, encode_bytes, encode_file, encode_with_dictionary, write_tokens_be16, Dictionary,
20 EncodeConfig, SortOrder,
21};
22use tufazip_prior::{read_archive, sha256_file, InitializationKind};
23
24const PAYLOAD_SHA256: &str = "198451a21f64386b487b8ea97e141562db355eec9cc9af34e7bbb6a022bcce77";
25const PAYLOAD_BYTES: u64 = 100_958_324;
26const VOCAB_SHA256: &str = "a63e517783e84a778f3f08c1f2321f3e2035e64d34c8e9206f03591460e44629";
27const VOCAB_BYTES: u64 = 34_421;
28const WEIGHTS_SHA256: &str = "e94c089f8eb985a87307dca3a7702385fcaccd14d95743811f8788f7da24448d";
29const WEIGHTS_BYTES: u64 = 4_334_741;
30const TOKENS_SHA256: &str = "102fa82b3f8932b37a1ec0d9262e555d23d17852685fe5fb33680dbee19dd694";
31const TOKENS_BYTES: u64 = 570_405_660;
32const RAW_SHA256: &str = "159b85351e5f76e60cbe32e04c677847a9ecba3adc79addab6f4c6c7aa3744bc";
33const RAW_BYTES: u64 = 1_000_000_000;
34
35#[derive(Debug, Parser)]
36#[command(
37 name = "tufazip",
38 version = "0.2.0",
39 about = "Pure-Rust port of tufazip v2 neural lossless compressor and decompressor"
40)]
41struct Cli {
42 #[command(subcommand)]
43 command: Command,
44}
45
46#[derive(Debug, Subcommand)]
47enum Command {
48 Compress {
50 input: PathBuf,
52 output: PathBuf,
54 #[arg(long)]
56 vocab: PathBuf,
57 #[arg(long)]
59 use_existing_vocab: bool,
60 #[arg(long, default_value_t = 4097)]
62 max_words: usize,
63 #[arg(long, default_value_t = 5)]
65 min_freq: u32,
66 #[arg(short = 's', long)]
68 sort_by_freq: bool,
69 #[arg(long)]
71 keep_tokens: Option<PathBuf>,
72 #[command(flatten)]
74 codec: CodecArgs,
75 },
76 Decompress {
78 input: PathBuf,
80 output: PathBuf,
82 #[arg(long)]
84 vocab: PathBuf,
85 #[arg(long)]
87 keep_tokens: Option<PathBuf>,
88 #[command(flatten)]
90 codec: CodecArgs,
91 },
92 Codec {
94 #[arg(long)]
96 input: PathBuf,
97 #[arg(long)]
99 output: Option<PathBuf>,
100 #[arg(long)]
102 decompress: bool,
103 #[arg(long)]
105 no_coding: bool,
106 #[command(flatten)]
108 codec: CodecArgs,
109 },
110 Preprocess {
112 #[command(subcommand)]
113 action: PreprocessAction,
114 },
115 V2Decode {
117 payload: PathBuf,
119 output: PathBuf,
121 #[arg(default_value = "tufazip-v2-state")]
123 state: PathBuf,
124 #[arg(long)]
126 vocab: Option<PathBuf>,
127 #[arg(long)]
129 weights: Option<PathBuf>,
130 #[arg(long, value_enum, default_value_t = CliDevice::Auto)]
132 device: CliDevice,
133 #[arg(long)]
135 skip_verify: bool,
136 },
137 InspectPrior {
139 archive: PathBuf,
141 #[arg(long)]
143 vocab: Option<PathBuf>,
144 #[arg(long)]
146 json: bool,
147 },
148}
149
150#[derive(Debug, Subcommand)]
151enum PreprocessAction {
152 Encode {
154 dict: PathBuf,
156 input: PathBuf,
158 output: PathBuf,
160 n_words: usize,
162 min_freq: u32,
164 #[arg(short = 's', long)]
166 sort_by_freq: bool,
167 #[arg(short = 'D', long)]
169 debug_dict: Option<PathBuf>,
170 },
171 Decode {
173 dict: PathBuf,
175 input: PathBuf,
177 output: PathBuf,
179 },
180}
181
182#[derive(Debug, Clone, Copy, ValueEnum)]
183enum Preset {
184 Tiny,
185 V2,
186}
187
188#[derive(Debug, Clone, Copy, ValueEnum)]
189enum CliPositionType {
190 Rope,
191 LearnedRelative,
192}
193
194#[derive(Debug, Clone, Copy, ValueEnum)]
195enum CliNormStyle {
196 Peri,
197 Post,
198}
199
200#[derive(Debug, Clone, Copy, ValueEnum)]
201enum CliOnlineLoss {
202 Code,
203 Window,
204}
205
206#[derive(Debug, Clone, Copy, ValueEnum)]
207enum CliDType {
208 Fp32,
209 Bf16,
210}
211
212#[derive(Debug, Clone, Copy, ValueEnum)]
213enum CliDevice {
214 Auto,
215 Cpu,
216 Metal,
217 Cuda,
218}
219
220#[derive(Debug, Args)]
221struct CodecArgs {
222 #[arg(long, value_enum, default_value_t = Preset::Tiny)]
224 preset: Preset,
225 #[arg(long)]
226 vocab_size: Option<usize>,
227 #[arg(long)]
228 n_layer: Option<usize>,
229 #[arg(long)]
230 d_model: Option<usize>,
231 #[arg(long)]
232 n_head: Option<usize>,
233 #[arg(long)]
234 d_inner: Option<usize>,
235 #[arg(long)]
236 ctx_len: Option<usize>,
237 #[arg(long)]
238 dropout: Option<f32>,
239 #[arg(long)]
240 tie_embeddings: Option<bool>,
241 #[arg(long, value_enum)]
242 position_type: Option<CliPositionType>,
243 #[arg(long)]
244 no_qk_norm: bool,
245 #[arg(long, value_enum)]
246 norm_style: Option<CliNormStyle>,
247 #[arg(long)]
248 batch_size: Option<usize>,
249 #[arg(long)]
250 block_len: Option<usize>,
251 #[arg(long, value_enum)]
252 online_loss: Option<CliOnlineLoss>,
253 #[arg(long)]
254 project_only_targets: Option<bool>,
255 #[arg(long)]
256 lr: Option<f64>,
257 #[arg(long)]
258 min_lr_ratio: Option<f64>,
259 #[arg(long)]
260 warmup: Option<usize>,
261 #[arg(long)]
262 clip: Option<f64>,
263 #[arg(long)]
264 adam_eps: Option<f64>,
265 #[arg(long)]
266 beta1: Option<f64>,
267 #[arg(long)]
268 beta2: Option<f64>,
269 #[arg(long)]
270 retrain_period: Option<usize>,
271 #[arg(long)]
272 retrain_len: Option<usize>,
273 #[arg(long)]
274 retrain_epochs: Option<usize>,
275 #[arg(long)]
276 retrain_early_epochs: Option<usize>,
277 #[arg(long)]
278 retrain_early_until: Option<usize>,
279 #[arg(long)]
280 retrain_lr: Option<String>,
281 #[arg(long)]
282 retrain_update_len: Option<usize>,
283 #[arg(long)]
284 retrain_batch_size: Option<usize>,
285 #[arg(long)]
286 seed: Option<u64>,
287 #[arg(long, value_enum)]
288 dtype: Option<CliDType>,
289 #[arg(long)]
290 force_bf16: bool,
291 #[arg(long, value_enum)]
292 device: Option<CliDevice>,
293 #[arg(long)]
294 max_tokens: Option<usize>,
295 #[arg(long)]
296 raw_bytes: Option<u64>,
297 #[arg(long)]
298 vocab_bytes: Option<u64>,
299 #[arg(long)]
300 vocab_file: Option<PathBuf>,
301 #[arg(long)]
302 initial_prior: Option<PathBuf>,
303 #[arg(long)]
304 log_interval: Option<usize>,
305 #[arg(long)]
306 checkpoint_dir: Option<PathBuf>,
307 #[arg(long)]
308 checkpoint_every_blocks: Option<usize>,
309 #[arg(long)]
310 stop_after_blocks: Option<usize>,
311 #[arg(long)]
312 run_dir: Option<PathBuf>,
313}
314
315impl CodecArgs {
316 fn build_config(&self, inferred_vocab_size: Option<usize>) -> Result<CodecConfig> {
317 let mut cfg = match self.preset {
318 Preset::V2 => CodecConfig::v2(),
319 Preset::Tiny => CodecConfig::tiny(inferred_vocab_size.unwrap_or(4097)),
320 };
321 if let Some(v) = self.vocab_size {
322 cfg.model.vocab_size = v;
323 } else if let Some(v) = inferred_vocab_size {
324 if matches!(self.preset, Preset::Tiny) {
325 cfg.model.vocab_size = v;
326 }
327 }
328 if let Some(v) = self.n_layer {
329 cfg.model.n_layer = v;
330 }
331 if let Some(v) = self.d_model {
332 cfg.model.d_model = v;
333 }
334 if let Some(v) = self.n_head {
335 cfg.model.n_head = v;
336 }
337 if let Some(v) = self.d_inner {
338 cfg.model.d_inner = v;
339 }
340 if let Some(v) = self.ctx_len {
341 cfg.model.ctx_len = v;
342 }
343 if let Some(v) = self.dropout {
344 cfg.model.dropout = v;
345 }
346 if let Some(v) = self.tie_embeddings {
347 cfg.model.tie_embeddings = v;
348 }
349 if let Some(v) = self.position_type {
350 cfg.model.position_type = match v {
351 CliPositionType::Rope => PositionType::Rope,
352 CliPositionType::LearnedRelative => PositionType::LearnedRelative,
353 };
354 }
355 if self.no_qk_norm {
356 cfg.model.qk_norm = false;
357 }
358 if let Some(v) = self.norm_style {
359 cfg.model.norm_style = match v {
360 CliNormStyle::Peri => NormStyle::Peri,
361 CliNormStyle::Post => NormStyle::Post,
362 };
363 }
364 if let Some(v) = self.batch_size {
365 cfg.batch_size = v;
366 }
367 if let Some(v) = self.block_len {
368 cfg.block_len = v;
369 }
370 if let Some(v) = self.online_loss {
371 cfg.online_loss = match v {
372 CliOnlineLoss::Code => OnlineLoss::Code,
373 CliOnlineLoss::Window => OnlineLoss::Window,
374 };
375 }
376 if let Some(v) = self.project_only_targets {
377 cfg.project_only_targets = v;
378 }
379 if let Some(v) = self.lr {
380 cfg.optim.lr = v;
381 }
382 if let Some(v) = self.min_lr_ratio {
383 cfg.optim.min_lr_ratio = v;
384 }
385 if let Some(v) = self.warmup {
386 cfg.optim.warmup = v;
387 }
388 if let Some(v) = self.clip {
389 cfg.optim.clip = v;
390 }
391 if let Some(v) = self.adam_eps {
392 cfg.optim.adam_eps = v;
393 }
394 if let Some(v) = self.beta1 {
395 cfg.optim.beta1 = v;
396 }
397 if let Some(v) = self.beta2 {
398 cfg.optim.beta2 = v;
399 }
400 if let Some(v) = self.retrain_period {
401 cfg.retrain.period = v;
402 }
403 if let Some(v) = self.retrain_len {
404 cfg.retrain.len = v;
405 }
406 if let Some(v) = self.retrain_epochs {
407 cfg.retrain.epochs = v;
408 }
409 if let Some(v) = self.retrain_early_epochs {
410 cfg.retrain.early_epochs = v;
411 }
412 if let Some(v) = self.retrain_early_until {
413 cfg.retrain.early_until = v;
414 }
415 if let Some(ref spec) = self.retrain_lr {
416 cfg.retrain.lr_schedule = PiecewiseLrSchedule::parse(spec)?;
417 }
418 if let Some(v) = self.retrain_update_len {
419 cfg.retrain.update_len = v;
420 }
421 if let Some(v) = self.retrain_batch_size {
422 cfg.retrain.batch_size = v;
423 }
424 if let Some(v) = self.seed {
425 cfg.seed = v;
426 }
427 if let Some(v) = self.dtype {
428 cfg.dtype = match v {
429 CliDType::Fp32 => ComputeDType::Fp32,
430 CliDType::Bf16 => ComputeDType::Bf16,
431 };
432 }
433 if self.force_bf16 {
434 cfg.autocast_policy = AutocastPolicy::Force;
435 }
436 if let Some(v) = self.device {
437 cfg.device = map_device(v);
438 }
439 if let Some(v) = self.max_tokens {
440 cfg.max_tokens = Some(v);
441 }
442 if let Some(v) = self.raw_bytes {
443 cfg.raw_bytes = Some(v);
444 }
445 if let Some(v) = self.vocab_bytes {
446 cfg.vocab_bytes = v;
447 }
448 if let Some(ref p) = self.vocab_file {
449 cfg.vocab_file = Some(p.clone());
450 }
451 if let Some(ref p) = self.initial_prior {
452 cfg.initial_prior = Some(p.clone());
453 }
454 if let Some(v) = self.log_interval {
455 cfg.log_interval = v;
456 }
457 if let Some(ref p) = self.checkpoint_dir {
458 cfg.checkpoint_dir = Some(p.clone());
459 }
460 if let Some(v) = self.checkpoint_every_blocks {
461 cfg.checkpoint_every_blocks = v;
462 }
463 if let Some(v) = self.stop_after_blocks {
464 cfg.stop_after_blocks = Some(v);
465 }
466 if let Some(ref p) = self.run_dir {
467 cfg.run_dir = Some(p.clone());
468 }
469 Ok(cfg)
470 }
471}
472
473fn map_device(d: CliDevice) -> DeviceChoice {
474 match d {
475 CliDevice::Auto => DeviceChoice::Auto,
476 CliDevice::Cpu => DeviceChoice::Cpu,
477 CliDevice::Metal => DeviceChoice::Metal,
478 CliDevice::Cuda => DeviceChoice::Cuda,
479 }
480}
481
482fn check_file(path: &Path, expected_size: u64, expected_sha256: &str) -> Result<()> {
483 if !path.is_file() {
484 bail!("missing file: {}", path.display());
485 }
486 let size = fs::metadata(path)?.len();
487 if size != expected_size {
488 bail!(
489 "wrong size for {}: expected {expected_size} bytes, found {size}",
490 path.display()
491 );
492 }
493 let digest = sha256_file(path)?;
494 if digest != expected_sha256 {
495 bail!(
496 "wrong SHA-256 for {}: expected {expected_sha256}, found {digest}",
497 path.display()
498 );
499 }
500 Ok(())
501}
502
503fn resolve_default_asset(explicit: Option<PathBuf>, filename: &str) -> PathBuf {
504 if let Some(p) = explicit {
505 return p;
506 }
507 let cwd_asset = PathBuf::from("assets").join(filename);
508 if cwd_asset.is_file() {
509 return cwd_asset;
510 }
511 if let Ok(exe) = std::env::current_exe() {
512 if let Some(dir) = exe.parent() {
513 let sibling = dir.join(filename);
514 if sibling.is_file() {
515 return sibling;
516 }
517 let sibling_assets = dir.join("assets").join(filename);
518 if sibling_assets.is_file() {
519 return sibling_assets;
520 }
521 }
522 }
523 PathBuf::from(filename)
524}
525
526fn install_stop_handlers(stop_flag: &Arc<AtomicBool>) {
527 let _ = signal_hook::flag::register(signal_hook::consts::SIGINT, Arc::clone(stop_flag));
528 #[cfg(unix)]
529 {
530 let _ = signal_hook::flag::register(signal_hook::consts::SIGTERM, Arc::clone(stop_flag));
531 let _ = signal_hook::flag::register(signal_hook::consts::SIGUSR1, Arc::clone(stop_flag));
532 }
533}
534
535fn run(cli: Cli, stop_flag: Arc<AtomicBool>) -> Result<ExitCode> {
536 match cli.command {
537 Command::Compress {
538 input,
539 output,
540 vocab,
541 use_existing_vocab,
542 max_words,
543 min_freq,
544 sort_by_freq,
545 keep_tokens,
546 codec,
547 } => {
548 let raw_bytes = fs::metadata(&input)
549 .with_context(|| format!("reading {}", input.display()))?
550 .len();
551 let raw = fs::read(&input)?;
552 let tokens = if use_existing_vocab {
553 let dict = Dictionary::load(&vocab)?;
554 encode_with_dictionary(&raw, &dict)?
555 } else {
556 let order = if sort_by_freq {
557 SortOrder::ByFrequency
558 } else {
559 SortOrder::Discovery
560 };
561 let (dict, tokens, _report) = encode_bytes(
562 &raw,
563 &EncodeConfig {
564 max_words,
565 min_freq,
566 sort_order: order,
567 debug_dict_path: None,
568 },
569 )?;
570 if let Some(parent) = vocab.parent() {
571 if !parent.as_os_str().is_empty() {
572 fs::create_dir_all(parent)?;
573 }
574 }
575 dict.save(&vocab)?;
576 tokens
577 };
578
579 let dict = Dictionary::load(&vocab)?;
580 let vocab_bytes = fs::metadata(&vocab)?.len();
581 let mut cfg = codec.build_config(Some(dict.len()))?;
582 if cfg.raw_bytes.is_none() {
583 cfg.raw_bytes = Some(raw_bytes);
584 }
585 if cfg.vocab_bytes == 0 {
586 cfg.vocab_bytes = vocab_bytes;
587 }
588
589 let tmp_dir;
590 let pp_path = match keep_tokens {
591 Some(p) => {
592 write_tokens_be16(&p, &tokens)?;
593 p
594 }
595 None => {
596 tmp_dir = tempfile::tempdir()?;
597 let p = tmp_dir.path().join("tokens.pp");
598 write_tokens_be16(&p, &tokens)?;
599 p
600 }
601 };
602
603 match run_codec(&pp_path, &output, &cfg, CodecMode::Compress, stop_flag)? {
604 RunOutcome::Completed(summary) => {
605 println!(
606 "Compressed {} ({} bytes) -> {} ({} bytes, vocab {} bytes, {:.4} bpb)",
607 input.display(),
608 raw_bytes,
609 output.display(),
610 summary.actual_compressed_bytes.unwrap_or(0),
611 vocab_bytes,
612 summary.actual_bits_per_byte.unwrap_or(0.0),
613 );
614 Ok(ExitCode::SUCCESS)
615 }
616 RunOutcome::RequeueRequested {
617 block_start,
618 n_symbols,
619 } => {
620 eprintln!(
621 "Stopped at block_start={block_start} (symbols={n_symbols}); repeat command to resume."
622 );
623 Ok(ExitCode::from(REQUEUE_EXIT_CODE as u8))
624 }
625 }
626 }
627 Command::Decompress {
628 input,
629 output,
630 vocab,
631 keep_tokens,
632 codec,
633 } => {
634 let dict = Dictionary::load(&vocab)?;
635 let cfg = codec.build_config(Some(dict.len()))?;
636
637 let tmp_dir;
638 let pp_path = match keep_tokens {
639 Some(p) => p,
640 None => {
641 tmp_dir = tempfile::tempdir()?;
642 tmp_dir.path().join("tokens.pp")
643 }
644 };
645
646 match run_codec(&input, &pp_path, &cfg, CodecMode::Decompress, stop_flag)? {
647 RunOutcome::Completed(_) => {
648 if let Some(parent) = output.parent() {
649 if !parent.as_os_str().is_empty() {
650 fs::create_dir_all(parent)?;
651 }
652 }
653 let token_count = decode_file(&pp_path, &output, &vocab)?;
654 let out_bytes = fs::metadata(&output)?.len();
655 println!(
656 "Decompressed {} ({token_count} tokens) -> {} ({out_bytes} bytes)",
657 input.display(),
658 output.display()
659 );
660 Ok(ExitCode::SUCCESS)
661 }
662 RunOutcome::RequeueRequested {
663 block_start,
664 n_symbols,
665 } => {
666 eprintln!(
667 "Stopped at block_start={block_start} (symbols={n_symbols}); repeat command to resume."
668 );
669 Ok(ExitCode::from(REQUEUE_EXIT_CODE as u8))
670 }
671 }
672 }
673 Command::Codec {
674 input,
675 output,
676 decompress,
677 no_coding,
678 codec,
679 } => {
680 if no_coding && decompress {
681 bail!("--no-coding cannot be combined with --decompress");
682 }
683 let mode = if decompress {
684 CodecMode::Decompress
685 } else if no_coding {
686 CodecMode::EstimateOnly
687 } else {
688 CodecMode::Compress
689 };
690 let cfg = codec.build_config(None)?;
691 let out_path = output.unwrap_or_else(|| {
692 cfg.run_dir
693 .clone()
694 .unwrap_or_else(|| PathBuf::from("."))
695 .join("out.bin")
696 });
697 match run_codec(&input, &out_path, &cfg, mode, stop_flag)? {
698 RunOutcome::Completed(summary) => {
699 println!("{}", serde_json::to_string_pretty(&summary)?);
700 Ok(ExitCode::SUCCESS)
701 }
702 RunOutcome::RequeueRequested {
703 block_start,
704 n_symbols,
705 } => {
706 eprintln!(
707 "Stopped at block_start={block_start} (symbols={n_symbols}); repeat command to resume."
708 );
709 Ok(ExitCode::from(REQUEUE_EXIT_CODE as u8))
710 }
711 }
712 }
713 Command::Preprocess { action } => match action {
714 PreprocessAction::Encode {
715 dict,
716 input,
717 output,
718 n_words,
719 min_freq,
720 sort_by_freq,
721 debug_dict,
722 } => {
723 let order = if sort_by_freq {
724 SortOrder::ByFrequency
725 } else {
726 SortOrder::Discovery
727 };
728 let report = encode_file(
729 &input,
730 &output,
731 &dict,
732 &EncodeConfig {
733 max_words: n_words,
734 min_freq,
735 sort_order: order,
736 debug_dict_path: debug_dict,
737 },
738 )?;
739 println!(
740 "Number of words={} Final length={} (entropy={:.0} bytes)",
741 report.vocab_size,
742 report.token_count,
743 report.entropy_bits / 8.0
744 );
745 Ok(ExitCode::SUCCESS)
746 }
747 PreprocessAction::Decode {
748 dict,
749 input,
750 output,
751 } => {
752 let tokens = decode_file(&input, &output, &dict)?;
753 println!("Decoded {tokens} tokens -> {}", output.display());
754 Ok(ExitCode::SUCCESS)
755 }
756 },
757 Command::V2Decode {
758 payload,
759 output,
760 state,
761 vocab,
762 weights,
763 device,
764 skip_verify,
765 } => {
766 let vocab_path = resolve_default_asset(vocab, "enwik9.voc");
767 let weights_path = resolve_default_asset(weights, "weights.bin");
768 if !skip_verify {
769 check_file(&payload, PAYLOAD_BYTES, PAYLOAD_SHA256)?;
770 check_file(&vocab_path, VOCAB_BYTES, VOCAB_SHA256)?;
771 check_file(&weights_path, WEIGHTS_BYTES, WEIGHTS_SHA256)?;
772 }
773
774 let ckpt_dir = state.join("checkpoint");
775 let run_dir = state.join("run");
776 let pp_path = state.join("enwik9.pp");
777 fs::create_dir_all(&ckpt_dir)?;
778 fs::create_dir_all(&run_dir)?;
779 if let Some(parent) = output.parent() {
780 if !parent.as_os_str().is_empty() {
781 fs::create_dir_all(parent)?;
782 }
783 }
784 clear_stop(&ckpt_dir)?;
785
786 if !is_done(&ckpt_dir) {
787 let mut cfg = CodecConfig::v2();
788 cfg.device = map_device(device);
789 cfg.vocab_file = Some(vocab_path.clone());
790 cfg.initial_prior = Some(weights_path);
791 cfg.checkpoint_dir = Some(ckpt_dir.clone());
792 cfg.run_dir = Some(run_dir);
793 match run_codec(&payload, &pp_path, &cfg, CodecMode::Decompress, stop_flag)? {
794 RunOutcome::Completed(_) => {}
795 RunOutcome::RequeueRequested { .. } => {
796 eprintln!(
797 "Stopping at checkpoint; repeat the identical command to resume."
798 );
799 return Ok(ExitCode::from(REQUEUE_EXIT_CODE as u8));
800 }
801 }
802 }
803
804 if !skip_verify {
805 check_file(&pp_path, TOKENS_BYTES, TOKENS_SHA256)?;
806 }
807 decode_file(&pp_path, &output, &vocab_path)?;
808 if !skip_verify {
809 check_file(&output, RAW_BYTES, RAW_SHA256)?;
810 }
811 println!("Decoded and verified: {}", output.display());
812 Ok(ExitCode::SUCCESS)
813 }
814 Command::InspectPrior {
815 archive,
816 vocab,
817 json,
818 } => {
819 let parsed = read_archive(&archive)?;
820 let mut bit_counts: BTreeMap<u8, usize> = BTreeMap::new();
821 for entry in &parsed.metadata.entries {
822 *bit_counts.entry(entry.bits).or_default() += 1;
823 }
824 let vocab_match = match (&parsed.metadata.initialization, &vocab) {
825 (Some(init), Some(voc)) => Some(sha256_file(voc)? == init.vocab_sha256),
826 _ => None,
827 };
828
829 if json {
830 let report = serde_json::json!({
831 "magic": parsed.magic.as_str(),
832 "archive_bytes": parsed.archive_bytes,
833 "archive_sha256": parsed.archive_sha256,
834 "version": parsed.metadata.version,
835 "policy": parsed.metadata.policy,
836 "initialization": parsed.metadata.initialization,
837 "tensor_count": parsed.tensors.len(),
838 "bit_width_counts": bit_counts,
839 "vocab_sha256_matches": vocab_match,
840 });
841 println!("{}", serde_json::to_string_pretty(&report)?);
842 } else {
843 println!("Magic: {}", parsed.magic.as_str());
844 println!("Size: {} bytes", parsed.archive_bytes);
845 println!("SHA-256: {}", parsed.archive_sha256);
846 println!("Policy: {:?}", parsed.metadata.policy);
847 println!(
848 "Tensors: {} (bits: {bit_counts:?})",
849 parsed.tensors.len()
850 );
851 if let Some(init) = &parsed.metadata.initialization {
852 let kind = match init.kind {
853 InitializationKind::DepthGrowth => "depth-growth",
854 InitializationKind::LoopUntying => "loop-untying",
855 };
856 println!("Init kind: {kind}");
857 println!("Seed: {}", init.initialization_seed);
858 println!(
859 "Source layers: {} -> Target layers: {} (d_model={}, n_head={}, d_inner={}, ctx_len={}, vocab={})",
860 init.source_geometry.n_layer,
861 init.target_geometry.n_layer,
862 init.target_geometry.d_model,
863 init.target_geometry.n_head,
864 init.target_geometry.d_inner,
865 init.target_geometry.ctx_len,
866 init.target_geometry.vocab_size
867 );
868 println!("Vocab SHA256: {}", init.vocab_sha256);
869 }
870 if let Some(matched) = vocab_match {
871 println!(
872 "Vocab check: {}",
873 if matched { "MATCH" } else { "MISMATCH" }
874 );
875 }
876 }
877 Ok(ExitCode::SUCCESS)
878 }
879 }
880}
881
882fn main() -> ExitCode {
883 let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
884 tracing_subscriber::fmt()
885 .with_env_filter(filter)
886 .with_target(false)
887 .without_time()
888 .with_writer(std::io::stderr)
889 .init();
890
891 let cli = Cli::parse();
892 let stop_flag = Arc::new(AtomicBool::new(false));
893 install_stop_handlers(&stop_flag);
894
895 match run(cli, stop_flag) {
896 Ok(code) => code,
897 Err(err) => {
898 eprintln!("tufazip: {err:#}");
899 ExitCode::FAILURE
900 }
901 }
902}