1use std::path::PathBuf;
2
3use candle_core::{DType, Device};
4use serde::{Deserialize, Serialize};
5use tufazip_prior::{validate_block_schedule, TransformerGeometry};
6
7use crate::error::CodecError;
8
9#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
11#[serde(rename_all = "snake_case")]
12pub enum PositionType {
13 Rope,
15 LearnedRelative,
17}
18
19impl PositionType {
20 #[must_use]
22 pub const fn as_str(self) -> &'static str {
23 match self {
24 Self::Rope => "rope",
25 Self::LearnedRelative => "learned_relative",
26 }
27 }
28}
29
30#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
32#[serde(rename_all = "snake_case")]
33pub enum NormStyle {
34 Peri,
36 Post,
38}
39
40impl NormStyle {
41 #[must_use]
43 pub const fn as_str(self) -> &'static str {
44 match self {
45 Self::Peri => "peri",
46 Self::Post => "post",
47 }
48 }
49}
50
51#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
53#[serde(rename_all = "snake_case")]
54pub enum OnlineLoss {
55 Code,
57 Window,
59}
60
61#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
63#[serde(rename_all = "snake_case")]
64pub enum ComputeDType {
65 Fp32,
67 Bf16,
69}
70
71#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
73#[serde(rename_all = "snake_case")]
74pub enum AutocastPolicy {
75 #[default]
77 Auto,
78 Force,
80}
81
82#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
84#[serde(rename_all = "snake_case")]
85pub enum DeviceChoice {
86 #[default]
88 Auto,
89 Cpu,
91 Metal,
93 Cuda,
95}
96
97#[derive(Debug, Clone, Copy, PartialEq, Eq)]
99pub enum CodecMode {
100 Compress,
102 Decompress,
104 EstimateOnly,
106}
107
108#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
110pub struct PiecewiseLrSchedule {
111 points: Vec<(usize, f64)>,
112}
113
114impl PiecewiseLrSchedule {
115 #[must_use]
117 pub fn constant(lr: f64) -> Self {
118 Self {
119 points: vec![(0, lr)],
120 }
121 }
122
123 pub fn parse(spec: &str) -> Result<Self, CodecError> {
129 let parts: Vec<&str> = spec.split(',').map(str::trim).collect();
130 if parts.is_empty() || parts.len() % 2 == 0 {
131 return Err(CodecError::InvalidConfig(format!(
132 "LR schedule must have syntax lr0[,step1,lr1,...], got {spec:?}"
133 )));
134 }
135 let lr0: f64 = parts[0].parse().map_err(|_| {
136 CodecError::InvalidConfig(format!("invalid initial LR {:?} in {spec:?}", parts[0]))
137 })?;
138 let mut points = vec![(0usize, lr0)];
139 let mut prev_step = 0usize;
140 for pair in parts[1..].chunks_exact(2) {
141 let step: usize = pair[0].parse().map_err(|_| {
142 CodecError::InvalidConfig(format!("invalid step {:?} in {spec:?}", pair[0]))
143 })?;
144 let lr: f64 = pair[1].parse().map_err(|_| {
145 CodecError::InvalidConfig(format!("invalid LR {:?} in {spec:?}", pair[1]))
146 })?;
147 if step <= prev_step {
148 return Err(CodecError::InvalidConfig(
149 "LR schedule steps must be strictly increasing".to_owned(),
150 ));
151 }
152 points.push((step, lr));
153 prev_step = step;
154 }
155 Ok(Self { points })
156 }
157
158 #[must_use]
160 pub fn lr_at(&self, step: usize) -> f64 {
161 if step <= self.points[0].0 {
162 return self.points[0].1;
163 }
164 for window in self.points.windows(2) {
165 let (start_step, start_lr) = window[0];
166 let (end_step, end_lr) = window[1];
167 if step <= end_step {
168 let fraction = (step - start_step) as f64 / (end_step - start_step) as f64;
169 return start_lr + fraction * (end_lr - start_lr);
170 }
171 }
172 self.points.last().map_or(0.0, |p| p.1)
173 }
174
175 #[must_use]
177 pub fn initial_lr(&self) -> f64 {
178 self.points[0].1
179 }
180
181 #[must_use]
183 pub fn to_spec(&self) -> String {
184 let mut out = format!("{}", self.points[0].1);
185 for &(step, lr) in &self.points[1..] {
186 out.push_str(&format!(",{step},{lr}"));
187 }
188 out
189 }
190}
191
192#[must_use]
194pub fn lr_at(step: usize, peak_lr: f64, min_lr: f64, warmup: usize, total_steps: usize) -> f64 {
195 if warmup > 0 && step < warmup {
196 return peak_lr * ((step + 1) as f64) / (warmup as f64);
197 }
198 if step >= total_steps {
199 return min_lr;
200 }
201 let denom = total_steps.saturating_sub(warmup).max(1) as f64;
202 let progress = (step.saturating_sub(warmup) as f64) / denom;
203 min_lr + 0.5 * (peak_lr - min_lr) * (1.0 + (std::f64::consts::PI * progress).cos())
204}
205
206#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
208pub struct ModelConfig {
209 pub vocab_size: usize,
211 pub n_layer: usize,
213 pub d_model: usize,
215 pub n_head: usize,
217 pub d_inner: usize,
219 pub ctx_len: usize,
221 pub dropout: f32,
223 pub rope_base: f64,
225 pub tie_embeddings: bool,
227 pub position_type: PositionType,
229 pub qk_norm: bool,
231 pub norm_style: NormStyle,
233 pub block_schedule: Option<Vec<usize>>,
235}
236
237impl ModelConfig {
238 #[must_use]
240 pub fn v2() -> Self {
241 Self {
242 vocab_size: 4097,
243 n_layer: 12,
244 d_model: 1024,
245 n_head: 8,
246 d_inner: 2816,
247 ctx_len: 192,
248 dropout: 0.25,
249 rope_base: 10_000.0,
250 tie_embeddings: false,
251 position_type: PositionType::Rope,
252 qk_norm: true,
253 norm_style: NormStyle::Peri,
254 block_schedule: None,
255 }
256 }
257
258 #[must_use]
260 pub fn tiny(vocab_size: usize) -> Self {
261 Self {
262 vocab_size,
263 n_layer: 2,
264 d_model: 64,
265 n_head: 2,
266 d_inner: 128,
267 ctx_len: 32,
268 dropout: 0.1,
269 rope_base: 10_000.0,
270 tie_embeddings: false,
271 position_type: PositionType::Rope,
272 qk_norm: true,
273 norm_style: NormStyle::Peri,
274 block_schedule: None,
275 }
276 }
277
278 pub fn validate(&self) -> Result<(), CodecError> {
284 if self.vocab_size == 0 || self.n_layer == 0 || self.d_model == 0 || self.n_head == 0 {
285 return Err(CodecError::InvalidConfig(
286 "vocab_size, n_layer, d_model, and n_head must be positive".to_owned(),
287 ));
288 }
289 if self.d_model % self.n_head != 0 {
290 return Err(CodecError::InvalidConfig(format!(
291 "d_model ({}) must be divisible by n_head ({})",
292 self.d_model, self.n_head
293 )));
294 }
295 if self.ctx_len == 0 || self.d_inner == 0 {
296 return Err(CodecError::InvalidConfig(
297 "ctx_len and d_inner must be positive".to_owned(),
298 ));
299 }
300 if !(0.0..1.0).contains(&self.dropout) {
301 return Err(CodecError::InvalidConfig(format!(
302 "dropout ({}) must be in [0.0, 1.0)",
303 self.dropout
304 )));
305 }
306 if let Some(schedule) = &self.block_schedule {
307 validate_block_schedule(schedule, self.n_layer)?;
308 }
309 Ok(())
310 }
311
312 #[must_use]
314 pub fn geometry(&self) -> TransformerGeometry {
315 TransformerGeometry {
316 vocab_size: self.vocab_size,
317 n_layer: self.n_layer,
318 d_model: self.d_model,
319 n_head: self.n_head,
320 d_inner: self.d_inner,
321 ctx_len: self.ctx_len,
322 tie_embeddings: self.tie_embeddings,
323 position_type: self.position_type.as_str().to_owned(),
324 qk_norm: self.qk_norm,
325 norm_style: self.norm_style.as_str().to_owned(),
326 }
327 }
328}
329
330#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
332pub struct OptimConfig {
333 pub lr: f64,
335 pub min_lr_ratio: f64,
337 pub warmup: usize,
339 pub clip: f64,
341 pub beta1: f64,
343 pub beta2: f64,
345 pub adam_eps: f64,
347}
348
349impl OptimConfig {
350 #[must_use]
352 pub fn v2() -> Self {
353 Self {
354 lr: 4e-5,
355 min_lr_ratio: 0.25,
356 warmup: 200,
357 clip: 0.25,
358 beta1: 0.0,
359 beta2: 0.9999,
360 adam_eps: 1e-9,
361 }
362 }
363
364 #[must_use]
366 pub fn tiny() -> Self {
367 Self {
368 lr: 5e-4,
369 min_lr_ratio: 0.1,
370 warmup: 16,
371 clip: 0.25,
372 beta1: 0.0,
373 beta2: 0.999,
374 adam_eps: 1e-8,
375 }
376 }
377}
378
379#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
381pub struct RetrainConfig {
382 pub period: usize,
384 pub len: usize,
386 pub epochs: usize,
388 pub early_epochs: usize,
390 pub early_until: usize,
392 pub lr_schedule: PiecewiseLrSchedule,
394 pub update_len: usize,
396 pub batch_size: usize,
398}
399
400impl RetrainConfig {
401 #[must_use]
407 pub fn v2() -> Self {
408 Self {
409 period: 1,
410 len: 35_000_000,
411 epochs: 1,
412 early_epochs: 0,
413 early_until: 0,
414 lr_schedule: PiecewiseLrSchedule::parse(
415 "1.6e-4,13000,1.6e-4,93000,7.9e-5,163000,4.0e-5,1911300,1.3e-5",
416 )
417 .expect("v2 retrain LR schedule literal is valid"),
418 update_len: 64,
419 batch_size: 32,
420 }
421 }
422
423 #[must_use]
425 pub fn tiny() -> Self {
426 Self {
427 period: 0,
428 len: 2048,
429 epochs: 1,
430 early_epochs: 0,
431 early_until: 0,
432 lr_schedule: PiecewiseLrSchedule::constant(3e-4),
433 update_len: 16,
434 batch_size: 4,
435 }
436 }
437}
438
439#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
441pub struct CodecConfig {
442 pub model: ModelConfig,
444 pub optim: OptimConfig,
446 pub retrain: RetrainConfig,
448 pub batch_size: usize,
450 pub block_len: usize,
452 pub online_loss: OnlineLoss,
454 pub project_only_targets: bool,
456 pub seed: u64,
458 pub dtype: ComputeDType,
460 pub autocast_policy: AutocastPolicy,
462 pub device: DeviceChoice,
464 pub max_tokens: Option<usize>,
466 pub raw_bytes: Option<u64>,
468 pub vocab_bytes: u64,
470 pub vocab_file: Option<PathBuf>,
472 pub initial_prior: Option<PathBuf>,
474 pub log_interval: usize,
476 pub checkpoint_dir: Option<PathBuf>,
478 pub checkpoint_every_blocks: usize,
480 pub stop_after_blocks: Option<usize>,
482 pub run_dir: Option<PathBuf>,
484}
485
486impl CodecConfig {
487 #[must_use]
489 pub fn v2() -> Self {
490 Self {
491 model: ModelConfig::v2(),
492 optim: OptimConfig::v2(),
493 retrain: RetrainConfig::v2(),
494 batch_size: 64,
495 block_len: 500_000,
496 online_loss: OnlineLoss::Code,
497 project_only_targets: true,
498 seed: 1111,
499 dtype: ComputeDType::Bf16,
500 autocast_policy: AutocastPolicy::Auto,
501 device: DeviceChoice::Auto,
502 max_tokens: None,
503 raw_bytes: Some(1_000_000_000),
504 vocab_bytes: 34_421,
505 vocab_file: None,
506 initial_prior: None,
507 log_interval: 100_000,
508 checkpoint_dir: None,
509 checkpoint_every_blocks: 4,
510 stop_after_blocks: None,
511 run_dir: None,
512 }
513 }
514
515 #[must_use]
517 pub fn tiny(vocab_size: usize) -> Self {
518 Self {
519 model: ModelConfig::tiny(vocab_size),
520 optim: OptimConfig::tiny(),
521 retrain: RetrainConfig::tiny(),
522 batch_size: 4,
523 block_len: 64,
524 online_loss: OnlineLoss::Code,
525 project_only_targets: true,
526 seed: 1111,
527 dtype: ComputeDType::Fp32,
528 autocast_policy: AutocastPolicy::Auto,
529 device: DeviceChoice::Cpu,
530 max_tokens: None,
531 raw_bytes: None,
532 vocab_bytes: 0,
533 vocab_file: None,
534 initial_prior: None,
535 log_interval: 64,
536 checkpoint_dir: None,
537 checkpoint_every_blocks: 2,
538 stop_after_blocks: None,
539 run_dir: None,
540 }
541 }
542
543 pub fn validate(&self, mode: CodecMode) -> Result<(), CodecError> {
549 self.model.validate()?;
550 if self.batch_size == 0 {
551 return Err(CodecError::InvalidConfig(
552 "batch_size must be positive".to_owned(),
553 ));
554 }
555 if self.checkpoint_every_blocks < 1 {
556 return Err(CodecError::InvalidConfig(
557 "checkpoint_every_blocks must be at least 1".to_owned(),
558 ));
559 }
560 if self.project_only_targets && self.online_loss != OnlineLoss::Code {
561 return Err(CodecError::InvalidConfig(
562 "project_only_targets requires online_loss = Code".to_owned(),
563 ));
564 }
565 if self.retrain.epochs < 1 {
566 return Err(CodecError::InvalidConfig(
567 "retrain.epochs must be at least 1".to_owned(),
568 ));
569 }
570 if (self.retrain.early_epochs == 0) != (self.retrain.early_until == 0) {
571 return Err(CodecError::InvalidConfig(
572 "retrain.early_epochs and retrain.early_until must either both be set or both be 0"
573 .to_owned(),
574 ));
575 }
576 if self.retrain.update_len > self.model.ctx_len {
577 return Err(CodecError::InvalidConfig(format!(
578 "retrain.update_len ({}) must be <= ctx_len ({})",
579 self.retrain.update_len, self.model.ctx_len
580 )));
581 }
582 if mode == CodecMode::Decompress && self.max_tokens.is_some() {
583 return Err(CodecError::InvalidConfig(
584 "max_tokens applies only when compressing".to_owned(),
585 ));
586 }
587 if let Some(prior) = &self.initial_prior {
588 if !prior.is_file() {
589 return Err(CodecError::InvalidConfig(format!(
590 "initial_prior does not exist: {}",
591 prior.display()
592 )));
593 }
594 if self.vocab_file.is_none() {
595 return Err(CodecError::InvalidConfig(
596 "vocab_file is required when initial_prior is specified".to_owned(),
597 ));
598 }
599 }
600 Ok(())
601 }
602
603 #[must_use]
605 pub fn checkpoint_signature(&self) -> String {
606 #[derive(Serialize)]
607 struct Sig<'a> {
608 adam_eps: f64,
609 batch_size: usize,
610 beta1: f64,
611 beta2: f64,
612 block_len: usize,
613 block_schedule: &'a Option<Vec<usize>>,
614 clip: f64,
615 ctx_len: usize,
616 d_inner: usize,
617 d_model: usize,
618 dropout: f32,
619 dtype: ComputeDType,
620 lr: f64,
621 min_lr_ratio: f64,
622 n_head: usize,
623 n_layer: usize,
624 norm_style: NormStyle,
625 online_loss: OnlineLoss,
626 position_type: PositionType,
627 qk_norm: bool,
628 retrain_batch_size: usize,
629 retrain_epochs: usize,
630 retrain_len: usize,
631 retrain_lr: String,
632 retrain_period: usize,
633 retrain_update_len: usize,
634 rope_base: f64,
635 seed: u64,
636 tie_embeddings: bool,
637 vocab_size: usize,
638 warmup: usize,
639 }
640
641 let sig = Sig {
642 adam_eps: self.optim.adam_eps,
643 batch_size: self.batch_size,
644 beta1: self.optim.beta1,
645 beta2: self.optim.beta2,
646 block_len: self.block_len,
647 block_schedule: &self.model.block_schedule,
648 clip: self.optim.clip,
649 ctx_len: self.model.ctx_len,
650 d_inner: self.model.d_inner,
651 d_model: self.model.d_model,
652 dropout: self.model.dropout,
653 dtype: self.dtype,
654 lr: self.optim.lr,
655 min_lr_ratio: self.optim.min_lr_ratio,
656 n_head: self.model.n_head,
657 n_layer: self.model.n_layer,
658 norm_style: self.model.norm_style,
659 online_loss: self.online_loss,
660 position_type: self.model.position_type,
661 qk_norm: self.model.qk_norm,
662 retrain_batch_size: self.retrain.batch_size,
663 retrain_epochs: self.retrain.epochs,
664 retrain_len: self.retrain.len,
665 retrain_lr: self.retrain.lr_schedule.to_spec(),
666 retrain_period: self.retrain.period,
667 retrain_update_len: self.retrain.update_len,
668 rope_base: self.model.rope_base,
669 seed: self.seed,
670 tie_embeddings: self.model.tie_embeddings,
671 vocab_size: self.model.vocab_size,
672 warmup: self.optim.warmup,
673 };
674 serde_json::to_string(&sig).unwrap_or_default()
675 }
676
677 #[must_use]
679 pub fn effective_compute_dtype(&self, device: &Device) -> DType {
680 match self.dtype {
681 ComputeDType::Fp32 => DType::F32,
682 ComputeDType::Bf16 => match self.autocast_policy {
683 AutocastPolicy::Force => DType::BF16,
684 AutocastPolicy::Auto => {
685 if matches!(device, Device::Cpu) {
686 DType::F32
687 } else {
688 DType::BF16
689 }
690 }
691 },
692 }
693 }
694}
695
696pub fn resolve_device(choice: DeviceChoice) -> Result<Device, CodecError> {
703 match choice {
704 DeviceChoice::Cpu => Ok(Device::Cpu),
705 DeviceChoice::Cuda => {
706 #[cfg(feature = "cuda")]
707 {
708 Ok(Device::new_cuda(0)?)
709 }
710 #[cfg(not(feature = "cuda"))]
711 {
712 Err(CodecError::InvalidConfig(
713 "CUDA device requested, but tufazip-codec was built without the `cuda` feature"
714 .to_owned(),
715 ))
716 }
717 }
718 DeviceChoice::Metal => {
719 #[cfg(feature = "metal")]
720 {
721 Ok(Device::new_metal(0)?)
722 }
723 #[cfg(not(feature = "metal"))]
724 {
725 Err(CodecError::InvalidConfig(
726 "Metal device requested, but tufazip-codec was built without the `metal` feature"
727 .to_owned(),
728 ))
729 }
730 }
731 DeviceChoice::Auto => {
732 #[cfg(feature = "cuda")]
733 if let Ok(dev) = Device::new_cuda(0) {
734 return Ok(dev);
735 }
736 #[cfg(feature = "metal")]
737 if let Ok(dev) = Device::new_metal(0) {
738 return Ok(dev);
739 }
740 Ok(Device::Cpu)
741 }
742 }
743}