1use std::fs;
4use std::path::Path;
5
6use serde::{Deserialize, Deserializer, Serialize};
7
8use crate::error::{ModelError, Result};
9
10#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
12pub enum ModelArchitecture {
13 Llama,
15 Qwen2,
17 Qwen3,
19 Phi3,
21 Gemma2,
23 Gemma3,
25 Gemma4,
27 Opt,
29}
30
31#[derive(Debug, Clone, Copy, PartialEq)]
33pub struct ResolvedRopeParams {
34 pub rope_theta: f64,
36 pub partial_rotary_factor: f64,
38 pub scale_factor: f64,
40 pub is_proportional: bool,
42}
43
44#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
46#[serde(untagged)]
47pub enum TokenIdSpec {
48 Single(u32),
50 Multiple(Vec<u32>),
52}
53
54impl TokenIdSpec {
55 #[must_use]
57 pub fn primary(&self) -> Option<u32> {
58 match self {
59 Self::Single(id) => Some(*id),
60 Self::Multiple(ids) => ids.first().copied(),
61 }
62 }
63
64 #[must_use]
66 pub fn contains(&self, id: u32) -> bool {
67 match self {
68 Self::Single(v) => *v == id,
69 Self::Multiple(vs) => vs.contains(&id),
70 }
71 }
72}
73
74impl<'de> Deserialize<'de> for TokenIdSpec {
75 fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
76 #[derive(Deserialize)]
77 #[serde(untagged)]
78 enum Raw {
79 One(u32),
80 Many(Vec<u32>),
81 }
82 match Raw::deserialize(deserializer)? {
83 Raw::One(v) => Ok(Self::Single(v)),
84 Raw::Many(vs) => Ok(Self::Multiple(vs)),
85 }
86 }
87}
88
89#[derive(Debug, Clone, Serialize, Deserialize)]
91#[serde(default)]
92pub struct ModelConfig {
93 pub model_type: String,
95 pub architectures: Vec<String>,
97 pub vocab_size: usize,
99 pub hidden_size: usize,
101 pub intermediate_size: usize,
103 pub ffn_dim: Option<usize>,
105 pub word_embed_proj_dim: Option<usize>,
107 pub num_hidden_layers: usize,
109 pub num_attention_heads: usize,
111 pub num_key_value_heads: Option<usize>,
113 pub head_dim: Option<usize>,
115 pub max_position_embeddings: usize,
117 pub n_ctx: Option<usize>,
119 pub rms_norm_eps: f64,
121 pub rope_theta: f64,
123 pub rope_local_base_freq: Option<f64>,
125 pub tie_word_embeddings: bool,
127 pub attention_bias: bool,
129 pub mlp_bias: bool,
131 pub enable_bias: Option<bool>,
133 pub do_layer_norm_before: Option<bool>,
135 pub layer_norm_elementwise_affine: Option<bool>,
137 pub hidden_act: String,
139 pub attn_logit_softcapping: Option<f64>,
141 pub final_logit_softcapping: Option<f64>,
143 pub query_pre_attn_scalar: Option<f64>,
145 pub sliding_window: Option<usize>,
147 pub sliding_window_pattern: Option<usize>,
149 #[serde(default, skip_serializing_if = "Option::is_none")]
151 pub layer_types: Option<Vec<String>>,
152 #[serde(default, skip_serializing_if = "Option::is_none")]
154 pub global_head_dim: Option<usize>,
155 #[serde(default, skip_serializing_if = "Option::is_none")]
157 pub num_global_key_value_heads: Option<usize>,
158 #[serde(default, skip_serializing_if = "Option::is_none")]
160 pub attention_k_eq_v: Option<bool>,
161 #[serde(default, skip_serializing_if = "Option::is_none")]
163 pub rope_parameters: Option<serde_json::Value>,
164 #[serde(default, skip_serializing_if = "Option::is_none")]
166 pub partial_rotary_factor: Option<f64>,
167 #[serde(default, skip_serializing_if = "Option::is_none")]
169 pub hidden_activation: Option<String>,
170 #[serde(default, skip_serializing_if = "Option::is_none")]
172 pub enable_moe_block: Option<bool>,
173 #[serde(default, skip_serializing_if = "Option::is_none")]
175 pub num_experts: Option<usize>,
176 #[serde(default, skip_serializing_if = "Option::is_none")]
178 pub top_k_experts: Option<usize>,
179 #[serde(default, skip_serializing_if = "Option::is_none")]
181 pub moe_intermediate_size: Option<usize>,
182 #[serde(default, skip_serializing_if = "Option::is_none")]
184 pub num_kv_shared_layers: Option<usize>,
185 #[serde(default, skip_serializing_if = "Option::is_none")]
187 pub use_double_wide_mlp: Option<bool>,
188 pub text_config: Option<Box<ModelConfig>>,
190 pub bos_token_id: Option<TokenIdSpec>,
192 pub eos_token_id: Option<TokenIdSpec>,
194 pub pad_token_id: Option<TokenIdSpec>,
196 #[serde(flatten)]
198 pub extra: serde_json::Map<String, serde_json::Value>,
199}
200
201impl Default for ModelConfig {
202 fn default() -> Self {
203 Self {
204 model_type: "llama".to_string(),
205 architectures: Vec::new(),
206 vocab_size: 32000,
207 hidden_size: 0,
208 intermediate_size: 0,
209 ffn_dim: None,
210 word_embed_proj_dim: None,
211 num_hidden_layers: 0,
212 num_attention_heads: 0,
213 num_key_value_heads: None,
214 head_dim: None,
215 max_position_embeddings: 2048,
216 n_ctx: None,
217 rms_norm_eps: 1e-5,
218 rope_theta: 10000.0,
219 rope_local_base_freq: None,
220 tie_word_embeddings: false,
221 attention_bias: false,
222 mlp_bias: false,
223 enable_bias: None,
224 do_layer_norm_before: None,
225 layer_norm_elementwise_affine: None,
226 hidden_act: "silu".to_string(),
227 attn_logit_softcapping: None,
228 final_logit_softcapping: None,
229 query_pre_attn_scalar: None,
230 sliding_window: None,
231 sliding_window_pattern: None,
232 layer_types: None,
233 global_head_dim: None,
234 num_global_key_value_heads: None,
235 attention_k_eq_v: None,
236 rope_parameters: None,
237 partial_rotary_factor: None,
238 hidden_activation: None,
239 enable_moe_block: None,
240 num_experts: None,
241 top_k_experts: None,
242 moe_intermediate_size: None,
243 num_kv_shared_layers: None,
244 use_double_wide_mlp: None,
245 text_config: None,
246 bos_token_id: None,
247 eos_token_id: None,
248 pad_token_id: None,
249 extra: serde_json::Map::new(),
250 }
251 }
252}
253
254impl ModelConfig {
255 pub fn from_file(path: &Path) -> Result<Self> {
260 let raw = fs::read_to_string(path)?;
261 let cfg: Self = serde_json::from_str(&raw)?;
262 Ok(cfg.resolve_effective())
263 }
264
265 #[must_use]
267 pub fn resolve_effective(mut self) -> Self {
268 if self.hidden_size == 0 {
269 if let Some(inner) = self.text_config.take() {
270 let mut resolved = *inner;
271 if resolved.model_type.is_empty()
272 || (resolved.model_type == "llama"
273 && !self.model_type.eq_ignore_ascii_case("llama"))
274 {
275 resolved.model_type = self.model_type;
276 }
277 if resolved.architectures.is_empty() {
278 resolved.architectures = self.architectures;
279 }
280 if !resolved.tie_word_embeddings && self.tie_word_embeddings {
281 resolved.tie_word_embeddings = true;
282 }
283 if resolved.bos_token_id.is_none() {
284 resolved.bos_token_id = self.bos_token_id;
285 }
286 if resolved.eos_token_id.is_none() {
287 resolved.eos_token_id = self.eos_token_id;
288 }
289 if resolved.pad_token_id.is_none() {
290 resolved.pad_token_id = self.pad_token_id;
291 }
292 return resolved.resolve_effective();
293 }
294 }
295 if self.intermediate_size == 0 {
296 if let Some(ffn) = self.ffn_dim {
297 self.intermediate_size = ffn;
298 }
299 }
300 self
301 }
302
303 pub fn detect_architecture(&self) -> Result<ModelArchitecture> {
308 let mt = self.model_type.to_ascii_lowercase();
309 match mt.as_str() {
310 "llama" => Ok(ModelArchitecture::Llama),
311 "qwen2" => Ok(ModelArchitecture::Qwen2),
312 "qwen3" => Ok(ModelArchitecture::Qwen3),
313 "phi3" => Ok(ModelArchitecture::Phi3),
314 "gemma2" => Ok(ModelArchitecture::Gemma2),
315 "gemma3" | "gemma3_text" => Ok(ModelArchitecture::Gemma3),
316 "gemma4" | "gemma4_text" => Ok(ModelArchitecture::Gemma4),
317 "opt" => Ok(ModelArchitecture::Opt),
318 _ => Err(ModelError::UnsupportedArchitecture(format!(
319 "unsupported model_type `{}`; supported: opt, llama, phi3, qwen2, qwen3, gemma2, gemma3, gemma4",
320 self.model_type
321 ))),
322 }
323 }
324
325 #[must_use]
327 pub fn effective_hidden_act(&self) -> &str {
328 match self.hidden_activation.as_deref() {
329 Some(act) if !act.is_empty() => act,
330 _ => &self.hidden_act,
331 }
332 }
333
334 #[must_use]
336 pub fn effective_head_dim(&self) -> usize {
337 self.head_dim.unwrap_or_else(|| {
338 self.hidden_size
339 .checked_div(self.num_attention_heads)
340 .unwrap_or(0)
341 })
342 }
343
344 #[must_use]
346 pub fn effective_global_head_dim(&self) -> usize {
347 self.global_head_dim
348 .unwrap_or_else(|| self.effective_head_dim())
349 }
350
351 #[must_use]
353 pub fn effective_kv_heads(&self) -> usize {
354 self.num_key_value_heads.unwrap_or(self.num_attention_heads)
355 }
356
357 #[must_use]
359 pub fn effective_global_kv_heads(&self) -> usize {
360 self.num_global_key_value_heads
361 .unwrap_or_else(|| self.effective_kv_heads())
362 }
363
364 #[must_use]
366 pub fn effective_layer_types(&self) -> Vec<String> {
367 if let Some(ref lt) = self.layer_types {
368 if lt.len() == self.num_hidden_layers {
369 return lt.clone();
370 }
371 }
372 if let Some(pattern) = self.sliding_window_pattern {
373 if pattern > 0 {
374 return (0..self.num_hidden_layers)
375 .map(|i| {
376 if (i + 1).is_multiple_of(pattern) {
377 "full_attention".to_string()
378 } else {
379 "sliding_attention".to_string()
380 }
381 })
382 .collect();
383 }
384 }
385 vec!["full_attention".to_string(); self.num_hidden_layers]
386 }
387
388 #[must_use]
390 pub fn gemma4_rope_params(&self, layer_type: &str) -> ResolvedRopeParams {
391 use serde_json::Value;
392
393 let default_theta = if layer_type == "sliding_attention" {
394 self.rope_local_base_freq.unwrap_or(self.rope_theta)
395 } else {
396 self.rope_theta
397 };
398 let default_partial = self.partial_rotary_factor.unwrap_or(1.0);
399
400 let entry = self
401 .rope_parameters
402 .as_ref()
403 .and_then(Value::as_object)
404 .and_then(|map| {
405 map.get(layer_type).and_then(Value::as_object).or_else(|| {
406 if map.contains_key("rope_type")
407 || map.contains_key("rope_theta")
408 || map.contains_key("partial_rotary_factor")
409 {
410 Some(map)
411 } else {
412 None
413 }
414 })
415 });
416
417 match entry {
418 Some(obj) => ResolvedRopeParams {
419 rope_theta: obj
420 .get("rope_theta")
421 .and_then(Value::as_f64)
422 .unwrap_or(default_theta),
423 partial_rotary_factor: obj
424 .get("partial_rotary_factor")
425 .and_then(Value::as_f64)
426 .unwrap_or(default_partial),
427 scale_factor: obj.get("factor").and_then(Value::as_f64).unwrap_or(1.0),
428 is_proportional: obj
429 .get("rope_type")
430 .and_then(Value::as_str)
431 .is_some_and(|s| s.eq_ignore_ascii_case("proportional")),
432 },
433 None => ResolvedRopeParams {
434 rope_theta: default_theta,
435 partial_rotary_factor: default_partial,
436 scale_factor: 1.0,
437 is_proportional: false,
438 },
439 }
440 }
441
442 #[must_use]
444 pub fn effective_seqlen(&self) -> usize {
445 self.n_ctx.unwrap_or(if self.max_position_embeddings > 0 {
446 self.max_position_embeddings
447 } else {
448 2048
449 })
450 }
451
452 #[must_use]
454 pub fn effective_intermediate_size(&self) -> usize {
455 if self.intermediate_size > 0 {
456 self.intermediate_size
457 } else {
458 self.ffn_dim.unwrap_or(0)
459 }
460 }
461
462 #[must_use]
464 pub fn tiny_llama(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
465 Self {
466 model_type: "llama".to_string(),
467 architectures: vec!["LlamaForCausalLM".to_string()],
468 vocab_size,
469 hidden_size,
470 intermediate_size: hidden_size * 2,
471 num_hidden_layers: layers,
472 num_attention_heads: 4,
473 num_key_value_heads: Some(2),
474 max_position_embeddings: 256,
475 rms_norm_eps: 1e-5,
476 rope_theta: 10000.0,
477 ..Default::default()
478 }
479 }
480
481 #[must_use]
483 pub fn tiny_qwen2(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
484 Self {
485 model_type: "qwen2".to_string(),
486 architectures: vec!["Qwen2ForCausalLM".to_string()],
487 vocab_size,
488 hidden_size,
489 intermediate_size: hidden_size * 2,
490 num_hidden_layers: layers,
491 num_attention_heads: 4,
492 num_key_value_heads: Some(2),
493 max_position_embeddings: 256,
494 rms_norm_eps: 1e-6,
495 rope_theta: 1000000.0,
496 attention_bias: true,
497 ..Default::default()
498 }
499 }
500
501 #[must_use]
503 pub fn tiny_qwen3(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
504 Self {
505 model_type: "qwen3".to_string(),
506 architectures: vec!["Qwen3ForCausalLM".to_string()],
507 vocab_size,
508 hidden_size,
509 intermediate_size: hidden_size * 2,
510 num_hidden_layers: layers,
511 num_attention_heads: 4,
512 num_key_value_heads: Some(2),
513 head_dim: Some(hidden_size / 4),
514 max_position_embeddings: 256,
515 rms_norm_eps: 1e-6,
516 rope_theta: 1000000.0,
517 ..Default::default()
518 }
519 }
520
521 #[must_use]
523 pub fn tiny_phi3(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
524 Self {
525 model_type: "phi3".to_string(),
526 architectures: vec!["Phi3ForCausalLM".to_string()],
527 vocab_size,
528 hidden_size,
529 intermediate_size: hidden_size * 2,
530 num_hidden_layers: layers,
531 num_attention_heads: 4,
532 num_key_value_heads: Some(2),
533 max_position_embeddings: 256,
534 rms_norm_eps: 1e-5,
535 rope_theta: 10000.0,
536 attention_bias: false,
537 ..Default::default()
538 }
539 }
540
541 #[must_use]
543 pub fn tiny_gemma2(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
544 Self {
545 model_type: "gemma2".to_string(),
546 architectures: vec!["Gemma2ForCausalLM".to_string()],
547 vocab_size,
548 hidden_size,
549 intermediate_size: hidden_size * 2,
550 num_hidden_layers: layers,
551 num_attention_heads: 4,
552 num_key_value_heads: Some(2),
553 head_dim: Some(hidden_size / 4),
554 max_position_embeddings: 256,
555 rms_norm_eps: 1e-6,
556 rope_theta: 10000.0,
557 tie_word_embeddings: true,
558 hidden_act: "gelu_pytorch_tanh".to_string(),
559 attn_logit_softcapping: Some(50.0),
560 final_logit_softcapping: Some(30.0),
561 query_pre_attn_scalar: Some((hidden_size / 4) as f64),
562 sliding_window: Some(64),
563 ..Default::default()
564 }
565 }
566
567 #[must_use]
569 pub fn tiny_gemma3(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
570 Self {
571 model_type: "gemma3_text".to_string(),
572 architectures: vec!["Gemma3ForCausalLM".to_string()],
573 vocab_size,
574 hidden_size,
575 intermediate_size: hidden_size * 2,
576 num_hidden_layers: layers,
577 num_attention_heads: 4,
578 num_key_value_heads: Some(2),
579 head_dim: Some(hidden_size / 4),
580 max_position_embeddings: 256,
581 rms_norm_eps: 1e-6,
582 rope_theta: 1_000_000.0,
583 rope_local_base_freq: Some(10000.0),
584 tie_word_embeddings: true,
585 hidden_act: "gelu_pytorch_tanh".to_string(),
586 query_pre_attn_scalar: Some((hidden_size / 4) as f64),
587 sliding_window: Some(64),
588 sliding_window_pattern: Some(2),
589 ..Default::default()
590 }
591 }
592
593 #[must_use]
595 pub fn tiny_gemma4_dense(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
596 let layer_types = (0..layers)
597 .map(|i| {
598 if (i + 1).is_multiple_of(2) {
599 "full_attention".to_string()
600 } else {
601 "sliding_attention".to_string()
602 }
603 })
604 .collect();
605 let head_dim = (hidden_size / 4).max(2);
606 let global_head_dim = (hidden_size / 2).max(8);
607 Self {
608 model_type: "gemma4_text".to_string(),
609 architectures: vec!["Gemma4ForCausalLM".to_string()],
610 vocab_size,
611 hidden_size,
612 intermediate_size: hidden_size * 2,
613 num_hidden_layers: layers,
614 num_attention_heads: 4,
615 num_key_value_heads: Some(2),
616 head_dim: Some(head_dim),
617 global_head_dim: Some(global_head_dim),
618 num_global_key_value_heads: Some(1),
619 attention_k_eq_v: Some(true),
620 max_position_embeddings: 256,
621 rms_norm_eps: 1e-6,
622 rope_theta: 1_000_000.0,
623 rope_local_base_freq: Some(10_000.0),
624 tie_word_embeddings: true,
625 hidden_act: "gelu_pytorch_tanh".to_string(),
626 hidden_activation: Some("gelu_pytorch_tanh".to_string()),
627 final_logit_softcapping: Some(30.0),
628 sliding_window: Some(64),
629 layer_types: Some(layer_types),
630 rope_parameters: Some(serde_json::json!({
631 "sliding_attention": {
632 "rope_type": "default",
633 "rope_theta": 10000.0
634 },
635 "full_attention": {
636 "rope_type": "proportional",
637 "partial_rotary_factor": 0.25,
638 "rope_theta": 1_000_000.0
639 }
640 })),
641 ..Default::default()
642 }
643 }
644
645 #[must_use]
647 pub fn tiny_gemma4_moe(
648 vocab_size: usize,
649 hidden_size: usize,
650 layers: usize,
651 num_experts: usize,
652 top_k: usize,
653 ) -> Self {
654 let mut cfg = Self::tiny_gemma4_dense(vocab_size, hidden_size, layers);
655 cfg.enable_moe_block = Some(true);
656 cfg.num_experts = Some(num_experts);
657 cfg.top_k_experts = Some(top_k);
658 cfg.moe_intermediate_size = Some(hidden_size);
659 cfg
660 }
661
662 #[must_use]
664 pub fn tiny_opt(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
665 Self {
666 model_type: "opt".to_string(),
667 architectures: vec!["OPTForCausalLM".to_string()],
668 vocab_size,
669 hidden_size,
670 intermediate_size: hidden_size * 2,
671 ffn_dim: Some(hidden_size * 2),
672 word_embed_proj_dim: Some(hidden_size),
673 num_hidden_layers: layers,
674 num_attention_heads: 4,
675 max_position_embeddings: 256,
676 tie_word_embeddings: true,
677 enable_bias: Some(true),
678 do_layer_norm_before: Some(true),
679 layer_norm_elementwise_affine: Some(true),
680 ..Default::default()
681 }
682 }
683}