Skip to main content

littlebit_models/
config.rs

1//! Hugging Face `config.json` parsing and architecture normalization across all supported LittleBit families.
2
3use std::fs;
4use std::path::Path;
5
6use serde::{Deserialize, Deserializer, Serialize};
7
8use crate::error::{ModelError, Result};
9
10/// Supported LLM architecture families matching `SamsungLabs/LittleBit` plus Gemma 4.
11#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
12pub enum ModelArchitecture {
13    /// Llama, Llama 2, and Llama 3 (`model_type = "llama"`).
14    Llama,
15    /// Qwen2, Qwen2.5, and QwQ (`model_type = "qwen2"`).
16    Qwen2,
17    /// Qwen3 (`model_type = "qwen3"`).
18    Qwen3,
19    /// Phi-3 and Phi-4 (`model_type = "phi3"`).
20    Phi3,
21    /// Gemma 2 (`model_type = "gemma2"`).
22    Gemma2,
23    /// Gemma 3 (`model_type = "gemma3" | "gemma3_text"`).
24    Gemma3,
25    /// Gemma 4 Dense and MoE (`model_type = "gemma4" | "gemma4_text"`).
26    Gemma4,
27    /// Meta OPT (`model_type = "opt"`).
28    Opt,
29}
30
31/// Resolved RoPE configuration for a specific attention layer type.
32#[derive(Debug, Clone, Copy, PartialEq)]
33pub struct ResolvedRopeParams {
34    /// Inverse frequency base (`rope_theta`).
35    pub rope_theta: f64,
36    /// Fraction of head dimensions rotated (`(0.0, 1.0]`).
37    pub partial_rotary_factor: f64,
38    /// Frequency scaling divisor (`factor`, `1.0` when unscaled).
39    pub scale_factor: f64,
40    /// Whether `rope_type == "proportional"` (full `head_dim` denominator in frequencies).
41    pub is_proportional: bool,
42}
43
44/// Token ID specification allowing either a single ID or a list (as in Llama 3 `eos_token_id`).
45#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
46#[serde(untagged)]
47pub enum TokenIdSpec {
48    /// Single token ID.
49    Single(u32),
50    /// Multiple valid token IDs (e.g., EOS + EOT).
51    Multiple(Vec<u32>),
52}
53
54impl TokenIdSpec {
55    /// Primary token ID (`Single(id)` or `Multiple[0]`).
56    #[must_use]
57    pub fn primary(&self) -> Option<u32> {
58        match self {
59            Self::Single(id) => Some(*id),
60            Self::Multiple(ids) => ids.first().copied(),
61        }
62    }
63
64    /// Return true if `id` is contained in this token spec.
65    #[must_use]
66    pub fn contains(&self, id: u32) -> bool {
67        match self {
68            Self::Single(v) => *v == id,
69            Self::Multiple(vs) => vs.contains(&id),
70        }
71    }
72}
73
74impl<'de> Deserialize<'de> for TokenIdSpec {
75    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
76        #[derive(Deserialize)]
77        #[serde(untagged)]
78        enum Raw {
79            One(u32),
80            Many(Vec<u32>),
81        }
82        match Raw::deserialize(deserializer)? {
83            Raw::One(v) => Ok(Self::Single(v)),
84            Raw::Many(vs) => Ok(Self::Multiple(vs)),
85        }
86    }
87}
88
89/// Unified Hugging Face `config.json` model configuration.
90#[derive(Debug, Clone, Serialize, Deserialize)]
91#[serde(default)]
92pub struct ModelConfig {
93    /// Architecture identifier (`llama`, `qwen2`, `qwen3`, `phi3`, `gemma2`, `gemma3`, `opt`).
94    pub model_type: String,
95    /// Hugging Face architecture class names.
96    pub architectures: Vec<String>,
97    /// Vocabulary size.
98    pub vocab_size: usize,
99    /// Transformer hidden dimension.
100    pub hidden_size: usize,
101    /// Feed-forward intermediate dimension (`ffn_dim` for OPT).
102    pub intermediate_size: usize,
103    /// OPT alias for `intermediate_size`.
104    pub ffn_dim: Option<usize>,
105    /// OPT projection dimension between embeddings and decoder hidden states.
106    pub word_embed_proj_dim: Option<usize>,
107    /// Number of transformer decoder layers.
108    pub num_hidden_layers: usize,
109    /// Number of query attention heads.
110    pub num_attention_heads: usize,
111    /// Number of key/value attention heads (GQA when smaller than `num_attention_heads`).
112    pub num_key_value_heads: Option<usize>,
113    /// Explicit per-head dimension (defaults to `hidden_size / num_attention_heads`).
114    pub head_dim: Option<usize>,
115    /// Maximum context length.
116    pub max_position_embeddings: usize,
117    /// Legacy context length alias (`n_ctx`).
118    pub n_ctx: Option<usize>,
119    /// RMSNorm / LayerNorm epsilon.
120    pub rms_norm_eps: f64,
121    /// RoPE base frequency (`rope_theta`).
122    pub rope_theta: f64,
123    /// Local sliding-window RoPE base frequency for Gemma 3.
124    pub rope_local_base_freq: Option<f64>,
125    /// Whether `lm_head` shares weights with `embed_tokens`.
126    pub tie_word_embeddings: bool,
127    /// Whether Q/K/V/O attention projections carry biases.
128    pub attention_bias: bool,
129    /// Whether MLP projections carry biases.
130    pub mlp_bias: bool,
131    /// OPT bias toggle (`enable_bias`).
132    pub enable_bias: Option<bool>,
133    /// OPT pre-norm vs. post-norm toggle (`do_layer_norm_before`).
134    pub do_layer_norm_before: Option<bool>,
135    /// OPT affine toggle on LayerNorm.
136    pub layer_norm_elementwise_affine: Option<bool>,
137    /// Hidden activation function name (`silu`, `gelu`, `gelu_pytorch_tanh`, `relu`).
138    pub hidden_act: String,
139    /// Gemma 2/3 attention logit softcapping threshold.
140    pub attn_logit_softcapping: Option<f64>,
141    /// Gemma 2/3 final logit softcapping threshold.
142    pub final_logit_softcapping: Option<f64>,
143    /// Gemma 2/3 attention scaling denominator (`1 / sqrt(query_pre_attn_scalar)`).
144    pub query_pre_attn_scalar: Option<f64>,
145    /// Sliding window size (Gemma 2/3, Phi).
146    pub sliding_window: Option<usize>,
147    /// Gemma 3 sliding window periodicity (every Nth layer is global).
148    pub sliding_window_pattern: Option<usize>,
149    /// Per-layer attention type schedule (`"sliding_attention"` vs `"full_attention"`), used by Gemma 4.
150    #[serde(default, skip_serializing_if = "Option::is_none")]
151    pub layer_types: Option<Vec<String>>,
152    /// Global (`full_attention`) head dimension when larger than sliding `head_dim` (Gemma 4).
153    #[serde(default, skip_serializing_if = "Option::is_none")]
154    pub global_head_dim: Option<usize>,
155    /// Number of key/value heads for global (`full_attention`) layers when distinct from sliding layers (Gemma 4).
156    #[serde(default, skip_serializing_if = "Option::is_none")]
157    pub num_global_key_value_heads: Option<usize>,
158    /// Whether global attention layers share keys and values (`V = K`, omitting `v_proj`), used by Gemma 4.
159    #[serde(default, skip_serializing_if = "Option::is_none")]
160    pub attention_k_eq_v: Option<bool>,
161    /// Per-layer-type RoPE parameters (`rope_type`, `rope_theta`, `partial_rotary_factor`, `factor`) for Gemma 4.
162    #[serde(default, skip_serializing_if = "Option::is_none")]
163    pub rope_parameters: Option<serde_json::Value>,
164    /// Default fraction of head dims rotated when partial RoPE is enabled without nested `rope_parameters`.
165    #[serde(default, skip_serializing_if = "Option::is_none")]
166    pub partial_rotary_factor: Option<f64>,
167    /// Alternative activation key (`hidden_activation` alias for `hidden_act`).
168    #[serde(default, skip_serializing_if = "Option::is_none")]
169    pub hidden_activation: Option<String>,
170    /// Enable Mixture-of-Experts FFN block alongside the shared dense MLP (Gemma 4 MoE).
171    #[serde(default, skip_serializing_if = "Option::is_none")]
172    pub enable_moe_block: Option<bool>,
173    /// Total number of routed experts in MoE layers (Gemma 4 MoE).
174    #[serde(default, skip_serializing_if = "Option::is_none")]
175    pub num_experts: Option<usize>,
176    /// Number of top-ranked experts activated per token (e.g., 8 in Gemma 4 MoE).
177    #[serde(default, skip_serializing_if = "Option::is_none")]
178    pub top_k_experts: Option<usize>,
179    /// Intermediate dimension of each routed MoE expert MLP (Gemma 4 MoE).
180    #[serde(default, skip_serializing_if = "Option::is_none")]
181    pub moe_intermediate_size: Option<usize>,
182    /// Number of trailing KV-shared edge layers (Gemma 4 E2B/E4B; `0` for Dense 31B / MoE 26B).
183    #[serde(default, skip_serializing_if = "Option::is_none")]
184    pub num_kv_shared_layers: Option<usize>,
185    /// Double-wide MLP flag for KV-shared layers (Gemma 4 edge variants).
186    #[serde(default, skip_serializing_if = "Option::is_none")]
187    pub use_double_wide_mlp: Option<bool>,
188    /// Optional nested `text_config` used by Gemma 3 / Gemma 4 multimodal checkpoints.
189    pub text_config: Option<Box<ModelConfig>>,
190    /// Beginning-of-sequence token ID.
191    pub bos_token_id: Option<TokenIdSpec>,
192    /// End-of-sequence token ID(s).
193    pub eos_token_id: Option<TokenIdSpec>,
194    /// Padding token ID.
195    pub pad_token_id: Option<TokenIdSpec>,
196    /// Preserved extra `config.json` keys for lossless round-trip serialization.
197    #[serde(flatten)]
198    pub extra: serde_json::Map<String, serde_json::Value>,
199}
200
201impl Default for ModelConfig {
202    fn default() -> Self {
203        Self {
204            model_type: "llama".to_string(),
205            architectures: Vec::new(),
206            vocab_size: 32000,
207            hidden_size: 0,
208            intermediate_size: 0,
209            ffn_dim: None,
210            word_embed_proj_dim: None,
211            num_hidden_layers: 0,
212            num_attention_heads: 0,
213            num_key_value_heads: None,
214            head_dim: None,
215            max_position_embeddings: 2048,
216            n_ctx: None,
217            rms_norm_eps: 1e-5,
218            rope_theta: 10000.0,
219            rope_local_base_freq: None,
220            tie_word_embeddings: false,
221            attention_bias: false,
222            mlp_bias: false,
223            enable_bias: None,
224            do_layer_norm_before: None,
225            layer_norm_elementwise_affine: None,
226            hidden_act: "silu".to_string(),
227            attn_logit_softcapping: None,
228            final_logit_softcapping: None,
229            query_pre_attn_scalar: None,
230            sliding_window: None,
231            sliding_window_pattern: None,
232            layer_types: None,
233            global_head_dim: None,
234            num_global_key_value_heads: None,
235            attention_k_eq_v: None,
236            rope_parameters: None,
237            partial_rotary_factor: None,
238            hidden_activation: None,
239            enable_moe_block: None,
240            num_experts: None,
241            top_k_experts: None,
242            moe_intermediate_size: None,
243            num_kv_shared_layers: None,
244            use_double_wide_mlp: None,
245            text_config: None,
246            bos_token_id: None,
247            eos_token_id: None,
248            pad_token_id: None,
249            extra: serde_json::Map::new(),
250        }
251    }
252}
253
254impl ModelConfig {
255    /// Load and parse a Hugging Face `config.json` file.
256    ///
257    /// # Errors
258    /// Returns an error if the file cannot be read or fails JSON deserialization.
259    pub fn from_file(path: &Path) -> Result<Self> {
260        let raw = fs::read_to_string(path)?;
261        let cfg: Self = serde_json::from_str(&raw)?;
262        Ok(cfg.resolve_effective())
263    }
264
265    /// Unwrap nested `text_config` when present (e.g., Gemma 3 / Gemma 4 multimodal checkpoints).
266    #[must_use]
267    pub fn resolve_effective(mut self) -> Self {
268        if self.hidden_size == 0 {
269            if let Some(inner) = self.text_config.take() {
270                let mut resolved = *inner;
271                if resolved.model_type.is_empty()
272                    || (resolved.model_type == "llama"
273                        && !self.model_type.eq_ignore_ascii_case("llama"))
274                {
275                    resolved.model_type = self.model_type;
276                }
277                if resolved.architectures.is_empty() {
278                    resolved.architectures = self.architectures;
279                }
280                if !resolved.tie_word_embeddings && self.tie_word_embeddings {
281                    resolved.tie_word_embeddings = true;
282                }
283                if resolved.bos_token_id.is_none() {
284                    resolved.bos_token_id = self.bos_token_id;
285                }
286                if resolved.eos_token_id.is_none() {
287                    resolved.eos_token_id = self.eos_token_id;
288                }
289                if resolved.pad_token_id.is_none() {
290                    resolved.pad_token_id = self.pad_token_id;
291                }
292                return resolved.resolve_effective();
293            }
294        }
295        if self.intermediate_size == 0 {
296            if let Some(ffn) = self.ffn_dim {
297                self.intermediate_size = ffn;
298            }
299        }
300        self
301    }
302
303    /// Detect the canonical [`ModelArchitecture`] family.
304    ///
305    /// # Errors
306    /// Returns [`ModelError::UnsupportedArchitecture`] for unrecognized `model_type`.
307    pub fn detect_architecture(&self) -> Result<ModelArchitecture> {
308        let mt = self.model_type.to_ascii_lowercase();
309        match mt.as_str() {
310            "llama" => Ok(ModelArchitecture::Llama),
311            "qwen2" => Ok(ModelArchitecture::Qwen2),
312            "qwen3" => Ok(ModelArchitecture::Qwen3),
313            "phi3" => Ok(ModelArchitecture::Phi3),
314            "gemma2" => Ok(ModelArchitecture::Gemma2),
315            "gemma3" | "gemma3_text" => Ok(ModelArchitecture::Gemma3),
316            "gemma4" | "gemma4_text" => Ok(ModelArchitecture::Gemma4),
317            "opt" => Ok(ModelArchitecture::Opt),
318            _ => Err(ModelError::UnsupportedArchitecture(format!(
319                "unsupported model_type `{}`; supported: opt, llama, phi3, qwen2, qwen3, gemma2, gemma3, gemma4",
320                self.model_type
321            ))),
322        }
323    }
324
325    /// Effective hidden activation identifier (`hidden_activation` when set, otherwise `hidden_act`).
326    #[must_use]
327    pub fn effective_hidden_act(&self) -> &str {
328        match self.hidden_activation.as_deref() {
329            Some(act) if !act.is_empty() => act,
330            _ => &self.hidden_act,
331        }
332    }
333
334    /// Effective per-head dimension.
335    #[must_use]
336    pub fn effective_head_dim(&self) -> usize {
337        self.head_dim.unwrap_or_else(|| {
338            self.hidden_size
339                .checked_div(self.num_attention_heads)
340                .unwrap_or(0)
341        })
342    }
343
344    /// Effective global (`full_attention`) head dimension for Gemma 4.
345    #[must_use]
346    pub fn effective_global_head_dim(&self) -> usize {
347        self.global_head_dim
348            .unwrap_or_else(|| self.effective_head_dim())
349    }
350
351    /// Effective number of key/value heads.
352    #[must_use]
353    pub fn effective_kv_heads(&self) -> usize {
354        self.num_key_value_heads.unwrap_or(self.num_attention_heads)
355    }
356
357    /// Effective number of key/value heads for global (`full_attention`) layers.
358    #[must_use]
359    pub fn effective_global_kv_heads(&self) -> usize {
360        self.num_global_key_value_heads
361            .unwrap_or_else(|| self.effective_kv_heads())
362    }
363
364    /// Per-layer attention type schedule (`"sliding_attention"` vs. `"full_attention"`).
365    #[must_use]
366    pub fn effective_layer_types(&self) -> Vec<String> {
367        if let Some(ref lt) = self.layer_types {
368            if lt.len() == self.num_hidden_layers {
369                return lt.clone();
370            }
371        }
372        if let Some(pattern) = self.sliding_window_pattern {
373            if pattern > 0 {
374                return (0..self.num_hidden_layers)
375                    .map(|i| {
376                        if (i + 1).is_multiple_of(pattern) {
377                            "full_attention".to_string()
378                        } else {
379                            "sliding_attention".to_string()
380                        }
381                    })
382                    .collect();
383            }
384        }
385        vec!["full_attention".to_string(); self.num_hidden_layers]
386    }
387
388    /// Resolve Gemma 4 RoPE hyperparameters for a specific attention layer type.
389    #[must_use]
390    pub fn gemma4_rope_params(&self, layer_type: &str) -> ResolvedRopeParams {
391        use serde_json::Value;
392
393        let default_theta = if layer_type == "sliding_attention" {
394            self.rope_local_base_freq.unwrap_or(self.rope_theta)
395        } else {
396            self.rope_theta
397        };
398        let default_partial = self.partial_rotary_factor.unwrap_or(1.0);
399
400        let entry = self
401            .rope_parameters
402            .as_ref()
403            .and_then(Value::as_object)
404            .and_then(|map| {
405                map.get(layer_type).and_then(Value::as_object).or_else(|| {
406                    if map.contains_key("rope_type")
407                        || map.contains_key("rope_theta")
408                        || map.contains_key("partial_rotary_factor")
409                    {
410                        Some(map)
411                    } else {
412                        None
413                    }
414                })
415            });
416
417        match entry {
418            Some(obj) => ResolvedRopeParams {
419                rope_theta: obj
420                    .get("rope_theta")
421                    .and_then(Value::as_f64)
422                    .unwrap_or(default_theta),
423                partial_rotary_factor: obj
424                    .get("partial_rotary_factor")
425                    .and_then(Value::as_f64)
426                    .unwrap_or(default_partial),
427                scale_factor: obj.get("factor").and_then(Value::as_f64).unwrap_or(1.0),
428                is_proportional: obj
429                    .get("rope_type")
430                    .and_then(Value::as_str)
431                    .is_some_and(|s| s.eq_ignore_ascii_case("proportional")),
432            },
433            None => ResolvedRopeParams {
434                rope_theta: default_theta,
435                partial_rotary_factor: default_partial,
436                scale_factor: 1.0,
437                is_proportional: false,
438            },
439        }
440    }
441
442    /// Effective sequence length matching `eval.py` fallback (`n_ctx` -> `max_position_embeddings` -> `2048`).
443    #[must_use]
444    pub fn effective_seqlen(&self) -> usize {
445        self.n_ctx.unwrap_or(if self.max_position_embeddings > 0 {
446            self.max_position_embeddings
447        } else {
448            2048
449        })
450    }
451
452    /// Effective intermediate size (`ffn_dim` for OPT or `intermediate_size` otherwise).
453    #[must_use]
454    pub fn effective_intermediate_size(&self) -> usize {
455        if self.intermediate_size > 0 {
456            self.intermediate_size
457        } else {
458            self.ffn_dim.unwrap_or(0)
459        }
460    }
461
462    /// Build a compact synthetic Llama config suitable for unit tests and quick prototyping.
463    #[must_use]
464    pub fn tiny_llama(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
465        Self {
466            model_type: "llama".to_string(),
467            architectures: vec!["LlamaForCausalLM".to_string()],
468            vocab_size,
469            hidden_size,
470            intermediate_size: hidden_size * 2,
471            num_hidden_layers: layers,
472            num_attention_heads: 4,
473            num_key_value_heads: Some(2),
474            max_position_embeddings: 256,
475            rms_norm_eps: 1e-5,
476            rope_theta: 10000.0,
477            ..Default::default()
478        }
479    }
480
481    /// Build a compact synthetic Qwen2 config.
482    #[must_use]
483    pub fn tiny_qwen2(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
484        Self {
485            model_type: "qwen2".to_string(),
486            architectures: vec!["Qwen2ForCausalLM".to_string()],
487            vocab_size,
488            hidden_size,
489            intermediate_size: hidden_size * 2,
490            num_hidden_layers: layers,
491            num_attention_heads: 4,
492            num_key_value_heads: Some(2),
493            max_position_embeddings: 256,
494            rms_norm_eps: 1e-6,
495            rope_theta: 1000000.0,
496            attention_bias: true,
497            ..Default::default()
498        }
499    }
500
501    /// Build a compact synthetic Qwen3 config.
502    #[must_use]
503    pub fn tiny_qwen3(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
504        Self {
505            model_type: "qwen3".to_string(),
506            architectures: vec!["Qwen3ForCausalLM".to_string()],
507            vocab_size,
508            hidden_size,
509            intermediate_size: hidden_size * 2,
510            num_hidden_layers: layers,
511            num_attention_heads: 4,
512            num_key_value_heads: Some(2),
513            head_dim: Some(hidden_size / 4),
514            max_position_embeddings: 256,
515            rms_norm_eps: 1e-6,
516            rope_theta: 1000000.0,
517            ..Default::default()
518        }
519    }
520
521    /// Build a compact synthetic Phi-3/Phi-4 config.
522    #[must_use]
523    pub fn tiny_phi3(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
524        Self {
525            model_type: "phi3".to_string(),
526            architectures: vec!["Phi3ForCausalLM".to_string()],
527            vocab_size,
528            hidden_size,
529            intermediate_size: hidden_size * 2,
530            num_hidden_layers: layers,
531            num_attention_heads: 4,
532            num_key_value_heads: Some(2),
533            max_position_embeddings: 256,
534            rms_norm_eps: 1e-5,
535            rope_theta: 10000.0,
536            attention_bias: false,
537            ..Default::default()
538        }
539    }
540
541    /// Build a compact synthetic Gemma 2 config.
542    #[must_use]
543    pub fn tiny_gemma2(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
544        Self {
545            model_type: "gemma2".to_string(),
546            architectures: vec!["Gemma2ForCausalLM".to_string()],
547            vocab_size,
548            hidden_size,
549            intermediate_size: hidden_size * 2,
550            num_hidden_layers: layers,
551            num_attention_heads: 4,
552            num_key_value_heads: Some(2),
553            head_dim: Some(hidden_size / 4),
554            max_position_embeddings: 256,
555            rms_norm_eps: 1e-6,
556            rope_theta: 10000.0,
557            tie_word_embeddings: true,
558            hidden_act: "gelu_pytorch_tanh".to_string(),
559            attn_logit_softcapping: Some(50.0),
560            final_logit_softcapping: Some(30.0),
561            query_pre_attn_scalar: Some((hidden_size / 4) as f64),
562            sliding_window: Some(64),
563            ..Default::default()
564        }
565    }
566
567    /// Build a compact synthetic Gemma 3 config.
568    #[must_use]
569    pub fn tiny_gemma3(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
570        Self {
571            model_type: "gemma3_text".to_string(),
572            architectures: vec!["Gemma3ForCausalLM".to_string()],
573            vocab_size,
574            hidden_size,
575            intermediate_size: hidden_size * 2,
576            num_hidden_layers: layers,
577            num_attention_heads: 4,
578            num_key_value_heads: Some(2),
579            head_dim: Some(hidden_size / 4),
580            max_position_embeddings: 256,
581            rms_norm_eps: 1e-6,
582            rope_theta: 1_000_000.0,
583            rope_local_base_freq: Some(10000.0),
584            tie_word_embeddings: true,
585            hidden_act: "gelu_pytorch_tanh".to_string(),
586            query_pre_attn_scalar: Some((hidden_size / 4) as f64),
587            sliding_window: Some(64),
588            sliding_window_pattern: Some(2),
589            ..Default::default()
590        }
591    }
592
593    /// Build a compact synthetic Dense Gemma 4 config (`sliding_attention` + `full_attention`, p-RoPE, `k_eq_v`).
594    #[must_use]
595    pub fn tiny_gemma4_dense(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
596        let layer_types = (0..layers)
597            .map(|i| {
598                if (i + 1).is_multiple_of(2) {
599                    "full_attention".to_string()
600                } else {
601                    "sliding_attention".to_string()
602                }
603            })
604            .collect();
605        let head_dim = (hidden_size / 4).max(2);
606        let global_head_dim = (hidden_size / 2).max(8);
607        Self {
608            model_type: "gemma4_text".to_string(),
609            architectures: vec!["Gemma4ForCausalLM".to_string()],
610            vocab_size,
611            hidden_size,
612            intermediate_size: hidden_size * 2,
613            num_hidden_layers: layers,
614            num_attention_heads: 4,
615            num_key_value_heads: Some(2),
616            head_dim: Some(head_dim),
617            global_head_dim: Some(global_head_dim),
618            num_global_key_value_heads: Some(1),
619            attention_k_eq_v: Some(true),
620            max_position_embeddings: 256,
621            rms_norm_eps: 1e-6,
622            rope_theta: 1_000_000.0,
623            rope_local_base_freq: Some(10_000.0),
624            tie_word_embeddings: true,
625            hidden_act: "gelu_pytorch_tanh".to_string(),
626            hidden_activation: Some("gelu_pytorch_tanh".to_string()),
627            final_logit_softcapping: Some(30.0),
628            sliding_window: Some(64),
629            layer_types: Some(layer_types),
630            rope_parameters: Some(serde_json::json!({
631                "sliding_attention": {
632                    "rope_type": "default",
633                    "rope_theta": 10000.0
634                },
635                "full_attention": {
636                    "rope_type": "proportional",
637                    "partial_rotary_factor": 0.25,
638                    "rope_theta": 1_000_000.0
639                }
640            })),
641            ..Default::default()
642        }
643    }
644
645    /// Build a compact synthetic MoE Gemma 4 config (dual shared MLP + top-k routed experts).
646    #[must_use]
647    pub fn tiny_gemma4_moe(
648        vocab_size: usize,
649        hidden_size: usize,
650        layers: usize,
651        num_experts: usize,
652        top_k: usize,
653    ) -> Self {
654        let mut cfg = Self::tiny_gemma4_dense(vocab_size, hidden_size, layers);
655        cfg.enable_moe_block = Some(true);
656        cfg.num_experts = Some(num_experts);
657        cfg.top_k_experts = Some(top_k);
658        cfg.moe_intermediate_size = Some(hidden_size);
659        cfg
660    }
661
662    /// Build a compact synthetic OPT config.
663    #[must_use]
664    pub fn tiny_opt(vocab_size: usize, hidden_size: usize, layers: usize) -> Self {
665        Self {
666            model_type: "opt".to_string(),
667            architectures: vec!["OPTForCausalLM".to_string()],
668            vocab_size,
669            hidden_size,
670            intermediate_size: hidden_size * 2,
671            ffn_dim: Some(hidden_size * 2),
672            word_embed_proj_dim: Some(hidden_size),
673            num_hidden_layers: layers,
674            num_attention_heads: 4,
675            max_position_embeddings: 256,
676            tie_word_embeddings: true,
677            enable_bias: Some(true),
678            do_layer_norm_before: Some(true),
679            layer_norm_elementwise_affine: Some(true),
680            ..Default::default()
681        }
682    }
683}