Skip to main content

littlebit_train/
eval.rs

1//! Perplexity and zero-shot multiple-choice evaluation (`eval.py` parity).
2
3use std::fs::File;
4use std::io::{BufRead, BufReader};
5use std::path::Path;
6
7use candle_core::{DType, Device, IndexOp, Tensor, D};
8use serde::{Deserialize, Serialize};
9use tokenizers::Tokenizer;
10
11use littlebit_models::{CausalLm, HiddenStatesMode};
12
13use crate::error::{Result, TrainError};
14
15/// Perplexity measurement result for one dataset split.
16#[derive(Debug, Clone, Serialize, Deserialize)]
17pub struct PplResult {
18    /// Dataset name.
19    pub dataset: String,
20    /// Exponentiated average negative log-likelihood (`exp(total_nll / tokens)`).
21    pub perplexity: f64,
22    /// Mean negative log-likelihood per predicted token.
23    pub avg_nll: f64,
24    /// Number of full `seqlen` windows evaluated.
25    pub num_samples: usize,
26    /// Evaluation context window length.
27    pub seqlen: usize,
28}
29
30/// Evaluate autoregressive perplexity on a flat token stream matching `eval.py`.
31///
32/// # Errors
33/// Returns an error if `seqlen < 2`, `token_ids.len() < seqlen`, or model forward passes fail.
34pub fn evaluate_perplexity(
35    model: &CausalLm,
36    token_ids: &[u32],
37    seqlen: usize,
38    dataset_name: &str,
39    device: &Device,
40) -> Result<PplResult> {
41    if seqlen < 2 {
42        return Err(TrainError::InvalidArgument(format!(
43            "seqlen must be >= 2 for perplexity evaluation, got {seqlen}"
44        )));
45    }
46    let nsamples = token_ids.len() / seqlen;
47    if nsamples == 0 {
48        return Err(TrainError::InvalidArgument(format!(
49            "not enough tokens ({}) for seqlen {seqlen} on `{dataset_name}`",
50            token_ids.len()
51        )));
52    }
53
54    let mut total_nll = 0.0_f64;
55    let pred_tokens_per_sample = seqlen - 1;
56
57    for i in 0..nsamples {
58        let chunk = &token_ids[i * seqlen..(i + 1) * seqlen];
59        let batch = Tensor::from_slice(chunk, (1, seqlen), device)?;
60        let out = model.forward(&batch, None, HiddenStatesMode::None)?;
61        let vocab_size = out.logits.dims()[2];
62
63        let shift_logits = out
64            .logits
65            .i((0, ..pred_tokens_per_sample, ..))?
66            .reshape((pred_tokens_per_sample, vocab_size))?
67            .to_dtype(DType::F32)?;
68        let shift_labels = Tensor::from_slice(&chunk[1..], pred_tokens_per_sample, device)?;
69        let loss = candle_nn::loss::cross_entropy(&shift_logits, &shift_labels)?
70            .to_scalar::<f32>()? as f64;
71        total_nll += loss * (pred_tokens_per_sample as f64);
72    }
73
74    let total_tokens = (nsamples * pred_tokens_per_sample) as f64;
75    let avg_nll = total_nll / total_tokens;
76    let perplexity = avg_nll.exp();
77
78    Ok(PplResult {
79        dataset: dataset_name.to_string(),
80        perplexity,
81        avg_nll,
82        num_samples: nsamples,
83        seqlen,
84    })
85}
86
87/// One multiple-choice zero-shot evaluation example (`boolq`, `piqa`, `hellaswag`, `arc`, etc.).
88#[derive(Debug, Clone, Serialize, Deserialize)]
89pub struct MultipleChoiceExample {
90    /// Prompt or conditioning context string.
91    pub context: String,
92    /// Candidate continuation strings.
93    pub choices: Vec<String>,
94    /// 0-based index of the correct choice in `choices`.
95    pub gold_index: usize,
96}
97
98/// Aggregated accuracy metrics for a zero-shot multiple-choice benchmark.
99#[derive(Debug, Clone, Serialize, Deserialize)]
100pub struct ZeroShotTaskResult {
101    /// Benchmark task name.
102    pub task: String,
103    /// Number of evaluated examples.
104    pub num_examples: usize,
105    /// Unnormalized log-likelihood accuracy (`acc`).
106    pub acc: f64,
107    /// Byte-length-normalized log-likelihood accuracy (`acc_norm`).
108    pub acc_norm: f64,
109}
110
111/// Load multiple-choice examples from a JSONL file.
112///
113/// Accepts records formatted as `{"context": "...", "choices": ["...", ...], "gold_index": 0}`.
114///
115/// # Errors
116/// Returns an error if the file cannot be read or contains invalid JSON lines.
117pub fn load_multiple_choice_jsonl(path: &Path) -> Result<Vec<MultipleChoiceExample>> {
118    let file = File::open(path)?;
119    let mut examples = Vec::new();
120    for line in BufReader::new(file).lines() {
121        let line = line?;
122        let trimmed = line.trim();
123        if trimmed.is_empty() {
124            continue;
125        }
126        let ex: MultipleChoiceExample = serde_json::from_str(trimmed)?;
127        examples.push(ex);
128    }
129    Ok(examples)
130}
131
132/// Evaluate conditional log-likelihood accuracy across multiple-choice examples,
133/// matching EleutherAI `lm-eval` scoring (`acc` and byte-normalized `acc_norm`).
134///
135/// # Errors
136/// Returns an error if `examples` is empty, tokenization fails, or model forward fails.
137pub fn evaluate_multiple_choice(
138    model: &CausalLm,
139    tokenizer: &Tokenizer,
140    task_name: &str,
141    examples: &[MultipleChoiceExample],
142    limit: Option<usize>,
143    device: &Device,
144) -> Result<ZeroShotTaskResult> {
145    let slice = match limit {
146        Some(n) if n > 0 && n < examples.len() => &examples[..n],
147        _ => examples,
148    };
149    if slice.is_empty() {
150        return Err(TrainError::InvalidArgument(format!(
151            "no examples provided for zero-shot task `{task_name}`"
152        )));
153    }
154
155    let mut correct_raw = 0usize;
156    let mut correct_norm = 0usize;
157
158    for ex in slice {
159        if ex.choices.is_empty() || ex.gold_index >= ex.choices.len() {
160            return Err(TrainError::InvalidArgument(format!(
161                "invalid multiple-choice example in task `{task_name}`"
162            )));
163        }
164
165        let ctx_ids = if ex.context.is_empty() {
166            Vec::new()
167        } else {
168            tokenizer
169                .encode(ex.context.as_str(), false)
170                .map_err(|err| TrainError::Tokenizer(format!("encode failed: {err}")))?
171                .get_ids()
172                .to_vec()
173        };
174
175        let mut best_raw_idx = 0usize;
176        let mut best_raw_score = f64::NEG_INFINITY;
177        let mut best_norm_idx = 0usize;
178        let mut best_norm_score = f64::NEG_INFINITY;
179
180        for (c_idx, choice) in ex.choices.iter().enumerate() {
181            let full_text = if ex.context.is_empty() {
182                choice.clone()
183            } else {
184                format!("{}{choice}", ex.context)
185            };
186            let full_ids = tokenizer
187                .encode(full_text.as_str(), false)
188                .map_err(|err| TrainError::Tokenizer(format!("encode failed: {err}")))?
189                .get_ids()
190                .to_vec();
191
192            if full_ids.len() < 2 {
193                continue;
194            }
195
196            let input_t = Tensor::from_slice(&full_ids, (1, full_ids.len()), device)?;
197            let logits = model
198                .forward(&input_t, None, HiddenStatesMode::None)?
199                .logits
200                .get(0)?
201                .to_dtype(DType::F32)?;
202            let log_probs = candle_nn::ops::log_softmax(&logits, D::Minus1)?;
203
204            let start_pos = ctx_ids.len().max(1).min(full_ids.len() - 1);
205            let mut sum_ll = 0.0_f64;
206            for (pos, &tok) in full_ids.iter().enumerate().skip(start_pos) {
207                let lp = log_probs
208                    .get(pos - 1)?
209                    .get(tok as usize)?
210                    .to_scalar::<f32>()? as f64;
211                sum_ll += lp;
212            }
213            let norm_ll = sum_ll / (choice.len().max(1) as f64);
214
215            if sum_ll > best_raw_score {
216                best_raw_score = sum_ll;
217                best_raw_idx = c_idx;
218            }
219            if norm_ll > best_norm_score {
220                best_norm_score = norm_ll;
221                best_norm_idx = c_idx;
222            }
223        }
224
225        if best_raw_idx == ex.gold_index {
226            correct_raw += 1;
227        }
228        if best_norm_idx == ex.gold_index {
229            correct_norm += 1;
230        }
231    }
232
233    let n = slice.len() as f64;
234    Ok(ZeroShotTaskResult {
235        task: task_name.to_string(),
236        num_examples: slice.len(),
237        acc: (correct_raw as f64) / n,
238        acc_norm: (correct_norm as f64) / n,
239    })
240}
241
242#[cfg(test)]
243mod tests {
244    use super::*;
245    use candle_core::Tensor;
246    use littlebit_models::ModelConfig;
247    use std::collections::HashMap;
248
249    fn test_tokenizer() -> Tokenizer {
250        let json = serde_json::json!({
251            "version": "1.0",
252            "model": {
253                "type": "WordLevel",
254                "vocab": {
255                    "<unk>": 0,
256                    "the": 1,
257                    "cat": 2,
258                    "sat": 3,
259                    "dog": 4,
260                    "flew": 5,
261                    "on": 6,
262                    "mat": 7
263                },
264                "unk_token": "<unk>"
265            },
266            "pre_tokenizer": { "type": "Whitespace" }
267        });
268        Tokenizer::from_bytes(json.to_string().as_bytes()).expect("test tokenizer")
269    }
270
271    #[test]
272    fn perplexity_on_zero_weights_equals_vocab_size() -> Result<()> {
273        let dev = Device::Cpu;
274        let cfg = ModelConfig::tiny_llama(32, 16, 1);
275        // Zero lm_head weights produce identically 0 logits -> uniform softmax 1/32 -> PPL = 32.0.
276        let mut tensors = HashMap::new();
277        tensors.insert(
278            "model.embed_tokens.weight".to_string(),
279            Tensor::zeros((32, 16), DType::F32, &dev)?,
280        );
281        tensors.insert(
282            "model.layers.0.input_layernorm.weight".to_string(),
283            Tensor::ones(16, DType::F32, &dev)?,
284        );
285        tensors.insert(
286            "model.layers.0.post_attention_layernorm.weight".to_string(),
287            Tensor::ones(16, DType::F32, &dev)?,
288        );
289        for name in ["self_attn.q_proj", "self_attn.o_proj"] {
290            tensors.insert(
291                format!("model.layers.0.{name}.weight"),
292                Tensor::zeros((16, 16), DType::F32, &dev)?,
293            );
294        }
295        for name in ["self_attn.k_proj", "self_attn.v_proj"] {
296            tensors.insert(
297                format!("model.layers.0.{name}.weight"),
298                Tensor::zeros((8, 16), DType::F32, &dev)?,
299            );
300        }
301        tensors.insert(
302            "model.layers.0.mlp.gate_proj.weight".to_string(),
303            Tensor::zeros((32, 16), DType::F32, &dev)?,
304        );
305        tensors.insert(
306            "model.layers.0.mlp.up_proj.weight".to_string(),
307            Tensor::zeros((32, 16), DType::F32, &dev)?,
308        );
309        tensors.insert(
310            "model.layers.0.mlp.down_proj.weight".to_string(),
311            Tensor::zeros((16, 32), DType::F32, &dev)?,
312        );
313        tensors.insert(
314            "model.norm.weight".to_string(),
315            Tensor::ones(16, DType::F32, &dev)?,
316        );
317        tensors.insert(
318            "lm_head.weight".to_string(),
319            Tensor::zeros((32, 16), DType::F32, &dev)?,
320        );
321
322        let model = CausalLm::from_tensors(
323            cfg,
324            &tensors,
325            None,
326            littlebit_core::WeightState::Raw,
327            littlebit_core::InferenceMode::Unpacked,
328            DType::F32,
329            &dev,
330        )?;
331
332        let tokens = vec![1u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12];
333        let ppl = evaluate_perplexity(&model, &tokens, 6, "uniform_test", &dev)?;
334        assert_eq!(ppl.num_samples, 2);
335        assert!(
336            (ppl.perplexity - 32.0).abs() < 1e-3,
337            "expected uniform PPL=32.0, got {}",
338            ppl.perplexity
339        );
340        Ok(())
341    }
342
343    #[test]
344    fn multiple_choice_scores_candidate_continuations() -> Result<()> {
345        let dev = Device::Cpu;
346        let tok = test_tokenizer();
347        let cfg = ModelConfig::tiny_llama(16, 16, 1);
348        let model = CausalLm::new_random(cfg, DType::F32, &dev, 7)?;
349        let examples = vec![MultipleChoiceExample {
350            context: "the cat".to_string(),
351            choices: vec![" sat on mat".to_string(), " flew".to_string()],
352            gold_index: 0,
353        }];
354        let res = evaluate_multiple_choice(&model, &tok, "synthetic", &examples, None, &dev)?;
355        assert_eq!(res.num_examples, 1);
356        assert!(res.acc == 0.0 || res.acc == 1.0);
357        assert!(res.acc_norm == 0.0 || res.acc_norm == 1.0);
358        Ok(())
359    }
360}