1use std::fs::File;
4use std::io::{BufRead, BufReader};
5use std::path::Path;
6
7use candle_core::{DType, Device, IndexOp, Tensor, D};
8use serde::{Deserialize, Serialize};
9use tokenizers::Tokenizer;
10
11use littlebit_models::{CausalLm, HiddenStatesMode};
12
13use crate::error::{Result, TrainError};
14
15#[derive(Debug, Clone, Serialize, Deserialize)]
17pub struct PplResult {
18 pub dataset: String,
20 pub perplexity: f64,
22 pub avg_nll: f64,
24 pub num_samples: usize,
26 pub seqlen: usize,
28}
29
30pub fn evaluate_perplexity(
35 model: &CausalLm,
36 token_ids: &[u32],
37 seqlen: usize,
38 dataset_name: &str,
39 device: &Device,
40) -> Result<PplResult> {
41 if seqlen < 2 {
42 return Err(TrainError::InvalidArgument(format!(
43 "seqlen must be >= 2 for perplexity evaluation, got {seqlen}"
44 )));
45 }
46 let nsamples = token_ids.len() / seqlen;
47 if nsamples == 0 {
48 return Err(TrainError::InvalidArgument(format!(
49 "not enough tokens ({}) for seqlen {seqlen} on `{dataset_name}`",
50 token_ids.len()
51 )));
52 }
53
54 let mut total_nll = 0.0_f64;
55 let pred_tokens_per_sample = seqlen - 1;
56
57 for i in 0..nsamples {
58 let chunk = &token_ids[i * seqlen..(i + 1) * seqlen];
59 let batch = Tensor::from_slice(chunk, (1, seqlen), device)?;
60 let out = model.forward(&batch, None, HiddenStatesMode::None)?;
61 let vocab_size = out.logits.dims()[2];
62
63 let shift_logits = out
64 .logits
65 .i((0, ..pred_tokens_per_sample, ..))?
66 .reshape((pred_tokens_per_sample, vocab_size))?
67 .to_dtype(DType::F32)?;
68 let shift_labels = Tensor::from_slice(&chunk[1..], pred_tokens_per_sample, device)?;
69 let loss = candle_nn::loss::cross_entropy(&shift_logits, &shift_labels)?
70 .to_scalar::<f32>()? as f64;
71 total_nll += loss * (pred_tokens_per_sample as f64);
72 }
73
74 let total_tokens = (nsamples * pred_tokens_per_sample) as f64;
75 let avg_nll = total_nll / total_tokens;
76 let perplexity = avg_nll.exp();
77
78 Ok(PplResult {
79 dataset: dataset_name.to_string(),
80 perplexity,
81 avg_nll,
82 num_samples: nsamples,
83 seqlen,
84 })
85}
86
87#[derive(Debug, Clone, Serialize, Deserialize)]
89pub struct MultipleChoiceExample {
90 pub context: String,
92 pub choices: Vec<String>,
94 pub gold_index: usize,
96}
97
98#[derive(Debug, Clone, Serialize, Deserialize)]
100pub struct ZeroShotTaskResult {
101 pub task: String,
103 pub num_examples: usize,
105 pub acc: f64,
107 pub acc_norm: f64,
109}
110
111pub fn load_multiple_choice_jsonl(path: &Path) -> Result<Vec<MultipleChoiceExample>> {
118 let file = File::open(path)?;
119 let mut examples = Vec::new();
120 for line in BufReader::new(file).lines() {
121 let line = line?;
122 let trimmed = line.trim();
123 if trimmed.is_empty() {
124 continue;
125 }
126 let ex: MultipleChoiceExample = serde_json::from_str(trimmed)?;
127 examples.push(ex);
128 }
129 Ok(examples)
130}
131
132pub fn evaluate_multiple_choice(
138 model: &CausalLm,
139 tokenizer: &Tokenizer,
140 task_name: &str,
141 examples: &[MultipleChoiceExample],
142 limit: Option<usize>,
143 device: &Device,
144) -> Result<ZeroShotTaskResult> {
145 let slice = match limit {
146 Some(n) if n > 0 && n < examples.len() => &examples[..n],
147 _ => examples,
148 };
149 if slice.is_empty() {
150 return Err(TrainError::InvalidArgument(format!(
151 "no examples provided for zero-shot task `{task_name}`"
152 )));
153 }
154
155 let mut correct_raw = 0usize;
156 let mut correct_norm = 0usize;
157
158 for ex in slice {
159 if ex.choices.is_empty() || ex.gold_index >= ex.choices.len() {
160 return Err(TrainError::InvalidArgument(format!(
161 "invalid multiple-choice example in task `{task_name}`"
162 )));
163 }
164
165 let ctx_ids = if ex.context.is_empty() {
166 Vec::new()
167 } else {
168 tokenizer
169 .encode(ex.context.as_str(), false)
170 .map_err(|err| TrainError::Tokenizer(format!("encode failed: {err}")))?
171 .get_ids()
172 .to_vec()
173 };
174
175 let mut best_raw_idx = 0usize;
176 let mut best_raw_score = f64::NEG_INFINITY;
177 let mut best_norm_idx = 0usize;
178 let mut best_norm_score = f64::NEG_INFINITY;
179
180 for (c_idx, choice) in ex.choices.iter().enumerate() {
181 let full_text = if ex.context.is_empty() {
182 choice.clone()
183 } else {
184 format!("{}{choice}", ex.context)
185 };
186 let full_ids = tokenizer
187 .encode(full_text.as_str(), false)
188 .map_err(|err| TrainError::Tokenizer(format!("encode failed: {err}")))?
189 .get_ids()
190 .to_vec();
191
192 if full_ids.len() < 2 {
193 continue;
194 }
195
196 let input_t = Tensor::from_slice(&full_ids, (1, full_ids.len()), device)?;
197 let logits = model
198 .forward(&input_t, None, HiddenStatesMode::None)?
199 .logits
200 .get(0)?
201 .to_dtype(DType::F32)?;
202 let log_probs = candle_nn::ops::log_softmax(&logits, D::Minus1)?;
203
204 let start_pos = ctx_ids.len().max(1).min(full_ids.len() - 1);
205 let mut sum_ll = 0.0_f64;
206 for (pos, &tok) in full_ids.iter().enumerate().skip(start_pos) {
207 let lp = log_probs
208 .get(pos - 1)?
209 .get(tok as usize)?
210 .to_scalar::<f32>()? as f64;
211 sum_ll += lp;
212 }
213 let norm_ll = sum_ll / (choice.len().max(1) as f64);
214
215 if sum_ll > best_raw_score {
216 best_raw_score = sum_ll;
217 best_raw_idx = c_idx;
218 }
219 if norm_ll > best_norm_score {
220 best_norm_score = norm_ll;
221 best_norm_idx = c_idx;
222 }
223 }
224
225 if best_raw_idx == ex.gold_index {
226 correct_raw += 1;
227 }
228 if best_norm_idx == ex.gold_index {
229 correct_norm += 1;
230 }
231 }
232
233 let n = slice.len() as f64;
234 Ok(ZeroShotTaskResult {
235 task: task_name.to_string(),
236 num_examples: slice.len(),
237 acc: (correct_raw as f64) / n,
238 acc_norm: (correct_norm as f64) / n,
239 })
240}
241
242#[cfg(test)]
243mod tests {
244 use super::*;
245 use candle_core::Tensor;
246 use littlebit_models::ModelConfig;
247 use std::collections::HashMap;
248
249 fn test_tokenizer() -> Tokenizer {
250 let json = serde_json::json!({
251 "version": "1.0",
252 "model": {
253 "type": "WordLevel",
254 "vocab": {
255 "<unk>": 0,
256 "the": 1,
257 "cat": 2,
258 "sat": 3,
259 "dog": 4,
260 "flew": 5,
261 "on": 6,
262 "mat": 7
263 },
264 "unk_token": "<unk>"
265 },
266 "pre_tokenizer": { "type": "Whitespace" }
267 });
268 Tokenizer::from_bytes(json.to_string().as_bytes()).expect("test tokenizer")
269 }
270
271 #[test]
272 fn perplexity_on_zero_weights_equals_vocab_size() -> Result<()> {
273 let dev = Device::Cpu;
274 let cfg = ModelConfig::tiny_llama(32, 16, 1);
275 let mut tensors = HashMap::new();
277 tensors.insert(
278 "model.embed_tokens.weight".to_string(),
279 Tensor::zeros((32, 16), DType::F32, &dev)?,
280 );
281 tensors.insert(
282 "model.layers.0.input_layernorm.weight".to_string(),
283 Tensor::ones(16, DType::F32, &dev)?,
284 );
285 tensors.insert(
286 "model.layers.0.post_attention_layernorm.weight".to_string(),
287 Tensor::ones(16, DType::F32, &dev)?,
288 );
289 for name in ["self_attn.q_proj", "self_attn.o_proj"] {
290 tensors.insert(
291 format!("model.layers.0.{name}.weight"),
292 Tensor::zeros((16, 16), DType::F32, &dev)?,
293 );
294 }
295 for name in ["self_attn.k_proj", "self_attn.v_proj"] {
296 tensors.insert(
297 format!("model.layers.0.{name}.weight"),
298 Tensor::zeros((8, 16), DType::F32, &dev)?,
299 );
300 }
301 tensors.insert(
302 "model.layers.0.mlp.gate_proj.weight".to_string(),
303 Tensor::zeros((32, 16), DType::F32, &dev)?,
304 );
305 tensors.insert(
306 "model.layers.0.mlp.up_proj.weight".to_string(),
307 Tensor::zeros((32, 16), DType::F32, &dev)?,
308 );
309 tensors.insert(
310 "model.layers.0.mlp.down_proj.weight".to_string(),
311 Tensor::zeros((16, 32), DType::F32, &dev)?,
312 );
313 tensors.insert(
314 "model.norm.weight".to_string(),
315 Tensor::ones(16, DType::F32, &dev)?,
316 );
317 tensors.insert(
318 "lm_head.weight".to_string(),
319 Tensor::zeros((32, 16), DType::F32, &dev)?,
320 );
321
322 let model = CausalLm::from_tensors(
323 cfg,
324 &tensors,
325 None,
326 littlebit_core::WeightState::Raw,
327 littlebit_core::InferenceMode::Unpacked,
328 DType::F32,
329 &dev,
330 )?;
331
332 let tokens = vec![1u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12];
333 let ppl = evaluate_perplexity(&model, &tokens, 6, "uniform_test", &dev)?;
334 assert_eq!(ppl.num_samples, 2);
335 assert!(
336 (ppl.perplexity - 32.0).abs() < 1e-3,
337 "expected uniform PPL=32.0, got {}",
338 ppl.perplexity
339 );
340 Ok(())
341 }
342
343 #[test]
344 fn multiple_choice_scores_candidate_continuations() -> Result<()> {
345 let dev = Device::Cpu;
346 let tok = test_tokenizer();
347 let cfg = ModelConfig::tiny_llama(16, 16, 1);
348 let model = CausalLm::new_random(cfg, DType::F32, &dev, 7)?;
349 let examples = vec![MultipleChoiceExample {
350 context: "the cat".to_string(),
351 choices: vec![" sat on mat".to_string(), " flew".to_string()],
352 gold_index: 0,
353 }];
354 let res = evaluate_multiple_choice(&model, &tok, "synthetic", &examples, None, &dev)?;
355 assert_eq!(res.num_examples, 1);
356 assert!(res.acc == 0.0 || res.acc == 1.0);
357 assert!(res.acc_norm == 0.0 || res.acc_norm == 1.0);
358 Ok(())
359 }
360}