feat(bench): implement LOCOMO, FRAMES, and GAIA dataset loaders

Closes #2836, #2837, #2839

Add shared Scenario/Evaluator traits and metric functions:
- token_f1: whitespace-token overlap F1 score
- exact_match: case-insensitive, punctuation-stripped equality
- gaia_normalized_exact_match: strips articles, punctuation, collapses whitespace

Loaders and evaluators:
- LocomoLoader: parses lmlab/locomo JSON array; one Scenario per QA pair;
  LocomoEvaluator uses token F1 with threshold 0.5
- FramesLoader: parses google/frames-benchmark JSONL; stores reasoning_types
  in metadata; FramesEvaluator uses exact match
- GaiaLoader: parses gaia-benchmark/GAIA JSONL with optional --level filter;
  GaiaEvaluator uses GAIA-normalized exact match

52 unit tests across all new modules; all 7788 workspace tests pass.

Author: bug-ops

crates/zeph-bench/src/lib.rs

@@ -6,11 +6,17 @@ pub mod cli;
 pub mod dataset;
 pub mod deterministic;
 pub mod error;
+pub mod loaders;
 pub mod results;
+pub mod scenario;
 
 pub use channel::BenchmarkChannel;
 pub use cli::BenchCommand;
 pub use dataset::{DatasetFormat, DatasetMeta, DatasetRegistry};
 pub use deterministic::apply_deterministic_overrides;
 pub use error::BenchError;
 pub use results::{Aggregate, BenchRun, ResultWriter, RunStatus, ScenarioResult};
+pub use scenario::{
+    DatasetLoader, EvalResult, Evaluator, Scenario, exact_match, gaia_normalized_exact_match,
+    token_f1,
+};

crates/zeph-bench/src/loaders/frames.rs

@@ -0,0 +1,160 @@
+// SPDX-FileCopyrightText: 2026 Andrei G <bug-ops>
+// SPDX-License-Identifier: MIT OR Apache-2.0
+
+use std::{
+    io::{BufRead as _, BufReader},
+    path::Path,
+};
+
+use serde::Deserialize;
+
+use crate::{
+    error::BenchError,
+    scenario::{DatasetLoader, EvalResult, Evaluator, Scenario, exact_match},
+};
+
+#[derive(Debug, Deserialize)]
+struct FramesRecord {
+    #[serde(rename = "Prompt")]
+    prompt: String,
+    #[serde(rename = "Answer")]
+    answer: String,
+    reasoning_types: Option<serde_json::Value>,
+}
+
+/// Loads FRAMES benchmark scenarios from a JSONL file.
+///
+/// Schema (google/frames-benchmark on HuggingFace):
+/// ```json
+/// {"Prompt": "...", "Answer": "...", "reasoning_types": [...], "wiki_links": [...]}
+/// ```
+///
+/// Each line becomes one [`Scenario`] with id `"frames_{line_number}"`.
+/// `reasoning_types` is stored in `metadata`.
+#[derive(Debug)]
+pub struct FramesLoader;
+
+impl DatasetLoader for FramesLoader {
+    fn name(&self) -> &'static str {
+        "frames"
+    }
+
+    /// # Errors
+    ///
+    /// Returns [`BenchError::Io`] when the file cannot be read and
+    /// [`BenchError::InvalidFormat`] when a JSONL line cannot be parsed.
+    fn load(&self, path: &Path) -> Result<Vec<Scenario>, BenchError> {
+        let file = std::fs::File::open(path)?;
+        let reader = BufReader::new(file);
+
+        let mut scenarios = Vec::new();
+        for (line_number, line) in reader.lines().enumerate() {
+            let line = line?;
+            let trimmed = line.trim();
+            if trimmed.is_empty() {
+                continue;
+            }
+            let record: FramesRecord = serde_json::from_str(trimmed)
+                .map_err(|e| BenchError::InvalidFormat(format!("line {line_number}: {e}")))?;
+
+            let metadata = record.reasoning_types.unwrap_or(serde_json::Value::Null);
+
+            scenarios.push(Scenario {
+                id: format!("frames_{line_number}"),
+                prompt: record.prompt,
+                expected: record.answer,
+                metadata,
+            });
+        }
+        Ok(scenarios)
+    }
+}
+
+/// Evaluates FRAMES responses using exact match.
+#[derive(Debug)]
+pub struct FramesEvaluator;
+
+impl Evaluator for FramesEvaluator {
+    fn evaluate(&self, scenario: &Scenario, agent_response: &str) -> EvalResult {
+        let passed = exact_match(agent_response, &scenario.expected);
+        EvalResult {
+            scenario_id: scenario.id.clone(),
+            score: if passed { 1.0 } else { 0.0 },
+            passed,
+            details: format!("exact_match={}", if passed { "true" } else { "false" }),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const FIXTURE: &str = r#"{"Prompt": "What is 2+2?", "Answer": "4", "reasoning_types": ["math"], "wiki_links": []}
+{"Prompt": "Capital of France?", "Answer": "Paris", "reasoning_types": ["geography"]}
+"#;
+
+    fn load_from_str(jsonl: &str) -> Vec<Scenario> {
+        let dir = tempfile::tempdir().unwrap();
+        let path = dir.path().join("frames.jsonl");
+        std::fs::write(&path, jsonl).unwrap();
+        FramesLoader.load(&path).unwrap()
+    }
+
+    #[test]
+    fn load_parses_scenario_count() {
+        let scenarios = load_from_str(FIXTURE);
+        assert_eq!(scenarios.len(), 2);
+    }
+
+    #[test]
+    fn load_builds_correct_ids() {
+        let scenarios = load_from_str(FIXTURE);
+        assert_eq!(scenarios[0].id, "frames_0");
+        assert_eq!(scenarios[1].id, "frames_1");
+    }
+
+    #[test]
+    fn load_maps_prompt_and_expected() {
+        let scenarios = load_from_str(FIXTURE);
+        assert_eq!(scenarios[0].prompt, "What is 2+2?");
+        assert_eq!(scenarios[0].expected, "4");
+    }
+
+    #[test]
+    fn load_stores_reasoning_types_in_metadata() {
+        let scenarios = load_from_str(FIXTURE);
+        assert!(scenarios[0].metadata.is_array());
+    }
+
+    #[test]
+    fn evaluator_exact_match_passes() {
+        let scenarios = load_from_str(FIXTURE);
+        let result = FramesEvaluator.evaluate(&scenarios[0], "4");
+        assert!(result.passed);
+        assert!((result.score - 1.0).abs() < f64::EPSILON);
+    }
+
+    #[test]
+    fn evaluator_wrong_answer_fails() {
+        let scenarios = load_from_str(FIXTURE);
+        let result = FramesEvaluator.evaluate(&scenarios[0], "5");
+        assert!(!result.passed);
+        assert!(result.score < f64::EPSILON);
+    }
+
+    #[test]
+    fn evaluator_case_insensitive_match() {
+        let scenarios = load_from_str(FIXTURE);
+        let result = FramesEvaluator.evaluate(&scenarios[1], "paris");
+        assert!(result.passed);
+    }
+
+    #[test]
+    fn load_invalid_jsonl_returns_error() {
+        let dir = tempfile::tempdir().unwrap();
+        let path = dir.path().join("bad.jsonl");
+        std::fs::write(&path, "not json\n").unwrap();
+        assert!(FramesLoader.load(&path).is_err());
+    }
+}

crates/zeph-bench/src/loaders/gaia.rs

@@ -0,0 +1,215 @@
+// SPDX-FileCopyrightText: 2026 Andrei G <bug-ops>
+// SPDX-License-Identifier: MIT OR Apache-2.0
+
+use std::{
+    io::{BufRead as _, BufReader},
+    path::Path,
+};
+
+use serde::Deserialize;
+
+use crate::{
+    error::BenchError,
+    scenario::{DatasetLoader, EvalResult, Evaluator, Scenario, gaia_normalized_exact_match},
+};
+
+#[derive(Debug, Deserialize)]
+struct GaiaRecord {
+    task_id: String,
+    #[serde(rename = "Question")]
+    question: String,
+    #[serde(rename = "Level")]
+    level: u8,
+    #[serde(rename = "Final answer")]
+    final_answer: String,
+    #[serde(rename = "Annotator Metadata")]
+    annotator_metadata: Option<serde_json::Value>,
+}
+
+/// Loads GAIA benchmark scenarios from a JSONL file.
+///
+/// Schema (gaia-benchmark/GAIA on HuggingFace):
+/// ```json
+/// {"task_id": "...", "Question": "...", "Level": 1, "Final answer": "...", "Annotator Metadata": {...}}
+/// ```
+///
+/// When `level` is `Some(n)`, only scenarios of that level are loaded.
+#[derive(Debug)]
+pub struct GaiaLoader {
+    /// Optional level filter. When `Some`, only scenarios with a matching `Level` are loaded.
+    pub level: Option<u8>,
+}
+
+impl GaiaLoader {
+    /// Create a loader that returns all levels.
+    #[must_use]
+    pub fn all_levels() -> Self {
+        Self { level: None }
+    }
+
+    /// Create a loader that filters to a specific difficulty level.
+    #[must_use]
+    pub fn with_level(level: u8) -> Self {
+        Self { level: Some(level) }
+    }
+}
+
+impl DatasetLoader for GaiaLoader {
+    fn name(&self) -> &'static str {
+        "gaia"
+    }
+
+    /// # Errors
+    ///
+    /// Returns [`BenchError::Io`] when the file cannot be read and
+    /// [`BenchError::InvalidFormat`] when a JSONL line cannot be parsed.
+    fn load(&self, path: &Path) -> Result<Vec<Scenario>, BenchError> {
+        let file = std::fs::File::open(path)?;
+        let reader = BufReader::new(file);
+
+        let mut scenarios = Vec::new();
+        for (line_number, line) in reader.lines().enumerate() {
+            let line = line?;
+            let trimmed = line.trim();
+            if trimmed.is_empty() {
+                continue;
+            }
+            let record: GaiaRecord = serde_json::from_str(trimmed)
+                .map_err(|e| BenchError::InvalidFormat(format!("line {line_number}: {e}")))?;
+
+            if let Some(filter_level) = self.level
+                && record.level != filter_level
+            {
+                continue;
+            }
+
+            let metadata = serde_json::json!({
+                "level": record.level,
+                "annotator_metadata": record.annotator_metadata,
+            });
+
+            scenarios.push(Scenario {
+                id: record.task_id,
+                prompt: record.question,
+                expected: record.final_answer,
+                metadata,
+            });
+        }
+        Ok(scenarios)
+    }
+}
+
+/// Evaluates GAIA responses using GAIA-normalized exact match.
+#[derive(Debug)]
+pub struct GaiaEvaluator;
+
+impl Evaluator for GaiaEvaluator {
+    fn evaluate(&self, scenario: &Scenario, agent_response: &str) -> EvalResult {
+        let passed = gaia_normalized_exact_match(agent_response, &scenario.expected);
+        EvalResult {
+            scenario_id: scenario.id.clone(),
+            score: if passed { 1.0 } else { 0.0 },
+            passed,
+            details: format!(
+                "gaia_normalized_exact_match={}",
+                if passed { "true" } else { "false" }
+            ),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const FIXTURE: &str = r#"{"task_id": "t1", "Question": "What year did WWII end?", "Level": 1, "Final answer": "1945", "Annotator Metadata": {"difficulty": "easy"}}
+{"task_id": "t2", "Question": "Who wrote Hamlet?", "Level": 2, "Final answer": "Shakespeare", "Annotator Metadata": null}
+{"task_id": "t3", "Question": "Capital of Japan?", "Level": 1, "Final answer": "Tokyo", "Annotator Metadata": null}
+"#;
+
+    fn load_from_str(jsonl: &str, level: Option<u8>) -> Vec<Scenario> {
+        let dir = tempfile::tempdir().unwrap();
+        let path = dir.path().join("gaia.jsonl");
+        std::fs::write(&path, jsonl).unwrap();
+        GaiaLoader { level }.load(&path).unwrap()
+    }
+
+    #[test]
+    fn load_all_levels_parses_scenario_count() {
+        let scenarios = load_from_str(FIXTURE, None);
+        assert_eq!(scenarios.len(), 3);
+    }
+
+    #[test]
+    fn load_filters_by_level() {
+        let scenarios = load_from_str(FIXTURE, Some(1));
+        assert_eq!(scenarios.len(), 2);
+        for s in &scenarios {
+            assert_eq!(s.metadata["level"], 1);
+        }
+    }
+
+    #[test]
+    fn load_maps_task_id_to_scenario_id() {
+        let scenarios = load_from_str(FIXTURE, None);
+        assert_eq!(scenarios[0].id, "t1");
+        assert_eq!(scenarios[1].id, "t2");
+    }
+
+    #[test]
+    fn load_maps_prompt_and_expected() {
+        let scenarios = load_from_str(FIXTURE, None);
+        assert_eq!(scenarios[0].prompt, "What year did WWII end?");
+        assert_eq!(scenarios[0].expected, "1945");
+    }
+
+    #[test]
+    fn load_stores_level_in_metadata() {
+        let scenarios = load_from_str(FIXTURE, None);
+        assert_eq!(scenarios[1].metadata["level"], 2);
+    }
+
+    #[test]
+    fn evaluator_normalized_match_passes() {
+        let scenarios = load_from_str(FIXTURE, None);
+        // "The 1945" should match "1945" after stripping article and comparing
+        let result = GaiaEvaluator.evaluate(&scenarios[0], "1945");
+        assert!(result.passed);
+    }
+
+    #[test]
+    fn evaluator_wrong_answer_fails() {
+        let scenarios = load_from_str(FIXTURE, None);
+        let result = GaiaEvaluator.evaluate(&scenarios[0], "1944");
+        assert!(!result.passed);
+        assert!(result.score < f64::EPSILON);
+    }
+
+    #[test]
+    fn evaluator_strips_article_the() {
+        let scenarios = load_from_str(FIXTURE, None);
+        // scenario[2]: expected = "Tokyo"
+        let result = GaiaEvaluator.evaluate(&scenarios[2], "The Tokyo");
+        assert!(result.passed);
+    }
+
+    #[test]
+    fn load_invalid_jsonl_returns_error() {
+        let dir = tempfile::tempdir().unwrap();
+        let path = dir.path().join("bad.jsonl");
+        std::fs::write(&path, "not json\n").unwrap();
+        assert!(GaiaLoader::all_levels().load(&path).is_err());
+    }
+
+    #[test]
+    fn all_levels_constructor() {
+        let loader = GaiaLoader::all_levels();
+        assert!(loader.level.is_none());
+    }
+
+    #[test]
+    fn with_level_constructor() {
+        let loader = GaiaLoader::with_level(2);
+        assert_eq!(loader.level, Some(2));
+    }
+}