merge: evaluation harness with 38 fixtures and aggregate reporting

- Extended EvalRunner with 5 assertion sections (evidence, prompt,
  connections, subject, breaking) and symbols.toml injection
- Added run_sync() for integration test usage
- 38 fixtures covering all 11 commit types, AST features, edge cases
- Per-type accuracy breakdown and overall score reporting
- 348 tests passing with eval feature

* feat/eval-harness:
  feat(eval): complete eval harness with 38 fixtures and aggregate reporting
  feat(eval): add 10 fixtures and integration test suite
  feat(eval): extend EvalRunner with assertion sections and symbols.toml loading

Author: Sephyi

src/eval.rs

@@ -0,0 +1,218 @@
+// SPDX-FileCopyrightText: 2026 Sephyi <me@sephy.io>
+//
+// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
+
+//! Integration tests for the evaluation harness.
+//!
+//! Runs all fixtures through the deterministic (no-LLM) pipeline and
+//! asserts type inference, evidence flags, prompt content, connections,
+//! and breaking change detection.
+
+#![cfg(feature = "eval")]
+
+use std::path::PathBuf;
+
+use commitbee::eval::EvalRunner;
+
+fn fixtures_dir() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/eval")
+}
+
+/// Run all fixtures and assert every one passes.
+#[test]
+fn all_fixtures_pass() {
+    let runner = EvalRunner::new(fixtures_dir(), None);
+    let results = runner.run_sync().expect("eval runner should not error");
+
+    assert!(!results.is_empty(), "should discover at least one fixture");
+
+    let mut failures = Vec::new();
+    for result in &results {
+        if !result.passed() {
+            let mut detail = format!("FIXTURE FAILED: {}\n", result.fixture_name);
+            if !result.type_passed {
+                detail.push_str(&format!(
+                    "  Type: expected={}, actual={}\n",
+                    result.expected_type, result.actual_type
+                ));
+            }
+            if !result.scope_passed {
+                detail.push_str(&format!(
+                    "  Scope: expected={:?}, actual={:?}\n",
+                    result.expected_scope, result.actual_scope
+                ));
+            }
+            if !result.prompt_assembled {
+                detail.push_str("  Prompt: failed to assemble\n");
+            }
+            for failure in &result.assertion_failures {
+                detail.push_str(&format!("  {}\n", failure));
+            }
+            if let Some(ref err) = result.error {
+                detail.push_str(&format!("  Error: {}\n", err));
+            }
+            failures.push(detail);
+        }
+    }
+
+    if !failures.is_empty() {
+        panic!(
+            "{} of {} fixtures failed:\n\n{}",
+            failures.len(),
+            results.len(),
+            failures.join("\n")
+        );
+    }
+}
+
+/// Each fixture category runs independently.
+#[test]
+fn type_inference_fixtures() {
+    let runner = EvalRunner::new(fixtures_dir(), None);
+    let results = runner.run_sync().expect("eval runner should not error");
+
+    for result in &results {
+        assert!(
+            result.type_passed,
+            "Type mismatch in {}: expected={}, actual={}",
+            result.fixture_name, result.expected_type, result.actual_type
+        );
+    }
+}
+
+#[test]
+fn evidence_flag_fixtures() {
+    let runner = EvalRunner::new(fixtures_dir(), None);
+    let results = runner.run_sync().expect("eval runner should not error");
+
+    for result in &results {
+        let evidence_failures: Vec<_> = result
+            .assertion_failures
+            .iter()
+            .filter(|f| f.category == "evidence")
+            .collect();
+
+        assert!(
+            evidence_failures.is_empty(),
+            "Evidence failures in {}: {:?}",
+            result.fixture_name,
+            evidence_failures
+                .iter()
+                .map(|f| &f.message)
+                .collect::<Vec<_>>()
+        );
+    }
+}
+
+#[test]
+fn prompt_content_fixtures() {
+    let runner = EvalRunner::new(fixtures_dir(), None);
+    let results = runner.run_sync().expect("eval runner should not error");
+
+    for result in &results {
+        assert!(
+            result.prompt_assembled,
+            "Prompt assembly failed for {}",
+            result.fixture_name
+        );
+
+        let prompt_failures: Vec<_> = result
+            .assertion_failures
+            .iter()
+            .filter(|f| f.category == "prompt")
+            .collect();
+
+        assert!(
+            prompt_failures.is_empty(),
+            "Prompt content failures in {}: {:?}",
+            result.fixture_name,
+            prompt_failures
+                .iter()
+                .map(|f| &f.message)
+                .collect::<Vec<_>>()
+        );
+    }
+}
+
+#[test]
+fn connection_detection_fixtures() {
+    let runner = EvalRunner::new(fixtures_dir(), None);
+    let results = runner.run_sync().expect("eval runner should not error");
+
+    for result in &results {
+        let conn_failures: Vec<_> = result
+            .assertion_failures
+            .iter()
+            .filter(|f| f.category == "connections")
+            .collect();
+
+        assert!(
+            conn_failures.is_empty(),
+            "Connection failures in {}: {:?}",
+            result.fixture_name,
+            conn_failures.iter().map(|f| &f.message).collect::<Vec<_>>()
+        );
+    }
+}
+
+#[test]
+fn breaking_change_fixtures() {
+    let runner = EvalRunner::new(fixtures_dir(), None);
+    let results = runner.run_sync().expect("eval runner should not error");
+
+    for result in &results {
+        let breaking_failures: Vec<_> = result
+            .assertion_failures
+            .iter()
+            .filter(|f| f.category == "breaking")
+            .collect();
+
+        assert!(
+            breaking_failures.is_empty(),
+            "Breaking change failures in {}: {:?}",
+            result.fixture_name,
+            breaking_failures
+                .iter()
+                .map(|f| &f.message)
+                .collect::<Vec<_>>()
+        );
+    }
+}
+
+/// Verify specific fixture count to catch accidental fixture deletion.
+#[test]
+fn fixture_count() {
+    let runner = EvalRunner::new(fixtures_dir(), None);
+    let results = runner.run_sync().expect("eval runner should not error");
+    // 12 original + 26 new = 38
+    assert!(
+        results.len() >= 38,
+        "Expected at least 38 fixtures, found {}",
+        results.len()
+    );
+}
+
+/// Print aggregate per-type accuracy report after running all fixtures.
+#[test]
+fn aggregate_summary() {
+    let runner = EvalRunner::new(fixtures_dir(), None);
+    let results = runner.run_sync().expect("eval runner should not error");
+
+    let summary = commitbee::eval::EvalSummary::from_results(&results);
+    let report = summary.format_report();
+
+    // Print the report so it's visible with --nocapture
+    eprintln!("\n{}", report);
+
+    // Verify the summary math is consistent
+    assert_eq!(
+        summary.total_passed + summary.total_failed,
+        summary.total_fixtures,
+        "passed + failed should equal total"
+    );
+    assert_eq!(
+        summary.total_fixtures,
+        results.len(),
+        "summary total should match results count"
+    );
+}

tests/eval.rs

@@ -0,0 +1,25 @@
+diff --git a/src/services/api.rs b/src/services/api.rs
+index abc1234..def5678 100644
+--- a/src/services/api.rs
++++ b/src/services/api.rs
+@@ -5,20 +5,5 @@ use crate::error::Result;
+
+ pub struct ApiClient {
+     base_url: String,
+ }
+
+-/// Send a GET request to the given path.
+-pub fn api_get(client: &ApiClient, path: &str) -> Result<Response> {
+-    let url = format!("{}/{}", client.base_url, path);
+-    reqwest::blocking::get(&url).map_err(|e| Error::Network(e.to_string()))
+-}
+-
+-/// Send a POST request with a JSON body.
+-pub fn api_post(client: &ApiClient, path: &str, body: &str) -> Result<Response> {
+-    let url = format!("{}/{}", client.base_url, path);
+-    reqwest::blocking::Client::new()
+-        .post(&url)
+-        .body(body.to_string())
+-        .send()
+-        .map_err(|e| Error::Network(e.to_string()))
+-}

tests/fixtures/eval/ast-breaking-public-removal/diff.patch

@@ -0,0 +1,20 @@
+# SPDX-FileCopyrightText: 2026 Sephyi <me@sephy.io>
+#
+# SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
+
+name = "ast-breaking-public-removal"
+description = "Removing public API without replacement should signal breaking change"
+language = "rust"
+category = "ast"
+expected_type = "refactor"
+expected_scope = "optional"
+
+[evidence]
+has_new_public_api = false
+public_api_removed_count = 2
+
+[prompt]
+must_contain = ["SYMBOLS CHANGED", "Removed"]
+
+[breaking]
+expected = true

tests/fixtures/eval/ast-breaking-public-removal/metadata.toml

@@ -0,0 +1,19 @@
+[[symbols]]
+kind = "Function"
+name = "api_get"
+file = "src/services/api.rs"
+line = 10
+end_line = 13
+is_public = true
+is_added = false
+signature = "pub fn api_get(client: &ApiClient, path: &str) -> Result<Response>"
+
+[[symbols]]
+kind = "Function"
+name = "api_post"
+file = "src/services/api.rs"
+line = 16
+end_line = 22
+is_public = true
+is_added = false
+signature = "pub fn api_post(client: &ApiClient, path: &str, body: &str) -> Result<Response>"

tests/fixtures/eval/ast-breaking-public-removal/symbols.toml

@@ -0,0 +1,39 @@
+diff --git a/src/services/validator.rs b/src/services/validator.rs
+new file mode 100644
+index 0000000..abc1234
+--- /dev/null
++++ b/src/services/validator.rs
+@@ -0,0 +1,12 @@
++use crate::error::Result;
++
++/// Validate user input before processing.
++pub fn validate_input(input: &str) -> Result<()> {
++    if input.is_empty() {
++        return Err(crate::error::Error::Config("empty input".into()));
++    }
++    if input.len() > 1024 {
++        return Err(crate::error::Error::Config("input too long".into()));
++    }
++    Ok(())
++}
+diff --git a/src/services/handler.rs b/src/services/handler.rs
+new file mode 100644
+index 0000000..def5678
+--- /dev/null
++++ b/src/services/handler.rs
+@@ -0,0 +1,15 @@
++use crate::error::Result;
++use super::validator::validate_input;
++
++pub struct RequestHandler;
++
++impl RequestHandler {
++    pub fn handle(&self, request: &str) -> Result<String> {
++        // Validate first, then process
++        validate_input(request)?;
++
++        // Process the validated input
++        let result = request.to_uppercase();
++        Ok(result)
++    }
++}

tests/fixtures/eval/ast-cross-file-connection/diff.patch

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2026 Sephyi <me@sephy.io>
+#
+# SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
+
+name = "ast-cross-file-connection"
+description = "Cross-file call should produce CONNECTIONS section in prompt"
+language = "rust"
+category = "feat"
+expected_type = "feat"
+expected_scope = "optional"
+
+[prompt]
+must_contain = ["CONNECTIONS", "calls validate_input"]
+
+[connections]
+min_count = 1
+must_contain = ["calls validate_input"]

tests/fixtures/eval/ast-cross-file-connection/metadata.toml

@@ -0,0 +1,29 @@
+[[symbols]]
+kind = "Function"
+name = "validate_input"
+file = "src/services/validator.rs"
+line = 4
+end_line = 11
+is_public = true
+is_added = true
+signature = "pub fn validate_input(input: &str) -> Result<()>"
+
+[[symbols]]
+kind = "Struct"
+name = "RequestHandler"
+file = "src/services/handler.rs"
+line = 4
+end_line = 4
+is_public = true
+is_added = true
+signature = "pub struct RequestHandler"
+
+[[symbols]]
+kind = "Function"
+name = "handle"
+file = "src/services/handler.rs"
+line = 7
+end_line = 14
+is_public = true
+is_added = true
+signature = "pub fn handle(&self, request: &str) -> Result<String>"

tests/fixtures/eval/ast-cross-file-connection/symbols.toml

@@ -0,0 +1,24 @@
+diff --git a/src/services/validator.rs b/src/services/validator.rs
+index abc1234..def5678 100644
+--- a/src/services/validator.rs
++++ b/src/services/validator.rs
+@@ -5,8 +5,8 @@ use crate::error::Result;
+
+ impl Validator {
+-    pub fn check(&self,  input: &str) -> bool {
+-        !input.is_empty()  &&  input.len() < 1024
++    pub fn check(&self, input: &str) -> bool {
++        !input.is_empty() && input.len() < 1024
+     }
+ }
+
+@@ -15,0 +16,10 @@
++/// Validate a batch of inputs, returning the first error.
++pub fn validate_batch(inputs: &[&str]) -> Result<()> {
++    for input in inputs {
++        if input.is_empty() {
++            return Err(crate::error::Error::Config("empty input in batch".into()));
++        }
++    }
++    Ok(())
++}