feat: get_means_by_benchmark — disaggregated projection the librarian asked for

sql_benchmarks/agent_orchestrator.py

@@ -126,8 +126,8 @@
 ANALYZER = SpecialistRole(
     name="analyzer",
     tool_names=[
-        "get_experiment_summary", "get_means_by_partition", "get_scaling_factor",
-        "get_replication_stability", "compare_engines",
+        "get_experiment_summary", "get_means_by_partition", "get_means_by_benchmark",
+        "get_scaling_factor", "get_replication_stability", "compare_engines",
         "compare_engines_by_partition", "get_experiment_result",
     ],
     system_prompt=_ANALYZER_PROMPT,
@@ -181,7 +181,8 @@
         "search_published_capsules", "list_lab_docs", "get_lab_doc",
         "list_categories",
         "get_experiment_summary", "get_means_by_partition",
-        "get_scaling_factor", "get_replication_stability",
+        "get_means_by_benchmark", "get_scaling_factor",
+        "get_replication_stability",
     ],
     system_prompt=_LIBRARIAN_PROMPT,
     max_turns=12,

sql_benchmarks/agent_tools.py

@@ -187,6 +187,18 @@
             }
         }
     },
+    {
+        "type": "function",
+        "function": {
+            "name": "get_means_by_benchmark",
+            "description": "Mean/std per (benchmark, partition, engine) — the DISAGGREGATED view. Use when the benchmarks within a suite are themselves the objects of comparison (e.g. different NULL-handling approaches, different selectivity levels); the pooled projections average across them.",
+            "parameters": {
+                "type": "object",
+                "properties": {"experiment_id": {"type": "string"}},
+                "required": ["experiment_id"]
+            }
+        }
+    },
     {
         "type": "function",
         "function": {
@@ -271,6 +283,9 @@ def execute_tool(name: str, args: dict) -> str:
         elif name == "get_scaling_factor":
             res = httpx.get(f"{API_BASE}/v1/results/{args['experiment_id']}/projections/scaling", timeout=30)
             return json.dumps(res.json(), indent=2)
+        elif name == "get_means_by_benchmark":
+            res = httpx.get(f"{API_BASE}/v1/results/{args['experiment_id']}/projections/benchmarks", timeout=30)
+            return json.dumps(res.json(), indent=2)
         elif name == "get_replication_stability":
             res = httpx.get(f"{API_BASE}/v1/results/{args['experiment_id']}/projections/stability", timeout=30)
             return json.dumps(res.json(), indent=2)

sql_benchmarks/api/logic/projections.py

@@ -214,3 +214,33 @@ def get_experiment_summary(exp_id: str, reader) -> dict:
         "narrative": "\n".join(lines),
         "provenance": _provenance(fragments),
     }
+
+
+def get_means_by_benchmark(exp_id: str, reader) -> dict:
+    """Mean duration per (benchmark, partition, engine) — the disaggregated
+    view. Suites that encode the OBJECT of comparison as the benchmark
+    (null_logic: 3VL vs hand-decomposed 2VL vs IS NOT DISTINCT FROM;
+    selectivity: q_0_1_percent vs q_10_percent) are invisible to the
+    partition-pooled projections — a reference-librarian run refused a
+    paper-prep question for exactly this reason (2026-07-06). Additive:
+    the pooled projections keep their shape."""
+    fragments = reader.get_fragments(exp_id)
+    grouped: dict = {}
+    for f in fragments:
+        raw = f.metrics.durations_raw or [f.metrics.duration_seconds]
+        key = (f.meta.asset, f.meta.partition, f.meta.engine)
+        grouped.setdefault(key, []).extend(raw)
+    benchmarks: dict = {}
+    for (asset, part, eng), raws in sorted(grouped.items()):
+        m = mean(raws)
+        s = stdev(raws) if len(raws) >= 2 else 0.0
+        benchmarks.setdefault(asset, {}).setdefault(part, {})[eng] = {
+            "mean_duration_seconds": round(m, 6),
+            "std_duration_seconds": round(s, 6),
+            "sample_count": len(raws),
+        }
+    return {
+        "experiment_id": exp_id,
+        "benchmarks": benchmarks,
+        "provenance": _provenance(fragments),
+    }

sql_benchmarks/api/routers/results.py

@@ -6,6 +6,7 @@
 from ..logic.comparator import compare_experiment, compare_experiment_by_partition
 from ..logic.projections import (
     get_experiment_summary,
+    get_means_by_benchmark,
     get_means_by_partition,
     get_replication_stability,
     get_scaling_factor,
@@ -122,3 +123,13 @@ def projection_summary(exp_id: str):
     if not _reader.results_exist(exp_id):
         raise HTTPException(status_code=404, detail=f"Experiment '{exp_id}' not found")
     return get_experiment_summary(exp_id, _reader)
+
+
+@router.get("/{exp_id}/projections/benchmarks")
+def projection_benchmarks(exp_id: str):
+    """Mean/std per (benchmark, partition, engine) — the disaggregated view
+    for suites whose benchmarks ARE the comparison (null_logic approaches,
+    selectivity levels)."""
+    if not _reader.results_exist(exp_id):
+        raise HTTPException(status_code=404, detail=f"Experiment '{exp_id}' not found")
+    return get_means_by_benchmark(exp_id, _reader)

sql_benchmarks_tests/test_orchestrator.py

@@ -69,7 +69,7 @@ def test_no_specialist_sees_the_full_tool_inventory():
     for role in (CONFIG_BUILDER, ANALYZER):
         subset = set(role.tool_names)
         assert subset < full  # strict subset
-        assert len(subset) <= 7  # kept small
+        assert len(subset) <= 9  # kept small (analyzer gained the per-benchmark projection)
 
 
 # ---------------------------------------------------------------------------

sql_benchmarks_tests/test_projections.py

@@ -284,3 +284,16 @@ def test_cli_legacy_positional_invocation_still_dispatches_to_run(monkeypatch):
             cli.main()
     assert exc_info.value.code == 0
     assert calls == [("queue", True)]
+
+
+def test_get_means_by_benchmark_disaggregates(three_scale_lab):
+    """Pooled projections average across benchmarks; this keeps them
+    apart — the view null_logic/selectivity suites need."""
+    from sql_benchmarks.api.logic.projections import get_means_by_benchmark
+    result = get_means_by_benchmark(EXP, ResultReader())
+    assert "duckdb_analytical" in result["benchmarks"]
+    small = result["benchmarks"]["duckdb_analytical"]["small"]["duckdb"]
+    assert small["sample_count"] == 3          # durations_raw, not pooled means
+    assert abs(small["mean_duration_seconds"] - 0.010) < 1e-6
+    assert small["std_duration_seconds"] > 0
+    assert result["provenance"]["fragment_count"] == 6