From 750cb38a907cd73fde62fe14c19b04005eb76f6a Mon Sep 17 00:00:00 2001 From: rctruta <17259677+rctruta@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:46:39 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20get=5Fmeans=5Fby=5Fbenchmark=20?= =?UTF-8?q?=E2=80=94=20disaggregated=20projection=20the=20librarian=20aske?= =?UTF-8?q?d=20for?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live reference-librarian run REFUSED a paper-prep question about the NULL-identity capsules because the pooled projections average across benchmarks — and in those suites the benchmarks ARE the comparison (3VL vs hand-decomposed 2VL vs IS NOT DISTINCT FROM; selectivity levels). Honest refusal -> instrument gap surfaced -> fixed. Same lesson the grader learned (asset-pooling false FAILs), now closed on the projection side too. Additive: new projection keyed (benchmark, partition, engine) with mean/std/sample_count + provenance; REST endpoint /projections/benchmarks; tool wired to LIBRARIAN + ANALYZER. Pooled projections keep their shape — no consumer breakage. --- sql_benchmarks/agent_orchestrator.py | 7 +++--- sql_benchmarks/agent_tools.py | 15 ++++++++++++ sql_benchmarks/api/logic/projections.py | 30 +++++++++++++++++++++++ sql_benchmarks/api/routers/results.py | 11 +++++++++ sql_benchmarks_tests/test_orchestrator.py | 2 +- sql_benchmarks_tests/test_projections.py | 13 ++++++++++ 6 files changed, 74 insertions(+), 4 deletions(-) diff --git a/sql_benchmarks/agent_orchestrator.py b/sql_benchmarks/agent_orchestrator.py index 4192c7e..755f92b 100644 --- a/sql_benchmarks/agent_orchestrator.py +++ b/sql_benchmarks/agent_orchestrator.py @@ -126,8 +126,8 @@ ANALYZER = SpecialistRole( name="analyzer", tool_names=[ - "get_experiment_summary", "get_means_by_partition", "get_scaling_factor", - "get_replication_stability", "compare_engines", + "get_experiment_summary", "get_means_by_partition", "get_means_by_benchmark", + "get_scaling_factor", "get_replication_stability", "compare_engines", "compare_engines_by_partition", "get_experiment_result", ], system_prompt=_ANALYZER_PROMPT, @@ -181,7 +181,8 @@ "search_published_capsules", "list_lab_docs", "get_lab_doc", "list_categories", "get_experiment_summary", "get_means_by_partition", - "get_scaling_factor", "get_replication_stability", + "get_means_by_benchmark", "get_scaling_factor", + "get_replication_stability", ], system_prompt=_LIBRARIAN_PROMPT, max_turns=12, diff --git a/sql_benchmarks/agent_tools.py b/sql_benchmarks/agent_tools.py index 6f1f86b..fa2e016 100644 --- a/sql_benchmarks/agent_tools.py +++ b/sql_benchmarks/agent_tools.py @@ -187,6 +187,18 @@ } } }, + { + "type": "function", + "function": { + "name": "get_means_by_benchmark", + "description": "Mean/std per (benchmark, partition, engine) — the DISAGGREGATED view. Use when the benchmarks within a suite are themselves the objects of comparison (e.g. different NULL-handling approaches, different selectivity levels); the pooled projections average across them.", + "parameters": { + "type": "object", + "properties": {"experiment_id": {"type": "string"}}, + "required": ["experiment_id"] + } + } + }, { "type": "function", "function": { @@ -271,6 +283,9 @@ def execute_tool(name: str, args: dict) -> str: elif name == "get_scaling_factor": res = httpx.get(f"{API_BASE}/v1/results/{args['experiment_id']}/projections/scaling", timeout=30) return json.dumps(res.json(), indent=2) + elif name == "get_means_by_benchmark": + res = httpx.get(f"{API_BASE}/v1/results/{args['experiment_id']}/projections/benchmarks", timeout=30) + return json.dumps(res.json(), indent=2) elif name == "get_replication_stability": res = httpx.get(f"{API_BASE}/v1/results/{args['experiment_id']}/projections/stability", timeout=30) return json.dumps(res.json(), indent=2) diff --git a/sql_benchmarks/api/logic/projections.py b/sql_benchmarks/api/logic/projections.py index f9489c3..a83a806 100644 --- a/sql_benchmarks/api/logic/projections.py +++ b/sql_benchmarks/api/logic/projections.py @@ -214,3 +214,33 @@ def get_experiment_summary(exp_id: str, reader) -> dict: "narrative": "\n".join(lines), "provenance": _provenance(fragments), } + + +def get_means_by_benchmark(exp_id: str, reader) -> dict: + """Mean duration per (benchmark, partition, engine) — the disaggregated + view. Suites that encode the OBJECT of comparison as the benchmark + (null_logic: 3VL vs hand-decomposed 2VL vs IS NOT DISTINCT FROM; + selectivity: q_0_1_percent vs q_10_percent) are invisible to the + partition-pooled projections — a reference-librarian run refused a + paper-prep question for exactly this reason (2026-07-06). Additive: + the pooled projections keep their shape.""" + fragments = reader.get_fragments(exp_id) + grouped: dict = {} + for f in fragments: + raw = f.metrics.durations_raw or [f.metrics.duration_seconds] + key = (f.meta.asset, f.meta.partition, f.meta.engine) + grouped.setdefault(key, []).extend(raw) + benchmarks: dict = {} + for (asset, part, eng), raws in sorted(grouped.items()): + m = mean(raws) + s = stdev(raws) if len(raws) >= 2 else 0.0 + benchmarks.setdefault(asset, {}).setdefault(part, {})[eng] = { + "mean_duration_seconds": round(m, 6), + "std_duration_seconds": round(s, 6), + "sample_count": len(raws), + } + return { + "experiment_id": exp_id, + "benchmarks": benchmarks, + "provenance": _provenance(fragments), + } diff --git a/sql_benchmarks/api/routers/results.py b/sql_benchmarks/api/routers/results.py index d938f81..7d1f0f6 100644 --- a/sql_benchmarks/api/routers/results.py +++ b/sql_benchmarks/api/routers/results.py @@ -6,6 +6,7 @@ from ..logic.comparator import compare_experiment, compare_experiment_by_partition from ..logic.projections import ( get_experiment_summary, + get_means_by_benchmark, get_means_by_partition, get_replication_stability, get_scaling_factor, @@ -122,3 +123,13 @@ def projection_summary(exp_id: str): if not _reader.results_exist(exp_id): raise HTTPException(status_code=404, detail=f"Experiment '{exp_id}' not found") return get_experiment_summary(exp_id, _reader) + + +@router.get("/{exp_id}/projections/benchmarks") +def projection_benchmarks(exp_id: str): + """Mean/std per (benchmark, partition, engine) — the disaggregated view + for suites whose benchmarks ARE the comparison (null_logic approaches, + selectivity levels).""" + if not _reader.results_exist(exp_id): + raise HTTPException(status_code=404, detail=f"Experiment '{exp_id}' not found") + return get_means_by_benchmark(exp_id, _reader) diff --git a/sql_benchmarks_tests/test_orchestrator.py b/sql_benchmarks_tests/test_orchestrator.py index d29683a..a51c7eb 100644 --- a/sql_benchmarks_tests/test_orchestrator.py +++ b/sql_benchmarks_tests/test_orchestrator.py @@ -69,7 +69,7 @@ def test_no_specialist_sees_the_full_tool_inventory(): for role in (CONFIG_BUILDER, ANALYZER): subset = set(role.tool_names) assert subset < full # strict subset - assert len(subset) <= 7 # kept small + assert len(subset) <= 9 # kept small (analyzer gained the per-benchmark projection) # --------------------------------------------------------------------------- diff --git a/sql_benchmarks_tests/test_projections.py b/sql_benchmarks_tests/test_projections.py index 070554c..fc4d4cc 100644 --- a/sql_benchmarks_tests/test_projections.py +++ b/sql_benchmarks_tests/test_projections.py @@ -284,3 +284,16 @@ def test_cli_legacy_positional_invocation_still_dispatches_to_run(monkeypatch): cli.main() assert exc_info.value.code == 0 assert calls == [("queue", True)] + + +def test_get_means_by_benchmark_disaggregates(three_scale_lab): + """Pooled projections average across benchmarks; this keeps them + apart — the view null_logic/selectivity suites need.""" + from sql_benchmarks.api.logic.projections import get_means_by_benchmark + result = get_means_by_benchmark(EXP, ResultReader()) + assert "duckdb_analytical" in result["benchmarks"] + small = result["benchmarks"]["duckdb_analytical"]["small"]["duckdb"] + assert small["sample_count"] == 3 # durations_raw, not pooled means + assert abs(small["mean_duration_seconds"] - 0.010) < 1e-6 + assert small["std_duration_seconds"] > 0 + assert result["provenance"]["fragment_count"] == 6