From 98b8e744346f0a06b1e96f1c37e68ec6788e71c3 Mon Sep 17 00:00:00 2001 From: rctruta <17259677+rctruta@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:18:02 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20reference=20librarian=20=E2=80=94=20kno?= =?UTF-8?q?wledge=20stage=20+=20ask()=20probe-the-lab=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New LIBRARIAN specialist: read-only reference desk over the lab's published knowledge — search_published_capsules, list_lab_docs / get_lab_doc (README, FAQ, methodology docs, the generated experiment catalog), plus the result projections. Three closes: FINAL ANSWER (cited, from corpus) | HANDOFF: build reason=... | HANDOFF: impossible. Wired twice: 1. Librarian-first in run() — structural fix for Finding 21 (0/5 unprompted adoption: schema exposure doesn't bind, a stage does). Corpus-answerable goals terminate as "answered" before config_builder spends anything. Librarian failure falls through to build — a broken reference desk never blocks measurement. 2. ask() mode (multi_agent.py --ask) — the user-delegation use case: surface information without running the lab. Build-worthy questions return "needs_experiment"; the CALLER decides whether to spend. Also: lab_docs logic + /v1/catalog/docs endpoints (progressive disclosure: list names/titles -> fetch one, size-capped with stated truncation; no path-traversal surface — names come from the listing). Live smoke: gemini-2.5-flash in ask() answered "are there published capsules measuring Quack?" with four correct capsule ids and the real published overhead numbers (attach 2.7x@100K -> 4.9x@1M, pushdown ~1.9x). Zero execution. Tests: 7 new (read-only desk pinned — no submit/template tools; three closes parse; ask answers/routes without executing; run() answers from corpus and skips build / proceeds on handoff / proceeds on librarian failure); 6 older orchestrator tests updated for the librarian-first sequence. 38 green + 4 library tests. --- docs/decisions_log.md | 14 ++ scripts/multi_agent.py | 13 +- sql_benchmarks/agent_orchestrator.py | 149 ++++++++++++++++++- sql_benchmarks/agent_tools.py | 28 +++- sql_benchmarks/api/logic/lab_docs.py | 76 ++++++++++ sql_benchmarks/api/routers/catalog.py | 19 +++ sql_benchmarks_tests/test_orchestrator.py | 165 +++++++++++++++++++--- 7 files changed, 444 insertions(+), 20 deletions(-) create mode 100644 sql_benchmarks/api/logic/lab_docs.py diff --git a/docs/decisions_log.md b/docs/decisions_log.md index 347fa9d..89723a3 100644 --- a/docs/decisions_log.md +++ b/docs/decisions_log.md @@ -350,3 +350,17 @@ Human-and-agent parity: `sqlbench project means ` (CLI), **Live validation of the recovery story (same day).** A session teardown killed the runner mid-poll and the API mid-execution. Traces survived (append-per-event JSONL); the stale running marker was the only manual surgery — and that class is now self-healing. **Cross-refs.** TODO #12, edge-3/edge-4 studies, Finding 18/19, `[[work-before-the-work]]` (states uncovered by pre-deployment probing — exactly the deliverable of the role Ramona is naming). + +--- + +## 2026-07-06 — Reference librarian: the knowledge stage + the probe-the-lab use case + +**Decision.** New LIBRARIAN specialist (read-only reference desk: published-capsule search, lab-docs discovery incl. the generated experiment catalog, result projections) with three closes: FINAL ANSWER (cited, from corpus), `HANDOFF: build reason=…`, `HANDOFF: impossible reason=…`. Wired twice: (1) **librarian-first in `run()`** — the structural fix for Finding 21 (0/5 unprompted adoption proved schema exposure doesn't bind; a workflow stage does); a corpus-answerable goal terminates as `answered` before config_builder ever spends. (2) **`ask()` mode** (`multi_agent.py --ask`) — Ramona's user-delegation use case: *"as a user, I want to delegate the agent to surface information, not to run the lab beginning-to-end. Are there published capsules for X? Probing the lab is a use case."* ask() never executes; build-worthy questions return `needs_experiment` and the caller decides whether to spend. + +**Fork closed.** Prompt-step-only integration (adding "check the library" to config_builder's prompt). The corpus-wide words-vs-structure record (SBD-3, llama3 gates, Finding 21) says stages bind and words drift; and a stage gives the probe-the-lab use case for free, which a prompt line does not. + +**Fork opened.** The librarian is the natural consumer for everything the publish-shape answer requires (derived summaries, honest taxonomy, docs catalog) — and the meta-cycle's first hop is now mechanical: publish → librarian reads → workflows reuse. A broken librarian never blocks measurement (failure falls through to build). + +**Live validation.** gemini-2.5-flash (cheapest ladder model) in ask() mode answered "are there published capsules measuring Quack?" with four correct capsule ids and the real published overhead numbers (attach-mode 2.7×@100K → 4.9×@1M; pushdown ~1.9×), zero execution. + +**Cross-refs.** Finding 21, edge-case 6 (study 9887ce57), `[[work-before-the-work]]`, FAQ.md + docs/experiments.md (the cataloguing feature, now agent-readable). diff --git a/scripts/multi_agent.py b/scripts/multi_agent.py index 87b9307..af5e6a1 100644 --- a/scripts/multi_agent.py +++ b/scripts/multi_agent.py @@ -42,6 +42,10 @@ def main(): help="litellm model identifier") parser.add_argument("--goal", type=str, required=True, help="Natural-language goal for the orchestrator") + parser.add_argument("--ask", action="store_true", + help="Reference-desk mode: the librarian answers from " + "published capsules + docs; nothing executes. For " + "questions like 'are there published capsules for X?'") parser.add_argument("--resume", metavar="EXP_ID", help="Resume a SUSPENDED run: skip config_builder, poll " "this experiment_id, then run the analyzer. The " @@ -59,14 +63,19 @@ def main(): poll_budget_seconds=args.poll_budget_seconds) console.print(f"[dim]Orchestrator trace: {orch.trace.path}[/dim]\n") - result = orch.resume(args.resume) if args.resume else orch.run() + if args.resume: + result = orch.resume(args.resume) + elif args.ask: + result = orch.ask() + else: + result = orch.run() console.print(f"\n[bold]Outcome:[/bold] {result.outcome}") console.print(f"[dim]experiment_id: {result.experiment_id}[/dim]") for stage, sub_id in result.sub_run_ids.items(): console.print(f"[dim] {stage}: {sub_id}[/dim]") - if result.outcome == "complete" and result.analysis: + if result.outcome in ("complete", "answered", "needs_experiment") and result.analysis: console.print(Panel(Markdown(result.analysis), title="📊 Analysis", border_style="green")) elif result.outcome == "suspended": diff --git a/sql_benchmarks/agent_orchestrator.py b/sql_benchmarks/agent_orchestrator.py index acebdf9..4192c7e 100644 --- a/sql_benchmarks/agent_orchestrator.py +++ b/sql_benchmarks/agent_orchestrator.py @@ -136,6 +136,59 @@ ) +_LIBRARIAN_PROMPT = """\ +You are the reference librarian for this benchmarking lab. Users and +other agents come to you with questions. Your job: answer from the +lab's PUBLISHED knowledge whenever it can, and route to experiment- +building only when it genuinely cannot. + +IMPORTANT: "librarian" is your ROLE, not a tool. Only call tools that +appear in your tool list. + +Your reference desk: +- `search_published_capsules` (optional category filter) — sealed, + verified experiment results. Read any hit with + `get_experiment_summary`, `get_scaling_factor`, etc. — published + results are read directly, nothing needs to run. +- `list_lab_docs` / `get_lab_doc` — README, FAQ, methodology docs, and + the generated experiment catalog. + +Decide, then close with EXACTLY one of: + +1. The question is answerable from published capsules/docs — produce a + final message starting with "FINAL ANSWER:" containing the answer, + citing capsule ids and doc names you drew from. Statements like + "yes, capsules X and Y cover this; here is what they found" are + exactly your job. +2. The question truly requires a NEW measurement (no published capsule + covers it) — produce: + + HANDOFF: build reason= + +3. The question cannot be satisfied by this lab at all (names an + engine/workload the lab does not have) — produce: + + HANDOFF: impossible reason= + +Never route to build for something the corpus already answers — a +verified published result is strictly better than a re-run. +""" + + +LIBRARIAN = SpecialistRole( + name="librarian", + tool_names=[ + "search_published_capsules", "list_lab_docs", "get_lab_doc", + "list_categories", + "get_experiment_summary", "get_means_by_partition", + "get_scaling_factor", "get_replication_stability", + ], + system_prompt=_LIBRARIAN_PROMPT, + max_turns=12, + parse_output=lambda content, tool_calls: _parse_librarian(content), +) + + _HANDOFF_RE = re.compile(r"HANDOFF:\s*experiment_id=([0-9a-f]{8})", re.IGNORECASE) _REFUSAL_RE = re.compile(r"HANDOFF:\s*impossible\s+reason=(.+)", re.IGNORECASE) @@ -164,6 +217,27 @@ def _parse_analyzer_final(content: str) -> Optional[dict]: return None +_BUILD_RE = re.compile(r"HANDOFF:\s*build\s+reason=(.+)", re.IGNORECASE) + + +def _parse_librarian(content: str) -> Optional[dict]: + """Librarian closes one of three ways: an answer from the published + corpus (FINAL ANSWER:), a routing handoff to config_builder + (HANDOFF: build reason=...), or a structured refusal + (HANDOFF: impossible reason=...).""" + if not content: + return None + b = _BUILD_RE.search(content) + if b: + return {"build": b.group(1).strip()} + r = _REFUSAL_RE.search(content) + if r: + return {"impossible": r.group(1).strip()} + if "final answer" in content.lower() and len(content.strip()) > 100: + return {"analysis": content} + return None + + # --- Poller (pure Python, no LLM) ------------------------------------------- def poll_until_terminal(experiment_id: str, @@ -190,7 +264,7 @@ def poll_until_terminal(experiment_id: str, @dataclass class OrchestratorResult: - outcome: str # "complete" | "refused" | "suspended" | "config_builder_failed" | "poll_failed" | "analyzer_failed" + outcome: str # "complete" | "answered" | "refused" | "needs_experiment" | "suspended" | "config_builder_failed" | "poll_failed" | "analyzer_failed" | "librarian_failed" experiment_id: Optional[str] analysis: Optional[str] orchestrator_run_id: str @@ -222,9 +296,82 @@ def __init__(self, goal: str, model: str, poll_budget_seconds: float = 180.0): agents_md_loaded=False, max_turns=0, ) + def ask(self) -> OrchestratorResult: + """Reference-desk mode: librarian ONLY — answer from published + capsules + docs, never execute. The user-delegation use case: + 'are there published capsules for X?', 'what does the FAQ say + about Y?'. A build-worthy question comes back as outcome + 'needs_experiment' with the librarian's reason — the caller + decides whether to spend on a run.""" + sub_ids: dict = {} + lib = run_specialist(LIBRARIAN, brief=self.goal, model=self.model) + sub_ids["librarian"] = lib.sub_run_id + self.trace.delegate( + stage="librarian", sub_run_id=lib.sub_run_id, + input_summary=self.goal[:200], outcome=lib.outcome, + output_summary=(str(lib.output)[:200] if lib.output else None), + ) + if lib.outcome == "ok" and lib.output and "analysis" in lib.output: + self.trace.run_end("answered", turns_used=0) + return OrchestratorResult( + outcome="answered", experiment_id=None, + analysis=lib.output["analysis"], + orchestrator_run_id=self.trace.run_id, sub_run_ids=sub_ids, + ) + if lib.outcome == "ok" and lib.output and "impossible" in lib.output: + self.trace.run_end("refused", turns_used=0) + return OrchestratorResult( + outcome="refused", experiment_id=None, + analysis=f"REFUSED: {lib.output['impossible']}", + orchestrator_run_id=self.trace.run_id, sub_run_ids=sub_ids, + ) + if lib.outcome == "ok" and lib.output and "build" in lib.output: + self.trace.run_end("needs_experiment", turns_used=0) + return OrchestratorResult( + outcome="needs_experiment", experiment_id=None, + analysis=f"NEEDS EXPERIMENT: {lib.output['build']}", + orchestrator_run_id=self.trace.run_id, sub_run_ids=sub_ids, + ) + self.trace.run_end("librarian_failed", turns_used=0, error=lib.error) + return OrchestratorResult( + outcome="librarian_failed", experiment_id=None, analysis=None, + orchestrator_run_id=self.trace.run_id, sub_run_ids=sub_ids, + error=lib.error or "librarian produced no parseable close", + ) + def run(self) -> OrchestratorResult: sub_ids: dict = {} + # Stage 0: reference librarian — check the lab's own literature + # BEFORE building anything. This is the structural fix for + # Finding 21 (0/5 unprompted library adoption: schema-only + # exposure doesn't bind; a workflow stage does). Three exits: + # answered-from-corpus (terminal, cheap), impossible (terminal, + # honest), build (continue to config_builder). + lib = run_specialist(LIBRARIAN, brief=self.goal, model=self.model) + sub_ids["librarian"] = lib.sub_run_id + self.trace.delegate( + stage="librarian", sub_run_id=lib.sub_run_id, + input_summary=self.goal[:200], outcome=lib.outcome, + output_summary=(str(lib.output)[:200] if lib.output else None), + ) + if lib.outcome == "ok" and lib.output and "analysis" in lib.output: + self.trace.run_end("answered", turns_used=0) + return OrchestratorResult( + outcome="answered", experiment_id=None, + analysis=lib.output["analysis"], + orchestrator_run_id=self.trace.run_id, sub_run_ids=sub_ids, + ) + if lib.outcome == "ok" and lib.output and "impossible" in lib.output: + self.trace.run_end("refused", turns_used=0) + return OrchestratorResult( + outcome="refused", experiment_id=None, + analysis=f"REFUSED: {lib.output['impossible']}", + orchestrator_run_id=self.trace.run_id, sub_run_ids=sub_ids, + ) + # 'build' handoff or librarian failure -> proceed to build either + # way (a broken reference desk must not block measurement). + # Stage 1: config_builder cb: SpecialistResult = run_specialist(CONFIG_BUILDER, brief=self.goal, model=self.model) sub_ids["config_builder"] = cb.sub_run_id diff --git a/sql_benchmarks/agent_tools.py b/sql_benchmarks/agent_tools.py index 535892f..6f1f86b 100644 --- a/sql_benchmarks/agent_tools.py +++ b/sql_benchmarks/agent_tools.py @@ -51,6 +51,26 @@ } } }, + { + "type": "function", + "function": { + "name": "list_lab_docs", + "description": "List the lab's published documents (README, FAQ, methodology docs, and the generated experiment catalog) — names, titles, sizes. Small payload.", + "parameters": {"type": "object", "properties": {}} + } + }, + { + "type": "function", + "function": { + "name": "get_lab_doc", + "description": "Fetch one published document's text by name (from list_lab_docs). Size-capped; truncation is stated.", + "parameters": { + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"] + } + } + }, { "type": "function", "function": { @@ -197,7 +217,13 @@ def execute_tool(name: str, args: dict) -> str: autonomous_agent.py used to have; extracted here so specialists reuse it without importing from a script.""" try: - if name == "search_published_capsules": + if name == "list_lab_docs": + res = httpx.get(f"{API_BASE}/v1/catalog/docs", timeout=30) + return json.dumps(res.json(), indent=2) + elif name == "get_lab_doc": + res = httpx.get(f"{API_BASE}/v1/catalog/docs/{args['name']}", timeout=30) + return json.dumps(res.json(), indent=2) + elif name == "search_published_capsules": params = {} if args.get("category"): params["category"] = args["category"] diff --git a/sql_benchmarks/api/logic/lab_docs.py b/sql_benchmarks/api/logic/lab_docs.py new file mode 100644 index 0000000..15ab6a5 --- /dev/null +++ b/sql_benchmarks/api/logic/lab_docs.py @@ -0,0 +1,76 @@ +"""Discovery + retrieval over the lab's published DOCUMENTS — the other +half of the reference desk (published capsules being the first half). + +Covers README.md, FAQ.md, and docs/*.md — including the generated +experiment catalog (docs/experiments.md, produced by +scripts/tools/gen_experiment_catalog.py: the lab's cataloguing feature). + +Progressive disclosure, as everywhere: `list_lab_docs` returns names + +titles + sizes (small); `get_lab_doc` returns one document's text, +size-capped so a single call can't flood the context window. +""" +import os +from typing import List, Optional + +from ...constants import ROOT_DIR + +DOC_CHAR_CAP = 20_000 # per-fetch cap; caller is told when truncated + + +def _doc_paths() -> dict: + """name -> absolute path for every published markdown document.""" + out = {} + for top in ("README.md", "FAQ.md"): + p = os.path.join(ROOT_DIR, top) + if os.path.isfile(p): + out[top] = p + docs_dir = os.path.join(ROOT_DIR, "docs") + if os.path.isdir(docs_dir): + for fn in sorted(os.listdir(docs_dir)): + if fn.endswith(".md"): + out[f"docs/{fn}"] = os.path.join(docs_dir, fn) + return out + + +def _title_of(path: str) -> str: + """First `# ` heading, else the filename.""" + try: + with open(path, encoding="utf-8") as f: + for line in f: + if line.startswith("# "): + return line[2:].strip() + if line.strip() and not line.startswith("#"): + break + except OSError: + pass + return os.path.basename(path) + + +def list_lab_docs() -> List[dict]: + """Names + titles + sizes of every published document. Small payload.""" + out = [] + for name, path in _doc_paths().items(): + out.append({ + "name": name, + "title": _title_of(path), + "bytes": os.path.getsize(path), + }) + return out + + +def get_lab_doc(name: str) -> Optional[dict]: + """One document's text, capped at DOC_CHAR_CAP chars (truncation is + stated, never silent). None if the name isn't in the published set — + names come from list_lab_docs, no path traversal surface.""" + path = _doc_paths().get(name) + if path is None: + return None + with open(path, encoding="utf-8") as f: + text = f.read() + truncated = len(text) > DOC_CHAR_CAP + return { + "name": name, + "content": text[:DOC_CHAR_CAP], + "truncated": truncated, + "total_chars": len(text), + } diff --git a/sql_benchmarks/api/routers/catalog.py b/sql_benchmarks/api/routers/catalog.py index 627a4eb..4527953 100644 --- a/sql_benchmarks/api/routers/catalog.py +++ b/sql_benchmarks/api/routers/catalog.py @@ -89,3 +89,22 @@ def search_published(category: str | None = None): hit with the existing result projections.""" from ..logic.published_library import search_published_capsules return {"capsules": search_published_capsules(category=category)} + + +@router.get("/docs") +def list_docs(): + """Names + titles of the lab's published documents (README, FAQ, + docs/*.md incl. the generated experiment catalog).""" + from ..logic.lab_docs import list_lab_docs + return {"docs": list_lab_docs()} + + +@router.get("/docs/{name:path}") +def get_doc(name: str): + """One published document's text (size-capped, truncation stated).""" + from fastapi import HTTPException + from ..logic.lab_docs import get_lab_doc + doc = get_lab_doc(name) + if doc is None: + raise HTTPException(status_code=404, detail=f"No published doc named '{name}'") + return doc diff --git a/sql_benchmarks_tests/test_orchestrator.py b/sql_benchmarks_tests/test_orchestrator.py index 601dd18..d29683a 100644 --- a/sql_benchmarks_tests/test_orchestrator.py +++ b/sql_benchmarks_tests/test_orchestrator.py @@ -179,27 +179,37 @@ def test_specialist_hits_max_turns_if_never_signals_done(): # Orchestrator state machine — short-circuits on any stage failure # --------------------------------------------------------------------------- + +def _librarian_routes_to_build(): + """run() now begins with the reference librarian (Finding 21's + structural fix); tests of the downstream stages prepend this.""" + return SpecialistResult(role="librarian", outcome="ok", + output={"build": "not in corpus"}, + sub_run_id="lib-run", turns_used=2) + def test_orchestrator_short_circuits_on_config_builder_failure(): """If config_builder fails, poll + analyzer must not run.""" with patch.object(agent_orchestrator, "run_specialist") as mock_run_spec, \ patch.object(agent_orchestrator, "poll_until_terminal") as mock_poll: # config_builder gives up - mock_run_spec.return_value = SpecialistResult( - role="config_builder", outcome="gave_up", output=None, - sub_run_id="cb-run", turns_used=15, error="stuck", - ) + mock_run_spec.side_effect = [ + _librarian_routes_to_build(), + SpecialistResult(role="config_builder", outcome="gave_up", output=None, + sub_run_id="cb-run", turns_used=15, error="stuck"), + ] result = Orchestrator(goal="test", model="test/model").run() assert result.outcome == "config_builder_failed" assert result.experiment_id is None assert mock_poll.call_count == 0 # never got to polling - assert mock_run_spec.call_count == 1 # analyzer never invoked + assert mock_run_spec.call_count == 2 # librarian + config_builder; analyzer never invoked def test_orchestrator_short_circuits_on_poll_failure(): with patch.object(agent_orchestrator, "run_specialist") as mock_run_spec, \ patch.object(agent_orchestrator, "poll_until_terminal") as mock_poll: mock_run_spec.side_effect = [ + _librarian_routes_to_build(), SpecialistResult(role="config_builder", outcome="ok", output={"experiment_id": "aabb0001"}, sub_run_id="cb-run", turns_used=5), @@ -209,13 +219,14 @@ def test_orchestrator_short_circuits_on_poll_failure(): assert result.outcome == "poll_failed" assert result.experiment_id == "aabb0001" - assert mock_run_spec.call_count == 1 # analyzer never invoked + assert mock_run_spec.call_count == 2 # analyzer never invoked def test_orchestrator_happy_path_returns_analysis_and_ids(): with patch.object(agent_orchestrator, "run_specialist") as mock_run_spec, \ patch.object(agent_orchestrator, "poll_until_terminal") as mock_poll: mock_run_spec.side_effect = [ + _librarian_routes_to_build(), SpecialistResult(role="config_builder", outcome="ok", output={"experiment_id": "aabb0002"}, sub_run_id="cb-run", turns_used=5), @@ -229,7 +240,8 @@ def test_orchestrator_happy_path_returns_analysis_and_ids(): assert result.outcome == "complete" assert result.experiment_id == "aabb0002" assert result.analysis == "FINAL ANSWER: DuckDB wins" - assert result.sub_run_ids == {"config_builder": "cb-run", "analyzer": "an-run"} + assert result.sub_run_ids == {"librarian": "lib-run", + "config_builder": "cb-run", "analyzer": "an-run"} def test_orchestrator_trace_records_delegate_events(tmp_path, monkeypatch): @@ -241,6 +253,7 @@ def test_orchestrator_trace_records_delegate_events(tmp_path, monkeypatch): with patch.object(agent_orchestrator, "run_specialist") as mock_run_spec, \ patch.object(agent_orchestrator, "poll_until_terminal") as mock_poll: mock_run_spec.side_effect = [ + _librarian_routes_to_build(), SpecialistResult(role="config_builder", outcome="ok", output={"experiment_id": "aabb0003"}, sub_run_id="cb-run", turns_used=1), @@ -254,10 +267,11 @@ def test_orchestrator_trace_records_delegate_events(tmp_path, monkeypatch): events = [json.loads(l) for l in open(orch.trace.path)] delegates = [e for e in events if e["event"] == "delegate"] - assert [d["stage"] for d in delegates] == ["config_builder", "poll", "analyzer"] - assert delegates[0]["sub_run_id"] == "cb-run" - assert delegates[1]["sub_run_id"] is None # poll is pure-Python - assert delegates[2]["sub_run_id"] == "an-run" + assert [d["stage"] for d in delegates] == ["librarian", "config_builder", "poll", "analyzer"] + assert delegates[0]["sub_run_id"] == "lib-run" + assert delegates[1]["sub_run_id"] == "cb-run" + assert delegates[2]["sub_run_id"] is None # poll is pure-Python + assert delegates[3]["sub_run_id"] == "an-run" # --------------------------------------------------------------------------- @@ -465,6 +479,7 @@ def test_poll_budget_is_contract_declarable(): with patch.object(agent_orchestrator, "run_specialist") as mock_run_spec, \ patch.object(agent_orchestrator, "poll_until_terminal") as mock_poll: mock_run_spec.side_effect = [ + _librarian_routes_to_build(), SpecialistResult(role="config_builder", outcome="ok", output={"experiment_id": "aabb0007"}, sub_run_id="cb", turns_used=1), @@ -491,17 +506,19 @@ def test_poll_timeout_suspends_not_fails(): analyzer NOT invoked (its tokens are spent later, on resume).""" with patch.object(agent_orchestrator, "run_specialist") as mock_run_spec, \ patch.object(agent_orchestrator, "poll_until_terminal") as mock_poll: - mock_run_spec.return_value = SpecialistResult( - role="config_builder", outcome="ok", - output={"experiment_id": "aabb0008"}, - sub_run_id="cb", turns_used=2) + mock_run_spec.side_effect = [ + _librarian_routes_to_build(), + SpecialistResult(role="config_builder", outcome="ok", + output={"experiment_id": "aabb0008"}, + sub_run_id="cb", turns_used=2), + ] mock_poll.return_value = {"status": "timeout", "polls": 600} result = Orchestrator(goal="g", model="m", poll_budget_seconds=1800).run() assert result.outcome == "suspended" assert result.experiment_id == "aabb0008" assert result.error is None # suspension is not an error - assert mock_run_spec.call_count == 1 # analyzer never ran + assert mock_run_spec.call_count == 2 # analyzer never ran def test_resume_completes_when_capsule_ready(): @@ -544,3 +561,119 @@ def test_resume_surfaces_execution_failure(): assert result.outcome == "poll_failed" assert "out of disk" in result.error assert mock_run_spec.call_count == 0 + + +# --------------------------------------------------------------------------- +# Reference librarian — ask() mode + librarian-first routing in run(). +# Structural fix for Finding 21: 0/5 unprompted library adoption showed +# schema-only exposure doesn't bind; a workflow STAGE does. +# --------------------------------------------------------------------------- + +from sql_benchmarks.agent_orchestrator import LIBRARIAN, _parse_librarian + + +def test_librarian_tool_desk_is_read_only(): + """The reference desk can discover and READ everything, and can + execute NOTHING — no submit, no templates, no build tools.""" + from sql_benchmarks.agent_tools import filter_tools + names = {t["function"]["name"] for t in filter_tools(LIBRARIAN.tool_names)} + assert "search_published_capsules" in names + assert "list_lab_docs" in names and "get_lab_doc" in names + assert "get_experiment_summary" in names + assert "submit_experiment" not in names + assert "get_template" not in names + + +def test_parse_librarian_three_closes(): + long_answer = "FINAL ANSWER: yes — capsules b8e2bfaf and 25b0e134 cover this. " + "x" * 100 + assert _parse_librarian(long_answer) == {"analysis": long_answer} + assert _parse_librarian("HANDOFF: build reason=no capsule covers 100M-row joins") \ + == {"build": "no capsule covers 100M-row joins"} + assert _parse_librarian("HANDOFF: impossible reason=no MongoDB engine") \ + == {"impossible": "no MongoDB engine"} + assert _parse_librarian("just thinking out loud") is None + + +def test_ask_mode_answers_without_executing(): + """The user-delegation use case: 'are there published capsules for X?' + Librarian answers; config_builder and poll never run.""" + with patch.object(agent_orchestrator, "run_specialist") as mock_run_spec, \ + patch.object(agent_orchestrator, "poll_until_terminal") as mock_poll: + mock_run_spec.return_value = SpecialistResult( + role="librarian", outcome="ok", + output={"analysis": "FINAL ANSWER: yes, b8e2bfaf covers quack transport" + "x" * 80}, + sub_run_id="lib", turns_used=3) + result = Orchestrator(goal="are there published capsules for quack?", + model="m").ask() + assert result.outcome == "answered" + assert "b8e2bfaf" in result.analysis + assert mock_run_spec.call_count == 1 + assert mock_poll.call_count == 0 + + +def test_ask_mode_routes_build_worthy_questions_without_building(): + """ask() never executes — a build-worthy question comes back as + needs_experiment and the CALLER decides whether to spend.""" + with patch.object(agent_orchestrator, "run_specialist") as mock_run_spec: + mock_run_spec.return_value = SpecialistResult( + role="librarian", outcome="ok", + output={"build": "no capsule covers recursive CTEs at scale"}, + sub_run_id="lib", turns_used=4) + result = Orchestrator(goal="how do recursive CTEs scale?", model="m").ask() + assert result.outcome == "needs_experiment" + assert "recursive CTEs" in result.analysis + assert mock_run_spec.call_count == 1 + + +def test_run_answers_from_corpus_and_skips_build(): + """Librarian-first in run(): corpus answer -> terminal, config_builder + never invoked. The Finding 21 economics, structurally enforced.""" + with patch.object(agent_orchestrator, "run_specialist") as mock_run_spec, \ + patch.object(agent_orchestrator, "poll_until_terminal") as mock_poll: + mock_run_spec.return_value = SpecialistResult( + role="librarian", outcome="ok", + output={"analysis": "FINAL ANSWER: published capsule answers this" + "x" * 80}, + sub_run_id="lib", turns_used=2) + result = Orchestrator(goal="quack overhead?", model="m").run() + assert result.outcome == "answered" + assert mock_run_spec.call_count == 1 # librarian only — no config_builder + assert mock_poll.call_count == 0 + + +def test_run_proceeds_to_build_on_librarian_handoff(): + with patch.object(agent_orchestrator, "run_specialist") as mock_run_spec, \ + patch.object(agent_orchestrator, "poll_until_terminal") as mock_poll: + mock_run_spec.side_effect = [ + SpecialistResult(role="librarian", outcome="ok", + output={"build": "nothing on sort spill at 100M"}, + sub_run_id="lib", turns_used=2), + SpecialistResult(role="config_builder", outcome="ok", + output={"experiment_id": "aabb000c"}, + sub_run_id="cb", turns_used=5), + SpecialistResult(role="analyzer", outcome="ok", + output={"analysis": "FINAL ANSWER: measured"}, + sub_run_id="an", turns_used=3), + ] + mock_poll.return_value = {"status": "complete", "polls": 1} + with patch.object(agent_orchestrator.httpx, "get") as mock_get: + mock_get.return_value.json.return_value = {"config": {}} + result = Orchestrator(goal="sort spill at 100M?", model="m").run() + assert result.outcome == "complete" + assert result.experiment_id == "aabb000c" + assert result.sub_run_ids["librarian"] == "lib" + + +def test_run_proceeds_to_build_when_librarian_fails(): + """A broken reference desk must not block measurement.""" + with patch.object(agent_orchestrator, "run_specialist") as mock_run_spec, \ + patch.object(agent_orchestrator, "poll_until_terminal") as mock_poll: + mock_run_spec.side_effect = [ + SpecialistResult(role="librarian", outcome="gave_up", output=None, + sub_run_id="lib", turns_used=12, error="stuck"), + SpecialistResult(role="config_builder", outcome="ok", + output={"impossible": "whatever"}, + sub_run_id="cb", turns_used=2), + ] + result = Orchestrator(goal="g", model="m").run() + assert result.outcome == "refused" # reached config_builder despite librarian failure + assert mock_run_spec.call_count == 2