From 533e9c174f6a70756ad90a62d6e8f70df29e3bce Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Tue, 27 Oct 2026 23:58:33 +0100 Subject: [PATCH 01/15] feat(captures): detect replies that claim a tool action Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/claims.py | 41 +++++++++++++++++++++++++ tests/unit/test_captures_claims.py | 45 ++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 src/evalshift_cli/captures/claims.py create mode 100644 tests/unit/test_captures_claims.py diff --git a/src/evalshift_cli/captures/claims.py b/src/evalshift_cli/captures/claims.py new file mode 100644 index 00000000..ac66a473 --- /dev/null +++ b/src/evalshift_cli/captures/claims.py @@ -0,0 +1,41 @@ +"""Spot a reply that claims a tool action, for refusing hallucinated turns at promotion. + +A captured turn that was offered tools, called none, and still says "my notes search didn't +find anything" records an action that never happened. Promoting it would make that +hallucination ground truth (``expected_no_tools: true`` plus the fabricated reply). The check is +wording only — deterministic, offline, English — and deliberately narrow: first-person action +phrasing, never a bare mention of a tool or a noun like "search bar". +""" + +from __future__ import annotations + +import re + +_APOS = "['\u2019]" # straight or curly apostrophe + +_CLAIM = re.compile( + rf"""\b(?: + I(?:{_APOS}ve|\s+have)?\s+(?:just\s+|already\s+)? + (?:searched|checked|looked\s+(?:up|through|in(?:to)?)|queried|pulled\s+up + |went\s+through|scanned) + | (?:my|the)\s+(?:\w+\s+)?(?:search|lookup|query)\s+ + (?:didn{_APOS}t|did\s+not|found|returned|came\s+back|turned\s+up|shows?|showed) + | let\s+me\s+(?:check|search|look|pull\s+up|find) + | I{_APOS}ll\s+(?:check|search|look) + )\b""", + re.IGNORECASE | re.VERBOSE, +) + + +def find_tool_claim(text: str) -> str | None: + """Return the first phrase in ``text`` that claims a tool action. + + Args: + text: A model reply. + + Returns: + The matched phrase exactly as it appears in ``text`` (for quoting in a message), or + ``None`` when the reply makes no such claim. + """ + match = _CLAIM.search(text) + return match.group(0) if match else None diff --git a/tests/unit/test_captures_claims.py b/tests/unit/test_captures_claims.py new file mode 100644 index 00000000..37748420 --- /dev/null +++ b/tests/unit/test_captures_claims.py @@ -0,0 +1,45 @@ +"""The wording check behind unbacked-tool-claim refusals.""" + +from __future__ import annotations + +import pytest + +from evalshift_cli.captures.claims import find_tool_claim + + +@pytest.mark.parametrize( + ("text", "phrase"), + [ + ( + "I don't have that information—my notes search didn't find anything specific", + "my notes search didn't", + ), + ("I searched your notes and found nothing.", "I searched"), + ("I've checked your calendar for Friday.", "I've checked"), + ("I’ve already looked through your tasks.", "I’ve already looked through"), # noqa: RUF001 — curly apostrophe + ("Let me check your notes for that.", "Let me check"), + ("I'll look into it.", "I'll look"), + ("The notes search returned no results.", "The notes search returned"), + ("My search didn’t turn up anything.", "My search didn’t"), # noqa: RUF001 — curly apostrophe + ], +) +def test_claims_are_found_and_quoted(text: str, phrase: str) -> None: + assert find_tool_claim(text) == phrase + + +@pytest.mark.parametrize( + "text", + [ + "Per your notes from Aug 20, the dentist told you to floss daily.", + "Your passport renewal is due by September 15, 2026, and it's still pending.", + "I can't search contract documents.", + "Would you like me to check?", + "Use the search bar at the top.", + "Let me know if you need anything else.", + "I looked at your question again: Friday is clear.", + "Checked items are done.", + "", + ], +) +def test_non_claims_are_ignored(text: str) -> None: + assert find_tool_claim(text) is None From 3b0b08a07b653ac2e4c8717191a208f2143c2c11 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Wed, 28 Oct 2026 00:00:01 +0100 Subject: [PATCH 02/15] feat(config): add captures.allow_unbacked_claims Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/cli/commands/init.py | 3 +++ src/evalshift_cli/config/models.py | 6 ++++++ tests/unit/test_config_captures.py | 14 ++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/src/evalshift_cli/cli/commands/init.py b/src/evalshift_cli/cli/commands/init.py index f5ed4ab3..d8c97488 100644 --- a/src/evalshift_cli/cli/commands/init.py +++ b/src/evalshift_cli/cli/commands/init.py @@ -182,11 +182,14 @@ # store: and name the same bucket. Does the agent answer after calling a # tool? rounds: all replays each recorded round, so that answer is scored too # (about twice the calls); the default replays the first model call only. +# A turn whose reply claims a tool call it never made is skipped; +# allow_unbacked_claims: true promotes it anyway. captures: max_age: 14d max_cases: 200 # store: s3:/// # rounds: all + # allow_unbacked_claims: true """ diff --git a/src/evalshift_cli/config/models.py b/src/evalshift_cli/config/models.py index 3fda8273..00c7dfae 100644 --- a/src/evalshift_cli/config/models.py +++ b/src/evalshift_cli/config/models.py @@ -875,12 +875,18 @@ class CapturesConfig(_StrictModel): ``capture sync --rounds`` overrides it for one sync. It applies to cases promoted from then on: ``capture sync --force`` rebuilds the existing ones whose capture is local, dropping hand edits and ``provenance: reviewed`` marks. + allow_unbacked_claims: Promote turns whose reply claims a tool call the model never + made (offered tools, called none, says it searched). Off by default: such a turn + records a hallucinated action, which is not ground truth. + ``capture sync --allow-unbacked-claims`` / ``--block-unbacked-claims`` override it + for one sync. """ store: str | None = None max_age: Annotated[timedelta | None, BeforeValidator(_duration_from_yaml)] = None max_cases: int | None = Field(default=None, ge=1) rounds: Literal["first", "all"] = "first" + allow_unbacked_claims: bool = False @field_validator("max_age") @classmethod diff --git a/tests/unit/test_config_captures.py b/tests/unit/test_config_captures.py index ce910e7f..75d9c1dd 100644 --- a/tests/unit/test_config_captures.py +++ b/tests/unit/test_config_captures.py @@ -148,6 +148,20 @@ def test_rounds_rejects_anything_else() -> None: EvalShiftConfig(prompts=[_PROMPT], captures={"rounds": "some"}) +def test_allow_unbacked_claims_defaults_to_false() -> None: + assert EvalShiftConfig(prompts=[_PROMPT]).captures.allow_unbacked_claims is False + + +def test_allow_unbacked_claims_accepts_true() -> None: + cfg = EvalShiftConfig(prompts=[_PROMPT], captures={"allow_unbacked_claims": True}) + assert cfg.captures.allow_unbacked_claims is True + + +def test_a_misspelt_allow_unbacked_claims_fails_loudly() -> None: + with pytest.raises(ValidationError, match="allow_unbacked_claim"): + EvalShiftConfig(prompts=[_PROMPT], captures={"allow_unbacked_claim": True}) + + @pytest.mark.parametrize("value", [0, -5]) def test_max_cases_must_be_positive(value: int) -> None: with pytest.raises(ValidationError, match="max_cases"): From 8af71a1354bcf6f8ed7714ca4c72ad4621fe33ae Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Wed, 28 Oct 2026 00:06:47 +0100 Subject: [PATCH 03/15] feat(captures): refuse to promote a turn that claims a tool call it never made Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/promote.py | 72 ++++++++++++++-- tests/unit/test_captures_promote.py | 118 +++++++++++++++++++++++++- 2 files changed, 184 insertions(+), 6 deletions(-) diff --git a/src/evalshift_cli/captures/promote.py b/src/evalshift_cli/captures/promote.py index d7009ed9..2d611c1d 100644 --- a/src/evalshift_cli/captures/promote.py +++ b/src/evalshift_cli/captures/promote.py @@ -30,6 +30,7 @@ from pydantic import ValidationError +from evalshift_cli.captures.claims import find_tool_claim from evalshift_cli.captures.models import CaptureEnvelope, PromotedCase from evalshift_cli.captures.native_messages import normalize_messages from evalshift_cli.captures.reader import CaptureRecord, suites_root, toolset_path @@ -62,7 +63,12 @@ #: Why :func:`build_example_from_capture` refused a capture -- see :class:`BuiltExample`. BlockedReason = Literal[ - "errored", "no_toolset", "multi_toolset", "unreplayable_content", "invalid_case" + "errored", + "no_toolset", + "multi_toolset", + "unreplayable_content", + "invalid_case", + "unbacked_claim", ] @@ -83,6 +89,12 @@ class PromoteOptions: event anyway. The case is still never given ``expected_no_tools=True`` — a failed turn is not evidence that calling nothing was correct. + allow_unbacked_claims: Promote a capture whose turn was offered tools, + called none, and still replied as if it had (see + :func:`~evalshift_cli.captures.claims.find_tool_claim`) anyway. The + refusal becomes a warning, and the case is still never given + ``expected_no_tools=True`` -- a turn that claimed a search is not + evidence that calling nothing was correct. rounds: How many recorded agent rounds the case is replayed over. ``first`` (default) writes no fixtures: replay is single-shot, scored against round 1, the only round it can reproduce. @@ -109,6 +121,7 @@ class PromoteOptions: names_only: bool = False tool_count: bool = False allow_errored: bool = False + allow_unbacked_claims: bool = False rounds: Literal["first", "all"] = "first" tool_properties: Mapping[str, frozenset[str]] | None = None @@ -145,7 +158,10 @@ class BuiltExample: ``example`` is an inert placeholder and ``blocked`` names pydantic's first error. None of the latter four is rescued by ``--allow-errored``: re-capturing fixes the first, and the others reproduce identically unless the app changes what - it records. ``None`` when ``blocked`` is ``None``. + it records. ``"unbacked_claim"`` covers a turn that was offered tools, called + none, and replied as if it had ("my notes search didn't find anything") -- + rescued only by ``allow_unbacked_claims``, never by ``--allow-errored``. + ``None`` when ``blocked`` is ``None``. promotion_source: Which yardstick the tool-call ground truth came from — ``"requested"`` when the capture recorded the calls the model itself asked for, ``"executed"`` when it fell back to the @@ -379,6 +395,22 @@ def _from_requested(call: RequestedToolCall) -> ExpectedToolCall: blocked = note blocked_reason = "errored" + # Not an `elif` of the chain above, so an allowed errored turn still falls through + # to here -- and is skipped: errors take precedence (a failed turn is never evidence + # about tool choice, and expected_no_tools is already off for it). Every earlier + # refusal keeps its own reason too; this only judges an otherwise-promotable turn. + unbacked = None if errors else _unbacked_claim(events, tools_offered) + if blocked is None and unbacked is not None: + note = ( + "turn was offered tools and called none, but its reply claims a tool action " + f'("{unbacked}") — promoting it would make a hallucinated action ground truth.' + ) + if opts.allow_unbacked_claims: + warnings.append(note) + else: + blocked = note + blocked_reason = "unbacked_claim" + failed_results = _failed_tool_results(events) if failed_results: names = ", ".join(sorted(set(failed_results))) @@ -442,8 +474,10 @@ def _from_requested(call: RequestedToolCall) -> ExpectedToolCall: # nothing a candidate model could have failed to call, and # _score_no_tools would award 1.0/1.0 to an unmeasurable row), and a # turn that errored produced no tool calls because it never ran, not - # because calling nothing was right. - expected_no_tools = bool(tools_offered) and not errors + # because calling nothing was right. Nor is a turn that claimed a tool + # action it never took (allowed via allow_unbacked_claims): its own reply + # says a call was the right move. + expected_no_tools = bool(tools_offered) and not errors and unbacked is None expected_tool_rounds = tool_rounds if tool_rounds else None if unwrapped_keys: keys = ", ".join(sorted(unwrapped_keys)) @@ -1324,7 +1358,15 @@ def _turn_assistant_text(record: CaptureRecord) -> str | None: Prefers the last :class:`FinalOutputEvent`; falls back to the last non-empty :class:`ModelCallEvent` output. ``None`` when neither exists. """ - events = record.envelope.trace.events + return _reply_text(record.envelope.trace.events) + + +def _reply_text(events: list[Any]) -> str | None: + """The text a turn replied with. + + Prefers the last :class:`FinalOutputEvent`; falls back to the last + non-empty :class:`ModelCallEvent` output. ``None`` when neither exists. + """ finals = [e for e in events if isinstance(e, FinalOutputEvent)] if finals: return finals[-1].text @@ -1334,6 +1376,26 @@ def _turn_assistant_text(record: CaptureRecord) -> str | None: return None +def _unbacked_claim(events: list[Any], tools_offered: Sequence[str] | None) -> str | None: + """The phrase a tool-free turn's reply uses to claim a tool action, if any. + + Fires only when the turn could have acted and didn't: tools were offered, + no ``tool_call`` event was recorded, and no model call requested one. A turn + that did call a tool is never flagged, however its reply is worded. + + Returns: + The matched phrase (see :func:`~evalshift_cli.captures.claims.find_tool_claim`), + or ``None`` when the turn is not an unbacked claim. + """ + if not tools_offered: + return None + if any(isinstance(e, ToolCallEvent) for e in events): + return None + if any(isinstance(e, ModelCallEvent) and e.requested_tool_calls for e in events): + return None + return find_tool_claim(_reply_text(events) or "") + + def _earliest_prior_system_message( prior: list[tuple[CaptureRecord, BuiltExample]], *, diff --git a/tests/unit/test_captures_promote.py b/tests/unit/test_captures_promote.py index 41040162..cf859f55 100644 --- a/tests/unit/test_captures_promote.py +++ b/tests/unit/test_captures_promote.py @@ -93,6 +93,7 @@ def _model_call( input_tokens: int = 0, output_tokens: int = 0, cost_usd: float = 0.0, + output: Any = "out", ) -> dict[str, Any]: event: dict[str, Any] = { "type": "model_call", @@ -101,7 +102,7 @@ def _model_call( "metadata": {}, "model_id": model_id, "input": model_input, - "output": "out", + "output": output, "input_tokens": input_tokens, "output_tokens": output_tokens, "cost_usd": cost_usd, @@ -564,6 +565,121 @@ def test_still_absent_on_an_allowed_errored_turn_with_no_tools_offered(self) -> assert built.example.expected_no_tools is False +# --------------------------------------------------------------------------- +# Unbacked tool claims -- a turn offered tools that called none, yet whose reply +# says it searched/checked/looked something up, records a hallucinated action. +# --------------------------------------------------------------------------- + +_CLAIM_REPLY = "I don't have that—my notes search didn't find anything." + + +def _claim_events(**model_call: Any) -> list[dict[str, Any]]: + """The turn-9 shape: one model call offered tools, no call, a reply claiming a search.""" + return [ + _model_call(0, model_input="PO number on the Acme contract?", **model_call), + _final(_CLAIM_REPLY, 1), + ] + + +class TestUnbackedClaim: + """A reply that claims a tool action the turn never took is not ground truth: + promoting it would score a candidate that *does* search as wrong + (``expected_no_tools``) and pin the fabricated reply as the expected output.""" + + def test_a_tool_free_turn_claiming_a_search_is_blocked(self) -> None: + built = build_example_from_capture(_envelope(events=_claim_events()), PromoteOptions()) + assert built.blocked_reason == "unbacked_claim" + assert built.blocked == ( + "turn was offered tools and called none, but its reply claims a tool action " + '("my notes search didn\'t") — promoting it would make a hallucinated action ' + "ground truth." + ) + + def test_a_claim_backed_by_a_tool_call_is_not_blocked(self) -> None: + """However the reply is worded, a turn that executed a tool did act.""" + events = [ + _claim_events()[0], + _tool_call("search_orders", 1, args={}), + _final(_CLAIM_REPLY, 2), + ] + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert built.blocked is None + assert built.blocked_reason is None + + def test_a_claim_backed_by_a_requested_call_is_not_blocked(self) -> None: + """A requested call is the model's own action, even if the app recorded no + ``tool_call`` event for it.""" + events = _claim_events(requested_tool_calls=[_requested("search_orders", {})]) + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert built.blocked is None + + def test_no_tools_offered_is_not_blocked(self) -> None: + """With nothing offered the turn could not have called a tool, so the check + has no "should have" to measure -- leave it to the reviewer.""" + events = _claim_events(tools_offered=[]) + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert built.blocked is None + assert built.example.expected_no_tools is False + + def test_a_plain_tool_free_answer_is_not_blocked(self) -> None: + """The common case must stay untouched: a tool-free answer that claims nothing + still promotes with ``expected_no_tools``.""" + events = [_model_call(0, model_input="q"), _final("Friday is clear.", 1)] + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert built.blocked is None + assert built.example.expected_no_tools is True + + def test_the_model_call_output_is_checked_without_a_final_event(self) -> None: + """Same reply rule as reconstructed history: last ``final_output``, else the + last non-empty model output.""" + events = [_model_call(0, model_input="q", output=_CLAIM_REPLY)] + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert built.blocked_reason == "unbacked_claim" + + def test_errored_wins_over_unbacked_claim(self) -> None: + """An errored turn keeps reporting as errored -- the failure is the root cause, + and ``--allow-errored`` is the hint the operator needs.""" + events = [*_claim_events(), _error("boom", 2)] + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert built.blocked_reason == "errored" + + def test_an_allowed_errored_turn_is_not_reblocked_as_unbacked(self) -> None: + """Errors take precedence even when allowed: the operator opted into a failed + turn, which is never evidence calling nothing was right, so ``expected_no_tools`` + is already off and an unbacked-claim block would only contradict the opt-in.""" + events = [*_claim_events(), _error("boom", 2)] + built = build_example_from_capture( + _envelope(events=events), PromoteOptions(allow_errored=True) + ) + assert built.blocked is None + assert built.blocked_reason is None + assert not any("claims a tool action" in w for w in built.warnings) + assert built.example.expected_no_tools is False + + def test_toolset_refusals_win_over_unbacked_claim(self) -> None: + """Only an otherwise-promotable turn is checked; earlier refusals keep their reason.""" + events = _claim_events(toolset_ref=None) + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert built.blocked_reason == "no_toolset" + + def test_allow_errored_does_not_rescue_it(self) -> None: + built = build_example_from_capture( + _envelope(events=_claim_events()), PromoteOptions(allow_errored=True) + ) + assert built.blocked_reason == "unbacked_claim" + + def test_allowed_it_promotes_with_a_warning_and_no_expected_no_tools(self) -> None: + """Mirrors ``allow_errored``: a turn that claimed a search is not evidence that + calling nothing was right, so the opt-in never asserts ``expected_no_tools``.""" + built = build_example_from_capture( + _envelope(events=_claim_events()), PromoteOptions(allow_unbacked_claims=True) + ) + assert built.blocked is None + assert built.blocked_reason is None + assert any("my notes search didn't" in w for w in built.warnings) + assert built.example.expected_no_tools is False + + def test_build_strict_args_uses_exact_match() -> None: built = build_example_from_capture(_envelope(), PromoteOptions(strict_args=True)) assert (built.example.expected_tools or [])[0].match_strategy == "exact" From 94ddd3ee79598f0294ffb78ea8fb33b288cc8d99 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Wed, 28 Oct 2026 00:12:31 +0100 Subject: [PATCH 04/15] feat(captures): warn when a case's recorded history carries an unbacked tool claim Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/claims.py | 39 +++++++++ src/evalshift_cli/captures/promote.py | 31 ++++++- tests/unit/test_captures_claims.py | 79 ++++++++++++++++- tests/unit/test_captures_promote.py | 120 +++++++++++++++++++++++++- 4 files changed, 266 insertions(+), 3 deletions(-) diff --git a/src/evalshift_cli/captures/claims.py b/src/evalshift_cli/captures/claims.py index ac66a473..940514ed 100644 --- a/src/evalshift_cli/captures/claims.py +++ b/src/evalshift_cli/captures/claims.py @@ -10,6 +10,9 @@ from __future__ import annotations import re +from collections.abc import Sequence + +from evalshift_cli.suite.models import ChatMessage _APOS = "['\u2019]" # straight or curly apostrophe @@ -39,3 +42,39 @@ def find_tool_claim(text: str) -> str | None: """ match = _CLAIM.search(text) return match.group(0) if match else None + + +def unbacked_history_claims(history: Sequence[ChatMessage]) -> list[tuple[int, str]]: + """Find the assistant replies in a recorded history that claim a tool action never taken. + + A reply is *backed* when a ``tool`` message or an ``assistant`` message with + ``tool_calls`` sits between it and the user turn it answers. A ``user`` message starts a + new turn, except one directly after a ``tool`` row: Claude delivers tool results inside + a user turn, and text riding in that same turn normalises to ``tool`` rows followed by a + ``user`` row -- still the answer to the call, not a fresh question. + + Only meaningful for history recorded with its tool rows intact. History rebuilt from + earlier turns' text carries no tool rows, so every past reply would look unbacked. + + Args: + history: A case's recorded conversation prefix. + + Returns: + ``(position, phrase)`` per unbacked claiming reply, in order. ``position`` is + 1-based within ``history``; ``phrase`` is quoted from the reply by + :func:`find_tool_claim`. + """ + claims: list[tuple[int, str]] = [] + backed = False + previous_role: str | None = None + for position, message in enumerate(history, start=1): + if message.role == "user" and previous_role != "tool": + backed = False + elif message.role == "tool" or (message.role == "assistant" and message.tool_calls): + backed = True + elif message.role == "assistant" and message.content and not backed: + phrase = find_tool_claim(message.content) + if phrase is not None: + claims.append((position, phrase)) + previous_role = message.role + return claims diff --git a/src/evalshift_cli/captures/promote.py b/src/evalshift_cli/captures/promote.py index 2d611c1d..0528156d 100644 --- a/src/evalshift_cli/captures/promote.py +++ b/src/evalshift_cli/captures/promote.py @@ -30,7 +30,7 @@ from pydantic import ValidationError -from evalshift_cli.captures.claims import find_tool_claim +from evalshift_cli.captures.claims import find_tool_claim, unbacked_history_claims from evalshift_cli.captures.models import CaptureEnvelope, PromotedCase from evalshift_cli.captures.native_messages import normalize_messages from evalshift_cli.captures.reader import CaptureRecord, suites_root, toolset_path @@ -236,6 +236,12 @@ def build_example_from_capture( loss = _context_loss_warning(events, recovered.inputs, recovered.history) if loss is not None: warnings.append(loss) + if recovered.history is not None: + # Recorded history only: it keeps its tool rows, so "backed" is decidable. History + # reconstructed from earlier turns is text-only and must never be scanned. + history_claims = _history_claims_warning(recovered.history) + if history_claims is not None: + warnings.append(history_claims) tags = _build_tags(envelope.suite, opts.tags) expected = _recover_expected(events, warnings) @@ -1254,6 +1260,29 @@ def _context_loss_warning( return None +def _history_claims_warning(history: Sequence[ChatMessage]) -> str | None: + """One warning naming every recorded history reply that claims a tool action never taken. + + See :func:`~evalshift_cli.captures.claims.unbacked_history_claims`. Positions are + 1-based, as in ``captures/repair.py``. + """ + claims = unbacked_history_claims(history) + if not claims: + return None + if len(claims) == 1: + position, phrase = claims[0] + return ( + f'history message {position} (assistant) claims a tool action ("{phrase}") with no ' + "tool call in its turn — this case replays with that claim in context." + ) + positions = ", ".join(str(position) for position, _ in claims) + phrases = ", ".join(f'"{phrase}"' for _, phrase in claims) + return ( + f"history messages {positions} (assistant) claim tool actions ({phrases}) with no " + "tool call in their turn — this case replays with those claims in context." + ) + + def _build_tags(suite: str, extra: tuple[str, ...]) -> list[str]: tags = [CAPTURED_TAG, suite] for tag in extra: diff --git a/tests/unit/test_captures_claims.py b/tests/unit/test_captures_claims.py index 37748420..3867b71b 100644 --- a/tests/unit/test_captures_claims.py +++ b/tests/unit/test_captures_claims.py @@ -4,7 +4,8 @@ import pytest -from evalshift_cli.captures.claims import find_tool_claim +from evalshift_cli.captures.claims import find_tool_claim, unbacked_history_claims +from evalshift_cli.suite.models import ChatMessage, HistoryToolCall @pytest.mark.parametrize( @@ -43,3 +44,79 @@ def test_claims_are_found_and_quoted(text: str, phrase: str) -> None: ) def test_non_claims_are_ignored(text: str) -> None: assert find_tool_claim(text) is None + + +def _user(text: str) -> ChatMessage: + return ChatMessage(role="user", content=text) + + +def _assistant(text: str, *, calls: list[HistoryToolCall] | None = None) -> ChatMessage: + return ChatMessage(role="assistant", content=text, tool_calls=calls) + + +def _tool(call_id: str, result: str) -> ChatMessage: + return ChatMessage(role="tool", content=result, tool_call_id=call_id) + + +_SEARCH = HistoryToolCall(id="c1", name="search_notes", arguments={"q": "PO"}) + + +def test_history_claim_without_a_tool_call_is_reported_by_position() -> None: + history = [_user("PO number?"), _assistant("Sorry, my notes search didn't find it.")] + assert unbacked_history_claims(history) == [(2, "my notes search didn't")] + + +def test_history_claim_with_tool_call_in_its_turn_is_backed() -> None: + """The native-shapes shape: "Let me look." beside a real call, then the answer.""" + history = [ + _user("PO number?"), + _assistant("Let me look.", calls=[_SEARCH]), + _tool("c1", '{"po": "PO-7"}'), + _assistant("I searched and found PO-7."), + ] + assert unbacked_history_claims(history) == [] + + +def test_history_reply_that_claims_nothing_is_ignored() -> None: + history = [ + ChatMessage(role="system", content="You are Kaila."), + _user("dentist?"), + _assistant("Per your notes, floss daily."), + ] + assert unbacked_history_claims(history) == [] + + +def test_unbacked_claims_in_different_turns_are_all_reported_in_order() -> None: + history = [ + _user("PO number?"), + _assistant("I searched your notes and found nothing."), + _user("Friday?"), + _assistant("Let me check your calendar."), + ] + assert unbacked_history_claims(history) == [(2, "I searched"), (4, "Let me check")] + + +def test_a_new_user_turn_resets_backing() -> None: + """A call in an earlier turn does not back a claim made after the user spoke again.""" + history = [ + _user("PO number?"), + _assistant("", calls=[_SEARCH]), + _tool("c1", '{"po": "PO-7"}'), + _assistant("It is PO-7."), + _user("And the Globex one?"), + _assistant("I checked and there is none."), + ] + assert unbacked_history_claims(history) == [(6, "I checked")] + + +def test_user_text_riding_on_a_tool_result_turn_keeps_backing() -> None: + """Claude delivers tool results in a user turn; text in that same turn normalises to + tool rows followed by a user row. It is still the answer to the call, not a new turn.""" + history = [ + _user("PO number?"), + _assistant("", calls=[_SEARCH]), + _tool("c1", '{"po": "PO-7"}'), + _user("(the user is waiting)"), + _assistant("I searched your notes: it is PO-7."), + ] + assert unbacked_history_claims(history) == [] diff --git a/tests/unit/test_captures_promote.py b/tests/unit/test_captures_promote.py index cf859f55..ba2fb537 100644 --- a/tests/unit/test_captures_promote.py +++ b/tests/unit/test_captures_promote.py @@ -234,6 +234,7 @@ def _messages_call( *, messages: list[dict[str, Any]], input_tokens: int = 0, + tools_offered: list[str] | None = None, ) -> dict[str, Any]: return { "type": "model_call", @@ -245,7 +246,7 @@ def _messages_call( "output": "out", "input_tokens": input_tokens, "toolset_ref": "sha256:" + "ab" * 32, - "tools_offered": [], + "tools_offered": tools_offered if tools_offered is not None else [], } @@ -776,6 +777,123 @@ def test_build_messages_list_model_role_coerced_to_assistant() -> None: assert built.example.history[1].content == "hello" +class TestUnbackedHistoryClaims: + """A case whose *recorded* history carries a reply claiming a tool action it never + took replays with that fabrication in context -- worth a warning, not a block: the + current turn may be fine.""" + + _HISTORY_WARNING = ( + 'history message 2 (assistant) claims a tool action ("my notes search didn\'t") ' + "with no tool call in its turn — this case replays with that claim in context." + ) + + @pytest.mark.parametrize("tools_offered", [["search_notes"], []]) + def test_recorded_history_with_an_unbacked_claim_warns(self, tools_offered: list[str]) -> None: + """The turn-10 shape. The warning is about the history, not the current turn, so it + fires whether or not this turn was offered tools.""" + messages = [ + {"role": "user", "content": "PO number on the Acme contract?"}, + {"role": "assistant", "content": _CLAIM_REPLY}, + {"role": "user", "content": "Try the Globex one then."}, + ] + events = [ + _messages_call(0, messages=messages, tools_offered=tools_offered), + _final("Globex is PO-7.", 1), + ] + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert built.blocked is None + assert built.warnings.count(self._HISTORY_WARNING) == 1 + + def test_several_unbacked_claims_aggregate_into_one_warning(self) -> None: + messages = [ + {"role": "user", "content": "PO number on the Acme contract?"}, + {"role": "assistant", "content": _CLAIM_REPLY}, + {"role": "user", "content": "Friday?"}, + {"role": "assistant", "content": "I've checked your calendar: it's clear."}, + {"role": "user", "content": "Thanks."}, + ] + events = [ + _messages_call(0, messages=messages, tools_offered=["search_notes"]), + _final("You're welcome.", 1), + ] + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + claim_warnings = [w for w in built.warnings if "claim" in w] + assert claim_warnings == [ + "history messages 2, 4 (assistant) claim tool actions " + '("my notes search didn\'t", "I\'ve checked") with no tool call in their ' + "turn — this case replays with those claims in context." + ] + + def test_a_claim_backed_by_a_recorded_call_does_not_warn(self) -> None: + messages = [ + {"role": "user", "content": "PO number on the Acme contract?"}, + { + "role": "assistant", + "content": "Let me look.", + "tool_calls": [{"id": "c1", "name": "search_notes", "arguments": {}}], + }, + {"role": "tool", "tool_call_id": "c1", "content": "nothing"}, + {"role": "assistant", "content": _CLAIM_REPLY}, + {"role": "user", "content": "Try the Globex one then."}, + ] + events = [ + _messages_call(0, messages=messages, tools_offered=["search_notes"]), + _final("Globex is PO-7.", 1), + ] + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert not any("claim" in w for w in built.warnings) + + def test_reconstructed_history_is_not_scanned(self) -> None: + """Reconstructed history is text-only: scanning it would flag every past reply.""" + turn0 = _envelope( + capture_id="cap_t0", + conversation_id="conv_1", + turn_index=0, + events=[ + _model_call(0, model_input="Am I free Friday?"), + _tool_call("search_orders", 1, args={}), + _final("I checked your calendar: Friday is clear.", 2), + ], + ) + turn1 = _envelope( + capture_id="cap_t1", + conversation_id="conv_1", + turn_index=1, + events=[_model_call(0, model_input="And Saturday?"), _final("Also clear.", 1)], + ) + + results = build_conversation_examples([_record(turn0), _record(turn1)], PromoteOptions()) + + _, built1 = results[1] + assert built1.example.history is not None + assert built1.example.history[1].content == "I checked your calendar: Friday is clear." + assert built1.blocked is None + assert not any("claim" in w for w in built1.warnings) + + def test_a_blocked_unbacked_turn_does_not_seed_reconstructed_history(self) -> None: + turn0 = _envelope( + capture_id="cap_t0", + conversation_id="conv_1", + turn_index=0, + events=_claim_events(), + ) + turn1 = _envelope( + capture_id="cap_t1", + conversation_id="conv_1", + turn_index=1, + events=[_model_call(0, model_input="And Globex?"), _final("Globex is PO-7.", 1)], + ) + + results = build_conversation_examples([_record(turn0), _record(turn1)], PromoteOptions()) + + (_, built0), (_, built1) = results + assert built0.blocked_reason == "unbacked_claim" + assert built1.blocked is None + history = built1.example.history or [] + assert all("Acme" not in m.content for m in history) + assert all(m.content != _CLAIM_REPLY for m in history) + + def test_promote_preserves_tool_turns_in_history() -> None: messages = [ {"role": "system", "content": "You are Kaila."}, From a268aa9683b17639ccd48880990d0abe1dbaf462 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Wed, 28 Oct 2026 00:15:58 +0100 Subject: [PATCH 05/15] feat(cli): capture promote --allow-unbacked-claims Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/cli/commands/capture.py | 20 +++++-- tests/unit/test_cli_capture.py | 66 +++++++++++++++++++++++ 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/src/evalshift_cli/cli/commands/capture.py b/src/evalshift_cli/cli/commands/capture.py index 971f1594..2f07133b 100644 --- a/src/evalshift_cli/cli/commands/capture.py +++ b/src/evalshift_cli/cli/commands/capture.py @@ -984,6 +984,16 @@ def capture_promote( ), ), ] = False, + allow_unbacked_claims: Annotated[ + bool, + typer.Option( + "--allow-unbacked-claims", + help=( + "Promote even if the turn's reply claims a tool call it never made. " + "The case still never asserts expected_no_tools." + ), + ), + ] = False, force: Annotated[ bool, typer.Option("--force", "-f", help="Overwrite an existing promoted case."), @@ -1015,6 +1025,7 @@ def capture_promote( names_only=names_only, tool_count=tool_count, allow_errored=allow_errored, + allow_unbacked_claims=allow_unbacked_claims, rounds="all" if rounds == "all" else "first", tool_properties=_declared_tool_properties([envelope], base=base), ) @@ -1026,11 +1037,14 @@ def capture_promote( # error worth failing on rather than a line in a sync summary. if built.blocked is not None: console.print(f"[red]✗[/red] refusing to promote {envelope.capture_id!r}: {built.blocked}") - # Only an errored turn is rescuable by the flag -- a capture with no - # usable recorded toolset needs re-capturing (already spelled out in - # `built.blocked`), so pointing at --allow-errored there is a dead end. + # Each rescuable refusal names its own flag -- a capture with no usable + # recorded toolset needs re-capturing (already spelled out in + # `built.blocked`), and --allow-errored never rescues an unbacked claim, + # so pointing at it there is a dead end. if built.blocked_reason == "errored": console.print("Re-run with [bold]--allow-errored[/bold] to promote it anyway.") + elif built.blocked_reason == "unbacked_claim": + console.print("Re-run with [bold]--allow-unbacked-claims[/bold] to promote it anyway.") raise typer.Exit(code=1) if envelope.conversation_id is not None: diff --git a/tests/unit/test_cli_capture.py b/tests/unit/test_cli_capture.py index 8ebe26fe..8104387b 100644 --- a/tests/unit/test_cli_capture.py +++ b/tests/unit/test_cli_capture.py @@ -1415,6 +1415,72 @@ def test_promote_allow_errored_promotes_without_expected_no_tools(tmp_path: Path assert case.example.expected_no_tools is False +_UNBACKED_REPLY = "I don't have that information—my notes search didn't find anything specific." + + +def _unbacked_events(model_input: Any = "what did the dentist say?") -> list[dict[str, Any]]: + """A turn offered a tool that called none, yet whose reply claims it searched.""" + return [ + { + "type": "model_call", + "sequence_index": 0, + "timestamp": "2026-06-16T12:00:00+00:00", + "metadata": {}, + "model_id": "m", + "input": model_input, + "output": "", + "toolset_ref": _TOOLSET_REF, + "tools_offered": ["search_orders"], + }, + { + "type": "final_output", + "sequence_index": 1, + "timestamp": "2026-06-16T12:00:03+00:00", + "metadata": {}, + "text": _UNBACKED_REPLY, + }, + ] + + +def test_promote_refuses_an_unbacked_claim(tmp_path: Path) -> None: + _write_capture(tmp_path, capture_id="cap_bad", suite="alpha", events=_unbacked_events()) + + result = _invoke(["promote", "cap_bad", "--as", "case_bad"], tmp_path) + + assert result.exit_code == 1 + flat = " ".join(result.stdout.split()) + assert "claims a tool action" in flat + assert "Re-run with --allow-unbacked-claims to promote it anyway." in flat + # --allow-errored can never rescue this refusal, so it is never offered. + assert "--allow-errored" not in flat + assert not (tmp_path / "suites" / "alpha" / "case_bad.json").exists() + + +def test_promote_allow_unbacked_claims_promotes_without_expected_no_tools( + tmp_path: Path, +) -> None: + _write_capture(tmp_path, capture_id="cap_bad", suite="alpha", events=_unbacked_events()) + + result = _invoke( + ["promote", "cap_bad", "--as", "case_bad", "--allow-unbacked-claims"], + tmp_path, + ) + + assert result.exit_code == 0, result.stdout + case_path = tmp_path / "suites" / "alpha" / "case_bad.json" + case = PromotedCase.model_validate_json(case_path.read_text(encoding="utf-8")) + assert case.example.expected_no_tools is False + + +def test_promote_allow_errored_does_not_rescue_an_unbacked_claim(tmp_path: Path) -> None: + _write_capture(tmp_path, capture_id="cap_bad", suite="alpha", events=_unbacked_events()) + + result = _invoke(["promote", "cap_bad", "--as", "case_bad", "--allow-errored"], tmp_path) + + assert result.exit_code == 1 + assert not (tmp_path / "suites" / "alpha" / "case_bad.json").exists() + + def test_sync_skips_errored_captures(tmp_path: Path) -> None: _write_capture(tmp_path, capture_id="cap_ok", suite="alpha") _write_capture(tmp_path, capture_id="cap_err", suite="alpha", events=_errored_events()) From 43c00cdcdca7e37c01a4f10938df2b5468239ca3 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Wed, 28 Oct 2026 00:19:19 +0100 Subject: [PATCH 06/15] feat(cli): capture sync skips unbacked tool claims, with YAML and flag overrides Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/cli/commands/capture.py | 61 ++++++++-- tests/unit/test_cli_capture.py | 135 ++++++++++++++++++++++ tests/unit/test_cli_capture_remote.py | 10 +- 3 files changed, 195 insertions(+), 11 deletions(-) diff --git a/src/evalshift_cli/cli/commands/capture.py b/src/evalshift_cli/cli/commands/capture.py index 2f07133b..4e42fbbe 100644 --- a/src/evalshift_cli/cli/commands/capture.py +++ b/src/evalshift_cli/cli/commands/capture.py @@ -602,6 +602,8 @@ class _SyncLimits: source_model: str | None = None #: ``captures.rounds``: not a limit, read here because sync reads its config here. rounds: Literal["first", "all"] = "first" + #: ``captures.allow_unbacked_claims``: not a limit either, read here for the same reason. + allow_unbacked_claims: bool = False @property def active(self) -> bool: @@ -640,12 +642,14 @@ def _sync_limits(config_path: Path, *, console: Console, warn: bool) -> _SyncLim max_cases=cfg.captures.max_cases, source_model=cfg.defaults.source_model, rounds=cfg.captures.rounds, + allow_unbacked_claims=cfg.captures.allow_unbacked_claims, ) if warn and _mentions_captures(config_path): console.print( f"[yellow]⚠[/yellow] could not read {escape(str(config_path))} ({escape(reason)}); " - "syncing without captures.max_age / captures.max_cases / captures.rounds " - "(replaying the first round only).", + "syncing without captures.max_age / captures.max_cases / captures.rounds / " + "captures.allow_unbacked_claims (replaying the first round only, blocking " + "unbacked tool claims).", ) return _SyncLimits() @@ -1450,6 +1454,18 @@ def capture_sync( ), ), ] = False, + allow_unbacked_claims: Annotated[ + bool | None, + typer.Option( + "--allow-unbacked-claims/--block-unbacked-claims", + help=( + "Promote captures whose reply claims a tool call the turn never made, or " + "block them. Defaults to captures.allow_unbacked_claims in evalshift.yaml, " + "else block: such a turn records a hallucinated action, which is not " + "ground truth." + ), + ), + ] = None, config_path: _ConfigOption = Path(CONFIG_FILENAME), force: Annotated[ bool, @@ -1500,6 +1516,8 @@ def capture_sync( keeps older captures out of a suite. The fetch from captures.store takes only captures within captures.max_age and, with captures.max_cases set, at most 3x max_cases per suite, a conversation at a time, newest first. + A capture whose reply claims a tool call its turn never made is skipped + unless --allow-unbacked-claims or captures.allow_unbacked_claims allows it. """ console = Console() if rounds is not None: @@ -1508,6 +1526,9 @@ def capture_sync( replay_rounds: Literal["first", "all"] = ( limits.rounds if rounds is None else ("all" if rounds == "all" else "first") ) + allow_unbacked = ( + limits.allow_unbacked_claims if allow_unbacked_claims is None else allow_unbacked_claims + ) now = _now() selection = _repair_selection(base=base, suite=suite, console=console) _maybe_fetch( @@ -1543,6 +1564,7 @@ def capture_sync( names_only=names_only, tool_count=tool_count, allow_errored=allow_errored, + allow_unbacked_claims=allow_unbacked, rounds=replay_rounds, tool_properties=_declared_tool_properties([r.envelope for r in records], base=base), ) @@ -1560,6 +1582,7 @@ def capture_sync( skipped_multi_toolset = 0 skipped_unreplayable = 0 skipped_invalid = 0 + skipped_unbacked = 0 repaired = 0 left_for_repair = 0 backfilled = 0 @@ -1715,6 +1738,9 @@ def capture_sync( if key is not None: seen_examples.setdefault((suite_name, key), case.name) repaired_here: list[str] = [] + # capture id -> the case file promoted from it; built only once a capture in this + # suite is refused as an unbacked claim, so the common path never pays for it. + stale_case_paths: dict[str, Path] | None = None for record, built in built_pairs: envelope = record.envelope backed = backed_by.get(envelope.capture_id) @@ -1771,17 +1797,30 @@ def capture_sync( repaired_here.append(case.name) continue if built.blocked is not None: - console.print( + skip_line = ( f"[yellow]⚠[/yellow] skipped [cyan]{envelope.capture_id}[/cyan]: " - f"{built.blocked}", + f"{built.blocked}" ) + if built.blocked_reason == "unbacked_claim": + # A case an earlier sync (or an override) promoted stays in the suite: + # sync never deletes a case file, so it names the file to delete. + if stale_case_paths is None: + stale_case_paths = {c.from_capture: path for path, c in kept} + stale_path = stale_case_paths.get(envelope.capture_id) + if stale_path is not None: + skip_line += ( + " Its case is still in the suite from an earlier sync: " + f"delete {escape(str(stale_path))} to drop it." + ) + console.print(skip_line, soft_wrap=True) # Routed by *why* it was blocked, not lumped under one counter: # --allow-errored rescues an errored turn but can never rescue # a capture with no usable recorded toolset, one that switched # toolsets mid-run, one whose messages carry content a text - # replay can't carry (unreplayable_content), or one no valid case - # can be built from (invalid_case), so the summary below must not - # offer that as a fix for any of them. + # replay can't carry (unreplayable_content), one no valid case + # can be built from (invalid_case), or one whose reply claims a + # tool call it never made (unbacked_claim), so the summary below + # must not offer that as a fix for any of them. if built.blocked_reason == "no_toolset": skipped_no_toolset += 1 elif built.blocked_reason == "multi_toolset": @@ -1790,6 +1829,8 @@ def capture_sync( skipped_unreplayable += 1 elif built.blocked_reason == "invalid_case": skipped_invalid += 1 + elif built.blocked_reason == "unbacked_claim": + skipped_unbacked += 1 else: skipped_errored += 1 continue @@ -1988,6 +2029,12 @@ def capture_sync( f", skipped {skipped_errored} errored capture(s) " "(use --allow-errored to promote anyway)" ) + if skipped_unbacked: + summary += ( + f", skipped {skipped_unbacked} capture(s) whose reply claims a tool call it never " + "made (use --allow-unbacked-claims or captures.allow_unbacked_claims to promote " + "anyway)" + ) if skipped_no_toolset: summary += ( f", skipped {skipped_no_toolset} capture(s) with no usable recorded toolset " diff --git a/tests/unit/test_cli_capture.py b/tests/unit/test_cli_capture.py index 8104387b..8a5fa56f 100644 --- a/tests/unit/test_cli_capture.py +++ b/tests/unit/test_cli_capture.py @@ -1509,6 +1509,141 @@ def test_sync_allow_errored_promotes_them(tmp_path: Path) -> None: } +_UNBACKED_SUMMARY = ( + ", skipped 1 capture(s) whose reply claims a tool call it never made " + "(use --allow-unbacked-claims or captures.allow_unbacked_claims to promote anyway)" +) + + +def _unbacked_sync_fixture(tmp_path: Path, *, yaml_allow: bool | None = None) -> Path: + """``cap_ok`` plus the unbacked ``cap_bad`` in suite alpha; returns the config path.""" + _write_capture(tmp_path, capture_id="cap_ok", suite="alpha") + _write_capture(tmp_path, capture_id="cap_bad", suite="alpha", events=_unbacked_events()) + config = tmp_path / "evalshift.yaml" + _write_min_config(config) + if yaml_allow is not None: + value = "true" if yaml_allow else "false" + config.write_text( + config.read_text(encoding="utf-8") + f"\ncaptures:\n allow_unbacked_claims: {value}\n", + encoding="utf-8", + ) + return config + + +def test_sync_skips_unbacked_claims(tmp_path: Path) -> None: + config = _unbacked_sync_fixture(tmp_path) + + result = _invoke(["sync", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + flat = " ".join(result.stdout.split()) + assert "claims a tool action" in flat + assert _UNBACKED_SUMMARY in flat + assert load_jsonl(tmp_path / "suites" / "alpha" / "golden.jsonl").ids() == {"cap_ok"} + + +def test_sync_allow_flag_promotes_them(tmp_path: Path) -> None: + config = _unbacked_sync_fixture(tmp_path) + + result = _invoke(["sync", "--allow-unbacked-claims", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + assert "whose reply claims a tool call" not in result.stdout + golden = load_jsonl(tmp_path / "suites" / "alpha" / "golden.jsonl") + assert golden.ids() == {"cap_bad", "cap_ok"} + case = PromotedCase.model_validate_json( + (tmp_path / "suites" / "alpha" / "cap_bad.json").read_text(encoding="utf-8") + ) + assert case.example.expected_no_tools is False + + +def test_sync_yaml_true_promotes_them(tmp_path: Path) -> None: + config = _unbacked_sync_fixture(tmp_path, yaml_allow=True) + + result = _invoke(["sync", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + assert load_jsonl(tmp_path / "suites" / "alpha" / "golden.jsonl").ids() == { + "cap_bad", + "cap_ok", + } + + +def test_sync_allow_flag_overrides_yaml_false(tmp_path: Path) -> None: + config = _unbacked_sync_fixture(tmp_path, yaml_allow=False) + + result = _invoke(["sync", "--allow-unbacked-claims", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + assert load_jsonl(tmp_path / "suites" / "alpha" / "golden.jsonl").ids() == { + "cap_bad", + "cap_ok", + } + + +def test_sync_block_flag_overrides_yaml(tmp_path: Path) -> None: + config = _unbacked_sync_fixture(tmp_path, yaml_allow=True) + + result = _invoke(["sync", "--block-unbacked-claims", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + assert _UNBACKED_SUMMARY in " ".join(result.stdout.split()) + assert load_jsonl(tmp_path / "suites" / "alpha" / "golden.jsonl").ids() == {"cap_ok"} + + +def test_sync_summary_does_not_offer_allow_errored_for_unbacked(tmp_path: Path) -> None: + """An unbacked claim used to fall into the errored counter, whose clause offers a flag + that cannot rescue it.""" + config = _unbacked_sync_fixture(tmp_path) + + result = _invoke(["sync", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + flat = " ".join(result.stdout.split()) + assert "errored capture(s)" not in flat + assert "--allow-errored" not in flat + + +def test_sync_names_the_stale_case_of_a_now_blocked_capture(tmp_path: Path) -> None: + """A case promoted before the check existed (here: under the override) stays in the + suite; sync names its file instead of deleting it.""" + config = _unbacked_sync_fixture(tmp_path) + promoted = _invoke(["promote", "cap_bad", "--allow-unbacked-claims"], tmp_path) + assert promoted.exit_code == 0, promoted.stdout + case_path = tmp_path / "suites" / "alpha" / "cap_bad.json" + assert case_path.exists() + + result = _invoke(["sync", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + flat = " ".join(result.stdout.split()) + assert ( + f" Its case is still in the suite from an earlier sync: delete {case_path} to drop it." + ) in flat + assert case_path.exists() + assert "cap_bad" in load_jsonl(tmp_path / "suites" / "alpha" / "golden.jsonl").ids() + + +def test_sync_stale_case_hint_only_for_a_promoted_capture(tmp_path: Path) -> None: + config = _unbacked_sync_fixture(tmp_path) + + result = _invoke(["sync", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + assert "still in the suite" not in result.stdout + + +def test_sync_unreadable_config_names_allow_unbacked_claims(tmp_path: Path) -> None: + config = tmp_path / "evalshift.yaml" + config.write_text("captures:\n allow_unbacked_claim: true\n", encoding="utf-8") + _write_capture(tmp_path, capture_id="cap_bad", suite="alpha", events=_unbacked_events()) + + result = _invoke(["sync", "--offline", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + assert "captures.allow_unbacked_claims" in " ".join(result.stdout.split()) + + def test_sync_skips_captures_with_no_toolset(tmp_path: Path) -> None: """capture sync's summary must count and describe a missing-toolset_ref refusal separately from an errored-capture refusal -- --allow-errored diff --git a/tests/unit/test_cli_capture_remote.py b/tests/unit/test_cli_capture_remote.py index bd4da04b..bd5a8cf8 100644 --- a/tests/unit/test_cli_capture_remote.py +++ b/tests/unit/test_cli_capture_remote.py @@ -479,8 +479,9 @@ def test_offline_sync_says_it_ignores_limits_it_cannot_read(tmp_path: Path) -> N result = _invoke(["sync", "--offline", "--config", str(config)], tmp_path) assert result.exit_code == 0, result.stdout assert ( - "syncing without captures.max_age / captures.max_cases / captures.rounds " - "(replaying the first round only)" + "syncing without captures.max_age / captures.max_cases / captures.rounds / " + "captures.allow_unbacked_claims (replaying the first round only, blocking unbacked " + "tool claims)" ) in " ".join(result.stdout.split()) @@ -499,8 +500,9 @@ def test_online_sync_says_it_ignores_limits_it_cannot_read( flat = " ".join(result.stdout.split()) assert ( f"could not read {config} (1 schema problem found); " - "syncing without captures.max_age / captures.max_cases / captures.rounds " - "(replaying the first round only)." + "syncing without captures.max_age / captures.max_cases / captures.rounds / " + "captures.allow_unbacked_claims (replaying the first round only, blocking unbacked " + "tool claims)." ) in flat assert "skipping captures.store" in flat assert remote.gets == [] From 1d069120e5656eebb8613abf8bc9165bc7906fbb Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Wed, 28 Oct 2026 00:26:19 +0100 Subject: [PATCH 07/15] docs: unbacked tool claims at capture promotion Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 31 +++++++++++++++++++++++++++ DOCS.md | 10 ++++----- docs/agents.md | 26 +++++++++++++++++++++- docs/configuration.md | 15 +++++++++++-- docs/conversations.md | 21 ++++++++++++++++++ llms-full.txt | 50 ++++++++++++++++++++++++++++++++++++++----- 6 files changed, 140 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5691366c..c3cd26cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Promotion refuses a turn that claims a tool call it never made.** A turn + that was offered tools, called none, and still replied "my notes search + didn't find anything" was promoted as ground truth: the fabricated reply + became `expected`, and the case asserted `expected_no_tools: true`. + `capture promote` now exits 1 on it (`Re-run with --allow-unbacked-claims to + promote it anyway.`), and `capture sync` skips it and counts it on its own + (`skipped 1 capture(s) whose reply claims a tool call it never made`). The + check reads the reply's English wording (first-person phrasing such as "I + searched", "let me check", "my search returned"; a bare tool name never + fires) and never flags a turn that called or requested a tool. + `capture promote --allow-unbacked-claims`, `capture sync + --allow-unbacked-claims/--block-unbacked-claims` and + `captures.allow_unbacked_claims` in `evalshift.yaml` (flag, then YAML, then + block) promote it with a warning instead; such a case never asserts + `expected_no_tools`. `--allow-errored` does not apply, and an errored turn + is still reported as errored. Cases an earlier sync promoted stay in the + suite: sync never deletes a case file, so its skip line names the file to + delete (`Its case is still in the suite from an earlier sync: delete + .evalshift/suites//.json to drop it.`). A refused turn + also drops out of later turns' reconstructed history. `init` scaffolds the + key commented out. Older CLIs reject the key, so update a CI + `evalshift-version:` pin first. +- Promotion warns when a case's recorded history carries an assistant reply + that claims a tool action with no tool call in its turn (`history message 2 + (assistant) claims a tool action ("my notes search didn't") with no tool + call in its turn — this case replays with that claim in context.`). The + case is still promoted. History reconstructed from sibling turns is not + checked. + ## [1.6.2] - 2026-10-27 ### Changed diff --git a/DOCS.md b/DOCS.md index 1becc779..81cc33da 100644 --- a/DOCS.md +++ b/DOCS.md @@ -187,9 +187,9 @@ and `capture sync` and `capture list` fetch what is new before they read the loc 4. Carries the recorded generation settings onto the promoted example: the first `model_call`'s `metadata["generation_config"]` is copied verbatim to the example's `generation_config`, so the replay dispatches under the temperature, structured-output, and tool-choice settings the production call actually used. `capture sync` appends `wired generation config for N case(s)` to its summary line when any promoted capture carried one. 5. Records what the captured run cost on the case file as `cost_usd` (its `model_call` events summed) plus `cost_source`. The SDK never prices anything — a `model_call`'s `cost_usd` is `0.0` unless the app's own instrumentation set it, and the provider client wrappers record tokens but no cost by design — so each event with tokens and no cost is priced at promotion from litellm's price table for its own `model_id` (aliases and provider prefixes resolve through the registry first). `cost_source` is `"recorded"` when every non-zero part came from the capture (a recorded cost is never re-estimated), `"estimated"` when any part was priced here, and absent with `cost_usd: 0.0` when nothing was priced — no tokens, or a model litellm does not price. An unpriced local / self-hosted model is the normal case, not a failure: it stays at 0 silently, with no warning. The example in `golden.jsonl` never carries the figure; it is provenance of the capture, not something a replay reproduces. 6. Skips captures whose replayed content duplicates an already-promoted case or an earlier capture in the same run — duplicate examples inflate *n* and corrupt the paired statistics (`--keep-duplicates` opts out). Dedup is seeded from the cases already in the suite dir, so it holds across repeated syncs. -7. Skips captures whose turn recorded an `error` event — a turn that died before the agent acted is not ground truth, and promoting it would assert `expected_no_tools: true` on a question that needed a tool. `--allow-errored` promotes it anyway (still never asserting `expected_no_tools`). `capture promote` exits non-zero on the same condition. Separately and unconditionally — `--allow-errored` does not help — a capture whose first `model_call` has no `toolset_ref` is refused: the SDK did not record what tools were offered, so there is nothing to carry, and re-capturing with a current `evalshift-sdk` is the only fix. Also unconditionally, a capture whose recorded messages carry content a text replay can't carry — an image, a document, audio, a server-tool result, an unknown block type — is skipped and counted on its own (`capture promote` refuses it): a replay without it is not ground truth. Last, as a safety net, a capture whose case fails the suite schema's own validation in a way none of these rules anticipates is skipped with pydantic's first error and counted on its own (`skipped N capture(s) that could not form a valid case`; `capture promote` refuses it) instead of aborting the sync; hand-write that case or report the capture. +7. Skips captures whose turn recorded an `error` event — a turn that died before the agent acted is not ground truth, and promoting it would assert `expected_no_tools: true` on a question that needed a tool. `--allow-errored` promotes it anyway (still never asserting `expected_no_tools`). `capture promote` exits non-zero on the same condition. Separately and unconditionally — `--allow-errored` does not help — a capture whose first `model_call` has no `toolset_ref` is refused: the SDK did not record what tools were offered, so there is nothing to carry, and re-capturing with a current `evalshift-sdk` is the only fix. Also unconditionally, a capture whose recorded messages carry content a text replay can't carry — an image, a document, audio, a server-tool result, an unknown block type — is skipped and counted on its own (`capture promote` refuses it): a replay without it is not ground truth. Also by default — `--allow-errored` does not help — a turn that was offered tools, called none, and still replies as if it had (`my notes search didn't find anything`) is skipped: promoting it would make a hallucinated action ground truth, with `expected_no_tools: true`. The check reads the reply's wording only (English first-person action phrasing — `I searched`, `I've checked`, `let me look`, `my search returned` — never a bare tool name), and a turn that recorded or requested any tool call is never flagged; an errored turn is reported as errored. It is counted on its own (`skipped N capture(s) whose reply claims a tool call it never made (use --allow-unbacked-claims or captures.allow_unbacked_claims to promote anyway)`). `--allow-unbacked-claims`, or `captures.allow_unbacked_claims: true` (`--block-unbacked-claims` overrides it for one run), promotes it with the refusal as a warning and never asserts `expected_no_tools`; `capture promote` refuses it unless given `--allow-unbacked-claims`. Sync never deletes a case file, so a case an earlier sync (or an override) promoted stays in the suite, and the skip line names it: `Its case is still in the suite from an earlier sync: delete .evalshift/suites//.json to drop it.` A case whose own recorded history carries such a reply with no tool call in its turn is promoted with a warning (`history message 2 (assistant) claims a tool action ("my notes search didn't") with no tool call in its turn — this case replays with that claim in context.`). Last, as a safety net, a capture whose case fails the suite schema's own validation in a way none of these rules anticipates is skipped with pydantic's first error and counted on its own (`skipped N capture(s) that could not form a valid case`; `capture promote` refuses it) instead of aborting the sync; hand-write that case or report the capture. 8. Warns when two captures claim the same `(conversation_id, turn_index)` (a retried turn), and when a promoted turn contains a failed tool result (`error`, or `{"success": false}`). Both stay warnings — see [Agent evals → What does not belong in a golden suite](docs/agents.md). -9. With `captures.max_age` / `captures.max_cases` set: filters, plans, retires. Captures recorded before now − `max_age`, or on another model than `defaults.source_model`, are not eligible. With `max_cases` set, sync plans each suite (every suite directory with case files, not only those with new captures; `--suite` narrows it). It fills the free slots route-balanced, a conversation at a time. It deletes the case files of the stale cases it retires and of any fresh cases it trims, and rebuilds `golden.jsonl` from what is left; reviewed cases stay. A case is known by its file, not the `name` inside it, so a renamed or copied case file is planned and deleted on its own; a deletion that fails is reported (`alpha: could not delete cap_0.json; not counted as retired.`) and not counted. A suite planned only because its directory holds case files is wired into `suites:` only if it already has an entry there. A suite whose case files cannot be read is left unchanged, with a warning (`alpha: could not read its promoted cases (ValidationError); left it unchanged.`). After the summary line, sync prints one line per suite: ` planner: 200 cases — kept 150, retired 50 (32 older than 14d, 18 on another model), added 50 of 2940 eligible; not eligible: 812 older than 14d, 40 on another model` (zero counts left out; `max_age` alone prints only ` planner: not eligible: …`). Syncing again with nothing new changes no file. If `evalshift.yaml` cannot be read, sync runs without the limits and, when the file has a top-level `captures:` key, says so: `could not read evalshift.yaml (); syncing without captures.max_age / captures.max_cases / captures.rounds (replaying the first round only).` Sync reads the file under `--offline` too. +9. With `captures.max_age` / `captures.max_cases` set: filters, plans, retires. Captures recorded before now − `max_age`, or on another model than `defaults.source_model`, are not eligible. With `max_cases` set, sync plans each suite (every suite directory with case files, not only those with new captures; `--suite` narrows it). It fills the free slots route-balanced, a conversation at a time. It deletes the case files of the stale cases it retires and of any fresh cases it trims, and rebuilds `golden.jsonl` from what is left; reviewed cases stay. A case is known by its file, not the `name` inside it, so a renamed or copied case file is planned and deleted on its own; a deletion that fails is reported (`alpha: could not delete cap_0.json; not counted as retired.`) and not counted. A suite planned only because its directory holds case files is wired into `suites:` only if it already has an entry there. A suite whose case files cannot be read is left unchanged, with a warning (`alpha: could not read its promoted cases (ValidationError); left it unchanged.`). After the summary line, sync prints one line per suite: ` planner: 200 cases — kept 150, retired 50 (32 older than 14d, 18 on another model), added 50 of 2940 eligible; not eligible: 812 older than 14d, 40 on another model` (zero counts left out; `max_age` alone prints only ` planner: not eligible: …`). Syncing again with nothing new changes no file. If `evalshift.yaml` cannot be read, sync runs without the limits and, when the file has a top-level `captures:` key, says so: `could not read evalshift.yaml (); syncing without captures.max_age / captures.max_cases / captures.rounds / captures.allow_unbacked_claims (replaying the first round only, blocking unbacked tool claims).` Sync reads the file under `--offline` too. 10. Writes `.evalshift/suites//golden.jsonl` and rewrites the managed `suites:` block in `evalshift.yaml` (between the `>>> evalshift suites` markers). 11. After the write (or after printing the block for you to paste), checks the CI pin: if a workflow under `.github/workflows/` uses `evalshift/evalshift-action` with an `evalshift-version` older than this CLI, with no pin at all, or with pins that are all newer than this CLI, it prints a warning naming the workflow and job plus the fix — the exact `evalshift-version: ""` line to set, or, for a newer pin, `pip install -U evalshift` locally. Advisory only — sync never edits a workflow and the exit code is unchanged. See [Pin drift](#pin-drift). @@ -344,7 +344,7 @@ suites: {} | `migration_policy` | block \| absent | Regression budgets, see [Migration policy](#migration-policy-and-ci-gating) | | `suites` | map | Named suites (`{name: {source: captured\|jsonl, path: ..., evaluators: ..., managed: true}}`); the block between the `>>> evalshift suites` markers is managed by `capture sync`. See [Per-suite evaluators](#per-suite-evaluators) | | `retention` | block | `max_runs_per_suite` (default 20, `0` disables), `run_ttl_days` (default off) | -| `captures` | block | `store`: `s3://`, `gs://` or `az://` URI of the object store the SDK ships captures to; `sync` and `list` mirror it locally first. Omit for local disk only. `max_age` (duration, `30m`/`24h`/`14d`) and `max_cases` (int ≥ 1), both default unset. Setting either turns bounded suites on: `max_age` filters what enters a suite and marks old cases stale, and `max_cases` caps and route-balances each suite; with either set, captures recorded on another model than `defaults.source_model` are filtered too. `rounds` (`first`, the default, or `all`): how many recorded agent rounds a case `capture sync` promotes is replayed over; `all` replays teacher-forced through the answer round, so `semantic` and `llm_judge` score the answer an agent writes after a tool call. `capture sync --rounds` overrides it for one run. See [Capturing from production](#capturing-from-production) | +| `captures` | block | `store`: `s3://`, `gs://` or `az://` URI of the object store the SDK ships captures to; `sync` and `list` mirror it locally first. Omit for local disk only. `max_age` (duration, `30m`/`24h`/`14d`) and `max_cases` (int ≥ 1), both default unset. Setting either turns bounded suites on: `max_age` filters what enters a suite and marks old cases stale, and `max_cases` caps and route-balances each suite; with either set, captures recorded on another model than `defaults.source_model` are filtered too. `rounds` (`first`, the default, or `all`): how many recorded agent rounds a case `capture sync` promotes is replayed over; `all` replays teacher-forced through the answer round, so `semantic` and `llm_judge` score the answer an agent writes after a tool call. `capture sync --rounds` overrides it for one run. `allow_unbacked_claims` (bool, default `false`): promote a turn that was offered tools, called none, and replied as if it had; off, `capture sync` skips it (`--allow-unbacked-claims` / `--block-unbacked-claims` override it for one run). See [Capturing from production](#capturing-from-production) | `thresholds` was a top-level field until it was removed. It was free-form and gated nothing — `migration_policy` is the single source of truth for gating — so rather than being silently ignored it is now rejected by name, and a config that still carries it fails to load: @@ -939,8 +939,8 @@ With a capture limit set, prints the `suite freshness` findings for a named or a **`evalshift capture list [suite]`** — `--json` · `-c/--config` · `--offline` **`evalshift capture fetch`** — `--suite` · `--since <30m|24h|7d|ISO>` · `-c/--config` (needs `captures.store`) -**`evalshift capture promote `** — `--as ` · `--suite` · `--input-var` (default `input`) · `--tag` (repeatable) · `--strict-args` · `--names-only` · `--tool-count` · `--rounds {first,all}` (default `first`) · `--allow-errored` · `-f/--force` -**`evalshift capture sync`** — `--suite` · `--input-var` · `--tag` · `--strict-args` · `--names-only` · `--tool-count` · `--rounds {first,all}` · `--allow-errored` · `-c/--config` · `-f/--force` · `--write/--print` (default write) · `--keep-duplicates` · `--since` · `--offline` · `--repair` +**`evalshift capture promote `** — `--as ` · `--suite` · `--input-var` (default `input`) · `--tag` (repeatable) · `--strict-args` · `--names-only` · `--tool-count` · `--rounds {first,all}` (default `first`) · `--allow-errored` · `--allow-unbacked-claims` · `-f/--force` +**`evalshift capture sync`** — `--suite` · `--input-var` · `--tag` · `--strict-args` · `--names-only` · `--tool-count` · `--rounds {first,all}` · `--allow-errored` · `--allow-unbacked-claims/--block-unbacked-claims` (default `captures.allow_unbacked_claims`, else block) · `-c/--config` · `-f/--force` · `--write/--print` (default write) · `--keep-duplicates` · `--since` · `--offline` · `--repair` With `captures.max_age` or `captures.max_cases` set, promotes only captures recorded on `defaults.source_model`. With `captures.max_age` set, also promotes only captures recorded within it, and its automatic fetch skips older captures (`--since` overrides). With `captures.max_cases` set, also keeps each suite a capped, route-balanced sample, retires stale cases as fresh ones replace them, and prints one `kept / retired / added` line per suite. Reads `evalshift.yaml` under `--offline` too. **`evalshift capture clean [suite]`** — `--promoted` (default) · `--all` · `-y/--yes` **`evalshift capture diff `** diff --git a/docs/agents.md b/docs/agents.md index cb90ac15..2aaa83fa 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -404,7 +404,7 @@ truth. ## What does not belong in a golden suite Ground truth is what the agent *did right*, and a capture is not automatically -that. Three shapes are actively filtered or flagged during promotion: +that. These shapes are actively filtered or flagged during promotion: - **Errored turns are refused.** A capture whose trace carries an `error` event died before the agent finished — an upstream 400, a timeout, a crash. Promoted @@ -413,6 +413,30 @@ that. Three shapes are actively filtered or flagged during promotion: `capture sync` skips it, both naming the first error. `--allow-errored` promotes it anyway; even then the case never gets `expected_no_tools: true`, because a turn that never ran is not evidence that inaction was right. +- **Turns that claim a tool call they never made are refused.** A turn that was + offered tools, called none, and still replied "I don't have that + information — my notes search didn't find anything" records an action that + never happened. Promoted, the fabricated reply becomes `expected` and the + turn asserts `expected_no_tools: true`. The check reads the reply's wording + (English first-person action phrasing such as "I searched", "let me check", + "my search returned"; a bare tool name never fires), and a turn that called + or requested any tool is never flagged: + + ```text + ⚠ skipped cap_bad: turn was offered tools and called none, but its reply claims a tool action ("my notes search didn't") — promoting it would make a hallucinated action ground truth. + ✓ promoted 1 capture(s) into 1 suite(s), skipped 1 capture(s) whose reply claims a tool call it never made (use --allow-unbacked-claims or captures.allow_unbacked_claims to promote anyway). + ``` + + `capture promote` exits non-zero with `Re-run with --allow-unbacked-claims to + promote it anyway.` For `capture sync`, `--allow-unbacked-claims` or + [`captures.allow_unbacked_claims: true`](configuration.md#captures) promotes + such turns (`--block-unbacked-claims` overrides the YAML for one run); the + refusal prints as a warning and the case never gets `expected_no_tools: true`. + `--allow-errored` does not help here, and an errored turn is reported as + errored. A case an earlier sync promoted stays in the suite — sync never + deletes a case file — and the skip line appends `Its case is still in the + suite from an earlier sync: delete .evalshift/suites//.json + to drop it.` - **Captures missing a recorded toolset are refused, unconditionally.** Every model call is required to record the toolset it was offered; a capture whose first `model_call` has no `toolset_ref` — the SDK failed to write the sidecar, diff --git a/docs/configuration.md b/docs/configuration.md index 4f6e99e5..a3fce875 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1096,6 +1096,7 @@ captures: max_age: 14d # optional: see Bounded suites below max_cases: 200 # optional, per suite rounds: all # optional: replay every recorded round, not just the first call + allow_unbacked_claims: true # optional: promote turns that claim a tool call they never made ``` | Field | Type | Default | Description | @@ -1104,6 +1105,7 @@ captures: | `max_age` | duration (`30m`, `24h`, `14d`; > 0, ≤ `36500d`) | (none) | Captures recorded longer ago than this never enter a suite through `capture sync` (`capture promote` ignores the limits), and promoted cases that old are stale: with `max_cases` also set, `capture sync` replaces them as fresh captures arrive. Also the window `capture sync` and `capture list` fetch from `store`. Omit for no age limit. | | `max_cases` | int (≥ 1) | (none) | The most cases `capture sync` keeps in each captured suite. Also caps what `capture sync` and `capture list` download from `store`: at most `3 × max_cases` captures per suite per run, a conversation at a time. Omit and suites grow with every sync. | | `rounds` | `first` \| `all` | `first` | How many recorded agent rounds a case `capture sync` promotes is replayed over. `first` replays the first model call single-shot: a turn that answers after a tool call is scored on that call alone, so `semantic` and `llm_judge` see no answer text. `all` replays every recorded round [teacher-forced](agents.md#agent-rounds-and-what-a-replay-can-reproduce), feeding back the recorded tool results, through the answer round; it costs about one call per round. `capture sync --rounds` overrides it for one run; `capture promote` takes only the flag. | +| `allow_unbacked_claims` | bool | `false` | Whether `capture sync` promotes a turn that was offered tools, called none, and still replied as if it had (`my notes search didn't find anything`). Off, it skips such a turn: promoting it would make a hallucinated action ground truth, with `expected_no_tools: true`. On, the turn is promoted with a warning and never asserts `expected_no_tools`. `capture sync --allow-unbacked-claims` / `--block-unbacked-claims` override it for one run; `capture promote` takes only `--allow-unbacked-claims`. `--allow-errored` does not apply. | `rounds` applies to the cases `capture sync` promotes from then on: sync never rewrites a promoted case unasked. When `rounds` is `all` and promoted cases still replay one round although their @@ -1116,6 +1118,14 @@ and ones promoted under another name (`capture promote --as`) are left out of th evalshift older than 1.6.0 rejects the key, so update a CI pin (`evalshift-version:`) before adding it. +`allow_unbacked_claims` judges the reply's wording only: English first-person action phrasing +(`I searched`, `I've checked`, `let me look`, `my search returned`), never a bare mention of a +tool. A turn that recorded or requested any tool call is never flagged, and an errored turn is +reported as errored. Sync never deletes a case file, so a case an earlier sync promoted from +such a capture stays in the suite; the skip line names the file to delete (`Its case is still +in the suite from an earlier sync: delete .evalshift/suites//.json to drop +it.`). A CLI released before the key rejects it; update a CI pin first. + Setting either `max_age` or `max_cases` turns [bounded suites](#bounded-suites) on. Both work without `store`: they govern how captures become suites, whether the captures came from disk or from a bucket. A `max_age` that is not a duration fails to load; `evalshift validate` reports it as @@ -1456,8 +1466,9 @@ paired statistics (`--keep-duplicates` opts out). The dedup set is seeded from the cases already in the suite dir, so re-syncing after recording more captures can't slip a duplicate past it. Captures whose messages carry an image or other non-text content are skipped too (see -[Provider message shapes](conversations.md#provider-message-shapes)), as is -a capture no valid case can be built from, and +[Provider message shapes](conversations.md#provider-message-shapes)), as are +a capture no valid case can be built from and one whose reply claims a tool +call it never made (see [`captures.allow_unbacked_claims`](#captures)), and `--repair` rebuilds cases an earlier sync built from stringified message blocks. Useful flags: `--input-var` (default `input`), `--suite ` to filter to one suite, `--tag`, diff --git a/docs/conversations.md b/docs/conversations.md index 4ed36115..23f4aac4 100644 --- a/docs/conversations.md +++ b/docs/conversations.md @@ -96,6 +96,27 @@ exchanges are absent), and the system prompt is only recovered when at least one turn in the conversation recorded a full messages list. Prefer the messages-list convention above when you can. +A turn that is not promoted — refused for any reason: an `error` event, no +recorded toolset, content a text replay can't carry, a reply claiming a tool +call it never made — also drops out of later turns' reconstructed history, so +a fabricated "my notes search didn't find anything" is never stitched into a +sibling's prefix. A turn promoted anyway (`--allow-errored`, +`--allow-unbacked-claims`) does seed it. + +Recovered history is checked for such claims too. When a turn's own messages +list carries an assistant reply that claims a tool action with no tool call +or tool result in its turn, promotion warns and still promotes — the current +turn may be fine, but the candidate will answer with that claim in context: + +```text +⚠ history message 2 (assistant) claims a tool action ("my notes search didn't") with no tool call in its turn — this case replays with that claim in context. +``` + +A `user` message directly after a `tool` message does not start a new turn: +Claude delivers tool results inside a user turn. Reconstructed history is +never checked — it carries no tool rows, so every earlier reply would look +unbacked. + With `captures.max_cases` set, a conversation's new turns enter a suite together when they fit; one larger than the suite's free slots is skipped for the next one. Once no whole conversation fits, the slots left take the diff --git a/llms-full.txt b/llms-full.txt index 0490890b..a281dd30 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -161,7 +161,7 @@ evalshift init [-f/--force] [-d/--directory DIR] [--ci] [--wire-agents/--no-wire Without --ci, warns after writing when an existing .github/workflows/*.yml pins an older or newer evalshift-version than this CLI, or none (advisory, exit 0; see CI pin drift). --ci writes the pin itself and does not warn about the file it just wrote. - Also scaffolds `captures: {max_age: 14d, max_cases: 200}` (store: and rounds: all commented out) between + Also scaffolds `captures: {max_age: 14d, max_cases: 200}` (store:, rounds: all and allow_unbacked_claims: true commented out) between migration_policy and the managed region, so new projects get bounded suites (see capture sync). Existing configs are never rewritten; doctor's `captures limits` row nudges them. PROFILE budgets (regression/critical/equivalence/arg-drift/tool-divergence/ @@ -401,13 +401,18 @@ evalshift capture fetch [--suite S] [--since 30m|24h|7d|ISO] [-c CONFIG] max_cases budget do NOT bound it; only the automatic fetch of sync and list applies them. evalshift capture promote CAPTURE_ID [--as CASE_ID] [--suite S] [--input-var NAME=input] [--tag T]... [--strict-args] [--names-only] [--tool-count] [--rounds first(default)|all] - [--allow-errored] [-f/--force] + [--allow-errored] [--allow-unbacked-claims] [-f/--force] One capture -> one golden case. History recovered only from that capture's own messages list (no cross-capture reconstruction; warns if unpromoted sibling turns exist -> use sync). EXITS 1 if the capture's trace carries an error event; --allow-errored promotes anyway. + EXITS 1 if the turn was offered tools, called none and its reply claims a tool action + (Promotion hygiene); --allow-unbacked-claims promotes anyway, --allow-errored does not. + promote reads no evalshift.yaml, so captures.allow_unbacked_claims does not apply to it. evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--strict-args] [--names-only] [--tool-count] [--rounds first|all (default captures.rounds, else first)] - [--allow-errored] [-c CONFIG] [-f] + [--allow-errored] + [--allow-unbacked-claims/--block-unbacked-claims (default captures.allow_unbacked_claims, + else block)] [-c CONFIG] [-f] [--write/--print (default write)] [--keep-duplicates] [--since 30m|24h|7d|ISO] [--offline] [--repair] captures.rounds: first|all in evalshift.yaml sets the sync default (promote takes only the @@ -472,7 +477,8 @@ evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--stri Config: sync loads evalshift.yaml under --offline too. Unreadable/invalid -> no limits; when the file has a top-level captures: key it prints (online AND offline) "could not read (); syncing without captures.max_age / captures.max_cases / captures.rounds - (replaying the first round only)." + / captures.allow_unbacked_claims (replaying the first round only, blocking unbacked tool + claims)." Staleness (one `now` per sync): too old / age-stale = recorded before now - max_age (case: captured_at, else promoted_at -- an upper bound, so a legacy case is retired later, never earlier; neither parseable -> never too old). Other model / model-stale = recorded model @@ -648,6 +654,13 @@ evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--stri no trace. Cost estimate counts one call per round; progress bar counts examples. --tool-count under --rounds all pins the total over the replayed rounds. NO flattening any more. Captures whose trace carries an error event are SKIPPED (--allow-errored promotes them). + Captures whose turn was offered tools, called none and replies claiming a tool action are + SKIPPED and counted on their own ("skipped N capture(s) whose reply claims a tool call it + never made (use --allow-unbacked-claims or captures.allow_unbacked_claims to promote + anyway)"); --allow-errored does not apply. A case already promoted from such a capture stays + (sync never deletes a case file); the skip line appends "Its case is still in the suite from + an earlier sync: delete .evalshift/suites//.json to drop it." (the path + as built from --base). Warns on duplicate (conversation_id, turn_index) and on failed tool results. Wrapper args (EXECUTED path only -- legacy captures): a capture recording a decorated function's parameters yields {"tool_args": {...}} where the model only saw the flat declared @@ -999,6 +1012,10 @@ captures: rounds: first|all = first # rounds capture sync promotes with (--rounds overrides; promote # takes the flag only). all = teacher-forced through the answer round, so semantic and # llm_judge score an agent's answer after a tool call. Applies to newly promoted cases. + allow_unbacked_claims: bool = false # capture sync promotes a turn offered tools that + # called none yet replies claiming a tool action (warning instead of skip; never + # expected_no_tools). --allow-unbacked-claims / --block-unbacked-claims override it for one + # sync; promote takes only --allow-unbacked-claims. # Setting max_age or max_cases turns bounded suites on (see capture sync): max_age filters what enters a # suite and marks old cases stale; max_cases caps and route-balances each suite; with either # set, captures/cases on another model than defaults.source_model are filtered/stale too. @@ -1675,7 +1692,8 @@ content leaves the machine. toolset_ref is copied verbatim onto the example's toolset_ref (a first-class event field, not metadata -- unlike generation_config, whose *shape* this carry mirrors). tools_offered (cheap tool-name list, also on the event) drives expected_no_tools: true only when it is - non-empty AND no tool was called AND the turn did not error -- never when nothing was + non-empty AND no tool was called AND the turn did not error AND its reply makes no + unbacked tool claim (below, even when allowed) -- never when nothing was offered, so `tool_selection`'s no-tools scoring (1.0/1.0 whenever both sides call zero tools) cannot fire on a row that measured nothing. - Promotion ALSO blocks unconditionally when the recorded messages carry content a text @@ -1689,6 +1707,28 @@ content leaves the machine. counts it separately ("skipped N capture(s) that could not form a valid case (--allow-errored does not apply)") and exits 0; promote exits 1 without suggesting --allow-errored. +- Promotion ALSO blocks an UNBACKED TOOL CLAIM: tools_offered non-empty, no tool_call event, + no model_call with requested_tool_calls, and the reply (last final_output, else last + non-empty model_call output) matches first-person action wording (English, regex in + captures/claims.py, [] optional: "I ['ve|have] [just|already] searched|checked|looked + up/through/in[to]|queried|pulled up|went through|scanned", "my|the [word] search|lookup|query + didn't|found|returned|came back|turned up|show(s|ed)", "let me check|search|look|pull up|find", "I'll check|search|look"; + a bare tool name never fires). blocked_reason "unbacked_claim", blocked "turn was offered + tools and called none, but its reply claims a tool action ("") — promoting it + would make a hallucinated action ground truth." promote exits 1 with "Re-run with + --allow-unbacked-claims to promote it anyway." (never --allow-errored); sync skips and + counts it on its own. Judged only on an otherwise-promotable turn: every earlier refusal + keeps its own reason, and an errored turn (even with --allow-errored) reports as errored, + never as this. allow_unbacked_claims (promote --allow-unbacked-claims; sync flag > + captures.allow_unbacked_claims > block) turns the refusal into a warning and the case never + gets expected_no_tools: true. +- Recorded history (the turn's own messages list, NEVER history reconstructed from sibling + turns, which has no tool rows) is scanned too: an assistant reply claiming a tool action with + no tool row or assistant tool_calls since the user message it answers WARNS (does not + block): "history message 2 (assistant) claims a tool action ("my notes search didn't") + with no tool call in its turn — this case replays with that claim in context." (several -> + "history messages 2, 4 (assistant) claim tool actions (...)"). A user row directly after a + tool row does not start a new turn (Claude delivers tool results inside a user turn). - `evalshift_cli.captures.reader.load_toolset(ref, *, base=None, cache=None) -> list[ToolSpec]` resolves a toolset sidecar (/toolsets/.json) via ToolSpec.from_dict -- NEVER via evaluators.tool_loader.load_tools, which rejects an empty list (wrong here: the empty From 55568847fa7fc7b9a9bc530d73f6d7cb34f7eb3a Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Wed, 28 Oct 2026 00:30:52 +0100 Subject: [PATCH 08/15] docs: spec and plan for unbacked tool claims Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-10-27-unbacked-tool-claims.md | 368 ++++++++++++++++++ .../2026-10-27-unbacked-tool-claims-design.md | 163 ++++++++ 2 files changed, 531 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-27-unbacked-tool-claims.md create mode 100644 docs/superpowers/specs/2026-10-27-unbacked-tool-claims-design.md diff --git a/docs/superpowers/plans/2026-10-27-unbacked-tool-claims.md b/docs/superpowers/plans/2026-10-27-unbacked-tool-claims.md new file mode 100644 index 00000000..b74b7b91 --- /dev/null +++ b/docs/superpowers/plans/2026-10-27-unbacked-tool-claims.md @@ -0,0 +1,368 @@ +# Unbacked Tool Claims Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `capture promote` / `capture sync` refuse a turn whose reply claims a tool action the model never took (it was offered tools and called none), warn when a case's verbatim history carries such a reply, and let `captures.allow_unbacked_claims` (YAML) or `--allow-unbacked-claims` (CLI) opt back in. + +**Architecture:** One pure module, `captures/claims.py`, holds the wording check (`find_tool_claim`) and the history scan (`unbacked_history_claims`). `build_example_from_capture` calls both: the first as a new last `elif` in its blocking chain (`BlockedReason` `"unbacked_claim"`), the second as a warning on recorded history. The CLI only threads the option through and routes the new reason to its own counter, hint and summary clause, exactly like `"errored"`. + +**Tech Stack:** Python 3.11+, pydantic v2, Typer + Rich, pytest, uv. + +**Spec:** `docs/superpowers/specs/2026-10-27-unbacked-tool-claims-design.md` — read it first; this plan argues from it. + +## Global Constraints + +- Python floor 3.11. Every module starts with `from __future__ import annotations`; public functions/classes get Google-style docstrings; `mypy --strict src/evalshift_cli` stays clean. +- Names, verbatim: `BlockedReason` value `"unbacked_claim"`; `PromoteOptions.allow_unbacked_claims`; `CapturesConfig.allow_unbacked_claims`; flags `--allow-unbacked-claims` (both commands) and `--block-unbacked-claims` (sync only, the negative of the pair). +- Precedence on `capture sync`: CLI flag > `captures.allow_unbacked_claims` > `False`. `capture promote` reads no config (as with `--rounds`), flag only. +- An allowed unbacked turn **never** gets `expected_no_tools: true` (mirrors `allow_errored`). +- The history scan runs on **recorded** history only (`recovered.history` inside `build_example_from_capture`), never on history `_reconstruct_history` builds — that one is text-only and would flag every reply. +- `--allow-errored` does not rescue `"unbacked_claim"` and is never suggested for it. +- `evaluator_config` / `eval_config_hash` must not change (`captures` is not in the snapshot) — no edits under `reports/` or the bundle builder. +- User-facing strings, verbatim: + - block: `turn was offered tools and called none, but its reply claims a tool action ("{phrase}") — promoting it would make a hallucinated action ground truth.` + - history warning (one per case, positions are 1-based like `repair.py:39`): `history message {n} (assistant) claims a tool action ("{phrase}") with no tool call in its turn — this case replays with that claim in context.` — several messages: `history messages {n1}, {n2} (assistant) claim tool actions ("{phrase1}", "{phrase2}") with no tool call in their turn — this case replays with those claims in context.` + - promote hint: `Re-run with [bold]--allow-unbacked-claims[/bold] to promote it anyway.` + - sync summary clause: `, skipped {n} capture(s) whose reply claims a tool call it never made (use --allow-unbacked-claims or captures.allow_unbacked_claims to promote anyway)` + - already-promoted hint (sync, appended to the skip line): ` Its case is still in the suite from an earlier sync: delete {path} to drop it.` +- Gate ("green" = all four, as CI): `uv run ruff check . && uv run ruff format --check . && uv run mypy --strict src/evalshift_cli && uv run pytest -q` (coverage floor 90). `make ci` is the exact CI mirror. +- Branch `feat/unbacked-tool-claims` (not `main`; confirm before every commit). Conventional Commits, each message ending with `Co-Authored-By: Claude Opus 5.5 `. Never `--no-verify`. +- `docs/superpowers/` is gitignored: add the spec and this plan with `git add -f` (Task 8). + +## Review Focus + +- A turn that *did* call a tool is never blocked, however its reply is worded — Task 3 `test_a_claim_backed_by_a_tool_call_is_not_blocked`, `test_a_claim_backed_by_a_requested_call_is_not_blocked`. +- Recorded history whose claims sit next to a real tool call ("Let me look." + `tool_use`, the native-shapes fixtures) raises no warning — Task 4 `test_history_claim_with_tool_call_in_its_turn_is_backed`, plus the existing native-shapes suites staying green. +- Reconstructed history never triggers the history warning, and a blocked turn drops out of later turns' reconstructed history — Task 4 `test_reconstructed_history_is_not_scanned`, `test_a_blocked_unbacked_turn_does_not_seed_reconstructed_history`. +- An errored turn with a claim reports `"errored"`, not `"unbacked_claim"` — Task 3 `test_errored_wins_over_unbacked_claim`. +- `--block-unbacked-claims` beats YAML `true` — Task 6 `test_sync_block_flag_overrides_yaml`. + +--- + +### Task 1: The wording check — `captures/claims.py` + +**Files:** +- Create: `src/evalshift_cli/captures/claims.py` +- Test: `tests/unit/test_captures_claims.py` (new) + +**Interfaces:** +- Produces: `find_tool_claim(text: str) -> str | None` (the matched phrase, verbatim from `text`). + +- [ ] **Step 1: Write the failing tests** + +```python +"""The wording check behind unbacked-tool-claim refusals.""" + +from __future__ import annotations + +import pytest + +from evalshift_cli.captures.claims import find_tool_claim + + +@pytest.mark.parametrize( + ("text", "phrase"), + [ + ( + "I don't have that information—my notes search didn't find anything specific", + "my notes search didn't", + ), + ("I searched your notes and found nothing.", "I searched"), + ("I've checked your calendar for Friday.", "I've checked"), + ("I’ve already looked through your tasks.", "I’ve already looked through"), + ("Let me check your notes for that.", "Let me check"), + ("I'll look into it.", "I'll look"), + ("The notes search returned no results.", "The notes search returned"), + ("My search didn’t turn up anything.", "My search didn’t"), + ], +) +def test_claims_are_found_and_quoted(text: str, phrase: str) -> None: + assert find_tool_claim(text) == phrase + + +@pytest.mark.parametrize( + "text", + [ + "Per your notes from Aug 20, the dentist told you to floss daily.", + "Your passport renewal is due by September 15, 2026, and it's still pending.", + "I can't search contract documents.", + "Would you like me to check?", + "Use the search bar at the top.", + "Let me know if you need anything else.", + "I looked at your question again: Friday is clear.", + "Checked items are done.", + "", + ], +) +def test_non_claims_are_ignored(text: str) -> None: + assert find_tool_claim(text) is None +``` + +- [ ] **Step 2: Run to see it fail** — `uv run pytest tests/unit/test_captures_claims.py -q` → `ModuleNotFoundError`. + +- [ ] **Step 3: Implement** + +```python +"""Spot a reply that claims a tool action, for refusing hallucinated turns at promotion. + +A captured turn that was offered tools, called none, and still says "my notes search didn't +find anything" records an action that never happened. Promoting it would make that +hallucination ground truth (``expected_no_tools: true`` plus the fabricated reply). The check is +wording only — deterministic, offline, English — and deliberately narrow: first-person action +phrasing, never a bare mention of a tool or a noun like "search bar". +""" + +from __future__ import annotations + +import re + +_APOS = "['’]" + +_CLAIM = re.compile( + rf"""\b(?: + I(?:{_APOS}ve|\s+have)?\s+(?:just\s+|already\s+)? + (?:searched|checked|looked\s+(?:up|through|in(?:to)?)|queried|pulled\s+up + |went\s+through|scanned) + | (?:my|the)\s+(?:\w+\s+)?(?:search|lookup|query)\s+ + (?:didn{_APOS}t|did\s+not|found|returned|came\s+back|turned\s+up|shows?|showed) + | let\s+me\s+(?:check|search|look|pull\s+up|find) + | I{_APOS}ll\s+(?:check|search|look) + )\b""", + re.IGNORECASE | re.VERBOSE, +) + + +def find_tool_claim(text: str) -> str | None: + """Return the first phrase in ``text`` that claims a tool action. + + Args: + text: A model reply. + + Returns: + The matched phrase exactly as it appears in ``text`` (for quoting in a message), or + ``None`` when the reply makes no such claim. + """ + match = _CLAIM.search(text) + return match.group(0) if match else None +``` + +- [ ] **Step 4: Run to see it pass**, then `uv run mypy --strict src/evalshift_cli/captures/claims.py`. + +- [ ] **Step 5: Commit** — `feat(captures): detect replies that claim a tool action`. + +--- + +### Task 2: `captures.allow_unbacked_claims` config field + +**Files:** +- Modify: `src/evalshift_cli/config/models.py` (`CapturesConfig`, ~L849-883, docstring + field) +- Modify: `src/evalshift_cli/cli/commands/init.py` (`CAPTURES_SCAFFOLD`, L177-190) +- Test: `tests/unit/test_config_captures.py` + +- [ ] **Step 1: Failing tests** (next to the `rounds` tests, L137-148): + +```python +def test_allow_unbacked_claims_defaults_to_false() -> None: + assert EvalShiftConfig(prompts=[_PROMPT]).captures.allow_unbacked_claims is False + + +def test_allow_unbacked_claims_accepts_true() -> None: + cfg = EvalShiftConfig(prompts=[_PROMPT], captures={"allow_unbacked_claims": True}) + assert cfg.captures.allow_unbacked_claims is True + + +def test_a_misspelt_allow_unbacked_claims_fails_loudly() -> None: + with pytest.raises(ValidationError, match="allow_unbacked_claim"): + EvalShiftConfig(prompts=[_PROMPT], captures={"allow_unbacked_claim": True}) +``` + +- [ ] **Step 2: Run, see the first two fail** (the third already passes via `extra="forbid"` — keep it as a guard). + +- [ ] **Step 3: Implement** — `allow_unbacked_claims: bool = False` after `rounds`, and an `Attributes:` docstring entry: "Promote turns whose reply claims a tool call the model never made (offered tools, called none, says it searched). Off by default: such a turn records a hallucinated action, which is not ground truth. ``capture sync --allow-unbacked-claims`` / ``--block-unbacked-claims`` override it for one sync." + +- [ ] **Step 4: Scaffold line.** In `CAPTURES_SCAFFOLD`, extend the comment with one sentence — `A turn whose reply claims a tool call it never made is skipped; allow_unbacked_claims: true promotes it anyway.` — and add ` # allow_unbacked_claims: true` after `# rounds: all`. Run `uv run pytest tests/unit/test_init.py tests/unit/test_config_suites.py -q` (they import the constant; confirm nothing pins the old text byte-for-byte). + +- [ ] **Step 5: Green + commit** — `feat(config): add captures.allow_unbacked_claims`. + +--- + +### Task 3: Block the turn — `BlockedReason` `"unbacked_claim"` + +**Files:** +- Modify: `src/evalshift_cli/captures/promote.py` (`BlockedReason` L64, `PromoteOptions` L70, `BuiltExample` docstring L117, blocking chain L319-381, `expected_no_tools` L440-447, `_turn_assistant_text` L1321) +- Test: `tests/unit/test_captures_promote.py` + +- [ ] **Step 1: Test helper.** Give `_model_call` (L85) an `output: str = "out"` kwarg written to `event["output"]`. + +- [ ] **Step 2: Failing tests** — new `class TestUnbackedClaim`: + +```python +_CLAIM_REPLY = "I don't have that—my notes search didn't find anything." + + +def _claim_events(**model_call: Any) -> list[dict[str, Any]]: + return [_model_call(0, model_input="PO number on the Acme contract?", **model_call), + _final(_CLAIM_REPLY, 1)] + + +class TestUnbackedClaim: + def test_a_tool_free_turn_claiming_a_search_is_blocked(self) -> None: + built = build_example_from_capture(_envelope(events=_claim_events()), PromoteOptions()) + assert built.blocked_reason == "unbacked_claim" + assert "my notes search didn't" in (built.blocked or "") + + def test_a_claim_backed_by_a_tool_call_is_not_blocked(self) -> None: + events = [*_claim_events()[:1], _tool_call("search_orders", 1, args={}), + _final(_CLAIM_REPLY, 2)] + assert build_example_from_capture(_envelope(events=events), PromoteOptions()).blocked is None + + def test_a_claim_backed_by_a_requested_call_is_not_blocked(self) -> None: + events = _claim_events( + requested_tool_calls=[{"name": "search_orders", "arguments": {}}] + ) + assert build_example_from_capture(_envelope(events=events), PromoteOptions()).blocked is None + + def test_no_tools_offered_is_not_blocked(self) -> None: + events = _claim_events(tools_offered=[]) + assert build_example_from_capture(_envelope(events=events), PromoteOptions()).blocked is None + + def test_a_plain_tool_free_answer_is_not_blocked(self) -> None: + events = [_model_call(0, model_input="q"), _final("Friday is clear.", 1)] + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert built.blocked is None + assert built.example.expected_no_tools is True + + def test_the_model_call_output_is_checked_without_a_final_event(self) -> None: + events = [_model_call(0, model_input="q", output=_CLAIM_REPLY)] + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert built.blocked_reason == "unbacked_claim" + + def test_errored_wins_over_unbacked_claim(self) -> None: + events = [*_claim_events(), _error("boom", 2)] + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert built.blocked_reason == "errored" + + def test_allow_errored_does_not_rescue_it(self) -> None: + built = build_example_from_capture( + _envelope(events=_claim_events()), PromoteOptions(allow_errored=True) + ) + assert built.blocked_reason == "unbacked_claim" + + def test_allowed_it_promotes_with_a_warning_and_no_expected_no_tools(self) -> None: + built = build_example_from_capture( + _envelope(events=_claim_events()), PromoteOptions(allow_unbacked_claims=True) + ) + assert built.blocked is None + assert any("my notes search didn't" in w for w in built.warnings) + assert built.example.expected_no_tools is False +``` + +(`RequestedToolCall` fields are `name` / `arguments`, `traces/models.py:75`; add an `id` if its validator requires one.) + +- [ ] **Step 3: Run, see them fail** (`TypeError` on the new option first). + +- [ ] **Step 4: Implement.** + - `BlockedReason`: add `"unbacked_claim"`; `BuiltExample` docstring: one sentence for it ("rescued only by `allow_unbacked_claims`, never by `--allow-errored`"). + - `PromoteOptions.allow_unbacked_claims: bool = False` + docstring entry mirroring `allow_errored`. + - Extract `_reply_text(events: list[Any]) -> str | None` from `_turn_assistant_text` (same rule: last `FinalOutputEvent`, else last non-empty `ModelCallEvent.output`); make `_turn_assistant_text(record)` return `_reply_text(record.envelope.trace.events)`. + - After the `elif errors:` branch, a new block (not `elif` of it, so the `allow_errored` warning path still falls through): compute `unbacked = _unbacked_claim(events, tools_offered)` where the helper returns the phrase when `tools_offered` is non-empty, no `ToolCallEvent` exists, no `ModelCallEvent.requested_tool_calls` is non-empty, and `find_tool_claim(_reply_text(events) or "")` matches. If `blocked is None and unbacked is not None`: build the verbatim block message; with `opts.allow_unbacked_claims` append it to `warnings`, else set `blocked` / `blocked_reason = "unbacked_claim"`. + - `expected_no_tools = bool(tools_offered) and not errors and unbacked is None`. + +- [ ] **Step 5: Full suite** — `uv run pytest -q`. Any existing fixture now blocked is a real collision: read it, and either change its reply text (if the test is not about wording) or record why it is a true positive. Do not loosen the regex to dodge a fixture. + +- [ ] **Step 6: Green + commit** — `feat(captures): refuse to promote a turn that claims a tool call it never made`. + +--- + +### Task 4: Warn on recorded history that carries a claim + +**Files:** +- Modify: `src/evalshift_cli/captures/claims.py` (add `unbacked_history_claims`) +- Modify: `src/evalshift_cli/captures/promote.py` (`build_example_from_capture`, after `_context_loss_warning` ~L220) +- Test: `tests/unit/test_captures_claims.py`, `tests/unit/test_captures_promote.py` + +**Interfaces:** +- Produces: `unbacked_history_claims(history: Sequence[ChatMessage]) -> list[tuple[int, str]]` — `(1-based position, phrase)` per unbacked claiming assistant message. + +- [ ] **Step 1: Failing unit tests** (`test_captures_claims.py`), building `ChatMessage` / `HistoryToolCall` directly: + - `user → assistant("my notes search didn't find it")` → `[(2, "my notes search didn't")]`. + - `user → assistant(tool_calls=[…], content="Let me look.") → tool → assistant("I searched and found X")` → `[]` (`test_history_claim_with_tool_call_in_its_turn_is_backed`). + - `system → user → assistant("Per your notes…")` → `[]`. + - two unbacked claims in different turns → both, in order. + +- [ ] **Step 2: Implement.** Walk the history keeping `backed = False`, reset on each `user` message; set `backed = True` on a `tool` message or an `assistant` with `tool_calls`; for an `assistant` with non-empty `content`, no `tool_calls` and `not backed`, record `find_tool_claim(content)` when it matches. + +- [ ] **Step 3: Wire it in.** In `build_example_from_capture`, when `recovered.history` is not `None` and the scan is non-empty, append the one aggregated warning (verbatim strings in Global Constraints). Nothing in `build_conversation_examples` changes. + +- [ ] **Step 4: Failing-then-passing promote tests** (`test_captures_promote.py`, use `_messages_call` with `tools_offered` set): + - `test_recorded_history_with_an_unbacked_claim_warns` — the turn-10 shape: recorded messages whose assistant reply at position 2 claims a search with no tool rows; warning names `history message 2`; `built.blocked is None`. + - `test_reconstructed_history_is_not_scanned` — a two-turn conversation of bare-string inputs (no recorded messages) where turn 0's reply is "I checked your calendar" *with* a tool call; turn 1's reconstructed history has no warning. + - `test_a_blocked_unbacked_turn_does_not_seed_reconstructed_history` — turn 0 blocked as `"unbacked_claim"`; turn 1's reconstructed history does not contain turn 0's user text (existing rule at `promote.py:1474-1479`). + +- [ ] **Step 5: Full suite** (native-shapes / repair suites carry "Let me look." next to a `tool_use` — they must stay green), green, commit — `feat(captures): warn when a case's recorded history carries an unbacked tool claim`. + +--- + +### Task 5: `capture promote --allow-unbacked-claims` + +**Files:** +- Modify: `src/evalshift_cli/cli/commands/capture.py` (`promote` options ~L977, `PromoteOptions(...)` ~L1010, blocked hint ~L1027-1034) +- Test: `tests/unit/test_cli_capture.py` + +- [ ] **Step 1: Failing tests** with a helper `_unbacked_events()` (copy `_errored_events` L1289: `tools_offered: ["search_orders"]`, `output: ""`, then a `final_output` with the claim reply, no error): + - `test_promote_refuses_an_unbacked_claim` — exit 1, stdout has `claims a tool action` and `--allow-unbacked-claims`, not `--allow-errored`. + - `test_promote_allow_unbacked_claims_promotes_without_expected_no_tools` — mirror L1404. + +- [ ] **Step 2: Implement** — the plain `bool` option (help: "Promote even if the turn's reply claims a tool call it never made. The case still never asserts expected_no_tools."), thread into `PromoteOptions`, and an `elif built.blocked_reason == "unbacked_claim":` printing the verbatim promote hint. + +- [ ] **Step 3: Green + commit** — `feat(cli): capture promote --allow-unbacked-claims`. + +--- + +### Task 6: `capture sync` — YAML + flag pair, counter, summary, stale-case hint + +**Files:** +- Modify: `src/evalshift_cli/cli/commands/capture.py` (`_SyncLimits` L595-604 + `_sync_limits` L642, sync options ~L1428, resolution ~L1494, `PromoteOptions(...)` ~L1525, counters ~L1544, blocked dispatch L1759-1781, summary ~L1972-1996, sync docstring) +- Test: `tests/unit/test_cli_capture.py` + +- [ ] **Step 1: Failing tests** (reuse `_unbacked_events`, `_write_min_config`; for YAML, append `"captures:\n allow_unbacked_claims: true\n"` to the written config): + - `test_sync_skips_unbacked_claims` — golden ids `{"cap_ok"}`, summary has the verbatim clause. + - `test_sync_allow_flag_promotes_them`. + - `test_sync_yaml_true_promotes_them`. + - `test_sync_block_flag_overrides_yaml` — YAML `true` + `--block-unbacked-claims` → skipped. + - `test_sync_summary_does_not_offer_allow_errored_for_unbacked`. + - `test_sync_names_the_stale_case_of_a_now_blocked_capture` — promote `cap_bad` first with `capture promote --allow-unbacked-claims` (case named after the capture), then plain `sync`: the skip line names `suites/alpha/cap_bad.json` with the verbatim already-promoted hint, and the file is still there (sync never deletes it). + +- [ ] **Step 2: Implement.** + - `_SyncLimits.allow_unbacked_claims: bool = False` (comment like the `rounds` one), filled from `cfg.captures.allow_unbacked_claims`; mention it in the unreadable-config warning's list. + - Option `allow_unbacked_claims: bool | None` with `typer.Option("--allow-unbacked-claims/--block-unbacked-claims", help="… Defaults to captures.allow_unbacked_claims in evalshift.yaml, else block.")`, default `None`; resolve `limits.allow_unbacked_claims if allow_unbacked_claims is None else allow_unbacked_claims` next to `replay_rounds`. + - `skipped_unbacked` counter, routed in the dispatch before the `else` (errored) branch; summary clause after `skipped_errored`'s. Update the dispatch comment's list. + - Stale-case hint: build once per suite a `{case.from_capture: path}` map from `iter_promoted_case_files(promoted_suite_dir(suite_name, base=base))` (only when a suite has an `"unbacked_claim"` refusal, to keep the happy path free); when the blocked capture has an entry, append the hint with the path relative to the base. + - Sync docstring: one sentence on unbacked claims and the override. + +- [ ] **Step 3: Green + commit** — `feat(cli): capture sync skips unbacked tool claims, with YAML and flag overrides`. + +--- + +### Task 7: Docs and changelog + +**Files:** `docs/configuration.md` (`captures:` block), `DOCS.md` (config schema `captures:`, sync step list ~L190, flag lists L942-943), `llms-full.txt` (config reference, promote/sync flags ~L404-410, sync skip rules ~L650, promotion block rules ~L1667-1691), `docs/agents.md` (~L413 skip list), `docs/conversations.md` (the later-turn history warning and that a blocked turn drops out of reconstructed history), `CHANGELOG.md` `## [Unreleased]` → `### Added`. + +- [ ] **Step 1:** Write each entry from the shipped behaviour (run the commands; quote real output), not from the spec. The changelog entry names the flag pair, the YAML key, that already-promoted cases are left in place with a hint, and that `--allow-errored` does not apply. +- [ ] **Step 2:** `grep -n "allow-errored" DOCS.md llms-full.txt docs/*.md` — every place listing promotion refusals now lists this one too. +- [ ] **Step 3: Commit** — `docs: unbacked tool claims at capture promotion`. + +--- + +### Task 8: Verify end to end, then review + +- [ ] **Step 1: Gate** — `make ci` and `pre-commit run --all-files`; show the output. +- [ ] **Step 2: Real data, read-only.** Copy `/Users/lukas/repos/testApp/.evalshift/{captures,toolsets}` and `evalshift.yaml` into a scratch dir (never write into testApp), then `uv run evalshift capture sync --offline --base /.evalshift --config /evalshift.yaml`. Expect: `cap_8e2ceeeceb084f1ca8c8b9a9336c0b5e` skipped with `("my notes search didn't")`; `cap_66c697d76a4b4961970a2f0ace04cb94` promoted with `history message 38 (assistant)`; no other planner/converter capture blocked. Repeat with `captures.allow_unbacked_claims: true` in the copied YAML → it is promoted, `expected_no_tools` false. +- [ ] **Step 3:** `git add -f docs/superpowers/specs/2026-10-27-unbacked-tool-claims-design.md docs/superpowers/plans/2026-10-27-unbacked-tool-claims.md`; commit — `docs: spec and plan for unbacked tool claims`. +- [ ] **Step 4:** superpowers:requesting-code-review over the branch diff (skill unavailable → a fresh reviewer subagent given the spec and the diff, not this plan's conclusions). + +## Out of scope + +- An LLM classifier, non-English phrasing, replay-time grounding evaluation (spec Non-goals). +- Deleting already-promoted cases automatically — sync only names them (Task 6). +- `evalshift-sdk`, `evalshift-action`, `evalshift-server`: no change. diff --git a/docs/superpowers/specs/2026-10-27-unbacked-tool-claims-design.md b/docs/superpowers/specs/2026-10-27-unbacked-tool-claims-design.md new file mode 100644 index 00000000..5ed0438d --- /dev/null +++ b/docs/superpowers/specs/2026-10-27-unbacked-tool-claims-design.md @@ -0,0 +1,163 @@ +# Unbacked tool claims: block hallucinated turns at promotion + +**Date:** 2026-10-27 · **Status:** decisions approved in chat, spec for maintainer review · **Repos:** evalshift-cli (behaviour + docs); evalshift-sdk, evalshift-action, evalshift-server (nothing) + +## Problem + +A captured turn can record a reply that *claims* a tool action the model never took. Found in +testApp's planner suite (`r_20261027_planner_47f8ef`), conversation +`planner:dee6c8bd-…`, turn 9: + +- user: "What's the PO number on the Acme contract?" +- production `claude-sonnet-5`, offered `get_schedule` / `get_tasks` / `search_notes`, made **one** + model call with `requested_tool_calls: []` and replied "I don't have that information—**my notes + search didn't find anything** specific…" + +`capture sync` promoted it as-is, so today: + +1. **The hallucination becomes ground truth.** The case gets `expected_no_tools: true` + (`captures/promote.py:440-447`) and the fabricated reply as `expected.final_output`. A + candidate that correctly calls `search_notes` fails `tool_selection` conformance. On replay + both models *did* search — the production turn was a one-off. +2. **It poisons later turns silently.** Turn 10's verbatim history carries the fabricated reply + as message 38. The target model, seeing an unanswered question and an unverified "I searched", + re-ran `search_notes("Acme PO")` on turn 10, which surfaced in the report as a puzzling + off-topic call and a 0.5 routing score — with nothing pointing at the cause. + +No evaluator catches this either: `llm_judge` never sees the turn's own tool calls +(`evaluators/llm_judge.py:113`), and no `trace_invariants` rule type can condition on reply text. + +## Decisions (approved in chat) + +| Question | Decision | +| --- | --- | +| Flagged turn | **Block** like an errored turn, with an opt-in `--allow-unbacked-claims` override. | +| Persistent setting | **`captures.allow_unbacked_claims`** in `evalshift.yaml`, overridable per sync. | +| Later turns carrying it in history | **Warn**, naming the history position; still promote. | +| Detection | **Deterministic wording check only.** No LLM, no cost, offline. | + +## Design + +### 1. Detector — new `captures/claims.py` + +```python +def find_tool_claim(text: str) -> str | None: + """Return the first phrase in ``text`` that claims a tool action, else ``None``.""" +``` + +One compiled, case-insensitive regex over first-person action phrasing. Starting set, validated +against testApp's three tool-free captures (1 true positive, 0 false positives): + +- past action: `I('ve| have)? (just )?(searched|checked|looked (up|through|at|in)|queried|pulled up|went through|scanned)` +- reported result: `(my|the) ( )?(search|lookup|query) (didn't|did not|found|returned|came back|turned up|shows?)` +- promise with nothing behind it: `let me (check|search|look|pull up|find)`, `I'll (check|search|look)` + +Returns the matched phrase so messages can quote it. English only (non-goal below). + +### 2. Current turn — new `BlockedReason` `"unbacked_claim"` + +In `build_example_from_capture`, as the **last** `elif` of the blocking chain (after `errors`, so +an errored turn keeps reporting as errored). It fires when all hold: + +- the first model call offered tools (`tools_offered` non-empty); +- the turn recorded no `tool_call` events and no model call has `requested_tool_calls`; +- `find_tool_claim()` matches, where the reply is the text + `_turn_assistant_text` already selects (last `FinalOutputEvent`, else last non-empty + `ModelCallEvent.output`). + +Message (exact wording settled in implementation): + +> turn was offered tools and called none, but its reply claims one ("my notes search didn't") — +> promoting it would make a hallucinated action ground truth. + +`PromoteOptions.allow_unbacked_claims: bool = False`. With it set, the note becomes a warning, +and — mirroring `allow_errored` — the case **never gets `expected_no_tools: true`**: a turn that +claimed a search is not evidence that calling nothing was right. + +### 3. Later turns — warning on verbatim history + +Also in `build_example_from_capture`, when `recovered.history` is not `None` (the capture's own +recorded messages, which keep `tool_calls` / `tool` rows): for each assistant message with text +and no `tool_calls`, it is *backed* if any `tool` message or assistant-with-`tool_calls` sits +between it and the preceding user message. An unbacked one whose text matches `find_tool_claim` +adds one aggregated warning: + +> history message 38 (assistant) claims a tool action ("my notes search didn't") with no tool +> call in its turn — this case replays with that claim in context. + +**Reconstructed history needs nothing new**: it is text-only, so the structural scan would flag +every past reply — it must *not* run there. It is already safe: a blocked turn never seeds later +turns' reconstructed history (`promote.py:1474-1479`), so blocking turn 9 removes it. + +### 4. Config — `captures.allow_unbacked_claims` + +New field on `CapturesConfig` (`config/models.py:849`), following the `captures.rounds` +precedent (a sync knob that lives in YAML with a per-run CLI override): + +```yaml +captures: + # Promote turns whose reply claims a tool call the model never made. Off by default: + # such a turn records a hallucinated action, which is not ground truth. + allow_unbacked_claims: false +``` + +- `allow_unbacked_claims: bool = False`; `extra="forbid"` already rejects typos. +- Read by `capture sync` only, alongside `rounds` in the limits read (`capture.py:603-642`). + `capture promote` does not read `evalshift.yaml` today (same as `--rounds`), so it stays + flag-only. +- **Hosted / hash impact: none.** `captures` is not part of the `evaluator_config` snapshot + (`../evalshift-server/BUNDLE_SPEC.md#evaluator-config`), so bundles and `eval_config_hash` + are unchanged. + +### 5. CLI + +- `capture sync`: `--allow-unbacked-claims/--block-unbacked-claims`, default `None` = use + `captures.allow_unbacked_claims` (else `false`); either flag overrides it for one sync — the + `--rounds` resolution pattern (`capture.py:1494`). The pair lets a sync re-check a project that + set `true` in YAML. +- `capture promote`: `--allow-unbacked-claims` (plain bool, help text parallel to + `--allow-errored`). On `blocked_reason == "unbacked_claim"`, print "Re-run with + `--allow-unbacked-claims` to promote it anyway." (`capture.py:1032` pattern). +- `capture sync` summary: new `skipped_unbacked` counter in the `blocked_reason` dispatch + (`capture.py:1771`) and clause `skipped N capture(s) whose reply claims a tool call it never + made (use --allow-unbacked-claims or captures.allow_unbacked_claims to promote anyway)`. + `--allow-errored` does not apply. + +### 6. Docs (same change) + +`docs/configuration.md` (`captures:` block — required for a new config field), `DOCS.md` +(config schema `captures:` entry, sync step list ~L190, flag lists L942-943), `llms-full.txt` +(config reference, promote/sync flags +~L404-410, sync skip rules ~L650, promotion block rules ~L1667-1691), `docs/agents.md` (~L413 +skip list), `docs/conversations.md` (later-turn history warning), `CHANGELOG.md` `[Unreleased]`. + +## Tests (TDD, `tests/unit/`) + +- `test_captures_claims.py` (new): positives — "my notes search didn't find anything", "I + searched your notes", "I've checked your calendar", "Let me check your notes for that."; + negatives — "Per your notes from Aug 20…", "Your passport renewal is due by September 15", + "I can't search contract documents", "Would you like me to check?", "the search bar", + empty string. +- `test_captures_promote.py`: blocks with `"unbacked_claim"` on the turn-9 shape; not blocked + when the turn called a tool, when no tools were offered, or when the reply makes no claim; + errored + claim reports `"errored"`; `allow_unbacked_claims` promotes with a warning and + `expected_no_tools` false; verbatim-history warning names the right position and does not fire + for a claim backed by a tool call in its turn; reconstructed history drops the blocked turn and + raises no history warning. +- CLI tests for both commands: exit 1 + hint on `promote`, summary clause on `sync`, flag rescues. +- Config (`tests/unit/test_config*.py`): `captures.allow_unbacked_claims` defaults to `false`, + accepts `true`, a misspelt key fails loudly. +- Sync resolution: YAML `true` promotes; YAML `true` + `--block-unbacked-claims` blocks; YAML + absent + `--allow-unbacked-claims` promotes. + +## Non-goals / risks + +- **False positives** — "As I checked earlier, …" referring to a real call in a prior turn would + block. Accepted: the flag rescues it and the message quotes the phrase. Revisit if reports land. +- **Misses** — unusual phrasings and non-English replies pass. An opt-in LLM classifier was + considered and declined for now. +- **Existing goldens stay.** Resolved while planning: sync's blocked check (`capture.py:1759`) + runs before its already-promoted check (`:1794`), so a newly blocked capture is reported as + skipped but its earlier case file is left in the suite, `--force` or not. Sync never deletes + it; the skip line names the file to delete (plan Task 6). +- Replay-time grounding evaluation (judge given the turn's own calls/results) is a separate idea. From 4aef6d4d3da30feac76ea72ac53dd9fd32336645 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Wed, 28 Oct 2026 00:39:32 +0100 Subject: [PATCH 09/15] fix(captures): narrow tool-claim detection to past-tense claims Promises ("let me check", "I'll search your notes once you confirm") are how a model asks a clarifying question before acting -- exactly the turn where calling no tool is right -- so they no longer count as claims. A reported result counts only as "my" search/lookup/query: "the query didn't include a WHERE clause" is as often about the user's query. Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/claims.py | 18 +++++++++++++----- tests/unit/test_captures_claims.py | 26 +++++++++++++++++++++----- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/evalshift_cli/captures/claims.py b/src/evalshift_cli/captures/claims.py index 940514ed..095da628 100644 --- a/src/evalshift_cli/captures/claims.py +++ b/src/evalshift_cli/captures/claims.py @@ -3,8 +3,18 @@ A captured turn that was offered tools, called none, and still says "my notes search didn't find anything" records an action that never happened. Promoting it would make that hallucination ground truth (``expected_no_tools: true`` plus the fabricated reply). The check is -wording only — deterministic, offline, English — and deliberately narrow: first-person action -phrasing, never a bare mention of a tool or a noun like "search bar". +wording only — deterministic, offline, English — and deliberately narrow: a first-person claim +that an action already *happened* ("I searched", "I've checked", "my notes search didn't …"), +never a bare mention of a tool or a noun like "search bar". + +Promises are not claims: "let me check" or "I'll search your notes once you confirm" is how a +model asks a clarifying question before acting -- exactly the turn where calling no tool is +right. A reported result counts only as *my* search/lookup/query: "the query didn't include a +WHERE clause" is about the user's query as often as the model's. + +Known false positive, accepted: past-tense review of content the user pasted ("I checked the +code you pasted") reads exactly like a claimed lookup. The refusal quotes the phrase and +``--allow-unbacked-claims`` rescues the turn. """ from __future__ import annotations @@ -21,10 +31,8 @@ I(?:{_APOS}ve|\s+have)?\s+(?:just\s+|already\s+)? (?:searched|checked|looked\s+(?:up|through|in(?:to)?)|queried|pulled\s+up |went\s+through|scanned) - | (?:my|the)\s+(?:\w+\s+)?(?:search|lookup|query)\s+ + | my\s+(?:\w+\s+)?(?:search|lookup|query)\s+ (?:didn{_APOS}t|did\s+not|found|returned|came\s+back|turned\s+up|shows?|showed) - | let\s+me\s+(?:check|search|look|pull\s+up|find) - | I{_APOS}ll\s+(?:check|search|look) )\b""", re.IGNORECASE | re.VERBOSE, ) diff --git a/tests/unit/test_captures_claims.py b/tests/unit/test_captures_claims.py index 3867b71b..d7c6b91b 100644 --- a/tests/unit/test_captures_claims.py +++ b/tests/unit/test_captures_claims.py @@ -18,10 +18,11 @@ ("I searched your notes and found nothing.", "I searched"), ("I've checked your calendar for Friday.", "I've checked"), ("I’ve already looked through your tasks.", "I’ve already looked through"), # noqa: RUF001 — curly apostrophe - ("Let me check your notes for that.", "Let me check"), - ("I'll look into it.", "I'll look"), - ("The notes search returned no results.", "The notes search returned"), ("My search didn’t turn up anything.", "My search didn’t"), # noqa: RUF001 — curly apostrophe + # Known, accepted false positive: past-tense review of content the user pasted reads + # exactly like a claimed lookup. Wording alone can't tell them apart; the turn is + # rescuable with --allow-unbacked-claims, and the refusal quotes the phrase. + ("I checked the code you pasted and it looks fine.", "I checked"), ], ) def test_claims_are_found_and_quoted(text: str, phrase: str) -> None: @@ -39,6 +40,21 @@ def test_claims_are_found_and_quoted(text: str, phrase: str) -> None: "Let me know if you need anything else.", "I looked at your question again: Friday is clear.", "Checked items are done.", + # A promise is not a claim that an action happened -- and a clarifying question + # before acting is exactly the turn where calling no tool is correct. + "Let me check your notes for that.", + "I'll look into it.", + "Let me know the date and I'll check your calendar.", + "Once you confirm the project name, I'll search your notes.", + "Sure — let me find a time that works. What duration do you need?", + "Let me check I understand: you want the Friday slot moved?", + "I'll look forward to it!", + "Let me check: 2+2=4.", + # Only *my* search reports a result the reply claims for itself; "the" search/query + # is as often the user's or a piece of code under discussion. + "The notes search returned no results.", + "The query didn't include a WHERE clause.", + "The search didn't match because the term is misspelled.", "", ], ) @@ -91,9 +107,9 @@ def test_unbacked_claims_in_different_turns_are_all_reported_in_order() -> None: _user("PO number?"), _assistant("I searched your notes and found nothing."), _user("Friday?"), - _assistant("Let me check your calendar."), + _assistant("I've checked your calendar: it's clear."), ] - assert unbacked_history_claims(history) == [(2, "I searched"), (4, "Let me check")] + assert unbacked_history_claims(history) == [(2, "I searched"), (4, "I've checked")] def test_a_new_user_turn_resets_backing() -> None: From ecb130cbff0d39045f1ed3fbeb044146d6eb2f7c Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Wed, 28 Oct 2026 00:40:12 +0100 Subject: [PATCH 10/15] fix(captures): treat tool results, retrieval and agent loops as backing A turn that recorded a tool result, a retrieval (a RAG answer did search), or more than one model call (an agent loop: a second call only follows something fed back, even in a pre-2.1.0 capture with no requested_tool_calls) is no longer flagged as an unbacked tool claim. Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/promote.py | 16 +++++++---- tests/unit/test_captures_promote.py | 40 +++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/src/evalshift_cli/captures/promote.py b/src/evalshift_cli/captures/promote.py index 0528156d..6fb3f427 100644 --- a/src/evalshift_cli/captures/promote.py +++ b/src/evalshift_cli/captures/promote.py @@ -47,6 +47,7 @@ FinalOutputEvent, ModelCallEvent, RequestedToolCall, + RetrievalEvent, ToolCallEvent, ToolResultEvent, ) @@ -1408,9 +1409,13 @@ def _reply_text(events: list[Any]) -> str | None: def _unbacked_claim(events: list[Any], tools_offered: Sequence[str] | None) -> str | None: """The phrase a tool-free turn's reply uses to claim a tool action, if any. - Fires only when the turn could have acted and didn't: tools were offered, - no ``tool_call`` event was recorded, and no model call requested one. A turn - that did call a tool is never flagged, however its reply is worded. + Fires only when the turn could have acted and shows no sign it did: tools were + offered, and the turn recorded no ``tool_call`` or ``tool_result`` event, no + ``retrieval`` event (a RAG answer did search), no model call requesting a tool, + and only one model call -- a second one happens only after something was fed + back, which is the only trace of an agent loop in a pre-2.1.0 capture with no + ``requested_tool_calls``. A turn with any of that evidence is never flagged, + however its reply is worded. Returns: The matched phrase (see :func:`~evalshift_cli.captures.claims.find_tool_claim`), @@ -1418,9 +1423,10 @@ def _unbacked_claim(events: list[Any], tools_offered: Sequence[str] | None) -> s """ if not tools_offered: return None - if any(isinstance(e, ToolCallEvent) for e in events): + if any(isinstance(e, ToolCallEvent | ToolResultEvent | RetrievalEvent) for e in events): return None - if any(isinstance(e, ModelCallEvent) and e.requested_tool_calls for e in events): + model_calls = [e for e in events if isinstance(e, ModelCallEvent)] + if len(model_calls) > 1 or any(call.requested_tool_calls for call in model_calls): return None return find_tool_claim(_reply_text(events) or "") diff --git a/tests/unit/test_captures_promote.py b/tests/unit/test_captures_promote.py index ba2fb537..f0d9f597 100644 --- a/tests/unit/test_captures_promote.py +++ b/tests/unit/test_captures_promote.py @@ -614,6 +614,46 @@ def test_a_claim_backed_by_a_requested_call_is_not_blocked(self) -> None: built = build_example_from_capture(_envelope(events=events), PromoteOptions()) assert built.blocked is None + def test_a_claim_backed_by_a_tool_result_is_not_blocked(self) -> None: + """A recorded tool result means a tool ran, even when the app logged no + ``tool_call`` event and the model call carries no ``requested_tool_calls``.""" + events = [ + _claim_events()[0], + _tool_result("search_orders", 1, result="nothing"), + _final(_CLAIM_REPLY, 2), + ] + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert built.blocked is None + + def test_a_claim_backed_by_retrieval_is_not_blocked(self) -> None: + """A RAG turn searched its notes through retrieval, not a tool: the claim is true.""" + events = [ + _claim_events()[0], + { + "type": "retrieval", + "sequence_index": 1, + "timestamp": "2026-06-16T12:00:01+00:00", + "metadata": {}, + "source": "notes", + "query": "Acme PO", + "documents": [], + }, + _final(_CLAIM_REPLY, 2), + ] + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert built.blocked is None + + def test_an_agent_loop_is_not_blocked(self) -> None: + """A second model call only happens after something was fed back -- a pre-2.1.0 + capture records no ``requested_tool_calls``, so the loop is the only trace of it.""" + events = [ + _claim_events()[0], + _model_call(1, model_input="PO number on the Acme contract?", output=_CLAIM_REPLY), + _final(_CLAIM_REPLY, 2), + ] + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert built.blocked is None + def test_no_tools_offered_is_not_blocked(self) -> None: """With nothing offered the turn could not have called a tool, so the check has no "should have" to measure -- leave it to the reviewer.""" From 71367bb70228aba67caf0906c383bb7540e52547 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Wed, 28 Oct 2026 00:41:12 +0100 Subject: [PATCH 11/15] fix(captures): read past a blank final_output for a turn's reply A blank final_output event is no reply: the claim check (and history reconstruction, which shares the rule) now falls back to the last non-blank model-call output. With nothing to fall back to, the blank final still counts as the reply, so reconstruction keeps its empty assistant message as before. Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/promote.py | 19 ++++++--- tests/unit/test_captures_promote.py | 59 +++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/src/evalshift_cli/captures/promote.py b/src/evalshift_cli/captures/promote.py index 6fb3f427..d2ace91b 100644 --- a/src/evalshift_cli/captures/promote.py +++ b/src/evalshift_cli/captures/promote.py @@ -1385,8 +1385,7 @@ def _turn_current_text(built: BuiltExample, opts: PromoteOptions) -> str: def _turn_assistant_text(record: CaptureRecord) -> str | None: """The text a reconstructed history entry should use for a prior turn's reply. - Prefers the last :class:`FinalOutputEvent`; falls back to the last - non-empty :class:`ModelCallEvent` output. ``None`` when neither exists. + See :func:`_reply_text`. """ return _reply_text(record.envelope.trace.events) @@ -1394,15 +1393,23 @@ def _turn_assistant_text(record: CaptureRecord) -> str | None: def _reply_text(events: list[Any]) -> str | None: """The text a turn replied with. - Prefers the last :class:`FinalOutputEvent`; falls back to the last - non-empty :class:`ModelCallEvent` output. ``None`` when neither exists. + Prefers the last :class:`FinalOutputEvent` with non-blank text; falls back to + the last non-blank :class:`ModelCallEvent` output. A blank final event with + nothing to fall back to still counts as the (empty) reply. ``None`` when the + turn has neither. """ finals = [e for e in events if isinstance(e, FinalOutputEvent)] - if finals: + if finals and finals[-1].text.strip(): return finals[-1].text - model_calls = [e for e in events if isinstance(e, ModelCallEvent) and _nonempty_text(e.output)] + model_calls = [ + e + for e in events + if isinstance(e, ModelCallEvent) and isinstance(e.output, str) and e.output.strip() + ] if model_calls: return str(model_calls[-1].output) + if finals: + return finals[-1].text return None diff --git a/tests/unit/test_captures_promote.py b/tests/unit/test_captures_promote.py index f0d9f597..d3ed115e 100644 --- a/tests/unit/test_captures_promote.py +++ b/tests/unit/test_captures_promote.py @@ -677,6 +677,15 @@ def test_the_model_call_output_is_checked_without_a_final_event(self) -> None: built = build_example_from_capture(_envelope(events=events), PromoteOptions()) assert built.blocked_reason == "unbacked_claim" + @pytest.mark.parametrize("final_text", ["", " \n"]) + def test_an_empty_final_event_falls_back_to_the_model_call_output( + self, final_text: str + ) -> None: + """An empty ``final_output`` is no reply: the model call's text is what was said.""" + events = [_model_call(0, model_input="q", output=_CLAIM_REPLY), _final(final_text, 1)] + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert built.blocked_reason == "unbacked_claim" + def test_errored_wins_over_unbacked_claim(self) -> None: """An errored turn keeps reporting as errored -- the failure is the root cause, and ``--allow-errored`` is the hint the operator needs.""" @@ -910,6 +919,56 @@ def test_reconstructed_history_is_not_scanned(self) -> None: assert built1.blocked is None assert not any("claim" in w for w in built1.warnings) + def test_reconstructed_history_skips_an_empty_final_event(self) -> None: + """Same reply rule as the claim check: an empty ``final_output`` falls back to the + model call's text rather than replaying an empty assistant message.""" + turn0 = _envelope( + capture_id="cap_t0", + conversation_id="conv_1", + turn_index=0, + events=[ + _model_call(0, model_input="Am I free Friday?", output="Friday is clear."), + _final("", 1), + ], + ) + turn1 = _envelope( + capture_id="cap_t1", + conversation_id="conv_1", + turn_index=1, + events=[_model_call(0, model_input="And Saturday?"), _final("Also clear.", 1)], + ) + + results = build_conversation_examples([_record(turn0), _record(turn1)], PromoteOptions()) + + _, built1 = results[1] + assert built1.example.history is not None + assert built1.example.history[1].content == "Friday is clear." + + def test_reconstructed_history_keeps_an_empty_final_with_nothing_to_fall_back_to( + self, + ) -> None: + """With no model-call text either, the empty reply is still the turn's reply.""" + turn0 = _envelope( + capture_id="cap_t0", + conversation_id="conv_1", + turn_index=0, + events=[_model_call(0, model_input="Am I free Friday?", output=""), _final("", 1)], + ) + turn1 = _envelope( + capture_id="cap_t1", + conversation_id="conv_1", + turn_index=1, + events=[_model_call(0, model_input="And Saturday?"), _final("Also clear.", 1)], + ) + + results = build_conversation_examples([_record(turn0), _record(turn1)], PromoteOptions()) + + _, built1 = results[1] + assert [(m.role, m.content) for m in built1.example.history or []] == [ + ("user", "Am I free Friday?"), + ("assistant", ""), + ] + def test_a_blocked_unbacked_turn_does_not_seed_reconstructed_history(self) -> None: turn0 = _envelope( capture_id="cap_t0", From 4ca343f6234e9b4affbc0d14c6861a0f2a2d18f8 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Wed, 28 Oct 2026 00:42:18 +0100 Subject: [PATCH 12/15] fix(captures): judge history claims per turn, skip tool-stripped history A reply is backed when its turn -- from one user message to the next -- holds any tool row or assistant tool_calls, before or after it, so "I searched your calendar." followed by the call no longer warns. A recorded history with no tool rows at all (stripped by the app when it resent history) is undecidable and yields no warning, instead of flagging every past claim. Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/claims.py | 42 ++++++++++++++++---------- src/evalshift_cli/captures/promote.py | 5 ++-- tests/unit/test_captures_claims.py | 43 ++++++++++++++++++++++++++- tests/unit/test_captures_promote.py | 30 +++++++++++++++++++ 4 files changed, 102 insertions(+), 18 deletions(-) diff --git a/src/evalshift_cli/captures/claims.py b/src/evalshift_cli/captures/claims.py index 095da628..6f154839 100644 --- a/src/evalshift_cli/captures/claims.py +++ b/src/evalshift_cli/captures/claims.py @@ -55,14 +55,18 @@ def find_tool_claim(text: str) -> str | None: def unbacked_history_claims(history: Sequence[ChatMessage]) -> list[tuple[int, str]]: """Find the assistant replies in a recorded history that claim a tool action never taken. - A reply is *backed* when a ``tool`` message or an ``assistant`` message with - ``tool_calls`` sits between it and the user turn it answers. A ``user`` message starts a - new turn, except one directly after a ``tool`` row: Claude delivers tool results inside - a user turn, and text riding in that same turn normalises to ``tool`` rows followed by a - ``user`` row -- still the answer to the call, not a fresh question. - - Only meaningful for history recorded with its tool rows intact. History rebuilt from - earlier turns' text carries no tool rows, so every past reply would look unbacked. + Backing is decided per turn: a turn runs from one ``user`` message to the next, and every + reply in a turn that holds a ``tool`` message or an ``assistant`` message with + ``tool_calls`` -- before *or after* the reply -- is backed ("I searched your calendar." + then the call is still that turn's action). A ``user`` message directly after a ``tool`` + row does not start a new turn: Claude delivers tool results inside a user turn, and text + riding in that same turn normalises to ``tool`` rows followed by a ``user`` row -- still + the answer to the call, not a fresh question. + + Only history recorded with its tool rows intact can be judged. One with no ``tool`` row + and no ``tool_calls`` anywhere either never called a tool or had its tool rows stripped + when the app resent it (or was rebuilt from earlier turns' text) -- undecidable, and + judging it would flag every past claim -- so it yields nothing. Args: history: A case's recorded conversation prefix. @@ -72,17 +76,25 @@ def unbacked_history_claims(history: Sequence[ChatMessage]) -> list[tuple[int, s 1-based within ``history``; ``phrase`` is quoted from the reply by :func:`find_tool_claim`. """ - claims: list[tuple[int, str]] = [] - backed = False + # Pass 1: number each message's turn and note which turns acted. + turns: list[int] = [] + backed_turns: set[int] = set() + turn = 0 previous_role: str | None = None - for position, message in enumerate(history, start=1): + for message in history: if message.role == "user" and previous_role != "tool": - backed = False + turn += 1 elif message.role == "tool" or (message.role == "assistant" and message.tool_calls): - backed = True - elif message.role == "assistant" and message.content and not backed: + backed_turns.add(turn) + turns.append(turn) + previous_role = message.role + if not backed_turns: + return [] + # Pass 2: a claiming reply in a turn that never acted is unbacked. + claims: list[tuple[int, str]] = [] + for position, (message, message_turn) in enumerate(zip(history, turns, strict=True), start=1): + if message.role == "assistant" and message.content and message_turn not in backed_turns: phrase = find_tool_claim(message.content) if phrase is not None: claims.append((position, phrase)) - previous_role = message.role return claims diff --git a/src/evalshift_cli/captures/promote.py b/src/evalshift_cli/captures/promote.py index d2ace91b..23ff20ce 100644 --- a/src/evalshift_cli/captures/promote.py +++ b/src/evalshift_cli/captures/promote.py @@ -238,8 +238,9 @@ def build_example_from_capture( if loss is not None: warnings.append(loss) if recovered.history is not None: - # Recorded history only: it keeps its tool rows, so "backed" is decidable. History - # reconstructed from earlier turns is text-only and must never be scanned. + # Recorded history only: it can keep its tool rows, so "backed" is decidable (and the + # scan itself yields nothing for one that kept none). History reconstructed from + # earlier turns is text-only and must never be scanned. history_claims = _history_claims_warning(recovered.history) if history_claims is not None: warnings.append(history_claims) diff --git a/tests/unit/test_captures_claims.py b/tests/unit/test_captures_claims.py index d7c6b91b..c5390327 100644 --- a/tests/unit/test_captures_claims.py +++ b/tests/unit/test_captures_claims.py @@ -77,11 +77,51 @@ def _tool(call_id: str, result: str) -> ChatMessage: _SEARCH = HistoryToolCall(id="c1", name="search_notes", arguments={"q": "PO"}) +#: A later turn that did call a tool: proof the app records tool rows in its history, so a +#: turn without them really called nothing (see the tool-stripped history test). +_BACKED_TURN = [ + _user("Friday?"), + _assistant("", calls=[HistoryToolCall(id="c9", name="get_schedule", arguments={})]), + _tool("c9", "[]"), + _assistant("Friday is clear."), +] + + def test_history_claim_without_a_tool_call_is_reported_by_position() -> None: - history = [_user("PO number?"), _assistant("Sorry, my notes search didn't find it.")] + history = [ + _user("PO number?"), + _assistant("Sorry, my notes search didn't find it."), + *_BACKED_TURN, + ] assert unbacked_history_claims(history) == [(2, "my notes search didn't")] +def test_tool_stripped_history_is_not_judged() -> None: + """With no tool row anywhere, the app may have dropped them when resending history: + backing is undecidable, and judging anyway would flag every past claim.""" + history = [ + _user("PO number?"), + _assistant("I searched your notes and found nothing."), + _user("Friday?"), + _assistant("I've checked your calendar: it's clear."), + ] + assert unbacked_history_claims(history) == [] + + +def test_a_claim_before_its_turns_tool_call_is_backed() -> None: + """Backing is per turn, not only backward: a reply that announces the search in the + past tense just before the recorded call is the same turn's action.""" + history = [ + _user("Am I free Friday?"), + _assistant("I searched your calendar."), + _assistant("", calls=[HistoryToolCall(id="c2", name="get_schedule", arguments={})]), + _tool("c2", "[]"), + _assistant("Friday is clear."), + _user("Thanks."), + ] + assert unbacked_history_claims(history) == [] + + def test_history_claim_with_tool_call_in_its_turn_is_backed() -> None: """The native-shapes shape: "Let me look." beside a real call, then the answer.""" history = [ @@ -108,6 +148,7 @@ def test_unbacked_claims_in_different_turns_are_all_reported_in_order() -> None: _assistant("I searched your notes and found nothing."), _user("Friday?"), _assistant("I've checked your calendar: it's clear."), + *_BACKED_TURN, ] assert unbacked_history_claims(history) == [(2, "I searched"), (4, "I've checked")] diff --git a/tests/unit/test_captures_promote.py b/tests/unit/test_captures_promote.py index d3ed115e..cb9e4de1 100644 --- a/tests/unit/test_captures_promote.py +++ b/tests/unit/test_captures_promote.py @@ -826,6 +826,19 @@ def test_build_messages_list_model_role_coerced_to_assistant() -> None: assert built.example.history[1].content == "hello" +#: A recorded history turn that did call a tool -- what tells the scan the app keeps tool rows. +_BACKED_TURN: list[dict[str, Any]] = [ + {"role": "user", "content": "Anything on Friday?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "c9", "name": "search_notes", "arguments": {}}], + }, + {"role": "tool", "tool_call_id": "c9", "content": "nothing"}, + {"role": "assistant", "content": "Friday is clear."}, +] + + class TestUnbackedHistoryClaims: """A case whose *recorded* history carries a reply claiming a tool action it never took replays with that fabrication in context -- worth a warning, not a block: the @@ -843,6 +856,7 @@ def test_recorded_history_with_an_unbacked_claim_warns(self, tools_offered: list messages = [ {"role": "user", "content": "PO number on the Acme contract?"}, {"role": "assistant", "content": _CLAIM_REPLY}, + *_BACKED_TURN, {"role": "user", "content": "Try the Globex one then."}, ] events = [ @@ -859,6 +873,7 @@ def test_several_unbacked_claims_aggregate_into_one_warning(self) -> None: {"role": "assistant", "content": _CLAIM_REPLY}, {"role": "user", "content": "Friday?"}, {"role": "assistant", "content": "I've checked your calendar: it's clear."}, + *_BACKED_TURN, {"role": "user", "content": "Thanks."}, ] events = [ @@ -873,6 +888,21 @@ def test_several_unbacked_claims_aggregate_into_one_warning(self) -> None: "turn — this case replays with those claims in context." ] + def test_tool_stripped_recorded_history_does_not_warn(self) -> None: + """No tool row anywhere: the app dropped them when resending, so a past claim + can't be judged and every one would warn.""" + messages = [ + {"role": "user", "content": "PO number on the Acme contract?"}, + {"role": "assistant", "content": _CLAIM_REPLY}, + {"role": "user", "content": "Try the Globex one then."}, + ] + events = [ + _messages_call(0, messages=messages, tools_offered=["search_notes"]), + _final("Globex is PO-7.", 1), + ] + built = build_example_from_capture(_envelope(events=events), PromoteOptions()) + assert not any("claim" in w for w in built.warnings) + def test_a_claim_backed_by_a_recorded_call_does_not_warn(self) -> None: messages = [ {"role": "user", "content": "PO number on the Acme contract?"}, From 2390a61ab75ea63b3c23bd28bbf55f33058a1a9b Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Wed, 28 Oct 2026 00:43:40 +0100 Subject: [PATCH 13/15] fix(cli): hint --allow-unbacked-claims on a blocked repair, spare reviewed cases `capture sync --repair` refusing a case whose rebuild is now an unbacked tool claim names the flag that lets the repair run. The stale-case hint for a newly blocked capture no longer asks to delete a case a human has marked provenance: reviewed. Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/cli/commands/capture.py | 17 +++++-- tests/unit/test_cli_capture.py | 21 +++++++++ tests/unit/test_cli_capture_repair.py | 55 +++++++++++++++++++++++ 3 files changed, 90 insertions(+), 3 deletions(-) diff --git a/src/evalshift_cli/cli/commands/capture.py b/src/evalshift_cli/cli/commands/capture.py index 4e42fbbe..3c4aaadd 100644 --- a/src/evalshift_cli/cli/commands/capture.py +++ b/src/evalshift_cli/cli/commands/capture.py @@ -1768,10 +1768,18 @@ def capture_sync( for warning in built.warnings: console.print(f"[yellow]⚠[/yellow] {envelope.capture_id}: {warning}") if built.blocked is not None: + # Unlike the other refusals, an unbacked claim is rescued by a sync + # flag: with it the rebuild is no longer blocked and the repair runs. + hint = ( + " Re-run with [bold]--allow-unbacked-claims[/bold] to repair it anyway." + if built.blocked_reason == "unbacked_claim" + else "" + ) for case in to_repair: console.print( f"[yellow]⚠[/yellow] can't repair " - f"[cyan]{escape(case.name)}[/cyan]: {escape(built.blocked)}", + f"[cyan]{escape(case.name)}[/cyan]: {escape(built.blocked)}" + f"{hint}", ) else: # A turn whose own capture yields no history got it from its sibling @@ -1803,9 +1811,12 @@ def capture_sync( ) if built.blocked_reason == "unbacked_claim": # A case an earlier sync (or an override) promoted stays in the suite: - # sync never deletes a case file, so it names the file to delete. + # sync never deletes a case file, so it names the file to delete -- + # unless a human already reviewed that case. if stale_case_paths is None: - stale_case_paths = {c.from_capture: path for path, c in kept} + stale_case_paths = { + c.from_capture: path for path, c in kept if not is_reviewed(c.example) + } stale_path = stale_case_paths.get(envelope.capture_id) if stale_path is not None: skip_line += ( diff --git a/tests/unit/test_cli_capture.py b/tests/unit/test_cli_capture.py index 8a5fa56f..6e8a3084 100644 --- a/tests/unit/test_cli_capture.py +++ b/tests/unit/test_cli_capture.py @@ -1624,6 +1624,27 @@ def test_sync_names_the_stale_case_of_a_now_blocked_capture(tmp_path: Path) -> N assert "cap_bad" in load_jsonl(tmp_path / "suites" / "alpha" / "golden.jsonl").ids() +def test_sync_stale_case_hint_leaves_out_a_reviewed_case(tmp_path: Path) -> None: + """A human already looked at a reviewed case (say, fixed it to expect the search), so + sync does not nag them to delete it.""" + config = _unbacked_sync_fixture(tmp_path) + assert _invoke(["promote", "cap_bad", "--allow-unbacked-claims"], tmp_path).exit_code == 0 + case_path = tmp_path / "suites" / "alpha" / "cap_bad.json" + case = json.loads(case_path.read_text(encoding="utf-8")) + case["example"]["expected_tools"] = [ + {"tool_name": "search_orders", "provenance": "reviewed"}, + ] + case_path.write_text(json.dumps(case), encoding="utf-8") + + result = _invoke(["sync", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + flat = " ".join(result.stdout.split()) + assert "skipped cap_bad" in flat + assert "still in the suite" not in flat + assert case_path.exists() + + def test_sync_stale_case_hint_only_for_a_promoted_capture(tmp_path: Path) -> None: config = _unbacked_sync_fixture(tmp_path) diff --git a/tests/unit/test_cli_capture_repair.py b/tests/unit/test_cli_capture_repair.py index 769b340a..c2bd2476 100644 --- a/tests/unit/test_cli_capture_repair.py +++ b/tests/unit/test_cli_capture_repair.py @@ -362,6 +362,61 @@ def test_repair_leaves_a_case_whose_rebuild_is_blocked( assert case.read_bytes() == before +def _put_unbacked(store: FakeRemoteStore, capture_id: str) -> None: + """``_put``, but the turn was offered a tool, called none, and claims it searched.""" + _put(store, capture_id) + key = f"captures/alpha/{capture_id}.json" + payload = json.loads(store.objects[key]) + model_call, final = payload["trace"]["events"] + model_call["tools_offered"] = ["search"] + final["text"] = "My search didn't turn up a price." + store.objects[key] = json.dumps(payload).encode("utf-8") + + +def test_repair_of_an_unbacked_claim_names_the_flag_that_allows_it( + tmp_path: Path, store: FakeRemoteStore +) -> None: + case = _write_case(tmp_path, "cap_1") + before = case.read_bytes() + _put_unbacked(store, "cap_1") + result = _sync(tmp_path, "--repair") + assert result.exit_code == 0, result.output + out = _out(result) + assert "can't repair cap_1: turn was offered tools and called none" in out + assert "Re-run with --allow-unbacked-claims to repair it anyway." in out + assert case.read_bytes() == before + + +def test_repair_of_an_unbacked_claim_proceeds_with_the_flag( + tmp_path: Path, store: FakeRemoteStore +) -> None: + case = _write_case(tmp_path, "cap_1") + _put_unbacked(store, "cap_1") + result = _sync(tmp_path, "--repair", "--allow-unbacked-claims") + assert result.exit_code == 0, result.output + assert "repaired 1 case(s)" in _out(result) + assert _stored(case).example.history is not None + assert [m.role for m in _stored(case).example.history or []] == [ + "system", + "user", + "assistant", + "tool", + "assistant", + ] + + +def test_another_refused_rebuild_gets_no_unbacked_claims_hint( + tmp_path: Path, store: FakeRemoteStore +) -> None: + """The hint is for unbacked claims only; other refusals keep their message as is.""" + _write_case(tmp_path, "cap_1") + _put(store, "cap_1") + del store.objects[TOOLSET_KEY] # refused as no_toolset + result = _sync(tmp_path, "--repair") + assert "can't repair cap_1:" in _out(result) + assert "--allow-unbacked-claims" not in _out(result) + + def test_repair_survives_a_corrupt_case_file_elsewhere( tmp_path: Path, store: FakeRemoteStore ) -> None: From d5eaac51e535a489707e4b189352374668dfb064 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Wed, 28 Oct 2026 00:45:05 +0100 Subject: [PATCH 14/15] docs: narrowed tool-claim detection and per-turn backing rules Promises no longer count as claims, tool results / retrieval / agent loops back a turn, history backing is per turn and skips tool-stripped history, and the repair and reviewed-case behaviour of sync. The spec gets an amended-after-review note. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 23 +++++---- DOCS.md | 2 +- docs/agents.md | 20 +++++--- docs/configuration.md | 17 ++++--- docs/conversations.md | 18 ++++--- .../2026-10-27-unbacked-tool-claims-design.md | 49 ++++++++++++++++--- llms-full.txt | 31 +++++++----- 7 files changed, 110 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3cd26cc..81d64085 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,9 +16,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `capture promote` now exits 1 on it (`Re-run with --allow-unbacked-claims to promote it anyway.`), and `capture sync` skips it and counts it on its own (`skipped 1 capture(s) whose reply claims a tool call it never made`). The - check reads the reply's English wording (first-person phrasing such as "I - searched", "let me check", "my search returned"; a bare tool name never - fires) and never flags a turn that called or requested a tool. + check reads the reply's English wording — a first-person claim that the + action already happened, such as "I searched", "I've checked" or "my notes + search didn't"; a promise ("let me check", "I'll search your notes once you + confirm") or a bare tool name never fires — and never flags a turn with any + other sign it acted: a tool call or tool result, a requested tool call, a + retrieval event, or more than one model call. `capture promote --allow-unbacked-claims`, `capture sync --allow-unbacked-claims/--block-unbacked-claims` and `captures.allow_unbacked_claims` in `evalshift.yaml` (flag, then YAML, then @@ -27,16 +30,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 is still reported as errored. Cases an earlier sync promoted stay in the suite: sync never deletes a case file, so its skip line names the file to delete (`Its case is still in the suite from an earlier sync: delete - .evalshift/suites//.json to drop it.`). A refused turn + .evalshift/suites//.json to drop it.`), unless the case + is marked `provenance: reviewed`. `capture sync --repair` refusing such a + case adds `Re-run with --allow-unbacked-claims to repair it anyway.` A refused turn also drops out of later turns' reconstructed history. `init` scaffolds the key commented out. Older CLIs reject the key, so update a CI `evalshift-version:` pin first. - Promotion warns when a case's recorded history carries an assistant reply - that claims a tool action with no tool call in its turn (`history message 2 - (assistant) claims a tool action ("my notes search didn't") with no tool - call in its turn — this case replays with that claim in context.`). The - case is still promoted. History reconstructed from sibling turns is not - checked. + that claims a tool action with no tool call or tool result anywhere in its + turn (`history message 2 (assistant) claims a tool action ("my notes search + didn't") with no tool call in its turn — this case replays with that claim + in context.`). The case is still promoted. A recorded history with no tool + rows at all, and history reconstructed from sibling turns, are not checked. ## [1.6.2] - 2026-10-27 diff --git a/DOCS.md b/DOCS.md index 81cc33da..637fed2c 100644 --- a/DOCS.md +++ b/DOCS.md @@ -187,7 +187,7 @@ and `capture sync` and `capture list` fetch what is new before they read the loc 4. Carries the recorded generation settings onto the promoted example: the first `model_call`'s `metadata["generation_config"]` is copied verbatim to the example's `generation_config`, so the replay dispatches under the temperature, structured-output, and tool-choice settings the production call actually used. `capture sync` appends `wired generation config for N case(s)` to its summary line when any promoted capture carried one. 5. Records what the captured run cost on the case file as `cost_usd` (its `model_call` events summed) plus `cost_source`. The SDK never prices anything — a `model_call`'s `cost_usd` is `0.0` unless the app's own instrumentation set it, and the provider client wrappers record tokens but no cost by design — so each event with tokens and no cost is priced at promotion from litellm's price table for its own `model_id` (aliases and provider prefixes resolve through the registry first). `cost_source` is `"recorded"` when every non-zero part came from the capture (a recorded cost is never re-estimated), `"estimated"` when any part was priced here, and absent with `cost_usd: 0.0` when nothing was priced — no tokens, or a model litellm does not price. An unpriced local / self-hosted model is the normal case, not a failure: it stays at 0 silently, with no warning. The example in `golden.jsonl` never carries the figure; it is provenance of the capture, not something a replay reproduces. 6. Skips captures whose replayed content duplicates an already-promoted case or an earlier capture in the same run — duplicate examples inflate *n* and corrupt the paired statistics (`--keep-duplicates` opts out). Dedup is seeded from the cases already in the suite dir, so it holds across repeated syncs. -7. Skips captures whose turn recorded an `error` event — a turn that died before the agent acted is not ground truth, and promoting it would assert `expected_no_tools: true` on a question that needed a tool. `--allow-errored` promotes it anyway (still never asserting `expected_no_tools`). `capture promote` exits non-zero on the same condition. Separately and unconditionally — `--allow-errored` does not help — a capture whose first `model_call` has no `toolset_ref` is refused: the SDK did not record what tools were offered, so there is nothing to carry, and re-capturing with a current `evalshift-sdk` is the only fix. Also unconditionally, a capture whose recorded messages carry content a text replay can't carry — an image, a document, audio, a server-tool result, an unknown block type — is skipped and counted on its own (`capture promote` refuses it): a replay without it is not ground truth. Also by default — `--allow-errored` does not help — a turn that was offered tools, called none, and still replies as if it had (`my notes search didn't find anything`) is skipped: promoting it would make a hallucinated action ground truth, with `expected_no_tools: true`. The check reads the reply's wording only (English first-person action phrasing — `I searched`, `I've checked`, `let me look`, `my search returned` — never a bare tool name), and a turn that recorded or requested any tool call is never flagged; an errored turn is reported as errored. It is counted on its own (`skipped N capture(s) whose reply claims a tool call it never made (use --allow-unbacked-claims or captures.allow_unbacked_claims to promote anyway)`). `--allow-unbacked-claims`, or `captures.allow_unbacked_claims: true` (`--block-unbacked-claims` overrides it for one run), promotes it with the refusal as a warning and never asserts `expected_no_tools`; `capture promote` refuses it unless given `--allow-unbacked-claims`. Sync never deletes a case file, so a case an earlier sync (or an override) promoted stays in the suite, and the skip line names it: `Its case is still in the suite from an earlier sync: delete .evalshift/suites//.json to drop it.` A case whose own recorded history carries such a reply with no tool call in its turn is promoted with a warning (`history message 2 (assistant) claims a tool action ("my notes search didn't") with no tool call in its turn — this case replays with that claim in context.`). Last, as a safety net, a capture whose case fails the suite schema's own validation in a way none of these rules anticipates is skipped with pydantic's first error and counted on its own (`skipped N capture(s) that could not form a valid case`; `capture promote` refuses it) instead of aborting the sync; hand-write that case or report the capture. +7. Skips captures whose turn recorded an `error` event — a turn that died before the agent acted is not ground truth, and promoting it would assert `expected_no_tools: true` on a question that needed a tool. `--allow-errored` promotes it anyway (still never asserting `expected_no_tools`). `capture promote` exits non-zero on the same condition. Separately and unconditionally — `--allow-errored` does not help — a capture whose first `model_call` has no `toolset_ref` is refused: the SDK did not record what tools were offered, so there is nothing to carry, and re-capturing with a current `evalshift-sdk` is the only fix. Also unconditionally, a capture whose recorded messages carry content a text replay can't carry — an image, a document, audio, a server-tool result, an unknown block type — is skipped and counted on its own (`capture promote` refuses it): a replay without it is not ground truth. Also by default — `--allow-errored` does not help — a turn that was offered tools, called none, and still replies as if it had (`my notes search didn't find anything`) is skipped: promoting it would make a hallucinated action ground truth, with `expected_no_tools: true`. The check reads the reply's wording only — an English first-person claim that the action already happened (`I searched`, `I've checked`, `my notes search didn't`), never a promise (`let me check`, `I'll search your notes once you confirm` — a clarifying question is a correct no-tool turn) and never a bare tool name. A turn with any other sign it acted is never flagged: a `tool_call` or `tool_result` event, a requested tool call, a `retrieval` event, or more than one model call (an agent loop). An errored turn is reported as errored. It is counted on its own (`skipped N capture(s) whose reply claims a tool call it never made (use --allow-unbacked-claims or captures.allow_unbacked_claims to promote anyway)`). `--allow-unbacked-claims`, or `captures.allow_unbacked_claims: true` (`--block-unbacked-claims` overrides it for one run), promotes it with the refusal as a warning and never asserts `expected_no_tools`; `capture promote` refuses it unless given `--allow-unbacked-claims`. Sync never deletes a case file, so a case an earlier sync (or an override) promoted stays in the suite, and the skip line names it unless the case is marked `provenance: reviewed`: `Its case is still in the suite from an earlier sync: delete .evalshift/suites//.json to drop it.` Under `--repair`, a case whose rebuild is refused this way prints `can't repair : … Re-run with --allow-unbacked-claims to repair it anyway.` A case whose own recorded history carries such a reply with no tool call or tool result anywhere in its turn (from one user message to the next) is promoted with a warning (`history message 2 (assistant) claims a tool action ("my notes search didn't") with no tool call in its turn — this case replays with that claim in context.`); a recorded history with no tool rows at all (the app stripped them when it resent history) is not checked. Last, as a safety net, a capture whose case fails the suite schema's own validation in a way none of these rules anticipates is skipped with pydantic's first error and counted on its own (`skipped N capture(s) that could not form a valid case`; `capture promote` refuses it) instead of aborting the sync; hand-write that case or report the capture. 8. Warns when two captures claim the same `(conversation_id, turn_index)` (a retried turn), and when a promoted turn contains a failed tool result (`error`, or `{"success": false}`). Both stay warnings — see [Agent evals → What does not belong in a golden suite](docs/agents.md). 9. With `captures.max_age` / `captures.max_cases` set: filters, plans, retires. Captures recorded before now − `max_age`, or on another model than `defaults.source_model`, are not eligible. With `max_cases` set, sync plans each suite (every suite directory with case files, not only those with new captures; `--suite` narrows it). It fills the free slots route-balanced, a conversation at a time. It deletes the case files of the stale cases it retires and of any fresh cases it trims, and rebuilds `golden.jsonl` from what is left; reviewed cases stay. A case is known by its file, not the `name` inside it, so a renamed or copied case file is planned and deleted on its own; a deletion that fails is reported (`alpha: could not delete cap_0.json; not counted as retired.`) and not counted. A suite planned only because its directory holds case files is wired into `suites:` only if it already has an entry there. A suite whose case files cannot be read is left unchanged, with a warning (`alpha: could not read its promoted cases (ValidationError); left it unchanged.`). After the summary line, sync prints one line per suite: ` planner: 200 cases — kept 150, retired 50 (32 older than 14d, 18 on another model), added 50 of 2940 eligible; not eligible: 812 older than 14d, 40 on another model` (zero counts left out; `max_age` alone prints only ` planner: not eligible: …`). Syncing again with nothing new changes no file. If `evalshift.yaml` cannot be read, sync runs without the limits and, when the file has a top-level `captures:` key, says so: `could not read evalshift.yaml (); syncing without captures.max_age / captures.max_cases / captures.rounds / captures.allow_unbacked_claims (replaying the first round only, blocking unbacked tool claims).` Sync reads the file under `--offline` too. 10. Writes `.evalshift/suites//golden.jsonl` and rewrites the managed `suites:` block in `evalshift.yaml` (between the `>>> evalshift suites` markers). diff --git a/docs/agents.md b/docs/agents.md index 2aaa83fa..cf7984e0 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -417,10 +417,14 @@ that. These shapes are actively filtered or flagged during promotion: offered tools, called none, and still replied "I don't have that information — my notes search didn't find anything" records an action that never happened. Promoted, the fabricated reply becomes `expected` and the - turn asserts `expected_no_tools: true`. The check reads the reply's wording - (English first-person action phrasing such as "I searched", "let me check", - "my search returned"; a bare tool name never fires), and a turn that called - or requested any tool is never flagged: + turn asserts `expected_no_tools: true`. The check reads the reply's wording: + an English first-person claim that the action already happened, such as "I + searched", "I've checked" or "my notes search didn't". A promise ("let me + check", "I'll search your notes once you confirm") never fires — a + clarifying question is a correct no-tool turn — nor does a bare tool name. + A turn with any other sign it acted is never flagged: a recorded tool call or + tool result, a requested tool call, a `retrieval` event, or more than one + model call (an agent loop): ```text ⚠ skipped cap_bad: turn was offered tools and called none, but its reply claims a tool action ("my notes search didn't") — promoting it would make a hallucinated action ground truth. @@ -434,9 +438,11 @@ that. These shapes are actively filtered or flagged during promotion: refusal prints as a warning and the case never gets `expected_no_tools: true`. `--allow-errored` does not help here, and an errored turn is reported as errored. A case an earlier sync promoted stays in the suite — sync never - deletes a case file — and the skip line appends `Its case is still in the - suite from an earlier sync: delete .evalshift/suites//.json - to drop it.` + deletes a case file — and, unless it is marked `provenance: reviewed`, the + skip line appends `Its case is still in the suite from an earlier sync: + delete .evalshift/suites//.json to drop it.` Under + `--repair`, a case whose rebuild is refused this way prints `can't repair + : … Re-run with --allow-unbacked-claims to repair it anyway.` - **Captures missing a recorded toolset are refused, unconditionally.** Every model call is required to record the toolset it was offered; a capture whose first `model_call` has no `toolset_ref` — the SDK failed to write the sidecar, diff --git a/docs/configuration.md b/docs/configuration.md index a3fce875..41451d97 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1118,13 +1118,16 @@ and ones promoted under another name (`capture promote --as`) are left out of th evalshift older than 1.6.0 rejects the key, so update a CI pin (`evalshift-version:`) before adding it. -`allow_unbacked_claims` judges the reply's wording only: English first-person action phrasing -(`I searched`, `I've checked`, `let me look`, `my search returned`), never a bare mention of a -tool. A turn that recorded or requested any tool call is never flagged, and an errored turn is -reported as errored. Sync never deletes a case file, so a case an earlier sync promoted from -such a capture stays in the suite; the skip line names the file to delete (`Its case is still -in the suite from an earlier sync: delete .evalshift/suites//.json to drop -it.`). A CLI released before the key rejects it; update a CI pin first. +`allow_unbacked_claims` judges the reply's wording only: an English first-person claim that the +action already happened (`I searched`, `I've checked`, `my notes search didn't`), never a +promise (`let me check`, `I'll search your notes once you confirm` — a clarifying question is a +correct no-tool turn) and never a bare mention of a tool. A turn with any other sign it acted is +never flagged: a recorded tool call or tool result, a requested tool call, a `retrieval` event, +or more than one model call (an agent loop). An errored turn is reported as errored. Sync never +deletes a case file, so a case an earlier sync promoted from such a capture stays in the suite; +unless that case is marked `provenance: reviewed`, the skip line names the file to delete (`Its +case is still in the suite from an earlier sync: delete +.evalshift/suites//.json to drop it.`). A CLI released before the key rejects it; update a CI pin first. Setting either `max_age` or `max_cases` turns [bounded suites](#bounded-suites) on. Both work without `store`: they govern how captures become suites, whether the captures came from disk or diff --git a/docs/conversations.md b/docs/conversations.md index 23f4aac4..0fb300b1 100644 --- a/docs/conversations.md +++ b/docs/conversations.md @@ -105,17 +105,21 @@ sibling's prefix. A turn promoted anyway (`--allow-errored`, Recovered history is checked for such claims too. When a turn's own messages list carries an assistant reply that claims a tool action with no tool call -or tool result in its turn, promotion warns and still promotes — the current -turn may be fine, but the candidate will answer with that claim in context: +or tool result anywhere in its turn — from one user message to the next, so a +reply that says "I searched your calendar." just before the call is backed — +promotion warns and still promotes. The current turn may be fine, but the +candidate will answer with that claim in context: ```text ⚠ history message 2 (assistant) claims a tool action ("my notes search didn't") with no tool call in its turn — this case replays with that claim in context. ``` A `user` message directly after a `tool` message does not start a new turn: -Claude delivers tool results inside a user turn. Reconstructed history is -never checked — it carries no tool rows, so every earlier reply would look -unbacked. +Claude delivers tool results inside a user turn. A recorded history with no +tool rows at all is not checked — the app may have stripped them when it +resent the conversation, and then every earlier claim would look unbacked. +Reconstructed history is never checked for the same reason: it carries no +tool rows. With `captures.max_cases` set, a conversation's new turns enter a suite together when they fit; one larger than the suite's free slots is skipped @@ -202,7 +206,9 @@ they are promoted and even past `--since`. A case whose capture is gone from both `.evalshift/captures/` and the store is named (`can't repair N case(s)`): delete it or re-capture. A case with stringified blocks whose rebuild is now refused — its turn carries an image, say — is named too -(`can't repair : …`) and left as it is; a case left with empty `inputs` +(`can't repair : …`) and left as it is — when the refusal is a reply +claiming a tool call it never made, the line adds `Re-run with +--allow-unbacked-claims to repair it anyway.`; a case left with empty `inputs` whose rebuild is refused is left as it is too, but reported like any other refused capture (`skipped : …`, counted in the summary). A turn whose capture recorded no messages list, or one with no user turn to split diff --git a/docs/superpowers/specs/2026-10-27-unbacked-tool-claims-design.md b/docs/superpowers/specs/2026-10-27-unbacked-tool-claims-design.md index 5ed0438d..0724db6d 100644 --- a/docs/superpowers/specs/2026-10-27-unbacked-tool-claims-design.md +++ b/docs/superpowers/specs/2026-10-27-unbacked-tool-claims-design.md @@ -2,6 +2,15 @@ **Date:** 2026-10-27 · **Status:** decisions approved in chat, spec for maintainer review · **Repos:** evalshift-cli (behaviour + docs); evalshift-sdk, evalshift-action, evalshift-server (nothing) +> **Amended after review (2026-10-28).** Review found false positives on exactly the turns +> where calling no tool is correct. Changes, folded into the sections below: promises (`let me +> …`, `I'll …`) and `the search/query …` no longer count as claims (§1); a tool result, a +> retrieval event or a second model call also backs a turn (§2); history backing is decided per +> turn, forward as well as backward, and a recorded history with no tool rows at all is not +> scanned (§3); a blank `final_output` falls back to the model-call text (§2); `--repair` names +> `--allow-unbacked-claims` when it can't repair such a case, and the stale-case hint skips +> reviewed cases (§5). + ## Problem A captured turn can record a reply that *claims* a tool action the model never took. Found in @@ -45,12 +54,18 @@ def find_tool_claim(text: str) -> str | None: """Return the first phrase in ``text`` that claims a tool action, else ``None``.""" ``` -One compiled, case-insensitive regex over first-person action phrasing. Starting set, validated -against testApp's three tool-free captures (1 true positive, 0 false positives): +One compiled, case-insensitive regex over first-person claims that an action already happened. +Validated against testApp's three tool-free captures (1 true positive, 0 false positives): + +- past action: `I('ve| have)? (just |already )?(searched|checked|looked (up|through|in|into)|queried|pulled up|went through|scanned)` +- reported result: `my ( )?(search|lookup|query) (didn't|did not|found|returned|came back|turned up|shows?|showed)` -- past action: `I('ve| have)? (just )?(searched|checked|looked (up|through|at|in)|queried|pulled up|went through|scanned)` -- reported result: `(my|the) ( )?(search|lookup|query) (didn't|did not|found|returned|came back|turned up|shows?)` -- promise with nothing behind it: `let me (check|search|look|pull up|find)`, `I'll (check|search|look)` +*Amended:* the original set also matched promises (`let me (check|search|look|pull up|find)`, +`I'll (check|search|look)`) and `the search/query …`. Both were dropped: a promise is not a +claim that an action happened, and "let me know the date and I'll check your calendar" is a +clarifying question — a correct no-tool turn; "the query didn't include a WHERE clause" is about +the user's query. Known residual false positive, accepted: past-tense review of pasted content +("I checked the code you pasted") — the flag rescues it. Returns the matched phrase so messages can quote it. English only (non-goal below). @@ -61,9 +76,14 @@ an errored turn keeps reporting as errored). It fires when all hold: - the first model call offered tools (`tools_offered` non-empty); - the turn recorded no `tool_call` events and no model call has `requested_tool_calls`; +- *(amended)* the turn recorded no `tool_result` and no `retrieval` event (a RAG answer did + search), and only one model call — a second one happens only after something was fed back, + the only trace of an agent loop in a pre-2.1.0 capture with no `requested_tool_calls`; - `find_tool_claim()` matches, where the reply is the text `_turn_assistant_text` already selects (last `FinalOutputEvent`, else last non-empty - `ModelCallEvent.output`). + `ModelCallEvent.output`). *(Amended:* a blank final text falls back to the model-call text; + with nothing to fall back to, the blank final still counts — reconstruction is unchanged + there.) Message (exact wording settled in implementation): @@ -82,6 +102,13 @@ and no `tool_calls`, it is *backed* if any `tool` message or assistant-with-`too between it and the preceding user message. An unbacked one whose text matches `find_tool_claim` adds one aggregated warning: +*Amended:* backing is per turn (one user message to the next, a user row right after a tool row +not starting one), forward as well as backward — "I searched your calendar." followed by the +call in the same turn is backed. Two passes: mark turns that hold a tool row or `tool_calls`, +then scan. A recorded history with no tool row and no `tool_calls` anywhere is not scanned at +all: the app may strip tool rows when it resends history, making backing undecidable. + + > history message 38 (assistant) claims a tool action ("my notes search didn't") with no tool > call in its turn — this case replays with that claim in context. @@ -118,6 +145,11 @@ captures: - `capture promote`: `--allow-unbacked-claims` (plain bool, help text parallel to `--allow-errored`). On `blocked_reason == "unbacked_claim"`, print "Re-run with `--allow-unbacked-claims` to promote it anyway." (`capture.py:1032` pattern). +- *(Amended)* `capture sync --repair`: a repair candidate whose rebuild is refused as + `unbacked_claim` prints `can't repair : … Re-run with --allow-unbacked-claims to repair + it anyway.` (the flag unblocks the rebuild, so the repair runs). Without `--repair` it is + still counted in `left N case(s) for --repair`, as an errored rebuild is. The stale-case hint + below skips cases marked `provenance: reviewed`. - `capture sync` summary: new `skipped_unbacked` counter in the `blocked_reason` dispatch (`capture.py:1771`) and clause `skipped N capture(s) whose reply claims a tool call it never made (use --allow-unbacked-claims or captures.allow_unbacked_claims to promote anyway)`. @@ -134,8 +166,9 @@ skip list), `docs/conversations.md` (later-turn history warning), `CHANGELOG.md` ## Tests (TDD, `tests/unit/`) - `test_captures_claims.py` (new): positives — "my notes search didn't find anything", "I - searched your notes", "I've checked your calendar", "Let me check your notes for that."; - negatives — "Per your notes from Aug 20…", "Your passport renewal is due by September 15", + searched your notes", "I've checked your calendar"; negatives — *(amended)* "Let me check + your notes for that.", "I'll look into it.", "The notes search returned no results.", "Per + your notes from Aug 20…", "Your passport renewal is due by September 15", "I can't search contract documents", "Would you like me to check?", "the search bar", empty string. - `test_captures_promote.py`: blocks with `"unbacked_claim"` on the turn-9 shape; not blocked diff --git a/llms-full.txt b/llms-full.txt index a281dd30..022ea1c8 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -435,7 +435,8 @@ evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--stri inputs rebuilds unchanged, so it is NOT a repair: sync treats it as an ordinary promoted case and --force refreshes it. A candidate whose capture is gone: "can't repair N case(s)". A stringified case whose rebuild is now blocked (e.g. an image): "can't repair : - ", left as is; an empty-inputs case whose rebuild is blocked is left as is + ", left as is (an unbacked tool claim appends " Re-run with + --allow-unbacked-claims to repair it anyway.", and that flag lets the repair run); an empty-inputs case whose rebuild is blocked is left as is too but gets the ordinary "skipped : ..." line and skip count. A case's single input key is kept (promoted with --input-var question -> stays question; a case left with empty inputs takes this run's --input-var), every other field too; other promoted cases are @@ -658,7 +659,8 @@ evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--stri SKIPPED and counted on their own ("skipped N capture(s) whose reply claims a tool call it never made (use --allow-unbacked-claims or captures.allow_unbacked_claims to promote anyway)"); --allow-errored does not apply. A case already promoted from such a capture stays - (sync never deletes a case file); the skip line appends "Its case is still in the suite from + (sync never deletes a case file); unless that case is_reviewed (any expected tool call + provenance: reviewed), the skip line appends "Its case is still in the suite from an earlier sync: delete .evalshift/suites//.json to drop it." (the path as built from --base). Warns on duplicate (conversation_id, turn_index) and on failed tool results. @@ -1707,13 +1709,16 @@ content leaves the machine. counts it separately ("skipped N capture(s) that could not form a valid case (--allow-errored does not apply)") and exits 0; promote exits 1 without suggesting --allow-errored. -- Promotion ALSO blocks an UNBACKED TOOL CLAIM: tools_offered non-empty, no tool_call event, - no model_call with requested_tool_calls, and the reply (last final_output, else last - non-empty model_call output) matches first-person action wording (English, regex in - captures/claims.py, [] optional: "I ['ve|have] [just|already] searched|checked|looked - up/through/in[to]|queried|pulled up|went through|scanned", "my|the [word] search|lookup|query - didn't|found|returned|came back|turned up|show(s|ed)", "let me check|search|look|pull up|find", "I'll check|search|look"; - a bare tool name never fires). blocked_reason "unbacked_claim", blocked "turn was offered +- Promotion ALSO blocks an UNBACKED TOOL CLAIM: tools_offered non-empty, no tool_call, + tool_result or retrieval event, exactly one model_call and it has no requested_tool_calls + (a second model_call means an agent loop ran, even in a pre-2.1.0 capture), and the reply + (last final_output with non-blank text, else last non-blank model_call output) matches + past-tense first-person claim wording (English, regex in captures/claims.py, [] optional: + "I ['ve|have] [just|already] searched|checked|looked up/through/in[to]|queried|pulled + up|went through|scanned", "my [word] search|lookup|query didn't|did not|found|returned|came + back|turned up|show(s|ed)"). Promises ("let me check", "I'll search") and "the search/query + ..." never fire; nor does a bare tool name. Known accepted false positive: "I checked the + code you pasted". blocked_reason "unbacked_claim", blocked "turn was offered tools and called none, but its reply claims a tool action ("") — promoting it would make a hallucinated action ground truth." promote exits 1 with "Re-run with --allow-unbacked-claims to promote it anyway." (never --allow-errored); sync skips and @@ -1723,9 +1728,11 @@ content leaves the machine. captures.allow_unbacked_claims > block) turns the refusal into a warning and the case never gets expected_no_tools: true. - Recorded history (the turn's own messages list, NEVER history reconstructed from sibling - turns, which has no tool rows) is scanned too: an assistant reply claiming a tool action with - no tool row or assistant tool_calls since the user message it answers WARNS (does not - block): "history message 2 (assistant) claims a tool action ("my notes search didn't") + turns, which has no tool rows) is scanned too, two-pass: history is split into turns (each + user message starts one), and an assistant reply claiming a tool action in a turn holding no + tool row and no assistant tool_calls -- before OR after the reply -- WARNS (does not + block). A recorded history with no tool row and no tool_calls anywhere (tool rows stripped by + the app when resending) is undecidable and never warns: "history message 2 (assistant) claims a tool action ("my notes search didn't") with no tool call in its turn — this case replays with that claim in context." (several -> "history messages 2, 4 (assistant) claim tool actions (...)"). A user row directly after a tool row does not start a new turn (Claude delivers tool results inside a user turn). From 4264473fcf42f238dd54d7484f87bda9577d0c29 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Sun, 1 Nov 2026 19:30:24 +0100 Subject: [PATCH 15/15] docs: link getting-started video at top of README Co-Authored-By: Claude Opus 5.5 --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 5fbdc668..2a2b7a27 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,10 @@ Open-source LLM migration and regression testing for AI agents. [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/) [![PyPI](https://img.shields.io/pypi/v/evalshift.svg)](https://pypi.org/project/evalshift/) +[![EvalShift getting started: catch model migration regressions in 4 minutes](https://img.youtube.com/vi/22-dRnu5fXs/sddefault.jpg)](https://www.youtube.com/watch?v=22-dRnu5fXs) + +▶ [Watch the 4-minute getting-started video on YouTube](https://www.youtube.com/watch?v=22-dRnu5fXs) + **The SDK captures what your agent really did. The CLI replays it against a candidate model and tells you what broke. The hosted app keeps the history.**