From 2e7e0f8d59b08272621761e78e1a0d7ce83d060c Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Mon, 26 Oct 2026 23:33:39 +0100 Subject: [PATCH] fix: stop scoring narration beside tool calls; hide untested effect placeholders semantic and llm_judge compared the text a model wrote beside a tool call. Without tool-result fixtures the replay stops at the call, so that text is narration ("Let me check!"), not an answer. On a real planner run the source narrated beside four calls and the target did not, and both evaluators scored each as a -1.0 loss: 8 of the 20 regressions behind an inconclusive verdict. A pair where either side's turn ended in a tool call now writes no record for these two evaluators and counts as not applicable; structural checks and the tool evaluators still score it. The report's evaluator table printed a skipped test's placeholder d and CI as "0.00 negligible [0.00, 0.00]" beside a uniform -1.000 drop. Effect and 95% range now read "-" when no test ran, the executive summary drops the magnitude word there, and Effect is shown signed so it sits inside its range. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 16 +++++ DOCS.md | 8 ++- docs/evaluators.md | 8 +++ docs/methodology.md | 4 +- llms-full.txt | 7 +- src/evalshift_cli/cli/commands/evaluate.py | 13 ++++ src/evalshift_cli/evaluators/tool_models.py | 10 +++ .../reports/templates/report.html.j2 | 14 ++-- tests/unit/test_evaluate_command.py | 64 +++++++++++++++++++ tests/unit/test_reports.py | 62 +++++++++++++++++- tests/unit/test_tool_models.py | 11 ++++ 11 files changed, 205 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12da905e..1c9e36d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **`semantic` and `llm_judge` no longer score a turn that ended in a tool + call.** Without tool-result fixtures the replay stops at the call, so the + text beside it is narration ("Let me check!"), not an answer. On a real + planner run the source narrated beside four of its calls and the target did + not, and both evaluators scored each of those as a −1.0 loss: 8 of the 20 + regressions behind an `inconclusive` verdict. Such pairs now write no record + and count as not applicable; the tool evaluators measure them. +- **The report's evaluator table no longer prints a skipped test's + placeholders as measurements.** A uniform −1.000 drop read + "0.00 negligible [0.00, 0.00]"; Effect and 95% range now read "—" when no + test ran, and the executive summary drops the "negligible" label there. + Effect is shown signed, so −0.44 no longer sits beside a range of + [−0.70, −0.22] as an unsigned 0.44. + ## [1.6.1] - 2026-10-23 ### Changed diff --git a/DOCS.md b/DOCS.md index 5957d072..77a44527 100644 --- a/DOCS.md +++ b/DOCS.md @@ -530,11 +530,13 @@ Embeds both outputs and scores the target by cosine similarity to the source (so Presentation is stripped before embedding, so formatting never reads as drift: a code fence wrapping the **whole** output (` ```json … ``` `) is dropped, and an output that is a JSON object or array is re-serialized canonically (sorted keys, uniform separators). A fenced, pretty-printed object and the same object returned compact score 1.0; a fence inside prose is part of the answer and kept. An empty fenced block counts as an empty output. Whether the output is *valid, unfenced* JSON is a separate question — `json_schema` and the judge answer it. +Both outputs empty (a tool-only turn) writes no record and makes no embedding call. One empty side still scores 0.0 — a target that went silent where the source answered is the regression this evaluator exists for. A turn that **ended in a tool call** on either side is skipped the same way: the replay stops at the call, so any text beside it is narration ("Let me check!"), not an answer, and comparing it scored a target that called tools without narrating as a total loss. The tool evaluators measure those turns. + On an agent turn where **both** models answered with tool calls and no prose, there is nothing to embed — the evaluator writes **no record at all** (no provider call) rather than erroring on an empty embedding input or inventing a score. A turn where only *one* side is empty is still scored: a target that went silent where the source answered is exactly the regression this evaluator exists to catch. Since the empty side cannot be embedded (the provider 400s on empty input), the pair scores **0.0 similarity by definition** with no embedding call — still gated by `min_similarity` as usual — and the record carries `empty_side: "source" | "target"` metadata plus an explanation the report shows verbatim. The mirrored case (source silent, target answered) scores identically. ### LLM-as-judge (`evaluators.llm_judge`, list) -Pairwise A/B comparison per criterion: the judge sees the two outputs anonymised and **order-randomised** (positional-bias mitigation) and picks a winner or a tie. Winner → scores (0, 1); tie → (0.5, 0.5). Config: `criterion_name`, `criterion_prompt`, `judge_model` (per criterion; built-in default `gemini-3.1-flash-lite-preview`). Write criteria symmetric (never mention "source"/"target") and include an explicit tie instruction; prefer a judge from a third model family so it isn't grading its own relatives. Multi-turn examples render the transcript to the judge (capped at 4,000 chars). Tool-only turns — both outputs empty — write **no record at all** and spend no judge call, since comparing two empty strings only ever returns a meaningless tie, and a fabricated tie is indistinguishable from a judged one. +Pairwise A/B comparison per criterion: the judge sees the two outputs anonymised and **order-randomised** (positional-bias mitigation) and picks a winner or a tie. Winner → scores (0, 1); tie → (0.5, 0.5). Config: `criterion_name`, `criterion_prompt`, `judge_model` (per criterion; built-in default `gemini-3.1-flash-lite-preview`). Write criteria symmetric (never mention "source"/"target") and include an explicit tie instruction; prefer a judge from a third model family so it isn't grading its own relatives. Multi-turn examples render the transcript to the judge (capped at 4,000 chars). Tool-only turns — both outputs empty — write **no record at all** and spend no judge call, since comparing two empty strings only ever returns a meaningless tie, and a fabricated tie is indistinguishable from a judged one. So is a turn that **ended in a tool call** on either side is skipped the same way: the replay stops at the call, so any text beside it is narration ("Let me check!"), not an answer, and comparing it scored a target that called tools without narrating as a total loss. The tool evaluators measure those turns. ### Tool-call evaluators — see [Agent evaluation](#agent-evaluation) @@ -718,7 +720,7 @@ Trace-aware debugging: `evalshift diff case ` (side-by-side trace Full contract in [docs/methodology.md](docs/methodology.md). Per `(prompt, evaluator, slice)` comparison over the paired per-example deltas: -1. **Sample guards.** A pair an evaluator measured nothing on (a tool-only turn, an example with no ground truth) has no row in `scores.jsonl` at all, so it is absent from `n`, from the slice aggregates and from the policy metrics by construction. The count is reconstructed from `state.json`'s `evaluator_coverage` and noted, so "8 of 10 rows not applicable" survives the rows themselves. n < 5 → skipped, severity `insufficient`. 5 ≤ n < 20 → tested, but flagged uncertain. Zero-variance deltas → skipped, severity `none`. A semantic row within `min_similarity` enters the test as a delta of 0 (noted `k of n rows within the evaluator's own threshold (min_similarity) — tested as no change`); `delta_avg_score` stays the raw mean. **No applicable rows left** → severity `insufficient` with a `nothing measured:` note, never `none`; a run whose blocking evaluators all landed there cannot return `pass` — it is downgraded to `conditional_pass` (or stays `inconclusive`) with those evaluators named in `recommendations`, since an evaluator that compared nothing is unknown, not equivalent. Advisory (`blocking: false`) evaluators are exempt — their silence gates nothing by design, so they are neither named nor demote the verdict: `evaluator_coverage` carries each axis's `blocking` flag, and the synthesized comparison gets an `advisory:` note (zero rows leave the policy layer nothing else to read the flag from). The HTML report gives that row its own headline — **Nothing measured**, not the **Not enough data** the rest of `insufficient` gets. A tool evaluator is never handed a pair that was offered no tools at all — like an `applies_to` miss, that pair is not attempted: no cell, no coverage entry, so it counts toward neither the "not applicable" note nor "nothing measured". +1. **Sample guards.** A pair an evaluator measured nothing on (a tool-only turn, a `semantic`/`llm_judge` pair where either side's turn ended in a tool call, an example with no ground truth) has no row in `scores.jsonl` at all, so it is absent from `n`, from the slice aggregates and from the policy metrics by construction. The count is reconstructed from `state.json`'s `evaluator_coverage` and noted, so "8 of 10 rows not applicable" survives the rows themselves. n < 5 → skipped, severity `insufficient`. 5 ≤ n < 20 → tested, but flagged uncertain. Zero-variance deltas → skipped, severity `none`. A semantic row within `min_similarity` enters the test as a delta of 0 (noted `k of n rows within the evaluator's own threshold (min_similarity) — tested as no change`); `delta_avg_score` stays the raw mean. **No applicable rows left** → severity `insufficient` with a `nothing measured:` note, never `none`; a run whose blocking evaluators all landed there cannot return `pass` — it is downgraded to `conditional_pass` (or stays `inconclusive`) with those evaluators named in `recommendations`, since an evaluator that compared nothing is unknown, not equivalent. Advisory (`blocking: false`) evaluators are exempt — their silence gates nothing by design, so they are neither named nor demote the verdict: `evaluator_coverage` carries each axis's `blocking` flag, and the synthesized comparison gets an `advisory:` note (zero rows leave the policy layer nothing else to read the flag from). The HTML report gives that row its own headline — **Nothing measured**, not the **Not enough data** the rest of `insufficient` gets. A tool evaluator is never handed a pair that was offered no tools at all — like an `applies_to` miss, that pair is not attempted: no cell, no coverage entry, so it counts toward neither the "not applicable" note nor "nothing measured". 2. **Normality screen.** Shapiro–Wilk on the deltas at α = 0.05: normal → **paired t-test**, non-normal → **Wilcoxon signed-rank**. n > 5000 skips the screen (CLT, t-test). 3. **Effect size.** Paired Cohen's d = mean(Δ)/std(Δ), with a 95% CI — analytical for the t-test, percentile bootstrap (2,000 resamples, seeded, deterministic) for Wilcoxon. 4. **Multiple comparisons.** Benjamini–Hochberg FDR correction at α = 0.05 across *all* testable comparisons in the run (equivalent to `statsmodels fdr_bh`). @@ -1035,7 +1037,7 @@ Usually correct behaviour: no statistically significant difference. Check n (n < Neither reading changes the severity, so neither gates the verdict. A zero-variance row with no shift (every pair scored the same) reads **✓ Equivalent — Both models scored the same on every pair.** -The report's **Confidence** column turns the corrected p-value into **Certain** (< 0.01), **Likely** (< 0.05) or **Unclear**. A row where no test ran shows **—**: too few pairs, or every pair scoring the same. Its p-value is a placeholder 1.0, not evidence. +The report's **Confidence** column turns the corrected p-value into **Certain** (< 0.01), **Likely** (< 0.05) or **Unclear**. A row where no test ran shows **—**: too few pairs, or every pair scoring the same. Its p-value is a placeholder 1.0, not evidence. The **Effect** and **95% range** columns show **—** on that row too: d and its interval are placeholder zeros there, and printed as "0.00 negligible [0.00, 0.00]" they called a uniform −1.000 drop negligible. Effect is shown signed (−0.44, not 0.44), so it sits inside its own range. ### The policy verdict is `inconclusive` diff --git a/docs/evaluators.md b/docs/evaluators.md index 474df206..bd6392d5 100644 --- a/docs/evaluators.md +++ b/docs/evaluators.md @@ -108,6 +108,14 @@ sentence ("which output preserves more factual detail?"). Multiple Tool-only turns (both outputs empty) are skipped without spending a judge call. +Both text evaluators (`semantic.cosine` and `llm_judge`) also skip a +pair where **either side's turn ended in a tool call**. Without +tool-result fixtures the replay stops at the call, so text beside it is +narration ("Let me check!"), not the answer a user would read. Scoring +it marked a target that called tools without narrating as a total loss. +The tool evaluators measure those turns; coverage counts the pair as +not applicable. + Pick the judge from a **third model family**. A judge prefers output that reads like its own (self-preference bias), and A/B randomisation does nothing against that. `doctor` and `validate` warn when a diff --git a/docs/methodology.md b/docs/methodology.md index 2a9cb8f0..233c49e5 100644 --- a/docs/methodology.md +++ b/docs/methodology.md @@ -122,7 +122,9 @@ severity, and so the verdict, is unchanged. The report's Confidence column reads the adjusted p-value: **Certain** below 0.01, **Likely** below 0.05, otherwise **Unclear**. A row where no test ran (n < 5, or zero variance) shows **—** instead, because its p-value is a -placeholder 1.0. +placeholder 1.0. Its Effect and 95% range columns show **—** for the same +reason: d and its interval are placeholder zeros. Effect is printed signed, +like its range. ### Drift inside `min_similarity` is tested as no change diff --git a/llms-full.txt b/llms-full.txt index 8656b245..6bcfb97c 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -809,8 +809,12 @@ evaluators: # every evaluator config also takes blocking # no provider call. ONE empty side still scores: 0.0 similarity by definition, # no embedding call (provider 400s on empty input); min_similarity gate applies; # metadata.empty_side = "source"|"target"; either direction scored the same. + # either side's turn ENDED IN A TOOL CALL (last round has calls) -> NO record, no + # provider call (semantic AND llm_judge): text beside a call is narration, not an + # answer; the replay stops at the call. Tool evaluators measure those turns. llm_judge: # list; pairwise A/B, order-randomized; win->(0,1), tie->(.5,.5) # both outputs empty (tool-only turn) -> NO record at all, no judge call + # either side's turn ended in a tool call -> NO record, no judge call (see semantic) - criterion_name: str criterion_prompt: str # keep symmetric + explicit tie instruction judge_model: str = "gemini-3.1-flash-lite-preview" # per-criterion; defaults.judge_model is NOT consulted @@ -1884,7 +1888,8 @@ All severities "none" -> usually genuinely no significant difference; check n (< "uniform drop:"/"uniform gain:", d is a placeholder 0) reads "⚠ Uniform drop/gain". Display only: severity, and so the verdict, is unchanged. Report "Confidence" = corrected p: Certain (<0.01), Likely (<0.05), Unclear; "—" when no - test ran (n<5, or zero variance: p is a placeholder 1.0, not evidence). + test ran (n<5, or zero variance: p is a placeholder 1.0, not evidence). Effect and 95% range + also read "—" on such a row (d and CI are placeholder 0s); Effect is shown SIGNED, like its range. Verdict "inconclusive" -> (1) all evaluators advisory (fresh init: flip blocking: true as the suite grows; recommendations name each advisory judge and whether its smallest per-prompt n has reached 20, or which prompt is short), (2) rate-budget breach unconfirmed by Wilson CI (grow the suite), (3) all n<5, diff --git a/src/evalshift_cli/cli/commands/evaluate.py b/src/evalshift_cli/cli/commands/evaluate.py index 2ad203e0..c97b5190 100644 --- a/src/evalshift_cli/cli/commands/evaluate.py +++ b/src/evalshift_cli/cli/commands/evaluate.py @@ -121,6 +121,10 @@ SCORES_FILENAME: str = "scores.jsonl" +#: Text evaluators that compare the two models' *answers*. A turn that ended +#: in a tool call has none yet, so these skip it; structural checks still run. +_ANSWER_KINDS: frozenset[str] = frozenset({"semantic", "llm_judge"}) + @dataclass(frozen=True, slots=True) class _PairedCalls: @@ -1346,6 +1350,15 @@ async def _score_one( if _is_tool_evaluator(evaluator): return await _score_one_tool(evaluator, pair, run_id, examples_by_id) + # A turn that ended in a tool call has no answer yet: the replay stops + # at the call, and any text beside it is narration. Comparing narration + # (or its absence) scored a target that called tools without chatting + # as a total loss. The tool evaluators above measure these turns. + if _evaluator_kind(evaluator) in _ANSWER_KINDS and any( + c.trace is not None and c.trace.ends_in_tool_calls for c in (pair.source, pair.target) + ): + return [] + example = examples_by_id.get(pair.example_id) input_vars = example.inputs if example is not None else {} history = ( diff --git a/src/evalshift_cli/evaluators/tool_models.py b/src/evalshift_cli/evaluators/tool_models.py index 815ba94a..d5584971 100644 --- a/src/evalshift_cli/evaluators/tool_models.py +++ b/src/evalshift_cli/evaluators/tool_models.py @@ -205,6 +205,16 @@ def call_count(self) -> int: """Number of tool calls in this trace.""" return len(self.calls) + @property + def ends_in_tool_calls(self) -> bool: + """True iff the last response called a tool, so it awaits results. + + Text beside such a call is narration ("Let me check!"), never the + answer a user would read: that answer comes after the results. + """ + last = self.round_count - 1 + return any(c.round_index == last for c in self.calls) + @property def tool_names(self) -> list[str]: """Tool names in the order they were called.""" diff --git a/src/evalshift_cli/reports/templates/report.html.j2 b/src/evalshift_cli/reports/templates/report.html.j2 index 81c903bc..cd018775 100644 --- a/src/evalshift_cli/reports/templates/report.html.j2 +++ b/src/evalshift_cli/reports/templates/report.html.j2 @@ -531,9 +531,9 @@ {{ blurb }} - + {{ "%+.0f"|format(row.delta_avg_score * 100) }}% - {{ effect_magnitude(row.effect_size) }} + {% if row.test != "skipped" %}{{ effect_magnitude(row.effect_size) }}{% endif %} {% set conf = confidence_label(row.severity, row.p_value_corrected, row.test) %} @@ -773,8 +773,14 @@ — {% else %} 0 else "" }}">{{ "%+.3f"|format(row.delta_avg_score) }} - {{ "%.2f"|format(row.effect_size|abs) }} {{ effect_magnitude(row.effect_size) }} + {% if row.test == "skipped" %} + {# No test ran: d and its interval are placeholder zeros, not measurements. #} + — + — + {% else %} + {{ "%.2f"|format(row.effect_size) }} {{ effect_magnitude(row.effect_size) }} [{{ "%.2f"|format(row.effect_size_ci_low) }}, {{ "%.2f"|format(row.effect_size_ci_high) }}] + {% endif %} {% if conf == "—" %}—{% else %}{{ conf }}{% endif %} {% endif %} @@ -823,7 +829,7 @@ {{ row.n }} 0 else "" }}">{{ "%+.3f"|format(row.delta_avg_score) }} - {{ "%.2f"|format(row.effect_size|abs) }} {{ effect_magnitude(row.effect_size) }} + {{ "%.2f"|format(row.effect_size) }} {{ effect_magnitude(row.effect_size) }} {% if conf == "—" %}—{% else %}{{ conf }}{% endif %} {{ verdict_glyph(row.severity, row.notes, row.effect_size) }} {{ verdict(row.severity, row.notes, row.effect_size)[0] }} diff --git a/tests/unit/test_evaluate_command.py b/tests/unit/test_evaluate_command.py index c6914e58..a00527ce 100644 --- a/tests/unit/test_evaluate_command.py +++ b/tests/unit/test_evaluate_command.py @@ -640,6 +640,70 @@ def _pair_for(example_id: str) -> _PairedCalls: ) +class _AnswerSpyEvaluator(_SpyEvaluator): + kind = "semantic" + + +class TestTextEvaluatorsSkipUnfinishedTurns: + """A turn that ended in a tool call has no answer yet. + + Without tool-result fixtures the replay stops at the first call, so any + text beside it is narration ("Let me check!"), not an answer. Comparing + it scored a target that called tools without narrating as a total loss. + """ + + def _pair(self, *, source_calls: bool, target_calls: bool) -> _PairedCalls: + pair = _pair_for("ex1") + + def trace(calls: bool, text: str) -> ToolTrace: + return ToolTrace( + calls=[ToolCall(tool_name="search_notes", sequence_index=0)] if calls else [], + final_text=text, + ) + + return replace( + pair, + source=pair.source.model_copy( + update={"text": "Let me check!", "trace": trace(source_calls, "Let me check!")}, + ), + target=pair.target.model_copy( + update={"text": "Your gift is due Sept 11.", "trace": trace(target_calls, "x")}, + ), + ) + + @pytest.mark.parametrize( + ("source_calls", "target_calls"), + [(True, True), (True, False), (False, True)], + ) + async def test_records_nothing_when_either_side_is_awaiting_tool_results( + self, source_calls: bool, target_calls: bool + ) -> None: + spy = _AnswerSpyEvaluator() + pair = self._pair(source_calls=source_calls, target_calls=target_calls) + + records = await _score_one(spy, pair, "r1", {}) # type: ignore[arg-type] + + assert records == [] + assert spy.received_input_vars is None # never asked to score + + async def test_still_scores_when_both_sides_answered(self) -> None: + spy = _AnswerSpyEvaluator() + pair = self._pair(source_calls=False, target_calls=False) + + records = await _score_one(spy, pair, "r1", {}) # type: ignore[arg-type] + + assert len(records) == 1 + + async def test_structural_checks_still_score_unfinished_turns(self) -> None: + spy = _SpyEvaluator() + spy.kind = "structural" # type: ignore[attr-defined] + pair = self._pair(source_calls=True, target_calls=True) + + records = await _score_one(spy, pair, "r1", {}) # type: ignore[arg-type] + + assert len(records) == 1 + + class TestScoreAllConcurrency: async def test_scores_pairs_concurrently(self) -> None: tracker = _Inflight() diff --git a/tests/unit/test_reports.py b/tests/unit/test_reports.py index d3fa2e32..83f9b5fd 100644 --- a/tests/unit/test_reports.py +++ b/tests/unit/test_reports.py @@ -584,9 +584,10 @@ def test_executive_summary_is_plain_language(self, tmp_path: Path) -> None: # mistaken for conclusive evidence. assert "small-sample" in html # Raw statistics survive as hover tooltips for power users. This - # prompt's row ran no test, so its confidence tooltip says that rather - # than quoting the placeholder p of 1.000 as a corrected p-value. - assert "Cohen's |d|" in html + # prompt's row ran no test, so its tooltips say that rather than + # quoting the placeholder p of 1.000 or d of 0.00 as measurements. + assert "Δ avg score" in html + assert "Cohen's |d|" not in html assert "No test ran: too few pairs, or every pair scored the same" in html def test_per_example_breakdown_is_plain_language(self, tmp_path: Path) -> None: @@ -1850,6 +1851,61 @@ def test_a_uniform_drop_outranks_a_small_tested_dip_for_the_summary(self) -> Non rows = _build_executive_summary({"comparisons": [dip, uniform]}, set()) assert [row["evaluator_name"] for row in rows] == ["routing"] + def test_an_untested_row_shows_no_effect_or_range(self, tmp_path: Path) -> None: + """A skipped test's d and interval are placeholder zeros, not measurements. + + A real run printed "0.00 negligible [0.00, 0.00]" beside a uniform + -1.000 drop: the largest possible drop, labelled negligible. + """ + run_dir, _ = _scaffold_two_axis_run(tmp_path) + analysis_path = run_dir / ANALYSIS_FILENAME + analysis = json.loads(analysis_path.read_text(encoding="utf-8")) + for comparison in analysis["comparisons"]: + comparison.update( + severity="none", + test="skipped", + effect_size=0.0, + effect_size_ci_low=0.0, + effect_size_ci_high=0.0, + delta_avg_score=-1.0, + p_value_corrected=1.0, + notes=[ + *comparison["notes"], + ZERO_VARIANCE_NOTE, + f"{UNIFORM_DROP_NOTE_PREFIX} every pair moved by -1.000", + ], + ) + analysis_path.write_text(json.dumps(analysis), encoding="utf-8") + html = render_html( + build_report_payload(run_dir, tool_evaluator_names=frozenset({"routing"})) + ) + + evaluators = _section(html, "Overall, by evaluator") + assert "Uniform drop" in evaluators + assert "negligible" not in evaluators + assert "[0.00, 0.00]" not in evaluators + summary = _section(html, "

Executive summary

") + assert "-100%" in summary + assert "negligible" not in summary + + def test_the_effect_keeps_the_sign_its_range_has(self, tmp_path: Path) -> None: + """An unsigned 0.44 beside [-0.70, -0.22] read as outside its own interval.""" + run_dir = self._rewrite_analysis( + tmp_path, + divergence={ + "effect_size": -0.44, + "effect_size_ci_low": -0.70, + "effect_size_ci_high": -0.22, + }, + ) + html = render_html( + build_report_payload(run_dir, tool_evaluator_names=frozenset({"routing"})) + ) + + evaluators = _section(html, "Overall, by evaluator") + assert '-0.44 small' in evaluators + assert "[-0.70, -0.22]" in evaluators + def _rewrite_analysis(self, tmp_path: Path, *, divergence: dict[str, object]) -> Path: """The two-axis run with both rows non-significant and the divergence row edited.""" run_dir, _ = _scaffold_two_axis_run(tmp_path) diff --git a/tests/unit/test_tool_models.py b/tests/unit/test_tool_models.py index 6f3126d7..3e37cabf 100644 --- a/tests/unit/test_tool_models.py +++ b/tests/unit/test_tool_models.py @@ -402,3 +402,14 @@ def test_has_parallel_calls_is_per_round(self) -> None: assert not any(r.has_parallel_calls() for r in trace.rounds()) fanout = ToolTrace(round_count=2, calls=[_tc("a", 0, 0), _tc("b", 1, 0)]) assert fanout.has_parallel_calls() + + def test_ends_in_tool_calls_when_the_last_round_called(self) -> None: + """A response that called a tool is still waiting on its results.""" + assert ToolTrace(calls=[_tc("a", 0)], final_text="Let me check!").ends_in_tool_calls + assert ToolTrace(round_count=2, calls=[_tc("a", 0, 0), _tc("b", 1, 1)]).ends_in_tool_calls + + def test_does_not_end_in_tool_calls_when_the_last_round_answered(self) -> None: + assert not ToolTrace(final_text="hi").ends_in_tool_calls + assert not ToolTrace( + round_count=2, calls=[_tc("a", 0, 0)], final_text="done" + ).ends_in_tool_calls