From 34f24259572da2ddb16266a0ff3558bb9bf95f7d Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Thu, 22 Oct 2026 22:47:30 +0200 Subject: [PATCH 1/2] feat(report): list every regressed example in Top regressions, one card each The list showed the five most negative scored rows, one per example and evaluator, so one example scored -1 by four evaluators filled four of five slots and drops inside an evaluator's own threshold (scored as no change by the analysis) were listed as regressions. Cards are now per example, worst first, capped at 50 per prompt; within-threshold drops sit in a closed line below. The report also gets more room between sections and blocks. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 22 ++ DOCS.md | 4 +- README.md | 4 +- docs/agents.md | 4 +- llms-full.txt | 8 +- src/evalshift_cli/reports/json.py | 354 +++++++++++------- .../reports/templates/report.css | 33 +- .../reports/templates/report.html.j2 | 353 +++++++++-------- tests/unit/test_reports.py | 178 +++++++++ 9 files changed, 654 insertions(+), 306 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e60e1e2..bca2ecac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **The report's Top regressions lists every example that regressed, one card + per example.** It showed the five most negative scored rows, one per example + and evaluator. On a real planner run four of the five went to one example + that four evaluators each scored −1, so of the ten examples with a drop of + 0.5 or more it showed two. It now shows every regressed example, worst + first, up to 50 per prompt (the per-example breakdown lists all). Each card + names every evaluator that dropped the example, gives each one's finding, + and shows the outputs, traces and context once. The header counts examples + ("15 of 21 examples regressed"), not rows. +- A drop inside an evaluator's own threshold is no longer listed as a + regression. The analysis already tests a `semantic` row above + `min_similarity` as no change, yet a 0.93 similarity under a 0.9 threshold + sat in the list. Those examples now sit in a closed "N more examples dropped + within the evaluator's threshold (scored as no change)" line under the + cards. `report.json` keeps `top_regressions`, one row per example and + evaluator for the examples shown, and adds `regressed_example_count`, + `within_threshold_regressions` and `within_threshold_example_count`. +- The report has more room between its sections, between the blocks of a + prompt's section, and between each heading and its table. + ## [1.6.0] - 2026-10-22 ### Added diff --git a/DOCS.md b/DOCS.md index dbcc8b1b..421018df 100644 --- a/DOCS.md +++ b/DOCS.md @@ -226,7 +226,7 @@ init → doctor → run → evaluate → analy - **`run`** parses prompts, validates every example against every prompt, estimates cost, then dispatches `(prompt × example × {source, target})` calls through an async orchestrator under a concurrency semaphore. Responses are cached — per call, and per replayed round for tool-calling examples (see [Response cache](#response-cache)); progress is checkpointed every 50 completions. - **`evaluate`** scores each (source, target) pair with the configured evaluators, one `EvalRecord` per pair × evaluator. Scoring runs under the same `defaults.concurrency` semaphore as `run`, and the embedding/judge calls it makes go through the same response cache. - **`analyze`** runs paired statistics per `(prompt, evaluator, slice)`, applies Benjamini–Hochberg FDR correction, classifies severities, and — when a `migration_policy` is configured — computes a pass/fail verdict. -- **`report`** renders the single-file HTML report (no external assets; works offline and attaches cleanly to a PR or email), and writes the machine-written [run insights](#run-insights) narrative unless `--no-insights` is passed. The page opens on a verdict / advisory-signal / economics panel row and a six-cell run strip (examples, calls, failed-or-truncated, spend, latency Δ, mean score Δ), then the executive summary (one row per prompt, drawn from blocking evaluators — advisory ones only on a prompt nothing else scored; the strip's mean score Δ averages those rows), the narrative, the **Trace rules broken** panel when the target broke a hand-written trace rule, one section per prompt, and the methodology. Every figure on it is derived from the run's own artefacts; the deltas in the header are the run-level rollup of the per-prompt economics. Top regressions are collapsed cards — expand one for the trace diff, the tool diffs and the conversation context. The report is dark-only. +- **`report`** renders the single-file HTML report (no external assets; works offline and attaches cleanly to a PR or email), and writes the machine-written [run insights](#run-insights) narrative unless `--no-insights` is passed. The page opens on a verdict / advisory-signal / economics panel row and a six-cell run strip (examples, calls, failed-or-truncated, spend, latency Δ, mean score Δ), then the executive summary (one row per prompt, drawn from blocking evaluators — advisory ones only on a prompt nothing else scored; the strip's mean score Δ averages those rows), the narrative, the **Trace rules broken** panel when the target broke a hand-written trace rule, one section per prompt, and the methodology. Every figure on it is derived from the run's own artefacts; the deltas in the header are the run-level rollup of the per-prompt economics. **Top regressions** lists every example that regressed, worst first, as one collapsed card per example that names each evaluator that dropped it — expand one for each evaluator's finding, then the trace diff, the tool diffs and the conversation context, shown once. It shows at most 50 cards per prompt and says so when it cuts; the per-example breakdown lists every example. A drop inside an evaluator's own threshold (`semantic`'s `min_similarity`) is tested as no change, so it is not a regression: those examples sit in a closed line under the cards. The report is dark-only. - **`compare`** chains everything end to end over one suite and adds `--gate`, `--policy-gate`, `--push`, `--open`. Warnings raised while the pipeline runs (LiteLLM deprecation notices, insights-retry notes) are deferred and printed as one `⚠` section directly under the pipeline block — also when a stage fails, since a held-back warning may explain the failure. Errors are never deferred. With `captures.max_age` or `captures.max_cases` set, `compare` prints doctor's `suite freshness` findings for the suite it runs (named with `--suite-name` or auto-selected; not for `--suite `) as `⚠` lines above the verdict, so a CI log shows a stale suite. ### Artefacts @@ -240,7 +240,7 @@ Run ids look like `r_20260722_golden_a1b2c3` (`r___`). Un | `scores.jsonl` | evaluate | One line per (pair × evaluator): source/target scores, delta, explanation | | `analysis.json` | analyze | Per-comparison statistics, severities, notes | | `migration_decision.json` | analyze | Policy verdict + per-budget detail (only when `migration_policy` set) | -| `report.json` | report | The report payload, including `invariant_violations`: one row per [trace rule](#trace-invariants) the target broke | +| `report.json` | report | The report payload, including `invariant_violations`: one row per [trace rule](#trace-invariants) the target broke. Each prompt section's `top_regressions` holds one row per example × evaluator for the regressed examples shown, `regressed_example_count` counts every regressed example, and `within_threshold_regressions` / `within_threshold_example_count` do the same for drops inside an evaluator's threshold | | `report.html` | report | Single-file HTML report | | `insights.json` | report | Cached [run insights](#run-insights) narrative (optional; skipped with `--no-insights` or no API key) | | `traces.jsonl` | traces import | Bring-your-own agent traces (optional) | diff --git a/README.md b/README.md index 2c5334d9..5fbdc668 100644 --- a/README.md +++ b/README.md @@ -303,8 +303,8 @@ assets, works offline) has: behavioural drift explained by `defaults.insights_model`. Every figure in it is copied from the computed statistics, never generated. Needs a provider key; skip it with `--no-insights`. -* **Per-prompt deep dive** — aggregate stats, per-slice breakdown, - top-5 worst regressions side-by-side. +* **Per-prompt deep dive** — aggregate stats, per-slice breakdown, and + every regressed example side-by-side, worst first, one card per example. * **Methodology appendix** — every test, p-value, effect size, and CI is documented. diff --git a/docs/agents.md b/docs/agents.md index 744144ab..cb90ac15 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -602,8 +602,8 @@ Two places name the tools themselves, both read off the evaluator's own record rather than re-derived from the call trace: * the **per-example breakdown** gains a `Tools called (source → target)` - column — one line per example, so a ten-example suite shows all ten - rather than only the five worst; + column — one line per example, so a ten-example suite shows all ten, + regressed or not; * each **top regression** card states both sides' tools in its "why flagged" line. diff --git a/llms-full.txt b/llms-full.txt index 096ff138..52563912 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -96,7 +96,7 @@ as one section under the pipeline block (also on stage failure); errors are neve | scores.jsonl | evaluate | one EvalRecord per (pair x evaluator): source/target scores in [0,1], delta | | analysis.json | analyze | per-(prompt,evaluator,slice) statistics + severity | | migration_decision.json | analyze | policy verdict (only when migration_policy configured) | -| report.json + report.html | report | payload + single-file HTML (no external assets, dark-only). report.json top-level invariant_violations: one row per trace rule the target broke (see evaluators.trace_invariants) | +| report.json + report.html | report | payload + single-file HTML (no external assets, dark-only). report.json top-level invariant_violations: one row per trace rule the target broke (see evaluators.trace_invariants). Per prompt section: top_regressions (one row per example x evaluator, regressed examples shown), regressed_example_count, within_threshold_regressions, within_threshold_example_count | | insights.json | report | cached machine-written narrative (optional; --no-insights skips) | | traces.jsonl | traces import | optional BYO agent traces | | run_bundle.json.gz | bundle/push | optional hosted upload bundle | @@ -253,7 +253,11 @@ evalshift report RUN_ID [-c CONFIG] [--open] [--insights/--no-insights (default evaluators; advisory ones only on a prompt nothing else scored) -> insights narrative -> "Trace rules broken" panel (only when the target broke a hand-written trace rule; blocking: false rows tagged advisory) -> one section per prompt (economics, per-example, - by-evaluator, slices, top regressions as collapsible cards) -> methodology. Dark-only, no external assets, no JS. Rendered surfaces show + by-evaluator, slices, top regressions) -> methodology. Top regressions: one collapsed card + per regressed example, worst first, naming every evaluator that dropped it; at most 50 per + prompt (the per-example table lists all); a drop inside an evaluator's own threshold + (semantic min_similarity) is tested as no change, so it sits in a closed line below the + cards, not among them. Dark-only, no external assets, no JS. Rendered surfaces show display names, never internal identifiers: failure categories as plain language (TOOL_SELECTION_DRIFT -> "Different tools chosen"; map in evaluators/failures.py), budgets as their labels (max_tool_divergence -> "Tool-selection divergence"; map in diff --git a/src/evalshift_cli/reports/json.py b/src/evalshift_cli/reports/json.py index 6b2f5515..76107586 100644 --- a/src/evalshift_cli/reports/json.py +++ b/src/evalshift_cli/reports/json.py @@ -27,6 +27,7 @@ AXIS_NOTE_PREFIX, uniform_change, ) +from evalshift_cli.analysis.tolerance import within_tolerance from evalshift_cli.cli.commands.analyze import ANALYSIS_FILENAME, MIGRATION_DECISION_FILENAME from evalshift_cli.cli.commands.evaluate import SCORES_FILENAME from evalshift_cli.evaluators.base import EvalRecord @@ -57,6 +58,12 @@ REPORT_JSON_FILENAME: str = "report.json" +#: The most regressed examples a prompt's top-regressions list shows as cards. +#: Every card carries both outputs and their traces, so a suite of thousands +#: would otherwise bloat the single-file report; the per-example breakdown +#: table still lists every example. +MAX_REGRESSION_CARDS: int = 50 + #: Prefix of the display note marking an aggregate row whose every #: measurement was a shared ground-truth miss. Written here rather than in #: ``analysis`` because it is a *rendering* decision — the statistics are @@ -193,6 +200,19 @@ class TopRegression: input_text: str | None = None +@dataclass(frozen=True, slots=True) +class RegressedExample: + """One example's card in the top-regressions list. + + ``regressions`` are its drops beyond each evaluator's own threshold, worst + first. They describe the same pair of outputs, so the card shows the + example's traces and context once, from the first. + """ + + example_id: str + regressions: list[TopRegression] + + @dataclass(frozen=True, slots=True) class ExampleRow: """One row in the per-example breakdown table. @@ -256,12 +276,22 @@ class PromptSection: prompt_id: str aggregate_rows: list[dict[str, Any]] # serialised SliceAggregate per evaluator slice_rows: list[dict[str, Any]] # serialised ComparisonResult rows + # Every row of ``regressed_examples``, card by card: the per-row view + # report.json has always carried. top_regressions: list[TopRegression] economics: PromptEconomics example_rows: list[ExampleRow] # v0.2 — when at least one Call has a populated trace, render a # "Tool Trace Comparison" section with example-level diffs. has_tool_traces: bool = False + # The examples that regressed, worst first, at most + # ``MAX_REGRESSION_CARDS``; the count is before that cut. + regressed_examples: list[RegressedExample] = field(default_factory=list) + regressed_example_count: int = 0 + # Examples whose every drop stayed inside its evaluator's own threshold: + # the analysis scores them as no change, so they are not regressions. + within_threshold_examples: list[RegressedExample] = field(default_factory=list) + within_threshold_example_count: int = 0 @dataclass(frozen=True, slots=True) @@ -677,71 +707,28 @@ def _build_prompt_sections( ] # Slice rows: every non-all comparison for this prompt. slice_rows = [c for c in prompt_comparisons if c["slice_name"] != "all"] - # Top regressions: the 5 records with the most-negative delta. - ranked = sorted( - prompt_records, - key=lambda r: ( - r.delta, - 0 - if ( - (r.prompt_id, r.example_id, "source") in agent_traces - and (r.prompt_id, r.example_id, "target") in agent_traces + # Top regressions: every example that regressed, one card each. + regressed, within = _group_drops(prompt_records, agent_traces) + + def cards(groups: list[tuple[str, list[EvalRecord]]]) -> list[RegressedExample]: + return [ + RegressedExample( + example_id=example_id, + regressions=[ + _top_regression( + r, + calls_by_pair=by_prompt_calls, + agent_traces=agent_traces, + examples_by_id=examples_by_id, + ) + for r in drops + ], ) - else 1, - ), - )[:5] - top: list[TopRegression] = [] - for r in ranked: - if r.delta >= 0 or r.error is not None: - continue - calls_for_pair = by_prompt_calls.get((r.prompt_id, r.example_id), {}) - src = calls_for_pair.get("source") - tgt = calls_for_pair.get("target") - source_agent_trace = agent_traces.get((r.prompt_id, r.example_id, "source")) - target_agent_trace = agent_traces.get((r.prompt_id, r.example_id, "target")) - trace_diff = ( - diff_traces(source_agent_trace, target_agent_trace) - if source_agent_trace is not None and target_agent_trace is not None - else None - ) - example = examples_by_id.get(r.example_id) - top.append( - TopRegression( - prompt_id=r.prompt_id, - example_id=r.example_id, - evaluator_name=r.evaluator_name, - delta=r.delta, - source_text=src.text if src else "", - target_text=tgt.text if tgt else "", - kind=r.kind, - tool_change=_tool_change(r.metadata), - source_score=r.source_score, - target_score=r.target_score, - explanation=r.explanation, - metadata=r.metadata, - source_trace=src.trace if src else None, - target_trace=tgt.trace if tgt else None, - tool_diffs=_build_tool_diffs( - src.trace if src else None, tgt.trace if tgt else None - ), - source_agent_trace=source_agent_trace, - target_agent_trace=target_agent_trace, - trace_diff=trace_diff, - truncated=(bool(src and src.truncated) or bool(tgt and tgt.truncated)), - target_empty_output=bool(tgt and is_empty_output(tgt)), - history=( - [m.model_dump(exclude_none=True) for m in example.history] - if example is not None and example.history is not None - else None - ), - turn_index=example.turn_index if example is not None else None, - input_text=( - _render_input_text(example.inputs, example_id=r.example_id) - if example is not None - else None - ), - ), - ) + for example_id, drops in groups[:MAX_REGRESSION_CARDS] + ] + + regressed_examples = cards(regressed) + within_threshold_examples = cards(within) # v0.2 — does this prompt have any traces at all? Drives the # "Tool Trace Comparison" subsection in the HTML template. has_traces = any( @@ -764,15 +751,115 @@ def _build_prompt_sections( prompt_id=prompt_id, aggregate_rows=aggregates_all, slice_rows=slice_rows, - top_regressions=top, + top_regressions=[tr for card in regressed_examples for tr in card.regressions], economics=build_economics(prompt_calls), example_rows=example_rows, has_tool_traces=has_traces, + regressed_examples=regressed_examples, + regressed_example_count=len(regressed), + within_threshold_examples=within_threshold_examples, + within_threshold_example_count=len(within), ), ) return sections +def _group_drops( + records: list[EvalRecord], + agent_traces: dict[TraceKey, AgentTrace], +) -> tuple[list[tuple[str, list[EvalRecord]]], list[tuple[str, list[EvalRecord]]]]: + """Split a prompt's drops into regressed examples and within-threshold ones. + + A drop is a scored row with ``delta < 0`` and no error. An example + regressed when any drop falls beyond its evaluator's own threshold, and + its card lists just those: a drop :func:`within_tolerance` forgives is + tested as no change, so it is not a regression. An example whose every + drop stayed inside a threshold goes to the second list instead. + + Both lists run worst example first. Ties go to an example with imported + agent traces on both sides (its card can show them), then to the one that + dropped on more evaluators, then to the example id, so the cut at + ``MAX_REGRESSION_CARDS`` is deterministic. + """ + drops_by_example: dict[str, list[EvalRecord]] = {} + for record in records: + if record.delta < 0 and record.error is None: + drops_by_example.setdefault(record.example_id, []).append(record) + + regressed: list[tuple[str, list[EvalRecord]]] = [] + within: list[tuple[str, list[EvalRecord]]] = [] + for example_id, drops in drops_by_example.items(): + beyond = [r for r in drops if not within_tolerance(r)] + shown = sorted(beyond or drops, key=lambda r: (r.delta, r.evaluator_name, r.kind)) + (regressed if beyond else within).append((example_id, shown)) + + def worst_first(group: tuple[str, list[EvalRecord]]) -> tuple[float, int, int, str]: + example_id, drops = group + prompt_id = drops[0].prompt_id + imported = (prompt_id, example_id, "source") in agent_traces and ( + prompt_id, + example_id, + "target", + ) in agent_traces + return (drops[0].delta, 0 if imported else 1, -len(drops), example_id) + + return sorted(regressed, key=worst_first), sorted(within, key=worst_first) + + +def _top_regression( + r: EvalRecord, + *, + calls_by_pair: dict[tuple[str, str], dict[str, Call]], + agent_traces: dict[TraceKey, AgentTrace], + examples_by_id: dict[str, SuiteExample], +) -> TopRegression: + """One drop with the pair's outputs, traces and context attached.""" + calls_for_pair = calls_by_pair.get((r.prompt_id, r.example_id), {}) + src = calls_for_pair.get("source") + tgt = calls_for_pair.get("target") + source_agent_trace = agent_traces.get((r.prompt_id, r.example_id, "source")) + target_agent_trace = agent_traces.get((r.prompt_id, r.example_id, "target")) + trace_diff = ( + diff_traces(source_agent_trace, target_agent_trace) + if source_agent_trace is not None and target_agent_trace is not None + else None + ) + example = examples_by_id.get(r.example_id) + return TopRegression( + prompt_id=r.prompt_id, + example_id=r.example_id, + evaluator_name=r.evaluator_name, + delta=r.delta, + source_text=src.text if src else "", + target_text=tgt.text if tgt else "", + kind=r.kind, + tool_change=_tool_change(r.metadata), + source_score=r.source_score, + target_score=r.target_score, + explanation=r.explanation, + metadata=r.metadata, + source_trace=src.trace if src else None, + target_trace=tgt.trace if tgt else None, + tool_diffs=_build_tool_diffs(src.trace if src else None, tgt.trace if tgt else None), + source_agent_trace=source_agent_trace, + target_agent_trace=target_agent_trace, + trace_diff=trace_diff, + truncated=(bool(src and src.truncated) or bool(tgt and tgt.truncated)), + target_empty_output=bool(tgt and is_empty_output(tgt)), + history=( + [m.model_dump(exclude_none=True) for m in example.history] + if example is not None and example.history is not None + else None + ), + turn_index=example.turn_index if example is not None else None, + input_text=( + _render_input_text(example.inputs, example_id=r.example_id) + if example is not None + else None + ), + ) + + def _build_example_rows( *, prompt_id: str, @@ -1133,6 +1220,69 @@ def _tool_change_to_dict(change: ToolChange | None) -> dict[str, Any] | None: return payload +def _top_regression_to_dict(tr: TopRegression) -> dict[str, Any]: + """One regression row as report.json writes it, in either list.""" + return { + "prompt_id": tr.prompt_id, + "example_id": tr.example_id, + "evaluator_name": tr.evaluator_name, + "kind": tr.kind, + "tool_change": _tool_change_to_dict(tr.tool_change), + "delta": tr.delta, + "truncated": tr.truncated, + "target_empty_output": tr.target_empty_output, + "history": tr.history, + "turn_index": tr.turn_index, + "input_text": tr.input_text, + "source_score": tr.source_score, + "target_score": tr.target_score, + "explanation": tr.explanation, + "metadata": tr.metadata, + "source_text": tr.source_text, + "target_text": tr.target_text, + "source_trace": (tr.source_trace.model_dump() if tr.source_trace else None), + "target_trace": (tr.target_trace.model_dump() if tr.target_trace else None), + "source_agent_trace": ( + tr.source_agent_trace.model_dump(mode="json") if tr.source_agent_trace else None + ), + "target_agent_trace": ( + tr.target_agent_trace.model_dump(mode="json") if tr.target_agent_trace else None + ), + "trace_diff": ( + { + "prompt_id": tr.trace_diff.prompt_id, + "example_id": tr.trace_diff.example_id, + "items": [ + { + "kind": item.kind, + "category": item.category, + "source_index": item.source_index, + "target_index": item.target_index, + "source_name": item.source_name, + "target_name": item.target_name, + "field": item.field, + "source_value": item.source_value, + "target_value": item.target_value, + } + for item in tr.trace_diff.items + ], + } + if tr.trace_diff + else None + ), + "tool_diffs": [ + { + "kind": d.kind, + "tool_name": d.tool_name, + "message": d.message, + "source_arguments": d.source_arguments, + "target_arguments": d.target_arguments, + } + for d in tr.tool_diffs + ], + } + + def _to_jsonable(report: ReportData) -> dict[str, Any]: return { "run_id": report.run_id, @@ -1172,72 +1322,14 @@ def _to_jsonable(report: ReportData) -> dict[str, Any]: } for er in ps.example_rows ], - "top_regressions": [ - { - "prompt_id": tr.prompt_id, - "example_id": tr.example_id, - "evaluator_name": tr.evaluator_name, - "kind": tr.kind, - "tool_change": _tool_change_to_dict(tr.tool_change), - "delta": tr.delta, - "truncated": tr.truncated, - "target_empty_output": tr.target_empty_output, - "history": tr.history, - "turn_index": tr.turn_index, - "input_text": tr.input_text, - "source_score": tr.source_score, - "target_score": tr.target_score, - "explanation": tr.explanation, - "metadata": tr.metadata, - "source_text": tr.source_text, - "target_text": tr.target_text, - "source_trace": (tr.source_trace.model_dump() if tr.source_trace else None), - "target_trace": (tr.target_trace.model_dump() if tr.target_trace else None), - "source_agent_trace": ( - tr.source_agent_trace.model_dump(mode="json") - if tr.source_agent_trace - else None - ), - "target_agent_trace": ( - tr.target_agent_trace.model_dump(mode="json") - if tr.target_agent_trace - else None - ), - "trace_diff": ( - { - "prompt_id": tr.trace_diff.prompt_id, - "example_id": tr.trace_diff.example_id, - "items": [ - { - "kind": item.kind, - "category": item.category, - "source_index": item.source_index, - "target_index": item.target_index, - "source_name": item.source_name, - "target_name": item.target_name, - "field": item.field, - "source_value": item.source_value, - "target_value": item.target_value, - } - for item in tr.trace_diff.items - ], - } - if tr.trace_diff - else None - ), - "tool_diffs": [ - { - "kind": d.kind, - "tool_name": d.tool_name, - "message": d.message, - "source_arguments": d.source_arguments, - "target_arguments": d.target_arguments, - } - for d in tr.tool_diffs - ], - } - for tr in ps.top_regressions + "top_regressions": [_top_regression_to_dict(tr) for tr in ps.top_regressions], + "regressed_example_count": ps.regressed_example_count, + "within_threshold_regressions": [ + _top_regression_to_dict(tr) + for card in ps.within_threshold_examples + for tr in card.regressions ], + "within_threshold_example_count": ps.within_threshold_example_count, } for ps in report.prompt_sections ], @@ -1251,11 +1343,13 @@ def _to_jsonable(report: ReportData) -> dict[str, Any]: __all__ = [ + "MAX_REGRESSION_CARDS", "REPORT_JSON_FILENAME", "SHARED_GROUND_TRUTH_NOTE_PREFIX", "InvariantViolationRow", "PromptEconomics", "PromptSection", + "RegressedExample", "ReportData", "RoleEconomics", "RoundToolChange", diff --git a/src/evalshift_cli/reports/templates/report.css b/src/evalshift_cli/reports/templates/report.css index 61d4c50c..11dda753 100644 --- a/src/evalshift_cli/reports/templates/report.css +++ b/src/evalshift_cli/reports/templates/report.css @@ -70,7 +70,7 @@ body { margin: 0 auto; display: flex; flex-direction: column; - gap: 30px; + gap: 56px; } a { color: var(--accent); text-underline-offset: 4px; } @@ -203,7 +203,7 @@ h1 { /* --- Section scaffolding ------------------------------------------------ */ -section { display: flex; flex-direction: column; gap: 12px; } +section { display: flex; flex-direction: column; gap: 16px; } /* Section heads rule above and below, like the landing's SectionHeader. */ .section-head { display: flex; @@ -260,7 +260,10 @@ h4 { max-width: 820px; text-wrap: pretty; } -.block { display: flex; flex-direction: column; gap: 10px; } +.block { display: flex; flex-direction: column; gap: 14px; } +/* Blocks inside one section are separate findings: give each its own room + rather than butting a heading against the table above it. */ +.block + .block { margin-top: 20px; } p.empty { margin: 0; padding: 22px; @@ -616,6 +619,15 @@ details[open] > summary .caret::after { content: "\25BE"; } } .reg-why { display: flex; flex-direction: column; gap: 8px; } .reg-why-line { display: flex; align-items: baseline; gap: 10px; flex-wrap: wrap; } +/* A card several evaluators flagged names each finding and its own delta. */ +.reg-why-evaluator { font-size: 14px; font-weight: 600; color: var(--fg); } +.reg-why-delta { + margin-left: auto; + font-family: var(--font-mono); + font-size: 13px; + font-weight: 600; + color: var(--bad); +} .reg-scores { font-family: var(--font-mono); font-size: 13px; color: var(--fg); } .reg-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 14px; } .reg-col { display: flex; flex-direction: column; gap: 8px; } @@ -665,12 +677,12 @@ details[open] > summary .caret::after { content: "\25BE"; } } .tool-diffs strong { color: var(--fg); } -.transcript, .reg-input { +.transcript, .reg-input, .within-threshold { margin: 0; background: var(--well); border: 1px solid var(--stroke); } -.transcript > summary, .reg-input > summary { +.transcript > summary, .reg-input > summary, .within-threshold > summary { display: flex; align-items: center; gap: 10px; @@ -679,6 +691,14 @@ details[open] > summary .caret::after { content: "\25BE"; } font-size: 13px; color: var(--accent); } +/* Drops the analysis scored as no change: secondary, so muted, not accent. */ +.within-threshold > summary { color: var(--dim); } +.within-threshold-body { + display: flex; + flex-direction: column; + gap: 10px; + padding: 0 14px 14px; +} .transcript-body { margin: 0; padding: 0 14px 14px 36px; @@ -780,7 +800,8 @@ footer p { margin: 0; } @media (max-width: 600px) { body { padding: 20px 12px 48px; } - .page { gap: 24px; } + .page { gap: 36px; } + .block + .block { margin-top: 8px; } h1 { font-size: 26px; } .hero-value .big { font-size: 24px; } .strip-value { font-size: 17px; } diff --git a/src/evalshift_cli/reports/templates/report.html.j2 b/src/evalshift_cli/reports/templates/report.html.j2 index 141cc1a9..a75a2b8b 100644 --- a/src/evalshift_cli/reports/templates/report.html.j2 +++ b/src/evalshift_cli/reports/templates/report.html.j2 @@ -34,6 +34,178 @@ {%- endif -%} {%- endmacro -%} +{#- One regressed example: why each evaluator flagged it, then the pair's + outputs and context once. The outputs are the same whichever evaluator + scored them, so a card per evaluator repeated them and let one example + fill the list. -#} +{%- macro regression_card(card, opened=False) -%} +{%- set first = card.regressions[0] -%} +{%- set single = card.regressions|length == 1 -%} +
+ + + {{ card.example_id }} + {% if first.turn_index is not none %}turn {{ first.turn_index }}{% endif %} + {% for tr in card.regressions %}{{ evaluator_label(tr.evaluator_name, tr.kind) }}{% if not loop.last %} · {% endif %}{% endfor %} + {% if first.truncated %}truncated (token cap){% endif %} + {% if first.target_empty_output %}empty output{% endif %} + {{ "%+.3f"|format(first.delta) }} + + +
+ {% for tr in card.regressions %} + {% set reason = regression_reason(tr) %} +
+
+ Why flagged + {% if not single %}{{ evaluator_label(tr.evaluator_name, tr.kind) }}{% endif %} + source {{ "%.2f"|format(tr.source_score) }} → target {{ "%.2f"|format(tr.target_score) }} (0–1) + {#- The evaluator's own failure category when it wrote one: it names + what went wrong, where the axis slug only names what was scored. -#} + {%- set categories = tr.metadata.get("failure_categories") or [] %} + {% for category in categories %}{{ category_label(category) }}{% endfor %} + {% if not categories and tr.kind %}{{ kind_label(tr.kind) }}{% endif %} + {% if not single %}{{ "%+.3f"|format(tr.delta) }}{% endif %} +
+ {% if reason %}

{{ reason }}

{% endif %} +
+ {% if tr.tool_change %} +
+ Tools called + {{ tool_change_arrows(tr.tool_change) }} +
+ {% endif %} + {% endfor %} + + {#- Each column head quotes a score only when one evaluator flagged the + card; with several, the scores are in their findings above. -#} + {% if first.source_agent_trace or first.target_agent_trace %} +
+
+
Source agent trace{% if single %}score {{ "%.3f"|format(first.source_score) }}{% endif %}
+ {% if first.source_agent_trace and first.source_agent_trace.tool_calls %} +
    + {% for c in first.source_agent_trace.tool_calls %} +
  1. {{ c.name }} ({{ c.arguments|tojson }})
    2. + {% endfor %} +
+ {% else %} +

(no tool calls)

+ {% endif %} +
+
+
Target agent trace{% if single %}score {{ "%.3f"|format(first.target_score) }}{% endif %}
+ {% if first.target_agent_trace and first.target_agent_trace.tool_calls %} + {% set source_names = (first.source_agent_trace.tool_calls if first.source_agent_trace else []) | map(attribute='name') | list %} +
    + {% for c in first.target_agent_trace.tool_calls %} +
  1. {{ c.name }} ({{ c.arguments|tojson }})
    2. + {% endfor %} +
+ {% else %} +

(no tool calls)

+ {% endif %} +
+
+ {% if first.trace_diff and first.trace_diff.items %} +
+ Tool diffs +
    + {% for diff in first.trace_diff.items %} +
  • {{ category_label(diff.category) }}: {{ diff.kind|replace("_", " ") }} + {% if diff.field %} · {{ diff.field }}: {{ diff.source_value|tojson }} → {{ diff.target_value|tojson }}{% endif %} + {% if diff.source_name or diff.target_name %} · {{ diff.source_name or "-" }} → {{ diff.target_name or "-" }}{% endif %} +
    • + {% endfor %} +
+
+ {% endif %} + + {% elif first.source_trace or first.target_trace %} + {# v0.2 — tool regression: render the trace diff in place of text. #} +
+
+
Source trace{% if single %}score {{ "%.3f"|format(first.source_score) }}{% endif %}
+ {% if first.source_trace and first.source_trace.calls %} +
    + {% for round_index, calls in first.source_trace.calls|groupby("round_index") %} + {% if first.source_trace.round_count > 1 %}
  1. round {{ round_index + 1 }}
    2. {% endif %} + {% for c in calls %} +
  2. {{ c.tool_name }} ({{ c.arguments|tojson }})
    3. + {% endfor %} + {% endfor %} +
+ {% else %} +

(no tool calls)

+ {% endif %} + {% if first.source_trace and first.source_trace.final_text %} +

final text: {{ first.source_trace.final_text }}

+ {% endif %} +
+
+
Target trace{% if single %}score {{ "%.3f"|format(first.target_score) }}{% endif %}
+ {% if first.target_trace and first.target_trace.calls %} + {% set source_names = (first.source_trace.calls if first.source_trace else []) | map(attribute='tool_name') | list %} +
    + {% for round_index, calls in first.target_trace.calls|groupby("round_index") %} + {% if first.target_trace.round_count > 1 %}
  1. round {{ round_index + 1 }}
    2. {% endif %} + {% for c in calls %} +
  2. {{ c.tool_name }} ({{ c.arguments|tojson }})
    3. + {% endfor %} + {% endfor %} +
+ {% else %} +

(no tool calls)

+ {% endif %} + {% if first.target_trace and first.target_trace.final_text %} +

final text: {{ first.target_trace.final_text }}

+ {% endif %} +
+
+ {% if first.tool_diffs %} +
+ Tool diffs +
    + {% for diff in first.tool_diffs %} +
  • {{ diff.kind|replace("_", " ") }}: {{ diff.message }}
    • + {% endfor %} +
+
+ {% endif %} + + {% else %} +
+
+
Source{% if single %}score {{ "%.3f"|format(first.source_score) }}{% endif %}
+ 
{{ first.source_text }}
+
+
+
Target{% if single %}score {{ "%.3f"|format(first.target_score) }}{% endif %}
+ 
{{ first.target_text }}
+
+
+ {% endif %} + + {% if first.history %} +
+ Conversation context ({{ first.history|length }} messages) +
    + {% for msg in first.history %} +
  1. [{{ msg.role }}] {{ msg.content }}{% for call in msg.tool_calls or [] %} → {{ call.name }}({{ call.arguments | tojson }}){% endfor %}
    2. + {% endfor %} +
+
+ {% endif %} + {% if first.input_text %} +
+ Input ({{ first.input_text|length }} chars) + 
{{ first.input_text }}
+
+ {% endif %} +
+
+{%- endmacro -%} + {%- set decision = migration_decision -%} {%- set budgets = decision.budget_results if decision else [] -%} {%- set rates = (decision.overall if decision and decision.overall.n_records > 0 @@ -637,176 +809,33 @@
-
+

Top regressions

- {% if section.top_regressions %}{{ section.top_regressions|length }} of {{ section.example_rows|length }} examples{% endif %} + {% if section.regressed_example_count or section.within_threshold_example_count %}{{ section.regressed_example_count }} of {{ section.example_rows|length }} examples regressed{% endif %} + {% if section.regressed_example_count > section.regressed_examples|length %}· showing the worst {{ section.regressed_examples|length }}; every example is in the Per-example breakdown above{% endif %}
- {% if section.top_regressions %} - {% for tr in section.top_regressions %} - {% set reason = regression_reason(tr) %} + {% if section.regressed_examples %} {#- The worst regression starts open: seeing one expanded is what tells the reader the rest expand too. -#} -
- - - {{ tr.example_id }} - {% if tr.turn_index is not none %}turn {{ tr.turn_index }}{% endif %} - {{ evaluator_label(tr.evaluator_name, tr.kind) }} - {% if tr.truncated %}truncated (token cap){% endif %} - {% if tr.target_empty_output %}empty output{% endif %} - {{ "%+.3f"|format(tr.delta) }} - - -
-
-
- Why flagged - source {{ "%.2f"|format(tr.source_score) }} → target {{ "%.2f"|format(tr.target_score) }} (0–1) - {#- The evaluator's own failure category when it wrote one: it names - what went wrong, where the axis slug only names what was scored. -#} - {%- set categories = tr.metadata.get("failure_categories") or [] %} - {% for category in categories %}{{ category_label(category) }}{% endfor %} - {% if not categories and tr.kind %}{{ kind_label(tr.kind) }}{% endif %} -
- {% if reason %}

{{ reason }}

{% endif %} -
- - {% if tr.tool_change %} -
- Tools called - {{ tool_change_arrows(tr.tool_change) }} -
- {% endif %} - - {% if tr.source_agent_trace or tr.target_agent_trace %} -
-
-
Source agent tracescore {{ "%.3f"|format(tr.source_score) }}
- {% if tr.source_agent_trace and tr.source_agent_trace.tool_calls %} -
    - {% for c in tr.source_agent_trace.tool_calls %} -
  1. {{ c.name }} ({{ c.arguments|tojson }})
    2. - {% endfor %} -
- {% else %} -

(no tool calls)

- {% endif %} -
-
-
Target agent tracescore {{ "%.3f"|format(tr.target_score) }}
- {% if tr.target_agent_trace and tr.target_agent_trace.tool_calls %} - {% set source_names = (tr.source_agent_trace.tool_calls if tr.source_agent_trace else []) | map(attribute='name') | list %} -
    - {% for c in tr.target_agent_trace.tool_calls %} -
  1. {{ c.name }} ({{ c.arguments|tojson }})
    2. - {% endfor %} -
- {% else %} -

(no tool calls)

- {% endif %} -
-
- {% if tr.trace_diff and tr.trace_diff.items %} -
- Tool diffs -
    - {% for diff in tr.trace_diff.items %} -
  • {{ category_label(diff.category) }}: {{ diff.kind|replace("_", " ") }} - {% if diff.field %} · {{ diff.field }}: {{ diff.source_value|tojson }} → {{ diff.target_value|tojson }}{% endif %} - {% if diff.source_name or diff.target_name %} · {{ diff.source_name or "-" }} → {{ diff.target_name or "-" }}{% endif %} -
    • - {% endfor %} -
-
- {% endif %} - - {% elif tr.source_trace or tr.target_trace %} - {# v0.2 — tool regression: render the trace diff in place of text. #} -
-
-
Source tracescore {{ "%.3f"|format(tr.source_score) }}
- {% if tr.source_trace and tr.source_trace.calls %} -
    - {% for round_index, calls in tr.source_trace.calls|groupby("round_index") %} - {% if tr.source_trace.round_count > 1 %}
  1. round {{ round_index + 1 }}
    2. {% endif %} - {% for c in calls %} -
  2. {{ c.tool_name }} ({{ c.arguments|tojson }})
    3. - {% endfor %} - {% endfor %} -
- {% else %} -

(no tool calls)

- {% endif %} - {% if tr.source_trace and tr.source_trace.final_text %} -

final text: {{ tr.source_trace.final_text }}

- {% endif %} -
-
-
Target tracescore {{ "%.3f"|format(tr.target_score) }}
- {% if tr.target_trace and tr.target_trace.calls %} - {% set source_names = (tr.source_trace.calls if tr.source_trace else []) | map(attribute='tool_name') | list %} -
    - {% for round_index, calls in tr.target_trace.calls|groupby("round_index") %} - {% if tr.target_trace.round_count > 1 %}
  1. round {{ round_index + 1 }}
    2. {% endif %} - {% for c in calls %} -
  2. {{ c.tool_name }} ({{ c.arguments|tojson }})
    3. - {% endfor %} - {% endfor %} -
- {% else %} -

(no tool calls)

- {% endif %} - {% if tr.target_trace and tr.target_trace.final_text %} -

final text: {{ tr.target_trace.final_text }}

- {% endif %} -
-
- {% if tr.tool_diffs %} -
- Tool diffs -
    - {% for diff in tr.tool_diffs %} -
  • {{ diff.kind|replace("_", " ") }}: {{ diff.message }}
    • - {% endfor %} -
-
- {% endif %} - - {% else %} -
-
-
Sourcescore {{ "%.3f"|format(tr.source_score) }}
- 
{{ tr.source_text }}
-
-
-
Targetscore {{ "%.3f"|format(tr.target_score) }}
- 
{{ tr.target_text }}
-
-
- {% endif %} - - {% if tr.history %} -
- Conversation context ({{ tr.history|length }} messages) -
    - {% for msg in tr.history %} -
  1. [{{ msg.role }}] {{ msg.content }}{% for call in msg.tool_calls or [] %} → {{ call.name }}({{ call.arguments | tojson }}){% endfor %}
    2. - {% endfor %} -
-
- {% endif %} - {% if tr.input_text %} -
- Input ({{ tr.input_text|length }} chars) - 
{{ tr.input_text }}
-
- {% endif %} -
-
+ {% for card in section.regressed_examples %} + {{ regression_card(card, opened=loop.first) }} {% endfor %} {% else %}

No specific regressions to surface.

{% endif %} + {#- Drops the analysis tests as no change: listed, closed, and named for + what they are, so the list above holds only regressions. -#} + {% if section.within_threshold_examples %} + {% set within_count = section.within_threshold_example_count %} +
+ {{ within_count }} {% if section.regressed_example_count %}more {% endif %}example{{ "s" if within_count != 1 }} dropped within the evaluator's threshold (scored as no change){% if within_count > section.within_threshold_examples|length %} · showing the largest {{ section.within_threshold_examples|length }}{% endif %} +
+ {% for card in section.within_threshold_examples %} + {{ regression_card(card) }} + {% endfor %} +
+
+ {% endif %}
{% endfor %} diff --git a/tests/unit/test_reports.py b/tests/unit/test_reports.py index a6641697..d9171da6 100644 --- a/tests/unit/test_reports.py +++ b/tests/unit/test_reports.py @@ -2949,3 +2949,181 @@ def test_evaluator_table_says_what_a_trace_rule_row_measures(self, tmp_path: Pat ) html = render_html(build_report_payload(run_dir)) assert "each side judged against the entry" in html + + +# --------------------------------------------------------------------------- +# Top regressions — every real regression, one card per example +# --------------------------------------------------------------------------- + + +class TestTopRegressionsListEveryRealRegression: + """Regressions the analysis counts, grouped by example, not the five worst rows.""" + + @staticmethod + def _record( + example_id: str, + delta: float, + *, + evaluator_name: str = "structural.length", + kind: str = "", + metadata: dict[str, Any] | None = None, + error: str | None = None, + ) -> dict[str, Any]: + return { + "run_id": "r_20260601_aaaaaa", + "prompt_id": "greet", + "example_id": example_id, + "evaluator_name": evaluator_name, + "kind": kind, + "source_score": 1.0 if delta <= 0 else 0.0, + "target_score": 1.0 + delta if delta <= 0 else delta, + "delta": delta, + "metadata": metadata or {}, + "error": error, + } + + @classmethod + def _semantic(cls, example_id: str, delta: float, *, beyond_threshold: bool) -> dict[str, Any]: + categories = ["SEMANTIC_REGRESSION"] if beyond_threshold else [] + return cls._record( + example_id, + delta, + evaluator_name="semantic.cosine", + kind="semantic", + metadata={"failure_categories": categories}, + ) + + @staticmethod + def _run(tmp_path: Path, records: list[dict[str, Any]]) -> Path: + """The scaffolded run with ``records`` as its only scores.""" + cwd, run_id = _scaffold_full_run(tmp_path) + run_dir = cwd / ".evalshift" / "runs" / run_id + (run_dir / SCORES_FILENAME).write_text( + "".join(EvalRecord.model_validate(r).model_dump_json() + "\n" for r in records), + encoding="utf-8", + ) + return run_dir + + def _threshold_run(self, tmp_path: Path) -> Path: + # ex1 drops inside min_similarity (scored as no change); ex2 breaches it. + return self._run( + tmp_path, + [ + self._semantic("ex1", -0.07, beyond_threshold=False), + self._semantic("ex2", -0.2, beyond_threshold=True), + ], + ) + + @staticmethod + def _top(html: str) -> str: + """The top-regressions block, from its heading to the methodology.""" + return html.split("

Top regressions

", 1)[1].split('

', 1)[0] + + def _staircase_run(self, tmp_path: Path, count: int) -> Path: + # e000 drops 0.01, e001 0.02, ... so the worst is the last one. + return self._run( + tmp_path, [self._record(f"e{i:03d}", -(i + 1) / 100) for i in range(count)] + ) + + def test_a_drop_inside_the_evaluators_threshold_is_not_a_regression( + self, tmp_path: Path + ) -> None: + section = build_report_payload(self._threshold_run(tmp_path)).prompt_sections[0] + assert [g.example_id for g in section.regressed_examples] == ["ex2"] + assert section.regressed_example_count == 1 + assert {tr.example_id for tr in section.top_regressions} == {"ex2"} + assert [g.example_id for g in section.within_threshold_examples] == ["ex1"] + assert section.within_threshold_example_count == 1 + + def test_one_card_lists_every_evaluator_that_dropped_worst_first(self, tmp_path: Path) -> None: + run_dir = self._run( + tmp_path, + [ + self._record("ex1", -0.5), + self._record( + "ex1", -1.0, evaluator_name="routing", kind="tool_selection.divergence" + ), + # Inside the threshold: no change, so not on the card. + self._semantic("ex1", -0.05, beyond_threshold=False), + # A gain and an errored row are not drops. + self._record("ex1", 1.0, evaluator_name="llm_judge.equivalence", kind="llm_judge"), + self._record("ex1", -1.0, evaluator_name="agent_contract", error="judge failed"), + ], + ) + section = build_report_payload(run_dir).prompt_sections[0] + [card] = section.regressed_examples + assert [(tr.evaluator_name, tr.kind, tr.delta) for tr in card.regressions] == [ + ("routing", "tool_selection.divergence", -1.0), + ("structural.length", "", -0.5), + ] + assert section.within_threshold_examples == [] + + def test_every_regressed_example_is_listed_worst_first(self, tmp_path: Path) -> None: + deltas = [-0.3, -0.7, -0.1, -0.6, -0.2, -0.5, -0.4] + run_dir = self._run(tmp_path, [self._record(f"e{i}", d) for i, d in enumerate(deltas)]) + section = build_report_payload(run_dir).prompt_sections[0] + assert [g.regressions[0].delta for g in section.regressed_examples] == sorted(deltas) + assert section.regressed_example_count == len(deltas) + + def test_cards_stop_at_the_cap_and_the_count_does_not(self, tmp_path: Path) -> None: + from evalshift_cli.reports.json import MAX_REGRESSION_CARDS + + total = MAX_REGRESSION_CARDS + 2 + section = build_report_payload(self._staircase_run(tmp_path, total)).prompt_sections[0] + assert len(section.regressed_examples) == MAX_REGRESSION_CARDS + assert section.regressed_example_count == total + # The cut keeps the worst: the last example leads, the two mildest go. + assert section.regressed_examples[0].example_id == f"e{total - 1:03d}" + assert {"e000", "e001"}.isdisjoint(g.example_id for g in section.regressed_examples) + + def test_header_counts_examples_not_rows(self, tmp_path: Path) -> None: + html = render_html(build_report_payload(self._threshold_run(tmp_path))) + assert '1 of 2 examples regressed' in html + + def test_drops_inside_the_threshold_sit_in_a_closed_list(self, tmp_path: Path) -> None: + html = render_html(build_report_payload(self._threshold_run(tmp_path))) + top = self._top(html) + line = "1 more example dropped within the evaluator's threshold (scored as no change)" + assert line in top + before, within = top.split('
', 1) + assert 'ex1' in within + assert 'ex1' not in before + + def test_a_card_shows_each_evaluators_finding_once(self, tmp_path: Path) -> None: + run_dir = self._run( + tmp_path, + [ + self._record("ex1", -0.5), + self._record( + "ex1", -1.0, evaluator_name="routing", kind="tool_selection.divergence" + ), + ], + ) + html = render_html(build_report_payload(run_dir)) + top = self._top(html) + assert top.count('ex1') == 1 + assert top.count("Why flagged") == 2 + # The example's own outputs show once, not once per evaluator. + assert top.count("
Hello there!
") == 1 + + def test_the_cap_names_where_the_rest_are(self, tmp_path: Path) -> None: + from evalshift_cli.reports.json import MAX_REGRESSION_CARDS + + html = render_html(build_report_payload(self._staircase_run(tmp_path, 60))) + note = f"showing the worst {MAX_REGRESSION_CARDS}" + assert note in html + assert "Per-example breakdown" in html.split(note, 1)[1].split("", 1)[0] + + def test_report_json_carries_the_counts_and_the_within_threshold_rows( + self, tmp_path: Path + ) -> None: + run_dir = self._threshold_run(tmp_path) + write_report_json(build_report_payload(run_dir), run_dir) + report = json.loads((run_dir / REPORT_JSON_FILENAME).read_text(encoding="utf-8")) + section = report["prompt_sections"][0] + assert section["regressed_example_count"] == 1 + assert section["within_threshold_example_count"] == 1 + assert [r["example_id"] for r in section["top_regressions"]] == ["ex2"] + assert [r["example_id"] for r in section["within_threshold_regressions"]] == ["ex1"] + # The same per-row shape as top_regressions. + assert set(section["within_threshold_regressions"][0]) == set(section["top_regressions"][0]) From b27658a87da9c7889c7c522d541534542cb6aa21 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Thu, 22 Oct 2026 22:59:33 +0200 Subject: [PATCH 2/2] fix(report): within-threshold cards and the per-example table stop claiming regressions Review of the previous commit: a card in the within-threshold list still read "Why flagged" in red and said the meaning drifted, and the per-example breakdown tinted every negative row, so a run with 15 regressed examples showed 21 red rows. Within cards now read "Within threshold" in a neutral tone; breakdown rows carry regressed, are tinted only when an evaluator dropped beyond its threshold, and quote that drop; forgiven drops show muted. Adds tests for tie order, the within list's cap and an empty regression list; fixes two stale comments and the report.json doc wording. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 16 +++-- DOCS.md | 4 +- llms-full.txt | 6 +- src/evalshift_cli/insights/facts.py | 2 +- src/evalshift_cli/reports/html.py | 19 +++++- src/evalshift_cli/reports/json.py | 21 ++++-- .../reports/templates/report.css | 3 + .../reports/templates/report.html.j2 | 36 +++++----- tests/unit/test_reports.py | 68 ++++++++++++++++++- 9 files changed, 144 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bca2ecac..c1a3ad11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,12 +20,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ("15 of 21 examples regressed"), not rows. - A drop inside an evaluator's own threshold is no longer listed as a regression. The analysis already tests a `semantic` row above - `min_similarity` as no change, yet a 0.93 similarity under a 0.9 threshold + `min_similarity` as no change, yet a 0.93 similarity with a 0.9 threshold sat in the list. Those examples now sit in a closed "N more examples dropped within the evaluator's threshold (scored as no change)" line under the - cards. `report.json` keeps `top_regressions`, one row per example and - evaluator for the examples shown, and adds `regressed_example_count`, - `within_threshold_regressions` and `within_threshold_example_count`. + cards, and their cards read "Within threshold" in a neutral tone rather than + "Why flagged" in red. `report.json` keeps `top_regressions`, one row per + example and evaluator for the examples shown, and adds + `regressed_example_count`, `within_threshold_regressions` and + `within_threshold_example_count`. +- The per-example breakdown applies the same rule. It tinted every row with a + negative delta, so all 21 rows of that planner run were red under a list of + 15 regressed examples. Only a regressed example's row is tinted now, it + quotes its worst drop beyond a threshold (the delta its card leads with), + and a forgiven drop shows muted. Each `example_rows` row in `report.json` + carries `regressed`. - The report has more room between its sections, between the blocks of a prompt's section, and between each heading and its table. diff --git a/DOCS.md b/DOCS.md index 421018df..73765a12 100644 --- a/DOCS.md +++ b/DOCS.md @@ -226,7 +226,7 @@ init → doctor → run → evaluate → analy - **`run`** parses prompts, validates every example against every prompt, estimates cost, then dispatches `(prompt × example × {source, target})` calls through an async orchestrator under a concurrency semaphore. Responses are cached — per call, and per replayed round for tool-calling examples (see [Response cache](#response-cache)); progress is checkpointed every 50 completions. - **`evaluate`** scores each (source, target) pair with the configured evaluators, one `EvalRecord` per pair × evaluator. Scoring runs under the same `defaults.concurrency` semaphore as `run`, and the embedding/judge calls it makes go through the same response cache. - **`analyze`** runs paired statistics per `(prompt, evaluator, slice)`, applies Benjamini–Hochberg FDR correction, classifies severities, and — when a `migration_policy` is configured — computes a pass/fail verdict. -- **`report`** renders the single-file HTML report (no external assets; works offline and attaches cleanly to a PR or email), and writes the machine-written [run insights](#run-insights) narrative unless `--no-insights` is passed. The page opens on a verdict / advisory-signal / economics panel row and a six-cell run strip (examples, calls, failed-or-truncated, spend, latency Δ, mean score Δ), then the executive summary (one row per prompt, drawn from blocking evaluators — advisory ones only on a prompt nothing else scored; the strip's mean score Δ averages those rows), the narrative, the **Trace rules broken** panel when the target broke a hand-written trace rule, one section per prompt, and the methodology. Every figure on it is derived from the run's own artefacts; the deltas in the header are the run-level rollup of the per-prompt economics. **Top regressions** lists every example that regressed, worst first, as one collapsed card per example that names each evaluator that dropped it — expand one for each evaluator's finding, then the trace diff, the tool diffs and the conversation context, shown once. It shows at most 50 cards per prompt and says so when it cuts; the per-example breakdown lists every example. A drop inside an evaluator's own threshold (`semantic`'s `min_similarity`) is tested as no change, so it is not a regression: those examples sit in a closed line under the cards. The report is dark-only. +- **`report`** renders the single-file HTML report (no external assets; works offline and attaches cleanly to a PR or email), and writes the machine-written [run insights](#run-insights) narrative unless `--no-insights` is passed. The page opens on a verdict / advisory-signal / economics panel row and a six-cell run strip (examples, calls, failed-or-truncated, spend, latency Δ, mean score Δ), then the executive summary (one row per prompt, drawn from blocking evaluators — advisory ones only on a prompt nothing else scored; the strip's mean score Δ averages those rows), the narrative, the **Trace rules broken** panel when the target broke a hand-written trace rule, one section per prompt, and the methodology. Every figure on it is derived from the run's own artefacts; the deltas in the header are the run-level rollup of the per-prompt economics. **Top regressions** lists every example that regressed, worst first, as one collapsed card per example that names each evaluator that dropped it — expand one for each evaluator's finding, then the trace diff, the tool diffs and the conversation context, shown once. It shows at most 50 cards per prompt and says so when it cuts; the per-example breakdown lists every example. A drop inside an evaluator's own threshold (`semantic`'s `min_similarity`) is tested as no change, so it is not a regression: those examples sit in a closed line under the cards, and their cards say "Within threshold" in a neutral tone. The per-example breakdown applies the same rule: only a regressed example's row is tinted, it quotes its worst drop beyond a threshold, and a forgiven drop shows muted. The report is dark-only. - **`compare`** chains everything end to end over one suite and adds `--gate`, `--policy-gate`, `--push`, `--open`. Warnings raised while the pipeline runs (LiteLLM deprecation notices, insights-retry notes) are deferred and printed as one `⚠` section directly under the pipeline block — also when a stage fails, since a held-back warning may explain the failure. Errors are never deferred. With `captures.max_age` or `captures.max_cases` set, `compare` prints doctor's `suite freshness` findings for the suite it runs (named with `--suite-name` or auto-selected; not for `--suite `) as `⚠` lines above the verdict, so a CI log shows a stale suite. ### Artefacts @@ -240,7 +240,7 @@ Run ids look like `r_20260722_golden_a1b2c3` (`r___`). Un | `scores.jsonl` | evaluate | One line per (pair × evaluator): source/target scores, delta, explanation | | `analysis.json` | analyze | Per-comparison statistics, severities, notes | | `migration_decision.json` | analyze | Policy verdict + per-budget detail (only when `migration_policy` set) | -| `report.json` | report | The report payload, including `invariant_violations`: one row per [trace rule](#trace-invariants) the target broke. Each prompt section's `top_regressions` holds one row per example × evaluator for the regressed examples shown, `regressed_example_count` counts every regressed example, and `within_threshold_regressions` / `within_threshold_example_count` do the same for drops inside an evaluator's threshold | +| `report.json` | report | The report payload, including `invariant_violations`: one row per [trace rule](#trace-invariants) the target broke. Each prompt section's `top_regressions` holds one row per example × evaluator for the regressed examples shown (at most 50), and `regressed_example_count` counts every regressed example. `within_threshold_regressions` / `within_threshold_example_count` do the same for examples whose *every* drop stayed inside its evaluator's threshold; a forgiven drop on an example that also regressed is in neither list. Each `example_rows` row carries `regressed` | | `report.html` | report | Single-file HTML report | | `insights.json` | report | Cached [run insights](#run-insights) narrative (optional; skipped with `--no-insights` or no API key) | | `traces.jsonl` | traces import | Bring-your-own agent traces (optional) | diff --git a/llms-full.txt b/llms-full.txt index 52563912..50fb7639 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -96,7 +96,7 @@ as one section under the pipeline block (also on stage failure); errors are neve | scores.jsonl | evaluate | one EvalRecord per (pair x evaluator): source/target scores in [0,1], delta | | analysis.json | analyze | per-(prompt,evaluator,slice) statistics + severity | | migration_decision.json | analyze | policy verdict (only when migration_policy configured) | -| report.json + report.html | report | payload + single-file HTML (no external assets, dark-only). report.json top-level invariant_violations: one row per trace rule the target broke (see evaluators.trace_invariants). Per prompt section: top_regressions (one row per example x evaluator, regressed examples shown), regressed_example_count, within_threshold_regressions, within_threshold_example_count | +| report.json + report.html | report | payload + single-file HTML (no external assets, dark-only). report.json top-level invariant_violations: one row per trace rule the target broke (see evaluators.trace_invariants). Per prompt section: top_regressions (one row per example x evaluator, regressed examples shown, at most 50), regressed_example_count (all of them), within_threshold_regressions + within_threshold_example_count (examples whose every drop stayed inside its evaluator's threshold; a forgiven drop on a regressed example is in neither), example_rows[].regressed | | insights.json | report | cached machine-written narrative (optional; --no-insights skips) | | traces.jsonl | traces import | optional BYO agent traces | | run_bundle.json.gz | bundle/push | optional hosted upload bundle | @@ -257,7 +257,9 @@ evalshift report RUN_ID [-c CONFIG] [--open] [--insights/--no-insights (default per regressed example, worst first, naming every evaluator that dropped it; at most 50 per prompt (the per-example table lists all); a drop inside an evaluator's own threshold (semantic min_similarity) is tested as no change, so it sits in a closed line below the - cards, not among them. Dark-only, no external assets, no JS. Rendered surfaces show + cards ("Within threshold", neutral tone), not among them; the per-example table tints + only regressed rows and shows a forgiven drop muted. Dark-only, no external assets, no + JS. Rendered surfaces show display names, never internal identifiers: failure categories as plain language (TOOL_SELECTION_DRIFT -> "Different tools chosen"; map in evaluators/failures.py), budgets as their labels (max_tool_divergence -> "Tool-selection divergence"; map in diff --git a/src/evalshift_cli/insights/facts.py b/src/evalshift_cli/insights/facts.py index bc8bac59..80622155 100644 --- a/src/evalshift_cli/insights/facts.py +++ b/src/evalshift_cli/insights/facts.py @@ -117,7 +117,7 @@ class ExampleFact: Deliberately not ``reports.json.ExampleRow`` — that row carries the deltas but no output text, and ``TopRegression`` carries the text but only for the - worst five of each prompt. The insights path needs both on every example. + examples that regressed. The insights path needs both on every example. """ example_id: str diff --git a/src/evalshift_cli/reports/html.py b/src/evalshift_cli/reports/html.py index f4a664a2..8c117bd6 100644 --- a/src/evalshift_cli/reports/html.py +++ b/src/evalshift_cli/reports/html.py @@ -333,7 +333,7 @@ def _kind_label(kind: str) -> str: return f"{label} — {axis}" if axis else label -def _regression_reason(regression: TopRegression) -> str: +def _regression_reason(regression: TopRegression, within: bool = False) -> str: """Explain, in one plain sentence, why an example was flagged. Prefers the evaluator's own written rationale (e.g. an llm_judge verdict). @@ -345,7 +345,24 @@ def _regression_reason(regression: TopRegression) -> str: user's: on the run this work came from both tool axes were called ``routing``, which matched no family prefix, so every tool regression rendered its scores and no reason at all. + + Args: + regression: The drop to explain. + within: The drop is inside its evaluator's own threshold, so the + analysis tests it as no change. The sentence says that instead + of claiming the meaning drifted. """ + if within: + raw = regression.metadata.get("raw_cosine") + similar = ( + f" ({raw * 100:.0f}% similar; 100% = identical)" + if isinstance(raw, (int, float)) + else "" + ) + return ( + f"The target reworded the answer but stayed inside the evaluator's threshold" + f"{similar}, so the analysis scores it as no change." + ) text = regression.explanation.strip() if text: return text diff --git a/src/evalshift_cli/reports/json.py b/src/evalshift_cli/reports/json.py index 76107586..33e32a58 100644 --- a/src/evalshift_cli/reports/json.py +++ b/src/evalshift_cli/reports/json.py @@ -234,11 +234,15 @@ class ExampleRow: tool_match: bool | None # Which tools each side called, when a divergence axis scored this # example. This column is the plan's table: one line per example, source - # called / target called, for every example rather than the worst five. + # called / target called, for every example, regressed or not. tool_change: ToolChange | None = None # Multi-turn — zero-based turn position within the source conversation. # None for single-turn examples or when the example isn't in the suite. turn_index: int | None = None + # Whether some evaluator dropped this example beyond its own threshold: + # the rule the top-regressions list uses. ``False`` on a row whose only + # drops the threshold forgives, though its worst delta is negative. + regressed: bool = False @dataclass(frozen=True, slots=True) @@ -897,7 +901,12 @@ def _build_example_rows( ex_records = records_by_example.get(example_id, []) scored = [r for r in ex_records if r.error is None] - worst = min((r.delta for r in scored), default=None) + # A drop inside its evaluator's own threshold is tested as no change, + # so a regressed row quotes the worst drop that counts, as its + # top-regressions card does. + counted = [r.delta for r in scored if not within_tolerance(r)] + regressed = any(d < 0 for d in counted) + worst = min(counted) if regressed else min((r.delta for r in scored), default=None) example = examples_by_id.get(example_id) rows.append( @@ -911,13 +920,16 @@ def _build_example_rows( tool_match=_tool_match(scored, tool_evaluator_names), tool_change=_example_tool_change(scored), turn_index=example.turn_index if example is not None else None, + regressed=regressed, ), ) - # Worst regression first (most-negative delta score), tiebreak on - # most-expensive (highest +delta cost), then example_id for stability. + # Regressed examples first, worst first (most-negative delta score), + # tiebreak on most-expensive (highest +delta cost), then example_id for + # stability. rows.sort( key=lambda r: ( + not r.regressed, r.worst_delta_score if r.worst_delta_score is not None else 0.0, -r.delta_cost_usd, r.example_id, @@ -1319,6 +1331,7 @@ def _to_jsonable(report: ReportData) -> dict[str, Any]: "tool_match": er.tool_match, "tool_change": _tool_change_to_dict(er.tool_change), "turn_index": er.turn_index, + "regressed": er.regressed, } for er in ps.example_rows ], diff --git a/src/evalshift_cli/reports/templates/report.css b/src/evalshift_cli/reports/templates/report.css index 11dda753..722d94df 100644 --- a/src/evalshift_cli/reports/templates/report.css +++ b/src/evalshift_cli/reports/templates/report.css @@ -327,6 +327,8 @@ p.empty { .meter { height: 100%; background: var(--ok); } .meter-warn { background: var(--warn); } .meter-bad { background: var(--bad); } +/* A drop the evaluator's own threshold forgives: shown, but not as a regression. */ +.meter-muted { background: var(--dim2); } .meter-track { height: 6px; background: var(--well); @@ -610,6 +612,7 @@ details[open] > summary .caret::after { content: "\25BE"; } background: var(--bad-tint); border: 1px solid var(--bad-edge); } +.reg-head .delta-muted { color: var(--dim); background: var(--panel-2); border-color: var(--stroke); } .reg-body { display: flex; flex-direction: column; diff --git a/src/evalshift_cli/reports/templates/report.html.j2 b/src/evalshift_cli/reports/templates/report.html.j2 index a75a2b8b..f0166624 100644 --- a/src/evalshift_cli/reports/templates/report.html.j2 +++ b/src/evalshift_cli/reports/templates/report.html.j2 @@ -38,7 +38,10 @@ outputs and context once. The outputs are the same whichever evaluator scored them, so a card per evaluator repeated them and let one example fill the list. -#} -{%- macro regression_card(card, opened=False) -%} +{%- macro regression_card(card, opened=False, within=False) -%} +{#- ``within``: a drop the evaluator's own threshold forgives. The card then + says so in a neutral tone instead of "Why flagged" in red. -#} +{%- set target_tone = "muted" if within else "bad" -%} {%- set first = card.regressions[0] -%} {%- set single = card.regressions|length == 1 -%}
@@ -49,22 +52,22 @@ {% for tr in card.regressions %}{{ evaluator_label(tr.evaluator_name, tr.kind) }}{% if not loop.last %} · {% endif %}{% endfor %} {% if first.truncated %}truncated (token cap){% endif %} {% if first.target_empty_output %}empty output{% endif %} - {{ "%+.3f"|format(first.delta) }} + {{ "%+.3f"|format(first.delta) }}
{% for tr in card.regressions %} - {% set reason = regression_reason(tr) %} -
+ {% set reason = regression_reason(tr, within) %} +
- Why flagged + {{ "Within threshold" if within else "Why flagged" }} {% if not single %}{{ evaluator_label(tr.evaluator_name, tr.kind) }}{% endif %} source {{ "%.2f"|format(tr.source_score) }} → target {{ "%.2f"|format(tr.target_score) }} (0–1) {#- The evaluator's own failure category when it wrote one: it names what went wrong, where the axis slug only names what was scored. -#} {%- set categories = tr.metadata.get("failure_categories") or [] %} - {% for category in categories %}{{ category_label(category) }}{% endfor %} - {% if not categories and tr.kind %}{{ kind_label(tr.kind) }}{% endif %} + {% for category in categories %}{{ category_label(category) }}{% endfor %} + {% if not categories and tr.kind %}{{ kind_label(tr.kind) }}{% endif %} {% if not single %}{{ "%+.3f"|format(tr.delta) }}{% endif %}
{% if reason %}

{{ reason }}

{% endif %} @@ -94,7 +97,7 @@ {% endif %}
-
Target agent trace{% if single %}score {{ "%.3f"|format(first.target_score) }}{% endif %}
+
Target agent trace{% if single %}score {{ "%.3f"|format(first.target_score) }}{% endif %}
{% if first.target_agent_trace and first.target_agent_trace.tool_calls %} {% set source_names = (first.source_agent_trace.tool_calls if first.source_agent_trace else []) | map(attribute='name') | list %}
    @@ -143,7 +146,7 @@ {% endif %}
-
Target trace{% if single %}score {{ "%.3f"|format(first.target_score) }}{% endif %}
+
Target trace{% if single %}score {{ "%.3f"|format(first.target_score) }}{% endif %}
{% if first.target_trace and first.target_trace.calls %} {% set source_names = (first.source_trace.calls if first.source_trace else []) | map(attribute='tool_name') | list %}
    @@ -180,7 +183,7 @@ 
    {{ first.source_text }}
-
Target{% if single %}score {{ "%.3f"|format(first.target_score) }}{% endif %}
+
Target{% if single %}score {{ "%.3f"|format(first.target_score) }}{% endif %}
{{ first.target_text }}
@@ -667,13 +670,13 @@ Tags Time Δ Cost Δ - Worst score Δ + Worst score Δ Tool match Tools called (source → target) {% for row in section.example_rows %} - + {{ row.example_id }}{% if row.turn_index is not none %}turn {{ row.turn_index }}{% endif %} {# One dot-separated line, not a chip per tag: a stack of chips in a narrow column pushes every other row taller. #} @@ -684,9 +687,10 @@ {%- if row.worst_delta_score is none %}— {%- elif row.worst_delta_score == 0 %}±0.00 {%- else %} -
-
- {{ "%+.2f"|format(row.worst_delta_score) }} + {%- set forgiven = row.worst_delta_score < 0 and not row.regressed %} +
+
+ {{ "%+.2f"|format(row.worst_delta_score) }}
{%- endif %} @@ -831,7 +835,7 @@ {{ within_count }} {% if section.regressed_example_count %}more {% endif %}example{{ "s" if within_count != 1 }} dropped within the evaluator's threshold (scored as no change){% if within_count > section.within_threshold_examples|length %} · showing the largest {{ section.within_threshold_examples|length }}{% endif %}
{% for card in section.within_threshold_examples %} - {{ regression_card(card) }} + {{ regression_card(card, within=True) }} {% endfor %}
diff --git a/tests/unit/test_reports.py b/tests/unit/test_reports.py index d9171da6..320c1f89 100644 --- a/tests/unit/test_reports.py +++ b/tests/unit/test_reports.py @@ -2990,7 +2990,7 @@ def _semantic(cls, example_id: str, delta: float, *, beyond_threshold: bool) -> delta, evaluator_name="semantic.cosine", kind="semantic", - metadata={"failure_categories": categories}, + metadata={"failure_categories": categories, "raw_cosine": 1.0 + delta}, ) @staticmethod @@ -3114,6 +3114,72 @@ def test_the_cap_names_where_the_rest_are(self, tmp_path: Path) -> None: assert note in html assert "Per-example breakdown" in html.split(note, 1)[1].split("", 1)[0] + def test_tied_examples_rank_by_evaluator_count_then_id(self, tmp_path: Path) -> None: + run_dir = self._run( + tmp_path, + [ + self._record("b", -0.5), + self._record("a", -0.5), + self._record("c", -0.5), + self._record("c", -0.5, evaluator_name="routing", kind="tool_selection.divergence"), + ], + ) + section = build_report_payload(run_dir).prompt_sections[0] + assert [g.example_id for g in section.regressed_examples] == ["c", "a", "b"] + + def test_a_within_threshold_card_does_not_claim_a_regression(self, tmp_path: Path) -> None: + html = render_html(build_report_payload(self._threshold_run(tmp_path))) + within = self._top(html).split('
', 1)[1] + assert "Within threshold" in within + assert "Why flagged" not in within + assert "notice-bad" not in within + assert "badge-bad" not in within + assert "drift" not in within + assert "93% similar" in within + + def test_the_within_list_has_its_own_cap(self, tmp_path: Path) -> None: + from evalshift_cli.reports.json import MAX_REGRESSION_CARDS + + total = MAX_REGRESSION_CARDS + 3 + run_dir = self._run( + tmp_path, + [ + self._semantic(f"e{i:03d}", -(i + 1) / 1000, beyond_threshold=False) + for i in range(total) + ], + ) + payload = build_report_payload(run_dir) + section = payload.prompt_sections[0] + assert len(section.within_threshold_examples) == MAX_REGRESSION_CARDS + assert section.within_threshold_example_count == total + html = render_html(payload) + # Nothing regressed: the header says so, and the line drops its "more". + assert '0 of 2 examples regressed' in html + assert "No specific regressions to surface." in html + assert ( + f"{total} examples dropped within the evaluator's threshold (scored as no change)" + f" · showing the largest {MAX_REGRESSION_CARDS}" + ) in html + + def test_the_breakdown_marks_only_examples_that_regressed(self, tmp_path: Path) -> None: + payload = build_report_payload(self._threshold_run(tmp_path)) + rows = {row.example_id: row for row in payload.prompt_sections[0].example_rows} + assert (rows["ex1"].regressed, rows["ex2"].regressed) == (False, True) + table = _section(render_html(payload), "

Per-example breakdown

") + assert table.count('class="ex-regression"') == 1 + + def test_a_regressed_rows_worst_delta_is_the_regression(self, tmp_path: Path) -> None: + # The forgiven semantic drop is larger; the breakdown quotes the one + # that counts, as the card does. + run_dir = self._run( + tmp_path, + [self._semantic("ex1", -0.08, beyond_threshold=False), self._record("ex1", -0.05)], + ) + section = build_report_payload(run_dir).prompt_sections[0] + row = next(r for r in section.example_rows if r.example_id == "ex1") + assert (row.regressed, row.worst_delta_score) == (True, -0.05) + assert section.regressed_examples[0].regressions[0].delta == -0.05 + def test_report_json_carries_the_counts_and_the_within_threshold_rows( self, tmp_path: Path ) -> None: