From 9db862e0e94a5392096ac48a7c75dc5adedecac8 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Thu, 22 Oct 2026 23:44:02 +0200 Subject: [PATCH] fix(report): tint a target trace red only when its tool calls differ A card flagged for its wording alone showed both sides making the same tool call with the target's in a red box, which read as a tool failure. The box and a target's "(no tool calls)" are now red only when the pair's tool diff is non-empty, and a within-threshold finding sits in a neutral notice instead of the warning tint. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 ++ .../reports/templates/report.css | 2 + .../reports/templates/report.html.j2 | 15 ++++-- tests/unit/test_reports.py | 54 +++++++++++++++++++ 4 files changed, 71 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1a3ad1..0f6f395 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 quotes its worst drop beyond a threshold (the delta its card leads with), and a forgiven drop shows muted. Each `example_rows` row in `report.json` carries `regressed`. +- A regression card outlines the target's trace in red only when its tool + calls differ from the source's, and reads a target's "(no tool calls)" in + red only when the source did call tools. A card flagged for its wording + alone showed both sides making the same call with the target's in a red + box, which read as a tool failure. - The report has more room between its sections, between the blocks of a prompt's section, and between each heading and its table. diff --git a/src/evalshift_cli/reports/templates/report.css b/src/evalshift_cli/reports/templates/report.css index 722d94d..9884345 100644 --- a/src/evalshift_cli/reports/templates/report.css +++ b/src/evalshift_cli/reports/templates/report.css @@ -154,6 +154,8 @@ pre { } .notice p { margin: 0; font-size: 14.5px; line-height: 1.65; color: var(--body); } .notice-bad { background: var(--bad-tint); border-color: var(--bad-edge); } +/* A finding that is not a problem (a drop inside its own threshold). */ +.notice-neutral { background: var(--well); border-color: var(--stroke); } /* --- Header ------------------------------------------------------------- */ diff --git a/src/evalshift_cli/reports/templates/report.html.j2 b/src/evalshift_cli/reports/templates/report.html.j2 index f016662..553a8a9 100644 --- a/src/evalshift_cli/reports/templates/report.html.j2 +++ b/src/evalshift_cli/reports/templates/report.html.j2 @@ -44,6 +44,11 @@ {%- set target_tone = "muted" if within else "bad" -%} {%- set first = card.regressions[0] -%} {%- set single = card.regressions|length == 1 -%} +{#- The target's trace reads red only where its calls differ from the + source's: a matching trace under a red box read as a tool failure on + cards flagged for their wording alone. -#} +{%- set agent_differs = first.trace_diff is not none and first.trace_diff.items|length > 0 -%} +{%- set tools_differ = first.tool_diffs|length > 0 -%}
@@ -58,7 +63,7 @@
{% for tr in card.regressions %} {% set reason = regression_reason(tr, within) %} -
+
{{ "Within threshold" if within else "Why flagged" }} {% if not single %}{{ evaluator_label(tr.evaluator_name, tr.kind) }}{% endif %} @@ -100,13 +105,13 @@
Target agent trace{% if single %}score {{ "%.3f"|format(first.target_score) }}{% endif %}
{% if first.target_agent_trace and first.target_agent_trace.tool_calls %} {% set source_names = (first.source_agent_trace.tool_calls if first.source_agent_trace else []) | map(attribute='name') | list %} -
    +
      {% for c in first.target_agent_trace.tool_calls %}
    1. {{ c.name }} ({{ c.arguments|tojson }})
      2. {% endfor %}
    {% else %} -

    (no tool calls)

    +

    (no tool calls)

    {% endif %}
@@ -149,7 +154,7 @@
Target trace{% if single %}score {{ "%.3f"|format(first.target_score) }}{% endif %}
{% if first.target_trace and first.target_trace.calls %} {% set source_names = (first.source_trace.calls if first.source_trace else []) | map(attribute='tool_name') | list %} -
    +
      {% for round_index, calls in first.target_trace.calls|groupby("round_index") %} {% if first.target_trace.round_count > 1 %}
    1. round {{ round_index + 1 }}
      2. {% endif %} {% for c in calls %} @@ -158,7 +163,7 @@ {% endfor %}
    {% else %} -

    (no tool calls)

    +

    (no tool calls)

    {% endif %} {% if first.target_trace and first.target_trace.final_text %}

    final text: {{ first.target_trace.final_text }}

    diff --git a/tests/unit/test_reports.py b/tests/unit/test_reports.py index 320c1f8..2c7d965 100644 --- a/tests/unit/test_reports.py +++ b/tests/unit/test_reports.py @@ -3180,6 +3180,60 @@ def test_a_regressed_rows_worst_delta_is_the_regression(self, tmp_path: Path) -> assert (row.regressed, row.worst_delta_score) == (True, -0.05) assert section.regressed_examples[0].regressions[0].delta == -0.05 + def _traced_run( + self, tmp_path: Path, *, source_tools: list[str], target_tools: list[str] + ) -> Path: + """ex1 regressed on semantic, with each side's recorded tool calls.""" + tmp_path.mkdir(parents=True, exist_ok=True) + run_dir = self._run(tmp_path, [self._semantic("ex1", -0.2, beyond_threshold=True)]) + (run_dir / "raw.jsonl").unlink() + for role, tools in (("source", source_tools), ("target", target_tools)): + calls = [ + ToolCall(tool_name=name, arguments={"value": 5}, sequence_index=i) + for i, name in enumerate(tools) + ] + append_call( + run_dir, + Call( + run_id="r_20260601_aaaaaa", + prompt_id="greet", + example_id="ex1", + model_id=f"model-{role}", + role=role, + text="5 kg is 11.02 lb.", + cost_usd=0.0001, + latency_ms=100, + input_tokens=10, + output_tokens=5, + trace=ToolTrace(calls=calls), + ), + ) + return run_dir + + def test_the_target_trace_is_red_only_when_its_calls_differ(self, tmp_path: Path) -> None: + same = self._traced_run( + tmp_path / "same", source_tools=["convert_units"], target_tools=["convert_units"] + ) + differ = self._traced_run( + tmp_path / "differ", source_tools=["convert_units"], target_tools=["convert_currency"] + ) + assert "trace-target" not in self._top(render_html(build_report_payload(same))) + assert "trace-target" in self._top(render_html(build_report_payload(differ))) + + def test_no_tool_calls_on_both_sides_is_not_flagged_missing(self, tmp_path: Path) -> None: + neither = self._traced_run(tmp_path / "neither", source_tools=[], target_tools=[]) + dropped = self._traced_run( + tmp_path / "dropped", source_tools=["convert_units"], target_tools=[] + ) + assert "trace-missing" not in self._top(render_html(build_report_payload(neither))) + assert "trace-missing" in self._top(render_html(build_report_payload(dropped))) + + def test_a_within_threshold_finding_is_not_tinted(self, tmp_path: Path) -> None: + html = render_html(build_report_payload(self._threshold_run(tmp_path))) + before, within = self._top(html).split('
    ', 1) + assert '
    ' in within + assert "notice-neutral" not in before + def test_report_json_carries_the_counts_and_the_within_threshold_rows( self, tmp_path: Path ) -> None: