fix(report): tint a target trace red only when its tool calls differ

CHANGELOG.md

@@ -34,6 +34,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   quotes its worst drop beyond a threshold (the delta its card leads with),
   and a forgiven drop shows muted. Each `example_rows` row in `report.json`
   carries `regressed`.
+- A regression card outlines the target's trace in red only when its tool
+  calls differ from the source's, and reads a target's "(no tool calls)" in
+  red only when the source did call tools. A card flagged for its wording
+  alone showed both sides making the same call with the target's in a red
+  box, which read as a tool failure.
 - The report has more room between its sections, between the blocks of a
   prompt's section, and between each heading and its table.
 

src/evalshift_cli/reports/templates/report.css

@@ -154,6 +154,8 @@ pre {
 }
 .notice p { margin: 0; font-size: 14.5px; line-height: 1.65; color: var(--body); }
 .notice-bad { background: var(--bad-tint); border-color: var(--bad-edge); }
+/* A finding that is not a problem (a drop inside its own threshold). */
+.notice-neutral { background: var(--well); border-color: var(--stroke); }
 
 /* --- Header ------------------------------------------------------------- */
 

src/evalshift_cli/reports/templates/report.html.j2

@@ -44,6 +44,11 @@
 {%- set target_tone = "muted" if within else "bad" -%}
 {%- set first = card.regressions[0] -%}
 {%- set single = card.regressions|length == 1 -%}
+{#- The target's trace reads red only where its calls differ from the
+    source's: a matching trace under a red box read as a tool failure on
+    cards flagged for their wording alone. -#}
+{%- set agent_differs = first.trace_diff is not none and first.trace_diff.items|length > 0 -%}
+{%- set tools_differ = first.tool_diffs|length > 0 -%}
 <details class="panel regression"{% if opened %} open{% endif %}>
   <summary class="reg-head">
     <span class="caret"></span>
@@ -58,7 +63,7 @@
   <div class="reg-body">
     {% for tr in card.regressions %}
     {% set reason = regression_reason(tr, within) %}
-    <div class="notice{{ " notice-bad" if not within }} reg-why">
+    <div class="notice {{ "notice-neutral" if within else "notice-bad" }} reg-why">
       <div class="reg-why-line">
         <span class="eyebrow">{{ "Within threshold" if within else "Why flagged" }}</span>
         {% if not single %}<span class="reg-why-evaluator">{{ evaluator_label(tr.evaluator_name, tr.kind) }}</span>{% endif %}
@@ -100,13 +105,13 @@
         <div class="reg-col-head"><span class="eyebrow">Target agent trace</span>{% if single %}<span class="reg-col-score {{ target_tone }}">score {{ "%.3f"|format(first.target_score) }}</span>{% endif %}</div>
         {% if first.target_agent_trace and first.target_agent_trace.tool_calls %}
         {% set source_names = (first.source_agent_trace.tool_calls if first.source_agent_trace else []) | map(attribute='name') | list %}
-        <ol class="trace-list trace-target">
+        <ol class="trace-list{{ ' trace-target' if agent_differs }}">
           {% for c in first.target_agent_trace.tool_calls %}
           <li class="trace-call {{ "" if c.name in source_names else "trace-extra" }}">{{ c.name }} <span class="trace-args">({{ c.arguments|tojson }})</span></li>
           {% endfor %}
         </ol>
         {% else %}
-        <p class="trace-empty trace-missing">(no tool calls)</p>
+        <p class="trace-empty{{ ' trace-missing' if agent_differs }}">(no tool calls)</p>
         {% endif %}
       </div>
     </div>
@@ -149,7 +154,7 @@
         <div class="reg-col-head"><span class="eyebrow">Target trace</span>{% if single %}<span class="reg-col-score {{ target_tone }}">score {{ "%.3f"|format(first.target_score) }}</span>{% endif %}</div>
         {% if first.target_trace and first.target_trace.calls %}
         {% set source_names = (first.source_trace.calls if first.source_trace else []) | map(attribute='tool_name') | list %}
-        <ol class="trace-list trace-target">
+        <ol class="trace-list{{ ' trace-target' if tools_differ }}">
           {% for round_index, calls in first.target_trace.calls|groupby("round_index") %}
           {% if first.target_trace.round_count > 1 %}<li class="trace-round-label" value="0">round {{ round_index + 1 }}</li>{% endif %}
           {% for c in calls %}
@@ -158,7 +163,7 @@
           {% endfor %}
         </ol>
         {% else %}
-        <p class="trace-empty trace-missing">(no tool calls)</p>
+        <p class="trace-empty{{ ' trace-missing' if tools_differ }}">(no tool calls)</p>
         {% endif %}
         {% if first.target_trace and first.target_trace.final_text %}
         <p class="trace-final"><em>final text:</em> {{ first.target_trace.final_text }}</p>

tests/unit/test_reports.py

@@ -3180,6 +3180,60 @@ def test_a_regressed_rows_worst_delta_is_the_regression(self, tmp_path: Path) ->
         assert (row.regressed, row.worst_delta_score) == (True, -0.05)
         assert section.regressed_examples[0].regressions[0].delta == -0.05
 
+    def _traced_run(
+        self, tmp_path: Path, *, source_tools: list[str], target_tools: list[str]
+    ) -> Path:
+        """ex1 regressed on semantic, with each side's recorded tool calls."""
+        tmp_path.mkdir(parents=True, exist_ok=True)
+        run_dir = self._run(tmp_path, [self._semantic("ex1", -0.2, beyond_threshold=True)])
+        (run_dir / "raw.jsonl").unlink()
+        for role, tools in (("source", source_tools), ("target", target_tools)):
+            calls = [
+                ToolCall(tool_name=name, arguments={"value": 5}, sequence_index=i)
+                for i, name in enumerate(tools)
+            ]
+            append_call(
+                run_dir,
+                Call(
+                    run_id="r_20260601_aaaaaa",
+                    prompt_id="greet",
+                    example_id="ex1",
+                    model_id=f"model-{role}",
+                    role=role,
+                    text="5 kg is 11.02 lb.",
+                    cost_usd=0.0001,
+                    latency_ms=100,
+                    input_tokens=10,
+                    output_tokens=5,
+                    trace=ToolTrace(calls=calls),
+                ),
+            )
+        return run_dir
+
+    def test_the_target_trace_is_red_only_when_its_calls_differ(self, tmp_path: Path) -> None:
+        same = self._traced_run(
+            tmp_path / "same", source_tools=["convert_units"], target_tools=["convert_units"]
+        )
+        differ = self._traced_run(
+            tmp_path / "differ", source_tools=["convert_units"], target_tools=["convert_currency"]
+        )
+        assert "trace-target" not in self._top(render_html(build_report_payload(same)))
+        assert "trace-target" in self._top(render_html(build_report_payload(differ)))
+
+    def test_no_tool_calls_on_both_sides_is_not_flagged_missing(self, tmp_path: Path) -> None:
+        neither = self._traced_run(tmp_path / "neither", source_tools=[], target_tools=[])
+        dropped = self._traced_run(
+            tmp_path / "dropped", source_tools=["convert_units"], target_tools=[]
+        )
+        assert "trace-missing" not in self._top(render_html(build_report_payload(neither)))
+        assert "trace-missing" in self._top(render_html(build_report_payload(dropped)))
+
+    def test_a_within_threshold_finding_is_not_tinted(self, tmp_path: Path) -> None:
+        html = render_html(build_report_payload(self._threshold_run(tmp_path)))
+        before, within = self._top(html).split('<details class="within-threshold">', 1)
+        assert '<div class="notice notice-neutral reg-why">' in within
+        assert "notice-neutral" not in before
+
     def test_report_json_carries_the_counts_and_the_within_threshold_rows(
         self, tmp_path: Path
     ) -> None: