From 3ec0fc563e5330d4594c6f43fb910d15a35ac2de Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Tue, 27 Oct 2026 00:29:45 +0100 Subject: [PATCH] fix(policy)!: gate cost/latency budgets on paired, spike-robust ratios max_cost_increase / max_latency_increase divided the target's mean over all its error-free calls by the source's. On one real 21-example suite that read +27.2% (pass) and +35.6% (fail) against a 30% latency budget; 86-95% of the excess was two or three 3-5 s target responses on different examples each run. Both budgets now pair calls by (prompt_id, example_id); an example counts only when each side has an error-free call. Latency gates the 20%-trimmed geometric mean of per-example target/source ratios (+13.2% / +21.6% on those runs, both pass); cost gates the paired total, untrimmed. With samples_per_example > 1 each side collapses to one value per example (median latency, mean cost). Cache hits keep counting with their recorded latency. Also: observed is signed (savings no longer clip to 0; the gate stays one-sided), the denominator counts example pairs instead of calls on both sides, and the report's economics card and insights FACTS show the figures the budgets were judged on beside the run-wide deltas. BREAKING CHANGE: the two budgets keep their config keys but measure a paired, spike-robust change, so the same run can get a different verdict; observed and PolicyMetricSummary cost/latency_increase_rate can now be negative; their denominator is example pairs. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 40 +++ DOCS.md | 13 +- docs/configuration.md | 35 ++- docs/methodology.md | 113 +++++++- llms-full.txt | 54 +++- src/evalshift_cli/analysis/policy.py | 238 +++++++++++----- src/evalshift_cli/config/models.py | 7 + src/evalshift_cli/insights/facts.py | 16 +- src/evalshift_cli/insights/generator.py | 6 + src/evalshift_cli/insights/templates.py | 4 +- .../reports/templates/report.html.j2 | 13 +- tests/unit/test_insight_facts.py | 58 ++++ tests/unit/test_policy.py | 267 ++++++++++++++++-- tests/unit/test_reports.py | 61 ++++ 14 files changed, 768 insertions(+), 157 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12da905e..7cb7ccd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **`max_cost_increase` and `max_latency_increase` now measure something + different. Same config keys, new meaning: the same run can get a different + verdict.** Both used to divide the target's mean over all its error-free + calls by the source's. That was unpaired and fragile. Two runs of one + 21-example suite read +27.2% (pass) and +35.6% (fail) against a 30% latency + budget, and 86–95% of the excess came from two or three 3–5 s target + responses that landed on different examples each run. Both budgets now + compare each example with itself, and an example counts only when both + models answered it without error: + - `max_latency_increase` caps the **typical example's** slowdown: the + 20%-trimmed geometric mean of per-example target/source latency ratios. + The two runs above now read +13.2% and +21.6%, and both pass. A slowdown + across more than a fifth of the examples still moves the value. One + confined to fewer is treated as noise. The median was rejected because it + ignores any slowdown on fewer than half the examples. + - `max_cost_increase` caps the change in **paired total cost**, untrimmed, + because an expensive answer is a real bill. With no failed calls this is + the same number as before. + - With `samples_per_example > 1` each example counts once: its median + latency and mean cost per side. + - Cache hits still count, with the latency recorded when they ran live. + - A latency of 0 is a missing timing, and that example is left out of the + latency ratio. A free source against a timed target is no longer read as + a measured latency. +- **The cost/latency budget rows report a signed value.** `observed` (and + `PolicyMetricSummary.cost_increase_rate` / `latency_increase_rate`) used to + be clipped at 0, so a target 29% cheaper read `+0%`. It now reads `-0.289`. + Only the pass/fail is one-sided (`observed <= allowed`), so any saving still + passes. Code that read these fields as never-negative must handle negatives. +- **The cost/latency `denominator` counts example pairs, not calls.** A + 21-example suite reports 21, where it used to report 42 (calls on both + sides). +- The report's economics card and the insights narrative now show the + figures the two budgets were judged on ("judged per example: cost −28.9%, + latency +21.6% for the typical example") beside the run-wide cost and + latency changes, which can differ. Before, a card could read "Latency + +35.6%. Both inside +30% latency". + ## [1.6.1] - 2026-10-23 ### Changed diff --git a/DOCS.md b/DOCS.md index 5957d072..6eef828e 100644 --- a/DOCS.md +++ b/DOCS.md @@ -757,8 +757,8 @@ Budgets (fractions, not percents): | `max_tool_divergence` | 0.20 | Share of `tool_selection.divergence` records where the target called different tools than the source | | `max_invariant_violations` | 0 | **Count** of examples on which the target broke a rule of a blocking [`trace_invariants`](#trace-invariants) evaluator, whatever the source did. A breach always fails | | `tool_argument_drift_floor` | 0.9 | Target argument score below which a call counts as drifted. Argument scores are continuous — without a floor every reworded query counts as drift. | -| `max_cost_increase` | 0.30 | Relative avg-cost increase, target vs source | -| `max_latency_increase` | 0.30 | Relative avg-latency increase | +| `max_cost_increase` | 0.30 | Relative increase in total cost over example pairs (examples both models answered without error), target vs source | +| `max_latency_increase` | 0.30 | Relative slowdown of the typical example: the 20%-trimmed geometric mean of per-example target/source latency ratios, so a few slow responses cannot flip it | | `fail_on_dropped_params` | `false` | Fail when `state.json` → `dropped_params` is non-empty — an arm could not honour a generation constraint the source capture recorded. Top level only: a model either accepts a parameter or does not, which no subset of examples can vary. | | `slices` | `{}` | Per-slice overrides keyed by example tag — each distinct tag is a slice (unset fields inherit the top level) | @@ -767,12 +767,13 @@ These defaults are a first-migration starting point, not a shipping gate: a fres How the verdict is computed: - **Only blocking evaluators gate quality.** Advisory records are summarised separately (`advisory`, `advisory_regressions`) and never flip the verdict. If *every* configured evaluator is advisory (the fresh-`init` state), the verdict is `inconclusive` with guidance to promote evaluators to blocking as the suite grows — **unless** the call-derived `max_cost_increase`/`max_latency_increase` budgets are breached, which still **fails** the run: those measure the calls, not the evaluators. The guidance names each advisory judge and reads its readiness from the judge's own smallest per-prompt `n` against the 20-pair reliability floor: `The equivalence judge scored at least 24 pairs on every prompt — enough to gate. Set blocking: true on it in evalshift.yaml to get a pass/fail verdict.`, or `The equivalence judge is advisory, so it does not gate this run. It becomes reliable at 20 pairs per prompt; summarize has 8. Collect more examples, then set blocking: true on it.` (`this run has 8` on a single-prompt run). A judge that measured nothing on a prompt (every call errored, so `n` is 0) is never told to collect more: `The equivalence judge measured nothing on summarize, so more examples will not help: find out why its calls produced no verdict before relying on it.` (`on this run` when it measured nothing anywhere). `semantic` is never suggested — it measures drift, not correctness — and with no advisory judge scored the generic line stays. Once anything gates, none of this is emitted, so a deliberately advisory judge is never nagged. Separately, each evaluator skipped for a missing API key adds one line under every verdict: `semantic was skipped: no API key for openai/text-embedding-3-small. Export OPENAI_API_KEY to enable it.` -- **The four rate budgets are Wilson-CI-aware.** `max_overall_regression_rate`, `min_equivalence_rate`, `max_tool_argument_drift` and `max_tool_divergence` are each a count of records over a count of records, so each carries a 95% Wilson interval (`ci_low`/`ci_high`) on its rate — the first three are also the ones the hosted gate computes an interval for. A breach of one of them only **fails** when the interval confirms the breach (the CI's favourable bound still clears the budget → the suite is too small to be sure → `inconclusive`, not `fail`). A budget the observation *held* stays conclusive however wide its interval: a thin sample must never turn a clean run into a caveat. Cost and latency budgets are exact — a measured breach of one is never softened — but they are conclusive only when they measured something: their ratio falls back to `0.00` both when either arm has no error-free call and when **both** arms average zero (unpriced models, whose calls carry `cost_usd`/`latency_ms` of `0` on every row), and either way `conclusive: false`. The record-derived budgets (regression rate, equivalence rate, critical count, tool-argument drift, tool-selection divergence) report `conclusive: false` on a scope that scored zero records, since their `0/0` default measures nothing. The two per-axis budgets need one of *their own* rows: a scope that scored plenty but ran no `tool_arguments` evaluator, or set `divergence: off`, is unmeasured too. +- **The four rate budgets are Wilson-CI-aware.** `max_overall_regression_rate`, `min_equivalence_rate`, `max_tool_argument_drift` and `max_tool_divergence` are each a count of records over a count of records, so each carries a 95% Wilson interval (`ci_low`/`ci_high`) on its rate — the first three are also the ones the hosted gate computes an interval for. A breach of one of them only **fails** when the interval confirms the breach (the CI's favourable bound still clears the budget → the suite is too small to be sure → `inconclusive`, not `fail`). A budget the observation *held* stays conclusive however wide its interval: a thin sample must never turn a clean run into a caveat. Cost and latency budgets carry no interval, and a measured breach of one is never softened. They are conclusive only when they measured something: their ratio falls back to `0.00` both when no example has an error-free call on each side and when **both** arms recorded zero (unpriced models, whose calls carry `cost_usd`/`latency_ms` of `0` on every row), and either way `conclusive: false`. The record-derived budgets (regression rate, equivalence rate, critical count, tool-argument drift, tool-selection divergence) report `conclusive: false` on a scope that scored zero records, since their `0/0` default measures nothing. The two per-axis budgets need one of *their own* rows: a scope that scored plenty but ran no `tool_arguments` evaluator, or set `divergence: off`, is unmeasured too. - **A broken trace rule fails, however small the suite.** `max_invariant_violations` counts distinct examples — `(prompt_id, example_id)` pairs with any blocking `trace_invariants` row whose target score is below `1.0`, so a row averaged over repeated samples counts when any sample broke a rule, and an example that broke rules in two entries (each entry writes its own row) counts once. Its `denominator` is the distinct examples with a blocking `trace_invariants` row. It is conclusive by construction — no interval, no `1/n` warning — and a breach overrides `inconclusive` and `conditional_pass` to **`fail`**, even when every comparison is `insufficient`. The same holds per slice: a slice whose own `max_invariant_violations` breaches reads `fail` however few comparisons it has. The `reason` names the worst scope and its count. A violation the source shares still counts; it also adds a `recommendations` line naming the rule, since either both models break it or the rule no longer matches the toolset and its owner should review it. The budget's row appears only in scopes that scored at least one blocking `trace_invariants` row; advisory-only rules add none. - **A rate ceiling finer than `1/n` is flagged as zero-tolerance.** Rates are counted over whole rows, so on a ten-row suite the achievable tool-argument drift rates are `{0.0, 0.1, …}` and a `0.01` budget is really "any drift at all". `analyze` adds a line to the recommendations — shown in the terminal and the HTML report — naming the budget, its value, the granularity and the denominator: `The tool-argument drift budget of 1% (max_tool_argument_drift in evalshift.yaml) is below the 10% granularity of 10 tool-argument comparisons — effective tolerance is zero at this sample size.` It applies to `max_overall_regression_rate`, `max_tool_argument_drift` and `max_tool_divergence`, per scope (slices use their own row counts). Nothing is emitted for a deliberate `0.0` budget, for a denominator of `0` (already reported by `conclusive: false`), for the count/ratio budgets, or for the `min_equivalence_rate` floor. No default changes and no verdict moves — see [methodology](docs/methodology.md). - **A shared ground-truth miss is not equivalence.** A `tool_selection.conformance` row grades each side *absolutely* against the ground truth the suite recorded, so both sides can miss it at the same height — `0.0 / 0.0` on an example both models answered with a tool call the recording never made. The delta is zero, which used to read as equivalence and is the whole of the `equivalent_rate: 1.0` a real run once shipped over a suite where nine pairs in ten routed differently. Those rows now leave **every** policy rate: they measure the harness (wrong toolset attached, wrong prompt, a suite promoted from a different agent), not the migration. They are still reported — as a `TOOL_GROUND_TRUTH_MISS` count in `failure_categories` and as a recommendations line naming how many were excluded. Only the *shared-height* case is dropped: a conformance row the target lost ground on (`0.8 / 0.3`) is still a regression and one it improved on (`0.2 / 0.6`) is still an improvement. A run whose every blocking row was a shared miss is `inconclusive`, and its recommendation is **Fix the eval harness before collecting more examples** — more pairs from the same setup are more excluded rows, so the denominator stays empty however many are added. The exclusion is deliberately narrower than the diagnosis, which is why `evaluate` runs its own broken-harness check over the **source side alone**: a suite the source fails and the target happens to satisfy has a *positive* delta and no `TOOL_GROUND_TRUTH_MISS` tag, so the exclusion rule cannot see it and it is exactly as misconfigured. -- **A cost/latency ratio measured only from zeroes says so.** When the calls exist but every `cost_usd` (or `latency_ms`) is `0` on both models, `analyze` adds a recommendations line beside the `conclusive: false`: `The cost increase budget could not be measured: all 4 error-free calls across both models recorded a cost of 0, so its observed 0.00 is a default, not a measurement.` A run with no calls gets no line — the empty `raw.jsonl` already explains itself. Emitted once per run, since every scope reads the same calls. -- **Every budget reports its own `denominator`** — the sample `observed` was computed over. Scored records for the regression rate, the equivalence rate and the critical count — counted over *measurements*, so an evaluator scoring two axes contributes two rows per example; `tool_arguments` rows for tool drift; `tool_selection.divergence` rows for tool divergence; the distinct examples with a blocking `trace_invariants` row for the trace-rule count; the error-free calls behind both averages for the cost and latency ratios. Slices report their own counts. `0` means "counted, and the sample was empty", so `observed` is a default; a *missing* `denominator` means "no sample size reported" and is **not** zero — only bundles written before the field say that, and the hosted gate falls back to `conclusive` for them. It is the same number the `1/n` granularity warning is judged on, and it is orthogonal to `conclusive`: an all-zero cost ratio counted every call it averaged and still measured nothing, so it reports a positive denominator beside `conclusive: false`. The hosted gate derives its own Wilson interval from these denominators, over the same three rate budgets and with the same confidence constant the CLI uses, so a local verdict and a hosted one now agree on whether a breach was confirmed — see [methodology](docs/methodology.md). +- **Cost and latency are compared per example, and latency is robust to spikes.** Calls are paired by `(prompt_id, example_id)`. An example counts only when each side has an error-free call, so an example the target failed on leaves *both* sides. `max_latency_increase` gates the 20%-trimmed geometric mean of per-example `target / source` latency ratios: `floor(0.2·n)` pairs are dropped from each end (4 of 21, none below 5), the rest of the log-ratios are averaged, and the result is exponentiated. Two or three 3–5 s provider responses can no longer flip the verdict, as they did between two runs of one real suite (+27% pass, +36% fail). A slowdown across more than a fifth of the examples still moves it, and a slowdown confined to fewer is treated as noise. Pairs with a `latency_ms` of 0 on either side are left out. `max_cost_increase` gates the paired total, `Σ target / Σ source − 1`, untrimmed: cost follows tokens, and an expensive answer is a real bill. With `samples_per_example > 1` each side collapses to one value per example (median latency, mean cost) and an errored repeat drops only that sample. Cache hits count, with the cost and latency recorded when they ran live, so re-running from cache reproduces the verdict instead of turning the budget unmeasured. `observed` is **signed** (`-0.289` = 28.9% cheaper; it used to be clipped at `0`), and only `passed` (`observed <= allowed`) is one-sided. The report's economics card shows these figures beside the run-wide cost and latency changes, which can differ. See [methodology](docs/methodology.md#cost-and-latency-are-compared-per-example). +- **A cost/latency ratio measured only from zeroes says so.** When example pairs exist but every `cost_usd` is `0` on both models (or no pair has a `latency_ms` above 0 on both), `analyze` adds a recommendations line beside the `conclusive: false`: `The cost increase budget could not be measured: both models recorded a cost of 0 on all 4 example pairs, so its observed 0.00 is a default, not a measurement.` A run with no calls gets no line — the empty `raw.jsonl` already explains itself. Emitted once per run, since every scope reads the same calls. +- **Every budget reports its own `denominator`** — the sample `observed` was computed over. Scored records for the regression rate, the equivalence rate and the critical count — counted over *measurements*, so an evaluator scoring two axes contributes two rows per example; `tool_arguments` rows for tool drift; `tool_selection.divergence` rows for tool divergence; the distinct examples with a blocking `trace_invariants` row for the trace-rule count; the example pairs compared for the cost and latency ratios (21 for a 21-example suite, whatever `samples_per_example` is). Slices report their own counts. `0` means "counted, and the sample was empty", so `observed` is a default; a *missing* `denominator` means "no sample size reported" and is **not** zero — only bundles written before the field say that, and the hosted gate falls back to `conclusive` for them. It is the same number the `1/n` granularity warning is judged on, and it is orthogonal to `conclusive`: an all-zero cost ratio counted every pair it compared and still measured nothing, so it reports a positive denominator beside `conclusive: false`. The hosted gate derives its own Wilson interval from these denominators, over the same three rate budgets and with the same confidence constant the CLI uses, so a local verdict and a hosted one now agree on whether a breach was confirmed — see [methodology](docs/methodology.md). - Any conclusive budget failure, or any blocking critical/high-severity comparison → **`fail`**. Any lower-severity blocking regression → **`conditional_pass`**. Otherwise → **`pass`**. When severity alone fails the run, `reason` names each blocking comparison. - **`migration_policy` is the single source of truth for these budgets.** `migration_decision.json`'s `policy` field is the resolved policy `analyze` computed the verdict under — every top-level budget with its default applied, plus `slices`. `bundle`/`push` do not read `migration_decision.json`; `bundle` re-resolves the verdict from `evalshift.yaml` at bundle time, so the bundle's `decision.policy` (see [Hosted EvalShift](#hosted-evalshift)) is the policy the config held when the bundle was built — editing `migration_policy` between `analyze` and `bundle` changes what the bundle carries. `push ` builds a bundle only when `run_bundle.json.gz` is missing — an existing bundle is uploaded as-is — so re-run `evalshift bundle ` after a config edit. Either way, that is what lets the hosted gate check a pull request against exactly the budgets the bundle's own verdict used, instead of a separate, web-edited policy. `null` when no `migration_policy` is configured, and on a `migration_decision.json` written before this field existed. `evalshift.yaml` is the source of truth for that policy, and the web app's project policy view is becoming a read-only display of the snapshot each run pushed. Because every budget is resolved, the snapshot carries `max_invariant_violations` (default `0`) even for a config with no trace rules; hosted EvalShift accepts and stores it (since 2026-10-08) but never re-evaluates it — a run that carries its own policy is answered with the CLI's decision, which did check it, and only runs pushed without a policy fall back to the server's six-budget re-check. - **A slice budget gates the run exactly like an overall one.** The budgets under `migration_policy.slices` are evaluated on the same terms as the top-level ones: a conclusively breached slice budget **fails** the run, and an unconfirmed breach makes it `inconclusive` — the same Wilson rule, counted over that slice's own denominator. Since the overall rows can all be green in a run a slice budget fails, `recommendations` names the one that blocked (`The 'security' slice breached its overall regression rate budget (the share of scored comparisons where the target did worse): 20% over n=20 vs the 0% limit.`) and the `inconclusive` `reason` scope-qualifies it the same way. Per-slice verdicts under `slices[*].verdict` are unchanged, and a slice that fails on *comparison severity* rather than a budget still only downgrades an overall `pass` to `conditional_pass`. @@ -794,6 +795,8 @@ CI wiring (on `analyze` and `compare`): **Internal identifiers never reach the narrative.** Budgets and failure categories enter the prompt under their display names ("Cost increase", "Different tools chosen"), and the output is scanned for the identifiers the prompt could otherwise leak — FACTS keys like `cost_delta_pct`, `evalshift.yaml` budget fields like `max_tool_divergence`, machine category labels like `TOOL_SELECTION_DRIFT`. An echoed identifier rejects the generation exactly like an invented number. User-chosen evaluator names are exempt: they are the reader's own vocabulary. +**The cost and latency ceilings come with the figures they were judged on.** The run-wide changes (`cost_delta_pct`, `latency_delta_pct`: total cost, and mean latency over live calls) are not what the budgets gate. The budgets compare each example with itself, and latency trims spikes (see [the policy](#migration-policy)). FACTS therefore also carries `cost_budget_change_pct` and `latency_budget_change_pct`, and the instruction tells the model to set each ceiling only beside its budget figure and to call the latency one the typical example's change. Without them a run could read "latency rose 36%, within its 30% ceiling". The templated fallback and the report's economics card show the same pair. + **An absent rate is handed over as "not measured", never as a figure.** The equivalence, regression and improved rates share one denominator, and over an empty one they default to `0%` — which reads as "nothing regressed" when the truth is "nothing was compared". A run whose blocking evaluators produced no comparable row, or whose every row was excluded as a shared ground-truth miss, therefore gets a digit-free marker plus the reason, and the instruction forbids describing it as equivalent, consistent or free of regressions. The permit-list does the rest: with no rate rendered, a claim like "achieved a 100% equivalence rate" carries a figure that is not a fact and the generation is rejected. **A gate that measured nothing is never counted as a gate that passed.** A budget handed an empty sample — a tool-divergence ceiling on a run that scored no divergence row, a cost ceiling on a run with no priced call — is within its limit by arithmetic: `0/0` is below every ceiling. Counted naively that renders "7 of 7 budgets passed", and a narrative restates it as "all hard constraints are met" while the report body two sections above names the blocking evaluators that scored nothing. So `budgets_passed` counts only budgets that were *measured*, the blind ones are named on their own `unmeasured_budgets` / `unmeasured_evaluators` facts with a digit-free `coverage_basis` saying why, and the instruction forbids claiming a clean sweep, a met constraint or a safe migration while that basis is present. The blind-gate set is the same one `migration_decision.json`'s own `recommendations` name, computed by one shared function, so the two surfaces cannot disagree. This is distinct from the rates rule above and not covered by it: that fires when the *whole run* measured nothing, this fires when only *some* gate did. diff --git a/docs/configuration.md b/docs/configuration.md index 2f26deaf..4f6e99e5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -130,8 +130,8 @@ migration_policy: max_tool_divergence: 0.20 # share of pairs where the target routed elsewhere max_invariant_violations: 0 # examples where the target broke a blocking trace rule tool_argument_drift_floor: 0.9 # below this a call counts as drifted - max_cost_increase: 0.30 # target may cost up to 30% more - max_latency_increase: 0.30 # …and be up to 30% slower + max_cost_increase: 0.30 # target may cost up to 30% more, paired per example + max_latency_increase: 0.30 # …and the typical example be up to 30% slower ``` Those values are the ones `evalshift init` writes, and they are also the @@ -158,6 +158,18 @@ fails), since migrating a small model to a much larger one is legitimately far costlier and slower. They are capped at `10.0` (1000%) only to reject obvious typos (e.g. writing `200` instead of `2.0`). +Both compare each example with itself: an example counts only when both models +answered it without error. `max_cost_increase` caps the change in the paired +total cost. `max_latency_increase` caps the **typical example's** slowdown, +the 20%-trimmed geometric mean of per-example latency ratios. A few slow +provider responses therefore cannot flip the verdict, while a slowdown across +more than a fifth of the examples still shows. The reported value is signed +(`-0.29` = 29% cheaper); only the pass/fail is one-sided. With +`samples_per_example > 1` each example still counts once (its median latency +and mean cost per side). See +[methodology](methodology.md#cost-and-latency-are-compared-per-example) for +why this statistic and what it cannot see. + Rates are counted over whole rows, so pick values your suite can actually express. With 10 tool-argument rows the achievable drift rates are `{0.0, 0.1, 0.2, …}` and `max_tool_argument_drift: 0.01` means "any drift at @@ -174,20 +186,20 @@ The same check covers `max_overall_regression_rate` and `max_tool_divergence`, and each slice is judged on its own row count. Capture more examples, or write a budget the suite can represent; `0.0` is silent, since an explicit zero-tolerance budget is a choice. -The two *increase* budgets have no row denominator, but they can go unmeasured +The two *increase* budgets are not rates, but they can go unmeasured in their own way: if neither model is priced (LiteLLM has no cost table entry for the id), every `cost_usd` is `0.0` and the ratio has nothing to divide. `analyze` reports that budget as `conclusive: false` rather than as a clean pass, and says why: ``` -The cost increase budget could not be measured: all 4 error-free calls -across both models recorded a cost of 0, so its observed 0.00 is a default, -not a measurement. +The cost increase budget could not be measured: both models recorded a cost +of 0 on all 4 example pairs, so its observed 0.00 is a default, not a +measurement. ``` -`max_latency_increase` behaves the same when every call reports `latency_ms: 0` -— typically an unpriced or unmetered model. Price the models, or re-run +`max_latency_increase` behaves the same when no example pair has a +`latency_ms` above 0 on both sides — typically an unmetered model. Price the models, or re-run live, if you need those budgets to bind. `max_tool_divergence` is the divergence axis's own budget: the share of @@ -283,9 +295,10 @@ Two behaviours to know: confirms it; if the suite is too small to be sure, the verdict is `inconclusive`, not `fail`. A budget the observation *held* stays conclusive however wide its interval. - Cost and latency budgets are exact, but report - `conclusive: false` when neither model priced its calls (both averages - zero is a default, not a measurement). Record-derived budgets report + Cost and latency budgets have no interval: they are robust paired + estimates (latency trims spikes; see above), and a measured breach + fails outright. They report `conclusive: false` when neither model + priced its calls (both sides zero is a default, not a measurement). Record-derived budgets report `conclusive: false` on a scope that scored zero records — their `0/0` default looks clean but measures nothing. diff --git a/docs/methodology.md b/docs/methodology.md index 2a9cb8f0..5056555d 100644 --- a/docs/methodology.md +++ b/docs/methodology.md @@ -285,23 +285,108 @@ is excluded because it is a *floor*: below one row's granularity it collapses to maximally lax — only a 0% rate could fail it — so "effective tolerance is zero" would be the opposite of the truth. The count budgets (`max_critical_regressions`, `max_invariant_violations`) are not rates, and the -cost/latency ratios have no row denominator, so `1/n` describes none of them. +cost/latency ratios land on no grid, so `1/n` describes none of them. Widening the budget is not the fix, and neither is a confidence interval: at 1 material drift in 10 the 95% Wilson lower bound is `0.0179`, still above `0.01`, so the breach is statistically confirmed. The fix is more rows — or a budget written at a size the suite can express. +### Cost and latency are compared per example + +`max_cost_increase` and `max_latency_increase` compare each example's target +call with that example's own source call. Until this change they divided the +target's mean over all its error-free calls by the source's, which was +unpaired and fragile. Two runs of the same 21-example suite (Sonnet 4.6 → 5.5, +latency budget `0.30`) read **+27.2%** (passed) and **+35.6%** (failed). In +both runs, 86–95% of the excess came from two or three target responses of +3–5 s. They landed on *different* examples each run, and the same examples +took 1.1–1.9 s in the other run. That is provider tail noise, not a property of +the model, and it decided the verdict. + +**The unit is the example pair.** Calls are grouped by `(prompt_id, +example_id)`. An example counts only when each side has at least one +error-free call. An example the target failed on drops out on *both* sides, +so the source's slowest example is no longer weighed against a target that +never answered it. With `samples_per_example > 1` each side's samples collapse +to one value per example: the **median** for latency, so one slow repeat +cannot move its example, and the **mean** for cost, the example's expected +spend. An errored repeat drops only that sample. `denominator` is the number of +example pairs (21 for a 21-example suite, whatever the sample count), not the +calls on both sides it used to report (42). + +**Latency is the typical example's change.** The gated value is the +**20%-trimmed geometric mean of per-example target/source latency ratios**: +take `log(target / source)` for every pair, drop the lowest and highest +`floor(0.2 · n)` (4 of 21, none below 5 pairs), average the rest and +exponentiate. Logs, so a 2× slower and a 2× faster example cancel instead of +averaging to +25%. A pair with a latency of 0 on either side is a missing +timing, not an instant answer, and is left out of the latency ratio. + +Two alternatives were measured on the runs above before choosing: + +| Statistic | run 1 | run 2 | catches a real 2× slowdown on 30% of examples | +| --- | --- | --- | --- | +| mean of each side (old) | +27.2% ✓ | +35.6% ✗ | yes, and also flips on three spikes | +| median of per-example ratios | +11.4% | +23.7% | 0–8% of the time | +| **20%-trimmed geometric mean** | **+13.2%** | **+21.6%** | **88–100% of the time** | + +The median is the most robust, but it ignores any slowdown confined to fewer +than half the examples, so a target three times slower on a third of the suite +passes. The trimmed mean absorbs spikes on up to a fifth of the examples at +either end and still moves when a slowdown reaches more of them. It is also +the steadier of the two between the runs above: 8 points apart, against 12 for +the median. The limit is deliberate and worth knowing: **a slowdown confined +to fewer than one example in five is treated as noise.** The gate cannot tell a +reproducible slowdown on three examples from three unlucky responses in a +single sample. Raise `samples_per_example` to separate them (each example then +keeps its median), or look at the report's p95 latency, which is reported but +not gated. + +**Cost is the paired total.** The gated value is `Σ target / Σ source − 1` over +the same example pairs: the change in the bill for the examples both models +answered. Cost follows token counts, not network weather, so there is no +provider noise to trim. An example that costs ten times as much is a real +expense, and trimming it would hide exactly the verbose-reasoning blow-up a +cost budget exists to catch. On the runs above it reads −28.5% and −28.9%, +as stable as before. A paired source total of zero against a positive target +total is reported as `1.0`, a full 100% increase. + +**The value is signed; the gate is one-sided.** `observed` used to be clipped +at `0`, so a target 29% cheaper rendered as `+0%`. It is now the signed change +(`-0.289`) in `migration_decision.json`, the bundle and the report. The budget +still only caps the increase: `passed` is `observed <= allowed`, and any +saving passes. `PolicyMetricSummary.cost_increase_rate` / +`latency_increase_rate` carry the same signed values under their old names. + +**Cached rows count.** A cache hit carries the cost and latency recorded when +the response was produced live, so it is a real measurement of that +response, just not one taken on this run. Leaving cache hits out would turn +a re-run from cache into "measured nothing", which passes, so re-running a +suite would clear a latency breach. The report's run-wide latency figures and +per-example "Time Δ" still exclude replayed latency, because they describe +this run's own timing. That is why the report shows the budget's figures +beside them. + +**No interval, on purpose.** A paired bootstrap interval was tried and +rejected as a gate. Copying the Wilson rule (a breach fails only when the +interval's lower bound clears the budget) would almost never fire at these +suite sizes. On run 2 above the 95% bootstrap interval of the mean ratio is ++2.4%..+76.4%, so a breach would have to be enormous before its lower bound +cleared 30%. The stability comes from the estimator instead. Like the +critical-regression count, these rows carry `ci_low`/`ci_high` of `null`, and +a measured breach fails conclusively. + ### A cost ratio computed from zeroes is not a measurement -`max_cost_increase` and `max_latency_increase` are ratios of two averages, so -unlike the rate budgets they have no row denominator — their denominator is the -*source average*, and that can itself be zero. The ratio falls back to `0.00` -in two different situations, and `0.00` clears every budget: +`max_cost_increase` and `max_latency_increase` are ratios, so unlike the rate +budgets their divisor is the *source* value, and that can itself be zero. The +ratio falls back to `0.00` in two different situations, and `0.00` clears +every budget: -- **No error-free call on one side.** Every target call errored, or the run has - no calls at all. Reported as `conclusive: false`. -- **Both sides average zero.** The models are not priced (LiteLLM returns +- **No example answered on both sides.** Every target call errored, or the run + has no calls at all. Reported as `conclusive: false`. +- **Both sides recorded zero.** The models are not priced (LiteLLM returns `cost_usd: 0.0` for an id it has no pricing for), or every call reported `latency_ms: 0` for the same reason. @@ -322,9 +407,9 @@ Because the flag alone cannot distinguish the two situations for a reader, `analyze` adds a line to the recommendations for the second one: ``` -The cost increase budget could not be measured: all 4 error-free calls -across both models recorded a cost of 0, so its observed 0.00 is a default, -not a measurement. +The cost increase budget could not be measured: both models recorded a cost +of 0 on all 4 example pairs, so its observed 0.00 is a default, not a +measurement. ``` The first situation stays silent: an empty `raw.jsonl` beside a @@ -351,7 +436,7 @@ last from the first two, but it does not say *how much* was counted. Each | `max_tool_argument_drift` | the scope's `tool_arguments` rows only | | `max_tool_divergence` | the scope's `tool_selection.divergence` rows only | | `max_invariant_violations` | the distinct examples (`prompt_id`, `example_id`) with a blocking `trace_invariants` row in the scope — one per example however many entries scored it; the row is emitted only when there is at least one | -| `max_cost_increase` / `max_latency_increase` | the error-free calls both averages were taken over, across both roles | +| `max_cost_increase` / `max_latency_increase` | the example pairs compared — examples with an error-free call on both sides (for latency, timed above 0 on both) | The first three are counted over *measurements*, not over examples. An evaluator that scores several axes contributes a row per axis per example — @@ -369,7 +454,7 @@ can never be reported against a sample it was not judged on. `denominator` and `conclusive` answer different questions, and the cost budgets are where they visibly disagree: a run whose calls all report `cost_usd: 0` -counted every one of them and still measured nothing, so it reports a positive +counted every pair and still measured nothing, so it reports a positive denominator beside `conclusive: false`. Reading either field as a stand-in for the other loses that. @@ -395,7 +480,7 @@ binomial proportion, and only those get a confidence interval. There are four: | `max_tool_divergence` | diverged rows over the scope's `tool_selection.divergence` rows | `max_critical_regressions` and `max_invariant_violations` are raw counts and -the cost/latency budgets are ratios of two averages; none describes a +the cost/latency budgets are paired ratios; none describes a proportion, so none is given an interval and all report `ci_low`/`ci_high` as `null`. diff --git a/llms-full.txt b/llms-full.txt index 8656b245..62505a5d 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -964,8 +964,8 @@ migration_policy: # optional; fractions not percents max_invariant_violations: int = 0 (>=0) # COUNT of examples where the target broke a # blocking trace_invariants rule; always conclusive tool_argument_drift_floor: float = 0.9 (0..1) # target arg score below which a call counts as drifted - max_cost_increase: float = 0.30 (0..10) - max_latency_increase: float = 0.30 (0..10) + max_cost_increase: float = 0.30 (0..10) # paired total cost change, see "Cost/latency" below + max_latency_increase: float = 0.30 (0..10) # typical example's slowdown (20%-trimmed geo-mean) fail_on_dropped_params: bool = false # fail when state.json dropped_params is non-empty, # i.e. an arm could not honour a generation constraint the capture recorded. Top level only # (a model either accepts a param or does not; not a per-slice property). @@ -1228,8 +1228,8 @@ Verdicts: pass | conditional_pass | fail | inconclusive. Written to migration_de ASYMMETRIC: a breach fails ONLY when the CI confirms it (favourable bound clears the budget); a breach with the CI still spanning the budget -> inconclusive (suite too small); a budget the observation HELD is conclusive however wide its CI (a wide CI must never - downgrade a clean run). max_critical_regressions is a raw count and cost/latency are ratios - of two averages - no proportion, so ci_low/ci_high are null and no CI softens them (a + downgrade a clean run). max_critical_regressions is a raw count and cost/latency are paired + ratios - no proportion, so ci_low/ci_high are null and no CI softens them (a measured breach is never softened), though they are NOT always conclusive - see the zero-valued rule below. Drift got its CI last, once bundles began reporting its denominator: the hosted gate had always scored drift as a proportion, so the same thin sample read fail @@ -1270,16 +1270,36 @@ Verdicts: pass | conditional_pass | fail | inconclusive. Written to migration_de stays empty however many are added. The DIAGNOSIS is wider than the exclusion: `evaluate` separately grades the SOURCE side alone and prints a red "broken eval harness" row when the source failed >= 50% of >= 4 conformance rows (see evaluators.tool_selection above). -- Cost/latency ratios default to 0.00 twice over, and BOTH are conclusive: false. (1) Either - arm has no error-free call (all target calls errored, or no calls at all). (2) Both arms - average zero - unpriced models (LiteLLM returns cost_usd 0.0 for an unpriced id, and - latency_ms 0 alongside it). Case 2 has non-empty call lists, so pairing alone called +- Cost/latency are compared PER EXAMPLE (analysis/policy.py::_relative_change). Calls grouped + by (prompt_id, example_id); an example counts only when EACH side has >=1 error-free call, + so an example one side failed leaves BOTH sides. samples_per_example > 1: each side + collapses to one value per example (latency: median of samples; cost: mean); an errored + repeat drops only that sample. Cache hits COUNT, with the cost/latency recorded when they + ran live (excluding them would make a cache re-run "measured nothing" = pass, clearing a + breach); the report's run-wide latency figures still exclude replayed latency. + latency: observed = exp(trimmed_mean(sorted log(target/source), drop floor(0.2*n) each + end; 4 of 21, none below 5)) - 1. Pairs with latency_ms 0 on either side are left out. + Why: provider tail spikes (2-3 target calls at 3-5 s, different examples each run) + flipped a real 21-example suite between +27% pass and +36% fail on the old mean ratio; + trimmed reads +13%/+22%. Median rejected: blind to a slowdown on < half the examples. + Limit: a slowdown confined to < 1 example in 5 is treated as noise. + cost: observed = sum(target)/sum(source) - 1 over the pairs, UNTRIMMED (cost follows + tokens, an expensive answer is a real bill). sum(source) <= 0 < sum(target) -> 1.0. + observed is SIGNED (-0.289 = 28.9% cheaper; used to be clipped at 0); passed = observed <= + allowed, so only the gate is one-sided. PolicyMetricSummary cost_increase_rate / + latency_increase_rate carry the same signed values. No interval: a bootstrap lower-bound + rule would almost never fire at n~21. +- Cost/latency ratios default to 0.00 twice over, and BOTH are conclusive: false. (1) No + example has an error-free call on both sides (all target calls errored, or no calls at + all). (2) Both sides recorded zero on every pair - unpriced models (LiteLLM returns cost_usd + 0.0 for an unpriced id, and latency_ms 0 alongside it). Case 2 has paired examples, so pairing alone called it measured and rendered "observed 0.00, passed, conclusive" for a cost never priced. Call.cost_usd is 0.0 both for unpriced and for genuinely free, and nothing separates them, so a genuinely free pair reads unmeasured too - deliberate: honest-uncertain over confident pass. Case 2 also adds a recommendations line: "The cost increase budget could not be - measured: all 4 error-free calls across both models recorded a cost of 0, so its observed - 0.00 is a default, not a measurement." Case 1 stays silent (empty raw.jsonl already says it). + measured: both models recorded a cost of 0 on all 4 example pairs, so its observed 0.00 is + a default, not a measurement." (latency: "none of the N example pairs recorded a latency + above 0 on both models") Case 1 stays silent (empty raw.jsonl already says it). Emitted once per run, not per scope (all scopes read the same run-level calls). Only conclusive changes; observed 0.00 still clears the budget, so no verdict moves. - Sub-granular rate CEILINGS are warned about, not silently enforced. A rate over n rows can @@ -1301,14 +1321,15 @@ Verdicts: pass | conditional_pass | fail | inconclusive. Written to migration_de tool_arguments rows for max_tool_argument_drift; tool_selection.divergence rows for max_tool_divergence; distinct examples with a blocking trace_invariants row for max_invariant_violations; the - error-free calls behind both averages (both roles summed, - 0 when either role has none) for max_cost_increase / max_latency_increase. Slices report + example pairs compared (NOT calls on both sides; 21 for a 21-example suite whatever + samples_per_example is; latency counts only pairs timed > 0 on both; 0 when no example is + paired) for max_cost_increase / max_latency_increase. Slices report their own counts. Three readings, all distinct: 0 = counted and the sample was empty (observed is a default, passed is vacuous); positive int = that many units counted; MISSING = no sample size reported, NOT zero - only bundles predating the field say that, and the hosted gate falls back to conclusive for them. This CLI always emits an integer. Same number the 1/n granularity warning is judged on (one map, two readers). Orthogonal - to conclusive: an all-zero cost ratio counted every call it averaged and still measured + to conclusive: an all-zero cost ratio counted every pair it compared and still measured nothing -> positive denominator, conclusive: false. The hosted gate computes its OWN Wilson interval from these denominators, over the same three proportion budgets and with the same z, so a local and a hosted verdict agree on whether a breach was confirmed. @@ -1376,6 +1397,13 @@ Model: defaults.insights_model, falling back to defaults.judge_model. evalshift.yaml budget fields (max_tool_divergence, ...), machine category labels (TOOL_SELECTION_DRIFT, ...). An echoed identifier rejects the generation like an invented number. User-chosen evaluator names are exempt (they are the reader's own vocabulary). +- Cost/latency CEILINGS are judged on their own figures, not on cost_delta_pct (run totals) / + latency_delta_pct (mean live latency): FACTS also carries cost_budget_change_pct and + latency_budget_change_pct (the budget rows' signed observed, "not measured" when the row + measured nothing), and the instruction says to set a ceiling only beside its budget figure + and call latency's "the typical example's change", never the average. The fallback and the + report's economics card ("judged per example: cost X, latency Y for the typical example") + show the same pair. - An ABSENT rate is not a figure. equivalence/regression/improved share one denominator and default to 0% over an empty one, which reads as "nothing regressed" when the truth is "nothing was compared". decision.overall.n_records == 0 (no blocking row, or every row diff --git a/src/evalshift_cli/analysis/policy.py b/src/evalshift_cli/analysis/policy.py index b0526faf..2444ac53 100644 --- a/src/evalshift_cli/analysis/policy.py +++ b/src/evalshift_cli/analysis/policy.py @@ -8,6 +8,7 @@ from __future__ import annotations import math +import statistics from collections import Counter from collections.abc import Mapping, Sequence from dataclasses import asdict, dataclass, field @@ -69,8 +70,8 @@ # denominator but is a *floor*, and a sub-granular floor collapses to # maximally lax — only a 0% rate could fail it — which is the opposite of zero # tolerance, so the warning's wording would be false. ``max_critical_regressions`` -# is a count on a granularity of 1, and the cost/latency budgets are ratios of -# two averages with no row denominator at all. +# is a count on a granularity of 1, and the cost/latency budgets are ratios +# across example pairs, which land on no grid. _RATE_CEILING_DENOMINATORS: dict[str, str] = { "max_overall_regression_rate": "scored comparisons", "max_tool_argument_drift": "tool-argument comparisons", @@ -122,19 +123,35 @@ # policy edit change what ``conclusive`` means. _WILSON_Z = 1.959963984540054 -# The call-derived budgets, and the ``Call`` field each one averages over both -# roles. Unlike the rate ceilings above these have no row denominator — their -# denominator is the *source average*, which can itself be zero. See -# :func:`_has_measured_ratio` and :func:`_zero_valued_call_warnings`. +# The call-derived budgets, and the ``Call`` field each one compares across +# example pairs. Unlike the rate ceilings above these are ratios, not +# proportions — their divisor is the *source* value, which can itself be zero. +# See :func:`_relative_change`, :func:`_has_measured_ratio` and +# :func:`_zero_valued_call_warnings`. _CALL_RATIO_FIELDS: dict[str, str] = { "max_cost_increase": "cost_usd", "max_latency_increase": "latency_ms", } +# Share of example pairs trimmed from *each* end before the latency +# log-ratios are averaged — 4 of 21, 0 below 5 pairs. 20% is the conventional +# robust default: it absorbs the handful of 3-5 s provider spikes that flipped +# a real suite's verdict between two runs, while a slowdown on more than a +# fifth of the examples survives into the value. Not configurable, for the +# same reason the Wilson z is not: a per-project trim would let a policy edit +# change what the latency budget measures. +_LATENCY_TRIM = 0.2 + @dataclass(frozen=True, slots=True) class PolicyMetricSummary: - """Aggregate migration outcome rates for a scope.""" + """Aggregate migration outcome rates for a scope. + + ``cost_increase_rate`` / ``latency_increase_rate`` keep their names — the + bundle schema is the server's — but are *signed* relative changes over + example pairs (see :func:`_relative_change`): ``-0.25`` is 25% cheaper or + faster. + """ n_records: int improved_rate: float @@ -165,13 +182,17 @@ class BudgetResult: but measures nothing. ``max_tool_argument_drift`` is counted over ``tool_arguments`` rows only, so it needs one of *those* — a scope that scored records but ran no tool-argument evaluator is unmeasured too. - Exact budgets (the critical-regression count, the cost/latency ratios of - two averages) describe no proportion and have - ``ci_low``/``ci_high`` of ``None``. The call-derived cost/latency budgets - follow the same rule from the other side: the ratio reads 0.0 — "no - increase" — both when a role has no error-free call to average and when - both roles average zero, so they are conclusive only when at least one - role contributed a positive average for that field + The critical-regression count and the cost/latency ratios describe no + proportion and have ``ci_low``/``ci_high`` of ``None``. The cost/latency + ratios are not exact either — they are robust paired estimates (see + :func:`_relative_change`) whose stability comes from comparing each example + with itself and trimming latency spikes, not from an interval: at the + suite sizes this runs at, an interval-confirmed breach would almost never + fire. Their ``observed`` is *signed* — a saving is negative — and only + ``passed`` is one-sided. They follow the empty-sample rule from the other + side: the ratio reads 0.0 both when no example has an error-free call on + each side and when both sides recorded zero on every pair, so they are + conclusive only when some pair recorded a positive value for that field (:func:`_has_measured_ratio`). ``denominator`` is how many units ``observed`` was computed over — the @@ -827,8 +848,8 @@ def _metrics( regression_rate=_rate(len(regressions), n), critical_regressions=len(critical), tool_argument_drift_rate=_rate(len(tool_argument_drift), len(tool_argument_records)), - cost_increase_rate=_relative_increase(calls, field="cost_usd"), - latency_increase_rate=_relative_increase(calls, field="latency_ms"), + cost_increase_rate=_relative_change(calls, field="cost_usd"), + latency_increase_rate=_relative_change(calls, field="latency_ms"), ) @@ -875,8 +896,8 @@ def _budget_results( when the denominator is positive. ``cost_measured`` / ``latency_measured`` are the call-derived budgets' conclusiveness flags and stay separate from their denominators: those ratios also default to 0.0 - with a perfectly good call count behind them, when both roles average zero - (see :func:`_has_measured_ratio`). + with a perfectly good pair count behind them, when both sides recorded + zero (see :func:`_has_measured_ratio`). All of it is passed in rather than read off ``metrics`` because ``PolicyMetricSummary`` is serialised into the bundle manifest, whose @@ -1136,13 +1157,16 @@ def _zero_valued_call_warnings(calls: list[Call]) -> list[str]: continue if _has_measured_ratio(calls, field=call_field): continue - source, target = _paired_call_values(calls, field=call_field) - noun = "a cost" if call_field == "cost_usd" else "a latency" + n = len(_example_pairs(calls, field=call_field)) + pairs = f"{n} example pair{'' if n == 1 else 's'}" + what = ( + f"both models recorded a cost of 0 on all {pairs}" + if call_field == "cost_usd" + else f"none of the {pairs} recorded a latency above 0 on both models" + ) out.append( - f"The {_label_lower(name)} budget could not be measured: all " - f"{len(source) + len(target)} error-free calls across both models " - f"recorded {noun} of 0, so its observed 0.00 is a default, not a " - f"measurement.", + f"The {_label_lower(name)} budget could not be measured: {what}, so " + f"its observed 0.00 is a default, not a measurement.", ) return out @@ -1299,7 +1323,7 @@ def _distinct_slice_budgets( ) -> list[BudgetResult]: """Slice budgets that measure something an overall row does not. - ``max_cost_increase`` / ``max_latency_increase`` are averaged over the + ``max_cost_increase`` / ``max_latency_increase`` are measured over the run's calls, so an inheriting slice restates the overall row number for number — and a run with three slices would report the same breach four times. A slice that *overrode* the budget, or one whose record-derived @@ -1769,65 +1793,96 @@ def _rate(count: int, total: int) -> float: return 0.0 if total <= 0 else count / total -def _paired_call_values(calls: list[Call], *, field: str) -> tuple[list[float], list[float]]: - """Error-free ``field`` values for the source and target roles. +def _example_pairs(calls: list[Call], *, field: str) -> list[tuple[float, float]]: + """One ``(source, target)`` value of ``field`` per example both roles answered. + + The unit is the example — ``(prompt_id, example_id)`` — not the call, so + each target is compared against its own source and an example one side + failed leaves the comparison on *both* sides. Comparing whatever each role + happened to complete compares two different sets of examples, which is a + difference in workload, not in models. + + Errored calls are dropped one sample at a time; an example stays paired + while each role kept at least one error-free sample. Cached rows count: + a cache hit carries the cost and latency recorded when the response was + produced live (``Call.latency_replayed``), and dropping them would turn a + re-run from cache into "measured nothing" — a pass, for a run whose gate + had failed. + + With ``samples_per_example > 1`` a role's samples collapse to one value: + the median for latency, so one spiked repeat cannot move its example, and + the mean for cost, the example's expected spend. + """ + samples: dict[tuple[str, str], dict[str, list[float]]] = {} + for call in calls: + if call.error is not None: + continue + roles = samples.setdefault((call.prompt_id, call.example_id), {}) + roles.setdefault(call.role, []).append(float(getattr(call, field))) + collapse = statistics.median if field == "latency_ms" else statistics.fmean + return [ + (collapse(roles["source"]), collapse(roles["target"])) + for _, roles in sorted(samples.items()) + if "source" in roles and "target" in roles + ] + + +def _ratio_pairs(calls: list[Call], *, field: str) -> list[tuple[float, float]]: + """The example pairs :func:`_relative_change` actually compares for ``field``. + + Every pair for cost. For latency only pairs timed on both sides: every + real call takes time, so a ``latency_ms`` of 0 is a missing timing, and a + log-ratio needs two positive values. - Both the cost/latency ratio and its ``conclusive`` flag read this one - function, so "we had samples to compare" can never disagree with the - early return in :func:`_relative_increase`. + Its length is the budget's ``denominator``; every reader of the ratio goes + through here, so the number a budget reports and the sample it was judged + on cannot drift apart. """ - source = [float(getattr(c, field)) for c in calls if c.role == "source" and c.error is None] - target = [float(getattr(c, field)) for c in calls if c.role == "target" and c.error is None] - return source, target + pairs = _example_pairs(calls, field=field) + if field == "latency_ms": + return [(source, target) for source, target in pairs if source > 0 and target > 0] + return pairs def _has_paired_calls(calls: list[Call], *, field: str) -> bool: - """Whether :func:`_relative_increase` had anything to compare for ``field``. + """Whether any example has an error-free call on both sides. - False means its 0.0 is the "nothing to divide" default rather than a - measured ratio — the budget row is unmeasured, not clean. + False means :func:`_relative_change`'s 0.0 is the "nothing to compare" + default rather than a measured ratio — the budget row is unmeasured, not + clean. Necessary but not sufficient for the budget to be conclusive; see :func:`_has_measured_ratio`, which is what ``_budget_results`` is given. """ - source, target = _paired_call_values(calls, field=field) - return bool(source) and bool(target) + return bool(_example_pairs(calls, field=field)) def _call_ratio_denominator(calls: list[Call], *, field: str) -> int: - """How many error-free calls the ``field`` ratio was averaged over. + """How many example pairs the ``field`` ratio was computed over. - Zero when either role contributed none: :func:`_relative_increase` divides - one role's average by the other's, so one-sided data compares nothing and - its 0.0 is the "nothing to divide" default (:func:`_has_paired_calls`) — - which is exactly the "counted, and there was nothing to count" that + Pairs, not calls: the ratio compares each example's target against its own + source, so a 21-example suite reports 21 whatever ``samples_per_example`` + is, and an example one side failed is not counted at all. Zero when no + example is paired — the "counted, and there was nothing to count" that ``BUNDLE_SPEC.md`` reserves ``0`` for. - Counts *calls on both sides*, not pairs. ``BUNDLE_SPEC.md`` calls this - "paired calls", but the CLI never pairs individual calls: it averages each - role independently over whatever error-free calls that role produced, and - the two counts need not match. There is therefore no pair count to report, - and the number of calls behind the two averages is the honest sample size. It is deliberately *not* the same question as :func:`_has_measured_ratio`: - a run whose calls all carry ``cost_usd = 0`` counted every one of them and - still measured nothing, so it reports a positive denominator beside + a run whose calls all carry ``cost_usd = 0`` counted every pair and still + measured nothing, so it reports a positive denominator beside ``conclusive: false``. """ - source, target = _paired_call_values(calls, field=field) - if not source or not target: - return 0 - return len(source) + len(target) + return len(_ratio_pairs(calls, field=field)) def _has_measured_ratio(calls: list[Call], *, field: str) -> bool: - """Whether :func:`_relative_increase` measured ``field``, rather than defaulting. + """Whether :func:`_relative_change` measured ``field``, rather than defaulting. - It defaults to 0.0 twice over: when a role has no error-free call at all - (:func:`_has_paired_calls`), and when both roles average zero — an - unpriced model pair, whose calls carry ``cost_usd = 0.0`` (or - ``latency_ms = 0``) on every row. The second case has non-empty call - lists, so pairing alone reports it as a confident "the target did not - cost more" for a quantity nobody ever priced. + It defaults to 0.0 twice over: when no example is paired + (:func:`_has_paired_calls`), and when both roles recorded zero on every + pair — an unpriced model pair, whose calls carry ``cost_usd = 0.0`` (or + ``latency_ms = 0``) on every row. The second case has paired examples, so + pairing alone reports it as a confident "the target did not cost more" for + a quantity nobody ever priced. Both-sides-zero is therefore treated as **unmeasured**. ``Call.cost_usd`` is ``0.0`` both when a model is unpriced and when it is genuinely free, and @@ -1836,25 +1891,52 @@ def _has_measured_ratio(calls: list[Call], *, field: str) -> bool: pass on an unpriced migration is the worse failure, and "could not tell" is recoverable by pricing the models. - Kept in lockstep with :func:`_relative_increase` — both read - :func:`_paired_call_values`, and the branch below mirrors its early - returns, so the flag can never disagree with the number it describes. + Kept in lockstep with :func:`_relative_change` — both read + :func:`_ratio_pairs`, so the flag can never disagree with the number it + describes. """ - source, target = _paired_call_values(calls, field=field) - if not source or not target: - return False - return (sum(source) / len(source)) > 0 or (sum(target) / len(target)) > 0 - - -def _relative_increase(calls: list[Call], *, field: str) -> float: - source, target = _paired_call_values(calls, field=field) - if not source or not target: + pairs = _ratio_pairs(calls, field=field) + return any(source > 0 or target > 0 for source, target in pairs) + + +def _relative_change(calls: list[Call], *, field: str) -> float: + """Signed relative change of ``field`` from source to target, over example pairs. + + ``-0.25`` is 25% cheaper or faster. Only the budget's pass/fail is + one-sided (``observed <= allowed``); the value itself is never clipped, so + a saving reads as one in the report instead of as ``+0%``. + + The two fields are measured differently because their noise is: + + * **Latency** — the 20%-trimmed geometric mean of per-example + target/source ratios (:data:`_LATENCY_TRIM`). Provider latency has a + heavy tail that lands on different examples each run, and a plain mean + let two or three slow responses flip a verdict. The trim absorbs up to + a fifth of the examples spiking on either side; a slowdown spread over + more of them than that still moves the value. Not the median: it cannot + see a slowdown confined to fewer than half the examples. Logs, so a 2x + slower and a 2x faster example cancel instead of averaging to +25%. + * **Cost** — the ratio of paired totals. Cost follows token counts, not + network weather, so it has no provider noise to trim, and an example + that costs ten times as much is a real bill the budget must see. + + A cost whose paired source total is zero against a positive target total + is reported as ``1.0`` — a full 100% increase rather than infinity or a + clean zero. + """ + pairs = _ratio_pairs(calls, field=field) + if not pairs: return 0.0 - source_avg = sum(source) / len(source) - target_avg = sum(target) / len(target) - if source_avg <= 0: - return 0.0 if target_avg <= 0 else 1.0 - return max(0.0, (target_avg - source_avg) / source_avg) + if field == "latency_ms": + logs = sorted(math.log(target / source) for source, target in pairs) + trim = int(_LATENCY_TRIM * len(logs)) + kept = logs[trim : len(logs) - trim] + return math.exp(statistics.fmean(kept)) - 1.0 + source_total = sum(source for source, _ in pairs) + target_total = sum(target for _, target in pairs) + if source_total <= 0: + return 0.0 if target_total <= 0 else 1.0 + return (target_total - source_total) / source_total __all__ = [ diff --git a/src/evalshift_cli/config/models.py b/src/evalshift_cli/config/models.py index c02b0af9..3fda8273 100644 --- a/src/evalshift_cli/config/models.py +++ b/src/evalshift_cli/config/models.py @@ -623,6 +623,13 @@ class MigrationPolicy(_StrictModel): ``max_cost_increase=0.30`` means the target may cost up to 30% more than the source before the policy fails. + The two *increase* budgets compare each example's target call with its own + source call. ``max_cost_increase`` caps the change in paired total cost; + ``max_latency_increase`` caps the typical example's slowdown — the + 20%-trimmed geometric mean of per-example latency ratios, so a few slow + provider responses cannot flip the verdict. See + ``analysis.policy._relative_change``. + The regression/equivalence rates (``max_overall_regression_rate``, ``min_equivalence_rate``, ``max_tool_argument_drift``, ``max_tool_divergence``) are true rates diff --git a/src/evalshift_cli/insights/facts.py b/src/evalshift_cli/insights/facts.py index 80622155..04d27b98 100644 --- a/src/evalshift_cli/insights/facts.py +++ b/src/evalshift_cli/insights/facts.py @@ -234,6 +234,14 @@ def build_facts( deltas = [e.worst_delta_score for e in examples if e.worst_delta_score is not None] worst_comparison = _worst_comparison(comparisons) budget_limits = {b.name: _budget_limit(b.name, b.allowed) for b in decision.budget_results} + # What each increase budget was judged on: paired per example, latency as + # the typical example with its spikes trimmed. Not the run-wide deltas + # below, which the prose once set against a ceiling the gate held. + budget_changes = { + b.name: _signed_pct(b.observed) if b.measured else NOT_MEASURED + for b in decision.budget_results + if b.name in _INCREASE_BUDGETS + } # The three outcome rates share one denominator, so they are measured or # missing together. ``_rate`` returns 0.0 over an empty one, which reads # as "nothing regressed" rather than "nothing was compared" — and that is @@ -304,6 +312,8 @@ def build_facts( ), "cost_ceiling_pct": budget_limits.get("max_cost_increase", NOT_AVAILABLE), "latency_ceiling_pct": budget_limits.get("max_latency_increase", NOT_AVAILABLE), + "cost_budget_change_pct": budget_changes.get("max_cost_increase", NOT_AVAILABLE), + "latency_budget_change_pct": budget_changes.get("max_latency_increase", NOT_AVAILABLE), "regression_rate_pct": ( _rate_pct(decision.overall.regression_rate) if measured else NOT_MEASURED ), @@ -396,9 +406,9 @@ def _signed_pct(value: float) -> str: def _relative_pct(source: float, target: float) -> str: """Render ``(target - source) / source``, or say it isn't measurable. - Not ``policy._relative_increase``: that clamps at zero because a budget - only cares about increases, which would render a genuinely cheaper target - as ``+0%``. + A run-wide ratio of totals or means. Not the budgets' own figure + (``policy._relative_change``), which compares each example with itself + and, for latency, trims the slowest and fastest fifth. """ if source <= 0: return NOT_COMPARABLE diff --git a/src/evalshift_cli/insights/generator.py b/src/evalshift_cli/insights/generator.py index c206bad1..cdc0fe3c 100644 --- a/src/evalshift_cli/insights/generator.py +++ b/src/evalshift_cli/insights/generator.py @@ -193,6 +193,12 @@ the tool names shown in SAMPLED REGRESSIONS. - The two cost figures are each model's total for the whole run. Never present either one as a per-call or per-example price. +- The cost and latency ceilings are judged on cost_budget_change_pct and + latency_budget_change_pct, not on the run-wide changes: each example is + compared with itself, and latency is the typical example's change with the + slowest and fastest fifth set aside. Set a ceiling only beside its budget + figure, and call the latency one the typical example's change, never the + average. - In SAMPLED REGRESSIONS, an output beginning "{tool_calls}" is a reply that called those tools, listed with their arguments, before any "{text}" it wrote. Describe it as calling them, never as an empty or missing answer. diff --git a/src/evalshift_cli/insights/templates.py b/src/evalshift_cli/insights/templates.py index 8fd8bd6e..be256403 100644 --- a/src/evalshift_cli/insights/templates.py +++ b/src/evalshift_cli/insights/templates.py @@ -154,7 +154,9 @@ def _economics_summary(rendered: dict[str, str]) -> str: f"{rendered['cost_target_total_usd']} ({rendered['n_calls']} calls, both " f"models). Cost {rendered['cost_delta_pct']}, " f"latency {rendered['latency_delta_pct']}. Ceilings: cost " - f"{rendered['cost_ceiling_pct']}, latency {rendered['latency_ceiling_pct']}." + f"{rendered['cost_ceiling_pct']}, latency {rendered['latency_ceiling_pct']}, " + f"judged per example: cost {rendered['cost_budget_change_pct']}, latency " + f"{rendered['latency_budget_change_pct']} for the typical example." ) diff --git a/src/evalshift_cli/reports/templates/report.html.j2 b/src/evalshift_cli/reports/templates/report.html.j2 index 81c903bc..d9d82f2e 100644 --- a/src/evalshift_cli/reports/templates/report.html.j2 +++ b/src/evalshift_cli/reports/templates/report.html.j2 @@ -11,6 +11,12 @@ {%- if value is none -%}—{%- else -%}{{ ("%+." ~ digits ~ "f")|format(value) }}%{%- endif -%} {%- endmacro -%} +{#- A cost/latency budget's own signed figure, or a dash when it measured + nothing: a passing row that is not conclusive is the 0/0 default. -#} +{%- macro budget_change(budget) -%} + {{ pct(budget.observed * 100 if budget.get("conclusive", true) or not budget.passed else none) }} +{%- endmacro -%} + {#- A tool change renders as one arrow, or as one arrow per round when the replay was teacher-forced: "a, b -> a, c" cannot say which round went wrong, and the round is the whole finding. -#} @@ -390,8 +396,11 @@ {%- set cost_cap = budget_limit(budgets, "max_cost_increase") -%} {%- set lat_cap = budget_limit(budgets, "max_latency_increase") -%} {% if cost_cap and lat_cap %} + {# The ceilings are judged per example, latency on the typical + one, so the run-wide figures above are not what they held to. #} {{ "Both inside" if cost_cap.passed and lat_cap.passed else "Ceilings" }} - +{{ "%.0f"|format(cost_cap.allowed * 100) }}% cost / +{{ "%.0f"|format(lat_cap.allowed * 100) }}% latency. + +{{ "%.0f"|format(cost_cap.allowed * 100) }}% cost / +{{ "%.0f"|format(lat_cap.allowed * 100) }}% latency, + judged per example: cost {{ budget_change(cost_cap) }}, latency {{ budget_change(lat_cap) }} for the typical example. {% endif %}

@@ -469,7 +478,7 @@ {%- if name in count_budgets -%} {{ "%.0f"|format(value) }} {%- else -%} - {{ "+" if name in increase_budgets else "" }}{{ "%.1f"|format(value * 100) }}% + {{ ("%+.1f" if name in increase_budgets else "%.1f")|format(value * 100) }}% {%- endif -%} {%- endmacro -%}
diff --git a/tests/unit/test_insight_facts.py b/tests/unit/test_insight_facts.py index acf43d35..e5098b47 100644 --- a/tests/unit/test_insight_facts.py +++ b/tests/unit/test_insight_facts.py @@ -83,6 +83,64 @@ def test_verdict_and_budget_counts_are_rendered(sample_run: dict[str, Any]) -> N assert facts.rendered["latency_ceiling_pct"] == "+30%" +def test_the_budgets_own_cost_and_latency_figures_are_rendered( + sample_run: dict[str, Any], +) -> None: + """The ceilings are judged on paired per-example figures, not the run totals. + + Rendered beside ``cost_delta_pct`` / ``latency_delta_pct`` so the prose + never sets a run-wide +36% latency against a 30% ceiling the gate held on + the typical example's +22%. + """ + sample_run["decision"] = replace( + sample_run["decision"], + budget_results=[ + *[b for b in budgets() if b.name not in {"max_cost_increase", "max_latency_increase"}], + BudgetResult( + name="max_cost_increase", observed=-0.289, allowed=0.5, passed=True, denominator=21 + ), + BudgetResult( + name="max_latency_increase", + observed=0.216, + allowed=0.3, + passed=True, + denominator=21, + ), + ], + ) + facts = build_facts(**sample_run) + assert facts.rendered["cost_budget_change_pct"] == "\u221228.9%" + assert facts.rendered["latency_budget_change_pct"] == "+21.6%" + + +def test_an_unmeasured_budget_figure_is_not_rendered_as_a_change( + sample_run: dict[str, Any], +) -> None: + sample_run["decision"] = replace( + sample_run["decision"], + budget_results=[ + *[b for b in budgets() if b.name != "max_latency_increase"], + BudgetResult( + name="max_latency_increase", + observed=0.0, + allowed=0.3, + passed=True, + conclusive=False, + denominator=0, + ), + ], + ) + assert build_facts(**sample_run).rendered["latency_budget_change_pct"] == "not measured" + + +def test_the_fallback_names_what_the_budgets_were_judged_on( + sample_run: dict[str, Any], +) -> None: + summary = fallback_insight(build_facts(**sample_run)).economics_summary + assert "judged per example" in summary + assert "typical example" in summary + + def test_delta_spread_is_rendered_from_the_examples(sample_run: dict[str, Any]) -> None: facts = build_facts(**sample_run) assert facts.rendered["worst_delta"] == "−0.0800" diff --git a/tests/unit/test_policy.py b/tests/unit/test_policy.py index 8935e1b7..a03544a8 100644 --- a/tests/unit/test_policy.py +++ b/tests/unit/test_policy.py @@ -96,16 +96,22 @@ def _call( cost_usd: float, latency_ms: int, error: str | None = None, + prompt_id: str = "p", + sample_index: int = 0, + cached: bool = False, ) -> Call: return Call( run_id="r1", - prompt_id="p", + prompt_id=prompt_id, example_id=example_id, model_id=role, role=role, # type: ignore[arg-type] + sample_index=sample_index, text="" if error else "ok", cost_usd=cost_usd, latency_ms=latency_ms, + cached=cached, + cached_rounds=1 if cached else 0, error=error, ) @@ -1135,8 +1141,8 @@ def test_the_bounds_match_the_servers_to_full_precision(self) -> None: class TestCallBudgetConclusiveness: """The cost/latency budgets are call-derived, so calls can be missing too. - ``_relative_increase`` returns 0.0 whenever either side has no error-free - call, and 0.0 clears every ``ge=0.0`` budget. That renders as + ``_relative_change`` returns 0.0 whenever no example has an error-free + call on both sides, and 0.0 clears every ``ge=0.0`` budget. That renders as ``observed 0.00, passed`` — "the target did not cost more" — derived from zero samples. Only ``conclusive`` can tell that apart from a genuine measurement of no increase. @@ -1232,11 +1238,205 @@ def test_a_real_cost_increase_still_fails_conclusively(self) -> None: assert decision.verdict == "fail" +class TestPairedCallBudgets: + """The cost/latency budgets compare each example against itself. + + They used to divide one role's mean by the other's. On real data that + flipped a migration verdict between two runs of the same 21-example suite + (+27% passed, +36% failed against a 30% latency budget): 86-95% of the + excess came from two or three 3-5 s target responses that landed on + different examples each run. These tests pin the replacement: latency is + the 20%-trimmed geometric mean of per-example target/source ratios, cost is + the ratio of paired totals, and both count example pairs. + """ + + def _budgets( + self, + calls: list[Call], + policy: MigrationPolicy | None = None, + ) -> dict[str, BudgetResult]: + decision = evaluate_migration_policy( + run_id="r1", + source_model="src", + target_model="tgt", + policy=policy + or MigrationPolicy(max_overall_regression_rate=1.0, min_equivalence_rate=0.0), + comparisons=[_comparison(severity="none", delta_avg_score=0.0)], + records=[_record(example_id="e1", delta=0.0)], + calls=calls, + ) + return {b.name: b for b in decision.budget_results} + + @staticmethod + def _latency_pairs(source_ms: list[int], target_ms: list[int]) -> list[Call]: + calls: list[Call] = [] + for i, (src, tgt) in enumerate(zip(source_ms, target_ms, strict=True)): + calls.append(_call(example_id=f"e{i}", role="source", cost_usd=0.01, latency_ms=src)) + calls.append(_call(example_id=f"e{i}", role="target", cost_usd=0.01, latency_ms=tgt)) + return calls + + def test_a_few_slow_target_responses_do_not_flip_the_latency_verdict(self) -> None: + # 18 examples 10% slower, three provider spikes at 5x. The old mean + # ratio read +65.7% and failed a 30% budget on three calls. + target = [1100] * 18 + [5000] * 3 + latency = self._budgets(self._latency_pairs([1000] * 21, target))["max_latency_increase"] + assert latency.observed == pytest.approx(0.10) + assert latency.passed is True + assert latency.conclusive is True + + def test_a_uniform_slowdown_still_fails(self) -> None: + latency = self._budgets(self._latency_pairs([1000] * 21, [1400] * 21))[ + "max_latency_increase" + ] + assert latency.observed == pytest.approx(0.40) + assert latency.passed is False + assert latency.conclusive is True + + def test_a_slowdown_on_more_than_a_fifth_of_examples_survives_the_trim(self) -> None: + # 8 of 21 examples 3x slower. Trimming drops 4 per side, so 4 of the + # slow examples stay in: exp((9*ln 1 + 4*ln 3) / 13) - 1 = +40.2%. + # The median of the same ratios is +0% — the reason it was not chosen. + target = [1000] * 13 + [3000] * 8 + latency = self._budgets(self._latency_pairs([1000] * 21, target))["max_latency_increase"] + assert latency.observed == pytest.approx(3 ** (4 / 13) - 1) + assert latency.passed is False + + def test_a_faster_target_reports_a_signed_saving(self) -> None: + """The pass/fail is one-sided; the number is not clipped to hide savings.""" + calls = [ + _call(example_id="e0", role="source", cost_usd=0.02, latency_ms=1000), + _call(example_id="e0", role="target", cost_usd=0.01, latency_ms=500), + ] + budgets = self._budgets(calls) + assert budgets["max_cost_increase"].observed == pytest.approx(-0.5) + assert budgets["max_latency_increase"].observed == pytest.approx(-0.5) + for name in ("max_cost_increase", "max_latency_increase"): + assert budgets[name].passed is True, name + assert budgets[name].conclusive is True, name + + def test_an_example_one_side_failed_leaves_both_sides(self) -> None: + # The target errored on the slowest, priciest example. Averaging each + # role on its own compared e0+e1 against e0 alone: -82% latency and + # -82% cost from a difference in *which* examples were averaged. + calls = [ + _call(example_id="e0", role="source", cost_usd=0.01, latency_ms=100), + _call(example_id="e1", role="source", cost_usd=0.10, latency_ms=1000), + _call(example_id="e0", role="target", cost_usd=0.01, latency_ms=100), + _call(example_id="e1", role="target", cost_usd=0.0, latency_ms=0, error="timeout"), + ] + budgets = self._budgets(calls) + for name in ("max_cost_increase", "max_latency_increase"): + assert budgets[name].observed == pytest.approx(0.0), name + assert budgets[name].denominator == 1, name + + def test_cost_is_the_paired_total_so_concentrated_spend_is_not_trimmed(self) -> None: + """Cost has no provider noise to trim: an expensive answer is a real bill.""" + calls: list[Call] = [] + for i in range(10): + calls.append(_call(example_id=f"e{i}", role="source", cost_usd=1.0, latency_ms=100)) + target_cost = 10.0 if i < 2 else 1.0 + calls.append( + _call(example_id=f"e{i}", role="target", cost_usd=target_cost, latency_ms=100) + ) + cost = self._budgets(calls)["max_cost_increase"] + assert cost.observed == pytest.approx(1.8) + assert cost.passed is False + assert cost.denominator == 10 + + def test_repeated_samples_collapse_to_one_pair_per_example(self) -> None: + """``samples_per_example`` adds evidence per example, not examples. + + Latency takes each side's median sample, so one spiked repeat cannot + move an example's ratio; cost takes each side's mean, so the pair is + the example's expected spend. + """ + calls = [ + _call(example_id="e0", role="source", cost_usd=0.01, latency_ms=100, sample_index=0), + _call(example_id="e0", role="source", cost_usd=0.01, latency_ms=100, sample_index=1), + _call(example_id="e0", role="source", cost_usd=0.01, latency_ms=100, sample_index=2), + _call(example_id="e0", role="target", cost_usd=0.01, latency_ms=110, sample_index=0), + _call(example_id="e0", role="target", cost_usd=0.02, latency_ms=110, sample_index=1), + _call(example_id="e0", role="target", cost_usd=0.03, latency_ms=9000, sample_index=2), + ] + budgets = self._budgets(calls) + assert budgets["max_latency_increase"].observed == pytest.approx(0.10) + assert budgets["max_cost_increase"].observed == pytest.approx(1.0) + for name in ("max_cost_increase", "max_latency_increase"): + assert budgets[name].denominator == 1, name + + def test_an_errored_repeat_drops_only_that_sample(self) -> None: + calls = [ + _call(example_id="e0", role="source", cost_usd=0.01, latency_ms=100, sample_index=0), + _call( + example_id="e0", + role="source", + cost_usd=0.0, + latency_ms=0, + sample_index=1, + error="boom", + ), + _call(example_id="e0", role="target", cost_usd=0.01, latency_ms=120, sample_index=0), + _call(example_id="e0", role="target", cost_usd=0.01, latency_ms=120, sample_index=1), + ] + latency = self._budgets(calls)["max_latency_increase"] + assert latency.observed == pytest.approx(0.20) + assert latency.denominator == 1 + + def test_the_same_example_id_under_two_prompts_is_two_pairs(self) -> None: + calls = [ + _call(example_id="e0", role="source", cost_usd=0.01, latency_ms=100, prompt_id="a"), + _call(example_id="e0", role="target", cost_usd=0.01, latency_ms=100, prompt_id="a"), + _call(example_id="e0", role="source", cost_usd=0.01, latency_ms=100, prompt_id="b"), + _call(example_id="e0", role="target", cost_usd=0.01, latency_ms=100, prompt_id="b"), + ] + budgets = self._budgets(calls) + for name in ("max_cost_increase", "max_latency_increase"): + assert budgets[name].denominator == 2, name + + def test_cached_rows_count_with_their_recorded_latency(self) -> None: + """A cache hit replays the latency recorded when it ran live. + + Excluding it would turn a re-run from cache into "measured nothing", + which passes — so re-running a failed suite would clear its gate. + """ + calls = [ + _call(example_id="e0", role="source", cost_usd=0.01, latency_ms=100, cached=True), + _call(example_id="e0", role="target", cost_usd=0.01, latency_ms=200, cached=True), + ] + latency = self._budgets(calls)["max_latency_increase"] + assert latency.observed == pytest.approx(1.0) + assert latency.conclusive is True + assert latency.denominator == 1 + + def test_an_untimed_example_is_left_out_of_the_latency_ratio(self) -> None: + """A ``latency_ms`` of 0 is a missing timing; it has no ratio to give.""" + calls = [ + _call(example_id="e0", role="source", cost_usd=0.01, latency_ms=0), + _call(example_id="e0", role="target", cost_usd=0.01, latency_ms=100), + _call(example_id="e1", role="source", cost_usd=0.01, latency_ms=100), + _call(example_id="e1", role="target", cost_usd=0.01, latency_ms=110), + ] + budgets = self._budgets(calls) + assert budgets["max_latency_increase"].observed == pytest.approx(0.10) + assert budgets["max_latency_increase"].denominator == 1 + assert budgets["max_cost_increase"].denominator == 2 + + def test_the_summary_carries_the_same_signed_values(self) -> None: + """``PolicyMetricSummary`` is what the hosted gate re-derives from.""" + calls = [ + _call(example_id="e0", role="source", cost_usd=0.02, latency_ms=1000), + _call(example_id="e0", role="target", cost_usd=0.01, latency_ms=1200), + ] + summary = _metrics(records=[], calls=calls) + assert summary.cost_increase_rate == pytest.approx(-0.5) + assert summary.latency_increase_rate == pytest.approx(0.2) + + class TestZeroValuedCallBudgets: """Paired calls are necessary for a cost/latency ratio, not sufficient. - ``_relative_increase`` also returns its ``0.0`` default when *both* roles - average zero — an unpriced model pair, or calls whose ``latency_ms`` is 0. + ``_relative_change`` also returns its ``0.0`` default when *both* roles + recorded zero — an unpriced model pair, or calls whose ``latency_ms`` is 0. The call lists are non-empty there, so the older pairing check said "measured" and the row rendered "observed 0.00, passed, conclusive" for a cost nobody ever priced. @@ -1287,7 +1487,12 @@ def test_each_field_is_judged_on_its_own_values(self) -> None: assert by_name["max_latency_increase"].conclusive is True def test_a_free_source_against_a_priced_target_stays_conclusive(self) -> None: - """Only one side has to have measured something for the ratio to mean one.""" + """Only one side has to have priced something for the cost ratio to mean one. + + Latency is stricter: every real call takes time, so a ``latency_ms`` of + 0 is a missing timing, and an example timed on one side only has no + ratio to contribute. + """ decision = self._decision( calls=[ _call(example_id="e1", role="source", cost_usd=0.0, latency_ms=0), @@ -1303,10 +1508,10 @@ def test_a_free_source_against_a_priced_target_stays_conclusive(self) -> None: by_name = {b.name: b for b in decision.budget_results} assert by_name["max_cost_increase"].observed == pytest.approx(1.0) assert by_name["max_cost_increase"].conclusive is True - assert by_name["max_latency_increase"].conclusive is True + assert by_name["max_latency_increase"].conclusive is False def test_a_priced_source_against_a_free_target_stays_conclusive(self) -> None: - """A target that costs nothing is a real, clean measurement.""" + """A target that costs nothing is a real, clean measurement — a 100% saving.""" decision = self._decision( calls=[ _call(example_id="e1", role="source", cost_usd=0.02, latency_ms=100), @@ -1314,9 +1519,10 @@ def test_a_priced_source_against_a_free_target_stays_conclusive(self) -> None: ], ) by_name = {b.name: b for b in decision.budget_results} - assert by_name["max_cost_increase"].observed == 0.0 + assert by_name["max_cost_increase"].observed == pytest.approx(-1.0) + assert by_name["max_cost_increase"].passed is True assert by_name["max_cost_increase"].conclusive is True - assert by_name["max_latency_increase"].conclusive is True + assert by_name["max_latency_increase"].conclusive is False def test_slice_budgets_are_unmeasured_too(self) -> None: """Slices read the same run-level calls, so they inherit the verdict.""" @@ -1376,9 +1582,9 @@ def test_an_unpriced_cost_budget_says_why(self) -> None: ], ) assert ( - "The cost increase budget could not be measured: all 4 error-free " - "calls across both models recorded a cost of 0, so its observed " - "0.00 is a default, not a measurement." + "The cost increase budget could not be measured: both models " + "recorded a cost of 0 on all 2 example pairs, so its observed 0.00 " + "is a default, not a measurement." ) in decision.recommendations assert not any("latency" in r for r in decision.recommendations) @@ -1390,16 +1596,16 @@ def test_an_untimed_latency_budget_says_why(self) -> None: ], ) assert ( - "The latency increase budget could not be measured: all 2 error-free " - "calls across both models recorded a latency of 0, so its observed " - "0.00 is a default, not a measurement." + "The latency increase budget could not be measured: none of the 1 " + "example pair recorded a latency above 0 on both models, so its " + "observed 0.00 is a default, not a measurement." ) in decision.recommendations assert not any("cost" in r for r in decision.recommendations) def test_missing_calls_are_not_double_reported(self) -> None: """No calls at all is already visible; a second note is noise.""" decision = self._decision(calls=[]) - assert not any("error-free calls" in r for r in decision.recommendations) + assert not any("could not be measured" in r for r in decision.recommendations) def test_a_measured_ratio_is_not_warned(self) -> None: decision = self._decision( @@ -1408,7 +1614,7 @@ def test_a_measured_ratio_is_not_warned(self) -> None: _call(example_id="e1", role="target", cost_usd=0.01, latency_ms=100), ], ) - assert not any("error-free calls" in r for r in decision.recommendations) + assert not any("could not be measured" in r for r in decision.recommendations) def test_the_note_is_emitted_once_per_run_not_once_per_scope(self) -> None: """Every scope reads the same run-level calls — repeating it is noise.""" @@ -1762,7 +1968,7 @@ def test_a_budget_that_measured_nothing_reports_zero_not_null(self) -> None: assert drift.denominator == 0 assert drift.conclusive is False - def test_call_ratio_budgets_count_the_error_free_calls(self) -> None: + def test_call_ratio_budgets_count_example_pairs(self) -> None: decision = evaluate_migration_policy( run_id="r1", source_model="src", @@ -1778,15 +1984,15 @@ def test_call_ratio_budgets_count_the_error_free_calls(self) -> None: ], ) by_name = {b.name: b for b in decision.budget_results} - # Three error-free calls: two source, one target. The errored target - # call is excluded from the averages, so it is excluded from the count. - assert by_name["max_cost_increase"].denominator == 3 - assert by_name["max_latency_increase"].denominator == 3 + # One pair: e1's target call errored, so e1 leaves the comparison on + # both sides. Pairs, not calls — the old count here was 3. + assert by_name["max_cost_increase"].denominator == 1 + assert by_name["max_latency_increase"].denominator == 1 def test_one_sided_calls_leave_the_ratio_denominator_at_zero(self) -> None: - """The ratio divides one role's average by the other's. + """The ratio compares each example's target against its own source. - With no target call there is nothing to divide, so nothing was counted + With no target call there is no pair, so nothing was counted — ``0``, matching the ``conclusive: false`` the same absence produces. """ decision = evaluate_migration_policy( @@ -1807,8 +2013,8 @@ def test_an_all_zero_ratio_still_reports_the_calls_it_counted(self) -> None: """Both-sides-zero is unmeasured (phase P5) but not uncounted. ``conclusive`` carries the doubt; the denominator stays honest about - how many calls were averaged, so the two fields say different things - rather than one of them lying. + how many example pairs were compared, so the two fields say different + things rather than one of them lying. """ decision = evaluate_migration_policy( run_id="r1", @@ -1823,7 +2029,7 @@ def test_an_all_zero_ratio_still_reports_the_calls_it_counted(self) -> None: ], ) cost = next(b for b in decision.budget_results if b.name == "max_cost_increase") - assert cost.denominator == 2 + assert cost.denominator == 1 assert cost.conclusive is False def test_slice_budgets_report_the_slice_denominator(self) -> None: @@ -2279,8 +2485,9 @@ def test_the_call_ratio_denominators_are_unmoved_by_the_second_axis(self) -> Non ], ) budgets = _budgets(decision) - assert budgets["max_cost_increase"].denominator == 2 - assert budgets["max_latency_increase"].denominator == 2 + # One example pair, however many record rows score it. + assert budgets["max_cost_increase"].denominator == 1 + assert budgets["max_latency_increase"].denominator == 1 # --------------------------------------------------------------------------- diff --git a/tests/unit/test_reports.py b/tests/unit/test_reports.py index d3fa2e32..327f9276 100644 --- a/tests/unit/test_reports.py +++ b/tests/unit/test_reports.py @@ -808,6 +808,67 @@ def test_renders_migration_verdict_when_present(self, tmp_path: Path) -> None: assert "≤ 0" in html # count budget stays an integer, no percent assert "0.030" not in html # no bare fractions anymore + def test_the_economics_card_shows_what_the_budgets_were_judged_on(self, tmp_path: Path) -> None: + """The run-wide latency mean is not the gated figure, so the card says both. + + Otherwise "Latency +35.6%. Both inside +30% latency" reads as a + contradiction — the gate held on the typical example's +21.6%. + """ + cwd, run_id = _scaffold_full_run(tmp_path) + run_dir = cwd / ".evalshift" / "runs" / run_id + _write_migration_decision(run_dir, run_id) + path = run_dir / "migration_decision.json" + decision = json.loads(path.read_text(encoding="utf-8")) + decision["budget_results"] = [ + { + "name": "max_cost_increase", + "observed": -0.289, + "allowed": 0.5, + "passed": True, + "scope": "overall", + "conclusive": True, + "denominator": 21, + }, + { + "name": "max_latency_increase", + "observed": 0.216, + "allowed": 0.3, + "passed": True, + "scope": "overall", + "conclusive": True, + "denominator": 21, + }, + ] + path.write_text(json.dumps(decision), encoding="utf-8") + + html = render_html(build_report_payload(run_dir)) + + assert "judged per example: cost -28.9%, latency +21.6% for the typical example" in html + + def test_a_failed_increase_budget_renders_signed(self, tmp_path: Path) -> None: + cwd, run_id = _scaffold_full_run(tmp_path) + run_dir = cwd / ".evalshift" / "runs" / run_id + _write_migration_decision(run_dir, run_id) + path = run_dir / "migration_decision.json" + decision = json.loads(path.read_text(encoding="utf-8")) + decision["budget_results"] = [ + { + "name": "max_latency_increase", + "observed": 0.402, + "allowed": 0.3, + "passed": False, + "scope": "overall", + "conclusive": True, + "denominator": 21, + }, + ] + path.write_text(json.dumps(decision), encoding="utf-8") + + html = render_html(build_report_payload(run_dir)) + + assert '+40.2%' in html + assert "≤ +30.0%" in html + def test_a_failed_trace_rule_budget_renders_as_a_count(self, tmp_path: Path) -> None: """``max_invariant_violations`` is a count of examples, never a percentage.""" cwd, run_id = _scaffold_full_run(tmp_path)