From 01b9fc90a6474edd6b9a0b9bda6b423e2fa54299 Mon Sep 17 00:00:00 2001 From: Arye Kogan Date: Sat, 4 Jul 2026 04:08:18 +0300 Subject: [PATCH] docs: align model-assisted examples with fail-closed configs --- docs/examples.md | 2 +- docs/guides/model-assisted-evals.md | 18 +++++++++++++----- docs/reference/cli.md | 10 +++++++--- docs/reference/results.md | 2 +- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index 0def3be..ca23343 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -133,7 +133,7 @@ Treat these as manual/advisory runs, not default CI gates. ```bash pnpm exec eval-kit judge-coverage \ - --config evals/eval-kit.config.json \ + --config evals/eval-kit.model-judge.config.json \ --case case-alpha \ --candidate evals/cases/case-alpha/candidate.md \ --model gpt-5.4 \ diff --git a/docs/guides/model-assisted-evals.md b/docs/guides/model-assisted-evals.md index 900d5c6..e49b09f 100644 --- a/docs/guides/model-assisted-evals.md +++ b/docs/guides/model-assisted-evals.md @@ -55,26 +55,34 @@ the consumer has calibrated pointwise evidence and a specific comparison need. ## Commands +Pointwise judging uses the standard manual model-judge config: + ```bash -pnpm exec eval-kit generate \ +pnpm exec eval-kit judge-coverage \ + --config evals/eval-kit.model-judge.config.json \ --case \ + --candidate \ --model \ --provider openai \ --effort medium \ --run-id +``` +Generation and pairwise are separate manual lanes. They require separate configs whose relevant +method is explicitly enabled; do not run them through `evals/eval-kit.config.json` or +`evals/eval-kit.model-judge.config.json`. -pnpm exec eval-kit judge-coverage \ - --config evals/eval-kit.model-judge.config.json \ +```bash +pnpm exec eval-kit generate \ + --config evals/eval-kit.generate.config.json \ --case \ - --candidate \ --model \ --provider openai \ --effort medium \ --run-id - pnpm exec eval-kit judge-pairwise \ + --config evals/eval-kit.pairwise.config.json \ --case \ --candidate-a \ --candidate-b \ diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 3ac936a..e204395 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -115,11 +115,13 @@ pnpm exec eval-kit generate \ --provider \ --effort \ --run-id \ - [--config ] + --config evals/eval-kit.generate.config.json ``` Requires Promptfoo and local Codex auth. This is manual/advisory evidence, not a default CI gate. Fails closed before those checks unless `methods.generate.enabled` is explicitly `true`. +Use a separate explicit config for generation; do not enable generation in the deterministic default +config. ## `judge-coverage` @@ -133,7 +135,7 @@ pnpm exec eval-kit judge-coverage \ --provider \ --effort \ [--run-id ] \ - [--config ] + --config evals/eval-kit.model-judge.config.json ``` Fails closed unless `methods.judge_coverage.enabled` is explicitly `true`. @@ -157,10 +159,12 @@ pnpm exec eval-kit judge-pairwise \ --effort \ --seed \ --run-id \ - [--config ] + --config evals/eval-kit.pairwise.config.json ``` Fails closed unless `methods.judge_pairwise.enabled` is explicitly `true`. +Use a separate explicit config for pairwise after pointwise calibration; do not enable pairwise in +the deterministic default config. ## `report` diff --git a/docs/reference/results.md b/docs/reference/results.md index d94d99d..df1a78a 100644 --- a/docs/reference/results.md +++ b/docs/reference/results.md @@ -39,7 +39,7 @@ Current schema: "run_type": "deterministic", "runner": { "id": "generic-eval-case", - "version": "0.1.2" + "version": "0.1.4" }, "case_ids": ["case-example-v1"], "started_at": "2026-07-03T00:00:00.000Z",