From 1a3c4bdc10111624de564350c69c0d5ca02c7a92 Mon Sep 17 00:00:00 2001 From: Arye Kogan Date: Sat, 4 Jul 2026 03:47:45 +0300 Subject: [PATCH] chore(evals): bump eval-kit v0.1.4 --- docs/design/evaluation-strategy.md | 9 +++-- docs/design/promptfoo-codex-eval-pilot.md | 45 ++++++++++------------- evals/implementation-plan.md | 14 +++---- package.json | 2 +- pnpm-lock.yaml | 12 +++--- 5 files changed, 39 insertions(+), 43 deletions(-) diff --git a/docs/design/evaluation-strategy.md b/docs/design/evaluation-strategy.md index 4906990..9a2e2a9 100644 --- a/docs/design/evaluation-strategy.md +++ b/docs/design/evaluation-strategy.md @@ -63,8 +63,9 @@ individual deterministic eval command that exercises the changed behavior, then Manual/advisory evals cover model-assisted candidate generation, pointwise LLM judge coverage, pairwise judging, long product-to-design or product-to-plan session evals, expensive full-case replay suites, and anything requiring auth, network, model calls, or manual calibration. These methods must -remain disabled by default in `evals/eval-kit.config.json` and require an intentional local edit -before use. +remain outside `pnpm check` and CI. The default `evals/eval-kit.config.json` stays deterministic and +CI-safe; manual pointwise judge runs use `evals/eval-kit.model-judge.config.json` through +`eval:judge:*` scripts. Generation and pairwise remain deferred to separate explicit configs. ## Evaluation Pyramid @@ -335,8 +336,8 @@ implementation-ready, and enforceable. The judge must explain the winning criter a winner. If deterministic grading and pointwise judging disagree, record the disagreement as calibration -evidence. Do not let an LLM judge override deterministic blockers until a human-reviewed calibration -set shows acceptable false-pass and false-fail behavior. +evidence. Calibrated model-judge results may inform manual review or future deterministic/rubric +changes, but they cannot upgrade or override deterministic red/yellow blockers. ### Human Calibration diff --git a/docs/design/promptfoo-codex-eval-pilot.md b/docs/design/promptfoo-codex-eval-pilot.md index a40a216..a2e4f09 100644 --- a/docs/design/promptfoo-codex-eval-pilot.md +++ b/docs/design/promptfoo-codex-eval-pilot.md @@ -48,8 +48,8 @@ judge rubric, result bundle convention, and final reporting story. false-fail behavior. - The reference design is a comparison anchor, not a certified great solution or exact-output target. - Model-graded commands remain manual and outside `pnpm check`. -- Model-assisted methods remain disabled by default and require an intentional local config edit - before each manual run. +- Pointwise model judging uses `evals/eval-kit.model-judge.config.json` and the `eval:judge:*` + scripts; do not edit the deterministic default config for pointwise runs. ## Tooling Decision @@ -129,8 +129,9 @@ evals/ # technical-design cases, adapter, rubrics, tests, resul `@agentic-workflow-kit/eval-kit` is shared infrastructure consumed from the GitHub tag pinned in `package.json` and `pnpm-lock.yaml`. It must stay private and must not be treated as part of the -product surface shipped to users. Root `pnpm eval:*` scripts are the consumer command surface and -call the installed `eval-kit` binary with `evals/eval-kit.config.json`. +product surface shipped to users. Root `pnpm eval:*` scripts are the consumer command surface: +deterministic scripts use `evals/eval-kit.config.json`, and pointwise judge scripts use +`evals/eval-kit.model-judge.config.json`. Eval-specific material is split by ownership: @@ -153,38 +154,32 @@ and eval-specific implementation docs belong in `evals/`. The root package stays small. It keeps `check` as the public deterministic repo gate and exposes only offline/static/local deterministic scripts such as `pnpm eval:case`, `pnpm eval:unit`, and -`pnpm eval:enforce`. Model-assisted commands remain direct `eval-kit` CLI usage because they require -local auth, explicit calibration, and human interpretation: +`pnpm eval:enforce`. Pointwise model judging remains manual/advisory and uses the separate +model-judge config: ```text pnpm eval:doctor pnpm eval:list pnpm eval:case -- -pnpm exec eval-kit generate --config evals/eval-kit.config.json -pnpm exec eval-kit judge-coverage --config evals/eval-kit.config.json -pnpm exec eval-kit judge-pairwise --config evals/eval-kit.config.json -pnpm exec eval-kit report --config evals/eval-kit.config.json +pnpm eval:judge:doctor +pnpm eval:judge:list +pnpm eval:judge:validate-fixtures +pnpm eval:judge:coverage -- ``` +Candidate generation, pairwise judging, and combined reports require separate explicit configs before +they are revived. Do not route provider-backed commands through `evals/eval-kit.config.json`. + The root `pnpm check` command remains the public repository gate. It should run formatting, root-visible static checks, adapter import and syntax validation, fixture validation, local unit tests, and seeded deterministic fixture checks only. ## Planned Suite Shape -Keep Promptfoo installed for optional direct CLI runs, but do not add package scripts for -model-assisted commands until there is a calibrated workflow. Because model-assisted methods are -disabled in `evals/eval-kit.config.json` by default, a manual run must intentionally enable the -specific method before invoking it: - -- `pnpm exec eval-kit generate` - run the candidate-generation Promptfoo suite after enabling - `methods.generate`. -- `pnpm exec eval-kit judge-coverage` - run the pointwise coverage judge Promptfoo suite after - enabling `methods.judge_coverage`. -- `pnpm exec eval-kit judge-pairwise` - run the pairwise judge Promptfoo suite after enabling - `methods.judge_pairwise`. -- `pnpm exec eval-kit report` - combine generation, deterministic, judge, and outcome results into - one final report. +Keep Promptfoo installed for optional manual model-backed runs. The standardized pointwise lane uses +`pnpm eval:judge:coverage` with `evals/eval-kit.model-judge.config.json`, where +`judge_coverage.enabled=true` and generation, pairwise, and report remain disabled. Do not +temporarily flip model methods in `evals/eval-kit.config.json`. Keep `pnpm check` deterministic-only. @@ -254,7 +249,7 @@ The final report command combines all run bundles and states: - whether judge JSON was schema-valid; - which candidate won the pairwise comparison, if any; - what token/runtime metadata Promptfoo and Codex reported; -- why the result is not yet a release signal without human calibration. +- why the result is advisory evidence for manual review, not an automatic release gate. ## Acceptance Criteria @@ -263,7 +258,7 @@ The final report command combines all run bundles and states: calls, network/auth/model calls, long replay suites, or manual calibration. - The pilot does not require or document `OPENAI_API_KEY`. - `codex login status` is checked before model-graded runs. -- Model-assisted methods remain disabled by default in `evals/eval-kit.config.json`. +- Model-assisted provider calls remain outside `pnpm check` and CI. - Promptfoo, not shell wrappers, owns provider execution, assertions, and JSON/HTML result exports. - Candidate generation produces a non-empty Markdown design for `case-tiny-laundry-pickup-v1`. - The existing deterministic grader runs against the generated candidate. diff --git a/evals/implementation-plan.md b/evals/implementation-plan.md index b02b57d..7ac1c0a 100644 --- a/evals/implementation-plan.md +++ b/evals/implementation-plan.md @@ -249,8 +249,8 @@ Ongoing enhancements: - Regenerate the aerial delivery candidate after `SRC-013` is visible in `product.md` and `source-map.md`; older generated artifacts cannot prove that newly visible non-goal. -- Build a small human-labeled pointwise calibration packet before treating model coverage judgments - as release signals. +- Build a small human-labeled pointwise calibration packet before using model coverage judgments as + advisory manual-review evidence. - Keep pairwise comparison secondary to deterministic and pointwise coverage results; use pairwise only for relative quality once blockers are understood. @@ -284,7 +284,7 @@ Implemented: Remaining: -- Calibrate on a small human-reviewed set before using judge verdicts as a release signal. +- Calibrate on a small human-reviewed set before using judge verdicts as advisory review evidence. Agents and review: @@ -296,10 +296,10 @@ Agents and review: Integration point: -- Judge runs should not block every PR until calibration demonstrates acceptable false-pass and - false-fail behavior. -- Pairwise regression should run before methodology template, rubric, or orchestrator behavior - changes. +- Judge runs remain manual/advisory. They may inform manual review and future deterministic/rubric + changes after calibration, but they cannot override deterministic blockers or become automatic PR + blockers. +- Pairwise regression remains deferred until pointwise calibration shows a specific comparison need. Verification: diff --git a/package.json b/package.json index 27f02ef..8b095c9 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "eval:enforce": "node scripts/run_enforce_eval.mjs" }, "devDependencies": { - "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3", + "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.4", "ajv": "^8.17.1", "dependency-cruiser": "^18.0.0", "prettier": "^3.6.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6b45c90..291e55d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: devDependencies: '@agentic-workflow-kit/eval-kit': - specifier: github:agentic-workflow-kit/eval-kit#v0.1.3 - version: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/1302ec0261674c2a3d14ba7ff49ec67e4b5fee27 + specifier: github:agentic-workflow-kit/eval-kit#v0.1.4 + version: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/39d9991ee41009f70a26155ee07886b105160abd ajv: specifier: ^8.17.1 version: 8.20.0 @@ -32,9 +32,9 @@ importers: packages: - '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/1302ec0261674c2a3d14ba7ff49ec67e4b5fee27': - resolution: {gitHosted: true, integrity: sha512-iEAejAn4igspvybA7fV9NpgFDMh4KTm2zKtpsec5NEyXN1CPsHoEuRW/MHggKcNYPhQlBkFvNEF/MRwNolsx+w==, tarball: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/1302ec0261674c2a3d14ba7ff49ec67e4b5fee27} - version: 0.1.3 + '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/39d9991ee41009f70a26155ee07886b105160abd': + resolution: {gitHosted: true, integrity: sha512-CIzIRKxxeMonTGM0pzPF5i3fc2UgLyB9x2wwMMyzWAo22hz+HvdQWTdfX1Y0GEikkktfCK+r2dFLfRGForGVAQ==, tarball: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/39d9991ee41009f70a26155ee07886b105160abd} + version: 0.1.4 engines: {node: '>=22.13.0', pnpm: '>=11.9.0'} hasBin: true @@ -4202,7 +4202,7 @@ packages: snapshots: - '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/1302ec0261674c2a3d14ba7ff49ec67e4b5fee27': + '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/39d9991ee41009f70a26155ee07886b105160abd': dependencies: ajv: 8.20.0