From ca33058465ae2f3c878f959d5e2f81026e4d3721 Mon Sep 17 00:00:00 2001 From: Arye Kogan Date: Sat, 4 Jul 2026 02:45:42 +0300 Subject: [PATCH] chore(evals): standardize model judge config --- evals/README.md | 49 +++++++++++++++++--------- evals/eval-kit.model-judge.config.json | 31 ++++++++++++++++ package.json | 7 +++- pnpm-lock.yaml | 12 +++---- 4 files changed, 75 insertions(+), 24 deletions(-) create mode 100644 evals/eval-kit.model-judge.config.json diff --git a/evals/README.md b/evals/README.md index b26ab25..bf235e9 100644 --- a/evals/README.md +++ b/evals/README.md @@ -16,7 +16,9 @@ The implementation plan lives in ``` evals/ - eval-kit.config.json # Eval suite configuration (entry point) + eval-kit.config.json # Deterministic/default eval suite configuration + eval-kit.model-judge.config.json + # Manual pointwise judge configuration adapter.mjs # Domain adapter: grader, reporter, validator, var resolvers rubric.md # Judge rubric for pointwise and pairwise semantic evaluation cases/ # Seven self-contained product-to-design cases @@ -94,50 +96,63 @@ case or eval slice that exercises the changed behavior, then widen to `pnpm chec ## Reading the evals -1. `eval-kit.config.json` — suite config; start here to understand what runs. -2. `adapter.mjs` — all domain-specific logic in one file. -3. `rubric.md` — judge rubric for semantic evaluation. -4. `cases/README.md` — authoring contract for product-to-design cases. -5. `results/README.md` — conventions for generated run outputs. +1. `eval-kit.config.json` — deterministic suite config; start here to understand default gates. +2. `eval-kit.model-judge.config.json` — manual pointwise judge config for local calibration. +3. `adapter.mjs` — all domain-specific logic in one file. +4. `rubric.md` — judge rubric for semantic evaluation. +5. `cases/README.md` — authoring contract for product-to-design cases. +6. `results/README.md` — conventions for generated run outputs. ## Manual Model-Assisted Product-to-Design Flow Manual model-graded runs use Promptfoo with the Codex App Server provider. They are advisory only and must stay outside `pnpm check`, CI, and required contributor gates. +This repo follows the eval-kit two-config standard: + +- `evals/eval-kit.config.json` stays deterministic, default, and CI-safe. Do not enable model + lanes there for local judging. +- `evals/eval-kit.model-judge.config.json` enables pointwise judge coverage for manual calibration. + It disables generation, pairwise judging, and reports. +- Pairwise judging remains deferred. Add a separate `evals/eval-kit.pairwise.config.json` only if a + future calibrated comparison workflow needs it. + Check local auth: ```bash codex login status ``` -Before running generation or pointwise coverage judging, set `methods.generate.enabled` and -`methods.judge_coverage.enabled` to `true` in `evals/eval-kit.config.json`. Pairwise judging also -requires `methods.judge_pairwise.enabled` to be set to `true`. Keep these methods disabled for the -default deterministic contributor gate. - -Generate a candidate design: +Before running pointwise coverage judging, validate the manual config: ```bash -pnpm exec eval-kit generate --config evals/eval-kit.config.json --case --model gpt-5.4 --provider openai --effort medium --run-id -generate +pnpm eval:judge:doctor +pnpm eval:judge:list +pnpm eval:judge:validate-fixtures ``` +Use any account-supported model approved for the local calibration run. Examples below use +`gpt-5.5`; that is an example, not repo policy. The standard manual config judges an existing +candidate. If a future workflow needs provider-backed candidate generation, add a separate explicit +generation config instead of enabling generation in the default deterministic config. + Grade deterministically: ```bash -pnpm eval:case -- --case --candidate evals/results/-generate/cases//candidate.md --run-id -deterministic +pnpm eval:case -- --case --candidate --run-id -deterministic ``` Run pointwise coverage judging: ```bash -pnpm exec eval-kit judge-coverage --config evals/eval-kit.config.json --case --candidate evals/results/-generate/cases//candidate.md --model gpt-5.4 --provider openai --effort medium --run-id -pointwise +pnpm eval:judge:coverage -- --case --candidate --model gpt-5.5 --provider openai --effort medium --run-id -pointwise ``` -Run pairwise judging only after deterministic blockers are understood: +Run pairwise judging only after deterministic blockers are understood, pointwise calibration exists, +and a separate pairwise config has been added: ```bash -pnpm exec eval-kit judge-pairwise --config evals/eval-kit.config.json --case --candidate-a --candidate-b --model gpt-5.4 --provider openai --effort medium --seed --run-id -pairwise +pnpm exec eval-kit judge-pairwise --config evals/eval-kit.pairwise.config.json --case --candidate-a --candidate-b --model gpt-5.5 --provider openai --effort medium --seed --run-id -pairwise ``` Write a combined manual report: diff --git a/evals/eval-kit.model-judge.config.json b/evals/eval-kit.model-judge.config.json new file mode 100644 index 0000000..3e79edf --- /dev/null +++ b/evals/eval-kit.model-judge.config.json @@ -0,0 +1,31 @@ +{ + "$schema": "../node_modules/@agentic-workflow-kit/eval-kit/schemas/eval-kit.config.schema.json", + "schema_version": "eval-kit.config.v1", + "suite_id": "technical-design", + "suite_root": ".", + "results_root": "results", + "adapter": "adapter.mjs", + "cases": { + "root": "cases", + "include": ["case-*-v1"] + }, + "methods": { + "deterministic": { + "enabled": true, + "grader": "facts-boundaries", + "reporter": "markdown" + }, + "generate": { + "enabled": false + }, + "judge_coverage": { + "enabled": true + }, + "judge_pairwise": { + "enabled": false + }, + "report": { + "enabled": false + } + } +} diff --git a/package.json b/package.json index 863616e..27f02ef 100644 --- a/package.json +++ b/package.json @@ -38,12 +38,17 @@ "check:eval-static": "node scripts/check_eval_static.mjs", "eval:doctor": "eval-kit doctor --config evals/eval-kit.config.json", "eval:list": "eval-kit list-cases --config evals/eval-kit.config.json", + "eval:validate-fixtures": "eval-kit validate-fixtures --config evals/eval-kit.config.json", + "eval:judge:doctor": "eval-kit doctor --config evals/eval-kit.model-judge.config.json", + "eval:judge:list": "eval-kit list-cases --config evals/eval-kit.model-judge.config.json", + "eval:judge:validate-fixtures": "eval-kit validate-fixtures --config evals/eval-kit.model-judge.config.json", + "eval:judge:coverage": "eval-kit judge-coverage --config evals/eval-kit.model-judge.config.json", "eval:unit": "vitest run evals/tests", "eval:case": "eval-kit run-case --config evals/eval-kit.config.json", "eval:enforce": "node scripts/run_enforce_eval.mjs" }, "devDependencies": { - "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.2", + "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3", "ajv": "^8.17.1", "dependency-cruiser": "^18.0.0", "prettier": "^3.6.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0d91af4..6b45c90 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: devDependencies: '@agentic-workflow-kit/eval-kit': - specifier: github:agentic-workflow-kit/eval-kit#v0.1.2 - version: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974 + specifier: github:agentic-workflow-kit/eval-kit#v0.1.3 + version: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/1302ec0261674c2a3d14ba7ff49ec67e4b5fee27 ajv: specifier: ^8.17.1 version: 8.20.0 @@ -32,9 +32,9 @@ importers: packages: - '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974': - resolution: {gitHosted: true, integrity: sha512-XNdPutRH/06q2cI2jxaZCBgh583JVoAc8W9uipxnpgNzcovcSY8TK5feFnk4L32RG5oGdl8wFGzpnXlFsAaRnA==, tarball: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974} - version: 0.1.2 + '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/1302ec0261674c2a3d14ba7ff49ec67e4b5fee27': + resolution: {gitHosted: true, integrity: sha512-iEAejAn4igspvybA7fV9NpgFDMh4KTm2zKtpsec5NEyXN1CPsHoEuRW/MHggKcNYPhQlBkFvNEF/MRwNolsx+w==, tarball: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/1302ec0261674c2a3d14ba7ff49ec67e4b5fee27} + version: 0.1.3 engines: {node: '>=22.13.0', pnpm: '>=11.9.0'} hasBin: true @@ -4202,7 +4202,7 @@ packages: snapshots: - '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974': + '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/1302ec0261674c2a3d14ba7ff49ec67e4b5fee27': dependencies: ajv: 8.20.0