chore(evals): standardize model judge config

evals/README.md

@@ -16,7 +16,9 @@ The implementation plan lives in
 
 ```
 evals/
-  eval-kit.config.json      # Eval suite configuration (entry point)
+  eval-kit.config.json      # Deterministic/default eval suite configuration
+  eval-kit.model-judge.config.json
+                             # Manual pointwise judge configuration
   adapter.mjs               # Domain adapter: grader, reporter, validator, var resolvers
   rubric.md                 # Judge rubric for pointwise and pairwise semantic evaluation
   cases/                    # Seven self-contained product-to-design cases
@@ -94,50 +96,63 @@ case or eval slice that exercises the changed behavior, then widen to `pnpm chec
 
 ## Reading the evals
 
-1. `eval-kit.config.json` — suite config; start here to understand what runs.
-2. `adapter.mjs` — all domain-specific logic in one file.
-3. `rubric.md` — judge rubric for semantic evaluation.
-4. `cases/README.md` — authoring contract for product-to-design cases.
-5. `results/README.md` — conventions for generated run outputs.
+1. `eval-kit.config.json` — deterministic suite config; start here to understand default gates.
+2. `eval-kit.model-judge.config.json` — manual pointwise judge config for local calibration.
+3. `adapter.mjs` — all domain-specific logic in one file.
+4. `rubric.md` — judge rubric for semantic evaluation.
+5. `cases/README.md` — authoring contract for product-to-design cases.
+6. `results/README.md` — conventions for generated run outputs.
 
 ## Manual Model-Assisted Product-to-Design Flow
 
 Manual model-graded runs use Promptfoo with the Codex App Server provider. They are advisory only
 and must stay outside `pnpm check`, CI, and required contributor gates.
 
+This repo follows the eval-kit two-config standard:
+
+- `evals/eval-kit.config.json` stays deterministic, default, and CI-safe. Do not enable model
+  lanes there for local judging.
+- `evals/eval-kit.model-judge.config.json` enables pointwise judge coverage for manual calibration.
+  It disables generation, pairwise judging, and reports.
+- Pairwise judging remains deferred. Add a separate `evals/eval-kit.pairwise.config.json` only if a
+  future calibrated comparison workflow needs it.
+
 Check local auth:
 
 ```bash
 codex login status
 ```
 
-Before running generation or pointwise coverage judging, set `methods.generate.enabled` and
-`methods.judge_coverage.enabled` to `true` in `evals/eval-kit.config.json`. Pairwise judging also
-requires `methods.judge_pairwise.enabled` to be set to `true`. Keep these methods disabled for the
-default deterministic contributor gate.
-
-Generate a candidate design:
+Before running pointwise coverage judging, validate the manual config:
 
 ```bash
-pnpm exec eval-kit generate --config evals/eval-kit.config.json --case <case-id> --model gpt-5.4 --provider openai --effort medium --run-id <run-id>-generate
+pnpm eval:judge:doctor
+pnpm eval:judge:list
+pnpm eval:judge:validate-fixtures
 ```
 
+Use any account-supported model approved for the local calibration run. Examples below use
+`gpt-5.5`; that is an example, not repo policy. The standard manual config judges an existing
+candidate. If a future workflow needs provider-backed candidate generation, add a separate explicit
+generation config instead of enabling generation in the default deterministic config.
+
 Grade deterministically:
 
 ```bash
-pnpm eval:case -- --case <case-id> --candidate evals/results/<run-id>-generate/cases/<case-id>/candidate.md --run-id <run-id>-deterministic
+pnpm eval:case -- --case <case-id> --candidate <candidate.md> --run-id <run-id>-deterministic
 ```
 
 Run pointwise coverage judging:
 
 ```bash
-pnpm exec eval-kit judge-coverage --config evals/eval-kit.config.json --case <case-id> --candidate evals/results/<run-id>-generate/cases/<case-id>/candidate.md --model gpt-5.4 --provider openai --effort medium --run-id <run-id>-pointwise
+pnpm eval:judge:coverage -- --case <case-id> --candidate <candidate.md> --model gpt-5.5 --provider openai --effort medium --run-id <run-id>-pointwise
 ```
 
-Run pairwise judging only after deterministic blockers are understood:
+Run pairwise judging only after deterministic blockers are understood, pointwise calibration exists,
+and a separate pairwise config has been added:
 
 ```bash
-pnpm exec eval-kit judge-pairwise --config evals/eval-kit.config.json --case <case-id> --candidate-a <a.md> --candidate-b <b.md> --model gpt-5.4 --provider openai --effort medium --seed <seed> --run-id <run-id>-pairwise
+pnpm exec eval-kit judge-pairwise --config evals/eval-kit.pairwise.config.json --case <case-id> --candidate-a <a.md> --candidate-b <b.md> --model gpt-5.5 --provider openai --effort medium --seed <seed> --run-id <run-id>-pairwise
 ```
 
 Write a combined manual report:

evals/eval-kit.model-judge.config.json

@@ -0,0 +1,31 @@
+{
+  "$schema": "../node_modules/@agentic-workflow-kit/eval-kit/schemas/eval-kit.config.schema.json",
+  "schema_version": "eval-kit.config.v1",
+  "suite_id": "technical-design",
+  "suite_root": ".",
+  "results_root": "results",
+  "adapter": "adapter.mjs",
+  "cases": {
+    "root": "cases",
+    "include": ["case-*-v1"]
+  },
+  "methods": {
+    "deterministic": {
+      "enabled": true,
+      "grader": "facts-boundaries",
+      "reporter": "markdown"
+    },
+    "generate": {
+      "enabled": false
+    },
+    "judge_coverage": {
+      "enabled": true
+    },
+    "judge_pairwise": {
+      "enabled": false
+    },
+    "report": {
+      "enabled": false
+    }
+  }
+}

package.json

@@ -38,12 +38,17 @@
     "check:eval-static": "node scripts/check_eval_static.mjs",
     "eval:doctor": "eval-kit doctor --config evals/eval-kit.config.json",
     "eval:list": "eval-kit list-cases --config evals/eval-kit.config.json",
+    "eval:validate-fixtures": "eval-kit validate-fixtures --config evals/eval-kit.config.json",
+    "eval:judge:doctor": "eval-kit doctor --config evals/eval-kit.model-judge.config.json",
+    "eval:judge:list": "eval-kit list-cases --config evals/eval-kit.model-judge.config.json",
+    "eval:judge:validate-fixtures": "eval-kit validate-fixtures --config evals/eval-kit.model-judge.config.json",
+    "eval:judge:coverage": "eval-kit judge-coverage --config evals/eval-kit.model-judge.config.json",
     "eval:unit": "vitest run evals/tests",
     "eval:case": "eval-kit run-case --config evals/eval-kit.config.json",
     "eval:enforce": "node scripts/run_enforce_eval.mjs"
   },
   "devDependencies": {
-    "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.2",
+    "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3",
     "ajv": "^8.17.1",
     "dependency-cruiser": "^18.0.0",
     "prettier": "^3.6.2",