diff --git a/docs/design/promptfoo-codex-eval-pilot.md b/docs/design/promptfoo-codex-eval-pilot.md index aabc742..10ceecb 100644 --- a/docs/design/promptfoo-codex-eval-pilot.md +++ b/docs/design/promptfoo-codex-eval-pilot.md @@ -146,14 +146,17 @@ product is designed and governed. Executable eval assets, fixture data, generate and eval-specific implementation docs belong in `evals/`. The root package stays small. It keeps `check` as the public deterministic repo gate and exposes -manual eval commands that call the internal kit: +only the basic deterministic eval-kit commands. Model-assisted commands remain direct `eval-kit` +CLI usage because they require local auth, explicit calibration, and human interpretation: ```text +pnpm eval:doctor +pnpm eval:list pnpm eval:case -- -pnpm eval:generate -- -pnpm eval:judge -- -pnpm eval:judge:coverage -- -pnpm eval:manual-report -- +pnpm exec eval-kit generate --config evals/eval-kit.config.json +pnpm exec eval-kit judge-coverage --config evals/eval-kit.config.json +pnpm exec eval-kit judge-pairwise --config evals/eval-kit.config.json +pnpm exec eval-kit report --config evals/eval-kit.config.json ``` The root `pnpm check` command remains the public repository gate. It should run formatting and the @@ -161,14 +164,19 @@ eval package's deterministic check. ## Planned Suite Shape -Add `promptfoo` as a development dependency of the eval package and add manual scripts: - -- `eval:generate` - run the candidate-generation Promptfoo suite. -- `eval:judge:coverage` - run the pointwise coverage judge Promptfoo suite. -- `eval:judge` - run the pairwise judge Promptfoo suite. -- `eval:outcome` - validate and report a redacted outcome-study summary. -- `eval:manual-report` - combine generation, deterministic, judge, and outcome results into one - final report. +Keep Promptfoo installed for optional direct CLI runs, but do not add package scripts for +model-assisted commands until there is a calibrated workflow. Because model-assisted methods are +disabled in `evals/eval-kit.config.json` by default, a manual run must intentionally enable the +specific method before invoking it: + +- `pnpm exec eval-kit generate` - run the candidate-generation Promptfoo suite after enabling + `methods.generate`. +- `pnpm exec eval-kit judge-coverage` - run the pointwise coverage judge Promptfoo suite after + enabling `methods.judge_coverage`. +- `pnpm exec eval-kit judge-pairwise` - run the pairwise judge Promptfoo suite after enabling + `methods.judge_pairwise`. +- `pnpm exec eval-kit report` - combine generation, deterministic, judge, and outcome results into + one final report. Keep `pnpm check` deterministic-only. diff --git a/evals/README.md b/evals/README.md index 0729965..907f2f7 100644 --- a/evals/README.md +++ b/evals/README.md @@ -101,10 +101,15 @@ Check local auth: codex login status ``` +Before running generation or pointwise coverage judging, set `methods.generate.enabled` and +`methods.judge_coverage.enabled` to `true` in `evals/eval-kit.config.json`. Pairwise judging also +requires `methods.judge_pairwise.enabled` to be set to `true`. Keep these methods disabled for the +default deterministic contributor gate. + Generate a candidate design: ```bash -pnpm eval:generate -- --case --model gpt-5.4 --provider openai --effort medium --run-id -generate +pnpm exec eval-kit generate --config evals/eval-kit.config.json --case --model gpt-5.4 --provider openai --effort medium --run-id -generate ``` Grade deterministically: @@ -116,20 +121,19 @@ pnpm eval:case -- --case --candidate evals/results/-generate/c Run pointwise coverage judging: ```bash -pnpm eval:judge:coverage -- --case --candidate evals/results/-generate/cases//candidate.md --model gpt-5.4 --provider openai --effort medium --run-id -pointwise +pnpm exec eval-kit judge-coverage --config evals/eval-kit.config.json --case --candidate evals/results/-generate/cases//candidate.md --model gpt-5.4 --provider openai --effort medium --run-id -pointwise ``` -Run pairwise judging only after deterministic blockers are understood and -`methods.judge_pairwise.enabled` is set to `true` in `eval-kit.config.json`: +Run pairwise judging only after deterministic blockers are understood: ```bash -pnpm eval:judge -- --case --candidate-a --candidate-b --model gpt-5.4 --provider openai --effort medium --seed --run-id -pairwise +pnpm exec eval-kit judge-pairwise --config evals/eval-kit.config.json --case --candidate-a --candidate-b --model gpt-5.4 --provider openai --effort medium --seed --run-id -pairwise ``` Write a combined manual report: ```bash -pnpm eval:manual-report -- --run-id --generate -generate --deterministic -deterministic --judge-coverage -pointwise +pnpm exec eval-kit report --config evals/eval-kit.config.json --run-id --generate -generate --deterministic -deterministic --judge-coverage -pointwise ``` Generated outputs go under `evals/results//` and are ignored by default. A result bundle diff --git a/evals/eval-kit.config.json b/evals/eval-kit.config.json index 42199ab..5b5bfc8 100644 --- a/evals/eval-kit.config.json +++ b/evals/eval-kit.config.json @@ -16,13 +16,10 @@ "reporter": "markdown" }, "generate": { - "enabled": true, - "prompt": "@eval-kit/generation" + "enabled": false }, "judge_coverage": { - "enabled": true, - "prompt": "@eval-kit/pointwise", - "rubric": "case:rubric.md" + "enabled": false }, "judge_pairwise": { "enabled": false diff --git a/evals/results/README.md b/evals/results/README.md index 53f2a7c..8779460 100644 --- a/evals/results/README.md +++ b/evals/results/README.md @@ -1,6 +1,6 @@ # Eval Results -Generated eval outputs belong here under `packages/evals/results//`. +Generated eval outputs belong here under `evals/results//`. This directory is intentionally ignored except for this README. Commit source fixtures, schemas, and small expected outputs; keep raw run outputs local unless a human reviewer explicitly promotes a @@ -9,7 +9,7 @@ redacted summary. Expected layout: ```text -packages/evals/results// +evals/results// manifest.json report.md grades.json # deterministic case runs only diff --git a/package.json b/package.json index 6cf6432..88a7239 100644 --- a/package.json +++ b/package.json @@ -36,16 +36,14 @@ "check": "pnpm lint && pnpm check:non-eval && pnpm smoke:enforce-generator && pnpm check:deterministic", "check:deterministic": "pnpm check:eval-static && pnpm eval:unit && pnpm eval:enforce", "check:eval-static": "node scripts/check_eval_static.mjs", + "eval:doctor": "eval-kit doctor --config evals/eval-kit.config.json", + "eval:list": "eval-kit list-cases --config evals/eval-kit.config.json", "eval:unit": "vitest run evals/tests", "eval:case": "eval-kit run-case --config evals/eval-kit.config.json", - "eval:enforce": "node scripts/run_enforce_eval.mjs", - "eval:generate": "eval-kit generate --config evals/eval-kit.config.json", - "eval:judge": "eval-kit judge-pairwise --config evals/eval-kit.config.json", - "eval:judge:coverage": "eval-kit judge-coverage --config evals/eval-kit.config.json", - "eval:manual-report": "eval-kit report --config evals/eval-kit.config.json" + "eval:enforce": "node scripts/run_enforce_eval.mjs" }, "devDependencies": { - "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.0", + "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.1", "ajv": "^8.17.1", "dependency-cruiser": "^18.0.0", "prettier": "^3.6.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4726496..4f12c80 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: devDependencies: '@agentic-workflow-kit/eval-kit': - specifier: github:agentic-workflow-kit/eval-kit#v0.1.0 - version: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/5a81b49553812d9a62ab49614d3b9fdaaef5d2ee + specifier: github:agentic-workflow-kit/eval-kit#v0.1.1 + version: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/2d3bf2da4533fa4442e0ae2a5a512c915c546d89 ajv: specifier: ^8.17.1 version: 8.20.0 @@ -32,9 +32,9 @@ importers: packages: - '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/5a81b49553812d9a62ab49614d3b9fdaaef5d2ee': - resolution: {gitHosted: true, integrity: sha512-cRe37903a+SdYW5pn66G74jDiQq+T5oMXvRe5Z8OpFc2MfXF+JWgNm8EgX9giW0wYzqZAqCAvXrsip2dHEbGdg==, tarball: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/5a81b49553812d9a62ab49614d3b9fdaaef5d2ee} - version: 0.1.0 + '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/2d3bf2da4533fa4442e0ae2a5a512c915c546d89': + resolution: {gitHosted: true, integrity: sha512-4sSEGmu2oZ0wiz3V0TEQ6qzq/G+YBppcfbKT3QRCWmakRx7kzqY93VFl5q9Wfw6GLUZ8OsETv8yiCQs3sGQZ3Q==, tarball: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/2d3bf2da4533fa4442e0ae2a5a512c915c546d89} + version: 0.1.1 engines: {node: '>=22.13.0', pnpm: '>=11.9.0'} hasBin: true @@ -4202,7 +4202,7 @@ packages: snapshots: - '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/5a81b49553812d9a62ab49614d3b9fdaaef5d2ee': + '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/2d3bf2da4533fa4442e0ae2a5a512c915c546d89': dependencies: ajv: 8.20.0