diff --git a/evals/model-judge/README.md b/evals/model-judge/README.md index f3328d2..b444e7b 100644 --- a/evals/model-judge/README.md +++ b/evals/model-judge/README.md @@ -24,6 +24,13 @@ judge semantics. ## Manual Commands +This repo follows the eval-kit two-config standard: + +- `evals/eval-kit.config.json` stays deterministic, default, and CI-safe. +- `evals/eval-kit.model-judge.config.json` enables the manual pointwise judge lane. +- Pairwise judging remains deferred unless a future calibrated comparison workflow adds a separate + config. + Use the deterministic suite first: ```bash @@ -33,15 +40,23 @@ pnpm eval:validate-fixtures pnpm eval:case -- --case case-contract-grounded-prd-v1 --candidate evals/cases/case-contract-grounded-prd-v1/candidate-good.md --run-id local-det-contract-good ``` +Validate the manual judge config before running provider-backed judging: + +```bash +pnpm eval:judge:doctor +pnpm eval:judge:list +pnpm eval:judge:validate-fixtures +``` + Run pointwise judging only when you explicitly want a local model-assisted review: ```bash -pnpm eval:judge:coverage -- --case case-contract-grounded-prd-v1 --candidate evals/cases/case-contract-grounded-prd-v1/candidate-good.md --model --provider openai --effort medium --run-id manual-pointwise-contract-good +pnpm eval:judge:coverage -- --case case-contract-grounded-prd-v1 --candidate evals/cases/case-contract-grounded-prd-v1/candidate-good.md --model gpt-5.5 --provider openai --effort medium --run-id manual-pointwise-contract-good ``` -This command can call a provider through Promptfoo. It expects a deliberate local Promptfoo setup -that provides `node_modules/.bin/promptfoo`; the default repo install and `pnpm check` do not require -Promptfoo. Do not run the command in CI, broad checks, or unattended automation. Do not commit raw +This command can call a provider through Promptfoo. `gpt-5.5` is an account-supported example for +local calibration, not a permanent model policy; use any approved model supported by the current +account. Do not run the command in CI, broad checks, or unattended automation. Do not commit raw result bundles unless they have been reviewed and intentionally curated. ## Calibration diff --git a/package.json b/package.json index 7d0f9d8..9d77289 100644 --- a/package.json +++ b/package.json @@ -28,11 +28,14 @@ "eval:list": "eval-kit list-cases --config evals/eval-kit.config.json", "eval:validate-fixtures": "eval-kit validate-fixtures --config evals/eval-kit.config.json", "eval:case": "eval-kit run-case --config evals/eval-kit.config.json", + "eval:judge:doctor": "eval-kit doctor --config evals/eval-kit.model-judge.config.json", + "eval:judge:list": "eval-kit list-cases --config evals/eval-kit.model-judge.config.json", + "eval:judge:validate-fixtures": "eval-kit validate-fixtures --config evals/eval-kit.model-judge.config.json", "eval:judge:coverage": "eval-kit judge-coverage --config evals/eval-kit.model-judge.config.json", "eval:report": "eval-kit report --config evals/eval-kit.config.json" }, "devDependencies": { - "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.2", + "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.3", "prettier": "^3.6.2", "promptfoo": "^0.121.17" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2ef5d54..f48b031 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,8 +15,8 @@ importers: .: devDependencies: '@agentic-workflow-kit/eval-kit': - specifier: github:agentic-workflow-kit/eval-kit#v0.1.2 - version: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974 + specifier: github:agentic-workflow-kit/eval-kit#v0.1.3 + version: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/1302ec0261674c2a3d14ba7ff49ec67e4b5fee27 prettier: specifier: ^3.6.2 version: 3.9.3 @@ -35,9 +35,9 @@ importers: packages: - '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974': - resolution: {gitHosted: true, tarball: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974} - version: 0.1.2 + '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/1302ec0261674c2a3d14ba7ff49ec67e4b5fee27': + resolution: {gitHosted: true, integrity: sha512-iEAejAn4igspvybA7fV9NpgFDMh4KTm2zKtpsec5NEyXN1CPsHoEuRW/MHggKcNYPhQlBkFvNEF/MRwNolsx+w==, tarball: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/1302ec0261674c2a3d14ba7ff49ec67e4b5fee27} + version: 0.1.3 engines: {node: '>=22.13.0', pnpm: '>=11.9.0'} hasBin: true @@ -3897,7 +3897,7 @@ packages: snapshots: - '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974': + '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/1302ec0261674c2a3d14ba7ff49ec67e4b5fee27': dependencies: ajv: 8.20.0