diff --git a/evals/model-judge/README.md b/evals/model-judge/README.md index b444e7b..8b0824f 100644 --- a/evals/model-judge/README.md +++ b/evals/model-judge/README.md @@ -69,15 +69,19 @@ Start calibration with the committed good and bad candidates for the P0 cases: For each run, compare `pointwise-result.json` against the expected human label: - good candidates should have no critical `missing` or `contradicted` items; -- bad candidates should expose the intended critical miss or contradiction; +- `partial` is non-covered calibration evidence unless a non-critical item is explicitly accepted; +- bad candidates should expose the intended critical miss, contradiction, partial coverage, or + ambiguity instead of receiving pass-like `covered` verdicts for the targeted defect; - `unknown` should be rare and should identify real ambiguity, not prompt weakness. Record curated observations here or in `evals/results/README.md`: false pass, false fail, ambiguity, verbosity bias, reference-wording bias, and any prompt or item wording adjustment. Model-judge output stays advisory even after calibration unless the repo policy changes explicitly. -Use `evals/model-judge/calibration-notes.template.md` for human-reviewed calibration notes. Keep raw -Promptfoo outputs local under `evals/results/`; commit only curated summaries. +This repo follows the shared eval-kit `v0.1.5` calibration/reporting policy while keeping +define-product PRD semantics local. Use `evals/model-judge/calibration-notes.template.md` for +human-reviewed calibration notes. Keep raw Promptfoo outputs local under `evals/results/`; commit +only curated summaries. ## Manual Reports @@ -92,5 +96,6 @@ Report policy: - include deterministic evidence first; - include pointwise evidence only after deterministic evidence for the same review context exists; +- include counts for `covered`, `partial`, `missing`, `contradicted`, and `unknown`; - state that model-judge results cannot upgrade deterministic red or yellow verdicts; - keep generated report bundles under ignored `evals/results/` unless a human curates a summary. diff --git a/evals/model-judge/calibration-notes.template.md b/evals/model-judge/calibration-notes.template.md index 9ffdca7..c2026f8 100644 --- a/evals/model-judge/calibration-notes.template.md +++ b/evals/model-judge/calibration-notes.template.md @@ -17,13 +17,13 @@ Date: `` ## Expected Human Label - Good candidate expectation: `` -- Bad candidate expectation: `` +- Bad candidate expectation: `` ## Item Review -| Item ID | Expected human label | Judge verdict | Disposition | Notes | -| ------- | -------------------- | ------------- | ---------------------- | -------------- | -| `` | `` | `` | `` | `` | +| Item ID | Expected human label | Judge verdict | Disposition | Notes | +| ------- | -------------------------------- | ------------- | ---------------------- | -------------- | +| `` | `` | `` | `` | `` | ## Bias And Failure Modes @@ -33,6 +33,8 @@ Date: `` - Verbosity bias: - Reference-wording bias: - Unknown rate: +- Critical partials: +- Expected-bad pass-like coverage: ## Decision diff --git a/evals/results/README.md b/evals/results/README.md index 090ba17..c416c40 100644 --- a/evals/results/README.md +++ b/evals/results/README.md @@ -6,3 +6,8 @@ only; transient run directories stay ignored by default. Manual report bundles may include deterministic grades and pointwise model-judge results copied from local runs. Treat them as local evidence unless a human reviews and curates a summary. Raw Promptfoo outputs, provider metadata, candidate text, and generated HTML reports should stay uncommitted. + +Read manual pointwise verdicts with the eval-kit `v0.1.5` calibration policy: `partial`, `missing`, +`contradicted`, and `unknown` are non-covered evidence unless a curated define-product note says a +non-critical partial is acceptable. Bad PRD fixtures are expected to remain adverse on their targeted +defect; pass-like `covered` verdicts there are false-pass risks. diff --git a/package.json b/package.json index f8482e0..9e68aea 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "eval:report": "eval-kit report --config evals/eval-kit.config.json" }, "devDependencies": { - "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.4", + "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.5", "prettier": "^3.6.2", "promptfoo": "^0.121.17" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 50b57dc..5092bbd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,8 +15,8 @@ importers: .: devDependencies: '@agentic-workflow-kit/eval-kit': - specifier: github:agentic-workflow-kit/eval-kit#v0.1.4 - version: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/39d9991ee41009f70a26155ee07886b105160abd + specifier: github:agentic-workflow-kit/eval-kit#v0.1.5 + version: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/272bad82ff99b8bc841f336d0348e154bc57f57c prettier: specifier: ^3.6.2 version: 3.9.3 @@ -35,9 +35,9 @@ importers: packages: - '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/39d9991ee41009f70a26155ee07886b105160abd': - resolution: {gitHosted: true, tarball: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/39d9991ee41009f70a26155ee07886b105160abd} - version: 0.1.4 + '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/272bad82ff99b8bc841f336d0348e154bc57f57c': + resolution: {gitHosted: true, integrity: sha512-j1gtvTiWuqitWbz8DeCyIvKn0ilHn6eJEjCRUYkOqSUpSFsW40KYtGkfpm5UgUcFzbMAigDs6/pb+0Bmr7BFWw==, tarball: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/272bad82ff99b8bc841f336d0348e154bc57f57c} + version: 0.1.5 engines: {node: '>=22.13.0', pnpm: '>=11.9.0'} hasBin: true @@ -1364,18 +1364,30 @@ packages: resolution: {integrity: sha512-sEvpvkBVoMxjoek35XyJFn2ZD3EJ1RpiZrT47WaZodxzAIWS44zkdvbqGE/ZlugtjiQp62cffYZ9ldyRkjAGnA==} engines: {node: '>=18.0.0'} - '@smithy/credential-provider-imds@4.4.5': - resolution: {integrity: sha512-LnjUTNG0GgQlKIq7IioeOrPaEmC5xOd1WtAz24TLSiYQnWX2uHr53GrFuQhkrJBktPYCMga/NbUOW7hFbSA2Cg==} + '@smithy/core@3.29.1': + resolution: {integrity: sha512-qoiY4nrk5OCu1+eIR1VB8l5DmON/oKiqrd5zZFAhXJXjJlLWQusKEW/SkBDAtGDcPaz86m9kfcE1lngU0GlM6A==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.4.6': + resolution: {integrity: sha512-B2WQ/PV/H6Jeg3lrIq6bKUfa6Hy01mtK7CGs6lhjzHA6k4aagldH6T6eEjnzKl4HI0cJnAsxfJ19pgb5PV+CVQ==} engines: {node: '>=18.0.0'} '@smithy/fetch-http-handler@5.6.2': resolution: {integrity: sha512-q96PSDOAGw+X+nuELd7Cjebps0SYr+YlPbviEX9sLVw+VM4M7VV8hn1nL1mGS6urDu33eQ5A7WhlphaDO6kUyQ==} engines: {node: '>=18.0.0'} + '@smithy/fetch-http-handler@5.6.3': + resolution: {integrity: sha512-CwCc/7SMTj45y97MUnDTbTaxvtAsiNNRm81z3abROIuMbMsC2Iy5EKfkkVdsKrz8WExQAAMx1EJapq+9j4fFTQ==} + engines: {node: '>=18.0.0'} + '@smithy/node-http-handler@4.9.2': resolution: {integrity: sha512-s0yAIRj6TVfHgl+QzVyqal1KMGZ9B5512IrxKc6+dOpw8fUmFL3CvuAhjv0J+aNjUPfVZ2IhqPEDvkB5Ncx9oA==} engines: {node: '>=18.0.0'} + '@smithy/node-http-handler@4.9.3': + resolution: {integrity: sha512-qZTa4gQFUo8RM02rk6q5UVTDLNrQ1oS20LsepBzqq1QBVc/EHJ03OOUADcqMZiXHArW+Y7+OGY0BpdTwZRq/Yg==} + engines: {node: '>=18.0.0'} + '@smithy/signature-v4@5.6.1': resolution: {integrity: sha512-SqvuP75p/DmgWWI7jv4kf/UW+V4LFmlUn19s604SgAcRuJRB1vDnWwzZMYCLUcmKxko9wDn6iLgGEIpTNgZbIQ==} engines: {node: '>=18.0.0'} @@ -3897,7 +3909,7 @@ packages: snapshots: - '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/39d9991ee41009f70a26155ee07886b105160abd': + '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/272bad82ff99b8bc841f336d0348e154bc57f57c': dependencies: ajv: 8.20.0 @@ -4052,7 +4064,7 @@ snapshots: '@aws-sdk/types': 3.973.15 '@aws-sdk/xml-builder': 3.972.33 '@aws/lambda-invoke-store': 0.3.0 - '@smithy/core': 3.29.0 + '@smithy/core': 3.29.1 '@smithy/signature-v4': 5.6.1 '@smithy/types': 4.15.1 bowser: 2.14.1 @@ -4063,7 +4075,7 @@ snapshots: dependencies: '@aws-sdk/core': 3.974.27 '@aws-sdk/types': 3.973.15 - '@smithy/core': 3.29.0 + '@smithy/core': 3.29.1 '@smithy/types': 4.15.1 tslib: 2.8.1 optional: true @@ -4072,9 +4084,9 @@ snapshots: dependencies: '@aws-sdk/core': 3.974.27 '@aws-sdk/types': 3.973.15 - '@smithy/core': 3.29.0 - '@smithy/fetch-http-handler': 5.6.2 - '@smithy/node-http-handler': 4.9.2 + '@smithy/core': 3.29.1 + '@smithy/fetch-http-handler': 5.6.3 + '@smithy/node-http-handler': 4.9.3 '@smithy/types': 4.15.1 tslib: 2.8.1 optional: true @@ -4090,8 +4102,8 @@ snapshots: '@aws-sdk/credential-provider-web-identity': 3.972.59 '@aws-sdk/nested-clients': 3.997.27 '@aws-sdk/types': 3.973.15 - '@smithy/core': 3.29.0 - '@smithy/credential-provider-imds': 4.4.5 + '@smithy/core': 3.29.1 + '@smithy/credential-provider-imds': 4.4.6 '@smithy/types': 4.15.1 tslib: 2.8.1 optional: true @@ -4101,7 +4113,7 @@ snapshots: '@aws-sdk/core': 3.974.27 '@aws-sdk/nested-clients': 3.997.27 '@aws-sdk/types': 3.973.15 - '@smithy/core': 3.29.0 + '@smithy/core': 3.29.1 '@smithy/types': 4.15.1 tslib: 2.8.1 optional: true @@ -4115,8 +4127,8 @@ snapshots: '@aws-sdk/credential-provider-sso': 3.972.59 '@aws-sdk/credential-provider-web-identity': 3.972.59 '@aws-sdk/types': 3.973.15 - '@smithy/core': 3.29.0 - '@smithy/credential-provider-imds': 4.4.5 + '@smithy/core': 3.29.1 + '@smithy/credential-provider-imds': 4.4.6 '@smithy/types': 4.15.1 tslib: 2.8.1 optional: true @@ -4125,7 +4137,7 @@ snapshots: dependencies: '@aws-sdk/core': 3.974.27 '@aws-sdk/types': 3.973.15 - '@smithy/core': 3.29.0 + '@smithy/core': 3.29.1 '@smithy/types': 4.15.1 tslib: 2.8.1 optional: true @@ -4136,7 +4148,7 @@ snapshots: '@aws-sdk/nested-clients': 3.997.27 '@aws-sdk/token-providers': 3.1079.0 '@aws-sdk/types': 3.973.15 - '@smithy/core': 3.29.0 + '@smithy/core': 3.29.1 '@smithy/types': 4.15.1 tslib: 2.8.1 optional: true @@ -4146,7 +4158,7 @@ snapshots: '@aws-sdk/core': 3.974.27 '@aws-sdk/nested-clients': 3.997.27 '@aws-sdk/types': 3.973.15 - '@smithy/core': 3.29.0 + '@smithy/core': 3.29.1 '@smithy/types': 4.15.1 tslib: 2.8.1 optional: true @@ -4193,9 +4205,9 @@ snapshots: '@aws-sdk/core': 3.974.27 '@aws-sdk/signature-v4-multi-region': 3.996.38 '@aws-sdk/types': 3.973.15 - '@smithy/core': 3.29.0 - '@smithy/fetch-http-handler': 5.6.2 - '@smithy/node-http-handler': 4.9.2 + '@smithy/core': 3.29.1 + '@smithy/fetch-http-handler': 5.6.3 + '@smithy/node-http-handler': 4.9.3 '@smithy/types': 4.15.1 tslib: 2.8.1 optional: true @@ -4213,7 +4225,7 @@ snapshots: '@aws-sdk/core': 3.974.27 '@aws-sdk/nested-clients': 3.997.27 '@aws-sdk/types': 3.973.15 - '@smithy/core': 3.29.0 + '@smithy/core': 3.29.1 '@smithy/types': 4.15.1 tslib: 2.8.1 optional: true @@ -5351,9 +5363,15 @@ snapshots: tslib: 2.8.1 optional: true - '@smithy/credential-provider-imds@4.4.5': + '@smithy/core@3.29.1': dependencies: - '@smithy/core': 3.29.0 + '@smithy/types': 4.15.1 + tslib: 2.8.1 + optional: true + + '@smithy/credential-provider-imds@4.4.6': + dependencies: + '@smithy/core': 3.29.1 '@smithy/types': 4.15.1 tslib: 2.8.1 optional: true @@ -5365,6 +5383,13 @@ snapshots: tslib: 2.8.1 optional: true + '@smithy/fetch-http-handler@5.6.3': + dependencies: + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.8.1 + optional: true + '@smithy/node-http-handler@4.9.2': dependencies: '@smithy/core': 3.29.0 @@ -5372,9 +5397,16 @@ snapshots: tslib: 2.8.1 optional: true + '@smithy/node-http-handler@4.9.3': + dependencies: + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.8.1 + optional: true + '@smithy/signature-v4@5.6.1': dependencies: - '@smithy/core': 3.29.0 + '@smithy/core': 3.29.1 '@smithy/types': 4.15.1 tslib: 2.8.1 optional: true