diff --git a/evals/adapter.mjs b/evals/adapter.mjs index f8ad677..81da450 100644 --- a/evals/adapter.mjs +++ b/evals/adapter.mjs @@ -1,3 +1,4 @@ +import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; @@ -843,6 +844,201 @@ export const renderDeterministicReport = ({ caseId, grades, findings }) => ), ].join("\n"); +const readReportJson = (filePath, label) => { + try { + return readJsonFile(filePath); + } catch (error) { + throw new Error(`failed to read ${label}: ${error.message}`); + } +}; + +const readReportText = (filePath, label) => { + try { + return fs.readFileSync(filePath, "utf8"); + } catch (error) { + throw new Error(`failed to read ${label}: ${error.message}`); + } +}; + +const writeReportJson = (filePath, value) => { + fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`); +}; + +const sha256Text = (value) => + crypto.createHash("sha256").update(value).digest("hex"); + +const manifestCandidateSha256 = (manifest, label) => { + const candidates = (manifest.artifacts ?? []).filter( + (artifact) => artifact.role === "candidate_markdown", + ); + if (candidates.length !== 1) { + throw new Error( + `${label} must include exactly one candidate_markdown artifact`, + ); + } + const candidateSha = candidates[0].sha256; + if (!isNonEmptyString(candidateSha)) { + throw new Error(`${label} candidate_markdown artifact must include sha256`); + } + return candidateSha; +}; + +const pointwiseCandidateFromConfig = (pointwiseDir) => { + const promptfooConfig = readReportJson( + path.join(pointwiseDir, "promptfooconfig.json"), + "pointwise promptfoo config", + ); + const vars = promptfooConfig.tests?.[0]?.vars; + if (!vars || typeof vars !== "object") { + throw new Error("pointwise promptfoo config must include tests[0].vars"); + } + if (!isNonEmptyString(vars.candidate)) { + throw new Error( + "pointwise promptfoo config must include candidate content", + ); + } + return { + path: isNonEmptyString(vars.candidate_path) + ? vars.candidate_path + : "unknown", + sha256: sha256Text(vars.candidate), + }; +}; + +export const compileReport = async ({ runs, resultDir, resolver }) => { + if (runs["judge-coverage"] && !runs.deterministic) { + throw new Error( + "manual reports with pointwise judge evidence must include a deterministic run", + ); + } + + const reportParts = [ + "# Manual Eval Report", + "", + "Deterministic evals remain authoritative. Model-judge results are manual, advisory evidence and cannot upgrade deterministic red or yellow results.", + "", + ]; + const caseIds = new Set(); + const artifacts = []; + const outputFiles = []; + let deterministicCaseId = ""; + let deterministicCandidateSha256 = ""; + + if (runs.deterministic) { + const deterministicDir = resolver.resolveRunDir(runs.deterministic); + const deterministicManifest = readReportJson( + path.join(deterministicDir, "manifest.json"), + "deterministic manifest", + ); + const grades = readReportJson( + path.join(deterministicDir, "grades.json"), + "deterministic grades", + ); + const report = readReportText( + path.join(deterministicDir, "report.md"), + "deterministic report", + ); + deterministicCaseId = grades.case_id ?? ""; + deterministicCandidateSha256 = manifestCandidateSha256( + deterministicManifest, + "deterministic manifest", + ); + if (deterministicCaseId) caseIds.add(deterministicCaseId); + + reportParts.push( + "## Deterministic Verdict", + "", + `- Run: ${runs.deterministic}`, + `- Case: ${grades.case_id ?? "unknown"}`, + `- Verdict: ${grades.verdict ?? "unknown"}`, + "", + report.trim(), + "", + ); + + writeReportJson(path.join(resultDir, "deterministic-grades.json"), grades); + artifacts.push({ + role: "deterministic_grades", + path: "deterministic-grades.json", + mediaType: "application/json", + }); + outputFiles.push("deterministic-grades.json"); + } else { + reportParts.push( + "## Deterministic Verdict", + "", + "No deterministic run was included. Do not use this report to interpret model-judge evidence.", + "", + ); + } + + if (runs["judge-coverage"]) { + const pointwiseDir = resolver.resolveRunDir(runs["judge-coverage"]); + const pointwiseResult = readReportJson( + path.join(pointwiseDir, "pointwise-result.json"), + "pointwise result", + ); + const pointwiseReport = readReportText( + path.join(pointwiseDir, "report.md"), + "pointwise report", + ); + if (pointwiseResult.case_id) caseIds.add(pointwiseResult.case_id); + if (pointwiseResult.case_id !== deterministicCaseId) { + throw new Error( + `pointwise case ${pointwiseResult.case_id} does not match deterministic case ${deterministicCaseId}`, + ); + } + const pointwiseCandidate = pointwiseCandidateFromConfig(pointwiseDir); + if (pointwiseCandidate.sha256 !== deterministicCandidateSha256) { + throw new Error( + `pointwise candidate ${pointwiseCandidate.path} does not match deterministic candidate`, + ); + } + const verdictCounts = Object.entries( + (pointwiseResult.items ?? []).reduce( + (counts, item) => ({ + ...counts, + [item.verdict]: (counts[item.verdict] ?? 0) + 1, + }), + {}, + ), + ) + .map(([verdict, count]) => `${verdict}: ${count}`) + .join(", "); + + reportParts.push( + "## Advisory Pointwise Judge", + "", + `- Run: ${runs["judge-coverage"]}`, + `- Case: ${pointwiseResult.case_id ?? "unknown"}`, + `- Item verdicts: ${verdictCounts || "none"}`, + "", + "This section is calibration evidence only. It cannot override deterministic blockers.", + "", + pointwiseReport.trim(), + "", + ); + + writeReportJson( + path.join(resultDir, "pointwise-result.json"), + pointwiseResult, + ); + artifacts.push({ + role: "pointwise_judge_result", + path: "pointwise-result.json", + mediaType: "application/json", + }); + outputFiles.push("pointwise-result.json"); + } + + return { + reportContent: reportParts.join("\n"), + caseIds: [...caseIds], + artifacts, + outputFiles, + }; +}; + export const validateFixtures = async ({ manifests }) => { assertExpectedCheckValidationFailsClosed(); assertBlockingQuestionMatcherBehavior(); diff --git a/evals/model-judge/README.md b/evals/model-judge/README.md index 544e442..f3328d2 100644 --- a/evals/model-judge/README.md +++ b/evals/model-judge/README.md @@ -60,3 +60,22 @@ For each run, compare `pointwise-result.json` against the expected human label: Record curated observations here or in `evals/results/README.md`: false pass, false fail, ambiguity, verbosity bias, reference-wording bias, and any prompt or item wording adjustment. Model-judge output stays advisory even after calibration unless the repo policy changes explicitly. + +Use `evals/model-judge/calibration-notes.template.md` for human-reviewed calibration notes. Keep raw +Promptfoo outputs local under `evals/results/`; commit only curated summaries. + +## Manual Reports + +Manual reports combine existing local run bundles. They are for reviewer handoff and calibration, +not CI. + +```bash +pnpm eval:report -- --run-id manual-report-contract-good --deterministic local-det-contract-good --judge-coverage manual-pointwise-contract-good +``` + +Report policy: + +- include deterministic evidence first; +- include pointwise evidence only after deterministic evidence for the same review context exists; +- state that model-judge results cannot upgrade deterministic red or yellow verdicts; +- keep generated report bundles under ignored `evals/results/` unless a human curates a summary. diff --git a/evals/model-judge/calibration-notes.template.md b/evals/model-judge/calibration-notes.template.md new file mode 100644 index 0000000..9ffdca7 --- /dev/null +++ b/evals/model-judge/calibration-notes.template.md @@ -0,0 +1,42 @@ +# Model-Judge Calibration Notes + +Case: `` +Candidate: `` +Deterministic run: `` +Pointwise judge run: `` +Manual report run: `` +Reviewer: `` +Date: `` + +## Deterministic Baseline + +- Verdict: `` +- Blocking findings: `` +- Deterministic result remains authoritative: `` + +## Expected Human Label + +- Good candidate expectation: `` +- Bad candidate expectation: `` + +## Item Review + +| Item ID | Expected human label | Judge verdict | Disposition | Notes | +| ------- | -------------------- | ------------- | ---------------------- | -------------- | +| `` | `` | `` | `` | `` | + +## Bias And Failure Modes + +- False pass: +- False fail: +- Ambiguity: +- Verbosity bias: +- Reference-wording bias: +- Unknown rate: + +## Decision + +- Prompt change needed: `` +- Expected item wording change needed: `` +- Keep lane advisory: `yes` +- Raw output reviewed and safe to summarize: `` diff --git a/evals/results/README.md b/evals/results/README.md index f7b0b82..090ba17 100644 --- a/evals/results/README.md +++ b/evals/results/README.md @@ -2,3 +2,7 @@ Eval-kit result bundles are written under this directory. Commit this README and curated summaries only; transient run directories stay ignored by default. + +Manual report bundles may include deterministic grades and pointwise model-judge results copied from +local runs. Treat them as local evidence unless a human reviews and curates a summary. Raw Promptfoo +outputs, provider metadata, candidate text, and generated HTML reports should stay uncommitted. diff --git a/package.json b/package.json index 22198e9..7d0f9d8 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,8 @@ "eval:list": "eval-kit list-cases --config evals/eval-kit.config.json", "eval:validate-fixtures": "eval-kit validate-fixtures --config evals/eval-kit.config.json", "eval:case": "eval-kit run-case --config evals/eval-kit.config.json", - "eval:judge:coverage": "eval-kit judge-coverage --config evals/eval-kit.model-judge.config.json" + "eval:judge:coverage": "eval-kit judge-coverage --config evals/eval-kit.model-judge.config.json", + "eval:report": "eval-kit report --config evals/eval-kit.config.json" }, "devDependencies": { "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.2",