feat: add model judge report calibration docs

evals/adapter.mjs

@@ -1,3 +1,4 @@
+import crypto from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
 
@@ -843,6 +844,201 @@ export const renderDeterministicReport = ({ caseId, grades, findings }) =>
     ),
   ].join("\n");
 
+const readReportJson = (filePath, label) => {
+  try {
+    return readJsonFile(filePath);
+  } catch (error) {
+    throw new Error(`failed to read ${label}: ${error.message}`);
+  }
+};
+
+const readReportText = (filePath, label) => {
+  try {
+    return fs.readFileSync(filePath, "utf8");
+  } catch (error) {
+    throw new Error(`failed to read ${label}: ${error.message}`);
+  }
+};
+
+const writeReportJson = (filePath, value) => {
+  fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`);
+};
+
+const sha256Text = (value) =>
+  crypto.createHash("sha256").update(value).digest("hex");
+
+const manifestCandidateSha256 = (manifest, label) => {
+  const candidates = (manifest.artifacts ?? []).filter(
+    (artifact) => artifact.role === "candidate_markdown",
+  );
+  if (candidates.length !== 1) {
+    throw new Error(
+      `${label} must include exactly one candidate_markdown artifact`,
+    );
+  }
+  const candidateSha = candidates[0].sha256;
+  if (!isNonEmptyString(candidateSha)) {
+    throw new Error(`${label} candidate_markdown artifact must include sha256`);
+  }
+  return candidateSha;
+};
+
+const pointwiseCandidateFromConfig = (pointwiseDir) => {
+  const promptfooConfig = readReportJson(
+    path.join(pointwiseDir, "promptfooconfig.json"),
+    "pointwise promptfoo config",
+  );
+  const vars = promptfooConfig.tests?.[0]?.vars;
+  if (!vars || typeof vars !== "object") {
+    throw new Error("pointwise promptfoo config must include tests[0].vars");
+  }
+  if (!isNonEmptyString(vars.candidate)) {
+    throw new Error(
+      "pointwise promptfoo config must include candidate content",
+    );
+  }
+  return {
+    path: isNonEmptyString(vars.candidate_path)
+      ? vars.candidate_path
+      : "unknown",
+    sha256: sha256Text(vars.candidate),
+  };
+};
+
+export const compileReport = async ({ runs, resultDir, resolver }) => {
+  if (runs["judge-coverage"] && !runs.deterministic) {
+    throw new Error(
+      "manual reports with pointwise judge evidence must include a deterministic run",
+    );
+  }
+
+  const reportParts = [
+    "# Manual Eval Report",
+    "",
+    "Deterministic evals remain authoritative. Model-judge results are manual, advisory evidence and cannot upgrade deterministic red or yellow results.",
+    "",
+  ];
+  const caseIds = new Set();
+  const artifacts = [];
+  const outputFiles = [];
+  let deterministicCaseId = "";
+  let deterministicCandidateSha256 = "";
+
+  if (runs.deterministic) {
+    const deterministicDir = resolver.resolveRunDir(runs.deterministic);
+    const deterministicManifest = readReportJson(
+      path.join(deterministicDir, "manifest.json"),
+      "deterministic manifest",
+    );
+    const grades = readReportJson(
+      path.join(deterministicDir, "grades.json"),
+      "deterministic grades",
+    );
+    const report = readReportText(
+      path.join(deterministicDir, "report.md"),
+      "deterministic report",
+    );
+    deterministicCaseId = grades.case_id ?? "";
+    deterministicCandidateSha256 = manifestCandidateSha256(
+      deterministicManifest,
+      "deterministic manifest",
+    );
+    if (deterministicCaseId) caseIds.add(deterministicCaseId);
+
+    reportParts.push(
+      "## Deterministic Verdict",
+      "",
+      `- Run: ${runs.deterministic}`,
+      `- Case: ${grades.case_id ?? "unknown"}`,
+      `- Verdict: ${grades.verdict ?? "unknown"}`,
+      "",
+      report.trim(),
+      "",
+    );
+
+    writeReportJson(path.join(resultDir, "deterministic-grades.json"), grades);
+    artifacts.push({
+      role: "deterministic_grades",
+      path: "deterministic-grades.json",
+      mediaType: "application/json",
+    });
+    outputFiles.push("deterministic-grades.json");
+  } else {
+    reportParts.push(
+      "## Deterministic Verdict",
+      "",
+      "No deterministic run was included. Do not use this report to interpret model-judge evidence.",
+      "",
+    );
+  }
+
+  if (runs["judge-coverage"]) {
+    const pointwiseDir = resolver.resolveRunDir(runs["judge-coverage"]);
+    const pointwiseResult = readReportJson(
+      path.join(pointwiseDir, "pointwise-result.json"),
+      "pointwise result",
+    );
+    const pointwiseReport = readReportText(
+      path.join(pointwiseDir, "report.md"),
+      "pointwise report",
+    );
+    if (pointwiseResult.case_id) caseIds.add(pointwiseResult.case_id);
+    if (pointwiseResult.case_id !== deterministicCaseId) {
+      throw new Error(
+        `pointwise case ${pointwiseResult.case_id} does not match deterministic case ${deterministicCaseId}`,
+      );
+    }
+    const pointwiseCandidate = pointwiseCandidateFromConfig(pointwiseDir);
+    if (pointwiseCandidate.sha256 !== deterministicCandidateSha256) {
+      throw new Error(
+        `pointwise candidate ${pointwiseCandidate.path} does not match deterministic candidate`,
+      );
+    }
+    const verdictCounts = Object.entries(
+      (pointwiseResult.items ?? []).reduce(
+        (counts, item) => ({
+          ...counts,
+          [item.verdict]: (counts[item.verdict] ?? 0) + 1,
+        }),
+        {},
+      ),
+    )
+      .map(([verdict, count]) => `${verdict}: ${count}`)
+      .join(", ");
+
+    reportParts.push(
+      "## Advisory Pointwise Judge",
+      "",
+      `- Run: ${runs["judge-coverage"]}`,
+      `- Case: ${pointwiseResult.case_id ?? "unknown"}`,
+      `- Item verdicts: ${verdictCounts || "none"}`,
+      "",
+      "This section is calibration evidence only. It cannot override deterministic blockers.",
+      "",
+      pointwiseReport.trim(),
+      "",
+    );
+
+    writeReportJson(
+      path.join(resultDir, "pointwise-result.json"),
+      pointwiseResult,
+    );
+    artifacts.push({
+      role: "pointwise_judge_result",
+      path: "pointwise-result.json",
+      mediaType: "application/json",
+    });
+    outputFiles.push("pointwise-result.json");
+  }
+
+  return {
+    reportContent: reportParts.join("\n"),
+    caseIds: [...caseIds],
+    artifacts,
+    outputFiles,
+  };
+};
+
 export const validateFixtures = async ({ manifests }) => {
   assertExpectedCheckValidationFailsClosed();
   assertBlockingQuestionMatcherBehavior();

evals/model-judge/README.md

@@ -60,3 +60,22 @@ For each run, compare `pointwise-result.json` against the expected human label:
 Record curated observations here or in `evals/results/README.md`: false pass, false fail, ambiguity,
 verbosity bias, reference-wording bias, and any prompt or item wording adjustment. Model-judge output
 stays advisory even after calibration unless the repo policy changes explicitly.
+
+Use `evals/model-judge/calibration-notes.template.md` for human-reviewed calibration notes. Keep raw
+Promptfoo outputs local under `evals/results/`; commit only curated summaries.
+
+## Manual Reports
+
+Manual reports combine existing local run bundles. They are for reviewer handoff and calibration,
+not CI.
+
+```bash
+pnpm eval:report -- --run-id manual-report-contract-good --deterministic local-det-contract-good --judge-coverage manual-pointwise-contract-good
+```
+
+Report policy:
+
+- include deterministic evidence first;
+- include pointwise evidence only after deterministic evidence for the same review context exists;
+- state that model-judge results cannot upgrade deterministic red or yellow verdicts;
+- keep generated report bundles under ignored `evals/results/` unless a human curates a summary.

evals/model-judge/calibration-notes.template.md

@@ -0,0 +1,42 @@
+# Model-Judge Calibration Notes
+
+Case: `<case-id>`
+Candidate: `<candidate path>`
+Deterministic run: `<run-id>`
+Pointwise judge run: `<run-id>`
+Manual report run: `<run-id>`
+Reviewer: `<name or handle>`
+Date: `<YYYY-MM-DD>`
+
+## Deterministic Baseline
+
+- Verdict: `<green|yellow|red>`
+- Blocking findings: `<none or finding ids>`
+- Deterministic result remains authoritative: `<yes>`
+
+## Expected Human Label
+
+- Good candidate expectation: `<critical items covered>`
+- Bad candidate expectation: `<intended critical misses or contradictions>`
+
+## Item Review
+
+| Item ID | Expected human label | Judge verdict | Disposition            | Notes          |
+| ------- | -------------------- | ------------- | ---------------------- | -------------- |
+| `<id>`  | `<covered/missing>`  | `<verdict>`   | `<ok/fp/fn/ambiguous>` | `<short note>` |
+
+## Bias And Failure Modes
+
+- False pass:
+- False fail:
+- Ambiguity:
+- Verbosity bias:
+- Reference-wording bias:
+- Unknown rate:
+
+## Decision
+
+- Prompt change needed: `<yes/no>`
+- Expected item wording change needed: `<yes/no>`
+- Keep lane advisory: `yes`
+- Raw output reviewed and safe to summarize: `<yes/no>`

evals/results/README.md

@@ -2,3 +2,7 @@
 
 Eval-kit result bundles are written under this directory. Commit this README and curated summaries
 only; transient run directories stay ignored by default.
+
+Manual report bundles may include deterministic grades and pointwise model-judge results copied from
+local runs. Treat them as local evidence unless a human reviews and curates a summary. Raw Promptfoo
+outputs, provider metadata, candidate text, and generated HTML reports should stay uncommitted.

package.json

@@ -28,7 +28,8 @@
     "eval:list": "eval-kit list-cases --config evals/eval-kit.config.json",
     "eval:validate-fixtures": "eval-kit validate-fixtures --config evals/eval-kit.config.json",
     "eval:case": "eval-kit run-case --config evals/eval-kit.config.json",
-    "eval:judge:coverage": "eval-kit judge-coverage --config evals/eval-kit.model-judge.config.json"
+    "eval:judge:coverage": "eval-kit judge-coverage --config evals/eval-kit.model-judge.config.json",
+    "eval:report": "eval-kit report --config evals/eval-kit.config.json"
   },
   "devDependencies": {
     "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.2",