feat: add define-product model judge lane

.github/workflows/check.yml

@@ -25,9 +25,9 @@ jobs:
       - name: Set up Node
         uses: actions/setup-node@v6
         with:
-          # supported floor — CI verifies the >=22.13.0 claim (.nvmrc=26 is the local dev line).
+          # supported floor — CI verifies the >=22.22.0 claim (.nvmrc=26 is the local dev line).
           # cache: pnpm uses setup-node's built-in pnpm store cache (action-setup runs first).
-          node-version: 22.13.0
+          node-version: 22.22.0
           cache: pnpm
 
       - name: Install dependencies

evals/adapter.mjs

@@ -4,6 +4,7 @@ import path from "node:path";
 import { validatePrdForHandoff } from "../packages/prd-kit/src/index.mjs";
 
 const EXPECTED_ITEMS_SCHEMA_VERSION = "define-product.expected-items.v1";
+const POINTWISE_ITEMS_SCHEMA_VERSION = "define-product.pointwise-items.v1";
 const CHECK_MODES = new Set([
   "valid_prd_handoff",
   "required_sections",
@@ -29,6 +30,24 @@ const REQUIRED_SECTIONS = [
   "Downstream Citation Map",
 ];
 const SEVERITIES = new Set(["critical", "high", "medium", "low"]);
+const POINTWISE_KINDS = new Set([
+  "contract",
+  "grounding",
+  "product-altitude",
+  "criterion-checkability",
+  "citation",
+  "scope",
+  "elicitation",
+  "id-stability",
+]);
+const POINTWISE_ITEM_FIELDS = new Set([
+  "item_id",
+  "kind",
+  "severity",
+  "source_refs",
+  "claim",
+  "judge_guidance",
+]);
 const REQUIRED_SECTION_SET = new Set(REQUIRED_SECTIONS);
 
 const formatCheckField = (check, field) => `${check.id}.${field}`;
@@ -313,6 +332,83 @@ const validateExpectedItems = (expectedItems, label) => {
   return expectedItems.checks;
 };
 
+const readJsonFile = (filePath) =>
+  JSON.parse(fs.readFileSync(filePath, "utf8"));
+
+const validatePointwiseItems = (pointwiseItems, label) => {
+  if (pointwiseItems?.schema_version !== POINTWISE_ITEMS_SCHEMA_VERSION) {
+    throw new Error(
+      `${label} must use schema_version ${POINTWISE_ITEMS_SCHEMA_VERSION}`,
+    );
+  }
+  if (
+    !Array.isArray(pointwiseItems.items) ||
+    pointwiseItems.items.length === 0
+  ) {
+    throw new Error(`${label} must define a non-empty items array`);
+  }
+
+  const seenIds = new Set();
+  const sanitizedItems = [];
+  for (const [index, item] of pointwiseItems.items.entries()) {
+    const itemLabel = `${label}.items[${index}]`;
+    const unexpectedFields = Object.keys(item).filter(
+      (field) => !POINTWISE_ITEM_FIELDS.has(field),
+    );
+    if (unexpectedFields.length > 0) {
+      throw new Error(
+        `${itemLabel} contains unsupported fields: ${unexpectedFields.join(", ")}`,
+      );
+    }
+    if (!isNonEmptyString(item.item_id)) {
+      throw new Error(`${itemLabel}.item_id must be a non-empty string`);
+    }
+    if (seenIds.has(item.item_id)) {
+      throw new Error(`${label} contains duplicate item_id ${item.item_id}`);
+    }
+    seenIds.add(item.item_id);
+    if (!POINTWISE_KINDS.has(item.kind)) {
+      throw new Error(`${itemLabel}.kind is unsupported: ${item.kind}`);
+    }
+    if (!SEVERITIES.has(item.severity)) {
+      throw new Error(`${itemLabel}.severity is unsupported: ${item.severity}`);
+    }
+    for (const field of ["claim", "judge_guidance"]) {
+      if (!isNonEmptyString(item[field])) {
+        throw new Error(`${itemLabel}.${field} must be a non-empty string`);
+      }
+    }
+    if (
+      !Array.isArray(item.source_refs) ||
+      item.source_refs.length === 0 ||
+      !item.source_refs.every(isNonEmptyString)
+    ) {
+      throw new Error(`${itemLabel}.source_refs must be a non-empty array`);
+    }
+    sanitizedItems.push({
+      item_id: item.item_id,
+      kind: item.kind,
+      severity: item.severity,
+      source_refs: item.source_refs,
+      claim: item.claim,
+      judge_guidance: item.judge_guidance,
+    });
+  }
+
+  return sanitizedItems;
+};
+
+const artifactFor = (artifacts, role, label) => {
+  const matches = artifacts.filter((artifact) => artifact.role === role);
+  if (matches.length !== 1) {
+    throw new Error(`${label} must define exactly one ${role} artifact`);
+  }
+  return matches[0];
+};
+
+const readArtifactText = (artifacts, role, label) =>
+  fs.readFileSync(artifactFor(artifacts, role, label).absolutePath, "utf8");
+
 const assertBlockingQuestionMatcherBehavior = () => {
   const expectedItems = {
     schema_version: EXPECTED_ITEMS_SCHEMA_VERSION,
@@ -651,6 +747,74 @@ const assessCheck = ({ candidateText, sections, rows, check }) => {
   }
 };
 
+export const resolvePointwiseVars = async ({
+  caseId,
+  artifacts,
+  candidateContent,
+  candidatePath,
+  promptVersion,
+  rubricVersion,
+  model,
+  provider,
+  resolver,
+}) => {
+  const pointwiseArtifact = artifactFor(
+    artifacts,
+    "pointwise_expected_items",
+    caseId,
+  );
+  const pointwiseItems = validatePointwiseItems(
+    readJsonFile(pointwiseArtifact.absolutePath),
+    resolver.relativeToRepo(pointwiseArtifact.absolutePath),
+  );
+
+  return {
+    case_id: caseId,
+    model,
+    provider,
+    prompt_version: promptVersion,
+    rubric_version: rubricVersion,
+    source_material: readArtifactText(artifacts, "generation_visible", caseId),
+    case_rubric: readArtifactText(artifacts, "rubric", caseId),
+    expected_items: JSON.stringify(pointwiseItems, null, 2),
+    candidate_path: resolver.relativeToRepo(candidatePath),
+    candidate: candidateContent,
+    _expectedItemsForCanonicalization: pointwiseItems,
+  };
+};
+
+export const canonicalizeExpectedItemMetadata = (
+  actualItems,
+  expectedItems,
+) => {
+  if (!Array.isArray(actualItems) || !Array.isArray(expectedItems)) {
+    throw new Error("pointwise canonicalization requires item arrays");
+  }
+
+  const actualById = new Map();
+  for (const item of actualItems) {
+    if (actualById.has(item.item_id)) {
+      throw new Error(`pointwise result duplicate item_id: ${item.item_id}`);
+    }
+    actualById.set(item.item_id, item);
+  }
+
+  const actualIds = [...actualById.keys()].sort();
+  const expectedIds = expectedItems.map((item) => item.item_id).sort();
+  if (JSON.stringify(actualIds) !== JSON.stringify(expectedIds)) {
+    throw new Error(
+      `pointwise result item_ids mismatch: expected ${JSON.stringify(expectedIds)}, received ${JSON.stringify(actualIds)}`,
+    );
+  }
+
+  return expectedItems.map((expected) => ({
+    ...actualById.get(expected.item_id),
+    kind: expected.kind,
+    severity: expected.severity,
+    source_refs: expected.source_refs,
+  }));
+};
+
 export const gradeCandidate = ({ candidateText, expectedItems }) => {
   const checks = validateExpectedItems(expectedItems, "expectedItems");
   const sections = extractSections(candidateText);
@@ -706,11 +870,28 @@ export const validateFixtures = async ({ manifests }) => {
       path.dirname(item.fullPath),
       graderInputs[0].path,
     );
-    const expectedItems = JSON.parse(
-      fs.readFileSync(expectedItemsPath, "utf8"),
-    );
+    const expectedItems = readJsonFile(expectedItemsPath);
     validateExpectedItems(expectedItems, expectedItemsPath);
 
+    const pointwiseInputs = item.manifest.artifacts.filter(
+      (artifact) => artifact.role === "pointwise_expected_items",
+    );
+    if (pointwiseInputs.length > 1) {
+      throw new Error(
+        `${item.relativePath} must define at most one pointwise_expected_items artifact`,
+      );
+    }
+    if (pointwiseInputs.length === 1) {
+      const pointwiseItemsPath = path.resolve(
+        path.dirname(item.fullPath),
+        pointwiseInputs[0].path,
+      );
+      validatePointwiseItems(
+        readJsonFile(pointwiseItemsPath),
+        pointwiseItemsPath,
+      );
+    }
+
     for (const artifact of item.manifest.artifacts) {
       if (!["candidate_good", "candidate_bad"].includes(artifact.role)) {
         continue;

evals/cases/case-altitude-and-future-scope-v1/case-manifest.json

@@ -6,6 +6,7 @@
   "artifacts": [
     { "role": "generation_visible", "path": "input.md" },
     { "role": "grader_input", "path": "expected-items.json" },
+    { "role": "pointwise_expected_items", "path": "pointwise-items.json" },
     { "role": "rubric", "path": "rubric.md" },
     { "role": "candidate_good", "path": "candidate-good.md" },
     { "role": "candidate_bad", "path": "candidate-bad.md" }

evals/cases/case-altitude-and-future-scope-v1/pointwise-items.json

@@ -0,0 +1,37 @@
+{
+  "schema_version": "define-product.pointwise-items.v1",
+  "items": [
+    {
+      "item_id": "design-leakage-rejected",
+      "kind": "product-altitude",
+      "severity": "critical",
+      "source_refs": ["input.md"],
+      "claim": "The PRD keeps architecture, package, CLI, schema, delivery sequencing, Jig, and runtime details out of product requirements and acceptance criteria.",
+      "judge_guidance": "Credit product-altitude phrasing that treats technical stakeholder suggestions as out-of-scope design/delivery material. Penalize semantic commitments to implementation even when banned keywords are absent."
+    },
+    {
+      "item_id": "future-work-not-current-scope",
+      "kind": "scope",
+      "severity": "critical",
+      "source_refs": ["input.md"],
+      "claim": "Mobile approvals, auto-approval rules, and manager analytics stay separated from current scope.",
+      "judge_guidance": "Credit current/future separation in non-goals or equivalent scope language. Penalize candidates that promote future work into active ACs or required current outcomes."
+    },
+    {
+      "item_id": "ac-checkable-product-success",
+      "kind": "criterion-checkability",
+      "severity": "high",
+      "source_refs": ["input.md"],
+      "claim": "Acceptance criteria describe externally recognizable reviewer success for the manual refund review queue.",
+      "judge_guidance": "Credit ACs about reviewers seeing high-value refunds, refund reasons, evidence presence, and readiness for approval. Penalize implementation milestones or delivery steps as ACs."
+    },
+    {
+      "item_id": "downstream-citation-exact-and-status-preserving",
+      "kind": "citation",
+      "severity": "critical",
+      "source_refs": ["input.md"],
+      "claim": "The downstream citation map cites exact product AC IDs without requiring downstream layers to inspect design or delivery internals.",
+      "judge_guidance": "Credit exact current-scope AC IDs and handoff-safe citation language. Penalize vague references or citations to technical implementation suggestions as product contract facts."
+    }
+  ]
+}

evals/cases/case-contract-grounded-prd-v1/case-manifest.json

@@ -6,6 +6,7 @@
   "artifacts": [
     { "role": "generation_visible", "path": "input.md" },
     { "role": "grader_input", "path": "expected-items.json" },
+    { "role": "pointwise_expected_items", "path": "pointwise-items.json" },
     { "role": "rubric", "path": "rubric.md" },
     { "role": "candidate_good", "path": "candidate-good.md" },
     { "role": "candidate_bad", "path": "candidate-bad.md" }

evals/cases/case-contract-grounded-prd-v1/pointwise-items.json

@@ -0,0 +1,45 @@
+{
+  "schema_version": "define-product.pointwise-items.v1",
+  "items": [
+    {
+      "item_id": "product-outcome-grounded",
+      "kind": "grounding",
+      "severity": "critical",
+      "source_refs": ["input.md"],
+      "claim": "The PRD states a product outcome grounded in support agents answering refund-status questions in under two minutes.",
+      "judge_guidance": "Credit only product-level outcome language tied to support agents, refund status, and the two-minute success signal. Do not credit implementation or workflow mechanics as the outcome."
+    },
+    {
+      "item_id": "user-job-source-faithful",
+      "kind": "product-altitude",
+      "severity": "high",
+      "source_refs": ["input.md"],
+      "claim": "The PRD identifies the support-agent job clearly enough for downstream design without inventing implementation details.",
+      "judge_guidance": "Look for a clear user/job statement about answering customer refund-status questions. Penalize invented personas, systems, or design-specific responsibilities."
+    },
+    {
+      "item_id": "ac-checkable-product-success",
+      "kind": "criterion-checkability",
+      "severity": "critical",
+      "source_refs": ["input.md"],
+      "claim": "Acceptance criteria describe externally recognizable product success rather than internal implementation completion.",
+      "judge_guidance": "Credit ACs that can be recognized from product behavior, cited facts, or user-visible/internal-agent outcomes. Penalize ACs that require reading architecture, schema, delivery, or tool internals."
+    },
+    {
+      "item_id": "uncertainty-visible-not-laundered",
+      "kind": "grounding",
+      "severity": "critical",
+      "source_refs": ["input.md"],
+      "claim": "Legal approval, 30-day exception uncertainty, and CSV export conflict remain visibly labeled instead of becoming settled facts.",
+      "judge_guidance": "Credit visible treatment of inferred facts, gaps/defaults, and conflicts. Penalize language that silently turns legal wording, exception paths, or export scope into confirmed current requirements."
+    },
+    {
+      "item_id": "downstream-citation-exact-and-status-preserving",
+      "kind": "citation",
+      "severity": "critical",
+      "source_refs": ["input.md"],
+      "claim": "The downstream citation map gives Technical Design and Planning exact AC IDs while preserving current-scope and uncertainty status.",
+      "judge_guidance": "Credit exact AC IDs and useful handoff notes. Penalize vague references, missing IDs, or citations that hide assumptions, conflicts, or non-goals."
+    }
+  ]
+}

evals/eval-kit.model-judge.config.json

@@ -0,0 +1,37 @@
+{
+  "$schema": "../node_modules/@agentic-workflow-kit/eval-kit/schemas/eval-kit.config.schema.json",
+  "schema_version": "eval-kit.config.v1",
+  "suite_id": "define-product",
+  "suite_root": ".",
+  "results_root": "results",
+  "adapter": "adapter.mjs",
+  "cases": {
+    "root": "cases",
+    "include": [
+      "case-altitude-and-future-scope-v1",
+      "case-contract-grounded-prd-v1"
+    ]
+  },
+  "prompt_templates": {
+    "pointwise_judge": "prompts/pointwise-prd.prompt.md"
+  },
+  "methods": {
+    "deterministic": {
+      "enabled": true,
+      "grader": "bootstrap-placeholder",
+      "reporter": "bootstrap-markdown"
+    },
+    "generate": {
+      "enabled": false
+    },
+    "judge_coverage": {
+      "enabled": true
+    },
+    "judge_pairwise": {
+      "enabled": false
+    },
+    "report": {
+      "enabled": false
+    }
+  }
+}