diff --git a/evals/adapter.mjs b/evals/adapter.mjs index 829d089..ab71bf1 100644 --- a/evals/adapter.mjs +++ b/evals/adapter.mjs @@ -1,32 +1,663 @@ import fs from "node:fs"; import path from "node:path"; -const expectedItemsFor = (expectedItems, label) => { +import { validatePrdForHandoff } from "../packages/prd-kit/src/index.mjs"; + +const EXPECTED_ITEMS_SCHEMA_VERSION = "define-product.expected-items.v1"; +const CHECK_MODES = new Set([ + "valid_prd_handoff", + "required_sections", + "section_includes", + "section_excludes", + "text_includes", + "text_excludes", + "ac_ids_present", + "ac_ids_preserved", + "ac_status", + "assumptions_labeled", + "downstream_cites_ids", + "forbidden_outside_sections", + "blocking_questions", +]); +const REQUIRED_SECTIONS = [ + "Product Outcome", + "User Job", + "Acceptance Criteria", + "Constraints", + "Assumptions", + "Non-Goals", + "Downstream Citation Map", +]; +const SEVERITIES = new Set(["critical", "high", "medium", "low"]); +const REQUIRED_SECTION_SET = new Set(REQUIRED_SECTIONS); + +const formatCheckField = (check, field) => `${check.id}.${field}`; + +const hasOwn = (value, field) => + Object.prototype.hasOwnProperty.call(value, field); + +const isNonEmptyString = (value) => + typeof value === "string" && value.trim().length > 0; + +const requireNonEmptyString = (check, field) => { + if (!isNonEmptyString(check[field])) { + throw new Error( + `${formatCheckField(check, field)} must be a non-empty string`, + ); + } +}; + +const validateStringArray = (check, field, { required }) => { + if (!hasOwn(check, field)) { + if (required) { + throw new Error( + `${formatCheckField(check, field)} must be a non-empty string array`, + ); + } + return false; + } if ( - !Array.isArray(expectedItems?.items) || - expectedItems.items.length === 0 + !Array.isArray(check[field]) || + check[field].length === 0 || + !check[field].every(isNonEmptyString) ) { - throw new Error(`${label} must define a non-empty items array`); + throw new Error( + `${formatCheckField(check, field)} must be a non-empty string array`, + ); } - return expectedItems.items; + return true; }; -export const gradeCandidate = ({ candidateText, expectedItems }) => { - const items = expectedItemsFor(expectedItems, "expectedItems"); - const findings = items.map((item) => { - const requiredText = item.required_text ?? ""; - const covered = - requiredText.length > 0 && candidateText.includes(requiredText); - return { - id: item.id, - kind: item.kind ?? "generic", - severity: item.severity ?? "medium", - verdict: covered ? "covered" : "missing", - evidence: covered - ? `found ${requiredText}` - : `missing ${requiredText || item.id}`, - }; +const validateOptionalStringArrays = (check, fields) => { + for (const field of fields) { + validateStringArray(check, field, { required: false }); + } +}; + +const requireAnyStringArray = (check, fields) => { + const validFields = fields.filter((field) => + validateStringArray(check, field, { required: false }), + ); + if (validFields.length === 0) { + throw new Error( + `${check.id} must define at least one of ${fields.join(", ")}`, + ); + } +}; + +const requireSection = (check) => { + requireNonEmptyString(check, "section"); + if (!REQUIRED_SECTION_SET.has(check.section)) { + throw new Error(`${check.id}.section must name a required PRD section`); + } +}; + +const requirePositiveInteger = (check, field) => { + if (!Number.isInteger(check[field]) || check[field] < 1) { + throw new Error( + `${formatCheckField(check, field)} must be a positive integer`, + ); + } +}; + +const requireBoolean = (check, field) => { + if (typeof check[field] !== "boolean") { + throw new Error(`${formatCheckField(check, field)} must be a boolean`); + } +}; + +const validatePreservedRows = (check) => { + if (!Array.isArray(check.rows) || check.rows.length === 0) { + throw new Error(`${check.id}.rows must be a non-empty array`); + } + for (const [index, row] of check.rows.entries()) { + const rowLabel = `${check.id}.rows[${index}]`; + if (!isNonEmptyString(row?.id)) { + throw new Error(`${rowLabel}.id must be a non-empty string`); + } + if ( + !Array.isArray(row.must_include_all) || + row.must_include_all.length === 0 || + !row.must_include_all.every(isNonEmptyString) + ) { + throw new Error( + `${rowLabel}.must_include_all must be a non-empty string array`, + ); + } + } +}; + +const validateCheckFields = (check) => { + switch (check.mode) { + case "valid_prd_handoff": + case "required_sections": + return; + case "section_includes": + requireSection(check); + requireAnyStringArray(check, ["must_include_all", "must_include_any"]); + validateOptionalStringArrays(check, ["must_not_include_any"]); + return; + case "section_excludes": + requireSection(check); + validateStringArray(check, "must_not_include_any", { required: true }); + return; + case "text_includes": + requireAnyStringArray(check, ["must_include_all", "must_include_any"]); + validateOptionalStringArrays(check, ["must_not_include_any"]); + return; + case "text_excludes": + validateStringArray(check, "must_not_include_any", { required: true }); + return; + case "ac_ids_present": + validateStringArray(check, "ids", { required: true }); + return; + case "ac_ids_preserved": + validatePreservedRows(check); + return; + case "ac_status": + requireNonEmptyString(check, "id_ref"); + requireNonEmptyString(check, "status"); + return; + case "assumptions_labeled": + validateStringArray(check, "labels", { required: true }); + return; + case "downstream_cites_ids": + validateStringArray(check, "ids", { required: true }); + return; + case "forbidden_outside_sections": + validateStringArray(check, "allowed_sections", { required: true }); + validateStringArray(check, "must_not_include_any", { required: true }); + return; + case "blocking_questions": + requirePositiveInteger(check, "min_questions"); + requireBoolean(check, "forbid_prd"); + requireAnyStringArray(check, ["must_include_all", "must_include_any"]); + validateOptionalStringArrays(check, ["must_not_include_any"]); + return; + default: + throw new Error(`unsupported check mode: ${check.mode}`); + } +}; + +const FAIL_CLOSED_VALIDATION_PROBES = [ + { id: "probe-section-includes", mode: "section_includes" }, + { + id: "probe-section-includes-empty", + mode: "section_includes", + section: "Product Outcome", + must_include_all: [], + }, + { + id: "probe-section-excludes", + mode: "section_excludes", + section: "Constraints", + }, + { id: "probe-text-includes", mode: "text_includes" }, + { + id: "probe-text-includes-empty", + mode: "text_includes", + must_include_any: [], + }, + { id: "probe-text-excludes", mode: "text_excludes" }, + { id: "probe-ac-ids-present", mode: "ac_ids_present" }, + { id: "probe-ac-ids-present-empty", mode: "ac_ids_present", ids: [] }, + { id: "probe-ac-ids-preserved", mode: "ac_ids_preserved" }, + { id: "probe-ac-status", mode: "ac_status" }, + { id: "probe-assumptions-labeled", mode: "assumptions_labeled" }, + { id: "probe-downstream-cites-ids", mode: "downstream_cites_ids" }, + { + id: "probe-forbidden-outside-sections", + mode: "forbidden_outside_sections", + }, + { id: "probe-blocking-questions", mode: "blocking_questions" }, +]; + +const assertExpectedCheckValidationFailsClosed = () => { + for (const check of FAIL_CLOSED_VALIDATION_PROBES) { + try { + validateExpectedItems( + { + schema_version: EXPECTED_ITEMS_SCHEMA_VERSION, + checks: [{ ...check, severity: "critical" }], + }, + `fail-closed probe ${check.id}`, + ); + } catch { + continue; + } + throw new Error(`fail-closed probe ${check.id} unexpectedly passed`); + } +}; + +const normalize = (value) => + String(value ?? "") + .toLowerCase() + .replace(/[`*_~]/g, "") + .replace(/[‘’‛′']/g, "") + .replace(/[“”«»"]/g, "") + .replace(/[‐‑‒–—―]/g, "-") + .replace(/&/g, " and ") + .replace(/[^a-z0-9]+/g, " ") + .replace(/\s+/g, " ") + .trim(); + +const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +const includesNormalized = (text, snippet) => { + const normalizedText = normalize(text); + const normalizedSnippet = normalize(snippet); + return ( + normalizedSnippet.length > 0 && + ` ${normalizedText} `.includes(` ${normalizedSnippet} `) + ); +}; + +const missingSnippets = (text, snippets = []) => + snippets.filter((snippet) => !includesNormalized(text, snippet)); + +const firstIncluded = (text, snippets = []) => + snippets.find((snippet) => includesNormalized(text, snippet)); + +const headingSnippetPattern = (snippet) => { + const match = String(snippet ?? "") + .trim() + .match(/^(#{1,6})\s+(.+?)\s*$/); + if (!match) return null; + return new RegExp( + `^\\s*${escapeRegExp(match[1])}\\s+${escapeRegExp(match[2])}\\s*$`, + "m", + ); +}; + +const includesForbiddenSnippet = (text, snippet) => { + const headingPattern = headingSnippetPattern(snippet); + return headingPattern + ? headingPattern.test(String(text ?? "")) + : includesNormalized(text, snippet); +}; + +const firstForbiddenIncluded = (text, snippets = []) => + snippets.find((snippet) => includesForbiddenSnippet(text, snippet)); + +const containsPrdSectionHeading = (text) => + REQUIRED_SECTIONS.some((section) => + new RegExp(`^\\s*##\\s+${escapeRegExp(section)}\\s*$`, "m").test( + String(text ?? ""), + ), + ); + +const validateExpectedItems = (expectedItems, label) => { + if (expectedItems?.schema_version !== EXPECTED_ITEMS_SCHEMA_VERSION) { + throw new Error( + `${label} must use schema_version ${EXPECTED_ITEMS_SCHEMA_VERSION}`, + ); + } + if ( + !Array.isArray(expectedItems.checks) || + expectedItems.checks.length === 0 + ) { + throw new Error(`${label} must define a non-empty checks array`); + } + for (const check of expectedItems.checks) { + if (typeof check.id !== "string" || check.id.length === 0) { + throw new Error(`${label} contains a check without an id`); + } + if (!CHECK_MODES.has(check.mode)) { + throw new Error(`${label} check ${check.id} has unknown mode`); + } + if (!SEVERITIES.has(check.severity ?? "medium")) { + throw new Error(`${label} check ${check.id} has invalid severity`); + } + validateCheckFields(check); + } + return expectedItems.checks; +}; + +const assertBlockingQuestionMatcherBehavior = () => { + const expectedItems = { + schema_version: EXPECTED_ITEMS_SCHEMA_VERSION, + checks: [ + { + id: "probe-blocking-heading-sensitive", + mode: "blocking_questions", + severity: "critical", + min_questions: 3, + forbid_prd: true, + must_include_all: ["blocking questions"], + must_include_any: ["product outcome"], + must_not_include_any: ["## Product Outcome", "## Acceptance Criteria"], + }, + ], + }; + + const blockingOnly = [ + "Blocking questions:", + "", + "1. What product outcome should this target?", + "2. Who is the user?", + "3. What recognizable success signal should the PRD use?", + ].join("\n"); + const blockingOnlyGrades = gradeCandidate({ + candidateText: blockingOnly, + expectedItems, }); + if (blockingOnlyGrades.verdict !== "green") { + throw new Error( + "heading-sensitive forbidden snippets rejected a blocking-only answer", + ); + } + + const partialPrd = [ + "Blocking questions:", + "", + "1. What product outcome should this target?", + "2. Who is the user?", + "3. What recognizable success signal should the PRD use?", + "", + "## User Job", + "", + "As a user, I need a better onboarding flow.", + ].join("\n"); + const partialPrdGrades = gradeCandidate({ + candidateText: partialPrd, + expectedItems, + }); + if (partialPrdGrades.verdict !== "red") { + throw new Error("forbid_prd allowed a partial PRD section heading"); + } +}; + +const extractSections = (markdown) => { + const matches = [...String(markdown ?? "").matchAll(/^##\s+(.+?)\s*$/gm)]; + const sections = new Map(); + for (const [index, match] of matches.entries()) { + const heading = match[1].trim(); + const start = match.index + match[0].length; + const end = + index + 1 < matches.length ? matches[index + 1].index : markdown.length; + sections.set(heading, markdown.slice(start, end).trim()); + } + return sections; +}; + +const parseAcceptanceCriteriaRows = (sections) => { + const tableRows = (sections.get("Acceptance Criteria") ?? "") + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.startsWith("|")) + .map((line) => + line + .split("|") + .slice(1, -1) + .map((cell) => cell.trim()), + ) + .filter((cells) => cells.length >= 3); + + const separatorIndex = tableRows.findIndex((cells) => + cells.every((cell) => /^:?-+:?$/.test(cell)), + ); + const dataRows = + separatorIndex === -1 ? [] : tableRows.slice(separatorIndex + 1); + return dataRows.map(([id, criterion, status]) => ({ id, criterion, status })); +}; + +const covered = (check, evidence) => ({ + id: check.id, + kind: check.kind ?? check.mode, + severity: check.severity ?? "medium", + verdict: "covered", + evidence, +}); + +const missing = (check, evidence, verdict = "missing") => ({ + id: check.id, + kind: check.kind ?? check.mode, + severity: check.severity ?? "medium", + verdict, + evidence, +}); + +const assessSectionIncludes = (candidateText, sections, check) => { + const target = sections.get(check.section) ?? ""; + if (!sections.has(check.section)) { + return missing(check, `missing section ${check.section}`); + } + const forbidden = firstForbiddenIncluded( + target, + check.must_not_include_any ?? [], + ); + if (forbidden) { + return missing( + check, + `forbidden text in ${check.section}: ${forbidden}`, + "contradicted", + ); + } + const missingAll = missingSnippets(target, check.must_include_all ?? []); + const anyRequired = check.must_include_any ?? []; + const anyHit = anyRequired.length === 0 || firstIncluded(target, anyRequired); + if (missingAll.length === 0 && anyHit) { + return covered(check, `matched ${check.section}`); + } + const missingEvidence = [ + ...missingAll, + ...(anyHit ? [] : [`one of: ${anyRequired.join(", ")}`]), + ].join("; "); + return missing(check, `missing from ${check.section}: ${missingEvidence}`); +}; + +const assessSectionExcludes = (sections, check) => { + const target = sections.get(check.section) ?? ""; + if (!sections.has(check.section)) { + return missing(check, `missing section ${check.section}`); + } + const forbidden = firstForbiddenIncluded( + target, + check.must_not_include_any ?? [], + ); + return forbidden + ? missing( + check, + `forbidden text in ${check.section}: ${forbidden}`, + "contradicted", + ) + : covered(check, `no forbidden text in ${check.section}`); +}; + +const assessTextIncludes = (candidateText, check) => { + const forbidden = firstForbiddenIncluded( + candidateText, + check.must_not_include_any ?? [], + ); + if (forbidden) { + return missing(check, `forbidden text found: ${forbidden}`, "contradicted"); + } + const missingAll = missingSnippets( + candidateText, + check.must_include_all ?? [], + ); + const anyRequired = check.must_include_any ?? []; + const anyHit = + anyRequired.length === 0 || firstIncluded(candidateText, anyRequired); + return missingAll.length === 0 && anyHit + ? covered(check, "matched required text") + : missing( + check, + `missing required text: ${[ + ...missingAll, + ...(anyHit ? [] : [`one of: ${anyRequired.join(", ")}`]), + ].join("; ")}`, + ); +}; + +const assessTextExcludes = (candidateText, check) => { + const forbidden = firstForbiddenIncluded( + candidateText, + check.must_not_include_any ?? [], + ); + return forbidden + ? missing(check, `forbidden text found: ${forbidden}`, "contradicted") + : covered(check, "no forbidden text found"); +}; + +const assessAcIdsPresent = (rows, check) => { + const ids = new Set(rows.map((row) => row.id)); + const missingIds = (check.ids ?? []).filter((id) => !ids.has(id)); + return missingIds.length === 0 + ? covered(check, `found IDs: ${(check.ids ?? []).join(", ")}`) + : missing(check, `missing IDs: ${missingIds.join(", ")}`); +}; + +const assessAcIdsPreserved = (rows, check) => { + const rowsById = new Map(rows.map((row) => [row.id, row])); + const failures = []; + for (const expected of check.rows ?? []) { + const row = rowsById.get(expected.id); + if (!row) { + failures.push(`${expected.id} missing`); + continue; + } + const missingMeaning = missingSnippets( + row.criterion, + expected.must_include_all ?? [], + ); + if (missingMeaning.length > 0) { + failures.push( + `${expected.id} missing meaning: ${missingMeaning.join(", ")}`, + ); + } + } + return failures.length === 0 + ? covered(check, "preserved expected AC IDs and meanings") + : missing(check, failures.join("; ")); +}; + +const assessAcStatus = (rows, check) => { + const rowsById = new Map(rows.map((row) => [row.id, row])); + const row = rowsById.get(check.id_ref); + if (!row) { + return missing(check, `missing ID ${check.id_ref}`); + } + return row.status === check.status + ? covered(check, `${check.id_ref} status is ${check.status}`) + : missing( + check, + `${check.id_ref} status was ${row.status}, expected ${check.status}`, + ); +}; + +const assessAssumptionsLabeled = (sections, check) => { + const assumptions = sections.get("Assumptions") ?? ""; + const missingLabels = (check.labels ?? []).filter( + (label) => + !new RegExp(`(^|\\n)\\s*[-*]?\\s*${label}:`, "i").test(assumptions), + ); + return missingLabels.length === 0 + ? covered(check, `found labels: ${(check.labels ?? []).join(", ")}`) + : missing(check, `missing labels: ${missingLabels.join(", ")}`); +}; + +const assessDownstreamCitesIds = (sections, check) => { + const citationMap = sections.get("Downstream Citation Map") ?? ""; + if (!sections.has("Downstream Citation Map")) { + return missing(check, "missing Downstream Citation Map"); + } + const missingIds = (check.ids ?? []).filter( + (id) => !includesNormalized(citationMap, id), + ); + return missingIds.length === 0 + ? covered(check, `citation map includes ${check.ids.join(", ")}`) + : missing(check, `citation map missing IDs: ${missingIds.join(", ")}`); +}; + +const assessForbiddenOutsideSections = (sections, check) => { + const allowed = new Set(check.allowed_sections ?? []); + const inspected = [...sections.entries()] + .filter(([heading]) => !allowed.has(heading)) + .map(([heading, body]) => `## ${heading}\n${body}`) + .join("\n\n"); + const forbidden = firstForbiddenIncluded( + inspected, + check.must_not_include_any ?? [], + ); + return forbidden + ? missing( + check, + `forbidden text outside allowed sections: ${forbidden}`, + "contradicted", + ) + : covered( + check, + "forbidden text appears only in allowed sections or not at all", + ); +}; + +const assessBlockingQuestions = (candidateText, check) => { + const questionCount = (candidateText.match(/\?/g) ?? []).length; + if (questionCount < (check.min_questions ?? 1)) { + return missing(check, `found ${questionCount} blocking question(s)`); + } + if (check.forbid_prd && containsPrdSectionHeading(candidateText)) { + return missing(check, "candidate emitted a PRD instead of blocking"); + } + return assessTextIncludes(candidateText, check); +}; + +const assessCheck = ({ candidateText, sections, rows, check }) => { + switch (check.mode) { + case "valid_prd_handoff": { + const result = validatePrdForHandoff(candidateText); + return result.valid + ? covered(check, "validatePrdForHandoff passed") + : missing( + check, + `handoff validation errors: ${result.errors + .map((error) => error.token) + .join(", ")}`, + ); + } + case "required_sections": { + const missingSections = REQUIRED_SECTIONS.filter( + (section) => !sections.has(section), + ); + return missingSections.length === 0 + ? covered(check, "all required sections present") + : missing(check, `missing sections: ${missingSections.join(", ")}`); + } + case "section_includes": + return assessSectionIncludes(candidateText, sections, check); + case "section_excludes": + return assessSectionExcludes(sections, check); + case "text_includes": + return assessTextIncludes(candidateText, check); + case "text_excludes": + return assessTextExcludes(candidateText, check); + case "ac_ids_present": + return assessAcIdsPresent(rows, check); + case "ac_ids_preserved": + return assessAcIdsPreserved(rows, check); + case "ac_status": + return assessAcStatus(rows, check); + case "assumptions_labeled": + return assessAssumptionsLabeled(sections, check); + case "downstream_cites_ids": + return assessDownstreamCitesIds(sections, check); + case "forbidden_outside_sections": + return assessForbiddenOutsideSections(sections, check); + case "blocking_questions": + return assessBlockingQuestions(candidateText, check); + default: + throw new Error(`unsupported check mode: ${check.mode}`); + } +}; + +export const gradeCandidate = ({ candidateText, expectedItems }) => { + const checks = validateExpectedItems(expectedItems, "expectedItems"); + const sections = extractSections(candidateText); + const rows = parseAcceptanceCriteriaRows(sections); + const findings = checks.map((check) => + assessCheck({ candidateText, sections, rows, check }), + ); return { findings, @@ -49,23 +680,54 @@ export const renderDeterministicReport = ({ caseId, grades, findings }) => ].join("\n"); export const validateFixtures = async ({ manifests }) => { + assertExpectedCheckValidationFailsClosed(); + assertBlockingQuestionMatcherBehavior(); + for (const item of manifests) { const graderInputs = item.manifest.artifacts.filter( (artifact) => artifact.role === "grader_input", ); - if (graderInputs.length === 0) { + if (graderInputs.length !== 1) { throw new Error( - `${item.relativePath} is missing a grader_input artifact`, + `${item.relativePath} must define exactly one grader_input artifact`, ); } + for (const role of [ + "generation_visible", + "candidate_good", + "candidate_bad", + ]) { + if (!item.manifest.artifacts.some((artifact) => artifact.role === role)) { + throw new Error(`${item.relativePath} is missing a ${role} artifact`); + } + } + + const expectedItemsPath = path.resolve( + path.dirname(item.fullPath), + graderInputs[0].path, + ); + const expectedItems = JSON.parse( + fs.readFileSync(expectedItemsPath, "utf8"), + ); + validateExpectedItems(expectedItems, expectedItemsPath); - for (const artifact of graderInputs) { - const artifactPath = path.resolve( + for (const artifact of item.manifest.artifacts) { + if (!["candidate_good", "candidate_bad"].includes(artifact.role)) { + continue; + } + const candidatePath = path.resolve( path.dirname(item.fullPath), artifact.path, ); - const expectedItems = JSON.parse(fs.readFileSync(artifactPath, "utf8")); - expectedItemsFor(expectedItems, artifactPath); + const candidateText = fs.readFileSync(candidatePath, "utf8"); + const grades = gradeCandidate({ candidateText, expectedItems }); + const expectedVerdict = + artifact.role === "candidate_good" ? "green" : "red"; + if (grades.verdict !== expectedVerdict) { + throw new Error( + `${item.relativePath} ${artifact.path} expected ${expectedVerdict}, got ${grades.verdict}`, + ); + } } } }; diff --git a/evals/cases/README.md b/evals/cases/README.md index a6e6cfc..8261ced 100644 --- a/evals/cases/README.md +++ b/evals/cases/README.md @@ -1,8 +1,20 @@ # Eval Cases -This directory will hold define-product-owned eval cases in follow-up PRs. Each case owns its -input material, expected fixture data, rubric, bad-candidate examples, and manifest. +Each case owns its input material, expected fixture data, rubric, good candidate, bad candidate, and +manifest. The deterministic adapter grades local Markdown candidates only; model-assisted generation +and judging remain disabled/manual. -This bootstrap PR intentionally does not add semantic PRD cases. The first case batch should cover -PRD contract completeness, AC-ID stability, grounding labels, downstream citeability, and product -altitude. +## Case Portfolio + +- `case-contract-grounded-prd-v1` checks complete PRD contract shape, clear audience/outcome, + source-grounded uncertainty labels, non-goals, and downstream citation. +- `case-revision-preserves-ac-ids-v1` checks stable AC-ID preservation during revision and one new ID + for new scope. +- `case-insufficient-input-blocks-v1` checks fail-closed blocking questions when source material + cannot support a coherent PRD. +- `case-altitude-and-future-scope-v1` checks product altitude, design/runtime leakage rejection, and + current-vs-future scope separation. + +`pnpm eval:validate-fixtures` validates manifest shape, expected-check schema, and that every +committed `candidate_good` grades green while every `candidate_bad` grades red. Do not add these +semantic eval runs to `pnpm check`; they are local, on-demand verification evidence. diff --git a/evals/cases/case-altitude-and-future-scope-v1/candidate-bad.md b/evals/cases/case-altitude-and-future-scope-v1/candidate-bad.md new file mode 100644 index 0000000..e0fbc08 --- /dev/null +++ b/evals/cases/case-altitude-and-future-scope-v1/candidate-bad.md @@ -0,0 +1,34 @@ +# High-Value Refund Review PRD + +## Product Outcome + +Billing operations gets a React app backed by a Postgres schema and CLI import so Jig execution can +start after two sprints. + +## User Job + +As a billing operations reviewer, I need a React queue so engineering can build the right screen. + +## Acceptance Criteria + +| ID | Criterion | Status | +| --------------- | -------------------------------------------------------------------------- | ------ | +| AC-QUEUE-001 | The React app displays high-value refunds from the Postgres schema. | Active | +| AC-EVIDENCE-001 | A CLI import loads evidence into the schema. | Active | +| AC-SCOPE-001 | Jig execution begins after two sprints with mobile approvals planned next. | Active | + +## Constraints + +- Use React, Postgres, a CLI import, and a two-sprint delivery sequence. + +## Assumptions + +- Inferred: Engineering wants these technologies. + +## Non-Goals + +- None. + +## Downstream Citation Map + +Technical Design and Planning may cite `AC-QUEUE-001`, `AC-EVIDENCE-001`, and `AC-SCOPE-001`. diff --git a/evals/cases/case-altitude-and-future-scope-v1/candidate-good.md b/evals/cases/case-altitude-and-future-scope-v1/candidate-good.md new file mode 100644 index 0000000..a47cf7c --- /dev/null +++ b/evals/cases/case-altitude-and-future-scope-v1/candidate-good.md @@ -0,0 +1,48 @@ +# High-Value Refund Review PRD + +## Product Outcome + +Billing operations reviewers can use a manual refund review queue to decide whether a high-value +refund is ready for approval. + +## User Job + +As a billing operations reviewer, I need to see each high-value refund, its stated reason, and +whether required evidence has been supplied, so I can make an approval-readiness decision. + +## Acceptance Criteria + +| ID | Criterion | Status | +| --------------- | --------------------------------------------------------------------------------------------------------------- | ------ | +| AC-QUEUE-001 | Billing operations reviewers can see each high-value refund that needs manual review in one review queue. | Active | +| AC-EVIDENCE-001 | Each queued refund shows its reason and whether required evidence has been supplied. | Active | +| AC-SCOPE-001 | The PRD separates current manual-review scope from future mobile approvals, auto-approval rules, and analytics. | Active | + +## Constraints + +- Success must be recognizable to a billing operations reviewer without reading implementation + details. +- The PRD describes product scope only and does not prescribe how the queue is built or delivered. + +## Assumptions + +- Inferred: The current owner priority is approval-readiness review, not automated approval. +- Gap/default: The exact threshold for "high-value refund" is not provided and must be supplied + before implementation planning. +- Gap/default: Technical stakeholder suggestions such as React, Postgres schema, CLI import, two + sprints, and Jig execution are design or delivery material rather than product commitments. + +## Non-Goals + +- No mobile approvals in the current scope. +- No auto-approval rules in the current scope. +- No manager analytics integration in the current scope. +- No commitment to a React implementation, Postgres schema, CLI import, two-sprint delivery plan, or + Jig execution mechanics. + +## Downstream Citation Map + +Technical Design and Planning may cite the Product Outcome, User Job, constraints, assumptions, +non-goals, and exact IDs `AC-QUEUE-001`, `AC-EVIDENCE-001`, and `AC-SCOPE-001`. They must preserve +the high-value threshold as a gap/default and must not treat the technical stakeholder suggestions as +product requirements. diff --git a/evals/cases/case-altitude-and-future-scope-v1/case-manifest.json b/evals/cases/case-altitude-and-future-scope-v1/case-manifest.json new file mode 100644 index 0000000..ec9e127 --- /dev/null +++ b/evals/cases/case-altitude-and-future-scope-v1/case-manifest.json @@ -0,0 +1,16 @@ +{ + "$schema": "../../../node_modules/@agentic-workflow-kit/eval-kit/schemas/case-manifest.schema.json", + "schema_version": "eval-kit.case.v1", + "case_id": "case-altitude-and-future-scope-v1", + "case_type": "deterministic", + "artifacts": [ + { "role": "generation_visible", "path": "input.md" }, + { "role": "grader_input", "path": "expected-items.json" }, + { "role": "rubric", "path": "rubric.md" }, + { "role": "candidate_good", "path": "candidate-good.md" }, + { "role": "candidate_bad", "path": "candidate-bad.md" } + ], + "metadata": { + "purpose": "Checks product altitude, design/runtime leakage rejection, and separation of current scope from future work." + } +} diff --git a/evals/cases/case-altitude-and-future-scope-v1/expected-items.json b/evals/cases/case-altitude-and-future-scope-v1/expected-items.json new file mode 100644 index 0000000..58f4eb2 --- /dev/null +++ b/evals/cases/case-altitude-and-future-scope-v1/expected-items.json @@ -0,0 +1,72 @@ +{ + "schema_version": "define-product.expected-items.v1", + "checks": [ + { + "id": "contract-handoff-valid", + "mode": "valid_prd_handoff", + "kind": "contract", + "severity": "critical" + }, + { + "id": "outcome-stays-product-altitude", + "mode": "section_includes", + "section": "Product Outcome", + "kind": "product-altitude", + "severity": "high", + "must_include_all": [ + "billing operations reviewers", + "manual refund review queue", + "ready for approval" + ], + "must_not_include_any": [ + "React", + "Postgres", + "CLI", + "schema", + "sprint", + "Jig" + ] + }, + { + "id": "expected-ac-ids", + "mode": "ac_ids_present", + "kind": "citation", + "severity": "critical", + "ids": ["AC-QUEUE-001", "AC-EVIDENCE-001", "AC-SCOPE-001"] + }, + { + "id": "future-work-is-not-current-scope", + "mode": "section_includes", + "section": "Non-Goals", + "kind": "scope", + "severity": "critical", + "must_include_all": [ + "mobile approvals", + "auto-approval rules", + "manager analytics" + ] + }, + { + "id": "design-leakage-contained", + "mode": "forbidden_outside_sections", + "kind": "product-altitude", + "severity": "critical", + "allowed_sections": ["Assumptions", "Non-Goals"], + "must_not_include_any": [ + "React", + "Postgres", + "CLI", + "schema", + "two sprints", + "Jig execution" + ] + }, + { + "id": "downstream-map-cites-product-ids", + "mode": "downstream_cites_ids", + "kind": "citation", + "severity": "critical", + "ids": ["AC-QUEUE-001", "AC-EVIDENCE-001", "AC-SCOPE-001"] + } + ] +} diff --git a/evals/cases/case-altitude-and-future-scope-v1/input.md b/evals/cases/case-altitude-and-future-scope-v1/input.md new file mode 100644 index 0000000..a9f8900 --- /dev/null +++ b/evals/cases/case-altitude-and-future-scope-v1/input.md @@ -0,0 +1,18 @@ +# Source Material + +Product owner notes: + +- Billing operations reviewers need a manual refund review queue so they can decide whether a + high-value refund is ready for approval. +- Current scope: reviewers can see each high-value refund, its reason, and whether required evidence + has been supplied. +- Future work: mobile approvals, auto-approval rules, and integration with manager analytics. +- A technical stakeholder suggested "React app, Postgres schema, CLI import, two sprints, then Jig + execution." Treat that as design/delivery material, not product scope. +- Success should be recognizable to a reviewer without reading implementation details. + +Task: + +Draft the product PRD. Keep product sections free of architecture, package layout, CLI behavior, +schema internals, delivery sequencing, downstream tool mechanics, and runtime details. Separate +current scope from future work. diff --git a/evals/cases/case-altitude-and-future-scope-v1/rubric.md b/evals/cases/case-altitude-and-future-scope-v1/rubric.md new file mode 100644 index 0000000..3870bfa --- /dev/null +++ b/evals/cases/case-altitude-and-future-scope-v1/rubric.md @@ -0,0 +1,10 @@ +# Rubric + +This case tests altitude discipline and current/future scope separation. + +The candidate should turn the billing-operations notes into product outcomes, user jobs, and +externally recognizable acceptance criteria. It may mention technical stakeholder suggestions only as +out-of-scope design/delivery material, not as PRD commitments. + +The candidate should fail if architecture, CLI, schema, delivery sequencing, or Jig mechanics appear +as product requirements or acceptance criteria. diff --git a/evals/cases/case-contract-grounded-prd-v1/candidate-bad.md b/evals/cases/case-contract-grounded-prd-v1/candidate-bad.md new file mode 100644 index 0000000..5624321 --- /dev/null +++ b/evals/cases/case-contract-grounded-prd-v1/candidate-bad.md @@ -0,0 +1,33 @@ +# Refund Center PRD + +## Product Outcome + +Customers can see refunds faster in a new refund center. + +## User Job + +As a customer, I want my refund resolved. + +## Acceptance Criteria + +| ID | Criterion | Status | +| ------------- | --------------------------------------------------- | ------ | +| AC-REFUND-001 | The system has refund details. | Active | +| AC-CITE-001 | Teams can read this PRD. | Active | +| AC-GROUND-001 | Legal and operations issues are handled internally. | Active | + +## Constraints + +- Build a dashboard. + +## Assumptions + +- Refunds older than 30 days are automatically resolved. + +## Non-Goals + +- None. + +## Downstream Citation Map + +- Cite this document. diff --git a/evals/cases/case-contract-grounded-prd-v1/candidate-good.md b/evals/cases/case-contract-grounded-prd-v1/candidate-good.md new file mode 100644 index 0000000..af119a7 --- /dev/null +++ b/evals/cases/case-contract-grounded-prd-v1/candidate-good.md @@ -0,0 +1,48 @@ +# Refund Status Assist PRD + +## Product Outcome + +Support agents can answer a customer refund-status question in under two minutes from one internal +view that summarizes refund status and the next owner-visible step. + +## User Job + +As a support agent, I need to answer a customer refund-status question without searching the order +tool, payment processor, and support history separately, so I can give the customer a clear status +and next step during the support interaction. + +## Acceptance Criteria + +| ID | Criterion | Status | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------- | ------ | +| AC-REFUND-001 | A support agent can identify refund status and the next owner-visible step from one internal view in under two minutes. | Active | +| AC-CITE-001 | Technical Design and Planning can cite the refund-status outcome and acceptance-criteria IDs without reading Product internals. | Active | +| AC-GROUND-001 | The PRD keeps legal wording approval, manual exception uncertainty, and CSV-export disagreement visible as uncertainty. | Active | +| AC-SCOPE-001 | The first release excludes automated refund issuance and agent performance analytics. | Active | + +## Constraints + +- The first release is an internal support-agent experience. +- Customer-facing status wording must not be treated as settled until Legal approves it. +- The PRD stays at product altitude and does not prescribe implementation mechanics. + +## Assumptions + +- Inferred: One internal view is sufficient to replace the three-system lookup for the first release. +- Gap/default: Refunds older than 30 days may need a manual exception path, but the owner has not + settled the product rule yet. +- Conflict: Manager CSV export was requested by one stakeholder, while the support lead says export + is useful later but should not block the first release. + +## Non-Goals + +- No automated refund issuance. +- No agent performance analytics. +- No manager CSV export in the first release unless the owner explicitly changes scope. + +## Downstream Citation Map + +Technical Design and Planning may cite the Product Outcome, User Job, constraints, assumptions, +non-goals, and exact IDs `AC-REFUND-001`, `AC-CITE-001`, `AC-GROUND-001`, and `AC-SCOPE-001`. +They must preserve the legal wording approval, manual exception path, and CSV-export disagreement as +uncertainty rather than settled product fact. diff --git a/evals/cases/case-contract-grounded-prd-v1/case-manifest.json b/evals/cases/case-contract-grounded-prd-v1/case-manifest.json new file mode 100644 index 0000000..2fd9377 --- /dev/null +++ b/evals/cases/case-contract-grounded-prd-v1/case-manifest.json @@ -0,0 +1,16 @@ +{ + "$schema": "../../../node_modules/@agentic-workflow-kit/eval-kit/schemas/case-manifest.schema.json", + "schema_version": "eval-kit.case.v1", + "case_id": "case-contract-grounded-prd-v1", + "case_type": "deterministic", + "artifacts": [ + { "role": "generation_visible", "path": "input.md" }, + { "role": "grader_input", "path": "expected-items.json" }, + { "role": "rubric", "path": "rubric.md" }, + { "role": "candidate_good", "path": "candidate-good.md" }, + { "role": "candidate_bad", "path": "candidate-bad.md" } + ], + "metadata": { + "purpose": "Checks a complete, grounded PRD with all contract sections, citable AC IDs, labeled uncertainty, and explicit non-goals." + } +} diff --git a/evals/cases/case-contract-grounded-prd-v1/expected-items.json b/evals/cases/case-contract-grounded-prd-v1/expected-items.json new file mode 100644 index 0000000..6496164 --- /dev/null +++ b/evals/cases/case-contract-grounded-prd-v1/expected-items.json @@ -0,0 +1,73 @@ +{ + "schema_version": "define-product.expected-items.v1", + "checks": [ + { + "id": "contract-handoff-valid", + "mode": "valid_prd_handoff", + "kind": "contract", + "severity": "critical" + }, + { + "id": "required-sections-present", + "mode": "required_sections", + "kind": "contract", + "severity": "critical" + }, + { + "id": "outcome-grounded-in-source", + "mode": "section_includes", + "section": "Product Outcome", + "kind": "grounding", + "severity": "high", + "must_include_all": [ + "support agents", + "refund status", + "under two minutes" + ] + }, + { + "id": "user-job-clear", + "mode": "section_includes", + "section": "User Job", + "kind": "product-altitude", + "severity": "high", + "must_include_all": [ + "support agent", + "answer", + "customer refund-status question" + ] + }, + { + "id": "expected-ac-ids", + "mode": "ac_ids_present", + "kind": "citation", + "severity": "critical", + "ids": ["AC-REFUND-001", "AC-CITE-001", "AC-GROUND-001", "AC-SCOPE-001"] + }, + { + "id": "visible-grounding-labels", + "mode": "assumptions_labeled", + "kind": "grounding", + "severity": "critical", + "labels": ["Inferred", "Gap/default", "Conflict"] + }, + { + "id": "non-goals-bound-scope", + "mode": "section_includes", + "section": "Non-Goals", + "kind": "scope", + "severity": "high", + "must_include_all": [ + "automated refund issuance", + "agent performance analytics" + ] + }, + { + "id": "downstream-map-cites-exact-ids", + "mode": "downstream_cites_ids", + "kind": "citation", + "severity": "critical", + "ids": ["AC-REFUND-001", "AC-CITE-001", "AC-GROUND-001", "AC-SCOPE-001"] + } + ] +} diff --git a/evals/cases/case-contract-grounded-prd-v1/input.md b/evals/cases/case-contract-grounded-prd-v1/input.md new file mode 100644 index 0000000..1854f41 --- /dev/null +++ b/evals/cases/case-contract-grounded-prd-v1/input.md @@ -0,0 +1,22 @@ +# Source Material + +Product owner notes: + +- Support agents spend too long answering customer refund-status questions because they search the + order tool, payment processor, and support history separately. +- The first release should give agents one internal view that summarizes refund status and the next + owner-visible step. +- Success means an agent can answer a refund-status question in under two minutes using the PRD's + published product facts. +- Do not automate refund issuance. Do not add agent performance analytics. +- Legal says customer-facing status wording still needs approval. +- Operations says refunds older than 30 days may need a manual exception path, but the details are + not settled. +- Conflicting notes: one stakeholder asked for manager CSV export in the first release; the support + lead says export is useful later but should not block the first release. + +Task: + +Draft a contract-conformant PRD at product altitude. Preserve visible grounding for inferred facts, +gaps/defaults, and conflicts. Include a downstream citation map that Technical Design and Planning +can cite without reading implementation internals. diff --git a/evals/cases/case-contract-grounded-prd-v1/rubric.md b/evals/cases/case-contract-grounded-prd-v1/rubric.md new file mode 100644 index 0000000..8533e79 --- /dev/null +++ b/evals/cases/case-contract-grounded-prd-v1/rubric.md @@ -0,0 +1,15 @@ +# Rubric + +This case rewards a complete PRD that is grounded in the source material and handoff-ready under +`prd-kit`. + +The candidate should: + +- include every required PRD contract section; +- write acceptance criteria with stable IDs; +- preserve uncertainty with visible `Inferred:`, `Gap/default:`, and `Conflict:` labels; +- name the current non-goals instead of letting downstream layers invent scope; +- include a downstream citation map with exact AC IDs. + +The candidate should fail if it emits a plausible PRD that hides source uncertainty, lacks a +citation map, or omits the non-goals that bound the first release. diff --git a/evals/cases/case-insufficient-input-blocks-v1/candidate-bad.md b/evals/cases/case-insufficient-input-blocks-v1/candidate-bad.md new file mode 100644 index 0000000..730543b --- /dev/null +++ b/evals/cases/case-insufficient-input-blocks-v1/candidate-bad.md @@ -0,0 +1,31 @@ +# Onboarding Dashboard PRD + +## Product Outcome + +New customers complete onboarding faster through an onboarding dashboard with Slack reminders. + +## User Job + +As a new customer, I need guided setup tasks. + +## Acceptance Criteria + +| ID | Criterion | Status | +| -------------- | --------------------------------------------------------- | ------ | +| AC-ONBOARD-001 | The onboarding dashboard improves completion rate by 25%. | Active | + +## Constraints + +- Use Slack reminders. + +## Assumptions + +- Inferred: Completion rate is the right metric. + +## Non-Goals + +- None. + +## Downstream Citation Map + +Technical Design may cite `AC-ONBOARD-001`. diff --git a/evals/cases/case-insufficient-input-blocks-v1/candidate-good.md b/evals/cases/case-insufficient-input-blocks-v1/candidate-good.md new file mode 100644 index 0000000..8bddcdb --- /dev/null +++ b/evals/cases/case-insufficient-input-blocks-v1/candidate-good.md @@ -0,0 +1,8 @@ +I cannot draft a coherent PRD from this source without inventing product facts. Blocking questions: + +1. Who is the user or audience whose onboarding is confusing? +2. What outcome should improve for that user after onboarding changes? +3. What recognizable success signal should the PRD use to tell whether onboarding is better? +4. What scope is explicitly out of bounds for this change? + +Once those are answered, remaining non-blocking unknowns can be recorded as assumptions in the PRD. diff --git a/evals/cases/case-insufficient-input-blocks-v1/case-manifest.json b/evals/cases/case-insufficient-input-blocks-v1/case-manifest.json new file mode 100644 index 0000000..9602795 --- /dev/null +++ b/evals/cases/case-insufficient-input-blocks-v1/case-manifest.json @@ -0,0 +1,16 @@ +{ + "$schema": "../../../node_modules/@agentic-workflow-kit/eval-kit/schemas/case-manifest.schema.json", + "schema_version": "eval-kit.case.v1", + "case_id": "case-insufficient-input-blocks-v1", + "case_type": "deterministic", + "artifacts": [ + { "role": "generation_visible", "path": "input.md" }, + { "role": "grader_input", "path": "expected-items.json" }, + { "role": "rubric", "path": "rubric.md" }, + { "role": "candidate_good", "path": "candidate-good.md" }, + { "role": "candidate_bad", "path": "candidate-bad.md" } + ], + "metadata": { + "purpose": "Checks fail-closed behavior when source material cannot support a coherent PRD." + } +} diff --git a/evals/cases/case-insufficient-input-blocks-v1/expected-items.json b/evals/cases/case-insufficient-input-blocks-v1/expected-items.json new file mode 100644 index 0000000..65af037 --- /dev/null +++ b/evals/cases/case-insufficient-input-blocks-v1/expected-items.json @@ -0,0 +1,42 @@ +{ + "schema_version": "define-product.expected-items.v1", + "checks": [ + { + "id": "blocks-instead-of-inventing-prd", + "mode": "blocking_questions", + "kind": "elicitation", + "severity": "critical", + "min_questions": 3, + "forbid_prd": true, + "must_include_all": ["blocking questions", "coherent PRD"], + "must_include_any": ["who is the user", "which user", "audience"], + "must_not_include_any": [ + "onboarding dashboard", + "Slack reminders", + "completion rate" + ] + }, + { + "id": "names-missing-success", + "mode": "text_includes", + "kind": "elicitation", + "severity": "high", + "must_include_any": [ + "recognizable success", + "success signal", + "what success means" + ] + }, + { + "id": "does-not-emit-template", + "mode": "text_excludes", + "kind": "contract", + "severity": "critical", + "must_not_include_any": [ + "## Product Outcome", + "## Acceptance Criteria", + "AC-ONBOARD-001" + ] + } + ] +} diff --git a/evals/cases/case-insufficient-input-blocks-v1/input.md b/evals/cases/case-insufficient-input-blocks-v1/input.md new file mode 100644 index 0000000..93ce7eb --- /dev/null +++ b/evals/cases/case-insufficient-input-blocks-v1/input.md @@ -0,0 +1,13 @@ +# Source Material + +Owner message: + +> We should make onboarding better. It is confusing right now. Please turn this into a PRD. + +No audience, target outcome, current workflow, success signal, constraints, assumptions, non-goals, +or downstream citation facts are supplied. + +Task: + +Apply define-product's blocking-only elicitation rule. If the source is insufficient for a coherent +PRD, stop and return blocking questions rather than inventing product facts. diff --git a/evals/cases/case-insufficient-input-blocks-v1/rubric.md b/evals/cases/case-insufficient-input-blocks-v1/rubric.md new file mode 100644 index 0000000..59b49d2 --- /dev/null +++ b/evals/cases/case-insufficient-input-blocks-v1/rubric.md @@ -0,0 +1,7 @@ +# Rubric + +This case verifies fail-closed elicitation. The source is too thin to know the user, target outcome, +success signal, or scope boundaries. + +The candidate should stop and ask only blocking questions. It should not draft a placeholder PRD, +invent onboarding channels, invent success metrics, or mint acceptance criteria. diff --git a/evals/cases/case-revision-preserves-ac-ids-v1/candidate-bad.md b/evals/cases/case-revision-preserves-ac-ids-v1/candidate-bad.md new file mode 100644 index 0000000..420bbdb --- /dev/null +++ b/evals/cases/case-revision-preserves-ac-ids-v1/candidate-bad.md @@ -0,0 +1,33 @@ +# Discovery Blocker Triage PRD + +## Product Outcome + +Product managers can see stale open discovery questions on one triage board before weekly planning. + +## User Job + +As a product manager, I need a planning dashboard. + +## Acceptance Criteria + +| ID | Criterion | Status | +| ------------- | ----------------------------------------------------------------------------- | ------ | +| AC-TRIAGE-001 | The dashboard shows blockers and exports them. | Active | +| AC-CITE-002 | Planning can cite the new dashboard requirement. | Active | +| AC-EXPORT-001 | Product managers can export the unresolved discovery blocker list for agenda. | Active | + +## Constraints + +- Use the new IDs because the document was revised. + +## Assumptions + +- Inferred: Renumbering is acceptable during revisions. + +## Non-Goals + +- None. + +## Downstream Citation Map + +Planning may cite `AC-EXPORT-001`. diff --git a/evals/cases/case-revision-preserves-ac-ids-v1/candidate-good.md b/evals/cases/case-revision-preserves-ac-ids-v1/candidate-good.md new file mode 100644 index 0000000..32f5327 --- /dev/null +++ b/evals/cases/case-revision-preserves-ac-ids-v1/candidate-good.md @@ -0,0 +1,38 @@ +# Discovery Blocker Triage PRD + +## Product Outcome + +Product managers can see stale open discovery questions on one triage board before weekly planning. + +## User Job + +As a product manager, I need to find unresolved discovery blockers before planning so I can decide +what needs owner follow-up and export the blocker list into the planning agenda. + +## Acceptance Criteria + +| ID | Criterion | Status | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------- | ------ | +| AC-BOARD-001 | The triage board shows unresolved discovery blockers that were not updated by the same business day before weekly planning. | Active | +| AC-CITE-001 | Technical Design and Planning can cite the triage-board outcome and exact blocker criteria without reinterpreting them locally. | Active | +| AC-EXPORT-001 | Product managers can export the unresolved discovery blocker list for a planning agenda. | Active | + +## Constraints + +- Existing published acceptance-criteria IDs remain stable when their meaning is unchanged. +- The export requirement is new scope and therefore receives a new acceptance-criteria ID. + +## Assumptions + +- Inferred: A same-business-day stale definition preserves the existing blocker-triage meaning while + making the review threshold explicit. + +## Non-Goals + +- No delivery sequencing or implementation mechanism for the export. +- No automatic owner follow-up workflow. + +## Downstream Citation Map + +Technical Design and Planning may cite `AC-BOARD-001`, `AC-CITE-001`, and `AC-EXPORT-001`, plus the +Product Outcome, User Job, constraints, assumptions, and non-goals above. diff --git a/evals/cases/case-revision-preserves-ac-ids-v1/case-manifest.json b/evals/cases/case-revision-preserves-ac-ids-v1/case-manifest.json new file mode 100644 index 0000000..696f4aa --- /dev/null +++ b/evals/cases/case-revision-preserves-ac-ids-v1/case-manifest.json @@ -0,0 +1,16 @@ +{ + "$schema": "../../../node_modules/@agentic-workflow-kit/eval-kit/schemas/case-manifest.schema.json", + "schema_version": "eval-kit.case.v1", + "case_id": "case-revision-preserves-ac-ids-v1", + "case_type": "deterministic", + "artifacts": [ + { "role": "generation_visible", "path": "input.md" }, + { "role": "grader_input", "path": "expected-items.json" }, + { "role": "rubric", "path": "rubric.md" }, + { "role": "candidate_good", "path": "candidate-good.md" }, + { "role": "candidate_bad", "path": "candidate-bad.md" } + ], + "metadata": { + "purpose": "Checks that revision keeps stable AC IDs when meaning is unchanged and adds a new ID for new scope." + } +} diff --git a/evals/cases/case-revision-preserves-ac-ids-v1/expected-items.json b/evals/cases/case-revision-preserves-ac-ids-v1/expected-items.json new file mode 100644 index 0000000..55141a7 --- /dev/null +++ b/evals/cases/case-revision-preserves-ac-ids-v1/expected-items.json @@ -0,0 +1,57 @@ +{ + "schema_version": "define-product.expected-items.v1", + "checks": [ + { + "id": "contract-handoff-valid", + "mode": "valid_prd_handoff", + "kind": "contract", + "severity": "critical" + }, + { + "id": "stable-and-new-ids-present", + "mode": "ac_ids_present", + "kind": "id-stability", + "severity": "critical", + "ids": ["AC-BOARD-001", "AC-CITE-001", "AC-EXPORT-001"] + }, + { + "id": "existing-id-meanings-preserved", + "mode": "ac_ids_preserved", + "kind": "id-stability", + "severity": "critical", + "rows": [ + { + "id": "AC-BOARD-001", + "must_include_all": [ + "triage board", + "unresolved discovery blockers", + "same business day" + ] + }, + { + "id": "AC-CITE-001", + "must_include_all": [ + "Technical Design and Planning", + "cite", + "triage-board outcome" + ] + } + ] + }, + { + "id": "new-export-id-active", + "mode": "ac_status", + "kind": "id-stability", + "severity": "high", + "id_ref": "AC-EXPORT-001", + "status": "Active" + }, + { + "id": "downstream-map-cites-revised-ids", + "mode": "downstream_cites_ids", + "kind": "citation", + "severity": "critical", + "ids": ["AC-BOARD-001", "AC-CITE-001", "AC-EXPORT-001"] + } + ] +} diff --git a/evals/cases/case-revision-preserves-ac-ids-v1/input.md b/evals/cases/case-revision-preserves-ac-ids-v1/input.md new file mode 100644 index 0000000..43d1769 --- /dev/null +++ b/evals/cases/case-revision-preserves-ac-ids-v1/input.md @@ -0,0 +1,31 @@ +# Source Material + +Existing published PRD excerpt: + +## Product Outcome + +Product managers can see stale open discovery questions on one triage board before weekly planning. + +## User Job + +As a product manager, I need to find unresolved discovery blockers before planning so I can decide +what needs owner follow-up. + +## Acceptance Criteria + +| ID | Criterion | Status | +| ------------ | ------------------------------------------------------------------------------------------- | ------ | +| AC-BOARD-001 | The triage board shows unresolved discovery blockers that are stale before weekly planning. | Active | +| AC-CITE-001 | Technical Design and Planning can cite the triage-board outcome and exact blocker criteria. | Active | + +Revision request: + +- Clarify "stale" to mean "not updated by the same business day before weekly planning." This is a + wording clarification, not a meaning change. +- Add a new product requirement: managers can export the unresolved blocker list for a planning + agenda. +- Preserve existing AC IDs when meaning stays stable. Add a new ID only for the export requirement. + +Task: + +Return the revised PRD. Do not renumber existing criteria. diff --git a/evals/cases/case-revision-preserves-ac-ids-v1/rubric.md b/evals/cases/case-revision-preserves-ac-ids-v1/rubric.md new file mode 100644 index 0000000..6de0cfd --- /dev/null +++ b/evals/cases/case-revision-preserves-ac-ids-v1/rubric.md @@ -0,0 +1,10 @@ +# Rubric + +This case focuses on the PRD contract's ID stability rule. + +The candidate should preserve `AC-BOARD-001` and `AC-CITE-001` because the revision says their +meaning is unchanged. It should clarify the stale-blocker wording in place and mint exactly one new +ID, `AC-EXPORT-001`, for the new export requirement. + +The candidate should fail if it renumbers old criteria, silently changes their meaning, or cites only +the new requirement downstream. diff --git a/package.json b/package.json index 0badc26..afb751a 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "eval:case": "eval-kit run-case --config evals/eval-kit.config.json" }, "devDependencies": { - "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.1", + "@agentic-workflow-kit/eval-kit": "github:agentic-workflow-kit/eval-kit#v0.1.2", "prettier": "^3.6.2" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1ccc7e5..f20757a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: devDependencies: '@agentic-workflow-kit/eval-kit': - specifier: github:agentic-workflow-kit/eval-kit#v0.1.1 - version: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/2d3bf2da4533fa4442e0ae2a5a512c915c546d89 + specifier: github:agentic-workflow-kit/eval-kit#v0.1.2 + version: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974 prettier: specifier: ^3.6.2 version: 3.9.3 @@ -26,9 +26,9 @@ importers: packages: - '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/2d3bf2da4533fa4442e0ae2a5a512c915c546d89': - resolution: {gitHosted: true, integrity: sha512-4sSEGmu2oZ0wiz3V0TEQ6qzq/G+YBppcfbKT3QRCWmakRx7kzqY93VFl5q9Wfw6GLUZ8OsETv8yiCQs3sGQZ3Q==, tarball: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/2d3bf2da4533fa4442e0ae2a5a512c915c546d89} - version: 0.1.1 + '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974': + resolution: {gitHosted: true, tarball: https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974} + version: 0.1.2 engines: {node: '>=22.13.0', pnpm: '>=11.9.0'} hasBin: true @@ -845,7 +845,7 @@ packages: snapshots: - '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/2d3bf2da4533fa4442e0ae2a5a512c915c546d89': + '@agentic-workflow-kit/eval-kit@https://codeload.github.com/agentic-workflow-kit/eval-kit/tar.gz/e45db38c0e5b151942ebdf579089d0bdd284f974': dependencies: ajv: 8.20.0