From 777f3995ee4a38133ea4798cf97883f66aa069e7 Mon Sep 17 00:00:00 2001 From: Arye Kogan Date: Fri, 3 Jul 2026 18:06:27 +0300 Subject: [PATCH] fix: align promptfoo adapter variables --- evals/adapter.mjs | 222 +++++++++++--------- evals/tests/public-reference-cases.test.mjs | 108 ++++++++++ 2 files changed, 236 insertions(+), 94 deletions(-) diff --git a/evals/adapter.mjs b/evals/adapter.mjs index db6d63f..f2de30f 100644 --- a/evals/adapter.mjs +++ b/evals/adapter.mjs @@ -64,64 +64,38 @@ const extractVersion = (promptText, label) => { return match[1]; }; -// Variable resolvers for Promptfoo - -export const resolveGenerationVars = async ({ - caseId, - caseDir, - artifacts, - resolver, -}) => { - const readText = (relPath) => - fs.readFileSync(path.resolve(resolver.repoRoot, relPath), "utf8"); +const readCaseText = ({ resolver, caseDir, file }) => + fs.readFileSync( + path.resolve(resolver.repoRoot, resolver.relativeToRepo(caseDir), file), + "utf8", + ); - return { - product_md: readText( - path.join(resolver.relativeToRepo(caseDir), "product.md"), - ), - source_map_md: readText( - path.join(resolver.relativeToRepo(caseDir), "source-map.md"), - ), - author_skill_md: readText("skills/author-technical-design/SKILL.md"), - technical_design_template_md: readText( - "methodologies/ddd/templates/technical-design.md", - ), - bounded_context_template_md: readText( - "methodologies/ddd/templates/bounded-context.md", - ), - enforcement_map_template_md: readText( - "methodologies/ddd/templates/enforcement-map.md", - ), - ddd_eval_expectations_md: readText( - "methodologies/ddd/eval-expectations.md", - ), - }; -}; +const readCaseJson = (input) => JSON.parse(readCaseText(input)); -export const resolvePointwiseVars = async ({ - caseId, - caseDir, - artifacts, - candidateContent, - candidatePath, - promptVersion, - rubricVersion, - model, - provider, - resolver, -}) => { - const readText = (relPath) => - fs.readFileSync(path.resolve(resolver.repoRoot, relPath), "utf8"); - const readJson = (relPath) => JSON.parse(readText(relPath)); +const sourceMaterialForCase = ({ resolver, caseDir }) => + [ + "# Product Brief", + "", + readCaseText({ resolver, caseDir, file: "product.md" }), + "", + "# Source Map", + "", + readCaseText({ resolver, caseDir, file: "source-map.md" }), + ].join("\n"); - const expectedFacts = readJson( - path.join(resolver.relativeToRepo(caseDir), "expected-facts.json"), - ); - const expectedBoundaries = readJson( - path.join(resolver.relativeToRepo(caseDir), "expected-boundaries.json"), - ); +const expectedItemsForCase = ({ resolver, caseDir }) => { + const expectedFacts = readCaseJson({ + resolver, + caseDir, + file: "expected-facts.json", + }); + const expectedBoundaries = readCaseJson({ + resolver, + caseDir, + file: "expected-boundaries.json", + }); - const expectedItems = [ + return [ ...expectedFacts.facts.map((fact) => ({ item_id: fact.id, kind: "fact", @@ -159,6 +133,85 @@ export const resolvePointwiseVars = async ({ required_concepts: context.required_concepts ?? [], })), ]; +}; + +// Variable resolvers for Promptfoo + +export const resolveGenerationVars = async ({ + caseId, + caseDir, + artifacts, + resolver, +}) => { + const readText = (relPath) => + fs.readFileSync(path.resolve(resolver.repoRoot, relPath), "utf8"); + const sourceMaterial = sourceMaterialForCase({ resolver, caseDir }); + const authorSkill = readText("skills/author-technical-design/SKILL.md"); + const technicalDesignTemplate = readText( + "methodologies/ddd/templates/technical-design.md", + ); + const boundedContextTemplate = readText( + "methodologies/ddd/templates/bounded-context.md", + ); + const enforcementMapTemplate = readText( + "methodologies/ddd/templates/enforcement-map.md", + ); + const dddEvalExpectations = readText( + "methodologies/ddd/eval-expectations.md", + ); + const candidateInstructions = [ + "# Author Skill", + "", + authorSkill, + "", + "# DDD Technical Design Template", + "", + technicalDesignTemplate, + "", + "# Bounded Context Template", + "", + boundedContextTemplate, + "", + "# Enforcement Map Template", + "", + enforcementMapTemplate, + "", + "# DDD Eval Expectations", + "", + dddEvalExpectations, + ].join("\n"); + + return { + source_material: sourceMaterial, + candidate_instructions: candidateInstructions, + output_format: + "Markdown technical design document following the DDD technical-design and bounded-context templates.", + product_md: readCaseText({ resolver, caseDir, file: "product.md" }), + source_map_md: readCaseText({ resolver, caseDir, file: "source-map.md" }), + author_skill_md: authorSkill, + technical_design_template_md: technicalDesignTemplate, + bounded_context_template_md: boundedContextTemplate, + enforcement_map_template_md: enforcementMapTemplate, + ddd_eval_expectations_md: dddEvalExpectations, + }; +}; + +export const resolvePointwiseVars = async ({ + caseId, + caseDir, + artifacts, + candidateContent, + candidatePath, + promptVersion, + rubricVersion, + model, + provider, + resolver, +}) => { + const readText = (relPath) => + fs.readFileSync(path.resolve(resolver.repoRoot, relPath), "utf8"); + const sourceMaterial = sourceMaterialForCase({ resolver, caseDir }); + const expectedItems = expectedItemsForCase({ resolver, caseDir }); return { case_id: caseId, @@ -166,15 +219,8 @@ export const resolvePointwiseVars = async ({ provider, prompt_version: promptVersion, rubric_version: rubricVersion, - source_facts: [ - "# Product Brief", - "", - readText(path.join(resolver.relativeToRepo(caseDir), "product.md")), - "", - "# Source Map", - "", - readText(path.join(resolver.relativeToRepo(caseDir), "source-map.md")), - ].join("\n"), + source_material: sourceMaterial, + source_facts: sourceMaterial, case_rubric: readText( path.join(resolver.relativeToRepo(caseDir), "rubric.md"), ), @@ -223,24 +269,18 @@ export const resolvePairwiseVars = async ({ }) => { const readText = (relPath) => fs.readFileSync(path.resolve(resolver.repoRoot, relPath), "utf8"); - const readJson = (relPath) => JSON.parse(readText(relPath)); - - const expectedFacts = readJson( - path.join(resolver.relativeToRepo(caseDir), "expected-facts.json"), - ); - const expectedBoundaries = readJson( - path.join(resolver.relativeToRepo(caseDir), "expected-boundaries.json"), - ); - - const [firstKey, secondKey] = randomizedOrder.candidate_order; - const candidate_a = - firstKey === "candidate_a" ? candidateAContent : candidateBContent; - const candidate_b = - firstKey === "candidate_a" ? candidateBContent : candidateAContent; - const candidate_a_path = - firstKey === "candidate_a" ? candidateAPath : candidateBPath; - const candidate_b_path = - firstKey === "candidate_a" ? candidateBPath : candidateAPath; + const expectedFacts = readCaseJson({ + resolver, + caseDir, + file: "expected-facts.json", + }); + const expectedBoundaries = readCaseJson({ + resolver, + caseDir, + file: "expected-boundaries.json", + }); + const sourceMaterial = sourceMaterialForCase({ resolver, caseDir }); + const expectedItems = expectedItemsForCase({ resolver, caseDir }); return { case_id: caseId, @@ -248,22 +288,16 @@ export const resolvePairwiseVars = async ({ provider, prompt_version: promptVersion, rubric_version: rubricVersion, - source_facts: [ - "# Product Brief", - "", - readText(path.join(resolver.relativeToRepo(caseDir), "product.md")), - "", - "# Source Map", - "", - readText(path.join(resolver.relativeToRepo(caseDir), "source-map.md")), - ].join("\n"), + source_material: sourceMaterial, + source_facts: sourceMaterial, case_rubric: readText( path.join(resolver.relativeToRepo(caseDir), "rubric.md"), ), - candidate_a_path: resolver.relativeToRepo(candidate_a_path), - candidate_b_path: resolver.relativeToRepo(candidate_b_path), - candidate_a, - candidate_b, + expected_items: JSON.stringify(expectedItems, null, 2), + candidate_a_path: resolver.relativeToRepo(candidateAPath), + candidate_b_path: resolver.relativeToRepo(candidateBPath), + candidate_a: candidateAContent, + candidate_b: candidateBContent, expected_facts: JSON.stringify( { facts: expectedFacts.facts, diff --git a/evals/tests/public-reference-cases.test.mjs b/evals/tests/public-reference-cases.test.mjs index db51f1e..e1b754b 100644 --- a/evals/tests/public-reference-cases.test.mjs +++ b/evals/tests/public-reference-cases.test.mjs @@ -7,6 +7,8 @@ import { fileURLToPath } from "node:url"; import { gradeFacts, loadConfig } from "@agentic-workflow-kit/eval-kit"; import { gradeBoundaries, + resolveGenerationVars, + resolvePairwiseVars, resolvePointwiseVars, verdictForFindings, } from "../adapter.mjs"; @@ -28,6 +30,13 @@ const findingsForCaseCandidate = (caseId, candidate) => [ ...gradeBoundaries(candidate, readJson(caseId, "expected-boundaries.json")), ]; +const expectKeys = (value, keys) => { + for (const key of keys) { + expect(value, `missing ${key}`).toHaveProperty(key); + expect(value[key], key).toBeDefined(); + } +}; + const newPublicCaseIds = [ "case-cloudevents-core-contract-v1", "case-openfeature-evaluation-api-v1", @@ -70,6 +79,105 @@ describe("public reference cases", () => { expect(vars.case_rubric).toContain("allows overlapping active bookings"); }); + it("returns generic bundled generation prompt variables", async () => { + const config = loadConfig(path.join(packageRoot, "eval-kit.config.json")); + const caseId = "case-tiny-laundry-pickup-v1"; + const caseDir = casePath(caseId); + + const vars = await resolveGenerationVars({ + caseId, + caseDir, + artifacts: [], + resolver: config.pathResolver, + }); + + expectKeys(vars, [ + "source_material", + "candidate_instructions", + "output_format", + ]); + expect(vars.source_material).toContain("# Product Brief"); + expect(vars.candidate_instructions).toContain("author-technical-design"); + expect(vars.output_format).toContain("technical design"); + }); + + it("returns generic bundled pointwise prompt variables", async () => { + const config = loadConfig(path.join(packageRoot, "eval-kit.config.json")); + const caseId = "case-tiny-laundry-pickup-v1"; + const caseDir = casePath(caseId); + const candidatePath = casePath(caseId, "reference-design.md"); + + const vars = await resolvePointwiseVars({ + caseId, + caseDir, + artifacts: [], + candidateContent: readText(caseId, "reference-design.md"), + candidatePath, + promptVersion: "pointwise-prompt-v1", + rubricVersion: "pointwise-coverage-rubric-v1", + model: "test-model", + provider: "test-provider", + resolver: config.pathResolver, + }); + + expectKeys(vars, [ + "source_material", + "case_rubric", + "expected_items", + "candidate_path", + "candidate", + ]); + expect(vars.source_material).toContain("# Product Brief"); + expect(vars.expected_items).toContain("FACT-001"); + expect(vars.candidate_path).toBe( + "evals/cases/case-tiny-laundry-pickup-v1/reference-design.md", + ); + }); + + it("returns generic bundled pairwise prompt variables", async () => { + const config = loadConfig(path.join(packageRoot, "eval-kit.config.json")); + const caseId = "case-tiny-laundry-pickup-v1"; + const caseDir = casePath(caseId); + const candidatePath = casePath(caseId, "reference-design.md"); + + const vars = await resolvePairwiseVars({ + caseId, + caseDir, + artifacts: [], + candidateAContent: "Displayed Candidate A", + candidateBContent: "Displayed Candidate B", + candidateAPath: candidatePath, + candidateBPath: candidatePath, + promptVersion: "pairwise-prompt-v1", + rubricVersion: "judge-rubric-v1", + model: "test-model", + provider: "test-provider", + randomizedOrder: { + method: "test", + seed: 42, + original_order: ["candidate_a", "candidate_b"], + candidate_order: ["candidate_b", "candidate_a"], + }, + resolver: config.pathResolver, + }); + + expectKeys(vars, [ + "source_material", + "case_rubric", + "expected_items", + "candidate_a", + "candidate_b", + "randomization_method", + "randomization_seed", + "original_order", + "candidate_order", + ]); + expect(vars.source_material).toContain("# Source Map"); + expect(vars.expected_items).toContain("FACT-001"); + expect(vars.candidate_a).toBe("Displayed Candidate A"); + expect(vars.candidate_b).toBe("Displayed Candidate B"); + }); + it("rejects 2PC as a customer-credit saga contradiction", () => { const expectedFacts = readJson( "case-customer-credit-order-saga-v1",