Skip to content

fix: align promptfoo adapter variables #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aryeko merged 1 commit into from
Jul 3, 2026
Merged

fix: align promptfoo adapter variables #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
222 changes: 128 additions & 94 deletions evals/adapter.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,64 +64,38 @@ const extractVersion = (promptText, label) => {
return match[1];
};

// Variable resolvers for Promptfoo

export const resolveGenerationVars = async ({
caseId,
caseDir,
artifacts,
resolver,
}) => {
const readText = (relPath) =>
fs.readFileSync(path.resolve(resolver.repoRoot, relPath), "utf8");
const readCaseText = ({ resolver, caseDir, file }) =>
fs.readFileSync(
path.resolve(resolver.repoRoot, resolver.relativeToRepo(caseDir), file),
"utf8",
);

return {
product_md: readText(
path.join(resolver.relativeToRepo(caseDir), "product.md"),
),
source_map_md: readText(
path.join(resolver.relativeToRepo(caseDir), "source-map.md"),
),
author_skill_md: readText("skills/author-technical-design/SKILL.md"),
technical_design_template_md: readText(
"methodologies/ddd/templates/technical-design.md",
),
bounded_context_template_md: readText(
"methodologies/ddd/templates/bounded-context.md",
),
enforcement_map_template_md: readText(
"methodologies/ddd/templates/enforcement-map.md",
),
ddd_eval_expectations_md: readText(
"methodologies/ddd/eval-expectations.md",
),
};
};
const readCaseJson = (input) => JSON.parse(readCaseText(input));

export const resolvePointwiseVars = async ({
caseId,
caseDir,
artifacts,
candidateContent,
candidatePath,
promptVersion,
rubricVersion,
model,
provider,
resolver,
}) => {
const readText = (relPath) =>
fs.readFileSync(path.resolve(resolver.repoRoot, relPath), "utf8");
const readJson = (relPath) => JSON.parse(readText(relPath));
const sourceMaterialForCase = ({ resolver, caseDir }) =>
[
"# Product Brief",
"",
readCaseText({ resolver, caseDir, file: "product.md" }),
"",
"# Source Map",
"",
readCaseText({ resolver, caseDir, file: "source-map.md" }),
].join("\n");

const expectedFacts = readJson(
path.join(resolver.relativeToRepo(caseDir), "expected-facts.json"),
);
const expectedBoundaries = readJson(
path.join(resolver.relativeToRepo(caseDir), "expected-boundaries.json"),
);
const expectedItemsForCase = ({ resolver, caseDir }) => {
const expectedFacts = readCaseJson({
resolver,
caseDir,
file: "expected-facts.json",
});
const expectedBoundaries = readCaseJson({
resolver,
caseDir,
file: "expected-boundaries.json",
});

const expectedItems = [
return [
...expectedFacts.facts.map((fact) => ({
item_id: fact.id,
kind: "fact",
Expand Down Expand Up @@ -159,22 +133,94 @@ export const resolvePointwiseVars = async ({
required_concepts: context.required_concepts ?? [],
})),
];
};

// Variable resolvers for Promptfoo

export const resolveGenerationVars = async ({
caseId,
caseDir,
artifacts,
resolver,
}) => {
const readText = (relPath) =>
fs.readFileSync(path.resolve(resolver.repoRoot, relPath), "utf8");
const sourceMaterial = sourceMaterialForCase({ resolver, caseDir });
const authorSkill = readText("skills/author-technical-design/SKILL.md");
const technicalDesignTemplate = readText(
"methodologies/ddd/templates/technical-design.md",
);
const boundedContextTemplate = readText(
"methodologies/ddd/templates/bounded-context.md",
);
const enforcementMapTemplate = readText(
"methodologies/ddd/templates/enforcement-map.md",
);
const dddEvalExpectations = readText(
"methodologies/ddd/eval-expectations.md",
);
const candidateInstructions = [
"# Author Skill",
"",
authorSkill,
"",
"# DDD Technical Design Template",
"",
technicalDesignTemplate,
"",
"# Bounded Context Template",
"",
boundedContextTemplate,
"",
"# Enforcement Map Template",
"",
enforcementMapTemplate,
"",
"# DDD Eval Expectations",
"",
dddEvalExpectations,
].join("\n");

return {
source_material: sourceMaterial,
candidate_instructions: candidateInstructions,
output_format:
"Markdown technical design document following the DDD technical-design and bounded-context templates.",
product_md: readCaseText({ resolver, caseDir, file: "product.md" }),
source_map_md: readCaseText({ resolver, caseDir, file: "source-map.md" }),
author_skill_md: authorSkill,
technical_design_template_md: technicalDesignTemplate,
bounded_context_template_md: boundedContextTemplate,
enforcement_map_template_md: enforcementMapTemplate,
ddd_eval_expectations_md: dddEvalExpectations,
};
};

export const resolvePointwiseVars = async ({
caseId,
caseDir,
artifacts,
candidateContent,
candidatePath,
promptVersion,
rubricVersion,
model,
provider,
resolver,
}) => {
const readText = (relPath) =>
fs.readFileSync(path.resolve(resolver.repoRoot, relPath), "utf8");
const sourceMaterial = sourceMaterialForCase({ resolver, caseDir });
const expectedItems = expectedItemsForCase({ resolver, caseDir });

return {
case_id: caseId,
model,
provider,
prompt_version: promptVersion,
rubric_version: rubricVersion,
source_facts: [
"# Product Brief",
"",
readText(path.join(resolver.relativeToRepo(caseDir), "product.md")),
"",
"# Source Map",
"",
readText(path.join(resolver.relativeToRepo(caseDir), "source-map.md")),
].join("\n"),
source_material: sourceMaterial,
source_facts: sourceMaterial,
case_rubric: readText(
path.join(resolver.relativeToRepo(caseDir), "rubric.md"),
),
Expand Down Expand Up @@ -223,47 +269,35 @@ export const resolvePairwiseVars = async ({
}) => {
const readText = (relPath) =>
fs.readFileSync(path.resolve(resolver.repoRoot, relPath), "utf8");
const readJson = (relPath) => JSON.parse(readText(relPath));

const expectedFacts = readJson(
path.join(resolver.relativeToRepo(caseDir), "expected-facts.json"),
);
const expectedBoundaries = readJson(
path.join(resolver.relativeToRepo(caseDir), "expected-boundaries.json"),
);

const [firstKey, secondKey] = randomizedOrder.candidate_order;
const candidate_a =
firstKey === "candidate_a" ? candidateAContent : candidateBContent;
const candidate_b =
firstKey === "candidate_a" ? candidateBContent : candidateAContent;
const candidate_a_path =
firstKey === "candidate_a" ? candidateAPath : candidateBPath;
const candidate_b_path =
firstKey === "candidate_a" ? candidateBPath : candidateAPath;
const expectedFacts = readCaseJson({
resolver,
caseDir,
file: "expected-facts.json",
});
const expectedBoundaries = readCaseJson({
resolver,
caseDir,
file: "expected-boundaries.json",
});
const sourceMaterial = sourceMaterialForCase({ resolver, caseDir });
const expectedItems = expectedItemsForCase({ resolver, caseDir });

return {
case_id: caseId,
model,
provider,
prompt_version: promptVersion,
rubric_version: rubricVersion,
source_facts: [
"# Product Brief",
"",
readText(path.join(resolver.relativeToRepo(caseDir), "product.md")),
"",
"# Source Map",
"",
readText(path.join(resolver.relativeToRepo(caseDir), "source-map.md")),
].join("\n"),
source_material: sourceMaterial,
source_facts: sourceMaterial,
case_rubric: readText(
path.join(resolver.relativeToRepo(caseDir), "rubric.md"),
),
candidate_a_path: resolver.relativeToRepo(candidate_a_path),
candidate_b_path: resolver.relativeToRepo(candidate_b_path),
candidate_a,
candidate_b,
expected_items: JSON.stringify(expectedItems, null, 2),
candidate_a_path: resolver.relativeToRepo(candidateAPath),
candidate_b_path: resolver.relativeToRepo(candidateBPath),
candidate_a: candidateAContent,
candidate_b: candidateBContent,
Comment thread
aryeko marked this conversation as resolved.
expected_facts: JSON.stringify(
{
facts: expectedFacts.facts,
Expand Down
108 changes: 108 additions & 0 deletions evals/tests/public-reference-cases.test.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import { fileURLToPath } from "node:url";
import { gradeFacts, loadConfig } from "@agentic-workflow-kit/eval-kit";
import {
gradeBoundaries,
resolveGenerationVars,
resolvePairwiseVars,
resolvePointwiseVars,
verdictForFindings,
} from "../adapter.mjs";
Expand All @@ -28,6 +30,13 @@ const findingsForCaseCandidate = (caseId, candidate) => [
...gradeBoundaries(candidate, readJson(caseId, "expected-boundaries.json")),
];

const expectKeys = (value, keys) => {
for (const key of keys) {
expect(value, `missing ${key}`).toHaveProperty(key);
expect(value[key], key).toBeDefined();
}
};

const newPublicCaseIds = [
"case-cloudevents-core-contract-v1",
"case-openfeature-evaluation-api-v1",
Expand Down Expand Up @@ -70,6 +79,105 @@ describe("public reference cases", () => {
expect(vars.case_rubric).toContain("allows overlapping active bookings");
});

it("returns generic bundled generation prompt variables", async () => {
const config = loadConfig(path.join(packageRoot, "eval-kit.config.json"));
const caseId = "case-tiny-laundry-pickup-v1";
const caseDir = casePath(caseId);

const vars = await resolveGenerationVars({
caseId,
caseDir,
artifacts: [],
resolver: config.pathResolver,
});

expectKeys(vars, [
"source_material",
"candidate_instructions",
"output_format",
]);
expect(vars.source_material).toContain("# Product Brief");
expect(vars.candidate_instructions).toContain("author-technical-design");
expect(vars.output_format).toContain("technical design");
});

it("returns generic bundled pointwise prompt variables", async () => {
const config = loadConfig(path.join(packageRoot, "eval-kit.config.json"));
const caseId = "case-tiny-laundry-pickup-v1";
const caseDir = casePath(caseId);
const candidatePath = casePath(caseId, "reference-design.md");

const vars = await resolvePointwiseVars({
caseId,
caseDir,
artifacts: [],
candidateContent: readText(caseId, "reference-design.md"),
candidatePath,
promptVersion: "pointwise-prompt-v1",
rubricVersion: "pointwise-coverage-rubric-v1",
model: "test-model",
provider: "test-provider",
resolver: config.pathResolver,
});

expectKeys(vars, [
"source_material",
"case_rubric",
"expected_items",
"candidate_path",
"candidate",
]);
expect(vars.source_material).toContain("# Product Brief");
expect(vars.expected_items).toContain("FACT-001");
expect(vars.candidate_path).toBe(
"evals/cases/case-tiny-laundry-pickup-v1/reference-design.md",
);
});

it("returns generic bundled pairwise prompt variables", async () => {
const config = loadConfig(path.join(packageRoot, "eval-kit.config.json"));
const caseId = "case-tiny-laundry-pickup-v1";
const caseDir = casePath(caseId);
const candidatePath = casePath(caseId, "reference-design.md");

const vars = await resolvePairwiseVars({
caseId,
caseDir,
artifacts: [],
candidateAContent: "Displayed Candidate A",
candidateBContent: "Displayed Candidate B",
candidateAPath: candidatePath,
candidateBPath: candidatePath,
promptVersion: "pairwise-prompt-v1",
rubricVersion: "judge-rubric-v1",
model: "test-model",
provider: "test-provider",
randomizedOrder: {
method: "test",
seed: 42,
original_order: ["candidate_a", "candidate_b"],
candidate_order: ["candidate_b", "candidate_a"],
},
resolver: config.pathResolver,
});

expectKeys(vars, [
"source_material",
"case_rubric",
"expected_items",
"candidate_a",
"candidate_b",
"randomization_method",
"randomization_seed",
"original_order",
"candidate_order",
]);
expect(vars.source_material).toContain("# Source Map");
expect(vars.expected_items).toContain("FACT-001");
expect(vars.candidate_a).toBe("Displayed Candidate A");
expect(vars.candidate_b).toBe("Displayed Candidate B");
});

it("rejects 2PC as a customer-credit saga contradiction", () => {
const expectedFacts = readJson(
"case-customer-credit-order-saga-v1",
Expand Down