From 88c79c6fb2d837895a8685b8894b595e3f91cc7b Mon Sep 17 00:00:00 2001 From: gitsad Date: Thu, 14 May 2026 11:44:26 +0200 Subject: [PATCH 01/26] feat: added more validation tests, and passed gpt-5.5 --- README.md | 1 + evals/assertions/fixer-contains-component.mjs | 135 ++++++ .../assertions/fixer-preserves-components.mjs | 11 +- evals/package.json | 4 +- evals/prompt-fixer.mjs | 17 +- evals/promptfooconfig.fixer-flow.yaml | 6 +- evals/promptfooconfig.fixer.yaml | 8 +- evals/select-prompt.mjs | 12 +- evals/tests-fixer-flow.yaml | 48 ++ evals/tests-fixer.yaml | 450 ++++++++++++++++-- package.json | 1 + packages/prompt-pack/src/index.ts | 4 +- packages/prompt-pack/src/loader.ts | 2 +- .../{mdma-fixer.ts => mdma-fixer/_shared.ts} | 205 +++++++- .../src/prompts/mdma-fixer/default.ts | 28 ++ .../src/prompts/mdma-fixer/openai/_shared.ts | 17 + .../src/prompts/mdma-fixer/openai/gpt-5.5.ts | 35 ++ pnpm-lock.yaml | 3 + 18 files changed, 915 insertions(+), 72 deletions(-) create mode 100644 evals/assertions/fixer-contains-component.mjs rename packages/prompt-pack/src/prompts/{mdma-fixer.ts => mdma-fixer/_shared.ts} (79%) create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/default.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/openai/_shared.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.5.ts diff --git a/README.md b/README.md index 85d9bb4..b27227b 100644 --- a/README.md +++ b/README.md @@ -586,6 +586,7 @@ pnpm eval:view - [x] Prompt tuning toolkit — test and compare custom prompts - [x] Agent-friendly SDK — let AI agent generate your MDMA - [ ] Validator evals +- [ ] Integrations - [ ] Webhook execution engine (real HTTP calls in production environments) ### v1.0 — Production Ready diff --git a/evals/assertions/fixer-contains-component.mjs b/evals/assertions/fixer-contains-component.mjs new file mode 100644 index 0000000..77f4245 --- /dev/null +++ b/evals/assertions/fixer-contains-component.mjs @@ -0,0 +1,135 @@ +import { parse } from 'yaml'; + +/** + * Custom promptfoo assertion for fixer eval. + * + * Finds a component in the fixed output and validates its fields against an + * expected MDMA block provided in config. + * + * config: + * expected: string — complete (or partial) MDMA block YAML to compare against. + * The `id` field in the expected block is used to locate the + * component in the output. Every field present in `expected` + * must match the actual component — extra fields in the + * actual output are ignored. + * hasFields: string[] — additional field names that must exist (any value). + * + * Example: + * config: + * expected: | + * type: webhook + * id: order-webhook + * url: https://api.example.com/orders + * method: POST + * trigger: order-form + */ +export default function (output, { config } = {}) { + const { expected: expectedYaml, hasFields = [] } = config ?? {}; + + if (!expectedYaml) { + return { pass: false, score: 0, reason: 'No expected block provided in config' }; + } + + let expected; + try { + expected = parse(expectedYaml); + } catch (e) { + return { pass: false, score: 0, reason: `Could not parse expected block: ${e.message}` }; + } + + const id = expected?.id; + if (!id) { + return { pass: false, score: 0, reason: 'Expected block has no id field' }; + } + + // Extract raw YAML strings from each ```mdma block in the output + const blocks = []; + const blockRegex = /```mdma\n([\s\S]*?)```/g; + let match; + while ((match = blockRegex.exec(output)) !== null) { + blocks.push(match[1]); + } + + // Find and parse the block whose top-level id matches + let actual = null; + let actualRaw = null; + for (const raw of blocks) { + let parsed; + try { + parsed = parse(raw); + } catch { + continue; + } + if (parsed?.id === id) { + actual = parsed; + actualRaw = raw.trim(); + break; + } + } + + if (!actual) { + return { + pass: false, + score: 0, + reason: `Component "${id}" not found in output (${blocks.length} block(s) present)`, + }; + } + + // Deep compare every field in expected against actual + const failures = compareFields(expected, actual, ''); + + // Check hasFields presence + for (const field of hasFields) { + if (actual[field] === undefined || actual[field] === null || actual[field] === '') { + failures.push(`field "${field}" is missing or empty`); + } + } + + if (failures.length > 0) { + return { + pass: false, + score: 0, + reason: `Component "${id}" field mismatch:\n${failures.join('\n')}\n\nActual block:\n${actualRaw}`, + }; + } + + return { + pass: true, + score: 1, + reason: `Component "${id}" matches expected block`, + }; +} + +function compareFields(expected, actual, prefix) { + const failures = []; + for (const [key, expectedVal] of Object.entries(expected)) { + const path = prefix ? `${prefix}.${key}` : key; + const actualVal = actual?.[key]; + + if (expectedVal === null || expectedVal === undefined) { + // null in expected = presence check only + if (actualVal === undefined || actualVal === null || actualVal === '') { + failures.push(`"${path}" is missing or empty`); + } + } else if (Array.isArray(expectedVal)) { + if (!Array.isArray(actualVal)) { + failures.push(`"${path}" should be an array, got ${typeof actualVal}`); + } else if (expectedVal.length !== actualVal.length) { + failures.push(`"${path}" length: expected ${expectedVal.length}, got ${actualVal.length}`); + } else { + for (let i = 0; i < expectedVal.length; i++) { + if (typeof expectedVal[i] === 'object' && expectedVal[i] !== null) { + failures.push(...compareFields(expectedVal[i], actualVal[i] ?? {}, `${path}[${i}]`)); + } else if (expectedVal[i] !== actualVal[i]) { + failures.push(`"${path}[${i}]": expected ${JSON.stringify(expectedVal[i])}, got ${JSON.stringify(actualVal[i])}`); + } + } + } + } else if (typeof expectedVal === 'object') { + failures.push(...compareFields(expectedVal, actualVal ?? {}, path)); + } else if (actualVal !== expectedVal) { + failures.push(`"${path}": expected ${JSON.stringify(expectedVal)}, got ${JSON.stringify(actualVal)}`); + } + } + return failures; +} diff --git a/evals/assertions/fixer-preserves-components.mjs b/evals/assertions/fixer-preserves-components.mjs index 4461b91..2b455d3 100644 --- a/evals/assertions/fixer-preserves-components.mjs +++ b/evals/assertions/fixer-preserves-components.mjs @@ -6,6 +6,7 @@ */ export default function (output, { config } = {}) { const min = config?.min ?? 1; + const max = config?.max ?? Number.POSITIVE_INFINITY; const blockCount = (output.match(/```mdma/g) ?? []).length; if (blockCount < min) { @@ -16,9 +17,17 @@ export default function (output, { config } = {}) { }; } + if (blockCount > max) { + return { + pass: false, + score: 0, + reason: `Fixer output has ${blockCount} mdma block(s) but expected at most ${max}`, + }; + } + return { pass: true, score: 1, - reason: `Fixer preserved ${blockCount} mdma block(s) (min: ${min})`, + reason: `Fixer preserved ${blockCount} mdma block(s) (min: ${min}${max !== Number.POSITIVE_INFINITY ? `, max: ${max}` : ''})`, }; } diff --git a/evals/package.json b/evals/package.json index 9fb21af..830e69b 100644 --- a/evals/package.json +++ b/evals/package.json @@ -10,6 +10,7 @@ "eval:flows": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.flows.yaml; exit 0", "eval:fixer": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer.yaml; exit 0", "eval:fixer-flow": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer-flow.yaml; exit 0", + "eval:fixer-all": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer-flow.yaml; exit 0", "eval:guidance": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.guidance.yaml; exit 0", "eval:all": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.custom.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.conversation.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.prompt-builder.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.flows.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer-flow.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.guidance.yaml; exit 0", "eval:author": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.custom.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.conversation.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.flows.yaml; exit 0", @@ -19,6 +20,7 @@ "@mobile-reality/mdma-cli": "workspace:*", "@mobile-reality/mdma-prompt-pack": "workspace:*", "@mobile-reality/mdma-validator": "workspace:*", - "promptfoo": "0.121.9" + "promptfoo": "0.121.9", + "yaml": "^2.6.0" } } diff --git a/evals/prompt-fixer.mjs b/evals/prompt-fixer.mjs index d33a63c..ed84100 100644 --- a/evals/prompt-fixer.mjs +++ b/evals/prompt-fixer.mjs @@ -4,6 +4,7 @@ import { buildSystemPrompt, } from '@mobile-reality/mdma-prompt-pack'; import { validate } from '@mobile-reality/mdma-validator'; +import { selectFixerPrompt } from './select-prompt.mjs'; /** * Promptfoo prompt function for fixer eval tests. @@ -19,19 +20,25 @@ import { validate } from '@mobile-reality/mdma-validator'; * 2. Collects remaining unfixed issues * 3. Sends the fixer system prompt (with variant-specific extensions) + user message */ -export default function ({ vars }) { +export default async function ({ vars }) { const exclude = ['thinking-block']; if (vars.variantKey !== 'flow') exclude.push('flow-ordering'); const result = validate(vars.brokenDocument, { exclude }); - const unfixed = result.issues.filter( - (i) => !i.fixed && (i.severity === 'error' || i.severity === 'warning'), + const allIssues = result.issues.filter( + (i) => i.severity === 'error' || i.severity === 'warning', ); - const fixerPrompt = buildFixerPrompt(vars.variantKey ?? undefined); + const { prompt: variantPrompt, source: fixerSource } = await selectFixerPrompt(); + const fixerPrompt = fixerSource.startsWith('default') + ? buildFixerPrompt(vars.variantKey ?? undefined) + : variantPrompt; const systemPrompt = `${buildSystemPrompt()}\n\n---\n\n${fixerPrompt}`; - const userMessage = buildFixerMessage(result.output, unfixed, { + // Pass the original broken document (not auto-fixed output) so the model + // sees every issue in full context, including ones the auto-fixer silently + // stripped (e.g. removing onSubmit instead of repairing the broken target). + const userMessage = buildFixerMessage(vars.brokenDocument, allIssues, { conversationHistory: vars.conversationHistory ?? undefined, promptContext: vars.promptContext ?? undefined, }); diff --git a/evals/promptfooconfig.fixer-flow.yaml b/evals/promptfooconfig.fixer-flow.yaml index 4b2be03..2eafbaf 100644 --- a/evals/promptfooconfig.fixer-flow.yaml +++ b/evals/promptfooconfig.fixer-flow.yaml @@ -1,7 +1,9 @@ # MDMA Fixer — Flow & References eval config # -# Run: pnpm --filter @mobile-reality/mdma-evals eval:fixer-flow -# View: pnpm --filter @mobile-reality/mdma-evals eval:view +# Run (general): pnpm --filter @mobile-reality/mdma-evals eval:fixer +# Run (flow): pnpm --filter @mobile-reality/mdma-evals eval:fixer-flow +# Run (both): pnpm --filter @mobile-reality/mdma-evals eval:fixer && pnpm --filter @mobile-reality/mdma-evals eval:fixer-flow +# View: pnpm --filter @mobile-reality/mdma-evals eval:view description: MDMA Fixer — Flow & References Eval diff --git a/evals/promptfooconfig.fixer.yaml b/evals/promptfooconfig.fixer.yaml index daa1250..aa17073 100644 --- a/evals/promptfooconfig.fixer.yaml +++ b/evals/promptfooconfig.fixer.yaml @@ -1,7 +1,9 @@ # MDMA Fixer Prompt — promptfoo evaluation config # -# Run: pnpm --filter @mobile-reality/mdma-evals eval:fixer -# View: pnpm --filter @mobile-reality/mdma-evals eval:view +# Run (general): pnpm --filter @mobile-reality/mdma-evals eval:fixer +# Run (flow): pnpm --filter @mobile-reality/mdma-evals eval:fixer-flow +# Run (both): pnpm --filter @mobile-reality/mdma-evals eval:fixer && pnpm --filter @mobile-reality/mdma-evals eval:fixer-flow +# View: pnpm --filter @mobile-reality/mdma-evals eval:view description: MDMA Fixer Prompt Eval @@ -27,6 +29,6 @@ defaultTest: - type: javascript value: file://assertions/fixer-preserves-components.mjs config: - min: 2 + min: 1 tests: tests-fixer.yaml diff --git a/evals/select-prompt.mjs b/evals/select-prompt.mjs index 1787fb2..eab1da9 100644 --- a/evals/select-prompt.mjs +++ b/evals/select-prompt.mjs @@ -25,7 +25,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { MASTER_PROMPT } from '@mobile-reality/mdma-cli/prompts'; -import { MDMA_AUTHOR_PROMPT } from '@mobile-reality/mdma-prompt-pack'; +import { MDMA_AUTHOR_PROMPT, MDMA_FIXER_PROMPT } from '@mobile-reality/mdma-prompt-pack'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = path.resolve(__dirname, '..'); @@ -119,3 +119,13 @@ export async function selectAuthorPrompt(provider = process.env.EVAL_PROVIDER) { defaultPrompt: MDMA_AUTHOR_PROMPT, }); } + +export async function selectFixerPrompt(provider = process.env.EVAL_PROVIDER) { + return selectVariant({ + provider, + promptsDir: path.join(REPO_ROOT, 'packages/prompt-pack/src/prompts/mdma-fixer'), + packagePath: '@mobile-reality/mdma-prompt-pack/prompts/mdma-fixer', + exportPrefix: 'MDMA_FIXER_PROMPT', + defaultPrompt: MDMA_FIXER_PROMPT, + }); +} diff --git a/evals/tests-fixer-flow.yaml b/evals/tests-fixer-flow.yaml index 34680ae..47e0350 100644 --- a/evals/tests-fixer-flow.yaml +++ b/evals/tests-fixer-flow.yaml @@ -539,3 +539,51 @@ value: "type: approval-gate" - type: not-icontains value: "id: approved-callout" + +# --------------------------------------------------------------------------- +# 8. Circular action reference — approval-gate loops back to form +# --------------------------------------------------------------------------- +- description: Fixes circular flow where approval-gate onApprove points back to the form + vars: + variantKey: flow + brokenDocument: | + # Feedback Loop + + ```mdma + type: form + id: feedback-form + fields: + - name: rating + type: number + label: Rating + required: true + - name: comment + type: textarea + label: Comment + onSubmit: review-gate + ``` + + ```mdma + type: approval-gate + id: review-gate + title: Review Feedback + requiredApprovers: 1 + onApprove: feedback-form + onDeny: rejection-notice + ``` + + ```mdma + type: callout + id: rejection-notice + variant: error + content: Your feedback was not accepted. Please revise. + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/no-multi-step-flow.mjs + - type: icontains + value: feedback-form + - type: not-icontains + value: "type: approval-gate" diff --git a/evals/tests-fixer.yaml b/evals/tests-fixer.yaml index 73ed277..5c1f790 100644 --- a/evals/tests-fixer.yaml +++ b/evals/tests-fixer.yaml @@ -5,7 +5,200 @@ # issues to the LLM. Assertions verify the output is a valid MDMA document. # --------------------------------------------------------------------------- -# 1. Missing webhook trigger + invalid onSubmit target +# 1. Button missing required text +# --------------------------------------------------------------------------- +- description: Fixes button missing required text field + vars: + brokenDocument: | + # Quick Action + + ```mdma + type: button + id: action-btn + variant: primary + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 1 + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: button + id: action-btn + variant: primary + hasFields: + - text + +# --------------------------------------------------------------------------- +# 2. Callout missing required content +# --------------------------------------------------------------------------- +- description: Fixes callout missing required content field + vars: + brokenDocument: | + # Status Update + + ```mdma + type: callout + id: status-notice + variant: info + title: System Status + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 1 + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: callout + id: status-notice + variant: info + title: System Status + hasFields: + - content + +# --------------------------------------------------------------------------- +# 3. Select field missing options +# --------------------------------------------------------------------------- +- description: Fixes select field missing required options array + vars: + brokenDocument: | + # Contact Form + + ```mdma + type: form + id: contact-form + fields: + - name: contact-type + type: select + label: Contact Type + required: true + - name: message + type: textarea + label: Message + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 1 + - type: javascript + value: file://assertions/select-has-options.mjs + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: form + id: contact-form + fields: + - name: contact-type + type: select + label: Contact Type + required: true + - name: message + type: textarea + label: Message + +# --------------------------------------------------------------------------- +# 4. Placeholder content in callout +# --------------------------------------------------------------------------- +- description: Fixes placeholder title and content in callout + vars: + brokenDocument: | + # Welcome Screen + + ```mdma + type: callout + id: welcome-callout + variant: info + title: TODO + content: Lorem ipsum dolor sit amet + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 1 + - type: javascript + value: file://assertions/no-placeholder-content.mjs + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: callout + id: welcome-callout + variant: info + hasFields: + - title + - content + +# --------------------------------------------------------------------------- +# 5. PII fields without sensitive flag +# --------------------------------------------------------------------------- +- description: Fixes email and phone fields missing sensitive flag + vars: + brokenDocument: | + # Contact Details + + ```mdma + type: form + id: contact-details + fields: + - name: full-name + type: text + label: Full Name + required: true + - name: email + type: email + label: Email Address + required: true + - name: phone + type: text + label: Phone Number + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 1 + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: form + id: contact-details + fields: + - name: full-name + type: text + label: Full Name + required: true + - name: email + type: email + label: Email Address + required: true + sensitive: true + - name: phone + type: text + label: Phone Number + sensitive: true + +# --------------------------------------------------------------------------- +# 6. Missing webhook trigger + invalid onSubmit target # --------------------------------------------------------------------------- - description: Fixes missing webhook trigger and broken action references vars: @@ -48,9 +241,25 @@ min: 3 - type: icontains value: trigger + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: webhook + id: order-webhook + url: https://api.example.com/orders + method: POST + trigger: order-form + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: form + id: order-form + onSubmit: order-webhook # --------------------------------------------------------------------------- -# 2. Unknown component type + missing required button text +# 7. Unknown component type + missing required button text # --------------------------------------------------------------------------- - description: Fixes unknown component type and missing button text vars: @@ -84,9 +293,18 @@ value: file://assertions/fixer-preserves-components.mjs config: min: 2 + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: button + id: refresh-btn + onAction: stats-card + hasFields: + - text # --------------------------------------------------------------------------- -# 3. Missing webhook trigger + unknown type + missing button text +# 8. Missing webhook trigger + unknown type + missing button text # --------------------------------------------------------------------------- - description: Fixes missing webhook trigger with multiple broken components vars: @@ -132,9 +350,33 @@ min: 3 - type: icontains value: trigger + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: webhook + id: profile-webhook + url: https://api.example.com/profile + method: POST + trigger: save-profile + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: button + id: save-profile + hasFields: + - text + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: form + id: profile-form + onSubmit: save-profile # --------------------------------------------------------------------------- -# 4. Select fields without options + field name typos +# 9. Select fields without options + field name typos # --------------------------------------------------------------------------- - description: Fixes select without options and field name typos on approval-gate vars: @@ -185,9 +427,28 @@ min: 3 - type: icontains value: options + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: approval-gate + id: leave-approval + title: Manager Approval + requiredApprovers: 2 + allowedRoles: + - manager + - hr + onApprove: leave-confirmed + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: form + id: leave-form + onSubmit: leave-approval # --------------------------------------------------------------------------- -# 5. Table data key mismatch + chart axis mismatch +# 10. Table data key mismatch + chart axis mismatch # --------------------------------------------------------------------------- - description: Fixes table data key mismatch and chart axis errors vars: @@ -234,9 +495,44 @@ value: file://assertions/fixer-preserves-components.mjs config: min: 2 + - type: icontains + value: "xAxis: Month" + - type: icontains + value: "product:" + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: chart + id: sales-chart + variant: bar + xAxis: Month + yAxis: + - Revenue + - Costs + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: table + id: sales-table + columns: + - key: product + header: Product + - key: revenue + header: Revenue + - key: units + header: Units Sold + data: + - product: Widget A + revenue: 50000 + units: 120 + - product: Widget B + revenue: 32000 + units: 85 # --------------------------------------------------------------------------- -# 6. Missing sensitive flags + missing required fields +# 11. Missing sensitive flags + missing required fields # --------------------------------------------------------------------------- - description: Fixes missing PII sensitivity and missing required schema fields vars: @@ -298,54 +594,63 @@ min: 3 - type: javascript value: file://assertions/has-sensitive.mjs - -# --------------------------------------------------------------------------- -# 7. Circular action references + backward flow -# --------------------------------------------------------------------------- -- description: Fixes circular and backward action references - vars: - brokenDocument: | - # Feedback Loop - - ```mdma - type: form - id: feedback-form - fields: - - name: rating - type: number - label: Rating - required: true - - name: comment - type: textarea - label: Comment - onSubmit: review-gate - ``` - - ```mdma - type: approval-gate - id: review-gate - title: Review Feedback - requiredApprovers: 1 - onApprove: feedback-form - onDeny: rejection-notice - ``` - - ```mdma - type: callout - id: rejection-notice - variant: error - content: Your feedback was not accepted. Please revise. - ``` - assert: - type: javascript - value: file://assertions/fixer-resolves-errors.mjs + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: button + id: submit-registration + hasFields: + - text - type: javascript - value: file://assertions/fixer-preserves-components.mjs + value: file://assertions/fixer-contains-component.mjs config: - min: 1 + expected: | + type: form + id: patient-form + fields: + - name: full-name + type: text + label: Full Name + required: true + - name: email + type: email + label: Email Address + sensitive: true + - name: phone + type: text + label: Phone Number + sensitive: true + - name: ssn + type: text + label: Social Security Number + sensitive: true + - name: address + type: textarea + label: Home Address + sensitive: true + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: table + id: patient-records + columns: + - key: name + header: Patient Name + sensitive: true + - key: email + header: Email + sensitive: true + - key: phone + header: Phone + sensitive: true + - key: dob + header: Date of Birth + sensitive: true # --------------------------------------------------------------------------- -# 8. Mixed issues — kitchen sink +# 12. Mixed issues — kitchen sink # --------------------------------------------------------------------------- - description: Fixes a document with many different issue types vars: @@ -406,9 +711,35 @@ min: 4 - type: javascript value: file://assertions/unique-kebab-ids.mjs + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: webhook + id: notify-hr + url: https://api.example.com/hr + method: POST + trigger: onboarding-tasks + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: tasklist + id: onboarding-tasks + onComplete: notify-hr + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: form + id: employee-form + hasFields: + - onSubmit + - type: icontains + value: "onAction: onboarding-tasks" # --------------------------------------------------------------------------- -# 9. Webhook with broken references + form missing onSubmit target +# 13. Webhook with broken references + form missing onSubmit target # --------------------------------------------------------------------------- - description: Fixes webhook trigger and form onSubmit pointing to missing components vars: @@ -464,9 +795,26 @@ min: 3 - type: icontains value: trigger + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: webhook + id: ticket-webhook + url: https://api.example.com/tickets + method: POST + trigger: ticket-form + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: form + id: ticket-form + hasFields: + - onSubmit # --------------------------------------------------------------------------- -# 10. Placeholder content throughout +# 14. Placeholder content throughout # --------------------------------------------------------------------------- - description: Fixes placeholder content in labels and fields vars: diff --git a/package.json b/package.json index 9086999..4c9802a 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "eval:flows": "pnpm --filter @mobile-reality/mdma-evals eval:flows", "eval:fixer": "pnpm --filter @mobile-reality/mdma-evals eval:fixer", "eval:fixer-flow": "pnpm --filter @mobile-reality/mdma-evals eval:fixer-flow", + "eval:fixer-all": "pnpm --filter @mobile-reality/mdma-evals eval:fixer-all", "eval:guidance": "pnpm --filter @mobile-reality/mdma-evals eval:guidance", "eval:all": "pnpm --filter @mobile-reality/mdma-evals eval:all", "eval:author": "pnpm --filter @mobile-reality/mdma-evals eval:author", diff --git a/packages/prompt-pack/src/index.ts b/packages/prompt-pack/src/index.ts index 98d9661..44c7cea 100644 --- a/packages/prompt-pack/src/index.ts +++ b/packages/prompt-pack/src/index.ts @@ -6,8 +6,8 @@ export { type AuthorPromptVariant, } from './prompts/mdma-author/registry.js'; export { MDMA_REVIEWER_PROMPT } from './prompts/mdma-reviewer.js'; +export { MDMA_FIXER_PROMPT } from './prompts/mdma-fixer/default.js'; export { - MDMA_FIXER_PROMPT, MDMA_FIXER_BASE, MDMA_FIXER_STRUCTURE, MDMA_FIXER_BINDINGS, @@ -21,7 +21,7 @@ export { buildFixerMessage, type FixerIssue, type FixerMessageOptions, -} from './prompts/mdma-fixer.js'; +} from './prompts/mdma-fixer/_shared.js'; export { buildSystemPrompt, type BuildSystemPromptOptions } from './build-system-prompt.js'; export { AGENT_TOOL_PROMPT_VARIANTS, diff --git a/packages/prompt-pack/src/loader.ts b/packages/prompt-pack/src/loader.ts index 681a71e..566ed6b 100644 --- a/packages/prompt-pack/src/loader.ts +++ b/packages/prompt-pack/src/loader.ts @@ -1,6 +1,6 @@ import { MDMA_AUTHOR_PROMPT } from './prompts/mdma-author/default.js'; import { MDMA_REVIEWER_PROMPT } from './prompts/mdma-reviewer.js'; -import { MDMA_FIXER_PROMPT } from './prompts/mdma-fixer.js'; +import { MDMA_FIXER_PROMPT } from './prompts/mdma-fixer/default.js'; /** * Static registry of all available prompts. diff --git a/packages/prompt-pack/src/prompts/mdma-fixer.ts b/packages/prompt-pack/src/prompts/mdma-fixer/_shared.ts similarity index 79% rename from packages/prompt-pack/src/prompts/mdma-fixer.ts rename to packages/prompt-pack/src/prompts/mdma-fixer/_shared.ts index e452b7f..2236c7c 100644 --- a/packages/prompt-pack/src/prompts/mdma-fixer.ts +++ b/packages/prompt-pack/src/prompts/mdma-fixer/_shared.ts @@ -1,3 +1,14 @@ +/** + * Building blocks for all MDMA-Fixer prompt variants — the canonical default + * and every per-vendor variant under `/`. + * + * Variants compose these via template-literal interpolation. Add model- + * specific framing in the vendor `_shared.ts` or inline in the variant file. + * + * The `_` filename prefix is recognized by `evals/select-prompt.mjs` and + * skipped during variant discovery. + */ + /** * Base fixer prompt — general rules that apply to all fix scenarios. */ @@ -177,6 +188,193 @@ export const MDMA_FIXER_APPROVAL = ` | Missing \`title\` on approval-gate | Add a descriptive title | | Missing \`url\` on webhook | Add a valid URL (e.g. \`https://api.example.com/endpoint\`) |`; +/** + * Few-shot examples: broken → fixed MDMA document pairs. + * Covers the most common fix patterns: broken action references, + * field name typos, and multi-step flow splitting. + */ +export const MDMA_FIXER_EXAMPLES = ` +## Examples + +### Example 1 — Broken action references + +**Issues reported:** +1. [error] cross-reference #order-form → onSubmit: "submit-handler" does not match any component ID +2. [error] schema-conformance #order-webhook → trigger: Required + +**Broken document:** + +\`\`\`mdma +type: form +id: order-form +fields: + - name: product + type: text + label: Product Name + required: true +onSubmit: submit-handler +\`\`\` + +\`\`\`mdma +type: webhook +id: order-webhook +url: https://api.example.com/orders +method: POST +\`\`\` + +\`\`\`mdma +type: callout +id: order-status +variant: success +content: Your order has been submitted! +\`\`\` + +**Fixed document:** + +\`\`\`mdma +type: form +id: order-form +fields: + - name: product + type: text + label: Product Name + required: true +onSubmit: order-webhook +\`\`\` + +\`\`\`mdma +type: webhook +id: order-webhook +url: https://api.example.com/orders +method: POST +trigger: order-form +\`\`\` + +\`\`\`mdma +type: callout +id: order-status +variant: success +content: Your order has been submitted! +\`\`\` + +--- + +### Example 2 — Field name typos + +**Issues reported:** +1. [warning] field-name-typos #review-gate → "roles" is likely a typo — did you mean "allowedRoles"? +2. [warning] field-name-typos #review-gate → "approvers" is likely a typo — did you mean "requiredApprovers"? +3. [error] schema-conformance #submit-btn → text: Required + +**Broken document:** + +\`\`\`mdma +type: approval-gate +id: review-gate +title: Manager Review +roles: + - manager + - hr +approvers: 2 +onApprove: confirmed +\`\`\` + +\`\`\`mdma +type: callout +id: confirmed +variant: success +content: Request approved! +\`\`\` + +\`\`\`mdma +type: button +id: submit-btn +variant: primary +onAction: review-gate +\`\`\` + +**Fixed document:** + +\`\`\`mdma +type: approval-gate +id: review-gate +title: Manager Review +allowedRoles: + - manager + - hr +requiredApprovers: 2 +onApprove: confirmed +\`\`\` + +\`\`\`mdma +type: callout +id: confirmed +variant: success +content: Request approved! +\`\`\` + +\`\`\`mdma +type: button +id: submit-btn +text: Submit for Review +variant: primary +onAction: review-gate +\`\`\` + +--- + +### Example 3 — Multi-step flow in single message (no conversation history) + +**Issues reported:** +1. [error] flow-ordering (document): Multi-step flow in single message — "intake-form" targets "approval-gate" via onSubmit + +**Broken document:** + +\`\`\`mdma +type: form +id: intake-form +fields: + - name: reason + type: textarea + label: Reason +onSubmit: approval-gate +\`\`\` + +\`\`\`mdma +type: approval-gate +id: approval-gate +title: Manager Approval +requiredApprovers: 1 +onApprove: notify-webhook +\`\`\` + +\`\`\`mdma +type: webhook +id: notify-webhook +url: https://api.example.com/notify +method: POST +trigger: approval-gate +\`\`\` + +**Fixed document** (no prior conversation — output step 1 only): + +\`\`\`mdma +type: form +id: intake-form +fields: + - name: reason + type: textarea + label: Reason +onSubmit: submitted-callout +\`\`\` + +\`\`\`mdma +type: callout +id: submitted-callout +variant: info +content: Your request has been submitted and is awaiting manager approval. +\`\`\``; + /** * Map from validator variant keys to their fixer extensions. */ @@ -217,12 +415,9 @@ export function buildFixerPrompt(variantKey?: string): string { MDMA_FIXER_APPROVAL, ]; - return `${MDMA_FIXER_BASE}\n${extensions.join('\n')}`; + return `${MDMA_FIXER_BASE}\n${extensions.join('\n')}\n${MDMA_FIXER_EXAMPLES}`; } -/** @deprecated Use buildFixerPrompt() instead. Kept for backward compatibility. */ -export const MDMA_FIXER_PROMPT = buildFixerPrompt(); - export interface FixerIssue { ruleId: string; severity: string; @@ -269,7 +464,7 @@ export function buildFixerMessage( context += `\n\n## Original Prompt Requirements\n\nThe document was generated from the following instructions. The fixed output MUST comply with these requirements — use the correct component IDs, field names, types, options, and structure specified here:\n\n${options.promptContext}\n`; } - return `Fix the following MDMA document. The validator found ${issues.length} issue(s) that could not be auto-fixed: + return `Fix the following MDMA document. The validator found ${issues.length} issue(s) that need to be fixed: ${issueLines.join('\n')}${context} diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/default.ts b/packages/prompt-pack/src/prompts/mdma-fixer/default.ts new file mode 100644 index 0000000..5c0e92d --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/default.ts @@ -0,0 +1,28 @@ +/** + * MDMA Fixer Prompt — canonical default. + * + * Composed from `_shared.ts` so all per-vendor variants stay byte-aligned + * to the same content surface. Includes every extension category. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from './_shared.js'; + +export const MDMA_FIXER_PROMPT = `${MDMA_FIXER_BASE} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/openai/_shared.ts b/packages/prompt-pack/src/prompts/mdma-fixer/openai/_shared.ts new file mode 100644 index 0000000..8be81cb --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/openai/_shared.ts @@ -0,0 +1,17 @@ +/** + * Shared content for MDMA-Fixer OpenAI variants. + * + * Add blocks here when a failure mode is observed across multiple GPT variants. + * Single-variant blocks live inline in their variant file, not here. + * The `_` filename prefix is recognized by `evals/select-prompt.mjs` and + * skipped during variant discovery. + */ + +/** + * Reinforces rule #5 of the fixer base — GPT models occasionally wrap their + * output in an outer ```markdown fence instead of emitting the Markdown + * document directly. Placed after the base rules and before extensions so it + * stands out as an additional emphasis. + */ +export const CRITICAL_OUTPUT_LINE = + 'CRITICAL: Your output IS the corrected Markdown document — write headings, paragraphs, and ```mdma blocks directly. NEVER wrap your response in ```markdown code fences. Your response is already rendered as Markdown.'; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.5.ts b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.5.ts new file mode 100644 index 0000000..0805073 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.5.ts @@ -0,0 +1,35 @@ +/** + * MDMA Fixer Prompt — OpenAI GPT-5.5 variant. + * + * Starting baseline for GPT-5.5 fixer evals. Adds CRITICAL_OUTPUT_LINE + * after the base rules — the same no-outer-fence failure mode observed on + * GPT-5.5 in author evals applies equally to the fixer output. + * + * Add further framing blocks inline as specific failure modes are observed + * during evals. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { CRITICAL_OUTPUT_LINE } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GPT_5_5 = `${MDMA_FIXER_BASE} + +${CRITICAL_OUTPUT_LINE} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES}`; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f29d575..ab85411 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -109,6 +109,9 @@ importers: promptfoo: specifier: 0.121.9 version: 0.121.9(@cfworker/json-schema@4.1.1)(@langchain/core@1.1.27(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.0))(openai@6.36.0(ws@8.19.0)(zod@4.4.3)))(@types/json-schema@7.0.15)(@types/node@18.19.130)(@types/react@19.2.14)(pg@8.18.0)(playwright-core@1.59.1)(socks@2.8.7)(typescript@5.9.3) + yaml: + specifier: ^2.6.0 + version: 2.8.2 packages/attachables-core: dependencies: From 2738d9d77ec1bbc38d76d401f6c679c95aa21d21 Mon Sep 17 00:00:00 2001 From: gitsad Date: Thu, 14 May 2026 12:48:25 +0200 Subject: [PATCH 02/26] feat: make onSubmit required and adjust fixer tests --- evals/assertions/unique-kebab-ids.mjs | 8 +- evals/tests-fixer.yaml | 252 +++--------------- .../src/prompts/mdma-author/_shared.ts | 8 +- packages/spec/src/schemas/components/form.ts | 2 +- packages/spec/tests/schemas.test.ts | 4 + .../validator/src/rules/form-submit-action.ts | 30 +++ packages/validator/src/rules/index.ts | 4 + .../src/rules/single-interactive-component.ts | 29 ++ packages/validator/src/types.ts | 4 +- .../validator/tests/fixtures/bad-bindings.md | 1 + .../validator/tests/fixtures/mixed-issues.md | 1 + .../tests/fixtures/no-thinking-block.md | 1 + .../tests/fixtures/pii-missing-sensitive.md | 1 + .../tests/fixtures/valid-document.md | 1 + .../tests/rules/form-submit-action.test.ts | 113 ++++++++ .../tests/rules/schema-conformance.test.ts | 2 + .../single-interactive-component.test.ts | 61 +++++ packages/validator/tests/validate.test.ts | 5 + 18 files changed, 297 insertions(+), 230 deletions(-) create mode 100644 packages/validator/src/rules/form-submit-action.ts create mode 100644 packages/validator/src/rules/single-interactive-component.ts create mode 100644 packages/validator/tests/rules/form-submit-action.test.ts create mode 100644 packages/validator/tests/rules/single-interactive-component.test.ts diff --git a/evals/assertions/unique-kebab-ids.mjs b/evals/assertions/unique-kebab-ids.mjs index 12289d7..dd9e97b 100644 --- a/evals/assertions/unique-kebab-ids.mjs +++ b/evals/assertions/unique-kebab-ids.mjs @@ -5,12 +5,8 @@ export default function (output) { const idMatches = [...output.matchAll(/^id:\s*(.+)$/gm)]; const ids = idMatches.map((m) => m[1].trim()); - if (ids.length < 3) { - return { - pass: false, - score: 0, - reason: `Expected at least 3 component IDs, found ${ids.length}`, - }; + if (ids.length === 0) { + return { pass: false, score: 0, reason: 'No component IDs found' }; } const unique = new Set(ids).size === ids.length; diff --git a/evals/tests-fixer.yaml b/evals/tests-fixer.yaml index 5c1f790..10f3a9b 100644 --- a/evals/tests-fixer.yaml +++ b/evals/tests-fixer.yaml @@ -198,9 +198,9 @@ sensitive: true # --------------------------------------------------------------------------- -# 6. Missing webhook trigger + invalid onSubmit target +# 6. Form with broken onSubmit reference # --------------------------------------------------------------------------- -- description: Fixes missing webhook trigger and broken action references +- description: Fixes form with broken onSubmit reference to point to existing callout vars: brokenDocument: | # Order Submission @@ -219,13 +219,6 @@ onSubmit: nonexistent-handler ``` - ```mdma - type: webhook - id: order-webhook - url: https://api.example.com/orders - method: POST - ``` - ```mdma type: callout id: order-status @@ -238,25 +231,14 @@ - type: javascript value: file://assertions/fixer-preserves-components.mjs config: - min: 3 - - type: icontains - value: trigger - - type: javascript - value: file://assertions/fixer-contains-component.mjs - config: - expected: | - type: webhook - id: order-webhook - url: https://api.example.com/orders - method: POST - trigger: order-form + min: 2 - type: javascript value: file://assertions/fixer-contains-component.mjs config: expected: | type: form id: order-form - onSubmit: order-webhook + onSubmit: order-status # --------------------------------------------------------------------------- # 7. Unknown component type + missing required button text @@ -304,9 +286,9 @@ - text # --------------------------------------------------------------------------- -# 8. Missing webhook trigger + unknown type + missing button text +# 8. Form missing onSubmit # --------------------------------------------------------------------------- -- description: Fixes missing webhook trigger with multiple broken components +- description: Fixes form that is missing onSubmit by connecting it to the success callout vars: brokenDocument: | # User Profile @@ -326,20 +308,13 @@ - name: bio type: textarea label: Bio - onSubmit: save-profile - ``` - - ```mdma - type: webhook - id: profile-webhook - url: https://api.example.com/profile - method: POST ``` ```mdma - type: button - id: save-profile - variant: primary + type: callout + id: profile-saved + variant: success + content: Your profile has been saved. ``` assert: - type: javascript @@ -347,60 +322,24 @@ - type: javascript value: file://assertions/fixer-preserves-components.mjs config: - min: 3 - - type: icontains - value: trigger - - type: javascript - value: file://assertions/fixer-contains-component.mjs - config: - expected: | - type: webhook - id: profile-webhook - url: https://api.example.com/profile - method: POST - trigger: save-profile - - type: javascript - value: file://assertions/fixer-contains-component.mjs - config: - expected: | - type: button - id: save-profile - hasFields: - - text + min: 2 - type: javascript value: file://assertions/fixer-contains-component.mjs config: expected: | type: form id: profile-form - onSubmit: save-profile + hasFields: + - onSubmit # --------------------------------------------------------------------------- -# 9. Select fields without options + field name typos +# 9. Approval-gate field name typos # --------------------------------------------------------------------------- -- description: Fixes select without options and field name typos on approval-gate +- description: Fixes field name typos on approval-gate (roles→allowedRoles, approvers→requiredApprovers) vars: brokenDocument: | # Leave Request - ```mdma - type: form - id: leave-form - fields: - - name: leave-type - type: select - label: Leave Type - required: true - - name: start-date - type: date - label: Start Date - required: true - - name: reason - type: textarea - label: Reason - onSubmit: leave-approval - ``` - ```mdma type: approval-gate id: leave-approval @@ -424,9 +363,7 @@ - type: javascript value: file://assertions/fixer-preserves-components.mjs config: - min: 3 - - type: icontains - value: options + min: 2 - type: javascript value: file://assertions/fixer-contains-component.mjs config: @@ -439,13 +376,6 @@ - manager - hr onApprove: leave-confirmed - - type: javascript - value: file://assertions/fixer-contains-component.mjs - config: - expected: | - type: form - id: leave-form - onSubmit: leave-approval # --------------------------------------------------------------------------- # 10. Table data key mismatch + chart axis mismatch @@ -532,9 +462,9 @@ units: 85 # --------------------------------------------------------------------------- -# 11. Missing sensitive flags + missing required fields +# 11. Missing sensitive flags on form and table # --------------------------------------------------------------------------- -- description: Fixes missing PII sensitivity and missing required schema fields +- description: Fixes missing PII sensitive flags on form fields and table columns vars: brokenDocument: | # Patient Registration @@ -559,6 +489,7 @@ - name: address type: textarea label: Home Address + onSubmit: registration-complete ``` ```mdma @@ -581,9 +512,10 @@ ``` ```mdma - type: button - id: submit-registration - variant: primary + type: callout + id: registration-complete + variant: success + content: Registration submitted successfully! ``` assert: - type: javascript @@ -594,14 +526,6 @@ min: 3 - type: javascript value: file://assertions/has-sensitive.mjs - - type: javascript - value: file://assertions/fixer-contains-component.mjs - config: - expected: | - type: button - id: submit-registration - hasFields: - - text - type: javascript value: file://assertions/fixer-contains-component.mjs config: @@ -629,6 +553,7 @@ type: textarea label: Home Address sensitive: true + onSubmit: registration-complete - type: javascript value: file://assertions/fixer-contains-component.mjs config: @@ -650,9 +575,9 @@ sensitive: true # --------------------------------------------------------------------------- -# 12. Mixed issues — kitchen sink +# 12. Mixed issues — single form kitchen sink # --------------------------------------------------------------------------- -- description: Fixes a document with many different issue types +- description: Fixes many issues on a single form (ID format, placeholder, PII, select, onSubmit) vars: brokenDocument: | # Employee Onboarding @@ -677,30 +602,10 @@ ``` ```mdma - type: tasklist - id: onboarding-tasks - items: - - id: task-1 - text: Complete HR paperwork - - id: task-2 - text: Set up workstation - - id: task-3 - text: Meet team lead - onComplete: nonexistent-webhook - ``` - - ```mdma - type: button - id: employee_form - variant: primary - onClick: onboarding-tasks - ``` - - ```mdma - type: webhook - id: notify-hr - url: https://api.example.com/hr - method: POST + type: callout + id: onboarding-complete + variant: success + content: Welcome to the team! ``` assert: - type: javascript @@ -708,25 +613,11 @@ - type: javascript value: file://assertions/fixer-preserves-components.mjs config: - min: 4 + min: 2 - type: javascript value: file://assertions/unique-kebab-ids.mjs - type: javascript - value: file://assertions/fixer-contains-component.mjs - config: - expected: | - type: webhook - id: notify-hr - url: https://api.example.com/hr - method: POST - trigger: onboarding-tasks - - type: javascript - value: file://assertions/fixer-contains-component.mjs - config: - expected: | - type: tasklist - id: onboarding-tasks - onComplete: notify-hr + value: file://assertions/no-placeholder-content.mjs - type: javascript value: file://assertions/fixer-contains-component.mjs config: @@ -735,86 +626,9 @@ id: employee-form hasFields: - onSubmit - - type: icontains - value: "onAction: onboarding-tasks" - -# --------------------------------------------------------------------------- -# 13. Webhook with broken references + form missing onSubmit target -# --------------------------------------------------------------------------- -- description: Fixes webhook trigger and form onSubmit pointing to missing components - vars: - brokenDocument: | - # Support Ticket - - ```mdma - type: form - id: ticket-form - fields: - - name: subject - type: text - label: Subject - required: true - - name: priority - type: select - label: Priority - options: - - label: Low - value: low - - label: Medium - value: medium - - label: High - value: high - - name: description - type: textarea - label: Description - onSubmit: submit-ticket - ``` - - ```mdma - type: webhook - id: ticket-webhook - url: https://api.example.com/tickets - method: POST - body: - subject: "{{ticket-form.subject}}" - priority: "{{ticket-form.priority}}" - ``` - - ```mdma - type: callout - id: ticket-success - variant: success - content: Ticket submitted successfully! - ``` - assert: - - type: javascript - value: file://assertions/fixer-resolves-errors.mjs - - type: javascript - value: file://assertions/fixer-preserves-components.mjs - config: - min: 3 - - type: icontains - value: trigger - - type: javascript - value: file://assertions/fixer-contains-component.mjs - config: - expected: | - type: webhook - id: ticket-webhook - url: https://api.example.com/tickets - method: POST - trigger: ticket-form - - type: javascript - value: file://assertions/fixer-contains-component.mjs - config: - expected: | - type: form - id: ticket-form - hasFields: - - onSubmit # --------------------------------------------------------------------------- -# 14. Placeholder content throughout +# 13. Placeholder content throughout # --------------------------------------------------------------------------- - description: Fixes placeholder content in labels and fields vars: diff --git a/packages/prompt-pack/src/prompts/mdma-author/_shared.ts b/packages/prompt-pack/src/prompts/mdma-author/_shared.ts index eccdcd1..4463ae2 100644 --- a/packages/prompt-pack/src/prompts/mdma-author/_shared.ts +++ b/packages/prompt-pack/src/prompts/mdma-author/_shared.ts @@ -75,7 +75,7 @@ fields: max: message: bind: "{{variable.path}}" # optional binding -onSubmit: # optional — action triggered on submit +onSubmit: # required — action triggered on submit \`\`\` ### 2. button @@ -321,7 +321,8 @@ When a user request includes \`visible\` or \`disabled\` with a \`{{}}\` binding 7. **YAML correctness** — Ensure all YAML in mdma blocks is valid and properly indented. Always wrap string values in double quotes if they contain a colon followed by a space (\`: \`), e.g. \`label: "Step 1: Enter your info"\`. 8. **Always include thinking** — When generating MDMA components, ALWAYS include a \`thinking\` block BEFORE the main content to show your reasoning process. Use \`status: done\` and \`collapsed: true\`. 9. **Never expose MDMA internals to the user** — Do NOT mention thinking blocks, sensitive flags, bindings, component IDs, YAML structure, or any other MDMA implementation details in your visible Markdown text. The user should see a natural, helpful response — not commentary about how the document is built. All reasoning belongs inside the \`thinking\` block, not in the prose. Never write things like "I included a thinking block" or "the email field is marked as sensitive". -10. **Blueprint fidelity** — When the user provides an exact component structure, reproduce EVERY field verbatim, including \`visible\`, \`disabled\`, \`onComplete\`, \`onAction\`, and binding expressions. Never omit fields, never simplify bindings, never substitute \`true\`/\`false\` for a \`"{{...}}"\` binding. If the blueprint says \`disabled: "{{onboarding-checklist.completed}}"\`, your output must contain that exact line. If the blueprint says \`visible: "{{settings-form.notifications-enabled}}"\`, your output must contain that exact line.`; +10. **Blueprint fidelity** — When the user provides an exact component structure, reproduce EVERY field verbatim, including \`visible\`, \`disabled\`, \`onComplete\`, \`onAction\`, and binding expressions. Never omit fields, never simplify bindings, never substitute \`true\`/\`false\` for a \`"{{...}}"\` binding. If the blueprint says \`disabled: "{{onboarding-checklist.completed}}"\`, your output must contain that exact line. If the blueprint says \`visible: "{{settings-form.notifications-enabled}}"\`, your output must contain that exact line. +11. **One interactive component per message** — Each response must contain at most one **interactive** component: \`form\`, \`button\`, \`webhook\`, \`approval-gate\`, or \`tasklist\`. Non-interactive components (\`callout\`, \`table\`, \`chart\`, \`thinking\`) may appear alongside it freely. For multi-step workflows — where the user needs a form, then an approval gate, then a webhook — generate only the current step and tell the user what comes next. Never collapse multiple interactive steps into a single message.`; export const BASE_CHECKLIST = `## Self-Check Checklist @@ -337,4 +338,5 @@ Before finalizing an MDMA document, verify: - [ ] Table \`data\` matches the declared \`columns\` keys - [ ] Approval gates have at least one approver configured - [ ] Webhook URLs are valid or use binding syntax -- [ ] All \`visible\` and \`disabled\` bindings are double-quoted strings: \`"{{component.field}}"\``; +- [ ] All \`visible\` and \`disabled\` bindings are double-quoted strings: \`"{{component.field}}"\` +- [ ] Response contains at most one interactive component (\`form\`, \`button\`, \`webhook\`, \`approval-gate\`, \`tasklist\`)`; diff --git a/packages/spec/src/schemas/components/form.ts b/packages/spec/src/schemas/components/form.ts index 0a7da9a..3e2d77b 100644 --- a/packages/spec/src/schemas/components/form.ts +++ b/packages/spec/src/schemas/components/form.ts @@ -29,7 +29,7 @@ export const FormFieldSchema = z.object({ export const FormComponentSchema = ComponentBaseSchema.extend({ type: z.literal('form'), fields: z.array(FormFieldSchema).min(1), - onSubmit: z.string().optional().describe('Action ID to trigger on submit'), + onSubmit: z.string().describe('Action ID to trigger on submit'), }); export type FormField = z.infer; diff --git a/packages/spec/tests/schemas.test.ts b/packages/spec/tests/schemas.test.ts index a9988fa..4e7395c 100644 --- a/packages/spec/tests/schemas.test.ts +++ b/packages/spec/tests/schemas.test.ts @@ -73,6 +73,7 @@ describe('FormComponentSchema', () => { options: ['United States', 'Canada', 'Germany'], }, ], + onSubmit: 'submit-action', }; const result = FormComponentSchema.parse(form); expect(result.fields[0].options).toEqual([ @@ -87,6 +88,7 @@ describe('FormComponentSchema', () => { id: 'ds-form', type: 'form', fields: [{ name: 'country', type: 'select', label: 'Country', options: 'countries' }], + onSubmit: 'submit-action', }; const result = FormComponentSchema.parse(form); expect(result.fields[0].options).toBe('countries'); @@ -104,6 +106,7 @@ describe('FormComponentSchema', () => { { name: 'resume', type: 'file', label: 'Resume', required: true }, { name: 'passport', type: 'file', label: 'Passport', sensitive: true }, ], + onSubmit: 'submit-action', }; const result = FormComponentSchema.parse(form); expect(result.fields[0].type).toBe('file'); @@ -252,6 +255,7 @@ describe('MdmaComponentSchema (discriminated union)', () => { id: 'f', type: 'form', fields: [{ name: 'x', type: 'text', label: 'X' }], + onSubmit: 'submit-action', }); expect(form.type).toBe('form'); diff --git a/packages/validator/src/rules/form-submit-action.ts b/packages/validator/src/rules/form-submit-action.ts new file mode 100644 index 0000000..0ea9979 --- /dev/null +++ b/packages/validator/src/rules/form-submit-action.ts @@ -0,0 +1,30 @@ +import type { ValidationRule } from '../types.js'; + +export const formSubmitActionRule: ValidationRule = { + id: 'form-submit-action', + name: 'Form Submit Action', + description: 'Checks that every type: form component has a non-empty onSubmit action', + defaultSeverity: 'error', + + validate(context) { + for (const block of context.blocks) { + if (block.data === null) continue; + if (block.data.type !== 'form') continue; + + const id = typeof block.data.id === 'string' ? block.data.id : null; + const onSubmit = block.data.onSubmit; + + if (!onSubmit || typeof onSubmit !== 'string' || onSubmit.trim() === '') { + context.issues.push({ + ruleId: 'form-submit-action', + severity: 'error', + message: 'Form must have an onSubmit action', + componentId: id, + field: 'onSubmit', + blockIndex: block.index, + fixed: false, + }); + } + } + }, +}; diff --git a/packages/validator/src/rules/index.ts b/packages/validator/src/rules/index.ts index 05f6fec..b11422a 100644 --- a/packages/validator/src/rules/index.ts +++ b/packages/validator/src/rules/index.ts @@ -21,6 +21,8 @@ import { placeholderContentRule } from './placeholder-content.js'; // import { unreferencedComponentsRule } from './unreferenced-components.js'; import { flowOrderingRule } from './flow-ordering.js'; import { expectedComponentsRule } from './expected-components.js'; +import { formSubmitActionRule } from './form-submit-action.js'; +import { singleInteractiveComponentRule } from './single-interactive-component.js'; /** * Ordered list of all validation rules. @@ -50,6 +52,8 @@ export const ALL_RULES: readonly ValidationRule[] = [ // unreferencedComponentsRule, flowOrderingRule, expectedComponentsRule, + formSubmitActionRule, + singleInteractiveComponentRule, ]; export function getRulesExcluding(exclude: ValidationRuleId[]): ValidationRule[] { diff --git a/packages/validator/src/rules/single-interactive-component.ts b/packages/validator/src/rules/single-interactive-component.ts new file mode 100644 index 0000000..927ff8a --- /dev/null +++ b/packages/validator/src/rules/single-interactive-component.ts @@ -0,0 +1,29 @@ +import type { ValidationRule } from '../types.js'; + +const INTERACTIVE_TYPES = new Set(['form', 'button', 'webhook', 'approval-gate', 'tasklist']); + +export const singleInteractiveComponentRule: ValidationRule = { + id: 'single-interactive-component', + name: 'Single Interactive Component', + description: 'Warns when a document contains more than one interactive component per message', + defaultSeverity: 'warning', + + validate(context) { + const interactive = context.blocks.filter( + (b) => b.data !== null && INTERACTIVE_TYPES.has(b.data.type as string), + ); + + if (interactive.length <= 1) return; + + const types = interactive.map((b) => `${b.data!.type}#${b.data!.id}`).join(', '); + + context.issues.push({ + ruleId: 'single-interactive-component', + severity: 'warning', + message: `Document contains ${interactive.length} interactive components (${types}) — use at most one interactive component per message`, + componentId: null, + blockIndex: -1, + fixed: false, + }); + }, +}; diff --git a/packages/validator/src/types.ts b/packages/validator/src/types.ts index e7aa99d..6f7f9e0 100644 --- a/packages/validator/src/types.ts +++ b/packages/validator/src/types.ts @@ -18,7 +18,9 @@ export type ValidationRuleId = | 'unreferenced-components' | 'flow-ordering' | 'field-name-typos' - | 'expected-components'; + | 'expected-components' + | 'form-submit-action' + | 'single-interactive-component'; export interface ValidationIssue { /** Which rule flagged this */ diff --git a/packages/validator/tests/fixtures/bad-bindings.md b/packages/validator/tests/fixtures/bad-bindings.md index 0c21c29..eee1252 100644 --- a/packages/validator/tests/fixtures/bad-bindings.md +++ b/packages/validator/tests/fixtures/bad-bindings.md @@ -9,6 +9,7 @@ fields: label: Email required: true sensitive: true +onSubmit: submit-action ``` ```mdma diff --git a/packages/validator/tests/fixtures/mixed-issues.md b/packages/validator/tests/fixtures/mixed-issues.md index 19666ee..0da2d95 100644 --- a/packages/validator/tests/fixtures/mixed-issues.md +++ b/packages/validator/tests/fixtures/mixed-issues.md @@ -11,6 +11,7 @@ fields: - name: phone type: text label: Phone Number +onSubmit: submitBtn ``` ```mdma diff --git a/packages/validator/tests/fixtures/no-thinking-block.md b/packages/validator/tests/fixtures/no-thinking-block.md index 5ae8c17..9dbb283 100644 --- a/packages/validator/tests/fixtures/no-thinking-block.md +++ b/packages/validator/tests/fixtures/no-thinking-block.md @@ -8,6 +8,7 @@ fields: type: text label: Title required: true +onSubmit: submit-btn ``` ```mdma diff --git a/packages/validator/tests/fixtures/pii-missing-sensitive.md b/packages/validator/tests/fixtures/pii-missing-sensitive.md index 8ffa602..7295e37 100644 --- a/packages/validator/tests/fixtures/pii-missing-sensitive.md +++ b/packages/validator/tests/fixtures/pii-missing-sensitive.md @@ -17,6 +17,7 @@ fields: - name: notes type: textarea label: Notes +onSubmit: submit-action ``` ```mdma diff --git a/packages/validator/tests/fixtures/valid-document.md b/packages/validator/tests/fixtures/valid-document.md index 4be037c..767487d 100644 --- a/packages/validator/tests/fixtures/valid-document.md +++ b/packages/validator/tests/fixtures/valid-document.md @@ -27,6 +27,7 @@ fields: - name: message type: textarea label: Message +onSubmit: submit-btn ``` ```mdma diff --git a/packages/validator/tests/rules/form-submit-action.test.ts b/packages/validator/tests/rules/form-submit-action.test.ts new file mode 100644 index 0000000..d41bcd0 --- /dev/null +++ b/packages/validator/tests/rules/form-submit-action.test.ts @@ -0,0 +1,113 @@ +import { describe, it, expect } from 'vitest'; +import { formSubmitActionRule } from '../../src/rules/form-submit-action.js'; +import type { ValidationRuleContext, ParsedBlock } from '../../src/types.js'; + +function createBlock(index: number, data: Record | null): ParsedBlock { + return { + index, + rawYaml: '', + data, + startOffset: 0, + endOffset: 0, + yamlStartOffset: 0, + yamlEndOffset: 0, + }; +} + +function createContext(blocks: ParsedBlock[]): ValidationRuleContext { + const idMap = new Map(); + for (const block of blocks) { + if (block.data && typeof block.data.id === 'string') { + idMap.set(block.data.id, block.index); + } + } + return { blocks, idMap, issues: [], options: {} }; +} + +describe('form-submit-action rule', () => { + it('flags a form missing onSubmit', () => { + const ctx = createContext([ + createBlock(0, { + type: 'form', + id: 'my-form', + fields: [{ name: 'email', type: 'email', label: 'Email' }], + }), + ]); + formSubmitActionRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].ruleId).toBe('form-submit-action'); + expect(ctx.issues[0].severity).toBe('error'); + expect(ctx.issues[0].message).toBe('Form must have an onSubmit action'); + expect(ctx.issues[0].componentId).toBe('my-form'); + expect(ctx.issues[0].field).toBe('onSubmit'); + }); + + it('flags a form with an empty onSubmit', () => { + const ctx = createContext([ + createBlock(0, { + type: 'form', + id: 'my-form', + fields: [{ name: 'email', type: 'email', label: 'Email' }], + onSubmit: '', + }), + ]); + formSubmitActionRule.validate(ctx); + expect(ctx.issues).toHaveLength(1); + expect(ctx.issues[0].ruleId).toBe('form-submit-action'); + }); + + it('passes when form has a valid onSubmit', () => { + const ctx = createContext([ + createBlock(0, { + type: 'form', + id: 'my-form', + fields: [{ name: 'email', type: 'email', label: 'Email' }], + onSubmit: 'submit-webhook', + }), + ]); + formSubmitActionRule.validate(ctx); + const formIssues = ctx.issues.filter((i) => i.ruleId === 'form-submit-action'); + expect(formIssues).toHaveLength(0); + }); + + it('does not flag non-form components', () => { + const ctx = createContext([ + createBlock(0, { type: 'button', id: 'btn', text: 'Submit', onAction: 'some-form' }), + createBlock(1, { type: 'callout', id: 'info', content: 'Hello' }), + ]); + formSubmitActionRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('skips blocks with null data', () => { + const ctx = createContext([createBlock(0, null)]); + formSubmitActionRule.validate(ctx); + expect(ctx.issues).toHaveLength(0); + }); + + it('produces exactly one issue per form missing onSubmit', () => { + const ctx = createContext([ + createBlock(0, { + type: 'form', + id: 'form-a', + fields: [{ name: 'name', type: 'text', label: 'Name' }], + }), + createBlock(1, { + type: 'form', + id: 'form-b', + fields: [{ name: 'email', type: 'email', label: 'Email' }], + onSubmit: 'some-action', + }), + createBlock(2, { + type: 'form', + id: 'form-c', + fields: [{ name: 'phone', type: 'text', label: 'Phone' }], + }), + ]); + formSubmitActionRule.validate(ctx); + const issues = ctx.issues.filter((i) => i.ruleId === 'form-submit-action'); + expect(issues).toHaveLength(2); + expect(issues[0].componentId).toBe('form-a'); + expect(issues[1].componentId).toBe('form-c'); + }); +}); diff --git a/packages/validator/tests/rules/schema-conformance.test.ts b/packages/validator/tests/rules/schema-conformance.test.ts index 58a7332..f618cc6 100644 --- a/packages/validator/tests/rules/schema-conformance.test.ts +++ b/packages/validator/tests/rules/schema-conformance.test.ts @@ -31,6 +31,7 @@ describe('schema-conformance rule', () => { type: 'form', id: 'my-form', fields: [{ name: 'email', type: 'email', label: 'Email' }], + onSubmit: 'submit-action', }), ]); schemaConformanceRule.validate(ctx); @@ -184,6 +185,7 @@ describe('schema-conformance rule', () => { type: 'form', id: 'upload-form', fields: [{ name: 'resume', type: 'file', label: 'Resume', required: true }], + onSubmit: 'submit-action', }), ]); schemaConformanceRule.validate(ctx); diff --git a/packages/validator/tests/rules/single-interactive-component.test.ts b/packages/validator/tests/rules/single-interactive-component.test.ts new file mode 100644 index 0000000..dcf9c9b --- /dev/null +++ b/packages/validator/tests/rules/single-interactive-component.test.ts @@ -0,0 +1,61 @@ +import { describe, it, expect } from 'vitest'; +import { validate } from '../../src/index.js'; + +const doc = (...blocks: string[]) => + blocks.map((b) => `\`\`\`mdma\n${b}\`\`\``).join('\n\n'); + +describe('single-interactive-component rule', () => { + it('passes for a single form', () => { + const result = validate( + doc('type: form\nid: f\nfields:\n - name: x\n type: text\n label: X\nonSubmit: done\n'), + ); + const issues = result.issues.filter((i) => i.ruleId === 'single-interactive-component'); + expect(issues).toHaveLength(0); + }); + + it('passes for one interactive + one non-interactive', () => { + const result = validate( + doc( + 'type: form\nid: f\nfields:\n - name: x\n type: text\n label: X\nonSubmit: c\n', + 'type: callout\nid: c\ncontent: Done\n', + ), + ); + const issues = result.issues.filter((i) => i.ruleId === 'single-interactive-component'); + expect(issues).toHaveLength(0); + }); + + it('warns for form + webhook in same document', () => { + const result = validate( + doc( + 'type: form\nid: f\nfields:\n - name: x\n type: text\n label: X\nonSubmit: w\n', + 'type: webhook\nid: w\nurl: https://example.com\ntrigger: f\n', + ), + ); + const issues = result.issues.filter((i) => i.ruleId === 'single-interactive-component'); + expect(issues).toHaveLength(1); + expect(issues[0].severity).toBe('warning'); + }); + + it('warns for form + approval-gate in same document', () => { + const result = validate( + doc( + 'type: form\nid: f\nfields:\n - name: x\n type: text\n label: X\nonSubmit: g\n', + 'type: approval-gate\nid: g\ntitle: Gate\n', + ), + ); + const issues = result.issues.filter((i) => i.ruleId === 'single-interactive-component'); + expect(issues).toHaveLength(1); + }); + + it('warns once regardless of how many interactive components are present', () => { + const result = validate( + doc( + 'type: form\nid: f\nfields:\n - name: x\n type: text\n label: X\nonSubmit: b\n', + 'type: button\nid: b\ntext: Go\nonAction: f\n', + 'type: tasklist\nid: t\nitems:\n - id: i1\n text: Item\n', + ), + ); + const issues = result.issues.filter((i) => i.ruleId === 'single-interactive-component'); + expect(issues).toHaveLength(1); + }); +}); diff --git a/packages/validator/tests/validate.test.ts b/packages/validator/tests/validate.test.ts index db37fbb..3ff8c51 100644 --- a/packages/validator/tests/validate.test.ts +++ b/packages/validator/tests/validate.test.ts @@ -161,6 +161,7 @@ fields: label: null - name: phone_number type: text +onSubmit: contact-form \`\`\` `; const result = validate(md); @@ -325,6 +326,7 @@ fields: type: email label: Email bind: other-form.email +onSubmit: my-form \`\`\` `; const result = validate(md); @@ -459,6 +461,7 @@ fields: - name: email type: email label: Email +onSubmit: submit-action \`\`\` `; const result = validate(md); @@ -518,6 +521,7 @@ fields: - name: email type: email label: Email +onSubmit: submit-action \`\`\` `; const result = validate(md, { @@ -542,6 +546,7 @@ fields: - name: email type: email label: Email +onSubmit: my-btn \`\`\` \`\`\`mdma From 94a0100221955361de60894ad3a250bfa92406ca Mon Sep 17 00:00:00 2001 From: gitsad Date: Thu, 14 May 2026 16:34:39 +0200 Subject: [PATCH 03/26] chore: WIP 5.5, 5.4 and 5.4-mini --- README.md | 4 +- demo/src/docs/sections/PromptMatrix.tsx | 17 +- evals/.env.example | 43 ++++- evals/tests-conversation.yaml | 4 +- evals/tests-custom-prompt.yaml | 29 ++- evals/tests.yaml | 178 +++++------------- .../src/prompts/mdma-author/_shared.ts | 3 +- .../src/prompts/mdma-author/openai/_shared.ts | 75 +++++++- .../mdma-author/openai/gpt-5.4-mini.ts | 75 ++++++-- .../src/prompts/mdma-author/openai/gpt-5.4.ts | 23 ++- .../src/prompts/mdma-author/openai/gpt-5.5.ts | 12 +- 11 files changed, 281 insertions(+), 182 deletions(-) diff --git a/README.md b/README.md index b27227b..56d7277 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ Each cell shows the pass rate of the model-specialized MDMA_AUTHOR prompt varian | :--- | :---: | :---: | :---: | :---: | | **OpenAI** | | | | | | `gpt-5.5` | ✅ | ✅ | ✅ | ✅ | -| `gpt-5.4` | ✅ | ✅ | ✅ | ✅ | +| `gpt-5.4` | ✅ | 🟡 † | 🟡 † | 🟡 † | | `gpt-5.4-mini` | ✅ | ✅ | ✅ \* | ✅ \* | | `gpt-5.4-nano` | ✅ | ✅ | ✅ \* | ✅ \* | | `gpt-5.2` | ✅ | ✅ | ✅ | ✅ | @@ -115,6 +115,8 @@ Each cell shows the pass rate of the model-specialized MDMA_AUTHOR prompt varian > **Don't see your model?** Add a prompt variant under `packages/prompt-pack/src/prompts/mdma-author//` and open a PR — we'll run the eval suite and add it to this table. +† **gpt-5.4 intermittent duplication bug** — `gpt-5.4` passes one-shot evals reliably but shows a non-deterministic output duplication in multi-turn, custom-prompt, and flow evals (~7–15% of runs). The model generates a complete, correct response and then immediately re-emits the entire output verbatim, causing `[duplicate-ids]` validation errors. This is a known model-level issue unrelated to the prompt variant. See the [OpenAI community thread](https://community.openai.com/t/seeing-intermittent-duplicate-strings-in-gpt-5-4-responses/1376651) for details. If this affects your use case, prefer `gpt-5.5` or `gpt-5.2`. + \* Smaller / lower-tier models from any lab (OpenAI mini · nano, Anthropic Haiku, Google Gemini Flash, etc.) pass our eval suites, which exercise short, structured test cases. In longer real-world conversations they tend to hallucinate, forget earlier turns, or drift from the spec. For production use that involves multi-turn dialogue or stateful flows, prefer the flagship-tier model from the same family. \[i] Noticeably slow response times — single-turn responses commonly take tens of seconds and full eval runs measure in minutes. diff --git a/demo/src/docs/sections/PromptMatrix.tsx b/demo/src/docs/sections/PromptMatrix.tsx index 04b0302..722067f 100644 --- a/demo/src/docs/sections/PromptMatrix.tsx +++ b/demo/src/docs/sections/PromptMatrix.tsx @@ -13,7 +13,7 @@ export function PromptMatrix() { headers={['Variant', 'one-shot', 'one-shot custom', 'conversation', 'specific flow']} rows={[ ['gpt-5.5', '✅', '✅', '✅', '✅'], - ['gpt-5.4', '✅', '✅', '✅', '✅'], + ['gpt-5.4', '✅', '🟡 †', '🟡 †', '🟡 †'], ['gpt-5.4-mini', '✅', '✅', '✅ *', '✅ *'], ['gpt-5.4-nano', '✅', '✅', '✅ *', '✅ *'], ['gpt-5.2', '✅', '✅', '✅', '✅'], @@ -46,6 +46,21 @@ export function PromptMatrix() {

[i] Noticeably slow response times — single-turn responses commonly take tens of seconds.

+

+ † gpt-5.4 intermittent duplication bug — passes one-shot evals reliably + but shows non-deterministic output duplication in multi-turn, custom-prompt, and flow evals + (~7–15% of runs). The model generates a correct response then immediately re-emits it + verbatim, causing [duplicate-ids] validation errors. This is a known + model-level issue unrelated to the prompt variant.{' '} + + See OpenAI community thread. + {' '} + Prefer gpt-5.5 or gpt-5.2 for production use. +

MDMA_AGENT Prompt Matrix

diff --git a/evals/.env.example b/evals/.env.example index 85d62b3..4d3940c 100644 --- a/evals/.env.example +++ b/evals/.env.example @@ -7,11 +7,40 @@ OPENAI_API_KEY= # Get one at https://openrouter.ai/keys OPENROUTER_API_KEY= -# Optional: pin the model used by every eval run. +# Pin the model used by every eval run. # Inline `EVAL_PROVIDER=... pnpm eval` overrides this for one-off runs. -# Examples: -# EVAL_PROVIDER=openai:gpt-4o -# EVAL_PROVIDER=openrouter:anthropic/claude-sonnet-4 -# EVAL_PROVIDER=openrouter:google/gemini-2.5-pro -# EVAL_PROVIDER=openrouter:meta-llama/llama-3.3-70b-instruct -# EVAL_PROVIDER= +# Uncomment exactly one line below: + +# --- OpenAI --- +#EVAL_PROVIDER=openai:gpt-5.5 +#EVAL_PROVIDER=openai:gpt-5.4 +#EVAL_PROVIDER=openai:gpt-5.4-mini +#EVAL_PROVIDER=openai:gpt-5.4-nano +#EVAL_PROVIDER=openai:gpt-5.2 +#EVAL_PROVIDER=openai:gpt-5.1 +#EVAL_PROVIDER=openai:gpt-5 +#EVAL_PROVIDER=openai:gpt-5-mini +#EVAL_PROVIDER=openai:gpt-5-nano +#EVAL_PROVIDER=openai:gpt-4.1 +#EVAL_PROVIDER=openai:gpt-4.1-mini +#EVAL_PROVIDER=openai:gpt-4.1-nano + +# --- Anthropic (via OpenRouter) --- +#EVAL_PROVIDER=openrouter:anthropic/claude-opus-4-7 +#EVAL_PROVIDER=openrouter:anthropic/claude-opus-4-6 +#EVAL_PROVIDER=openrouter:anthropic/claude-sonnet-4-5 +#EVAL_PROVIDER=openrouter:anthropic/claude-haiku-4-5 + +# --- Google (via OpenRouter) --- +#EVAL_PROVIDER=openrouter:google/gemini-3.1-pro-preview +#EVAL_PROVIDER=openrouter:google/gemini-3.1-flash-lite-preview +#EVAL_PROVIDER=openrouter:google/gemini-3-flash-preview +#EVAL_PROVIDER=openrouter:google/gemini-2.5-pro +#EVAL_PROVIDER=openrouter:google/gemini-2.5-flash +#EVAL_PROVIDER=openrouter:google/gemini-2.5-flash-lite + +# --- xAI (via OpenRouter) --- +#EVAL_PROVIDER=openrouter:x-ai/grok-4.20 +#EVAL_PROVIDER=openrouter:x-ai/grok-4.3 + +EVAL_PROVIDER=openai:gpt-5.5 diff --git a/evals/tests-conversation.yaml b/evals/tests-conversation.yaml index 2e54449..86f00df 100644 --- a/evals/tests-conversation.yaml +++ b/evals/tests-conversation.yaml @@ -678,13 +678,11 @@ - type: javascript value: file://assertions/only-components.mjs config: - allowed: [form, approval-gate, button] + allowed: [form] - type: javascript value: file://assertions/exact-field-count.mjs config: expected: 3 - - type: contains - value: "requiredApprovers: 1" - description: "Conv 10/T2: User asks about approval process — no regeneration" vars: diff --git a/evals/tests-custom-prompt.yaml b/evals/tests-custom-prompt.yaml index 214cc59..c384875 100644 --- a/evals/tests-custom-prompt.yaml +++ b/evals/tests-custom-prompt.yaml @@ -44,6 +44,7 @@ - name: actual type: textarea label: "Actual Behavior" + onSubmit: bug-submitted ``` Generate only this form. No buttons, callouts, or other components. @@ -106,6 +107,7 @@ type: date label: "Start Date" required: true + onSubmit: onboarding-checklist ``` ```mdma @@ -130,7 +132,7 @@ - type: javascript value: file://assertions/only-components.mjs config: - allowed: [form, tasklist] + allowed: [form] - type: javascript value: file://assertions/exact-field-count.mjs config: @@ -143,8 +145,6 @@ value: file://assertions/has-required-fields.mjs config: min: 3 - - type: contains - value: "type: tasklist" # --------------------------------------------------------------------------- # 3. Customer feedback — form + pie chart MDMA blueprint @@ -183,6 +183,7 @@ - name: feedback type: textarea label: "Comments" + onSubmit: feedback-submitted ``` ```mdma @@ -283,19 +284,13 @@ - type: javascript value: file://assertions/only-components.mjs config: - allowed: [form, approval-gate, button] + allowed: [form] - type: javascript value: file://assertions/exact-field-count.mjs config: expected: 5 - type: javascript value: file://assertions/select-has-options.mjs - - type: contains - value: "type: approval-gate" - - type: contains - value: "requiredApprovers: 1" - - type: contains - value: "type: button" # --------------------------------------------------------------------------- # 5. IT ticket — form + webhook MDMA blueprint @@ -362,13 +357,11 @@ - type: javascript value: file://assertions/only-components.mjs config: - allowed: [form, webhook] + allowed: [form] - type: javascript value: file://assertions/exact-field-count.mjs config: expected: 4 - - type: javascript - value: file://assertions/has-webhook.mjs - type: javascript value: file://assertions/has-sensitive.mjs - type: javascript @@ -479,6 +472,7 @@ type: textarea label: "Chief Complaint" required: true + onSubmit: patient-registered ``` Generate only this form. No other components. @@ -580,6 +574,7 @@ value: amendment - label: Renewal value: renewal + onSubmit: review-checklist ``` ```mdma @@ -617,7 +612,7 @@ - type: javascript value: file://assertions/only-components.mjs config: - allowed: [form, tasklist, approval-gate] + allowed: [form] - type: javascript value: file://assertions/exact-field-count.mjs config: @@ -628,10 +623,6 @@ value: file://assertions/has-required-fields.mjs config: min: 4 - - type: contains - value: "requiredApprovers: 2" - - type: contains - value: "allowedRoles:" # --------------------------------------------------------------------------- # 10b. Recruiting — job application with file uploads (resume + portfolio) @@ -666,6 +657,7 @@ - name: cover-letter type: textarea label: "Cover Letter" + onSubmit: application-submitted ``` Mark only the email as sensitive. Generate only this form — no @@ -719,6 +711,7 @@ label: "Proof of Address (utility bill or bank statement)" required: true sensitive: true + onSubmit: kyc-identity-verified ``` Every field in this form is PII and MUST have `sensitive: true`. diff --git a/evals/tests.yaml b/evals/tests.yaml index 6b4081c..acdf8c7 100644 --- a/evals/tests.yaml +++ b/evals/tests.yaml @@ -29,6 +29,7 @@ type: textarea label: "Message" required: true + onSubmit: contact-submitted ``` assert: - type: javascript @@ -72,6 +73,7 @@ label: "Social Security Number" required: true sensitive: true + onSubmit: employee-pii-submitted ``` assert: - type: javascript @@ -157,6 +159,7 @@ value: au - label: "Germany" value: de + onSubmit: country-form-submitted ``` assert: - type: javascript @@ -261,10 +264,10 @@ # --------------------------------------------------------------------------- # 9. Multi-component incident triage workflow # --------------------------------------------------------------------------- -- description: Generates a multi-component incident triage workflow +- description: Generates an incident triage severity form vars: request: | - Create an incident triage workflow with these exact components: + Create an incident triage form matching this exact structure: ```mdma type: form @@ -292,37 +295,13 @@ value: medium - label: Low value: low - ``` - - ```mdma - type: tasklist - id: response-checklist - items: - - id: identify - text: "Identify affected systems" - - id: assess - text: "Assess blast radius" - - id: communicate - text: "Notify stakeholders" - - id: mitigate - text: "Apply mitigation steps" - - id: document - text: "Document root cause" - onComplete: checklist-done - ``` - - ```mdma - type: button - id: notify-team-btn - text: "Notify Team" - variant: primary - onAction: notify-team + onSubmit: triage-submitted ``` assert: - type: javascript value: file://assertions/only-components.mjs config: - allowed: [form, tasklist, button] + allowed: [form] - type: javascript value: file://assertions/exact-field-count.mjs config: @@ -336,58 +315,29 @@ - description: All component IDs are unique and kebab-case vars: request: | - Create three separate forms matching these exact structures: + Create a user settings workflow with these exact components: ```mdma type: form - id: login-form + id: settings-form fields: + - name: display-name + type: text + label: "Display Name" + required: true - name: email type: email label: "Email" required: true sensitive: true - - name: password - type: text - label: "Password" - required: true - sensitive: true - ``` - - ```mdma - type: form - id: feedback-form - fields: - - name: rating - type: select - label: "Rating" - options: - - label: "1" - value: "1" - - label: "2" - value: "2" - - label: "3" - value: "3" - - label: "4" - value: "4" - - label: "5" - value: "5" - - name: comment - type: textarea - label: "Comment" + onSubmit: settings-saved ``` ```mdma - type: form - id: profile-form - fields: - - name: display-name - type: text - label: "Display Name" - required: true - - name: bio - type: textarea - label: "Bio" + type: callout + id: settings-saved + variant: success + content: "Your settings have been saved successfully." ``` assert: - type: javascript @@ -395,7 +345,7 @@ - type: javascript value: file://assertions/only-components.mjs config: - allowed: [form] + allowed: [form, callout] - type: javascript value: file://assertions/component-count.mjs config: @@ -541,10 +491,10 @@ # --------------------------------------------------------------------------- # 16. Webhook component # --------------------------------------------------------------------------- -- description: Generates a form with webhook matching blueprint +- description: Generates a support ticket form with submission confirmation vars: request: | - Create a support ticket form with a webhook matching these exact structures: + Create a support ticket form matching these exact structures: ```mdma type: form @@ -558,28 +508,24 @@ type: textarea label: "Description" required: true - onSubmit: submit-ticket + onSubmit: ticket-submitted ``` ```mdma - type: webhook - id: ticket-webhook - url: "https://api.example.com/tickets" - method: POST - headers: - Content-Type: application/json - body: - subject: "{{ticket-form.subject}}" - description: "{{ticket-form.description}}" - trigger: submit-ticket + type: callout + id: ticket-submitted + variant: success + content: "Your support ticket has been submitted. We'll get back to you shortly." ``` assert: - type: javascript value: file://assertions/only-components.mjs config: - allowed: [form, webhook] + allowed: [form, callout] - type: javascript - value: file://assertions/has-webhook.mjs + value: file://assertions/component-count.mjs + config: + min: 3 # --------------------------------------------------------------------------- # 17. Table with sortable and filterable features @@ -658,6 +604,7 @@ label: "Billing Address" required: true sensitive: true + onSubmit: payment-submitted ``` assert: - type: javascript @@ -700,6 +647,7 @@ - name: bio type: textarea label: "Bio" + onSubmit: registration-submitted ``` assert: - type: javascript @@ -767,6 +715,7 @@ - name: notifications-enabled type: checkbox label: "Enable Notifications" + onSubmit: notification-info ``` ```mdma @@ -788,10 +737,10 @@ # --------------------------------------------------------------------------- # 22. Complex multi-component — HR onboarding workflow # --------------------------------------------------------------------------- -- description: Generates a large multi-component HR onboarding document +- description: Generates a large multi-field HR personal info form with sensitive data vars: request: | - Create an HR onboarding workflow with these exact components: + Create the first step of an HR onboarding workflow with these exact components: ```mdma type: callout @@ -827,66 +776,24 @@ label: "Social Security Number" required: true sensitive: true + onSubmit: info-submitted ``` ```mdma - type: form - id: equipment-form - fields: - - name: laptop - type: select - label: "Laptop Model" - required: true - options: - - label: "MacBook Pro" - value: macbook-pro - - label: "ThinkPad X1" - value: thinkpad - - label: "Dell XPS 15" - value: dell-xps - ``` - - ```mdma - type: tasklist - id: onboarding-tasks - items: - - id: badge-photo - text: "Upload badge photo" - - id: parking-pass - text: "Request parking pass" - - id: nda-signed - text: "Sign NDA" - required: true - - id: orientation - text: "Attend orientation session" - required: true - ``` - - ```mdma - type: approval-gate - id: manager-approval - title: "Manager Approval" - requiredApprovers: 1 - ``` - - ```mdma - type: button - id: complete-onboarding-btn - text: "Complete Onboarding" - variant: primary - onAction: finish-onboarding + type: callout + id: info-submitted + variant: success + content: "Personal information submitted. We'll continue with equipment selection next." ``` assert: - type: javascript value: file://assertions/component-count.mjs config: - min: 7 + min: 3 - type: javascript value: file://assertions/unique-kebab-ids.mjs - type: javascript value: file://assertions/has-sensitive.mjs - - type: contains - value: "type: approval-gate" # --------------------------------------------------------------------------- # 23. Approval gate with role restrictions @@ -961,6 +868,7 @@ type: textarea label: "Known Allergies" sensitive: true + onSubmit: patient-intake-submitted ``` assert: - type: javascript @@ -1063,6 +971,7 @@ type: file label: "Resume" required: true + onSubmit: resume-submitted ``` assert: - type: javascript @@ -1098,6 +1007,7 @@ label: "Passport Scan" required: true sensitive: true + onSubmit: kyc-submitted ``` assert: - type: javascript diff --git a/packages/prompt-pack/src/prompts/mdma-author/_shared.ts b/packages/prompt-pack/src/prompts/mdma-author/_shared.ts index 4463ae2..8994912 100644 --- a/packages/prompt-pack/src/prompts/mdma-author/_shared.ts +++ b/packages/prompt-pack/src/prompts/mdma-author/_shared.ts @@ -315,7 +315,7 @@ When a user request includes \`visible\` or \`disabled\` with a \`{{}}\` binding 1. **Unique IDs** — Every component \`id\` must be unique within the document. Use descriptive kebab-case names (e.g., \`employee-onboarding-form\`, \`submit-btn\`). 2. **Sensitive data** — Set \`sensitive: true\` on any field or column that contains PII (personally identifiable information) such as email addresses, phone numbers, SSNs, addresses, or financial data. 3. **Required fields** — Mark form fields as \`required: true\` when the workflow cannot proceed without them. -4. **Action references** — All \`onSubmit\`, \`onAction\`, \`onComplete\`, \`onApprove\`, \`onDeny\`, and \`trigger\` values should reference valid action IDs within the document. +4. **Action references** — Every \`type: form\` MUST include an \`onSubmit\` field pointing to a valid component ID in the document (e.g., a confirmation callout). All other action fields (\`onAction\`, \`onComplete\`, \`onApprove\`, \`onDeny\`, \`trigger\`) must also reference valid IDs. If no target exists yet, create a \`type: callout\` as the submission confirmation target. 5. **Binding validity** — Every \`{{binding}}\` must reference a valid source. Do not leave unresolved bindings. 6. **Minimal components** — Only include components that are necessary for the workflow. Avoid empty or placeholder components. 7. **YAML correctness** — Ensure all YAML in mdma blocks is valid and properly indented. Always wrap string values in double quotes if they contain a colon followed by a space (\`: \`), e.g. \`label: "Step 1: Enter your info"\`. @@ -332,6 +332,7 @@ Before finalizing an MDMA document, verify: - [ ] All PII fields have \`sensitive: true\` - [ ] All \`{{bindings}}\` reference valid sources - [ ] Required form fields are marked \`required: true\` +- [ ] Every \`type: form\` has an \`onSubmit\` field pointing to a valid component ID - [ ] Action IDs referenced in event handlers exist in the document - [ ] Select fields include an \`options\` array - [ ] YAML syntax is valid in all mdma blocks diff --git a/packages/prompt-pack/src/prompts/mdma-author/openai/_shared.ts b/packages/prompt-pack/src/prompts/mdma-author/openai/_shared.ts index 09ff18b..571f6ec 100644 --- a/packages/prompt-pack/src/prompts/mdma-author/openai/_shared.ts +++ b/packages/prompt-pack/src/prompts/mdma-author/openai/_shared.ts @@ -9,9 +9,9 @@ * * Variant matrix (which blocks each variant pulls in): * - * gpt-5.5 CRITICAL_OUTPUT_LINE - * gpt-5.4 CRITICAL_OUTPUT_LINE + SCOPE_DISCIPLINE_BLOCK - * gpt-5.4-mini CRITICAL_OUTPUT_LINE + FENCE_CLOSING_BLOCK + SELECT_OPTIONS_BLOCK + * gpt-5.5 CRITICAL_OUTPUT_LINE + SCOPE_DISCIPLINE_BLOCK + INTERACTIVE_TYPES_BLOCK + SINGLE_INTERACTIVE_BLOCK + SELECT_OPTIONS_BLOCK + * gpt-5.4 CRITICAL_OUTPUT_LINE + FENCE_CLOSING_BLOCK + SCOPE_DISCIPLINE_BLOCK + INTERACTIVE_TYPES_BLOCK + SINGLE_INTERACTIVE_BLOCK + THINKING_ROLE_BLOCK + NO_REPEAT_BLOCK + NO_DUPLICATES_BLOCK + * gpt-5.4-mini CRITICAL_OUTPUT_LINE + FENCE_CLOSING_BLOCK + SCOPE_DISCIPLINE_BLOCK + INTERACTIVE_TYPES_BLOCK + SINGLE_INTERACTIVE_BLOCK + SELECT_OPTIONS_BLOCK + THINKING_ROLE_BLOCK + NO_REPEAT_BLOCK + NO_DUPLICATES_BLOCK * gpt-5.4-nano all of the above */ @@ -69,7 +69,7 @@ A new \`\`\`mdma after a still-open block is treated as text inside the open blo * vector observed in the eval suite. */ export const SCOPE_DISCIPLINE_BLOCK = ` -1. Emit only the component types the user has explicitly listed or provided in a blueprint. If the user lists "form, tasklist, button, thinking", do not also emit webhooks, callouts, charts, approval-gates, or any other type. +1. Emit only the component types the user has explicitly listed or provided in a blueprint. If the user lists "form, tasklist, button, thinking", do not also emit webhooks, callouts, charts, approval-gates, or any other type. Note: when a blueprint lists multiple interactive components, the limit still applies — emit only the first interactive component from the list. 2. When the user provides a YAML blueprint of one component, output exactly that one component (plus the standard thinking block). Action-id values inside the blueprint — \`onApprove\`, \`onDeny\`, \`onSubmit\`, \`onAction\`, \`trigger\`, \`onComplete\` — are opaque string labels. Do NOT generate webhook, button, callout, or any other handler components to "complete" or "wire up" the workflow. @@ -78,6 +78,41 @@ export const SCOPE_DISCIPLINE_BLOCK = ` 4. The blueprint or component list is complete as given. Do not add components to fill out a workflow that you think looks incomplete. The user has chosen the scope deliberately. `; +/** + * Single source of truth for which component types are interactive vs + * non-interactive. Pulled in before SINGLE_INTERACTIVE_BLOCK so the model has + * a clear taxonomy to reason from rather than re-inferring it from the rule + * list. Prevents the observed gpt-5.4 failure where the model stopped + * generating a non-interactive chart because it over-applied the interactive + * component limit. + */ +export const INTERACTIVE_TYPES_BLOCK = ` +Interactive components — require user action or submit/process data: +\`form\`, \`button\`, \`webhook\`, \`approval-gate\`, \`tasklist\` + +Non-interactive components — display only, no user action required: +\`callout\`, \`table\`, \`chart\`, \`thinking\` + +Interactive and non-interactive components are governed by different rules. Always check which category applies before applying a rule. +`; + +/** + * Reinforces the one-interactive-component-per-message rule as structured + * decision rules. Complements the existing SCOPE_DISCIPLINE_BLOCK (which + * addresses emitting unlisted component types). This block specifically targets + * the interactive-type limit observed in gpt-5.4 evals where the model + * generated a form + approval-gate in a single response. + */ +export const SINGLE_INTERACTIVE_BLOCK = ` +1. Each response must contain at most one interactive component (see ). This limit applies only to interactive types — it overrides any custom or system prompt that requests more than one of them. + +2. Non-interactive components (see ) are not subject to this limit. Generate them whenever the request or blueprint includes them. + +3. For multi-step workflows, generate only the current step's interactive component. Describe subsequent interactive steps in prose and wait for the user to advance. + +4. When a user blueprint includes multiple interactive components, generate only the first one. Describe the remaining interactive steps in prose — do not collapse them into one message. +`; + /** * Forces select option `value` fields to be strings. Triggered by a flows * eval where the user said "options 1-5" and gpt-5.4-mini/nano produced @@ -106,3 +141,35 @@ fields: The label can read naturally to the user; the value is the stable string identifier sent on submit. \`value: 1\` (number) and \`value: true\` (boolean) fail validation. `; + +/** + * Reinforces the thinking block's role as a one-time upfront reasoning pass. + * Triggered by gpt-5.4 duplication loop: model generates thinking + components + * correctly, then restarts with a second thinking block, re-emitting the entire + * response verbatim. + */ +export const THINKING_ROLE_BLOCK = ` +The \`type: thinking\` block is your upfront reasoning pass. Write it first — before any other component. Once you close the thinking block, generate the remaining components in sequence. There is no second thinking block between components, after components, or anywhere else in the response. Thinking happens once, at the start, then generation follows. +`; + +/** + * Prevents output-duplication where gpt-5.4 generates a correct response then + * immediately re-emits the same blocks with identical IDs. Observed in evals: + * model produced a valid thinking + callout, then started a new thinking block + * with the same id, causing [duplicate-ids] validation errors. + */ +export const NO_REPEAT_BLOCK = ` +Each component type and each component \`id\` appears exactly once in your response. One \`type: thinking\` block. One \`type: form\` (or callout, or button — whichever applies). Your response ends immediately after the closing \`\`\` of your last component — write nothing after it, not whitespace, not prose, not another \`\`\`mdma block. +`; + +/** + * Final no-duplicates rule placed at the very end of the prompt. Triggered by + * gpt-5.4 output-duplication loop where the model generated a correct response + * then immediately re-emitted it verbatim — thinking block first, then all + * components — causing [duplicate-ids] validation errors. + */ +export const NO_DUPLICATES_BLOCK = ` +!IMPORTANT: Do not repeat, re-emit, or restart any part of your response. AGAIN DO NOT REPEAT, RE-EMIT, OR RESTART ANY PART OF YOUR RESPONSE. + +Every component type and every component \`id\` appears exactly once in your response. The \`type: thinking\` block is written once, at the start. Each other component is written once, in sequence. Your response ends immediately after the closing \`\`\` of your last component — do not repeat, restart, or re-emit anything already written. +`; diff --git a/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.4-mini.ts b/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.4-mini.ts index 8e1bbfa..a450f09 100644 --- a/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.4-mini.ts +++ b/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.4-mini.ts @@ -1,31 +1,84 @@ /** * MDMA Author Prompt — OpenAI GPT-5.4-mini variant. * - * Adds + on top of the canonical opening: + * Full gpt-5.4 block set plus : * - * - — when emitting a thinking block (or any component) - * with a YAML \`content: |\` block scalar followed - * by another component, gpt-5.4-mini sometimes - * forgets to close the \`\`\`mdma fence with three - * backticks before the next block, breaking - * CommonMark parsing. - * - — for \`type: select\` fields, mini defaulted to - * \`value: 1\` (number) when the user described - * options as 1–5; the schema requires string values. + * - — mini forgets to close ```mdma fences after + * YAML `content: |` block scalars. + * - — same over-elaboration pattern as gpt-5.4. + * - — interactive vs non-interactive taxonomy. + * - — one interactive component per response. + * - — mini produced `value: 1` (number) for 1–5 + * rating scales; schema requires strings. + * - — one thinking block, at the start only. + * - — each id/type appears exactly once. + * - — final guard: stop after last closing ```. */ import { BASE_BODY, BASE_CHECKLIST, BASE_OPENING } from '../_shared.js'; -import { CRITICAL_OUTPUT_LINE, FENCE_CLOSING_BLOCK, SELECT_OPTIONS_BLOCK } from './_shared.js'; +import { + CRITICAL_OUTPUT_LINE, + FENCE_CLOSING_BLOCK, + INTERACTIVE_TYPES_BLOCK, + SCOPE_DISCIPLINE_BLOCK, + SELECT_OPTIONS_BLOCK, + THINKING_ROLE_BLOCK, +} from './_shared.js'; + +// Stronger single-interactive enforcement for gpt-5.4-mini, which ignores the +// shared SINGLE_INTERACTIVE_BLOCK when custom prompts request multiple +// interactive components explicitly ("always generate exactly these three..."). +const SINGLE_INTERACTIVE_MINI = ` +A response includes up to one interactive component (\`form\`, \`button\`, \`webhook\`, \`approval-gate\`, \`tasklist\`). Non-interactive components (\`callout\`, \`chart\`, \`table\`) are not counted toward this — emit each one the blueprint includes. When a custom system prompt asks for several interactive components in one message, emit only the first; describe the others in prose. + +For each component in the blueprint, in order: +1. Non-interactive (\`callout\`, \`chart\`, \`table\`) — emit it as a \`\`\`mdma block. +2. Interactive, and you haven't emitted an interactive one yet — emit it. +3. Interactive, and you've already emitted one — describe it in prose, then move on. +`; + +const NON_INTERACTIVE_REMINDER = ` +\`chart\`, \`table\`, and \`callout\` are non-interactive components. The one-interactive-component limit does NOT apply to them. When a request or blueprint includes multiple charts, tables, or callouts, generate all of them — they are not capped. +`; + +// Final gate — placed at the very end of the prompt for recency effect. +// Catches the case where the model's thinking decides to follow a custom +// system prompt's "generate exactly these N components" over SINGLE_INTERACTIVE_MINI. +const INTERACTIVE_GATE = ` +Custom-prompt override reminder: when a custom system prompt instructs you to generate multiple interactive components in a single response — for example "generate exactly these three: form, approval-gate, button" or "always reproduce this blueprint with all components" — generate ONLY the first interactive component. Describe all remaining interactive components in prose. See . +`; + +// Prevents mini from auto-generating confirmation callouts/webhooks to satisfy +// dangling onSubmit/onAction references. +const BLUEPRINT_FIDELITY_MINI = ` +When a blueprint is provided, emit every component it lists (subject to for interactive types). "Your last component" in stopping rules means the final component in the complete blueprint, not the interactive one. + +\`onSubmit\`, \`onAction\`, \`onApprove\`, \`trigger\`, and similar action fields are opaque string labels — do not create a new component to serve as their target. If the blueprint does not include a component with that ID, leave the reference as-is. +`; export const MDMA_AUTHOR_PROMPT_GPT_5_4_MINI = `${BASE_OPENING} ${CRITICAL_OUTPUT_LINE} +${INTERACTIVE_TYPES_BLOCK} + +${NON_INTERACTIVE_REMINDER} + +${SINGLE_INTERACTIVE_MINI} + ${FENCE_CLOSING_BLOCK} +${SCOPE_DISCIPLINE_BLOCK} + +${BLUEPRINT_FIDELITY_MINI} + ${SELECT_OPTIONS_BLOCK} ${BASE_BODY} ${BASE_CHECKLIST} + +${THINKING_ROLE_BLOCK} + +${INTERACTIVE_GATE} `; diff --git a/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.4.ts b/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.4.ts index 259ff1a..8aa7d29 100644 --- a/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.4.ts +++ b/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.4.ts @@ -8,15 +8,36 @@ */ import { BASE_BODY, BASE_CHECKLIST, BASE_OPENING } from '../_shared.js'; -import { CRITICAL_OUTPUT_LINE, SCOPE_DISCIPLINE_BLOCK } from './_shared.js'; +import { + CRITICAL_OUTPUT_LINE, + FENCE_CLOSING_BLOCK, + INTERACTIVE_TYPES_BLOCK, + NO_DUPLICATES_BLOCK, + NO_REPEAT_BLOCK, + SCOPE_DISCIPLINE_BLOCK, + SINGLE_INTERACTIVE_BLOCK, + THINKING_ROLE_BLOCK, +} from './_shared.js'; export const MDMA_AUTHOR_PROMPT_GPT_5_4 = `${BASE_OPENING} ${CRITICAL_OUTPUT_LINE} +${FENCE_CLOSING_BLOCK} + ${SCOPE_DISCIPLINE_BLOCK} +${INTERACTIVE_TYPES_BLOCK} + +${SINGLE_INTERACTIVE_BLOCK} + ${BASE_BODY} ${BASE_CHECKLIST} + +${THINKING_ROLE_BLOCK} + +${NO_REPEAT_BLOCK} + +${NO_DUPLICATES_BLOCK} `; diff --git a/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.5.ts b/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.5.ts index 92b7292..7fd4603 100644 --- a/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.5.ts +++ b/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.5.ts @@ -16,7 +16,13 @@ */ import { BASE_BODY, BASE_CHECKLIST, BASE_OPENING } from '../_shared.js'; -import { CRITICAL_OUTPUT_LINE, SCOPE_DISCIPLINE_BLOCK, SELECT_OPTIONS_BLOCK } from './_shared.js'; +import { + CRITICAL_OUTPUT_LINE, + INTERACTIVE_TYPES_BLOCK, + SCOPE_DISCIPLINE_BLOCK, + SELECT_OPTIONS_BLOCK, + SINGLE_INTERACTIVE_BLOCK, +} from './_shared.js'; export const MDMA_AUTHOR_PROMPT_GPT_5_5 = `${BASE_OPENING} @@ -24,6 +30,10 @@ ${CRITICAL_OUTPUT_LINE} ${SCOPE_DISCIPLINE_BLOCK} +${INTERACTIVE_TYPES_BLOCK} + +${SINGLE_INTERACTIVE_BLOCK} + ${SELECT_OPTIONS_BLOCK} ${BASE_BODY} From f2d0ad07e8ae32fae45e4dabd4da824b9e72f818 Mon Sep 17 00:00:00 2001 From: gitsad Date: Thu, 14 May 2026 17:28:44 +0200 Subject: [PATCH 04/26] chore: WIP gpt-5.4-min --- evals/tests-conversation.yaml | 23 +++++-- evals/tests-custom-prompt.yaml | 62 +++++++++++++++---- evals/tests.yaml | 6 +- .../mdma-author/openai/gpt-5.4-mini.ts | 4 +- 4 files changed, 71 insertions(+), 24 deletions(-) diff --git a/evals/tests-conversation.yaml b/evals/tests-conversation.yaml index 86f00df..7073f73 100644 --- a/evals/tests-conversation.yaml +++ b/evals/tests-conversation.yaml @@ -623,8 +623,11 @@ - description: "Conv 10/T1: Generate expense workflow from blueprint" vars: customPrompt: &conv10_prompt | - You are a finance assistant. When the user asks for an expense report - workflow, generate exactly these three components: + You are a finance assistant. The expense report workflow has three + turns: + + Turn 1 — When the user first asks for the workflow, generate this + form to capture the expense details: ```mdma type: form @@ -652,6 +655,9 @@ label: "Receipt Description" ``` + Turn 2 — After the user submits the form, the next assistant message + will present this approval gate for manager sign-off: + ```mdma type: approval-gate id: expense-approval @@ -659,6 +665,9 @@ requiredApprovers: 1 ``` + Turn 3 — Once the approval is in, the final assistant message will + offer this submit button: + ```mdma type: button id: submit-btn @@ -667,10 +676,12 @@ onAction: submit-expense ``` - Generate these three components only once when the user first requests - it. No callouts, charts, or webhooks beyond these three. For any - follow-up questions, respond conversationally in plain text without - regenerating the components. + For the initial response (Turn 1), generate only the form. The + approval gate and button are follow-up steps in later turns — do not + include them now. No callouts, charts, or webhooks beyond these + three. For any follow-up questions about the workflow itself, + respond conversationally in plain text without regenerating any + components. message: I need an expense report workflow. metadata: conversationId: conv-10 diff --git a/evals/tests-custom-prompt.yaml b/evals/tests-custom-prompt.yaml index c384875..3cbad7d 100644 --- a/evals/tests-custom-prompt.yaml +++ b/evals/tests-custom-prompt.yaml @@ -73,8 +73,11 @@ - description: "Generates prescribed onboarding form and checklist" vars: customPrompt: | - You are an HR onboarding assistant. For every new hire, generate - exactly these two components: + You are an HR onboarding assistant. The onboarding workflow has two + turns: + + Turn 1 — In the initial response, generate this form to collect new + hire details: ```mdma type: form @@ -110,6 +113,9 @@ onSubmit: onboarding-checklist ``` + Turn 2 — After the new hire submits the form, the next assistant + message will show this onboarding checklist: + ```mdma type: tasklist id: onboarding-checklist @@ -126,7 +132,9 @@ text: "Meet your team lead" ``` - Generate only these two components. No buttons, callouts, or others. + For the initial response, generate only the form. The tasklist is + a follow-up step and appears in the next turn — do not include it + now. No buttons, callouts, or other components. request: We have a new hire starting in the Design department next Monday. assert: - type: javascript @@ -224,8 +232,11 @@ - description: "Generates exact expense workflow from MDMA blueprint" vars: customPrompt: | - You are a finance assistant. When a user submits an expense, always - generate exactly these three components in this order: + You are a finance assistant. The expense submission workflow has three + turns: + + Turn 1 — In the initial response, generate this form to collect the + expense details: ```mdma type: form @@ -262,6 +273,9 @@ onSubmit: approve-expense ``` + Turn 2 — After the user submits the form, the next assistant message + will present this approval gate for manager sign-off: + ```mdma type: approval-gate id: expense-approval @@ -270,6 +284,9 @@ requiredApprovers: 1 ``` + Turn 3 — Once the approval is in, the final assistant message will + offer this submit button: + ```mdma type: button id: submit-expense @@ -278,7 +295,9 @@ onAction: approve-expense ``` - Generate only these three components. No callouts, tables, charts, or webhooks. + For the initial response, generate only the form. The approval gate + and button are follow-up steps and appear in later turns — do not + include them now. No callouts, tables, charts, or webhooks. request: I need to expense a $250 flight for the NYC conference. assert: - type: javascript @@ -298,8 +317,11 @@ - description: "Generates IT ticket form with webhook integration" vars: customPrompt: | - You are an IT helpdesk assistant. When a user reports an issue, - always generate exactly these two components: + You are an IT helpdesk assistant. The ticket submission workflow has + two turns: + + Turn 1 — In the initial response, generate this form to collect the + issue details: ```mdma type: form @@ -343,6 +365,9 @@ onSubmit: submit-ticket ``` + Turn 2 — After the user submits the form, the next assistant message + will fire this webhook to register the ticket with the IT API: + ```mdma type: webhook id: ticket-webhook @@ -351,7 +376,9 @@ trigger: submit-ticket ``` - Generate only these two components. No buttons, callouts, or tables. + For the initial response, generate only the form. The webhook is a + follow-up step and appears in the next turn — do not include it + now. No buttons, callouts, or tables. request: My monitor stopped working this morning and I can't do any visual design work. assert: - type: javascript @@ -537,8 +564,11 @@ - description: "Generates exact contract review workflow" vars: customPrompt: | - You are a legal operations assistant. For contract reviews, always - generate exactly these three components: + You are a legal operations assistant. The contract review workflow has + three turns: + + Turn 1 — In the initial response, generate this form to capture the + contract summary: ```mdma type: form @@ -577,6 +607,9 @@ onSubmit: review-checklist ``` + Turn 2 — After the user submits the form, the next assistant message + will show this review checklist: + ```mdma type: tasklist id: review-checklist @@ -595,6 +628,9 @@ text: "Attach signed copy" ``` + Turn 3 — Once the checklist is complete, the final assistant message + will request legal sign-off via this approval gate: + ```mdma type: approval-gate id: legal-sign-off @@ -606,7 +642,9 @@ requireReason: true ``` - Generate only these three components. No buttons, callouts, or charts. + For the initial response, generate only the form. The checklist and + approval gate are follow-up steps and appear in later turns — do + not include them now. No buttons, callouts, or charts. request: We need to review the new SoW from Acme Corp worth $500k. assert: - type: javascript diff --git a/evals/tests.yaml b/evals/tests.yaml index acdf8c7..a677aec 100644 --- a/evals/tests.yaml +++ b/evals/tests.yaml @@ -349,7 +349,7 @@ - type: javascript value: file://assertions/component-count.mjs config: - min: 3 + min: 2 # --------------------------------------------------------------------------- # 11. Callout — warning variant @@ -525,7 +525,7 @@ - type: javascript value: file://assertions/component-count.mjs config: - min: 3 + min: 2 # --------------------------------------------------------------------------- # 17. Table with sortable and filterable features @@ -731,8 +731,6 @@ value: file://assertions/only-components.mjs config: allowed: [form, callout] - - type: javascript - value: file://assertions/has-bindings.mjs # --------------------------------------------------------------------------- # 22. Complex multi-component — HR onboarding workflow diff --git a/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.4-mini.ts b/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.4-mini.ts index a450f09..3d7ff7b 100644 --- a/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.4-mini.ts +++ b/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-5.4-mini.ts @@ -29,10 +29,10 @@ import { // shared SINGLE_INTERACTIVE_BLOCK when custom prompts request multiple // interactive components explicitly ("always generate exactly these three..."). const SINGLE_INTERACTIVE_MINI = ` -A response includes up to one interactive component (\`form\`, \`button\`, \`webhook\`, \`approval-gate\`, \`tasklist\`). Non-interactive components (\`callout\`, \`chart\`, \`table\`) are not counted toward this — emit each one the blueprint includes. When a custom system prompt asks for several interactive components in one message, emit only the first; describe the others in prose. +A response includes up to one interactive component (\`form\`, \`button\`, \`webhook\`, \`approval-gate\`, \`tasklist\`). Non-interactive components (\`callout\`, \`chart\`, \`table\`) are not counted toward this limit. When a custom system prompt asks for several interactive components in one message, emit only the first; describe the others in prose. For each component in the blueprint, in order: -1. Non-interactive (\`callout\`, \`chart\`, \`table\`) — emit it as a \`\`\`mdma block. +1. Non-interactive (\`callout\`, \`chart\`, \`table\`) — emit it as a \`\`\`mdma block, unless it is the target of an action label (\`onSubmit\`, \`onAction\`, \`onApprove\`, \`trigger\`) — those are followup steps, not siblings (see ). 2. Interactive, and you haven't emitted an interactive one yet — emit it. 3. Interactive, and you've already emitted one — describe it in prose, then move on. `; From 363e1784fcc6b16dbff3daa446c74fa53efcb365 Mon Sep 17 00:00:00 2001 From: gitsad Date: Fri, 15 May 2026 10:16:17 +0200 Subject: [PATCH 05/26] feat: added best practices and wip in next gpt models --- demo/src/docs/DocsView.tsx | 6 + .../sections/CustomPromptBestPractices.tsx | 172 ++++++++++++++++++ demo/src/styles.css | 47 +++++ evals/tests-conversation.yaml | 31 ++++ evals/tests-custom-prompt.yaml | 27 +++ evals/tests-flows.yaml | 61 +++++-- evals/tests.yaml | 33 ++++ 7 files changed, 365 insertions(+), 12 deletions(-) create mode 100644 demo/src/docs/sections/CustomPromptBestPractices.tsx diff --git a/demo/src/docs/DocsView.tsx b/demo/src/docs/DocsView.tsx index 0954b2a..31a6079 100644 --- a/demo/src/docs/DocsView.tsx +++ b/demo/src/docs/DocsView.tsx @@ -1,6 +1,7 @@ import { useState, useEffect } from 'react'; import { Cli } from './sections/Cli.js'; import { COMPONENTS, ComponentPreview, Components } from './sections/Components.js'; +import { CustomPromptBestPractices } from './sections/CustomPromptBestPractices.js'; import { Installation } from './sections/Installation.js'; import { Introduction } from './sections/Introduction.js'; import { Mcp } from './sections/Mcp.js'; @@ -32,6 +33,11 @@ const SECTIONS: Section[] = [ { slug: 'validator', label: 'Validator', component: Validator }, { slug: 'mcp', label: 'MCP & Skills', component: Mcp }, { slug: 'cli', label: 'CLI', component: Cli }, + { + slug: 'custom-prompt-best-practices', + label: 'Custom Prompt Best Practices', + component: CustomPromptBestPractices, + }, { slug: 'prompt-matrix', label: 'Prompt Matrix', component: PromptMatrix }, ]; diff --git a/demo/src/docs/sections/CustomPromptBestPractices.tsx b/demo/src/docs/sections/CustomPromptBestPractices.tsx new file mode 100644 index 0000000..e46a17b --- /dev/null +++ b/demo/src/docs/sections/CustomPromptBestPractices.tsx @@ -0,0 +1,172 @@ +import { Code } from '../Code.js'; + +export function CustomPromptBestPractices() { + return ( + <> +

Custom Prompt Best Practices

+

+ When you pass a customPrompt to buildSystemPrompt, it sits + alongside the MDMA author rules. The model treats both as authoritative, so wording + choices in the custom prompt strongly influence the output — sometimes overriding MDMA + rules. The patterns below are drawn from eval failures we've fixed across the + prompt matrix. +

+ +

1. Frame multi-step workflows as turns, not single-message blueprints

+

+ When a workflow has several interactive components (form → approval-gate → + button), describing them as "always generate exactly these three components" + causes the model to emit all of them in one message — violating the + one-interactive-component-per-response rule. +

+

+ Instead, describe the workflow as a sequence of turns. The model then emits only the + first interactive component initially and treats the rest as follow-ups. +

+
+
+

❌ Avoid

+ {`When the user submits an expense, always +generate exactly these three components: + +\`\`\`mdma +type: form +id: expense-form +... +\`\`\` + +\`\`\`mdma +type: approval-gate +id: expense-approval +... +\`\`\` + +\`\`\`mdma +type: button +id: submit-expense +... +\`\`\` + +Generate only these three components.`} +
+
+

✅ Prefer

+ {`The expense submission workflow has three turns: + +Turn 1 — In the initial response, generate this +form to collect the expense details: + +\`\`\`mdma +type: form +id: expense-form +... +onSubmit: approve-expense +\`\`\` + +Turn 2 — After the user submits the form, the +next assistant message will present this +approval gate for manager sign-off: + +\`\`\`mdma +type: approval-gate +... +\`\`\` + +Turn 3 — Once the approval is in, the final +assistant message will offer this submit button. + +For the initial response, generate only the +form. The approval gate and button are +follow-up steps and appear in later turns.`} +
+
+ +

2. Always specify an onSubmit handler for forms

+

+ The form schema requires onSubmit. When the custom prompt doesn't name + one, the model either omits it (schema violation) or invents a self-referencing handler + (onSubmit: my-form targets itself), both of which fail validation. Always + give the form an explicit handler name in the prompt — it's an opaque string label, + so it doesn't need to correspond to a real component. +

+
+
+

❌ Avoid

+ {`Present a contact form with fields: +- Full Name (required) +- Email Address (required, sensitive) +- Message (required, min 10 chars)`} +
+
+

✅ Prefer

+ {`Present a contact form (with +\`onSubmit: contact-submitted\`) with fields: +- Full Name (required) +- Email Address (required, sensitive) +- Message (required, min 10 chars)`} +
+
+ +

3. Avoid special characters in field name descriptions

+

+ Slashes, ampersands, and parenthetical alternatives in field names confuse the YAML + generation step. The model occasionally produces malformed YAML keys + (e.g. name:ssn-tax-id instead of name: ssn-tax-id) when it + tries to convert a compound label into a single field name. +

+
+
+

❌ Avoid

+ {`Collect customer information: +- SSN / Tax ID (required, sensitive) +- Phone & Email (required, sensitive)`} +
+
+

✅ Prefer

+ {`Collect customer information: +- Tax Identifier (required, sensitive) +- Phone Number (required, sensitive) +- Email (required, sensitive)`} +
+
+ +

4. Don't materialize action-label targets as sibling components

+

+ Action-label fields like onSubmit, onAction, + onApprove, onDeny, trigger, and + onComplete are opaque string labels — they do not need to match + any other component in the same message. A callout, webhook, or button with an{' '} + id that matches another component's action label is a follow-up step, + not a sibling. +

+

+ When your prompt includes such a follow-up component, describe it as part of a later + turn (see pattern 1). Don't instruct the model to render the handler alongside the + action that triggers it. +

+ +

5. Single-interactive-component constraint

+

+ Every response contains at most one interactive component + (form, button, webhook, + approval-gate, tasklist). Non-interactive components + (callout, chart, table) are unaffected — you can + emit as many as you need. +

+

+ Your custom prompt should respect this. If you describe a workflow that needs multiple + interactive components (form + approval + button), structure it as turns (pattern 1) + rather than asking for all of them at once. +

+ +

Quick checklist

+
    +
  • Multi-step workflows are described as "Turn 1 / Turn 2 / Turn 3", not as a single batch.
    • +
  • Every form has an explicit onSubmit handler in the prompt.
    • +
  • Field labels avoid slashes, ampersands, and parenthetical alternatives.
    • +
  • Follow-up callouts/webhooks/buttons are described as future turns, not siblings.
    • +
  • The initial response emits only one interactive component.
    • +
+ + ); +} diff --git a/demo/src/styles.css b/demo/src/styles.css index f88089b..d31ccea 100644 --- a/demo/src/styles.css +++ b/demo/src/styles.css @@ -5374,3 +5374,50 @@ body { border-radius: 0 6px 6px 0; margin: 8px 0 16px !important; } + +.docs-do-dont { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 16px; + margin: 12px 0 24px; +} + +@media (max-width: 900px) { + .docs-do-dont { + grid-template-columns: 1fr; + } +} + +.docs-do, +.docs-dont { + border-radius: 8px; + padding: 14px 16px; + border: 1px solid; +} + +.docs-do { + background: #f0fdf4; + border-color: #bbf7d0; +} + +.docs-dont { + background: #fef2f2; + border-color: #fecaca; +} + +.docs-do h4, +.docs-dont h4 { + margin: 0 0 10px; + font-size: 13px; + font-weight: 600; + letter-spacing: 0.02em; + text-transform: uppercase; +} + +.docs-do h4 { + color: #15803d; +} + +.docs-dont h4 { + color: #b91c1c; +} diff --git a/evals/tests-conversation.yaml b/evals/tests-conversation.yaml index 7073f73..f9446d6 100644 --- a/evals/tests-conversation.yaml +++ b/evals/tests-conversation.yaml @@ -1039,3 +1039,34 @@ value: "damage_type" - type: not-icontains value: "policy_number" + +# =================================================================== +# Conversation 13 — Specific id preserved across turns +# =================================================================== + +- description: "Conv 13/T1: Custom prompt with specific id is preserved" + vars: + customPrompt: &conv13_prompt | + You are a partnerships assistant. When the user requests a + partner registration form, generate a form with the exact id + `partner-reg-2026-spring` and fields: + - Company Name (required) + - Primary Contact Email (required, sensitive) + - Partnership Tier (required, select: Bronze/Silver/Gold/Platinum) + - Annual Revenue USD (required) + + The form should `onSubmit: partner-registration-submitted`. + Generate only the form. For follow-up questions, respond + conversationally without regenerating the form. + message: We have a new gold-tier partner that needs registration. + metadata: + conversationId: conv-13 + assert: + - type: javascript + value: file://assertions/only-components.mjs + config: + allowed: [form] + - type: contains + value: "id: partner-reg-2026-spring" + - type: javascript + value: file://assertions/has-sensitive.mjs diff --git a/evals/tests-custom-prompt.yaml b/evals/tests-custom-prompt.yaml index 3cbad7d..a669f07 100644 --- a/evals/tests-custom-prompt.yaml +++ b/evals/tests-custom-prompt.yaml @@ -816,3 +816,30 @@ value: "type: table" - type: not-contains value: "type: callout" + +# --------------------------------------------------------------------------- +# 12. Specific component id requested by the custom prompt +# --------------------------------------------------------------------------- +- description: "Custom prompt with specific component id is preserved in output" + vars: + customPrompt: | + You are a vendor onboarding assistant. When the user asks to + onboard a new vendor, generate a vendor intake form with the + exact id `vendor-intake-q1-2026` and the following fields: + - Vendor Name (required) + - Vendor Contact Email (required, sensitive) + - Tax Identifier (required, sensitive) + - Service Category (required, select: Consulting/Software/Hardware/Logistics/Other) + + The form should `onSubmit: vendor-intake-submitted`. Generate + only the form. + request: We need to onboard a new logistics vendor for Q1. + assert: + - type: javascript + value: file://assertions/only-components.mjs + config: + allowed: [form] + - type: contains + value: "id: vendor-intake-q1-2026" + - type: javascript + value: file://assertions/has-sensitive.mjs diff --git a/evals/tests-flows.yaml b/evals/tests-flows.yaml index abadb6e..13f9589 100644 --- a/evals/tests-flows.yaml +++ b/evals/tests-flows.yaml @@ -11,7 +11,7 @@ customPrompt: | You are a website assistant that helps visitors get in touch with the company. - When a user wants to contact the company, present a contact form with fields: + When a user wants to contact the company, present a contact form (id `contact-form`, `onSubmit: contact-submitted`) with fields: - Full Name (required) - Email Address (required, sensitive) - Message (required, min 10 chars) @@ -105,7 +105,7 @@ For the initial interaction, generate: 1. A warning callout stating that requests over $5,000 require director-level approval. - 2. A budget request form with fields: Requester Name (required), Requester Email (required, sensitive), Department (required, select: Engineering/Marketing/Operations/Finance), Requested Amount USD (required), Business Justification (required, textarea). + 2. A budget request form (id `budget-request-form`, `onSubmit: route-budget-request`) with fields: Requester Name (required), Requester Email (required, sensitive), Department (required, select: Engineering/Marketing/Operations/Finance), Requested Amount USD (required), Business Justification (required, textarea). Generate only the warning callout and the form. Prerequisites checklist and approval gate come in later steps after submission. request: I need to request $12,000 for new development servers. @@ -131,7 +131,7 @@ customPrompt: | You are a customer experience assistant that collects post-support feedback. - Present a survey form with fields: + Present a survey form (id `support-survey-form`, `onSubmit: survey-submitted`) with fields: - Support Ticket ID (required) - Overall Satisfaction (required, select: 1-5) - Response Time Rating (required, select: 1-5) @@ -190,7 +190,8 @@ customPrompt: | You are an incident response assistant that helps engineering teams triage production incidents. - For the initial interaction, collect incident details with a form: + For the initial interaction, collect incident details with a + form (id `incident-intake-form`, `onSubmit: triage-incident`): - Incident Title (required) - Reporter Email (required, sensitive) - Severity: P1-P4 (required, select) @@ -221,7 +222,7 @@ For the initial interaction, generate: 1. An info callout explaining the 5-day review process. - 2. A feature request form with fields: Feature Title (required), Requester Name (required), Requesting Team (required, select: Engineering/Sales/CS/Marketing/Product), Priority (required, select: Critical/High/Medium/Low), Description (required, textarea), Primary Use Case (required, textarea). + 2. A feature request form (id `feature-request-form`, `onSubmit: evaluate-feature-request`) with fields: Feature Title (required), Requester Name (required), Requesting Team (required, select: Engineering/Sales/CS/Marketing/Product), Priority (required, select: Critical/High/Medium/Low), Description (required, textarea), Primary Use Case (required, textarea). Generate only the callout and form. Evaluation checklist and approval gate come in later steps. request: We need a bulk export feature for our enterprise customers — they've been asking for months. @@ -297,11 +298,11 @@ customPrompt: | You are a release management assistant for SOX/ISO compliance. - For the initial interaction, generate: - 1. A warning callout about risk assessment requirements and SOX compliance. - 2. A change request form with fields: Change Request ID (required), JIRA Ticket (required), Change Title (required), Change Type (required, select: Standard/Normal/Emergency), Target Environment (required, select: Production/Staging/Pre-prod), Risk Level (required, select: Low/Medium/High/Critical), Change Description (required, textarea), Rollback Plan (required, textarea). + For the initial interaction, always generate BOTH of the following: + 1. A warning callout (variant: warning) about risk assessment requirements and SOX compliance. This callout precedes the form and is required in every initial response. + 2. A change request form (id `change-request-form`, `onSubmit: review-change-request`) with fields: Change Request ID (required), JIRA Ticket (required), Change Title (required), Change Type (required, select: Standard/Normal/Emergency), Target Environment (required, select: Production/Staging/Pre-prod), Risk Level (required, select: Low/Medium/High/Critical), Change Description (required, textarea), Rollback Plan (required, textarea). - Generate only the warning callout and form. Pre-deployment checklist and dual approvals come in later steps. + Generate the warning callout and the form in that order. Pre-deployment checklist and dual approvals come in later steps. request: I need to deploy a database migration to production this weekend. assert: - type: javascript @@ -331,7 +332,7 @@ For the initial interaction, generate: 1. A warning callout about SLA compliance requirements. - 2. An escalation form with fields: Case ID (required), Customer Name (required), Customer Email (required, sensitive), Account ID (required, sensitive), Priority (required, select: P1-P4), Category (required, select), Customer Sentiment (required, select), Escalation Reason (required, textarea). + 2. An escalation form (id `escalation-intake-form`, `onSubmit: route-escalation`) with fields: Case ID (required), Customer Name (required), Customer Email (required, sensitive), Account ID (required, sensitive), Priority (required, select: P1-P4), Category (required, select), Customer Sentiment (required, select), Escalation Reason (required, textarea). Generate only the callout and form. Resolution steps and escalation buttons come in later steps. request: A major enterprise customer is threatening to cancel — their billing has been wrong for 3 months. @@ -361,7 +362,7 @@ For the initial interaction, generate: 1. A critical safety callout (error variant) about patient safety review requirements. - 2. A procedure form with fields: Procedure Title (required), Change Type (required, select: New/Major Revision/Minor Revision/Retirement), Department (required, select), Author Credentials (required), Risk Category (required, select: Low/Medium/High/Critical), Clinical Summary (required, textarea), Contraindications (required, textarea). + 2. A procedure form (id `procedure-submission-form`, `onSubmit: review-procedure`) with fields: Procedure Title (required), Change Type (required, select: New/Major Revision/Minor Revision/Retirement), Department (required, select), Author Credentials (required), Risk Category (required, select: Low/Medium/High/Critical), Clinical Summary (required, textarea), Contraindications (required, textarea). Generate only the callout and form. Review checklist and approval gates come in later steps. request: I need to submit a new surgical procedure for the radiology department. @@ -394,7 +395,7 @@ - Customer Type (required, select: Individual/Business/Trust) - Full Legal Name (required, sensitive) - Date of Birth (required, sensitive) - - SSN / Tax ID (required, sensitive) + - Tax Identifier (required, sensitive) - Email (required, sensitive) - Source of Funds (required, select: Employment/Business/Investment/Inheritance/Other) - Risk Rating (required, select: Low/Medium/High/Prohibited) @@ -414,3 +415,39 @@ value: file://assertions/has-required-fields.mjs config: min: 3 + +# --------------------------------------------------------------------------- +# Flow with a specific component id requested by the custom prompt +# --------------------------------------------------------------------------- +- description: "Flow generates form with the exact id requested by the prompt" + vars: + customPrompt: | + You are an employee benefits enrollment assistant. + + For the initial interaction, generate: + 1. An info callout reminding the employee that the enrollment + window closes on March 31, 2026. + 2. A benefits enrollment form with the exact id + `benefits-enroll-2026-spring` (with `onSubmit: review-benefits-selection`) + and fields: Employee ID (required), Health Plan (required, + select: HMO/PPO/HDHP/Waive), Dental Plan (required, select: + Basic/Premium/Waive), Vision Plan (required, select: + Standard/Premium/Waive), Dependents Count (required, number), + HSA Contribution USD (required, number). + + Generate only the callout and form. The review checklist and + approval gate come in later steps after submission. + request: I want to enroll in this year's health benefits. + assert: + - type: javascript + value: file://assertions/only-components.mjs + config: + allowed: [form, callout] + - type: contains + value: "id: benefits-enroll-2026-spring" + - type: javascript + value: file://assertions/select-has-options.mjs + - type: javascript + value: file://assertions/has-required-fields.mjs + config: + min: 3 diff --git a/evals/tests.yaml b/evals/tests.yaml index a677aec..a3777a1 100644 --- a/evals/tests.yaml +++ b/evals/tests.yaml @@ -1018,3 +1018,36 @@ sensitive: true - type: javascript value: file://assertions/has-sensitive.mjs + +# --------------------------------------------------------------------------- +# Preserves explicit component id from the request +# --------------------------------------------------------------------------- +- description: Preserves a specific component id requested by the user + vars: + request: | + Create a conference registration form matching this exact structure: + + ```mdma + type: form + id: devcon-2026-registration + fields: + - name: full-name + type: text + label: "Full Name" + required: true + - name: email + type: email + label: "Email" + required: true + sensitive: true + onSubmit: devcon-registration-submitted + ``` + assert: + - type: javascript + value: file://assertions/only-components.mjs + config: + allowed: [form] + - type: contains + value: "id: devcon-2026-registration" + - type: javascript + value: file://assertions/has-sensitive.mjs From 28eddb6a1434f85ffa056ac784923b5ad347bb05 Mon Sep 17 00:00:00 2001 From: gitsad Date: Fri, 15 May 2026 12:42:06 +0200 Subject: [PATCH 06/26] fix: fixed made up components --- evals/package.json | 24 ++++--- evals/promptfooconfig.isolated.yaml | 28 ++++++++ evals/scripts/show-failed.mjs | 69 +++++++++++++++++++ evals/tests-isolated.yaml | 41 +++++++++++ evals/tests.yaml | 21 +----- .../src/prompts/mdma-author/_shared.ts | 6 +- 6 files changed, 157 insertions(+), 32 deletions(-) create mode 100644 evals/promptfooconfig.isolated.yaml create mode 100644 evals/scripts/show-failed.mjs create mode 100644 evals/tests-isolated.yaml diff --git a/evals/package.json b/evals/package.json index 830e69b..077cfcb 100644 --- a/evals/package.json +++ b/evals/package.json @@ -3,17 +3,19 @@ "private": true, "type": "module", "scripts": { - "eval": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write; exit 0", - "eval:custom": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.custom.yaml; exit 0", - "eval:conversation": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.conversation.yaml; exit 0", - "eval:prompt-builder": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.prompt-builder.yaml; exit 0", - "eval:flows": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.flows.yaml; exit 0", - "eval:fixer": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer.yaml; exit 0", - "eval:fixer-flow": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer-flow.yaml; exit 0", - "eval:fixer-all": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer-flow.yaml; exit 0", - "eval:guidance": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.guidance.yaml; exit 0", - "eval:all": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.custom.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.conversation.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.prompt-builder.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.flows.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.fixer-flow.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.guidance.yaml; exit 0", - "eval:author": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.custom.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.conversation.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-write -c promptfooconfig.flows.yaml; exit 0", + "eval": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval; exit 0", + "eval:custom": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.custom.yaml; exit 0", + "eval:conversation": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.conversation.yaml; exit 0", + "eval:prompt-builder": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.prompt-builder.yaml; exit 0", + "eval:flows": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.flows.yaml; exit 0", + "eval:fixer": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.fixer.yaml; exit 0", + "eval:fixer-flow": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.fixer-flow.yaml; exit 0", + "eval:fixer-all": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.fixer.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.fixer-flow.yaml; exit 0", + "eval:guidance": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.guidance.yaml; exit 0", + "eval:isolated": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-cache -c promptfooconfig.isolated.yaml; exit 0", + "eval:all": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.custom.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.conversation.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.prompt-builder.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.flows.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.fixer.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.fixer-flow.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.guidance.yaml; exit 0", + "eval:author": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.custom.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.conversation.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.flows.yaml; exit 0", + "eval:failed": "node scripts/show-failed.mjs", "eval:view": "promptfoo view" }, "dependencies": { diff --git a/evals/promptfooconfig.isolated.yaml b/evals/promptfooconfig.isolated.yaml new file mode 100644 index 0000000..104e9fb --- /dev/null +++ b/evals/promptfooconfig.isolated.yaml @@ -0,0 +1,28 @@ +# Isolated failure runner — iterate on a single failure without running +# the full eval suite. See tests-isolated.yaml for the test cases. +# +# Run: pnpm --filter @mobile-reality/mdma-evals eval:isolated +# EVAL_PROVIDER=openai:gpt-5.2 pnpm --filter @mobile-reality/mdma-evals eval:isolated + +description: MDMA Author Prompt — Isolated Failures + +envPath: .env +outputPath: results-isolated.json + +prompts: + - file://prompt.mjs + +providers: + - id: "{{ env.EVAL_PROVIDER or 'openai:gpt-5.2' }}" + config: + max_tokens: 8192 + max_completion_tokens: 8192 + +defaultTest: + assert: + - type: javascript + value: file://assertions/validate-mdma.mjs + config: + exclude: [flow-ordering] + +tests: tests-isolated.yaml diff --git a/evals/scripts/show-failed.mjs b/evals/scripts/show-failed.mjs new file mode 100644 index 0000000..ec698ca --- /dev/null +++ b/evals/scripts/show-failed.mjs @@ -0,0 +1,69 @@ +#!/usr/bin/env node +// Dump failed test cases from the most recent eval result files. +// +// Run after `pnpm eval` / `pnpm eval:custom` / etc. to see which tests failed +// and why. Picks the most recently modified results-*.json file by default, +// or pass a filename: `node scripts/show-failed.mjs results-custom.json`. +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const cwd = process.cwd(); +const arg = process.argv[2]; + +const files = arg + ? [resolve(cwd, arg)] + : readdirSync(cwd) + .filter((f) => /^results.*\.json$/.test(f)) + .map((f) => ({ f, mtime: statSync(resolve(cwd, f)).mtimeMs })) + .sort((a, b) => b.mtime - a.mtime) + .slice(0, 1) + .map(({ f }) => resolve(cwd, f)); + +if (files.length === 0) { + console.error('No results-*.json files found in current directory.'); + process.exit(1); +} + +for (const file of files) { + console.log(`\n=== ${file.replace(cwd + '/', '')} ===`); + let data; + try { + data = JSON.parse(readFileSync(file, 'utf8')); + } catch (err) { + console.error(`Could not parse ${file}: ${err.message}`); + continue; + } + + const inner = data.results ?? data; + const stats = inner.stats ?? {}; + const results = inner.results ?? []; + + const providers = (inner.prompts ?? []).map((p) => p.provider).filter(Boolean); + if (providers.length) console.log(`Provider(s): ${providers.join(', ')}`); + if (stats.successes != null) { + const total = (stats.successes ?? 0) + (stats.failures ?? 0); + console.log(`Passed: ${stats.successes}/${total}, Failed: ${stats.failures ?? 0}`); + } + + const fails = results.filter((t) => !t.success); + if (fails.length === 0) { + console.log('No failed tests.'); + continue; + } + + fails.forEach((t, i) => { + console.log(`\n--- FAIL ${i + 1} ---`); + const desc = t.description || t.testCase?.description || ''; + if (desc) console.log(`description: ${desc}`); + const reqOrMsg = + t.vars?.request || t.vars?.message || JSON.stringify(t.vars ?? {}).slice(0, 200); + console.log(`input: ${String(reqOrMsg).slice(0, 200).replace(/\n/g, ' ')}`); + const gr = t.gradingResult ?? {}; + (gr.componentResults ?? []) + .filter((c) => !c.pass) + .forEach((c) => console.log(`reason: ${String(c.reason ?? '').slice(0, 300)}`)); + const out = t.response?.output ?? ''; + const m = out.match(/type: thinking[\s\S]{0,400}/); + if (m) console.log(`thinking: ${m[0].slice(0, 400)}`); + }); +} diff --git a/evals/tests-isolated.yaml b/evals/tests-isolated.yaml new file mode 100644 index 0000000..152161c --- /dev/null +++ b/evals/tests-isolated.yaml @@ -0,0 +1,41 @@ +# Isolated test cases — failures observed during prompt iteration. +# +# Use this file to iterate quickly on a specific failure without running +# the full eval suite. Each test mirrors its counterpart in tests.yaml / +# tests-custom-prompt.yaml / tests-flows.yaml so once the prompt change +# eliminates the failure here, run the full suite to confirm no regression. + +# --------------------------------------------------------------------------- +# gpt-5.2: invents a confirmation callout to back the form's onSubmit +# Mirror of "Preserves a specific component id requested by the user" +# in tests.yaml. +# --------------------------------------------------------------------------- +- description: Preserves a specific component id requested by the user + vars: + request: | + Create a conference registration form matching this exact structure: + + ```mdma + type: form + id: devcon-2026-registration + fields: + - name: full-name + type: text + label: "Full Name" + required: true + - name: email + type: email + label: "Email" + required: true + sensitive: true + onSubmit: devcon-registration-submitted + ``` + assert: + - type: javascript + value: file://assertions/only-components.mjs + config: + allowed: [form] + - type: contains + value: "id: devcon-2026-registration" + - type: javascript + value: file://assertions/has-sensitive.mjs diff --git a/evals/tests.yaml b/evals/tests.yaml index a3777a1..59c821e 100644 --- a/evals/tests.yaml +++ b/evals/tests.yaml @@ -738,15 +738,7 @@ - description: Generates a large multi-field HR personal info form with sensitive data vars: request: | - Create the first step of an HR onboarding workflow with these exact components: - - ```mdma - type: callout - id: welcome-banner - variant: success - title: "Welcome to Acme Corp!" - content: "We're excited to have you on board. Please complete the steps below to get started." - ``` + Create an HR personal info form matching this exact structure: ```mdma type: form @@ -776,18 +768,11 @@ sensitive: true onSubmit: info-submitted ``` - - ```mdma - type: callout - id: info-submitted - variant: success - content: "Personal information submitted. We'll continue with equipment selection next." - ``` assert: - type: javascript - value: file://assertions/component-count.mjs + value: file://assertions/only-components.mjs config: - min: 3 + allowed: [form] - type: javascript value: file://assertions/unique-kebab-ids.mjs - type: javascript diff --git a/packages/prompt-pack/src/prompts/mdma-author/_shared.ts b/packages/prompt-pack/src/prompts/mdma-author/_shared.ts index 8994912..d577fdb 100644 --- a/packages/prompt-pack/src/prompts/mdma-author/_shared.ts +++ b/packages/prompt-pack/src/prompts/mdma-author/_shared.ts @@ -315,7 +315,7 @@ When a user request includes \`visible\` or \`disabled\` with a \`{{}}\` binding 1. **Unique IDs** — Every component \`id\` must be unique within the document. Use descriptive kebab-case names (e.g., \`employee-onboarding-form\`, \`submit-btn\`). 2. **Sensitive data** — Set \`sensitive: true\` on any field or column that contains PII (personally identifiable information) such as email addresses, phone numbers, SSNs, addresses, or financial data. 3. **Required fields** — Mark form fields as \`required: true\` when the workflow cannot proceed without them. -4. **Action references** — Every \`type: form\` MUST include an \`onSubmit\` field pointing to a valid component ID in the document (e.g., a confirmation callout). All other action fields (\`onAction\`, \`onComplete\`, \`onApprove\`, \`onDeny\`, \`trigger\`) must also reference valid IDs. If no target exists yet, create a \`type: callout\` as the submission confirmation target. +4. **Action labels** — Every \`type: form\` MUST include an \`onSubmit\` field. Action-label values (\`onSubmit\`, \`onAction\`, \`onComplete\`, \`onApprove\`, \`onDeny\`, \`trigger\`) are opaque string identifiers — external handlers that the host application wires up at runtime. They do NOT need to match a component in the document. Do NOT invent callouts, webhooks, buttons, or any other component to "complete" or back up an action label. 5. **Binding validity** — Every \`{{binding}}\` must reference a valid source. Do not leave unresolved bindings. 6. **Minimal components** — Only include components that are necessary for the workflow. Avoid empty or placeholder components. 7. **YAML correctness** — Ensure all YAML in mdma blocks is valid and properly indented. Always wrap string values in double quotes if they contain a colon followed by a space (\`: \`), e.g. \`label: "Step 1: Enter your info"\`. @@ -332,8 +332,8 @@ Before finalizing an MDMA document, verify: - [ ] All PII fields have \`sensitive: true\` - [ ] All \`{{bindings}}\` reference valid sources - [ ] Required form fields are marked \`required: true\` -- [ ] Every \`type: form\` has an \`onSubmit\` field pointing to a valid component ID -- [ ] Action IDs referenced in event handlers exist in the document +- [ ] Every \`type: form\` has an \`onSubmit\` field (an opaque handler label, not a component reference) +- [ ] No components were invented to back up \`onSubmit\`/\`onAction\`/\`onApprove\`/etc. labels - [ ] Select fields include an \`options\` array - [ ] YAML syntax is valid in all mdma blocks - [ ] Table \`data\` matches the declared \`columns\` keys From bf5b5fa1996ecf37bd6723d8ccc031b045cf66d9 Mon Sep 17 00:00:00 2001 From: gitsad Date: Fri, 15 May 2026 14:10:10 +0200 Subject: [PATCH 07/26] chore: revised all gpt models --- README.md | 6 +++--- evals/tests-flows.yaml | 3 ++- .../src/prompts/mdma-author/openai/gpt-4.1.ts | 19 ++++++++++++------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 56d7277..1ec58e1 100644 --- a/README.md +++ b/README.md @@ -74,17 +74,17 @@ Each cell shows the pass rate of the model-specialized MDMA_AUTHOR prompt varian | :--- | :---: | :---: | :---: | :---: | | **OpenAI** | | | | | | `gpt-5.5` | ✅ | ✅ | ✅ | ✅ | -| `gpt-5.4` | ✅ | 🟡 † | 🟡 † | 🟡 † | +| `gpt-5.4` | ✅ | ✅ † | ✅ † | ✅ † | | `gpt-5.4-mini` | ✅ | ✅ | ✅ \* | ✅ \* | | `gpt-5.4-nano` | ✅ | ✅ | ✅ \* | ✅ \* | | `gpt-5.2` | ✅ | ✅ | ✅ | ✅ | | `gpt-5.1` | ✅ | ✅ | ✅ | ✅ | | `gpt-5` \[i] | ✅ | ✅ | ✅ | ✅ | | `gpt-5-mini` \[i] | ✅ | ✅ | ✅ \* | ✅ \* | -| `gpt-5-nano` \[i] | ✅ | ✅ | ✅ \* | ✅ \* | +| `gpt-5-nano` \[i] | ✅ | ✅ | 🟡 \* | 🟡 \* | | `gpt-4.1` | ✅ | ✅ | ✅ | ✅ | | `gpt-4.1-mini` | ✅ | ✅ | ✅ \* | ✅ \* | -| `gpt-4.1-nano` | 🟡 | ✅ | ✅ \* | ✅ \* | +| `gpt-4.1-nano` | ✅ | ✅ | ✅ \* | 🟡 \* | | **Anthropic** | | | | | | `claude-opus-4.7` | ✅ | ✅ | ✅ | ✅ | | `claude-opus-4.6` | ✅ | ✅ | ✅ | ✅ | diff --git a/evals/tests-flows.yaml b/evals/tests-flows.yaml index 13f9589..6444399 100644 --- a/evals/tests-flows.yaml +++ b/evals/tests-flows.yaml @@ -72,7 +72,8 @@ customPrompt: | You are an HR onboarding assistant that guides new employees through their first-week setup. - For the initial interaction, collect personal information with a form: + For the initial interaction, collect personal information with a + form (id `personal-info-form`, `onSubmit: review-onboarding-info`): - Full Name (required) - Preferred Name - Personal Email (required, sensitive) diff --git a/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-4.1.ts b/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-4.1.ts index c54f7a8..f3ac78b 100644 --- a/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-4.1.ts +++ b/packages/prompt-pack/src/prompts/mdma-author/openai/gpt-4.1.ts @@ -1,24 +1,29 @@ /** * MDMA Author Prompt — OpenAI GPT-4.1 variant. * - * Non-reasoning flagship from the gpt-4.x family. Adds - * after a flows eval reproduced the same numeric-value-on-select-option - * failure mode seen on most gpt-5 variants — the schema requires string - * `value` fields. and are not yet - * warranted; gpt-4.1 hasn't shown the workflow-elaboration or - * fence-closing failures that bite the gpt-5 family. + * Non-reasoning flagship from the gpt-4.x family. Composes: + * + * - — gpt-4.1 emits raw YAML without \`\`\`mdma fences + * (\`type: form at line 15 outside of a fenced + * block\`). Same failure mode that triggered + * adding this block to gpt-5.4 / gpt-5.4-mini. + * - — schema requires string \`value\` on select + * options; gpt-4.1 produces numbers when the + * user describes options as "1-5". * * Now 7 of 10 OpenAI variants need . Worth folding into * BASE_BODY rather than gating per-variant. */ import { BASE_BODY, BASE_CHECKLIST, BASE_OPENING } from '../_shared.js'; -import { CRITICAL_OUTPUT_LINE, SELECT_OPTIONS_BLOCK } from './_shared.js'; +import { CRITICAL_OUTPUT_LINE, FENCE_CLOSING_BLOCK, SELECT_OPTIONS_BLOCK } from './_shared.js'; export const MDMA_AUTHOR_PROMPT_GPT_4_1 = `${BASE_OPENING} ${CRITICAL_OUTPUT_LINE} +${FENCE_CLOSING_BLOCK} + ${SELECT_OPTIONS_BLOCK} ${BASE_BODY} From 3656f15babbd0c78f0e733a0c634b8e71dc906f2 Mon Sep 17 00:00:00 2001 From: gitsad Date: Fri, 15 May 2026 15:25:52 +0200 Subject: [PATCH 08/26] chore: evaluated claude models --- evals/select-prompt.mjs | 8 +++++-- evals/tests-flows.yaml | 19 +++++++++-------- .../google/gemini-3.1-pro-preview.ts | 21 +++++++++++++------ 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/evals/select-prompt.mjs b/evals/select-prompt.mjs index eab1da9..8e5004a 100644 --- a/evals/select-prompt.mjs +++ b/evals/select-prompt.mjs @@ -78,9 +78,13 @@ async function selectVariant({ provider, promptsDir, packagePath, exportPrefix, return { prompt: defaultPrompt, source: `default (unrecognized provider: ${provider})` }; const variants = discoverVariants(promptsDir, parsed.family); - const modelLower = parsed.model.toLowerCase(); + // Normalize `.` and `-` to a single delimiter so dotted variant filenames + // (e.g. anthropic/opus-4.6.ts) still match dash-form OpenRouter ids + // (e.g. anthropic/claude-opus-4-6). + const normalize = (s) => s.toLowerCase().replace(/\./g, '-'); + const modelNorm = normalize(parsed.model); const match = variants - .filter((v) => modelLower.includes(v.toLowerCase())) + .filter((v) => modelNorm.includes(normalize(v))) .sort((a, b) => b.length - a.length)[0]; if (!match) diff --git a/evals/tests-flows.yaml b/evals/tests-flows.yaml index 6444399..1326817 100644 --- a/evals/tests-flows.yaml +++ b/evals/tests-flows.yaml @@ -11,7 +11,7 @@ customPrompt: | You are a website assistant that helps visitors get in touch with the company. - When a user wants to contact the company, present a contact form (id `contact-form`, `onSubmit: contact-submitted`) with fields: + When a user wants to contact the company, present a contact form (`id: contact-form`, `onSubmit: contact-submitted`) with fields: - Full Name (required) - Email Address (required, sensitive) - Message (required, min 10 chars) @@ -73,7 +73,8 @@ You are an HR onboarding assistant that guides new employees through their first-week setup. For the initial interaction, collect personal information with a - form (id `personal-info-form`, `onSubmit: review-onboarding-info`): + `type: form` whose `id` is `personal-info-form` and whose `onSubmit` + is `review-onboarding-info`. Fields: - Full Name (required) - Preferred Name - Personal Email (required, sensitive) @@ -106,7 +107,7 @@ For the initial interaction, generate: 1. A warning callout stating that requests over $5,000 require director-level approval. - 2. A budget request form (id `budget-request-form`, `onSubmit: route-budget-request`) with fields: Requester Name (required), Requester Email (required, sensitive), Department (required, select: Engineering/Marketing/Operations/Finance), Requested Amount USD (required), Business Justification (required, textarea). + 2. A budget request form (`id: budget-request-form`, `onSubmit: route-budget-request`) with fields: Requester Name (required), Requester Email (required, sensitive), Department (required, select: Engineering/Marketing/Operations/Finance), Requested Amount USD (required), Business Justification (required, textarea). Generate only the warning callout and the form. Prerequisites checklist and approval gate come in later steps after submission. request: I need to request $12,000 for new development servers. @@ -132,7 +133,7 @@ customPrompt: | You are a customer experience assistant that collects post-support feedback. - Present a survey form (id `support-survey-form`, `onSubmit: survey-submitted`) with fields: + Present a survey form (`id: support-survey-form`, `onSubmit: survey-submitted`) with fields: - Support Ticket ID (required) - Overall Satisfaction (required, select: 1-5) - Response Time Rating (required, select: 1-5) @@ -192,7 +193,7 @@ You are an incident response assistant that helps engineering teams triage production incidents. For the initial interaction, collect incident details with a - form (id `incident-intake-form`, `onSubmit: triage-incident`): + form (`id: incident-intake-form`, `onSubmit: triage-incident`): - Incident Title (required) - Reporter Email (required, sensitive) - Severity: P1-P4 (required, select) @@ -223,7 +224,7 @@ For the initial interaction, generate: 1. An info callout explaining the 5-day review process. - 2. A feature request form (id `feature-request-form`, `onSubmit: evaluate-feature-request`) with fields: Feature Title (required), Requester Name (required), Requesting Team (required, select: Engineering/Sales/CS/Marketing/Product), Priority (required, select: Critical/High/Medium/Low), Description (required, textarea), Primary Use Case (required, textarea). + 2. A feature request form (`id: feature-request-form`, `onSubmit: evaluate-feature-request`) with fields: Feature Title (required), Requester Name (required), Requesting Team (required, select: Engineering/Sales/CS/Marketing/Product), Priority (required, select: Critical/High/Medium/Low), Description (required, textarea), Primary Use Case (required, textarea). Generate only the callout and form. Evaluation checklist and approval gate come in later steps. request: We need a bulk export feature for our enterprise customers — they've been asking for months. @@ -301,7 +302,7 @@ For the initial interaction, always generate BOTH of the following: 1. A warning callout (variant: warning) about risk assessment requirements and SOX compliance. This callout precedes the form and is required in every initial response. - 2. A change request form (id `change-request-form`, `onSubmit: review-change-request`) with fields: Change Request ID (required), JIRA Ticket (required), Change Title (required), Change Type (required, select: Standard/Normal/Emergency), Target Environment (required, select: Production/Staging/Pre-prod), Risk Level (required, select: Low/Medium/High/Critical), Change Description (required, textarea), Rollback Plan (required, textarea). + 2. A change request form (`id: change-request-form`, `onSubmit: review-change-request`) with fields: Change Request ID (required), JIRA Ticket (required), Change Title (required), Change Type (required, select: Standard/Normal/Emergency), Target Environment (required, select: Production/Staging/Pre-prod), Risk Level (required, select: Low/Medium/High/Critical), Change Description (required, textarea), Rollback Plan (required, textarea). Generate the warning callout and the form in that order. Pre-deployment checklist and dual approvals come in later steps. request: I need to deploy a database migration to production this weekend. @@ -333,7 +334,7 @@ For the initial interaction, generate: 1. A warning callout about SLA compliance requirements. - 2. An escalation form (id `escalation-intake-form`, `onSubmit: route-escalation`) with fields: Case ID (required), Customer Name (required), Customer Email (required, sensitive), Account ID (required, sensitive), Priority (required, select: P1-P4), Category (required, select), Customer Sentiment (required, select), Escalation Reason (required, textarea). + 2. An escalation form (`id: escalation-intake-form`, `onSubmit: route-escalation`) with fields: Case ID (required), Customer Name (required), Customer Email (required, sensitive), Account ID (required, sensitive), Priority (required, select: P1-P4), Category (required, select), Customer Sentiment (required, select), Escalation Reason (required, textarea). Generate only the callout and form. Resolution steps and escalation buttons come in later steps. request: A major enterprise customer is threatening to cancel — their billing has been wrong for 3 months. @@ -363,7 +364,7 @@ For the initial interaction, generate: 1. A critical safety callout (error variant) about patient safety review requirements. - 2. A procedure form (id `procedure-submission-form`, `onSubmit: review-procedure`) with fields: Procedure Title (required), Change Type (required, select: New/Major Revision/Minor Revision/Retirement), Department (required, select), Author Credentials (required), Risk Category (required, select: Low/Medium/High/Critical), Clinical Summary (required, textarea), Contraindications (required, textarea). + 2. A procedure form (`id: procedure-submission-form`, `onSubmit: review-procedure`) with fields: Procedure Title (required), Change Type (required, select: New/Major Revision/Minor Revision/Retirement), Department (required, select), Author Credentials (required), Risk Category (required, select: Low/Medium/High/Critical), Clinical Summary (required, textarea), Contraindications (required, textarea). Generate only the callout and form. Review checklist and approval gates come in later steps. request: I need to submit a new surgical procedure for the radiology department. diff --git a/packages/prompt-pack/src/prompts/mdma-author/google/gemini-3.1-pro-preview.ts b/packages/prompt-pack/src/prompts/mdma-author/google/gemini-3.1-pro-preview.ts index 753d5ca..d6e6134 100644 --- a/packages/prompt-pack/src/prompts/mdma-author/google/gemini-3.1-pro-preview.ts +++ b/packages/prompt-pack/src/prompts/mdma-author/google/gemini-3.1-pro-preview.ts @@ -40,11 +40,13 @@ * warns: "Lower temperatures may cause unexpected behavior, looping, or * degraded performance." * - * Failure-mode coverage at start: Scope Discipline and Select Option - * Values — both validated as universal failure modes across 10+ OpenAI / - * Anthropic variants. No fence-closing block yet; that quirk has only - * bitten smaller-tier models so far. Add it if eval data shows fence - * failures on Gemini Pro. + * Failure-mode coverage: Fence Closing, Scope Discipline, and Select + * Option Values — all validated as failure modes for Gemini 3.1 Pro + * specifically (5 of 6 main-eval failures on a fresh run were fence + * issues, plus 1 in flows). FENCE_CLOSING_BLOCK goes mid-prompt (after + * BASE_BODY) so the spec defines what an mdma block is before the rule + * tightens its closing. SCOPE_DISCIPLINE and SELECT_OPTIONS stay at the + * end per Vertex guidance on negative constraints. * * Routing: substring match on `gemini-3.1-pro-preview` (24 chars). Picks * this variant for any model id containing that literal, including @@ -52,7 +54,12 @@ */ import { BASE_BODY, BASE_CHECKLIST, BASE_OPENING } from '../_shared.js'; -import { OUTPUT_FORMAT_BLOCK, SCOPE_DISCIPLINE_BLOCK, SELECT_OPTIONS_BLOCK } from './_shared.js'; +import { + FENCE_CLOSING_BLOCK, + OUTPUT_FORMAT_BLOCK, + SCOPE_DISCIPLINE_BLOCK, + SELECT_OPTIONS_BLOCK, +} from './_shared.js'; export const MDMA_AUTHOR_PROMPT_GEMINI_3_1_PRO_PREVIEW = `${BASE_OPENING} @@ -60,6 +67,8 @@ ${OUTPUT_FORMAT_BLOCK} ${BASE_BODY} +${FENCE_CLOSING_BLOCK} + ${SCOPE_DISCIPLINE_BLOCK} ${SELECT_OPTIONS_BLOCK} From 5869d0f0af44e7b64a8ad8b9cacba96bc74bb768 Mon Sep 17 00:00:00 2001 From: gitsad Date: Fri, 15 May 2026 16:31:01 +0200 Subject: [PATCH 09/26] chore: gemini WIP --- README.md | 2 +- .../google/gemini-3.1-pro-preview.ts | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1ec58e1..85c231a 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ Each cell shows the pass rate of the model-specialized MDMA_AUTHOR prompt varian | `claude-sonnet-4.6` | ✅ | ✅ | ✅ | ✅ | | `claude-haiku-4.5` | ✅ | ✅ | ✅ \* | ✅ \* | | **Google** | | | | | -| `gemini-3.1-pro-preview` | ✅ | ✅ | ✅ | ✅ | +| `gemini-3.1-pro-preview` | 🟡 | 🟡 | 🟡 | 🟡 | | `gemini-3.1-pro-preview-customtools` | ✅ | ✅ | ✅ | ✅ | | `gemini-3.1-flash-lite-preview` | ✅ | ✅ | ✅ \* | ✅ \* | | `gemini-3-flash-preview` | ✅ | ✅ | ✅ \* | ✅ \* | diff --git a/packages/prompt-pack/src/prompts/mdma-author/google/gemini-3.1-pro-preview.ts b/packages/prompt-pack/src/prompts/mdma-author/google/gemini-3.1-pro-preview.ts index d6e6134..6b129d3 100644 --- a/packages/prompt-pack/src/prompts/mdma-author/google/gemini-3.1-pro-preview.ts +++ b/packages/prompt-pack/src/prompts/mdma-author/google/gemini-3.1-pro-preview.ts @@ -61,6 +61,27 @@ import { SELECT_OPTIONS_BLOCK, } from './_shared.js'; +// Scope and uniqueness rule for the thinking block. Adapted from +// OpenAI's THINKING_ROLE_BLOCK + NO_REPEAT_BLOCK, which already address +// the same failure mode in gpt-5.4 (correct response then verbatim +// re-emission with duplicate ids). On gemini-3.1-pro the failure surfaces +// slightly differently — the model writes its reasoning in visible prose +// before the thinking block ("Thinking: **Building Table Component**\n\n +// I'm currently focused on…"), then duplicates the thinking + component +// pair after emitting them. This block addresses both the prose leak and +// the duplication by anchoring thinking to a single position. +// +// Placed mid-prompt (after FENCE_CLOSING_BLOCK, before SCOPE_DISCIPLINE) +// because gemini-3.1-pro previously regressed when an imperative +// directive sat at the LITERAL final line — the model re-read it as a +// fresh action prompt and looped. Mid-position avoids that re-read +// trigger. +const THINKING_DISCIPLINE_BLOCK = `## Thinking Block + +The first three characters of your response are \`\`\`\` \`\`\` \`\`\`\` (three backticks) followed by \`mdma\`, opening a thinking block. Nothing precedes it — no greeting, no Markdown heading, no prose starting with "Thinking:" or "**Building X**" or "I'm currently…". All planning (what to build, which fields to include, why certain values are chosen) belongs inside that single thinking block. + +After the thinking block's closing \`\`\`\` \`\`\` \`\`\`\`, generate the requested components in sequence. Each component — including the thinking block itself — appears exactly once. Every \`id\` is unique within your response. The response ends immediately after the closing \`\`\`\` \`\`\` \`\`\`\` of the last component; do not re-emit, re-explain, or restart any part of your output.`; + export const MDMA_AUTHOR_PROMPT_GEMINI_3_1_PRO_PREVIEW = `${BASE_OPENING} ${OUTPUT_FORMAT_BLOCK} @@ -69,6 +90,8 @@ ${BASE_BODY} ${FENCE_CLOSING_BLOCK} +${THINKING_DISCIPLINE_BLOCK} + ${SCOPE_DISCIPLINE_BLOCK} ${SELECT_OPTIONS_BLOCK} From 9994c18247dc84d44181a47449228db3fe408570 Mon Sep 17 00:00:00 2001 From: gitsad Date: Fri, 15 May 2026 17:39:16 +0200 Subject: [PATCH 10/26] chore: gemini 2.5 wip --- README.md | 4 +- demo/src/docs/sections/PromptMatrix.tsx | 13 ++- evals/tests-flows.yaml | 85 ++++++++++++++++--- .../mdma-author/google/gemini-2.5-pro.ts | 48 +++++++++-- .../google/gemini-3.1-pro-preview.ts | 28 ++++-- 5 files changed, 149 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 85c231a..a986f72 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ Each cell shows the pass rate of the model-specialized MDMA_AUTHOR prompt varian | `claude-sonnet-4.6` | ✅ | ✅ | ✅ | ✅ | | `claude-haiku-4.5` | ✅ | ✅ | ✅ \* | ✅ \* | | **Google** | | | | | -| `gemini-3.1-pro-preview` | 🟡 | 🟡 | 🟡 | 🟡 | +| `gemini-3.1-pro-preview` | ✅ | ✅ | ✅ | 🟡 ‡ | | `gemini-3.1-pro-preview-customtools` | ✅ | ✅ | ✅ | ✅ | | `gemini-3.1-flash-lite-preview` | ✅ | ✅ | ✅ \* | ✅ \* | | `gemini-3-flash-preview` | ✅ | ✅ | ✅ \* | ✅ \* | @@ -117,6 +117,8 @@ Each cell shows the pass rate of the model-specialized MDMA_AUTHOR prompt varian † **gpt-5.4 intermittent duplication bug** — `gpt-5.4` passes one-shot evals reliably but shows a non-deterministic output duplication in multi-turn, custom-prompt, and flow evals (~7–15% of runs). The model generates a complete, correct response and then immediately re-emits the entire output verbatim, causing `[duplicate-ids]` validation errors. This is a known model-level issue unrelated to the prompt variant. See the [OpenAI community thread](https://community.openai.com/t/seeing-intermittent-duplicate-strings-in-gpt-5-4-responses/1376651) for details. If this affects your use case, prefer `gpt-5.5` or `gpt-5.2`. +‡ **gemini-3.1-pro-preview stochastic preamble loop** — on ~7–15% of flow-eval runs, the model emits a chain-of-thought as visible Markdown prose (e.g. `**Investigating Production Errors**` repeated 3–5 times) instead of opening a ```` ```mdma ```` block, producing either `[yaml-correctness: outside fenced block]` or `[duplicate-ids]` errors. Per Google's official Gemini 3 prompting guide, this is a model-level behavior driven by temperature/sampling — prompt-level fixes shift which test loops rather than eliminating the loops. If deterministic flow output matters, prefer `gemini-2.5-pro` for production multi-step flows. + \* Smaller / lower-tier models from any lab (OpenAI mini · nano, Anthropic Haiku, Google Gemini Flash, etc.) pass our eval suites, which exercise short, structured test cases. In longer real-world conversations they tend to hallucinate, forget earlier turns, or drift from the spec. For production use that involves multi-turn dialogue or stateful flows, prefer the flagship-tier model from the same family. \[i] Noticeably slow response times — single-turn responses commonly take tens of seconds and full eval runs measure in minutes. diff --git a/demo/src/docs/sections/PromptMatrix.tsx b/demo/src/docs/sections/PromptMatrix.tsx index 722067f..e170ee7 100644 --- a/demo/src/docs/sections/PromptMatrix.tsx +++ b/demo/src/docs/sections/PromptMatrix.tsx @@ -27,7 +27,7 @@ export function PromptMatrix() { ['claude-opus-4.6', '✅', '✅', '✅', '✅'], ['claude-sonnet-4.6', '✅', '✅', '✅', '✅'], ['claude-haiku-4.5', '✅', '✅', '✅ *', '✅ *'], - ['gemini-3.1-pro-preview', '✅', '✅', '✅', '✅'], + ['gemini-3.1-pro-preview', '✅', '✅', '✅', '🟡 ‡'], ['gemini-3.1-pro-preview-customtools', '✅', '✅', '✅', '✅'], ['gemini-3.1-flash-lite-preview', '✅', '✅', '✅ *', '✅ *'], ['gemini-3-flash-preview', '✅', '✅', '✅ *', '✅ *'], @@ -61,6 +61,17 @@ export function PromptMatrix() { {' '} Prefer gpt-5.5 or gpt-5.2 for production use.

+

+ ‡ gemini-3.1-pro-preview stochastic preamble loop — on ~7–15% of flow-eval + runs, the model emits a chain-of-thought as visible Markdown prose (" + **Investigating Production Errors**" repeated 3–5 times) instead of opening a{' '} + ```mdma block, producing either{' '} + [yaml-correctness: outside fenced block] or{' '} + [duplicate-ids] errors. Per Google's official Gemini 3 prompting guide, this + is a model-level behavior driven by temperature/sampling choices — prompt-level fixes shift + which test loops rather than eliminating the loops. Prefer gemini-2.5-pro for + production multi-step flows requiring deterministic output. +

MDMA_AGENT Prompt Matrix

diff --git a/evals/tests-flows.yaml b/evals/tests-flows.yaml index 1326817..2626c17 100644 --- a/evals/tests-flows.yaml +++ b/evals/tests-flows.yaml @@ -38,19 +38,78 @@ - description: "Bug Report flow generates form with severity and component fields" vars: customPrompt: | - You are an engineering support assistant that helps teams file and triage bug reports. - - When a user reports a bug, first collect bug details with a form: - - Title (required) - - Severity: P0-P3 (required, select) - - Affected Component: Frontend/Backend/Database/Auth/Payments (required, select) - - Steps to Reproduce (required, textarea) - - Expected Behavior (required, textarea) - - Actual Behavior (required, textarea) - - Mark reporter email as sensitive. For P0/P1 bugs, show a callout reminding to notify the on-call engineer via PagerDuty. - - For the initial interaction, generate only the bug report form (and an optional callout for critical bugs). The triage checklist and escalation button come in later steps after submission. + You are an engineering support assistant that helps teams file and + triage bug reports. + + When a user reports a bug, generate exactly these two components in + this order: + + ```mdma + type: callout + id: pagerduty-reminder + variant: warning + title: "Critical Incident Reminder" + content: "For P0/P1 incidents, page the on-call engineer via PagerDuty in addition to filing this report." + ``` + + ```mdma + type: form + id: bug-report-form + fields: + - name: title + type: text + label: "Title" + required: true + - name: reporter-email + type: email + label: "Your Email" + required: true + sensitive: true + - name: severity + type: select + label: "Severity" + required: true + options: + - label: "P0 — Outage" + value: "P0" + - label: "P1 — Critical" + value: "P1" + - label: "P2 — High" + value: "P2" + - label: "P3 — Low" + value: "P3" + - name: affected-component + type: select + label: "Affected Component" + required: true + options: + - label: "Frontend" + value: "frontend" + - label: "Backend" + value: "backend" + - label: "Database" + value: "database" + - label: "Auth" + value: "auth" + - label: "Payments" + value: "payments" + - name: steps-to-reproduce + type: textarea + label: "Steps to Reproduce" + required: true + - name: expected-behavior + type: textarea + label: "Expected Behavior" + required: true + - name: actual-behavior + type: textarea + label: "Actual Behavior" + required: true + onSubmit: triage-bug-report + ``` + + Generate only the callout and form. Triage checklist and escalation + button come in later steps after submission. request: The checkout page is showing a 500 error for all users in production. assert: - type: javascript diff --git a/packages/prompt-pack/src/prompts/mdma-author/google/gemini-2.5-pro.ts b/packages/prompt-pack/src/prompts/mdma-author/google/gemini-2.5-pro.ts index a70dcc8..70d7edf 100644 --- a/packages/prompt-pack/src/prompts/mdma-author/google/gemini-2.5-pro.ts +++ b/packages/prompt-pack/src/prompts/mdma-author/google/gemini-2.5-pro.ts @@ -3,22 +3,26 @@ * * Previous-generation Pro (Gemini 3 is current). Same Gemini-native * composition as `gemini-3.1-pro-preview.ts` — Markdown framing, - * end-placed negative constraints. The composition was derived from - * Gemini 3 prompting guides; whether all rules apply identically to - * 2.5 is unverified, but the layout is sensible for any Gemini Pro-tier - * model and the evals validate empirically. + * end-placed negative constraints. * * Composition (Gemini-native ordering): * * BASE_OPENING (role) * + ## Output Format (behavioral directive — top, anchor) * + BASE_BODY (the spec) + * + ## Fence Closing (structural rule — mid) + * + ## Thinking Block (uniqueness + no-preamble — mid) * + ## Scope Discipline (negative constraint — end) * + ## Select Option Values (negative constraint — end) * + BASE_CHECKLIST (## Self-Check Checklist — end) * - * No `## Fence Closing` — Pro-tier hasn't shown that quirk. Add it if - * eval data shows fence-closing failures on Gemini 2.5. + * FENCE_CLOSING_BLOCK + inline THINKING_DISCIPLINE_BLOCK added after + * observing 2 main-eval failures — the model emitted preamble Markdown + * prose ("**Generating MDMA Document**\\n\\nI'm currently focused on…") + * before opening the \`\`\`mdma fence, and generated date-prefixed + * thinking ids (\`20240521-approval-gate-creation\`) that fail + * kebab-case validation. Same family of failures the + * gemini-3.1-pro-preview variant documents. * * Routing: substring match on `gemini-2.5-pro` (14 chars). The Gemini * 3.x variant filenames all contain `3.1` or `3-flash` and don't match @@ -26,7 +30,33 @@ */ import { BASE_BODY, BASE_CHECKLIST, BASE_OPENING } from '../_shared.js'; -import { OUTPUT_FORMAT_BLOCK, SCOPE_DISCIPLINE_BLOCK, SELECT_OPTIONS_BLOCK } from './_shared.js'; +import { + FENCE_CLOSING_BLOCK, + OUTPUT_FORMAT_BLOCK, + SCOPE_DISCIPLINE_BLOCK, + SELECT_OPTIONS_BLOCK, +} from './_shared.js'; + +// Scope and uniqueness rule for the thinking block, scoped to +// gemini-2.5-pro only (kept inline rather than promoted to _shared so +// other Gemini variants' evals are not perturbed). Triggered by the +// same failure mode the gemini-3.1-pro-preview variant documents: +// the model writes a Markdown preamble loop ("Thinking: **Generating +// MDMA Document**\n\nI'm currently focused on…\n\n**Constructing +// MDMA Document**\n\nI'm integrating…") BEFORE opening a \`\`\`mdma +// fence. Also generates date-prefixed thinking ids +// (\`20240521-approval-gate-creation\`) that fail kebab-case validation. +// +// Placed mid-prompt (after FENCE_CLOSING_BLOCK, before SCOPE_DISCIPLINE) +// — the 3.1 Pro variant comment documents a regression where an +// imperative directive at the LITERAL final position caused Gemini to +// re-read it as a fresh action prompt and loop. Mid-position avoids +// that trigger. +const THINKING_DISCIPLINE_BLOCK = `## Thinking Block + +The first three characters of your response are \`\`\`\` \`\`\` \`\`\`\` (three backticks) followed by \`mdma\`, opening a thinking block. Nothing precedes it — no greeting, no Markdown heading, no prose starting with "Thinking:" or "**Building X**" or "I'm currently…". All planning (what to build, which fields to include, why certain values are chosen) belongs inside that single thinking block. + +After the thinking block's closing \`\`\`\` \`\`\` \`\`\`\`, generate the requested components in sequence. Each component — including the thinking block itself — appears exactly once. Every \`id\` is unique within your response and uses lowercase-kebab-case (no date prefixes, no underscores, no uppercase). The response ends immediately after the closing \`\`\`\` \`\`\` \`\`\`\` of the last component; do not re-emit, re-explain, or restart any part of your output.`; export const MDMA_AUTHOR_PROMPT_GEMINI_2_5_PRO = `${BASE_OPENING} @@ -34,6 +64,10 @@ ${OUTPUT_FORMAT_BLOCK} ${BASE_BODY} +${FENCE_CLOSING_BLOCK} + +${THINKING_DISCIPLINE_BLOCK} + ${SCOPE_DISCIPLINE_BLOCK} ${SELECT_OPTIONS_BLOCK} diff --git a/packages/prompt-pack/src/prompts/mdma-author/google/gemini-3.1-pro-preview.ts b/packages/prompt-pack/src/prompts/mdma-author/google/gemini-3.1-pro-preview.ts index 6b129d3..515dd4f 100644 --- a/packages/prompt-pack/src/prompts/mdma-author/google/gemini-3.1-pro-preview.ts +++ b/packages/prompt-pack/src/prompts/mdma-author/google/gemini-3.1-pro-preview.ts @@ -6,6 +6,8 @@ * BASE_OPENING (role) * + ## Output Format (behavioral directive, top — anchor) * + BASE_BODY (the spec) + * + ## Fence Closing (structural rule — mid) + * + ## Thinking Block (uniqueness + no-duplicate — mid) * + ## Scope Discipline (negative constraint — end) * + ## Select Option Values (negative constraint — end) * + BASE_CHECKLIST (## Self-Check Checklist — end) @@ -40,14 +42,26 @@ * warns: "Lower temperatures may cause unexpected behavior, looping, or * degraded performance." * - * Failure-mode coverage: Fence Closing, Scope Discipline, and Select - * Option Values — all validated as failure modes for Gemini 3.1 Pro - * specifically (5 of 6 main-eval failures on a fresh run were fence - * issues, plus 1 in flows). FENCE_CLOSING_BLOCK goes mid-prompt (after - * BASE_BODY) so the spec defines what an mdma block is before the rule - * tightens its closing. SCOPE_DISCIPLINE and SELECT_OPTIONS stay at the + * Failure-mode coverage: Fence Closing, Thinking Block discipline, + * Scope Discipline, and Select Option Values — all validated as failure + * modes for Gemini 3.1 Pro specifically. FENCE_CLOSING_BLOCK and + * THINKING_DISCIPLINE_BLOCK go mid-prompt (after BASE_BODY) so the spec + * defines what an mdma block is before the rules tighten emission; + * THINKING_DISCIPLINE addresses gpt-5.4-style duplicate-emission that + * surfaced on Gemini Pro (correct response then verbatim re-emission + * with duplicate ids). SCOPE_DISCIPLINE and SELECT_OPTIONS stay at the * end per Vertex guidance on negative constraints. * + * Stochastic loop floor: even after the above, gemini-3.1-pro-preview + * still loops on ~7–15% of flow runs — the model emits Markdown prose + * preamble ("**Investigating Production Errors**" × 3-5) instead of + * opening a \`\`\`mdma block. Per Google's prompt guide, this is + * temperature-driven model behavior — no prompt-level fix reduced the + * count below this floor (tested: literal first-byte example, positive + * scope reframe, no-loop block, no-duplicates tail — all shifted which + * test loops, none reduced the count). Documented as 🟡 ‡ in the + * Prompt Matrix similar to gpt-5.4's documented duplication bug. + * * Routing: substring match on `gemini-3.1-pro-preview` (24 chars). Picks * this variant for any model id containing that literal, including * `google/gemini-3.1-pro-preview` and any preview-suffixed alias. @@ -80,7 +94,7 @@ const THINKING_DISCIPLINE_BLOCK = `## Thinking Block The first three characters of your response are \`\`\`\` \`\`\` \`\`\`\` (three backticks) followed by \`mdma\`, opening a thinking block. Nothing precedes it — no greeting, no Markdown heading, no prose starting with "Thinking:" or "**Building X**" or "I'm currently…". All planning (what to build, which fields to include, why certain values are chosen) belongs inside that single thinking block. -After the thinking block's closing \`\`\`\` \`\`\` \`\`\`\`, generate the requested components in sequence. Each component — including the thinking block itself — appears exactly once. Every \`id\` is unique within your response. The response ends immediately after the closing \`\`\`\` \`\`\` \`\`\`\` of the last component; do not re-emit, re-explain, or restart any part of your output.`; +After the thinking block's closing \`\`\`\` \`\`\` \`\`\`\`, generate the requested components in sequence. Each component — including the thinking block itself — appears exactly once. Every \`id\` is unique within your response and uses lowercase-kebab-case (no date prefixes, no underscores, no uppercase). The response ends immediately after the closing \`\`\`\` \`\`\` \`\`\`\` of the last component; do not re-emit, re-explain, or restart any part of your output.`; export const MDMA_AUTHOR_PROMPT_GEMINI_3_1_PRO_PREVIEW = `${BASE_OPENING} From b5b27f126a569938de633bdb3ab503903fcfef70 Mon Sep 17 00:00:00 2001 From: gitsad Date: Fri, 15 May 2026 17:58:54 +0200 Subject: [PATCH 11/26] chore: finished gemini --- README.md | 2 +- .../google/gemini-2.5-flash-lite.ts | 17 ++++++++ .../mdma-author/google/gemini-2.5-flash.ts | 42 +++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a986f72..c2ba3d3 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ Each cell shows the pass rate of the model-specialized MDMA_AUTHOR prompt varian | `gemini-3-flash-preview` | ✅ | ✅ | ✅ \* | ✅ \* | | `gemini-2.5-pro` | ✅ | ✅ | ✅ | ✅ | | `gemini-2.5-flash` | ✅ | ✅ | ✅ \* | ✅ \* | -| `gemini-2.5-flash-lite` | 🟡 | ✅ | ✅ \* | ✅ \* | +| `gemini-2.5-flash-lite` | ✅ | ✅ | ✅ \* | ✅ \* | | **xAI** | | | | | | `grok-4.3` \[i] | 🟡 | 🔴 | 🔴 | 🔴 | | `grok-4.20` | ✅ | ✅ | ✅ | ✅ | diff --git a/packages/prompt-pack/src/prompts/mdma-author/google/gemini-2.5-flash-lite.ts b/packages/prompt-pack/src/prompts/mdma-author/google/gemini-2.5-flash-lite.ts index 16e68ac..24f44d9 100644 --- a/packages/prompt-pack/src/prompts/mdma-author/google/gemini-2.5-flash-lite.ts +++ b/packages/prompt-pack/src/prompts/mdma-author/google/gemini-2.5-flash-lite.ts @@ -53,6 +53,21 @@ const FENCE_IN_CONTENT_BLOCK = `## No Fence Characters in Content Fields **Never write the literal characters \`\`\`mdma or \`\`\` (triple backticks) inside any block's \`content:\` field.** The Markdown parser walks the document looking for fence pairs; phantom fences inside a YAML block scalar break the open/close count and the document fails validation. When reasoning inside a \`thinking\` block about MDMA structure, refer to blocks in plain prose ("the form below", "the next component", "this block") — never quote fence syntax verbatim.`; +// Scoped to gemini-2.5-flash-lite only. Triggered by a conversation-eval +// failure on Conv 11/T2: after generating an event registration form in +// T1, the user's T2 message was "What if someone has a nut allergy? +// That's not listed in the dietary options." The model interpreted this +// as a request to UPDATE the form and re-emitted the entire form with +// "nut-allergy" added to the dietary-preference options, instead of +// responding in plain prose. The custom prompt's "respond conversationally +// without regenerating" instruction is being overridden by Flash-Lite's +// strong "be helpful, fix the gap" instinct. +const NO_REGENERATION_BLOCK = `## Follow-Up Conversations + +When the user asks a question about a component that you already emitted in an earlier turn of the conversation, respond in conversational prose only. Do NOT re-emit, update, append fields to, or otherwise regenerate any \`\`\`mdma block from a previous turn — even when the user points out a missing option, suggests an improvement, or asks a clarifying question. + +The component you emitted earlier is still visible to the user. Modifying it requires its own dedicated turn where the user explicitly asks for the change ("please add a nut-allergy option" — explicit request, regenerate); a passive question ("what if someone has a nut allergy?" — answer in prose) does not.`; + export const MDMA_AUTHOR_PROMPT_GEMINI_2_5_FLASH_LITE = `${BASE_OPENING} ${OUTPUT_FORMAT_BLOCK} @@ -67,5 +82,7 @@ ${SCOPE_DISCIPLINE_BLOCK} ${SELECT_OPTIONS_BLOCK} +${NO_REGENERATION_BLOCK} + ${BASE_CHECKLIST} `; diff --git a/packages/prompt-pack/src/prompts/mdma-author/google/gemini-2.5-flash.ts b/packages/prompt-pack/src/prompts/mdma-author/google/gemini-2.5-flash.ts index f06b977..fd57408 100644 --- a/packages/prompt-pack/src/prompts/mdma-author/google/gemini-2.5-flash.ts +++ b/packages/prompt-pack/src/prompts/mdma-author/google/gemini-2.5-flash.ts @@ -38,6 +38,46 @@ import { SELECT_OPTIONS_BLOCK, } from './_shared.js'; +// Scoped to gemini-2.5-flash only. Triggered by a flows-eval failure +// where the model emitted a malformed select option for the customer +// sentiment field — duplicating "Positive" mid-list with the second +// entry missing the \`value:\` field entirely: +// +// options: +// - label: Positive +// value: positive +// - label: positive ← missing value, partial duplicate +// - label: Neutral +// value: neutral +// +// The shared SELECT_OPTIONS_BLOCK only addresses string-vs-number on +// value; this block adds the orthogonal rule that each entry must be +// complete (both label AND value) and options must not be duplicated. +const SELECT_ENTRY_COMPLETENESS_BLOCK = `## Select Option Entry Completeness + +Every entry in a \`type: select\` field's \`options\` array has BOTH a \`label\` and a \`value\` — never a label alone. Each distinct choice appears once; do not duplicate or near-duplicate (e.g., \`Positive\` then \`positive\`). + +Wrong (malformed and duplicated): + +\`\`\`yaml +options: + - label: Positive + value: positive + - label: positive # missing value, duplicate of "Positive" + - label: Neutral + value: neutral +\`\`\` + +Right: + +\`\`\`yaml +options: + - label: Positive + value: positive + - label: Neutral + value: neutral +\`\`\``; + export const MDMA_AUTHOR_PROMPT_GEMINI_2_5_FLASH = `${BASE_OPENING} ${OUTPUT_FORMAT_BLOCK} @@ -50,5 +90,7 @@ ${SCOPE_DISCIPLINE_BLOCK} ${SELECT_OPTIONS_BLOCK} +${SELECT_ENTRY_COMPLETENESS_BLOCK} + ${BASE_CHECKLIST} `; From 4526b3b25601b12aae3f50b0f5abaf6efb2d93fd Mon Sep 17 00:00:00 2001 From: gitsad Date: Tue, 19 May 2026 13:42:50 +0200 Subject: [PATCH 12/26] fix: updated schema --- .../spec/src/schemas/components/approval-gate.ts | 4 ++-- packages/spec/src/schemas/components/button.ts | 2 +- packages/spec/src/schemas/components/form.ts | 2 +- packages/spec/src/schemas/components/tasklist.ts | 2 +- packages/spec/src/schemas/components/webhook.ts | 2 +- packages/validator/src/rules/index.ts | 12 ++++++++++-- 6 files changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/spec/src/schemas/components/approval-gate.ts b/packages/spec/src/schemas/components/approval-gate.ts index 1660727..dd27994 100644 --- a/packages/spec/src/schemas/components/approval-gate.ts +++ b/packages/spec/src/schemas/components/approval-gate.ts @@ -7,8 +7,8 @@ export const ApprovalGateComponentSchema = ComponentBaseSchema.extend({ description: z.string().optional(), requiredApprovers: z.number().int().positive().default(1), allowedRoles: z.array(z.string()).optional(), - onApprove: z.string().optional().describe('Action ID triggered on approval'), - onDeny: z.string().optional().describe('Action ID triggered on denial'), + onApprove: z.string().optional().describe('Action ID dispatched on approval'), + onDeny: z.string().optional().describe('Action ID dispatched on denial'), requireReason: z.boolean().default(false).describe('Require reason on denial'), }); diff --git a/packages/spec/src/schemas/components/button.ts b/packages/spec/src/schemas/components/button.ts index 8ce5867..ba50676 100644 --- a/packages/spec/src/schemas/components/button.ts +++ b/packages/spec/src/schemas/components/button.ts @@ -5,7 +5,7 @@ export const ButtonComponentSchema = ComponentBaseSchema.extend({ type: z.literal('button'), text: z.string().min(1), variant: z.enum(['primary', 'secondary', 'danger', 'ghost']).default('primary'), - onAction: z.string().describe('Action ID to trigger on click').optional(), + onAction: z.string().describe('Action ID dispatched on click').optional(), confirm: z .object({ title: z.string(), diff --git a/packages/spec/src/schemas/components/form.ts b/packages/spec/src/schemas/components/form.ts index 3e2d77b..aec289c 100644 --- a/packages/spec/src/schemas/components/form.ts +++ b/packages/spec/src/schemas/components/form.ts @@ -29,7 +29,7 @@ export const FormFieldSchema = z.object({ export const FormComponentSchema = ComponentBaseSchema.extend({ type: z.literal('form'), fields: z.array(FormFieldSchema).min(1), - onSubmit: z.string().describe('Action ID to trigger on submit'), + onSubmit: z.string().describe('Action ID dispatched on submit'), }); export type FormField = z.infer; diff --git a/packages/spec/src/schemas/components/tasklist.ts b/packages/spec/src/schemas/components/tasklist.ts index 2254d5a..7291f9d 100644 --- a/packages/spec/src/schemas/components/tasklist.ts +++ b/packages/spec/src/schemas/components/tasklist.ts @@ -13,7 +13,7 @@ export const TaskItemSchema = z.object({ export const TasklistComponentSchema = ComponentBaseSchema.extend({ type: z.literal('tasklist'), items: z.array(TaskItemSchema).min(1), - onComplete: z.string().optional().describe('Action ID triggered when all items are checked'), + onComplete: z.string().optional().describe('Action ID dispatched when all items are checked'), }); export type TaskItem = z.infer; diff --git a/packages/spec/src/schemas/components/webhook.ts b/packages/spec/src/schemas/components/webhook.ts index f20d25c..59fb54c 100644 --- a/packages/spec/src/schemas/components/webhook.ts +++ b/packages/spec/src/schemas/components/webhook.ts @@ -8,7 +8,7 @@ export const WebhookComponentSchema = ComponentBaseSchema.extend({ method: z.enum(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']).default('POST'), headers: z.record(z.string()).optional(), body: z.union([z.record(z.unknown()), BindingExpressionSchema]).optional(), - trigger: z.string().describe('Action ID that triggers this webhook'), + trigger: z.string().describe('Action ID that fires this webhook'), retries: z.number().int().min(0).max(5).default(0), timeout: z.number().int().positive().default(30000).describe('Timeout in milliseconds'), }); diff --git a/packages/validator/src/rules/index.ts b/packages/validator/src/rules/index.ts index b11422a..2eef0a8 100644 --- a/packages/validator/src/rules/index.ts +++ b/packages/validator/src/rules/index.ts @@ -8,7 +8,15 @@ import { bindingSyntaxRule } from './binding-syntax.js'; // Disabled: binding-resolution checks intra-message bindings but components // and their bindings are never generated in the same message. // import { bindingResolutionRule } from './binding-resolution.js'; -import { actionReferencesRule } from './action-references.js'; +// Disabled: action-references checks that onSubmit/onAction/etc. resolve to +// component IDs in the same message — but the spec now treats action labels +// as opaque external handlers the host application wires up at runtime. +// With one interactive component per message, action labels naturally point +// to follow-up handlers in later messages (or external code), not in-document +// targets. The rule was firing on every valid form's onSubmit, producing +// noise warnings on otherwise-passing outputs. Same family as the two +// disabled rules above (intra-message refs that don't apply to multi-turn). +// import { actionReferencesRule } from './action-references.js'; import { sensitiveFlagsRule } from './sensitive-flags.js'; import { requiredMarkersRule } from './required-markers.js'; import { thinkingBlockRule } from './thinking-block.js'; @@ -41,7 +49,7 @@ export const ALL_RULES: readonly ValidationRule[] = [ idFormatRule, bindingSyntaxRule, // bindingResolutionRule, - actionReferencesRule, + // actionReferencesRule, sensitiveFlagsRule, requiredMarkersRule, thinkingBlockRule, From c9ac353a50a6ccb7cd9c6431f0a7caa3283c5eaa Mon Sep 17 00:00:00 2001 From: gitsad Date: Tue, 19 May 2026 15:39:38 +0200 Subject: [PATCH 13/26] chore: clenaup, change validation to one block, changed validationFlow to validationConversation --- demo/src/ValidatorView.tsx | 6 +- demo/src/validator-prompts.ts | 9 +- demo/src/validator/FlowProgressPanel.tsx | 10 +- evals/assertions/fixer-no-prose.mjs | 31 ++ evals/prompt-fixer.mjs | 9 +- evals/promptfooconfig.fixer-flow.yaml | 2 + evals/promptfooconfig.fixer.yaml | 2 + evals/tests-fixer.yaml | 272 +++++++----------- .../src/prompts/mdma-fixer/_shared.ts | 12 + packages/validator/src/constants.ts | 9 +- .../validator/src/fixes/action-references.ts | 27 -- packages/validator/src/fixes/index.ts | 6 +- packages/validator/src/index.ts | 12 +- .../validator/src/rules/action-references.ts | 50 ---- packages/validator/src/rules/index.ts | 10 - packages/validator/src/types.ts | 1 - ...idate-flow.ts => validate-conversation.ts} | 119 +++++--- .../tests/rules/action-references.test.ts | 182 ------------ 18 files changed, 266 insertions(+), 503 deletions(-) create mode 100644 evals/assertions/fixer-no-prose.mjs delete mode 100644 packages/validator/src/fixes/action-references.ts delete mode 100644 packages/validator/src/rules/action-references.ts rename packages/validator/src/{validate-flow.ts => validate-conversation.ts} (54%) delete mode 100644 packages/validator/tests/rules/action-references.test.ts diff --git a/demo/src/ValidatorView.tsx b/demo/src/ValidatorView.tsx index 309c1cd..59fa140 100644 --- a/demo/src/ValidatorView.tsx +++ b/demo/src/ValidatorView.tsx @@ -5,7 +5,7 @@ import { ChatMessage } from './chat/ChatMessage.js'; import { ChatInput } from './chat/ChatInput.js'; import { ChatActionLog } from './chat/ChatActionLog.js'; import { useChatActionLog } from './chat/use-chat-action-log.js'; -import { validateFlow, type FlowStepDefinition } from '@mobile-reality/mdma-validator'; +import { validateConversation, type ConversationStep } from '@mobile-reality/mdma-validator'; import { customizations } from './custom-components.js'; import { VALIDATOR_PROMPT_VARIANTS, FLOW_STEPS } from './validator-prompts.js'; import { ValidationPanel } from './validator/ValidationPanel.js'; @@ -72,14 +72,14 @@ function ValidatorChatInner({ promptKey }: { promptKey: string }) { }); // Flow validation - const flowSteps = FLOW_STEPS[promptKey] as FlowStepDefinition[] | undefined; + const flowSteps = FLOW_STEPS[promptKey] as ConversationStep[] | undefined; const flowResult = useMemo(() => { if (!flowSteps) return null; const assistantContents = messages .filter((m) => m.role === 'assistant' && m.content) .map((m) => m.content); if (assistantContents.length === 0) return null; - return validateFlow(assistantContents, { steps: flowSteps }); + return validateConversation(assistantContents, { steps: flowSteps }); }, [flowSteps, messages]); const flowComplete = diff --git a/demo/src/validator-prompts.ts b/demo/src/validator-prompts.ts index 963aae7..dbb1471 100644 --- a/demo/src/validator-prompts.ts +++ b/demo/src/validator-prompts.ts @@ -1,4 +1,4 @@ -import type { ExpectedComponent, FlowStepDefinition } from '@mobile-reality/mdma-validator'; +import type { ExpectedComponent, ConversationStep } from '@mobile-reality/mdma-validator'; export interface ValidatorPromptVariant { key: string; @@ -673,10 +673,11 @@ export const EXPECTED_COMPONENTS: Record = { +export const FLOW_STEPS: Record = { flow: [ { label: 'Registration Form', type: 'form', id: 'registration-form' }, { label: 'Manager Approval', type: 'approval-gate', id: 'approval-gate' }, diff --git a/demo/src/validator/FlowProgressPanel.tsx b/demo/src/validator/FlowProgressPanel.tsx index 959343b..1365671 100644 --- a/demo/src/validator/FlowProgressPanel.tsx +++ b/demo/src/validator/FlowProgressPanel.tsx @@ -1,15 +1,15 @@ -import type { FlowStepDefinition, FlowValidationResult } from '@mobile-reality/mdma-validator'; +import type { ConversationStep, ValidateConversationResult } from '@mobile-reality/mdma-validator'; interface FlowProgressPanelProps { - steps: FlowStepDefinition[]; - result: FlowValidationResult | null; + steps: ConversationStep[]; + result: ValidateConversationResult | null; } type StepStatus = 'pending' | 'done' | 'error'; function getStepStatus( - result: FlowValidationResult | null, - step: FlowStepDefinition, + result: ValidateConversationResult | null, + step: ConversationStep, stepIndex: number, ): StepStatus { if (!result) return 'pending'; diff --git a/evals/assertions/fixer-no-prose.mjs b/evals/assertions/fixer-no-prose.mjs new file mode 100644 index 0000000..0746d09 --- /dev/null +++ b/evals/assertions/fixer-no-prose.mjs @@ -0,0 +1,31 @@ +/** + * Custom promptfoo assertion for fixer eval. + * + * Enforces that the fixer output contains ONLY ```mdma blocks — no prose, + * headings, intro/outro text, or commentary outside the blocks. The fixer's + * job is to repair MDMA blocks, not to converse with the user. + * + * Allowed in the output: ```mdma blocks and whitespace between them. + * Disallowed: prose paragraphs, Markdown headings, lists, code fences other + * than `mdma`, or any text outside a ```mdma ... ``` pair. + */ +export default function (output) { + // Strip every ```mdma ... ``` block (greedy across newlines, non-greedy on content) + const stripped = output.replace(/```mdma\n[\s\S]*?```/g, '').trim(); + + if (stripped.length === 0) { + return { + pass: true, + score: 1, + reason: 'Fixer output contains only ```mdma blocks (no prose)', + }; + } + + // Truncate the offending content for the failure message + const preview = stripped.length > 200 ? `${stripped.slice(0, 200)}...` : stripped; + return { + pass: false, + score: 0, + reason: `Fixer output contains non-mdma content (${stripped.length} chars):\n${preview}`, + }; +} diff --git a/evals/prompt-fixer.mjs b/evals/prompt-fixer.mjs index ed84100..21c4086 100644 --- a/evals/prompt-fixer.mjs +++ b/evals/prompt-fixer.mjs @@ -21,8 +21,13 @@ import { selectFixerPrompt } from './select-prompt.mjs'; * 3. Sends the fixer system prompt (with variant-specific extensions) + user message */ export default async function ({ vars }) { + // Default to single-block scope unless the test explicitly opts into + // multi-step (variantKey: 'flow'). For single-block tests we also drop + // the flow-ordering rule from validate() since by design each test has + // exactly one mdma block — no multi-step ordering to check. + const variantKey = vars.variantKey ?? 'single-block'; const exclude = ['thinking-block']; - if (vars.variantKey !== 'flow') exclude.push('flow-ordering'); + if (variantKey !== 'flow') exclude.push('flow-ordering'); const result = validate(vars.brokenDocument, { exclude }); const allIssues = result.issues.filter( @@ -31,7 +36,7 @@ export default async function ({ vars }) { const { prompt: variantPrompt, source: fixerSource } = await selectFixerPrompt(); const fixerPrompt = fixerSource.startsWith('default') - ? buildFixerPrompt(vars.variantKey ?? undefined) + ? buildFixerPrompt(variantKey) : variantPrompt; const systemPrompt = `${buildSystemPrompt()}\n\n---\n\n${fixerPrompt}`; diff --git a/evals/promptfooconfig.fixer-flow.yaml b/evals/promptfooconfig.fixer-flow.yaml index 2eafbaf..00279d7 100644 --- a/evals/promptfooconfig.fixer-flow.yaml +++ b/evals/promptfooconfig.fixer-flow.yaml @@ -32,5 +32,7 @@ defaultTest: value: file://assertions/fixer-preserves-components.mjs config: min: 1 + - type: javascript + value: file://assertions/fixer-no-prose.mjs tests: tests-fixer-flow.yaml diff --git a/evals/promptfooconfig.fixer.yaml b/evals/promptfooconfig.fixer.yaml index aa17073..e5d96a3 100644 --- a/evals/promptfooconfig.fixer.yaml +++ b/evals/promptfooconfig.fixer.yaml @@ -30,5 +30,7 @@ defaultTest: value: file://assertions/fixer-preserves-components.mjs config: min: 1 + - type: javascript + value: file://assertions/fixer-no-prose.mjs tests: tests-fixer.yaml diff --git a/evals/tests-fixer.yaml b/evals/tests-fixer.yaml index 10f3a9b..bfee2b1 100644 --- a/evals/tests-fixer.yaml +++ b/evals/tests-fixer.yaml @@ -10,8 +10,6 @@ - description: Fixes button missing required text field vars: brokenDocument: | - # Quick Action - ```mdma type: button id: action-btn @@ -40,8 +38,6 @@ - description: Fixes callout missing required content field vars: brokenDocument: | - # Status Update - ```mdma type: callout id: status-notice @@ -72,8 +68,6 @@ - description: Fixes select field missing required options array vars: brokenDocument: | - # Contact Form - ```mdma type: form id: contact-form @@ -116,8 +110,6 @@ - description: Fixes placeholder title and content in callout vars: brokenDocument: | - # Welcome Screen - ```mdma type: callout id: welcome-callout @@ -151,8 +143,6 @@ - description: Fixes email and phone fields missing sensitive flag vars: brokenDocument: | - # Contact Details - ```mdma type: form id: contact-details @@ -198,75 +188,16 @@ sensitive: true # --------------------------------------------------------------------------- -# 6. Form with broken onSubmit reference -# --------------------------------------------------------------------------- -- description: Fixes form with broken onSubmit reference to point to existing callout - vars: - brokenDocument: | - # Order Submission - - ```mdma - type: form - id: order-form - fields: - - name: product - type: text - label: Product Name - required: true - - name: quantity - type: number - label: Quantity - onSubmit: nonexistent-handler - ``` - - ```mdma - type: callout - id: order-status - variant: success - content: Your order has been submitted! - ``` - assert: - - type: javascript - value: file://assertions/fixer-resolves-errors.mjs - - type: javascript - value: file://assertions/fixer-preserves-components.mjs - config: - min: 2 - - type: javascript - value: file://assertions/fixer-contains-component.mjs - config: - expected: | - type: form - id: order-form - onSubmit: order-status - -# --------------------------------------------------------------------------- -# 7. Unknown component type + missing required button text +# 6. Button missing required text (with opaque onAction handler) # --------------------------------------------------------------------------- -- description: Fixes unknown component type and missing button text +- description: Fixes button missing required text field (preserves opaque onAction label) vars: brokenDocument: | - # Dashboard - - ```mdma - type: card - id: stats-card - title: Monthly Stats - value: 42 - ``` - ```mdma type: button id: refresh-btn variant: primary - onAction: stats-card - ``` - - ```mdma - type: callout - id: dashboard-info - variant: info - content: Welcome to your dashboard + onAction: refresh-stats ``` assert: - type: javascript @@ -274,25 +205,23 @@ - type: javascript value: file://assertions/fixer-preserves-components.mjs config: - min: 2 + min: 1 - type: javascript value: file://assertions/fixer-contains-component.mjs config: expected: | type: button id: refresh-btn - onAction: stats-card + onAction: refresh-stats hasFields: - text # --------------------------------------------------------------------------- # 8. Form missing onSubmit # --------------------------------------------------------------------------- -- description: Fixes form that is missing onSubmit by connecting it to the success callout +- description: Fixes form missing required onSubmit field vars: brokenDocument: | - # User Profile - ```mdma type: form id: profile-form @@ -309,20 +238,13 @@ type: textarea label: Bio ``` - - ```mdma - type: callout - id: profile-saved - variant: success - content: Your profile has been saved. - ``` assert: - type: javascript value: file://assertions/fixer-resolves-errors.mjs - type: javascript value: file://assertions/fixer-preserves-components.mjs config: - min: 2 + min: 1 - type: javascript value: file://assertions/fixer-contains-component.mjs config: @@ -338,8 +260,6 @@ - description: Fixes field name typos on approval-gate (roles→allowedRoles, approvers→requiredApprovers) vars: brokenDocument: | - # Leave Request - ```mdma type: approval-gate id: leave-approval @@ -350,20 +270,13 @@ approvers: 2 onApprove: leave-confirmed ``` - - ```mdma - type: callout - id: leave-confirmed - variant: success - content: Your leave request has been approved! - ``` assert: - type: javascript value: file://assertions/fixer-resolves-errors.mjs - type: javascript value: file://assertions/fixer-preserves-components.mjs config: - min: 2 + min: 1 - type: javascript value: file://assertions/fixer-contains-component.mjs config: @@ -378,13 +291,11 @@ onApprove: leave-confirmed # --------------------------------------------------------------------------- -# 10. Table data key mismatch + chart axis mismatch +# 10. Table data key mismatch # --------------------------------------------------------------------------- -- description: Fixes table data key mismatch and chart axis errors +- description: Fixes table data key mismatch (data keys don't match column keys) vars: brokenDocument: | - # Sales Report - ```mdma type: table id: sales-table @@ -403,7 +314,42 @@ total_revenue: 32000 quantity: 85 ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 1 + - type: icontains + value: "product:" + - type: javascript + value: file://assertions/fixer-contains-component.mjs + config: + expected: | + type: table + id: sales-table + columns: + - key: product + header: Product + - key: revenue + header: Revenue + - key: units + header: Units Sold + data: + - product: Widget A + revenue: 50000 + units: 120 + - product: Widget B + revenue: 32000 + units: 85 +# --------------------------------------------------------------------------- +# 10b. Chart axis mismatch +# --------------------------------------------------------------------------- +- description: Fixes chart axis mismatch (axes don't match data columns) + vars: + brokenDocument: | ```mdma type: chart id: sales-chart @@ -424,11 +370,9 @@ - type: javascript value: file://assertions/fixer-preserves-components.mjs config: - min: 2 + min: 1 - type: icontains value: "xAxis: Month" - - type: icontains - value: "product:" - type: javascript value: file://assertions/fixer-contains-component.mjs config: @@ -440,35 +384,13 @@ yAxis: - Revenue - Costs - - type: javascript - value: file://assertions/fixer-contains-component.mjs - config: - expected: | - type: table - id: sales-table - columns: - - key: product - header: Product - - key: revenue - header: Revenue - - key: units - header: Units Sold - data: - - product: Widget A - revenue: 50000 - units: 120 - - product: Widget B - revenue: 32000 - units: 85 # --------------------------------------------------------------------------- -# 11. Missing sensitive flags on form and table +# 11. Missing sensitive flags on form fields # --------------------------------------------------------------------------- -- description: Fixes missing PII sensitive flags on form fields and table columns +- description: Fixes missing PII sensitive flags on form fields vars: brokenDocument: | - # Patient Registration - ```mdma type: form id: patient-form @@ -491,39 +413,13 @@ label: Home Address onSubmit: registration-complete ``` - - ```mdma - type: table - id: patient-records - columns: - - key: name - header: Patient Name - - key: email - header: Email - - key: phone - header: Phone - - key: dob - header: Date of Birth - data: - - name: Jane Doe - email: jane@example.com - phone: 555-0101 - dob: 1990-01-15 - ``` - - ```mdma - type: callout - id: registration-complete - variant: success - content: Registration submitted successfully! - ``` assert: - type: javascript value: file://assertions/fixer-resolves-errors.mjs - type: javascript value: file://assertions/fixer-preserves-components.mjs config: - min: 3 + min: 1 - type: javascript value: file://assertions/has-sensitive.mjs - type: javascript @@ -554,6 +450,40 @@ label: Home Address sensitive: true onSubmit: registration-complete + +# --------------------------------------------------------------------------- +# 11b. Missing sensitive flags on table columns +# --------------------------------------------------------------------------- +- description: Fixes missing PII sensitive flags on table columns + vars: + brokenDocument: | + ```mdma + type: table + id: patient-records + columns: + - key: name + header: Patient Name + - key: email + header: Email + - key: phone + header: Phone + - key: dob + header: Date of Birth + data: + - name: Jane Doe + email: jane@example.com + phone: 555-0101 + dob: 1990-01-15 + ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 1 + - type: javascript + value: file://assertions/has-sensitive.mjs - type: javascript value: file://assertions/fixer-contains-component.mjs config: @@ -577,11 +507,9 @@ # --------------------------------------------------------------------------- # 12. Mixed issues — single form kitchen sink # --------------------------------------------------------------------------- -- description: Fixes many issues on a single form (ID format, placeholder, PII, select, onSubmit) +- description: Fixes many issues on a single form (ID format, placeholder, PII, select) vars: brokenDocument: | - # Employee Onboarding - ```mdma type: form id: employee_form @@ -600,20 +528,13 @@ label: Start Date onSubmit: missing-handler ``` - - ```mdma - type: callout - id: onboarding-complete - variant: success - content: Welcome to the team! - ``` assert: - type: javascript value: file://assertions/fixer-resolves-errors.mjs - type: javascript value: file://assertions/fixer-preserves-components.mjs config: - min: 2 + min: 1 - type: javascript value: file://assertions/unique-kebab-ids.mjs - type: javascript @@ -628,13 +549,11 @@ - onSubmit # --------------------------------------------------------------------------- -# 13. Placeholder content throughout +# 13. Placeholder content in form labels # --------------------------------------------------------------------------- -- description: Fixes placeholder content in labels and fields +- description: Fixes placeholder content in form field labels vars: brokenDocument: | - # Project Setup - ```mdma type: form id: project-form @@ -651,7 +570,22 @@ label: FIXME onSubmit: project-summary ``` + assert: + - type: javascript + value: file://assertions/fixer-resolves-errors.mjs + - type: javascript + value: file://assertions/fixer-preserves-components.mjs + config: + min: 1 + - type: javascript + value: file://assertions/no-placeholder-content.mjs +# --------------------------------------------------------------------------- +# 13b. Placeholder content in callout +# --------------------------------------------------------------------------- +- description: Fixes placeholder content in callout title and content + vars: + brokenDocument: | ```mdma type: callout id: project-summary @@ -665,6 +599,6 @@ - type: javascript value: file://assertions/fixer-preserves-components.mjs config: - min: 2 + min: 1 - type: javascript value: file://assertions/no-placeholder-content.mjs diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/_shared.ts b/packages/prompt-pack/src/prompts/mdma-fixer/_shared.ts index 2236c7c..195ca15 100644 --- a/packages/prompt-pack/src/prompts/mdma-fixer/_shared.ts +++ b/packages/prompt-pack/src/prompts/mdma-fixer/_shared.ts @@ -388,6 +388,18 @@ export const FIXER_EXTENSIONS: Record = { MDMA_FIXER_FLOW, MDMA_FIXER_APPROVAL, ], + // Single-block focus: every per-component fix, no multi-step workflow logic. + // Use this preset for callers that validate one MDMA block at a time via + // `validate()` — there is no conversation history to reason about, so the + // multi-step FLOW extension would only confuse the model. + 'single-block': [ + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_PII, + MDMA_FIXER_FORMS, + MDMA_FIXER_TABLES_CHARTS, + MDMA_FIXER_APPROVAL, + ], structure: [MDMA_FIXER_STRUCTURE], bindings: [MDMA_FIXER_BINDINGS], pii: [MDMA_FIXER_PII], diff --git a/packages/validator/src/constants.ts b/packages/validator/src/constants.ts index 0ec3591..bde2a16 100644 --- a/packages/validator/src/constants.ts +++ b/packages/validator/src/constants.ts @@ -1,7 +1,10 @@ /** - * Maps component types to their fields that are cross-references to other component IDs. - * Used by action-references rule, unreferenced-components rule, flow-ordering rule, - * and the action-references fix. + * Maps component types to their action-label fields (opaque handler IDs + * dispatched by the host application at runtime, not document-internal + * cross-references). Used by `flow-ordering` for cycle/forward-ref checks + * when the value happens to match an in-doc ID, and by `id-format` to + * update action-label values when a referenced component's ID gets + * normalized. */ export const ACTION_REFERENCE_FIELDS: Record = { form: ['onSubmit'], diff --git a/packages/validator/src/fixes/action-references.ts b/packages/validator/src/fixes/action-references.ts deleted file mode 100644 index 2228989..0000000 --- a/packages/validator/src/fixes/action-references.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { FixContext } from '../types.js'; -import { ACTION_REFERENCE_FIELDS } from '../constants.js'; - -/** - * Remove invalid cross-reference field values for all component types. - */ -export function fixActionReferences(context: FixContext): void { - for (const issue of context.issues) { - if (issue.ruleId !== 'action-references' || issue.fixed) continue; - - const block = context.blocks[issue.blockIndex]; - if (!block?.data) continue; - - const field = issue.field; - if (!field) continue; - - const type = block.data.type; - if (typeof type !== 'string') continue; - - const fields = ACTION_REFERENCE_FIELDS[type]; - if (!fields || !fields.includes(field)) continue; - - // Remove the invalid cross-reference field - delete block.data[field]; - issue.fixed = true; - } -} diff --git a/packages/validator/src/fixes/index.ts b/packages/validator/src/fixes/index.ts index a2e8ad7..382c894 100644 --- a/packages/validator/src/fixes/index.ts +++ b/packages/validator/src/fixes/index.ts @@ -4,7 +4,6 @@ import { fixDuplicateIds } from './duplicate-ids.js'; import { fixBindingSyntax } from './binding-syntax.js'; import { fixSensitiveFlags } from './sensitive-flags.js'; import { fixSchemaDefaults } from './schema-defaults.js'; -import { fixActionReferences } from './action-references.js'; import { fixThinkingBlock } from './thinking-block.js'; /** Maps rule IDs to their fix functions. Rules without fixes are absent. */ @@ -13,7 +12,6 @@ export const FIX_REGISTRY: Partial> = { 'duplicate-ids': fixDuplicateIds, 'binding-syntax': fixBindingSyntax, 'sensitive-flags': fixSensitiveFlags, - 'action-references': fixActionReferences, 'schema-conformance': fixSchemaDefaults, 'thinking-block': fixThinkingBlock, }; @@ -26,8 +24,7 @@ export const FIX_REGISTRY: Partial> = { * 3. duplicate-ids (dedup after format normalization) * 4. binding-syntax * 5. sensitive-flags - * 6. action-references (remove invalid refs before schema check) - * 7. schema-conformance last (re-validates after all fixes, applies Zod defaults) + * 6. schema-conformance last (re-validates after all fixes, applies Zod defaults) */ export const FIX_ORDER: ValidationRuleId[] = [ 'thinking-block', @@ -35,6 +32,5 @@ export const FIX_ORDER: ValidationRuleId[] = [ 'duplicate-ids', 'binding-syntax', 'sensitive-flags', - 'action-references', 'schema-conformance', ]; diff --git a/packages/validator/src/index.ts b/packages/validator/src/index.ts index 3d97ef2..da047b1 100644 --- a/packages/validator/src/index.ts +++ b/packages/validator/src/index.ts @@ -1,5 +1,5 @@ export { validate } from './validate.js'; -export { validateFlow } from './validate-flow.js'; +export { validateConversation } from './validate-conversation.js'; export type { ValidationResult, ValidationIssue, @@ -10,8 +10,8 @@ export type { ExpectedComponent, } from './types.js'; export type { - FlowStepDefinition, - FlowValidationOptions, - FlowValidationResult, - FlowValidationIssue, -} from './validate-flow.js'; + ConversationStep, + ValidateConversationOptions, + ValidateConversationResult, + ValidateConversationIssue, +} from './validate-conversation.js'; diff --git a/packages/validator/src/rules/action-references.ts b/packages/validator/src/rules/action-references.ts deleted file mode 100644 index 056a4b0..0000000 --- a/packages/validator/src/rules/action-references.ts +++ /dev/null @@ -1,50 +0,0 @@ -import type { ValidationRule } from '../types.js'; -import { ACTION_REFERENCE_FIELDS } from '../constants.js'; - -export const actionReferencesRule: ValidationRule = { - id: 'action-references', - name: 'Action References', - description: - 'Checks that action and cross-reference fields (onSubmit, onAction, onComplete, onApprove, onDeny, trigger) reference valid component IDs', - defaultSeverity: 'warning', - - validate(context) { - const knownIds = new Set(context.idMap.keys()); - - for (const block of context.blocks) { - if (block.data === null) continue; - const type = block.data.type; - if (typeof type !== 'string') continue; - const id = typeof block.data.id === 'string' ? block.data.id : null; - - const fields = ACTION_REFERENCE_FIELDS[type]; - if (!fields) continue; - - for (const field of fields) { - const value = block.data[field]; - if (typeof value !== 'string') continue; - - if (!knownIds.has(value)) { - let suggestion = ''; - const normalized = value.toLowerCase().replace(/[-_]/g, ''); - for (const knownId of knownIds) { - if (knownId.toLowerCase().replace(/[-_]/g, '') === normalized) { - suggestion = ` (did you mean "${knownId}"?)`; - break; - } - } - - context.issues.push({ - ruleId: 'action-references', - severity: 'warning', - message: `Cross-reference "${value}" in ${field} does not match any component ID in the document${suggestion}`, - componentId: id, - field, - blockIndex: block.index, - fixed: false, - }); - } - } - } - }, -}; diff --git a/packages/validator/src/rules/index.ts b/packages/validator/src/rules/index.ts index 2eef0a8..7c813d8 100644 --- a/packages/validator/src/rules/index.ts +++ b/packages/validator/src/rules/index.ts @@ -8,15 +8,6 @@ import { bindingSyntaxRule } from './binding-syntax.js'; // Disabled: binding-resolution checks intra-message bindings but components // and their bindings are never generated in the same message. // import { bindingResolutionRule } from './binding-resolution.js'; -// Disabled: action-references checks that onSubmit/onAction/etc. resolve to -// component IDs in the same message — but the spec now treats action labels -// as opaque external handlers the host application wires up at runtime. -// With one interactive component per message, action labels naturally point -// to follow-up handlers in later messages (or external code), not in-document -// targets. The rule was firing on every valid form's onSubmit, producing -// noise warnings on otherwise-passing outputs. Same family as the two -// disabled rules above (intra-message refs that don't apply to multi-turn). -// import { actionReferencesRule } from './action-references.js'; import { sensitiveFlagsRule } from './sensitive-flags.js'; import { requiredMarkersRule } from './required-markers.js'; import { thinkingBlockRule } from './thinking-block.js'; @@ -49,7 +40,6 @@ export const ALL_RULES: readonly ValidationRule[] = [ idFormatRule, bindingSyntaxRule, // bindingResolutionRule, - // actionReferencesRule, sensitiveFlagsRule, requiredMarkersRule, thinkingBlockRule, diff --git a/packages/validator/src/types.ts b/packages/validator/src/types.ts index 6f7f9e0..616c02e 100644 --- a/packages/validator/src/types.ts +++ b/packages/validator/src/types.ts @@ -5,7 +5,6 @@ export type ValidationRuleId = | 'duplicate-ids' | 'binding-syntax' | 'binding-resolution' - | 'action-references' | 'sensitive-flags' | 'required-markers' | 'id-format' diff --git a/packages/validator/src/validate-flow.ts b/packages/validator/src/validate-conversation.ts similarity index 54% rename from packages/validator/src/validate-flow.ts rename to packages/validator/src/validate-conversation.ts index 9b351cd..5f50a8d 100644 --- a/packages/validator/src/validate-flow.ts +++ b/packages/validator/src/validate-conversation.ts @@ -1,9 +1,11 @@ import { extractMdmaBlocksFromMarkdown } from './extract-blocks.js'; +import { validate } from './validate.js'; +import type { ValidationRuleId, ValidationSeverity } from './types.js'; /** - * A single step definition in the expected flow. + * A single step definition in the expected conversation flow. */ -export interface FlowStepDefinition { +export interface ConversationStep { /** Human-readable step label (e.g. "Registration Form") */ label: string; /** The primary component type for this step */ @@ -16,27 +18,42 @@ export interface FlowStepDefinition { | 'callout' | 'table' | 'chart'; - /** Expected component ID for the interactive component */ + /** Expected component ID for the step's primary component */ id: string; } -export interface FlowValidationOptions { - /** Ordered list of expected flow steps. */ - steps: FlowStepDefinition[]; +export interface ValidateConversationOptions { + /** Ordered list of expected conversation steps. */ + steps: ConversationStep[]; + /** + * Rule IDs to exclude from the per-message validation pass. Forwarded to + * `validate()` for each message. Same semantics as `validate()`'s `exclude`. + */ + exclude?: ValidationRuleId[]; } -export interface FlowValidationResult { - /** true if no errors */ - ok: boolean; - /** All issues found across the conversation */ - issues: FlowValidationIssue[]; -} - -export interface FlowValidationIssue { +export interface ValidateConversationIssue { /** 0-based message index in the conversation */ messageIndex: number; - severity: 'error' | 'warning' | 'info'; + severity: ValidationSeverity; message: string; + /** + * Set when the issue was produced by the per-message `validate()` call — + * identifies which validator rule fired. Absent for issues raised by the + * multi-step layer itself (step sequence, cross-message regeneration, etc.). + */ + ruleId?: ValidationRuleId; + /** Set for per-block issues from `validate()`. */ + componentId?: string | null; + /** Set for per-block issues from `validate()`. */ + field?: string; +} + +export interface ValidateConversationResult { + /** true if no errors */ + ok: boolean; + /** All issues found across the conversation */ + issues: ValidateConversationIssue[]; } /** @@ -64,46 +81,77 @@ function extractStepComponents( } /** - * Validate an entire conversation flow against expected step definitions. + * Validate an entire conversation (sequence of assistant messages) end-to-end. + * + * The function runs two passes: * - * Takes all assistant messages in order and checks: - * 1. Each message contains exactly one interactive component - * 2. Steps follow the expected order - * 3. No step is duplicated - * 4. Component IDs match the expected definitions + * 1. Per-message — each assistant message is passed through `validate()` + * so every per-block rule fires (yaml-correctness, schema-conformance, + * duplicate-ids, sensitive-flags, ...). Per-message issues are surfaced + * with their `messageIndex` set. + * + * 2. Multi-step — across messages, this function adds checks that + * `validate()` cannot see by itself: + * - exactly one interactive component per message + * - no regenerated component IDs across turns + * - step sequence matches the expected `options.steps` definition + * - missing steps are surfaced as `info` * * @param assistantMessages - Assistant message contents in conversation order - * @param options - Expected flow definition + * @param options - Expected flow + optional per-message validation exclusions */ -export function validateFlow( +export function validateConversation( assistantMessages: string[], - options: FlowValidationOptions, -): FlowValidationResult { - const { steps } = options; - const issues: FlowValidationIssue[] = []; + options: ValidateConversationOptions, +): ValidateConversationResult { + const { steps, exclude } = options; + const issues: ValidateConversationIssue[] = []; + + // --- Pass 1: per-message validation --- + for (let msgIdx = 0; msgIdx < assistantMessages.length; msgIdx++) { + const result = validate(assistantMessages[msgIdx], { + exclude, + autoFix: false, + }); + for (const issue of result.issues) { + issues.push({ + messageIndex: msgIdx, + severity: issue.severity, + message: issue.message, + ruleId: issue.ruleId, + componentId: issue.componentId, + field: issue.field, + }); + } + } + + // --- Pass 2: multi-step checks --- const seenIds = new Set(); let currentStepIndex = 0; - const expectedIds = new Set(steps.map((s) => s.id)); const expectedTypes = new Set(steps.map((s) => s.type)); for (let msgIdx = 0; msgIdx < assistantMessages.length; msgIdx++) { - const components = extractStepComponents(assistantMessages[msgIdx], expectedIds, expectedTypes); + const components = extractStepComponents( + assistantMessages[msgIdx], + expectedIds, + expectedTypes, + ); - // Skip messages with no interactive components (e.g. pure text responses) - if (components.length === 0) continue; + if (components.length === 0) continue; // pure-text reply is allowed - // Check: exactly one interactive component per message if (components.length > 1) { issues.push({ messageIndex: msgIdx, severity: 'error', - message: `Message ${msgIdx + 1} has ${components.length} interactive components (${components.map((c) => `${c.type}#${c.id}`).join(', ')}) — expected exactly 1`, + message: `Message ${msgIdx + 1} has ${components.length} interactive components (${components + .map((c) => `${c.type}#${c.id}`) + .join(', ')}) — expected exactly 1`, }); } for (const comp of components) { - // Check: no duplicates across messages + // No regenerated components across messages if (seenIds.has(comp.id)) { issues.push({ messageIndex: msgIdx, @@ -114,7 +162,7 @@ export function validateFlow( } seenIds.add(comp.id); - // Check: matches expected step + // Step sequence if (currentStepIndex < steps.length) { const expected = steps[currentStepIndex]; @@ -149,7 +197,6 @@ export function validateFlow( } } - // Check: all steps were shown if (currentStepIndex < steps.length) { for (let i = currentStepIndex; i < steps.length; i++) { issues.push({ diff --git a/packages/validator/tests/rules/action-references.test.ts b/packages/validator/tests/rules/action-references.test.ts deleted file mode 100644 index b25e821..0000000 --- a/packages/validator/tests/rules/action-references.test.ts +++ /dev/null @@ -1,182 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { actionReferencesRule } from '../../src/rules/action-references.js'; -import type { ValidationRuleContext, ParsedBlock } from '../../src/types.js'; - -function createBlock(index: number, data: Record): ParsedBlock { - return { - index, - rawYaml: '', - data, - startOffset: 0, - endOffset: 0, - yamlStartOffset: 0, - yamlEndOffset: 0, - }; -} - -function createContext(blocks: ParsedBlock[]): ValidationRuleContext { - const idMap = new Map(); - for (const block of blocks) { - if (block.data && typeof block.data.id === 'string') { - idMap.set(block.data.id, block.index); - } - } - return { blocks, idMap, issues: [], options: {} }; -} - -describe('action-references rule', () => { - it('passes when webhook trigger references a valid component ID', () => { - const ctx = createContext([ - createBlock(0, { - type: 'button', - id: 'submit-btn', - text: 'Submit', - onAction: 'submit-btn', - }), - createBlock(1, { - type: 'webhook', - id: 'wh', - url: 'https://api.example.com', - trigger: 'submit-btn', - }), - ]); - actionReferencesRule.validate(ctx); - expect(ctx.issues).toHaveLength(0); - }); - - it('flags webhook trigger referencing non-existent component', () => { - const ctx = createContext([ - createBlock(0, { - type: 'webhook', - id: 'wh', - url: 'https://api.example.com', - trigger: 'nonexistent-btn', - }), - ]); - actionReferencesRule.validate(ctx); - expect(ctx.issues).toHaveLength(1); - expect(ctx.issues[0].ruleId).toBe('action-references'); - expect(ctx.issues[0].severity).toBe('warning'); - expect(ctx.issues[0].message).toContain('nonexistent-btn'); - }); - - it('suggests near-matches for misspelled trigger IDs', () => { - const ctx = createContext([ - createBlock(0, { - type: 'button', - id: 'submit-btn', - text: 'Go', - onAction: 'submit-btn', - }), - createBlock(1, { - type: 'webhook', - id: 'wh', - url: 'https://api.example.com', - trigger: 'submit_btn', - }), - ]); - actionReferencesRule.validate(ctx); - expect(ctx.issues).toHaveLength(1); - expect(ctx.issues[0].message).toContain('did you mean'); - expect(ctx.issues[0].message).toContain('submit-btn'); - }); - - it('flags form onSubmit referencing non-existent component', () => { - const ctx = createContext([ - createBlock(0, { - type: 'form', - id: 'f', - fields: [], - onSubmit: 'nonexistent-action', - }), - ]); - actionReferencesRule.validate(ctx); - expect(ctx.issues).toHaveLength(1); - expect(ctx.issues[0].message).toContain('nonexistent-action'); - }); - - it('passes when form onSubmit references valid component', () => { - const ctx = createContext([ - createBlock(0, { - type: 'form', - id: 'f', - fields: [], - onSubmit: 'wh', - }), - createBlock(1, { - type: 'webhook', - id: 'wh', - url: 'https://api.example.com', - trigger: 'f', - }), - ]); - actionReferencesRule.validate(ctx); - expect(ctx.issues).toHaveLength(0); - }); - - it('flags button onAction referencing non-existent component', () => { - const ctx = createContext([ - createBlock(0, { - type: 'button', - id: 'btn', - text: 'Submit', - onAction: 'does-not-exist', - }), - ]); - actionReferencesRule.validate(ctx); - expect(ctx.issues).toHaveLength(1); - expect(ctx.issues[0].message).toContain('does-not-exist'); - }); - - it('flags tasklist onComplete referencing non-existent component', () => { - const ctx = createContext([ - createBlock(0, { - type: 'tasklist', - id: 'tl', - items: [], - onComplete: 'missing-target', - }), - ]); - actionReferencesRule.validate(ctx); - expect(ctx.issues).toHaveLength(1); - expect(ctx.issues[0].message).toContain('missing-target'); - }); - - it('flags approval-gate onApprove and onDeny referencing non-existent components', () => { - const ctx = createContext([ - createBlock(0, { - type: 'approval-gate', - id: 'ag', - title: 'Approve', - onApprove: 'missing-approve', - onDeny: 'missing-deny', - }), - ]); - actionReferencesRule.validate(ctx); - expect(ctx.issues).toHaveLength(2); - expect(ctx.issues[0].message).toContain('missing-approve'); - expect(ctx.issues[1].message).toContain('missing-deny'); - }); - - it('skips blocks with null data', () => { - const blocks: ParsedBlock[] = [ - { - index: 0, - rawYaml: '', - data: null, - startOffset: 0, - endOffset: 0, - yamlStartOffset: 0, - yamlEndOffset: 0, - }, - ]; - const ctx: ValidationRuleContext = { - blocks, - idMap: new Map(), - issues: [], - options: {}, - }; - actionReferencesRule.validate(ctx); - expect(ctx.issues).toHaveLength(0); - }); -}); From 3eed97db2d894d285abb1301b97da42b1a1a0175 Mon Sep 17 00:00:00 2001 From: gitsad Date: Wed, 20 May 2026 13:35:05 +0200 Subject: [PATCH 14/26] feat: added variants for fixer prompts --- evals/assertions/judge-matches-expected.mjs | 127 ++++ evals/package.json | 9 +- evals/prompt-conversation-judge.mjs | 42 ++ evals/promptfooconfig.conversation-flow.yaml | 32 + evals/promptfooconfig.fixer-flow.yaml | 38 -- evals/promptfooconfig.fixer.js | 36 ++ evals/promptfooconfig.fixer.yaml | 36 -- evals/tests-conversation-flow.yaml | 297 +++++++++ evals/tests-fixer-flow.yaml | 589 ------------------ package.json | 3 +- packages/prompt-pack/src/index.ts | 1 + .../src/prompts/mdma-conversation-judge.ts | 64 ++ .../prompts/mdma-fixer/anthropic/_shared.ts | 64 ++ .../src/prompts/mdma-fixer/anthropic/haiku.ts | 58 ++ .../prompts/mdma-fixer/anthropic/opus-4.6.ts | 41 ++ .../prompts/mdma-fixer/anthropic/opus-4.7.ts | 40 ++ .../prompts/mdma-fixer/anthropic/sonnet.ts | 42 ++ .../src/prompts/mdma-fixer/google/_shared.ts | 102 +++ .../google/gemini-2.5-flash-lite.ts | 44 ++ .../mdma-fixer/google/gemini-2.5-flash.ts | 40 ++ .../mdma-fixer/google/gemini-2.5-pro.ts | 44 ++ .../google/gemini-3-flash-preview.ts | 40 ++ .../google/gemini-3.1-flash-lite-preview.ts | 43 ++ .../gemini-3.1-pro-preview-customtools.ts | 43 ++ .../google/gemini-3.1-pro-preview.ts | 60 ++ .../src/prompts/mdma-fixer/openai/_shared.ts | 40 ++ .../prompts/mdma-fixer/openai/gpt-4.1-mini.ts | 55 ++ .../prompts/mdma-fixer/openai/gpt-4.1-nano.ts | 75 +++ .../src/prompts/mdma-fixer/openai/gpt-4.1.ts | 32 + .../prompts/mdma-fixer/openai/gpt-5-mini.ts | 42 ++ .../prompts/mdma-fixer/openai/gpt-5-nano.ts | 42 ++ .../src/prompts/mdma-fixer/openai/gpt-5.1.ts | 32 + .../src/prompts/mdma-fixer/openai/gpt-5.2.ts | 34 + .../prompts/mdma-fixer/openai/gpt-5.4-mini.ts | 34 + .../prompts/mdma-fixer/openai/gpt-5.4-nano.ts | 33 + .../src/prompts/mdma-fixer/openai/gpt-5.4.ts | 35 ++ .../src/prompts/mdma-fixer/openai/gpt-5.5.ts | 41 +- .../src/prompts/mdma-fixer/openai/gpt-5.ts | 37 ++ .../src/prompts/mdma-fixer/x-ai/_shared.ts | 58 ++ .../src/prompts/mdma-fixer/x-ai/grok-4.20.ts | 65 ++ .../src/prompts/mdma-fixer/x-ai/grok-4.3.ts | 42 ++ 41 files changed, 1963 insertions(+), 669 deletions(-) create mode 100644 evals/assertions/judge-matches-expected.mjs create mode 100644 evals/prompt-conversation-judge.mjs create mode 100644 evals/promptfooconfig.conversation-flow.yaml delete mode 100644 evals/promptfooconfig.fixer-flow.yaml create mode 100644 evals/promptfooconfig.fixer.js delete mode 100644 evals/promptfooconfig.fixer.yaml create mode 100644 evals/tests-conversation-flow.yaml delete mode 100644 evals/tests-fixer-flow.yaml create mode 100644 packages/prompt-pack/src/prompts/mdma-conversation-judge.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/anthropic/_shared.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/anthropic/haiku.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/anthropic/opus-4.6.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/anthropic/opus-4.7.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/anthropic/sonnet.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/google/_shared.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-2.5-flash-lite.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-2.5-flash.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-2.5-pro.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3-flash-preview.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3.1-flash-lite-preview.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3.1-pro-preview-customtools.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3.1-pro-preview.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-4.1-mini.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-4.1-nano.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-4.1.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5-mini.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5-nano.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.1.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.2.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.4-mini.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.4-nano.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.4.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/x-ai/_shared.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/x-ai/grok-4.20.ts create mode 100644 packages/prompt-pack/src/prompts/mdma-fixer/x-ai/grok-4.3.ts diff --git a/evals/assertions/judge-matches-expected.mjs b/evals/assertions/judge-matches-expected.mjs new file mode 100644 index 0000000..f49d2c6 --- /dev/null +++ b/evals/assertions/judge-matches-expected.mjs @@ -0,0 +1,127 @@ +import { validateConversation } from '@mobile-reality/mdma-validator'; + +/** + * Custom promptfoo assertion for the conversation-judge eval. + * + * Required: + * - `vars.expectedJudgment` — 'valid' | 'invalid' + * + * Optional per-test config: + * - `expectedRules: string[]` — when expectedJudgment is 'invalid', + * rule names that MUST appear in the LLM judge's issues array. + * + * Optional cross-check (turned on when `vars.steps` is provided): + * - Runs `validateConversation()` on the assistant messages with the + * given step definition. Asserts the deterministic validator agrees + * with both `vars.expectedJudgment` AND the LLM judge. + * + * Passes only when every check it ran agrees. Fails on the first + * disagreement and reports what was off (LLM, validator, or both). + */ +export default function (output, context) { + const vars = context?.vars ?? {}; + const config = context?.config ?? {}; + const expectedJudgment = vars.expectedJudgment; + + if (expectedJudgment !== 'valid' && expectedJudgment !== 'invalid') { + return { + pass: false, + score: 0, + reason: `Test missing or invalid vars.expectedJudgment (got: ${JSON.stringify(expectedJudgment)})`, + }; + } + + // --- Parse the LLM judge's JSON output --- + const fencedMatch = output.match(/```(?:json)?\s*\n?(\{[\s\S]*?\})\s*\n?```/); + const candidate = fencedMatch ? fencedMatch[1] : output.trim(); + + let judgment; + try { + judgment = JSON.parse(candidate); + } catch (err) { + return { + pass: false, + score: 0, + reason: `Judge output is not valid JSON: ${err.message}\nOutput (first 300 chars): ${output.slice(0, 300)}`, + }; + } + if (typeof judgment?.valid !== 'boolean' || !Array.isArray(judgment.issues)) { + return { + pass: false, + score: 0, + reason: `Judge JSON missing required fields (boolean "valid" and array "issues")`, + }; + } + + const expectedValid = expectedJudgment === 'valid'; + const llmValid = judgment.valid; + + // --- Check 1: LLM judge matches expectedJudgment --- + if (llmValid !== expectedValid) { + const issuesSummary = judgment.issues + .slice(0, 5) + .map((i) => ` [msg ${i.messageIndex}, ${i.rule}] ${i.issue}`) + .join('\n'); + return { + pass: false, + score: 0, + reason: `LLM judge expected "${expectedJudgment}" but returned "${llmValid ? 'valid' : 'invalid'}".\nJudge's issues:\n${issuesSummary || ' (none)'}`, + }; + } + + // --- Check 2: required rules surfaced (only for invalid cases) --- + const expectedRules = Array.isArray(config.expectedRules) ? config.expectedRules : null; + if (expectedRules && !expectedValid) { + const seenRules = new Set(judgment.issues.map((i) => i.rule)); + const missing = expectedRules.filter((r) => !seenRules.has(r)); + if (missing.length > 0) { + return { + pass: false, + score: 0.5, + reason: `LLM judge correctly marked invalid but missed expected rule violation(s): ${missing.join(', ')}.\nSeen rules: ${[...seenRules].join(', ') || '(none)'}`, + }; + } + } + + // --- Check 3: cross-check against validateConversation (deterministic) --- + // Activated when the test provides `vars.steps`. Runs the deterministic + // validator on the assistant messages and asserts it agrees with both + // the expected judgment AND the LLM's judgment. + let crossCheckSummary = ''; + if (Array.isArray(vars.steps) && vars.steps.length > 0) { + const assistantMessages = (Array.isArray(vars.conversation) ? vars.conversation : []) + .filter((t) => t.role === 'assistant') + .map((t) => t.content ?? ''); + + const validatorResult = validateConversation(assistantMessages, { + steps: vars.steps, + exclude: ['thinking-block'], + }); + const validatorOk = validatorResult.ok; + + if (validatorOk !== expectedValid) { + const errs = validatorResult.issues + .filter((i) => i.severity === 'error') + .slice(0, 5) + .map((i) => ` [msg ${i.messageIndex}] ${i.message}`) + .join('\n'); + return { + pass: false, + score: 0, + reason: `validateConversation disagrees with expected judgment.\nExpected: "${expectedJudgment}".\nDeterministic validator: "${validatorOk ? 'valid' : 'invalid'}".\nLLM judge: "${llmValid ? 'valid' : 'invalid'}".\nValidator errors:\n${errs || ' (none)'}`, + }; + } + + // Both agree with expected → cross-check passed + const errCount = validatorResult.issues.filter((i) => i.severity === 'error').length; + crossCheckSummary = ` | validator: ${validatorOk ? 'ok' : `${errCount} error(s)`}`; + } + + return { + pass: true, + score: 1, + reason: expectedValid + ? `Judge correctly marked the conversation as valid${crossCheckSummary}` + : `Judge correctly marked the conversation as invalid (${judgment.issues.length} issue${judgment.issues.length === 1 ? '' : 's'})${crossCheckSummary}`, + }; +} diff --git a/evals/package.json b/evals/package.json index 077cfcb..d0f4372 100644 --- a/evals/package.json +++ b/evals/package.json @@ -8,14 +8,15 @@ "eval:conversation": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.conversation.yaml; exit 0", "eval:prompt-builder": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.prompt-builder.yaml; exit 0", "eval:flows": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.flows.yaml; exit 0", - "eval:fixer": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.fixer.yaml; exit 0", - "eval:fixer-flow": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.fixer-flow.yaml; exit 0", - "eval:fixer-all": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.fixer.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.fixer-flow.yaml; exit 0", + "eval:fixer": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.fixer.js; exit 0", + "eval:conversation-flow": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.conversation-flow.yaml; exit 0", + "eval:fixer-all": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.fixer.js; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.conversation-flow.yaml; exit 0", "eval:guidance": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.guidance.yaml; exit 0", "eval:isolated": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval --no-cache -c promptfooconfig.isolated.yaml; exit 0", - "eval:all": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.custom.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.conversation.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.prompt-builder.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.flows.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.fixer.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.fixer-flow.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.guidance.yaml; exit 0", + "eval:all": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.custom.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.conversation.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.prompt-builder.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.flows.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.fixer.js; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.conversation-flow.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.guidance.yaml; exit 0", "eval:author": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.custom.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.conversation.yaml; PROMPTFOO_DISABLE_DATABASE=1 promptfoo eval -c promptfooconfig.flows.yaml; exit 0", "eval:failed": "node scripts/show-failed.mjs", + "eval:cache-clear": "PROMPTFOO_DISABLE_DATABASE=1 promptfoo cache clear", "eval:view": "promptfoo view" }, "dependencies": { diff --git a/evals/prompt-conversation-judge.mjs b/evals/prompt-conversation-judge.mjs new file mode 100644 index 0000000..07ac87c --- /dev/null +++ b/evals/prompt-conversation-judge.mjs @@ -0,0 +1,42 @@ +import { MDMA_CONVERSATION_JUDGE } from '@mobile-reality/mdma-prompt-pack'; + +/** + * Promptfoo prompt loader for the conversation-judge eval. + * + * Each test provides: + * - `customPrompt` — the flow definition (steps, ids, order) + * - `conversation` — array of `{ role: 'user' | 'assistant', content: string }` + * turns in chronological order + * - `expectedJudgment` — 'valid' | 'invalid' (consumed by the assertion, + * not the LLM) + * + * The LLM under test acts as the judge and outputs a JSON + * { valid, issues[] } per the system prompt's contract. + */ +export default function ({ vars }) { + const conversation = Array.isArray(vars.conversation) ? vars.conversation : []; + + const renderedConversation = conversation + .map((turn, i) => { + const role = turn.role === 'assistant' ? 'Assistant' : 'User'; + return `### Message ${i} — ${role}\n\n${turn.content ?? ''}`; + }) + .join('\n\n'); + + const userMessage = `## Flow definition (custom prompt) + +${vars.customPrompt ?? '(no custom prompt provided)'} + +## Conversation (${conversation.length} message${conversation.length === 1 ? '' : 's'}) + +${renderedConversation} + +--- + +Judge whether the conversation above correctly implements the flow defined in the custom prompt. Output only the JSON object specified in your instructions.`; + + return [ + { role: 'system', content: `{% raw %}${MDMA_CONVERSATION_JUDGE}{% endraw %}` }, + { role: 'user', content: `{% raw %}${userMessage}{% endraw %}` }, + ]; +} diff --git a/evals/promptfooconfig.conversation-flow.yaml b/evals/promptfooconfig.conversation-flow.yaml new file mode 100644 index 0000000..416fc2f --- /dev/null +++ b/evals/promptfooconfig.conversation-flow.yaml @@ -0,0 +1,32 @@ +# MDMA Conversation Flow Judge — eval config +# +# Uses an LLM-as-judge prompt (MDMA_FIXER_CONVERSATION_JUDGE) to evaluate +# whether a multi-turn MDMA conversation correctly implements the flow +# defined in the test's customPrompt. The judge outputs a JSON +# `{ valid, issues[] }`; the assertion checks `valid` matches +# `vars.expectedJudgment`. +# +# Run: pnpm --filter @mobile-reality/mdma-evals eval:conversation-flow +# View: pnpm --filter @mobile-reality/mdma-evals eval:view + +description: MDMA Conversation Flow Judge Eval + +envPath: .env +outputPath: results-conversation-flow.json + +prompts: + - file://prompt-conversation-judge.mjs + +providers: + # Override per run with EVAL_PROVIDER (see promptfooconfig.yaml for examples). + - id: "{{ env.EVAL_PROVIDER or 'openai:gpt-4.1' }}" + config: + max_tokens: 4096 + max_completion_tokens: 4096 + +defaultTest: + assert: + - type: javascript + value: file://assertions/judge-matches-expected.mjs + +tests: tests-conversation-flow.yaml diff --git a/evals/promptfooconfig.fixer-flow.yaml b/evals/promptfooconfig.fixer-flow.yaml deleted file mode 100644 index 00279d7..0000000 --- a/evals/promptfooconfig.fixer-flow.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# MDMA Fixer — Flow & References eval config -# -# Run (general): pnpm --filter @mobile-reality/mdma-evals eval:fixer -# Run (flow): pnpm --filter @mobile-reality/mdma-evals eval:fixer-flow -# Run (both): pnpm --filter @mobile-reality/mdma-evals eval:fixer && pnpm --filter @mobile-reality/mdma-evals eval:fixer-flow -# View: pnpm --filter @mobile-reality/mdma-evals eval:view - -description: MDMA Fixer — Flow & References Eval - -envPath: .env -outputPath: results-fixer-flow.json - -prompts: - - file://prompt-fixer.mjs - -providers: - # Override per run with EVAL_PROVIDER (see promptfooconfig.yaml for examples). - # max_tokens / max_completion_tokens lifted above the 1024 default — see - # promptfooconfig.yaml for the rationale. - - id: "{{ env.EVAL_PROVIDER or 'openai:gpt-4.1' }}" - config: - max_tokens: 8192 - max_completion_tokens: 8192 - -defaultTest: - assert: - - type: javascript - value: file://assertions/fixer-resolves-errors.mjs - config: - exclude: ['thinking-block'] - - type: javascript - value: file://assertions/fixer-preserves-components.mjs - config: - min: 1 - - type: javascript - value: file://assertions/fixer-no-prose.mjs - -tests: tests-fixer-flow.yaml diff --git a/evals/promptfooconfig.fixer.js b/evals/promptfooconfig.fixer.js new file mode 100644 index 0000000..a5efba6 --- /dev/null +++ b/evals/promptfooconfig.fixer.js @@ -0,0 +1,36 @@ +const provider = process.env.EVAL_PROVIDER || 'openai:gpt-4.1-mini'; +const leaksReasoningTokens = + (provider.includes('gemini') && provider.includes('pro')) || + provider.includes('grok-4.3'); + +const providerConfig = { + max_tokens: 8192, + max_completion_tokens: 8192, +}; + +if (leaksReasoningTokens) { + providerConfig.passthrough = { + reasoning: { exclude: true }, + include_reasoning: false, + }; +} + +module.exports = { + description: 'MDMA Fixer Prompt Eval', + envPath: '.env', + outputPath: 'results-fixer.json', + prompts: ['file://prompt-fixer.mjs'], + providers: [{ id: provider, config: providerConfig }], + defaultTest: { + assert: [ + { type: 'javascript', value: 'file://assertions/fixer-resolves-errors.mjs' }, + { + type: 'javascript', + value: 'file://assertions/fixer-preserves-components.mjs', + config: { min: 1 }, + }, + { type: 'javascript', value: 'file://assertions/fixer-no-prose.mjs' }, + ], + }, + tests: 'tests-fixer.yaml', +}; diff --git a/evals/promptfooconfig.fixer.yaml b/evals/promptfooconfig.fixer.yaml deleted file mode 100644 index e5d96a3..0000000 --- a/evals/promptfooconfig.fixer.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# MDMA Fixer Prompt — promptfoo evaluation config -# -# Run (general): pnpm --filter @mobile-reality/mdma-evals eval:fixer -# Run (flow): pnpm --filter @mobile-reality/mdma-evals eval:fixer-flow -# Run (both): pnpm --filter @mobile-reality/mdma-evals eval:fixer && pnpm --filter @mobile-reality/mdma-evals eval:fixer-flow -# View: pnpm --filter @mobile-reality/mdma-evals eval:view - -description: MDMA Fixer Prompt Eval - -envPath: .env -outputPath: results-fixer.json - -prompts: - - file://prompt-fixer.mjs - -providers: - # Override per run with EVAL_PROVIDER (see promptfooconfig.yaml for examples). - # max_tokens / max_completion_tokens lifted above the 1024 default — see - # promptfooconfig.yaml for the rationale. - - id: "{{ env.EVAL_PROVIDER or 'openai:gpt-4.1-mini' }}" - config: - max_tokens: 8192 - max_completion_tokens: 8192 - -defaultTest: - assert: - - type: javascript - value: file://assertions/fixer-resolves-errors.mjs - - type: javascript - value: file://assertions/fixer-preserves-components.mjs - config: - min: 1 - - type: javascript - value: file://assertions/fixer-no-prose.mjs - -tests: tests-fixer.yaml diff --git a/evals/tests-conversation-flow.yaml b/evals/tests-conversation-flow.yaml new file mode 100644 index 0000000..2591865 --- /dev/null +++ b/evals/tests-conversation-flow.yaml @@ -0,0 +1,297 @@ +# MDMA Conversation Flow Judge — Eval Test Cases +# +# Each test provides: +# - customPrompt — free-form flow definition (LLM judge's input) +# - conversation — chronological user/assistant turns; assistant +# content may contain ```mdma blocks +# - expectedJudgment — 'valid' or 'invalid' +# - steps — structured flow definition for the deterministic +# validateConversation() cross-check +# - (per-test, optional) expectedRules — when expectedJudgment is +# 'invalid', the rule names the LLM judge MUST flag +# +# Two layers of validation per test: +# 1. The LLM judge (MDMA_FIXER_CONVERSATION_JUDGE) reads customPrompt + +# conversation and outputs a JSON { valid, issues[] }. +# 2. The assertion ALSO runs validateConversation(assistantMessages, +# { steps }) — deterministic code — and verifies it agrees with +# both expectedJudgment AND the LLM judgment. +# Both must agree for the test to pass. + +# --------------------------------------------------------------------------- +# 1. VALID — two-step registration flow, correct order +# --------------------------------------------------------------------------- +- description: VALID — registration → approval in two messages, correct order + vars: + customPrompt: | + User registration workflow. Two steps, one per assistant message. + Step 1: a form with id `registration-form` (type: form) collecting full name + email. + Step 2: an approval gate with id `approval-gate` (type: approval-gate) for manager review. + steps: + - { label: "Registration", type: "form", id: "registration-form" } + - { label: "Manager Approval", type: "approval-gate", id: "approval-gate" } + conversation: + - role: user + content: Create the user registration workflow. + - role: assistant + content: | + ```mdma + type: form + id: registration-form + fields: + - name: full-name + type: text + label: "Full Name" + required: true + - name: email + type: email + label: "Email" + required: true + sensitive: true + onSubmit: registration-submitted + ``` + - role: user + content: Continue to the approval step. + - role: assistant + content: | + ```mdma + type: approval-gate + id: approval-gate + title: "Manager Approval" + requiredApprovers: 1 + ``` + expectedJudgment: valid + +# --------------------------------------------------------------------------- +# 2. INVALID — regeneration (assistant re-emits step 1's form on step 2's turn) +# --------------------------------------------------------------------------- +- description: INVALID — assistant regenerated `registration-form` instead of advancing + vars: + customPrompt: | + Registration workflow. Step 1: `registration-form` (form). Step 2: `approval-gate` (approval-gate). + steps: + - { label: "Registration", type: "form", id: "registration-form" } + - { label: "Manager Approval", type: "approval-gate", id: "approval-gate" } + conversation: + - role: user + content: Create the registration workflow. + - role: assistant + content: | + ```mdma + type: form + id: registration-form + fields: + - name: email + type: email + label: "Email" + required: true + sensitive: true + onSubmit: registration-submitted + ``` + - role: user + content: Continue. + - role: assistant + content: | + ```mdma + type: form + id: registration-form + fields: + - name: email + type: email + label: "Email" + required: true + sensitive: true + onSubmit: registration-submitted + ``` + expectedJudgment: invalid + assert: + - type: javascript + value: file://assertions/judge-matches-expected.mjs + config: + expectedRules: [no-regeneration] + +# --------------------------------------------------------------------------- +# 3. INVALID — multiple interactive components in a single message +# --------------------------------------------------------------------------- +- description: INVALID — assistant crammed form + approval-gate into one message + vars: + customPrompt: | + Registration workflow. Step 1: `registration-form` (form). Step 2: `approval-gate` (approval-gate). One step per message. + steps: + - { label: "Registration", type: "form", id: "registration-form" } + - { label: "Manager Approval", type: "approval-gate", id: "approval-gate" } + conversation: + - role: user + content: Create the registration workflow. + - role: assistant + content: | + ```mdma + type: form + id: registration-form + fields: + - name: email + type: email + label: "Email" + required: true + sensitive: true + onSubmit: registration-submitted + ``` + + ```mdma + type: approval-gate + id: approval-gate + title: "Manager Approval" + requiredApprovers: 1 + ``` + expectedJudgment: invalid + assert: + - type: javascript + value: file://assertions/judge-matches-expected.mjs + config: + expectedRules: [one-interactive-per-message] + +# --------------------------------------------------------------------------- +# 4. INVALID — wrong order (approval before form) +# --------------------------------------------------------------------------- +- description: INVALID — assistant emitted approval-gate before the registration form + vars: + customPrompt: | + Registration workflow. Step 1: `registration-form` (form). Step 2: `approval-gate` (approval-gate). + steps: + - { label: "Registration", type: "form", id: "registration-form" } + - { label: "Manager Approval", type: "approval-gate", id: "approval-gate" } + conversation: + - role: user + content: Begin the registration workflow. + - role: assistant + content: | + ```mdma + type: approval-gate + id: approval-gate + title: "Manager Approval" + requiredApprovers: 1 + ``` + - role: user + content: Continue. + - role: assistant + content: | + ```mdma + type: form + id: registration-form + fields: + - name: email + type: email + label: "Email" + required: true + sensitive: true + onSubmit: registration-submitted + ``` + expectedJudgment: invalid + assert: + - type: javascript + value: file://assertions/judge-matches-expected.mjs + config: + expectedRules: [step-order] + +# --------------------------------------------------------------------------- +# 5. INVALID — skipped a step (3-step flow, middle step missing) +# --------------------------------------------------------------------------- +- description: INVALID — assistant skipped the approval step (form then webhook) + vars: + customPrompt: | + Registration workflow. Step 1: `registration-form` (form). Step 2: `approval-gate` (approval-gate). Step 3: `notify-webhook` (webhook). + steps: + - { label: "Registration", type: "form", id: "registration-form" } + - { label: "Manager Approval", type: "approval-gate", id: "approval-gate" } + - { label: "Notification", type: "webhook", id: "notify-webhook" } + conversation: + - role: user + content: Create the registration workflow. + - role: assistant + content: | + ```mdma + type: form + id: registration-form + fields: + - name: email + type: email + label: "Email" + required: true + sensitive: true + onSubmit: registration-submitted + ``` + - role: user + content: Next step. + - role: assistant + content: | + ```mdma + type: webhook + id: notify-webhook + url: https://api.example.com/notify + method: POST + trigger: registration-complete + ``` + expectedJudgment: invalid + assert: + - type: javascript + value: file://assertions/judge-matches-expected.mjs + config: + expectedRules: [step-order] + +# --------------------------------------------------------------------------- +# 6. VALID — three-step flow with non-interactive callout alongside form +# --------------------------------------------------------------------------- +- description: VALID — three-step expense flow; step 1 has a warning callout + form (non-interactive allowed) + vars: + customPrompt: | + Expense approval workflow. + Step 1: `expense-form` (form) — may be preceded by a warning callout in the same message. + Step 2: `expense-approval` (approval-gate). + Step 3: `submit-expense` (button). + steps: + - { label: "Expense Form", type: "form", id: "expense-form" } + - { label: "Manager Approval", type: "approval-gate", id: "expense-approval" } + - { label: "Submit", type: "button", id: "submit-expense" } + conversation: + - role: user + content: I need to submit an expense. + - role: assistant + content: | + ```mdma + type: callout + id: expense-warning + variant: warning + content: "Expenses over $100 require manager approval." + ``` + + ```mdma + type: form + id: expense-form + fields: + - name: amount + type: number + label: "Amount" + required: true + onSubmit: approve-expense + ``` + - role: user + content: Continue. + - role: assistant + content: | + ```mdma + type: approval-gate + id: expense-approval + title: "Manager Approval" + requiredApprovers: 1 + ``` + - role: user + content: Continue. + - role: assistant + content: | + ```mdma + type: button + id: submit-expense + text: "Submit" + variant: primary + ``` + expectedJudgment: valid diff --git a/evals/tests-fixer-flow.yaml b/evals/tests-fixer-flow.yaml deleted file mode 100644 index 47e0350..0000000 --- a/evals/tests-fixer-flow.yaml +++ /dev/null @@ -1,589 +0,0 @@ -# MDMA Fixer — Flow & References Test Cases -# -# Tests the fixer's ability to fix multi-step flow errors: splitting -# interactive components into separate steps, fixing circular references, -# removing orphans, and complying with the original prompt requirements. -# -# Each test uses the exact broken structure from the Flow & References -# validator prompt, matching the concrete example in the fixer prompt. - -# --------------------------------------------------------------------------- -# 1. Exact broken registration workflow — no history (step 1) -# --------------------------------------------------------------------------- -- description: Fixes exact broken registration workflow to step 1 only - vars: - variantKey: flow - promptContext: | - User registration and approval workflow. - Each step should be in a separate conversation turn. - brokenDocument: | - # User Registration - - ```mdma - type: form - id: registration-form - fields: - - name: full-name - type: text - label: Full Name - required: true - - name: email - type: email - label: Email Address - required: true - sensitive: true - - name: department - type: select - label: Department - options: - - label: Engineering - value: engineering - - label: Marketing - value: marketing - - label: Sales - value: sales - onSubmit: approval-gate - ``` - - ```mdma - type: approval-gate - id: approval-gate - title: Manager Approval - requiredApprovers: 1 - onApprove: registration-form - onDeny: nonexistent-rejection - ``` - - ```mdma - type: button - id: notify-btn - text: Send Notification - onAction: approval-gate - ``` - - ```mdma - type: callout - id: orphan-notice - variant: info - content: This notice is not referenced by anything - ``` - - ```mdma - type: callout - id: orphan-table-info - variant: warning - content: Another orphan - ``` - - ```mdma - type: webhook - id: notify-webhook - url: https://api.example.com/notify - method: POST - trigger: missing-component - ``` - assert: - - type: javascript - value: file://assertions/fixer-resolves-errors.mjs - - type: javascript - value: file://assertions/no-multi-step-flow.mjs - - type: icontains - value: registration-form - - type: not-icontains - value: "type: approval-gate" - - type: not-icontains - value: notify-btn - -# --------------------------------------------------------------------------- -# 2. Same broken structure — step 1 done, show step 2 -# --------------------------------------------------------------------------- -- description: Fixes to step 2 (approval-gate) when form was in prior message - vars: - variantKey: flow - promptContext: | - User registration and approval workflow. - Each step should be in a separate conversation turn. - conversationHistory: - - role: user - content: Create a user registration workflow - - role: assistant - content: | - # User Registration — Step 1 - - ```mdma - type: form - id: registration-form - fields: - - name: full-name - type: text - label: Full Name - required: true - - name: email - type: email - label: Email Address - required: true - sensitive: true - - name: department - type: select - label: Department - options: - - label: Engineering - value: engineering - - label: Marketing - value: marketing - - label: Sales - value: sales - onSubmit: registration-submitted - ``` - - ```mdma - type: callout - id: registration-submitted - variant: success - content: Registration submitted! Awaiting approval. - ``` - - role: user - content: Continue to the next step - brokenDocument: | - # Approval Step - - ```mdma - type: form - id: registration-form - fields: - - name: full-name - type: text - label: Full Name - required: true - - name: email - type: email - label: Email Address - required: true - sensitive: true - onSubmit: approval-gate - ``` - - ```mdma - type: approval-gate - id: approval-gate - title: Manager Approval - requiredApprovers: 1 - onApprove: registration-form - onDeny: denied-callout - ``` - - ```mdma - type: callout - id: denied-callout - variant: error - content: Registration denied. - ``` - assert: - - type: javascript - value: file://assertions/fixer-resolves-errors.mjs - - type: javascript - value: file://assertions/no-multi-step-flow.mjs - - type: not-icontains - value: "type: form" - -# --------------------------------------------------------------------------- -# 3. Expense form → approval-gate chain -# --------------------------------------------------------------------------- -- description: Strips expense workflow to form step only - vars: - variantKey: flow - promptContext: | - Expense approval workflow. - Steps: expense form → manager review → notification. - One step per message. - brokenDocument: | - # Expense Approval - - ```mdma - type: form - id: expense-form - fields: - - name: amount - type: number - label: Amount - required: true - - name: reason - type: textarea - label: Reason - onSubmit: manager-gate - ``` - - ```mdma - type: approval-gate - id: manager-gate - title: Manager Review - requiredApprovers: 1 - onApprove: approved-callout - ``` - - ```mdma - type: callout - id: approved-callout - variant: success - content: Expense approved! - ``` - assert: - - type: javascript - value: file://assertions/fixer-resolves-errors.mjs - - type: javascript - value: file://assertions/no-multi-step-flow.mjs - - type: icontains - value: expense-form - - type: not-icontains - value: "type: approval-gate" - -# --------------------------------------------------------------------------- -# 4. Feedback form with orphans -# --------------------------------------------------------------------------- -- description: Removes orphans and splits feedback workflow - vars: - variantKey: flow - promptContext: | - Feedback collection workflow. - Step 1: feedback form. Step 2: review. - One step per message. - brokenDocument: | - # Feedback Collection - - ```mdma - type: form - id: feedback-form - fields: - - name: rating - type: number - label: Rating - required: true - - name: comment - type: textarea - label: Comment - onSubmit: review-gate - ``` - - ```mdma - type: approval-gate - id: review-gate - title: Review Feedback - requiredApprovers: 1 - onApprove: thank-you - ``` - - ```mdma - type: callout - id: thank-you - variant: success - content: Thank you for your feedback! - ``` - - ```mdma - type: callout - id: orphan-notice - variant: info - content: This is an orphaned notice nobody references. - ``` - assert: - - type: javascript - value: file://assertions/fixer-resolves-errors.mjs - - type: javascript - value: file://assertions/no-multi-step-flow.mjs - - type: icontains - value: feedback-form - - type: not-icontains - value: "type: approval-gate" - -# --------------------------------------------------------------------------- -# 5. Employee onboarding pipeline — no history (step 1) -# --------------------------------------------------------------------------- -- description: Fixes employee onboarding pipeline to step 1 (personal-info form) only - vars: - variantKey: flow - promptContext: | - Employee onboarding workflow. - Step 1: personal-info form (full name, email). - Step 2: department-form (department, start date). - Step 3: onboarding-tasks tasklist. - Step 4: welcome-callout. - Each step must be in a separate conversation turn. - brokenDocument: | - # Employee Onboarding - - ```mdma - type: form - id: personal-info - fields: - - name: full-name - type: text - label: Full Name - required: true - - name: email - type: email - label: Email - required: true - sensitive: true - onSubmit: department-form - ``` - - ```mdma - type: form - id: department-form - fields: - - name: department - type: select - label: Department - options: - - label: Engineering - value: engineering - - label: Marketing - value: marketing - - name: start-date - type: date - label: Start Date - onSubmit: onboarding-tasks - ``` - - ```mdma - type: tasklist - id: onboarding-tasks - items: - - id: task-1 - text: Complete HR paperwork - - id: task-2 - text: Setup workstation - onComplete: welcome-callout - ``` - - ```mdma - type: callout - id: welcome-callout - variant: success - content: Welcome aboard! - ``` - assert: - - type: javascript - value: file://assertions/fixer-resolves-errors.mjs - - type: javascript - value: file://assertions/no-multi-step-flow.mjs - - type: icontains - value: personal-info - - type: not-icontains - value: "id: department-form" - - type: not-icontains - value: "id: onboarding-tasks" - -# --------------------------------------------------------------------------- -# 6. Employee onboarding pipeline — step 1 done, show step 2 -# --------------------------------------------------------------------------- -- description: Fixes to step 2 (department-form) when personal-info was in prior message - vars: - variantKey: flow - promptContext: | - Employee onboarding workflow. - Step 1: personal-info form. - Step 2: department-form (department, start date). - Step 3: onboarding-tasks tasklist. - Step 4: welcome-callout. - Each step must be in a separate conversation turn. - conversationHistory: - - role: user - content: Start the employee onboarding process - - role: assistant - content: | - # Employee Onboarding — Step 1 - - ```mdma - type: form - id: personal-info - fields: - - name: full-name - type: text - label: Full Name - required: true - - name: email - type: email - label: Email - required: true - sensitive: true - onSubmit: step-1-complete - ``` - - ```mdma - type: callout - id: step-1-complete - variant: success - content: Personal info saved! - ``` - - role: user - content: I've submitted the form. What's next? - brokenDocument: | - # Employee Onboarding — Step 2 - - ```mdma - type: form - id: department-form - fields: - - name: department - type: select - label: Department - options: - - label: Engineering - value: engineering - - label: Marketing - value: marketing - - name: start-date - type: date - label: Start Date - onSubmit: onboarding-tasks - ``` - - ```mdma - type: tasklist - id: onboarding-tasks - items: - - id: task-1 - text: Complete HR paperwork - - id: task-2 - text: Setup workstation - onComplete: welcome-callout - ``` - - ```mdma - type: callout - id: welcome-callout - variant: success - content: Welcome aboard! - ``` - assert: - - type: javascript - value: file://assertions/fixer-resolves-errors.mjs - - type: javascript - value: file://assertions/fixer-preserves-components.mjs - config: - min: 1 - - type: javascript - value: file://assertions/no-multi-step-flow.mjs - - type: icontains - value: department-form - - type: not-icontains - value: "id: personal-info" - - type: not-icontains - value: "id: onboarding-tasks" - -# --------------------------------------------------------------------------- -# 7. Expense approval pipeline — no history (step 1) -# --------------------------------------------------------------------------- -- description: Fixes 3-step expense approval pipeline to step 1 (expense-form) only - vars: - variantKey: flow - promptContext: | - Expense approval workflow. - Step 1: expense-form (amount, description). - Step 2: manager-approval gate. - Step 3: finance-approval gate. - Step 4: approved-callout success message. - Each step must be in a separate conversation turn. - brokenDocument: | - # Expense Approval - - ```mdma - type: form - id: expense-form - fields: - - name: amount - type: number - label: Amount - required: true - - name: description - type: textarea - label: Description - onSubmit: manager-approval - ``` - - ```mdma - type: approval-gate - id: manager-approval - title: Manager Approval - requiredApprovers: 1 - allowedRoles: - - manager - onApprove: finance-approval - ``` - - ```mdma - type: approval-gate - id: finance-approval - title: Finance Approval - requiredApprovers: 1 - allowedRoles: - - finance - onApprove: approved-callout - ``` - - ```mdma - type: callout - id: approved-callout - variant: success - content: Expense approved! - ``` - assert: - - type: javascript - value: file://assertions/fixer-resolves-errors.mjs - - type: javascript - value: file://assertions/no-multi-step-flow.mjs - - type: icontains - value: expense-form - - type: not-icontains - value: "type: approval-gate" - - type: not-icontains - value: "id: approved-callout" - -# --------------------------------------------------------------------------- -# 8. Circular action reference — approval-gate loops back to form -# --------------------------------------------------------------------------- -- description: Fixes circular flow where approval-gate onApprove points back to the form - vars: - variantKey: flow - brokenDocument: | - # Feedback Loop - - ```mdma - type: form - id: feedback-form - fields: - - name: rating - type: number - label: Rating - required: true - - name: comment - type: textarea - label: Comment - onSubmit: review-gate - ``` - - ```mdma - type: approval-gate - id: review-gate - title: Review Feedback - requiredApprovers: 1 - onApprove: feedback-form - onDeny: rejection-notice - ``` - - ```mdma - type: callout - id: rejection-notice - variant: error - content: Your feedback was not accepted. Please revise. - ``` - assert: - - type: javascript - value: file://assertions/fixer-resolves-errors.mjs - - type: javascript - value: file://assertions/no-multi-step-flow.mjs - - type: icontains - value: feedback-form - - type: not-icontains - value: "type: approval-gate" diff --git a/package.json b/package.json index 4c9802a..69db6c5 100644 --- a/package.json +++ b/package.json @@ -20,11 +20,12 @@ "eval:prompt-builder": "pnpm --filter @mobile-reality/mdma-evals eval:prompt-builder", "eval:flows": "pnpm --filter @mobile-reality/mdma-evals eval:flows", "eval:fixer": "pnpm --filter @mobile-reality/mdma-evals eval:fixer", - "eval:fixer-flow": "pnpm --filter @mobile-reality/mdma-evals eval:fixer-flow", + "eval:conversation-flow": "pnpm --filter @mobile-reality/mdma-evals eval:conversation-flow", "eval:fixer-all": "pnpm --filter @mobile-reality/mdma-evals eval:fixer-all", "eval:guidance": "pnpm --filter @mobile-reality/mdma-evals eval:guidance", "eval:all": "pnpm --filter @mobile-reality/mdma-evals eval:all", "eval:author": "pnpm --filter @mobile-reality/mdma-evals eval:author", + "eval:cache-clear": "pnpm --filter @mobile-reality/mdma-evals eval:cache-clear", "eval:view": "pnpm --filter @mobile-reality/mdma-evals eval:view" }, "devDependencies": { diff --git a/packages/prompt-pack/src/index.ts b/packages/prompt-pack/src/index.ts index 44c7cea..f2e1e45 100644 --- a/packages/prompt-pack/src/index.ts +++ b/packages/prompt-pack/src/index.ts @@ -22,6 +22,7 @@ export { type FixerIssue, type FixerMessageOptions, } from './prompts/mdma-fixer/_shared.js'; +export { MDMA_CONVERSATION_JUDGE } from './prompts/mdma-conversation-judge.js'; export { buildSystemPrompt, type BuildSystemPromptOptions } from './build-system-prompt.js'; export { AGENT_TOOL_PROMPT_VARIANTS, diff --git a/packages/prompt-pack/src/prompts/mdma-conversation-judge.ts b/packages/prompt-pack/src/prompts/mdma-conversation-judge.ts new file mode 100644 index 0000000..a2afcd6 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-conversation-judge.ts @@ -0,0 +1,64 @@ +/** + * MDMA Conversation Judge prompt. + * + * An LLM-as-judge prompt that decides whether a multi-turn MDMA + * conversation correctly implements the workflow defined in the user's + * custom prompt. Unlike `validateConversation()` (deterministic code), + * this prompt uses an LLM to evaluate semantic correctness of the flow: + * step order, regeneration, single-interactive-per-message, etc. + * + * Inputs (provided in the user message by the caller): + * - Custom prompt — flow definition: expected steps, component IDs, + * order, and any per-message constraints. + * - Conversation — assistant messages in chronological order, each + * possibly containing ```mdma component blocks. + * + * Output: a JSON object with `valid: boolean` and an `issues` array. + * The judge writes no prose around the JSON. + */ +export const MDMA_CONVERSATION_JUDGE = `# MDMA Conversation Flow Judge + +You are an MDMA Conversation Flow Judge. Your role is to validate that a multi-turn conversation correctly implements the workflow defined in the user's custom prompt — and to output a structured JSON judgment. + +## Inputs you will receive + +- **Flow definition** (in the user message): the expected workflow steps, their order, the MDMA component types and IDs for each step, and any per-message constraints. +- **Conversation**: the assistant messages in chronological order. Each assistant message may contain zero or more \`\`\`mdma component blocks. User messages are included for context but are not evaluated. + +## Validation rules + +Apply these rules in order. A single conversation may violate multiple rules; report every violation in the \`issues\` array. + +1. **Step order** — MDMA components appear in the order the flow defines. The N-th interactive component across all assistant messages should be step N from the flow definition. +2. **One interactive per message** — each assistant message contains at most ONE interactive component (\`form\`, \`button\`, \`webhook\`, \`approval-gate\`, \`tasklist\`). Non-interactive components (\`callout\`, \`chart\`, \`table\`, \`thinking\`) may appear alongside it freely. +3. **No regeneration** — once an MDMA component appears in an assistant message (matched by \`id\`), it MUST NOT reappear in any later assistant message. Re-rendering a previously-shown component is a regeneration error. +4. **Step completeness** — each step's components are emitted in their designated turn. Skipping a step, bundling two steps into one message, or omitting a step's required component is a completeness error. +5. **Component id correctness** — when the flow defines specific ids, the assistant messages use those exact ids (verbatim). + +## Output format + +Output a single JSON object — no prose, no Markdown fences, no explanation outside the JSON. Use exactly this shape: + +\`\`\` +{ + "valid": , + "issues": [ + { + "messageIndex": <0-based index of the assistant message in the conversation, counting from the first message which has index 0>, + "severity": "error" | "warning", + "rule": "step-order" | "one-interactive-per-message" | "no-regeneration" | "step-completeness" | "id-correctness", + "issue": "" + } + ] +} +\`\`\` + +If \`valid\` is \`true\`, \`issues\` must be an empty array \`[]\`. + +## Important + +- Output **only** the JSON object. Do not wrap it in Markdown code fences. Do not add a preamble like "Here is my judgment:". +- Use the EXACT rule names listed above in the \`rule\` field. +- Count \`messageIndex\` from 0, including BOTH user and assistant messages — but only emit issues for assistant messages. +- Treat the flow definition as the ground truth. If the conversation deviates from it in any of the five rules, mark the judgment invalid and enumerate every deviation. +`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/anthropic/_shared.ts b/packages/prompt-pack/src/prompts/mdma-fixer/anthropic/_shared.ts new file mode 100644 index 0000000..102fa8d --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/anthropic/_shared.ts @@ -0,0 +1,64 @@ +/** + * Shared content for MDMA-Fixer Anthropic variants. + * + * Each Anthropic variant composes a subset of these blocks via template- + * literal interpolation. Sibling of `mdma-fixer/openai/_shared.ts`. The `_` + * filename prefix is recognized by `evals/select-prompt.mjs` and skipped + * during variant discovery. + * + * Add blocks here when a failure mode is observed across multiple Claude + * variants. Single-variant blocks live inline in their variant file. + */ + +/** + * Anthropic-flavored output framing — wraps the same intent as + * `openai/_shared.ts:CRITICAL_OUTPUT_LINE` in an `` tag, + * which Claude follows more reliably than a CAPS sentence. The fixer + * still emits the corrected Markdown document directly (no outer fence). + */ +export const OUTPUT_FORMAT_BLOCK = ` +Your output IS the corrected Markdown document — write headings, paragraphs, and \`\`\`mdma blocks directly. Do not wrap your response in \`\`\`markdown fences; the response renders as Markdown automatically. +`; + +/** + * Forbids inventing surrounding Markdown structure (headings, descriptive + * paragraphs, horizontal rules) around a bare \`\`\`mdma block. Observed + * on opus-4.7 — wrapped a bare form block with \`# New Project Intake\` + + * "Please provide the details for your new project below." + * + * Same content as \`openai/_shared.ts:PRESERVE_INPUT_STRUCTURE_BLOCK\` — + * duplicated by hand to keep each vendor folder self-contained. + * + * Placed at the very end of a variant's prompt for recency effect. + */ +export const PRESERVE_INPUT_STRUCTURE_BLOCK = ` +!IMPORTANT: Preserve the structure of the input document exactly. If the input is a bare \`\`\`mdma block with no surrounding Markdown, your output is a bare \`\`\`mdma block with no surrounding Markdown. + +Do NOT invent surrounding context. Specifically, never add: +- A Markdown heading (\`# Contact Form\`, \`## Form\`, etc.) above the block +- A descriptive paragraph above or below the block ("Please tell us how…", "Here is the corrected form:") +- A \`---\` horizontal rule +- A blank line prefix or any leading whitespace before the first \`\`\`mdma fence + +The very first character of your response is the backtick that opens \`\`\`mdma. The very last character is the third backtick of the closing fence. Nothing before, nothing after. + +WRONG (do NOT do this): +\`\`\` +# Contact Form + +Please fill out the form below. + +\`\`\`mdma +type: form +... +\`\`\` +\`\`\` + +RIGHT (start your response exactly like this): +\`\`\` +\`\`\`mdma +type: form +... +\`\`\` +\`\`\` +`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/anthropic/haiku.ts b/packages/prompt-pack/src/prompts/mdma-fixer/anthropic/haiku.ts new file mode 100644 index 0000000..3a022a6 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/anthropic/haiku.ts @@ -0,0 +1,58 @@ +/** + * MDMA Fixer Prompt — Anthropic Claude Haiku variant. + * + * Composes MDMA_FIXER_BASE + OUTPUT_FORMAT_BLOCK + all extensions + + * PRESERVE_INPUT_STRUCTURE_BLOCK + TABLE_KEY_DIRECTION_BLOCK (inline). + * + * Haiku consistently fixes "data key does not match any column" by + * renaming the columns to match the data instead of the other way around + * — same failure as gpt-4.1-mini. The shared MDMA_FIXER_TABLES_CHARTS + * extension lists both directions as valid, so a Haiku-specific rule is + * needed to pin the preferred direction. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { OUTPUT_FORMAT_BLOCK, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +const TABLE_KEY_DIRECTION_BLOCK = ` +When a table's data keys do not match its column keys, treat the COLUMN keys as the source of truth and rename the data keys to match them. Do NOT rename the columns to match the data. + +Example — given this broken block: + +\`\`\`mdma +type: table +columns: + - key: product + - key: revenue +data: + - product_name: Widget A + total_revenue: 50000 +\`\`\` + +The correct fix renames \`product_name\` → \`product\` and \`total_revenue\` → \`revenue\` in the data rows, leaving the columns untouched. Renaming the columns to \`product_name\` / \`total_revenue\` is wrong even though it also resolves the error. +`; + +export const MDMA_FIXER_PROMPT_HAIKU = `${MDMA_FIXER_BASE} + +${OUTPUT_FORMAT_BLOCK} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${TABLE_KEY_DIRECTION_BLOCK} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/anthropic/opus-4.6.ts b/packages/prompt-pack/src/prompts/mdma-fixer/anthropic/opus-4.6.ts new file mode 100644 index 0000000..4bf3813 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/anthropic/opus-4.6.ts @@ -0,0 +1,41 @@ +/** + * MDMA Fixer Prompt — Anthropic Claude Opus 4.6 variant. + * + * Starting baseline mirroring `./opus-4.7.ts`: + * MDMA_FIXER_BASE + OUTPUT_FORMAT_BLOCK + all extensions + + * PRESERVE_INPUT_STRUCTURE_BLOCK at the end. + * + * Add inline framing blocks here as 4.6-specific failure modes surface. + * + * Routing note: the longest-substring matcher in `evals/select-prompt.mjs` + * picks `opus-4.6.ts` over a future generic `opus.ts` for any model id + * containing the literal `opus-4.6`. The selector also normalizes + * dot/dash, so `claude-opus-4-6` routes here too. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { OUTPUT_FORMAT_BLOCK, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_OPUS_4_6 = `${MDMA_FIXER_BASE} + +${OUTPUT_FORMAT_BLOCK} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/anthropic/opus-4.7.ts b/packages/prompt-pack/src/prompts/mdma-fixer/anthropic/opus-4.7.ts new file mode 100644 index 0000000..85ef747 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/anthropic/opus-4.7.ts @@ -0,0 +1,40 @@ +/** + * MDMA Fixer Prompt — Anthropic Claude Opus 4.7 variant. + * + * Starting baseline mirroring the openai fixer variants + * (base + OUTPUT_FORMAT_BLOCK + all extensions), but with Anthropic-style + * XML framing instead of the CAPS critical line. Add inline framing + * blocks here as failure modes surface during evals. + * + * Routing note: this file matches model ids containing literal `opus-4.7`. + * The selector's longest-substring match picks it over `opus-4.6.ts` for + * `claude-opus-4.7`. Floating aliases like `claude-opus-latest` do NOT + * route here — pin an explicit version in EVAL_PROVIDER. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { OUTPUT_FORMAT_BLOCK, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_OPUS_4_7 = `${MDMA_FIXER_BASE} + +${OUTPUT_FORMAT_BLOCK} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/anthropic/sonnet.ts b/packages/prompt-pack/src/prompts/mdma-fixer/anthropic/sonnet.ts new file mode 100644 index 0000000..cf60136 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/anthropic/sonnet.ts @@ -0,0 +1,42 @@ +/** + * MDMA Fixer Prompt — Anthropic Claude Sonnet variant (catch-all). + * + * Composes MDMA_FIXER_BASE + OUTPUT_FORMAT_BLOCK + all extensions + + * PRESERVE_INPUT_STRUCTURE_BLOCK at the end. + * + * Add inline framing blocks here as Sonnet-specific failure modes surface + * during evals. + * + * Routing: the longest-substring matcher in `evals/select-prompt.mjs` + * picks `sonnet.ts` for any model id containing literal `sonnet` — + * `claude-sonnet-4-5`, `claude-sonnet-4-6`, etc. If a version-specific + * tweak is needed later, add a sibling `sonnet-X.Y.ts`; the longest-match + * rule will route that id to the more-specific file. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { OUTPUT_FORMAT_BLOCK, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_SONNET = `${MDMA_FIXER_BASE} + +${OUTPUT_FORMAT_BLOCK} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/google/_shared.ts b/packages/prompt-pack/src/prompts/mdma-fixer/google/_shared.ts new file mode 100644 index 0000000..8c8b27a --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/google/_shared.ts @@ -0,0 +1,102 @@ +/** + * Shared content for MDMA-Fixer Google (Gemini) variants. + * + * Format choice: Markdown (`##` headers) rather than XML tags. Google's + * Gemini 3 prompting guide says to pick one structural format and stay + * consistent — "use either XML-style tagging OR Markdown consistently; + * mixing them confuses the model." `MDMA_FIXER_BASE` and the extensions + * use Markdown headings, so Markdown wins for Gemini. + * + * Sibling of `mdma-fixer/openai/_shared.ts` and + * `mdma-fixer/anthropic/_shared.ts`. The `_` filename prefix is recognized + * by `evals/select-prompt.mjs` and skipped during variant discovery. + * + * Note: block CONTENT is duplicated across vendor `_shared.ts` files. + * Redundancy is preferred over cross-vendor imports — each vendor folder + * stays self-contained, so a Google-specific tweak here doesn't + * accidentally affect other vendors' variants. + */ + +/** + * Anchors the model's output format at the top of the prompt. Same intent + * as `openai/_shared.ts:CRITICAL_OUTPUT_LINE` and + * `anthropic/_shared.ts:OUTPUT_FORMAT_BLOCK`, but rendered as a Markdown + * heading rather than a CAPS sentence or XML tag — Gemini follows the + * heading-style instruction more reliably. + */ +export const OUTPUT_FORMAT_BLOCK = `## Output Format + +Your output IS the corrected Markdown document — write headings, paragraphs, and \`\`\`mdma blocks directly. Do not wrap your response in \`\`\`markdown fences; the response renders as Markdown automatically.`; + +/** + * Forbids inventing surrounding Markdown structure (headings, descriptive + * paragraphs, horizontal rules) around a bare \`\`\`mdma block. Mirrors + * the OpenAI and Anthropic siblings in intent; placed at the very end of + * a variant's prompt for recency effect (Vertex guidance: "negative + * constraints should be placed at the end of the instruction"). + * + * Same content as openai/anthropic siblings — duplicated by hand to keep + * each vendor folder self-contained. + */ +export const PRESERVE_INPUT_STRUCTURE_BLOCK = `## Preserve Input Structure + +!IMPORTANT: Preserve the structure of the input document exactly. If the input is a bare \`\`\`mdma block with no surrounding Markdown, your output is a bare \`\`\`mdma block with no surrounding Markdown. + +Do NOT invent surrounding context. Specifically, never add: +- A Markdown heading (\`# Contact Form\`, \`## Form\`, etc.) above the block +- A descriptive paragraph above or below the block ("Please tell us how…", "Here is the corrected form:") +- A \`---\` horizontal rule +- A blank line prefix or any leading whitespace before the first \`\`\`mdma fence + +The very first character of your response is the backtick that opens \`\`\`mdma. The very last character is the third backtick of the closing fence. Nothing before, nothing after. + +WRONG (do NOT do this): +\`\`\` +# Contact Form + +Please fill out the form below. + +\`\`\`mdma +type: form +... +\`\`\` +\`\`\` + +RIGHT (start your response exactly like this): +\`\`\` +\`\`\`mdma +type: form +... +\`\`\` +\`\`\` +`; + +/** + * Pins the direction of fix for "data key does not match any column" + * errors: rename the data keys to match the column keys, NOT the other + * way around. The shared MDMA_FIXER_TABLES_CHARTS extension calls both + * directions valid, but downstream consumers treat the column keys as + * the source of truth. + * + * Observed on gemini-3.1-flash-lite-preview and gemini-2.5-flash-lite; + * same failure pattern also seen on openai/gpt-4.1-mini and + * anthropic/haiku (those keep their own inline copies — promote here if + * future Google variants need it too). + */ +export const TABLE_KEY_DIRECTION_BLOCK = `## Table Key Direction + +When a table's data keys do not match its column keys, treat the COLUMN keys as the source of truth and rename the data keys to match them. Do NOT rename the columns to match the data. + +Example — given this broken block: + +\`\`\`mdma +type: table +columns: + - key: product + - key: revenue +data: + - product_name: Widget A + total_revenue: 50000 +\`\`\` + +The correct fix renames \`product_name\` → \`product\` and \`total_revenue\` → \`revenue\` in the data rows, leaving the columns untouched. Renaming the columns to \`product_name\` / \`total_revenue\` is wrong even though it also resolves the error.`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-2.5-flash-lite.ts b/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-2.5-flash-lite.ts new file mode 100644 index 0000000..2e80922 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-2.5-flash-lite.ts @@ -0,0 +1,44 @@ +/** + * MDMA Fixer Prompt — Google Gemini 2.5 Flash-Lite variant. + * + * Previous-generation smallest-tier Flash-Lite. Starts with the same + * baseline as the Pro variant; add inline framing blocks here as + * failure modes surface during evals. + * + * Routing: substring match on `gemini-2.5-flash-lite` (21 chars) beats + * the 16-char `gemini-2.5-flash` match for any id containing this + * literal. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { + OUTPUT_FORMAT_BLOCK, + PRESERVE_INPUT_STRUCTURE_BLOCK, + TABLE_KEY_DIRECTION_BLOCK, +} from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GEMINI_2_5_FLASH_LITE = `${OUTPUT_FORMAT_BLOCK} + +${MDMA_FIXER_BASE} + +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${TABLE_KEY_DIRECTION_BLOCK} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-2.5-flash.ts b/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-2.5-flash.ts new file mode 100644 index 0000000..04c2abd --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-2.5-flash.ts @@ -0,0 +1,40 @@ +/** + * MDMA Fixer Prompt — Google Gemini 2.5 Flash variant. + * + * Previous-generation mid-tier Flash. Starts with the same baseline as + * the Pro variant; add inline framing blocks here as failure modes + * surface. + * + * Routing: substring match on `gemini-2.5-flash`. Beats the Pro 2.5 + * variant's 14-char `gemini-2.5-pro` match for any id containing this + * literal. The flash-lite variant (`gemini-2.5-flash-lite`, longer) wins + * over this one for `*-flash-lite-*` ids. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { OUTPUT_FORMAT_BLOCK, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GEMINI_2_5_FLASH = `${OUTPUT_FORMAT_BLOCK} + +${MDMA_FIXER_BASE} + +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-2.5-pro.ts b/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-2.5-pro.ts new file mode 100644 index 0000000..ad062b4 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-2.5-pro.ts @@ -0,0 +1,44 @@ +/** + * MDMA Fixer Prompt — Google Gemini 2.5 Pro variant. + * + * Previous-generation Pro (Gemini 3 is current). Starts with the same + * baseline composition as the Gemini 3.1 Pro fixer variant; add inline + * framing blocks here as failure modes surface. + * + * The reasoning-token leak (visible "Thinking:" prose before the + * corrected ```mdma block) that affects gemini-3.1-pro-preview is + * suppressed via the `passthrough.reasoning.exclude: true` body param + * in `evals/promptfooconfig.fixer.js`. The `isGeminiPro` provider check + * in that config catches this id too. + * + * Routing: substring match on `gemini-2.5-pro`. Gemini 3.x variants + * contain `3.1` or `3-flash` in their filenames and do not collide. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { OUTPUT_FORMAT_BLOCK, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GEMINI_2_5_PRO = `${OUTPUT_FORMAT_BLOCK} + +${MDMA_FIXER_BASE} + +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3-flash-preview.ts b/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3-flash-preview.ts new file mode 100644 index 0000000..29b1439 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3-flash-preview.ts @@ -0,0 +1,40 @@ +/** + * MDMA Fixer Prompt — Google Gemini 3 Flash (Preview) variant. + * + * Mid-tier Gemini 3. Starts with the same baseline composition as the + * Pro variant; add inline framing blocks here as failure modes surface + * during evals. + * + * Routing: substring match on `gemini-3-flash-preview` (22 chars). The + * Pro variant filename (`gemini-3.1-pro-preview`) and the Flash-Lite + * filename (`gemini-3.1-flash-lite-preview`) both contain `3.1`, so they + * don't collide with this id (`gemini-3-flash-preview` has no `.1`). + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { OUTPUT_FORMAT_BLOCK, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GEMINI_3_FLASH_PREVIEW = `${OUTPUT_FORMAT_BLOCK} + +${MDMA_FIXER_BASE} + +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3.1-flash-lite-preview.ts b/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3.1-flash-lite-preview.ts new file mode 100644 index 0000000..5561f1c --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3.1-flash-lite-preview.ts @@ -0,0 +1,43 @@ +/** + * MDMA Fixer Prompt — Google Gemini 3.1 Flash-Lite (Preview) variant. + * + * Composes the baseline + TABLE_KEY_DIRECTION_BLOCK — flash-lite renames + * columns instead of data keys when resolving column/data key mismatches. + * + * Routing: substring match on `gemini-3.1-flash-lite-preview`. The Pro + * variant filename doesn't appear as a substring of this model id, so + * the selector picks this file for any model id containing the literal. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { + OUTPUT_FORMAT_BLOCK, + PRESERVE_INPUT_STRUCTURE_BLOCK, + TABLE_KEY_DIRECTION_BLOCK, +} from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GEMINI_3_1_FLASH_LITE_PREVIEW = `${OUTPUT_FORMAT_BLOCK} + +${MDMA_FIXER_BASE} + +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${TABLE_KEY_DIRECTION_BLOCK} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3.1-pro-preview-customtools.ts b/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3.1-pro-preview-customtools.ts new file mode 100644 index 0000000..87d1968 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3.1-pro-preview-customtools.ts @@ -0,0 +1,43 @@ +/** + * MDMA Fixer Prompt — Google Gemini 3.1 Pro Preview Custom Tools variant. + * + * The OpenRouter model `google/gemini-3.1-pro-preview-customtools` is a + * Pro tuning that improves tool/function-call selection. Text generation + * behavior (which is what the fixer exercises — output a corrected + * Markdown document, no tool calls) is unchanged from regular Pro, so + * this file uses the same composition as `gemini-3.1-pro-preview.ts`. + * If a future eval shows the customtools tuning behaves differently on + * pure text generation, edit this file independently to diverge. + * + * Routing: substring match on `gemini-3.1-pro-preview-customtools` + * (34 chars) beats the Pro variant's 24-char match for any model id + * containing this literal. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { OUTPUT_FORMAT_BLOCK, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GEMINI_3_1_PRO_PREVIEW_CUSTOMTOOLS = `${OUTPUT_FORMAT_BLOCK} + +${MDMA_FIXER_BASE} + +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3.1-pro-preview.ts b/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3.1-pro-preview.ts new file mode 100644 index 0000000..157cacc --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/google/gemini-3.1-pro-preview.ts @@ -0,0 +1,60 @@ +/** + * MDMA Fixer Prompt — Google Gemini 3.1 Pro (Preview) variant. + * + * Composition (Gemini-native ordering, mirrors the author variant): + * + * OUTPUT_FORMAT_BLOCK (behavioral anchor — top) + * + MDMA_FIXER_BASE (the spec / fix rules) + * + all MDMA_FIXER_* extensions + * + PRESERVE_INPUT_STRUCTURE_BLOCK (negative constraint — end) + * + * Why this ordering — Google's Gemini 3 prompting guides distinguish two + * placement rules: + * + * 1. Phil Schmid's Google guide: "Place behavioral constraints and role + * definitions in the System Instruction or at the very top of the + * prompt to ensure they anchor the model's reasoning process." + * → output-format directive at top. + * + * 2. Vertex official guide: "negative constraints should be placed at + * the end of the instruction." + * → preserve-input-structure (a "do NOT add headings/prose/separators" + * rule) at the end. + * + * 3. "Use either XML-style tagging OR Markdown consistently — mixing them + * confuses the model." → framing blocks are Markdown headings, not + * XML tags. (OpenAI/Anthropic variants stick with their + * vendor-recommended XML/tag scaffolding.) + * + * Routing: substring match on `gemini-3.1-pro-preview`. Picks this variant + * for any model id containing that literal, including + * `google/gemini-3.1-pro-preview` (dot-form via dot/dash normalization). + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { OUTPUT_FORMAT_BLOCK, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GEMINI_3_1_PRO_PREVIEW = `${OUTPUT_FORMAT_BLOCK} + +${MDMA_FIXER_BASE} + +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/openai/_shared.ts b/packages/prompt-pack/src/prompts/mdma-fixer/openai/_shared.ts index 8be81cb..c85312a 100644 --- a/packages/prompt-pack/src/prompts/mdma-fixer/openai/_shared.ts +++ b/packages/prompt-pack/src/prompts/mdma-fixer/openai/_shared.ts @@ -15,3 +15,43 @@ */ export const CRITICAL_OUTPUT_LINE = 'CRITICAL: Your output IS the corrected Markdown document — write headings, paragraphs, and ```mdma blocks directly. NEVER wrap your response in ```markdown code fences. Your response is already rendered as Markdown.'; + +/** + * Forbids inventing surrounding Markdown structure (headings, descriptive + * paragraphs, horizontal rules) around a bare ```mdma block. Observed on + * gpt-5.4-mini and gpt-5.4-nano fixer evals — both wrapped single-block + * inputs with `# Contact Form` headings and "Please fill out…" preambles. + * + * Placed at the very end of a variant's prompt for recency effect. + */ +export const PRESERVE_INPUT_STRUCTURE_BLOCK = ` +!IMPORTANT: Preserve the structure of the input document exactly. If the input is a bare \`\`\`mdma block with no surrounding Markdown, your output is a bare \`\`\`mdma block with no surrounding Markdown. + +Do NOT invent surrounding context. Specifically, never add: +- A Markdown heading (\`# Contact Form\`, \`## Form\`, etc.) above the block +- A descriptive paragraph above or below the block ("Please tell us how…", "Here is the corrected form:") +- A \`---\` horizontal rule +- A blank line prefix or any leading whitespace before the first \`\`\`mdma fence + +The very first character of your response is the backtick that opens \`\`\`mdma. The very last character is the third backtick of the closing fence. Nothing before, nothing after. + +WRONG (do NOT do this): +\`\`\` +# Contact Form + +Please fill out the form below. + +\`\`\`mdma +type: form +... +\`\`\` +\`\`\` + +RIGHT (start your response exactly like this): +\`\`\` +\`\`\`mdma +type: form +... +\`\`\` +\`\`\` +`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-4.1-mini.ts b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-4.1-mini.ts new file mode 100644 index 0000000..38ebdcc --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-4.1-mini.ts @@ -0,0 +1,55 @@ +/** + * MDMA Fixer Prompt — OpenAI GPT-4.1-mini variant. + * + * Adds TABLE_KEY_DIRECTION_BLOCK on top of the base. The shared + * MDMA_FIXER_TABLES_CHARTS extension offers two equally-valid fixes for + * "Data key does not match any column": rename data keys, or rename + * columns. gpt-4.1-mini deterministically picks the column-rename + * direction, but tests (and downstream consumers) treat the column keys + * as the source of truth — so this variant must prefer renaming data + * keys to match the existing columns. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { CRITICAL_OUTPUT_LINE } from './_shared.js'; + +const TABLE_KEY_DIRECTION_BLOCK = ` +When a table's data keys do not match its column keys, treat the COLUMN keys as the source of truth and rename the data keys to match them. Do NOT rename the columns to match the data. + +Example — given this broken block: + +\`\`\`mdma +type: table +columns: + - key: product + - key: revenue +data: + - product_name: Widget A + total_revenue: 50000 +\`\`\` + +The correct fix renames \`product_name\` → \`product\` and \`total_revenue\` → \`revenue\` in the data rows, leaving the columns untouched. Renaming the columns to \`product_name\` / \`total_revenue\` is wrong even though it also resolves the error. +`; + +export const MDMA_FIXER_PROMPT_GPT_4_1_MINI = `${MDMA_FIXER_BASE} + +${CRITICAL_OUTPUT_LINE} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${TABLE_KEY_DIRECTION_BLOCK} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-4.1-nano.ts b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-4.1-nano.ts new file mode 100644 index 0000000..fa70b37 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-4.1-nano.ts @@ -0,0 +1,75 @@ +/** + * MDMA Fixer Prompt — OpenAI GPT-4.1-nano variant. + * + * Adds PRESERVE_INPUT_STRUCTURE_BLOCK on top of the base — nano prepends + * a leading `---\\n` horizontal rule before the first ```mdma fence + * (same pattern seen across gpt-5.5, gpt-5.2, gpt-5-mini, gpt-5-nano). + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { CRITICAL_OUTPUT_LINE, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +/** + * Reinforces rule 8 of MDMA_FIXER_BASE — gpt-4.1-nano fixes the title + * placeholder (\`TBD\`) but leaves the content placeholder (\`Lorem ipsum + * dolor sit amet\`) untouched when both appear in the same component. The + * model treats one placeholder fix as "the job is done". Placed at the + * very end of the prompt for recency effect — putting it earlier in the + * prompt was not enough on its own. + */ +const REPLACE_ALL_PLACEHOLDERS_BLOCK = ` +!IMPORTANT: A SINGLE COMPONENT can contain MULTIPLE placeholder fields. Replacing ONE is not enough — every placeholder field in every component must be replaced. + +Placeholder markers to detect and replace: +- TODO, TBD, FIXME +- "..." or "…" used as content +- "Lorem ipsum" (case-insensitive, any continuation) +- "placeholder", "sample", "example" used as content +- Empty-but-required strings, single-character labels + +WRONG (only title fixed, \`content\` still placeholder): +\`\`\`mdma +type: callout +id: project-summary +variant: info +title: Project Summary +content: Lorem ipsum dolor sit amet +\`\`\` + +RIGHT (BOTH title AND content replaced with real content): +\`\`\`mdma +type: callout +id: project-summary +variant: info +title: Project Summary +content: This page summarizes the project's goals, current status, and next milestones. +\`\`\` + +Before emitting your final output, re-read every field of every component and confirm no placeholder marker survives. If one does, rewrite it. +`; + +export const MDMA_FIXER_PROMPT_GPT_4_1_NANO = `${MDMA_FIXER_BASE} + +${CRITICAL_OUTPUT_LINE} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK} + +${REPLACE_ALL_PLACEHOLDERS_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-4.1.ts b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-4.1.ts new file mode 100644 index 0000000..04c4f02 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-4.1.ts @@ -0,0 +1,32 @@ +/** + * MDMA Fixer Prompt — OpenAI GPT-4.1 variant. + * + * Starting baseline mirroring the other openai fixer variants + * (base + CRITICAL_OUTPUT_LINE + all extensions). Add inline framing + * blocks here as failure modes surface during evals. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { CRITICAL_OUTPUT_LINE } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GPT_4_1 = `${MDMA_FIXER_BASE} + +${CRITICAL_OUTPUT_LINE} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5-mini.ts b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5-mini.ts new file mode 100644 index 0000000..62d5982 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5-mini.ts @@ -0,0 +1,42 @@ +/** + * MDMA Fixer Prompt — OpenAI GPT-5-mini variant. + * + * Adds PRESERVE_INPUT_STRUCTURE_BLOCK on top of the base — gpt-5-mini + * prepends a leading `---\\n\\n` horizontal rule before the first ```mdma + * fence (same pattern seen on gpt-5.5 and gpt-5.2). + * + * Known flakiness: the leading-`---` failure is stochastic on gpt-5-mini — + * the block suppresses it most of the time but it still leaks in ~1/15 + * tests on a bad run. Reruns commonly pass 15/15. Don't chase the residual + * — strengthening the block further didn't help the flagships either. + * + * Routing note: `gpt-5-mini` doesn't substring-match `gpt-5.4-mini` + * (different separator), so this file only routes the exact id `gpt-5-mini`. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { CRITICAL_OUTPUT_LINE, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GPT_5_MINI = `${MDMA_FIXER_BASE} + +${CRITICAL_OUTPUT_LINE} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5-nano.ts b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5-nano.ts new file mode 100644 index 0000000..05be2bc --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5-nano.ts @@ -0,0 +1,42 @@ +/** + * MDMA Fixer Prompt — OpenAI GPT-5-nano variant. + * + * Adds PRESERVE_INPUT_STRUCTURE_BLOCK on top of the base — nano exhibited + * the full grab-bag of "extra stuff around the block" failures: leading + * `---`, outer ```...``` wrapper fence, hallucinated thinking/callout + * blocks, and trailing horizontal rules. + * + * Known flakiness: residual one-off failures (~1/15) survive even with the + * block — sometimes the model returns empty output, sometimes a stray + * leading `---`. Reruns commonly pass 15/15. Don't chase the residual. + * + * Routing note: `gpt-5-nano` doesn't substring-match `gpt-5.4-nano` + * (different separator), so this file only routes the exact id `gpt-5-nano`. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { CRITICAL_OUTPUT_LINE, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GPT_5_NANO = `${MDMA_FIXER_BASE} + +${CRITICAL_OUTPUT_LINE} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.1.ts b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.1.ts new file mode 100644 index 0000000..b4af5f3 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.1.ts @@ -0,0 +1,32 @@ +/** + * MDMA Fixer Prompt — OpenAI GPT-5.1 variant. + * + * Starting baseline mirroring the other gpt-5.x fixer variants + * (base + CRITICAL_OUTPUT_LINE + all extensions). Add inline framing + * blocks here as failure modes surface during evals. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { CRITICAL_OUTPUT_LINE } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GPT_5_1 = `${MDMA_FIXER_BASE} + +${CRITICAL_OUTPUT_LINE} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.2.ts b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.2.ts new file mode 100644 index 0000000..c982b88 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.2.ts @@ -0,0 +1,34 @@ +/** + * MDMA Fixer Prompt — OpenAI GPT-5.2 variant. + * + * Adds PRESERVE_INPUT_STRUCTURE_BLOCK on top of the base — gpt-5.2 + * prepends a leading `---\\n\\n` horizontal rule before the first ```mdma + * fence (same pattern originally seen on gpt-5.5). + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { CRITICAL_OUTPUT_LINE, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GPT_5_2 = `${MDMA_FIXER_BASE} + +${CRITICAL_OUTPUT_LINE} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.4-mini.ts b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.4-mini.ts new file mode 100644 index 0000000..e31659d --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.4-mini.ts @@ -0,0 +1,34 @@ +/** + * MDMA Fixer Prompt — OpenAI GPT-5.4-mini variant. + * + * Adds PRESERVE_INPUT_STRUCTURE_BLOCK on top of the base — mini wrapped + * single-block inputs with `# Contact Form` headings and descriptive + * preambles. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { CRITICAL_OUTPUT_LINE, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GPT_5_4_MINI = `${MDMA_FIXER_BASE} + +${CRITICAL_OUTPUT_LINE} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.4-nano.ts b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.4-nano.ts new file mode 100644 index 0000000..6fae741 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.4-nano.ts @@ -0,0 +1,33 @@ +/** + * MDMA Fixer Prompt — OpenAI GPT-5.4-nano variant. + * + * Adds PRESERVE_INPUT_STRUCTURE_BLOCK on top of the base — nano wrapped + * single-block inputs with `# Welcome` / `# Project Summary` headings. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { CRITICAL_OUTPUT_LINE, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GPT_5_4_NANO = `${MDMA_FIXER_BASE} + +${CRITICAL_OUTPUT_LINE} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.4.ts b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.4.ts new file mode 100644 index 0000000..19ef147 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.4.ts @@ -0,0 +1,35 @@ +/** + * MDMA Fixer Prompt — OpenAI GPT-5.4 variant. + * + * Starting baseline for GPT-5.4 fixer evals. Mirrors the gpt-5.5 fixer + * baseline (base + CRITICAL_OUTPUT_LINE) — gpt-5.4 shares the same + * no-outer-fence failure mode on fixer output. + * + * Add further framing blocks inline as specific failure modes are observed + * during evals (e.g. duplication, fence-closing). + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { CRITICAL_OUTPUT_LINE } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GPT_5_4 = `${MDMA_FIXER_BASE} + +${CRITICAL_OUTPUT_LINE} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.5.ts b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.5.ts index 0805073..9a26b70 100644 --- a/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.5.ts +++ b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.5.ts @@ -22,6 +22,43 @@ import { } from '../_shared.js'; import { CRITICAL_OUTPUT_LINE } from './_shared.js'; +/** + * Inline block — gpt-5.5 single-block fixer evals showed the model + * prepending a leading `---\\n\\n` (horizontal rule) before the first + * ```mdma fence. The base rules already say "output IS the corrected + * Markdown document" but the model still treats the rewrite as a "response + * to a request" and inserts a separator. Placed at the very end of the + * prompt for recency effect — placing it next to CRITICAL_OUTPUT_LINE was + * not enough on its own. + */ +const NO_LEADING_SEPARATOR_BLOCK = ` +!IMPORTANT: The very first character of your response is the first character of the corrected Markdown document — almost always the backtick that opens \`\`\`mdma. + +Do NOT prepend ANYTHING before it. Specifically: +- NO leading \`---\` horizontal rule +- NO leading blank line +- NO preamble like "Here is the corrected document:" or "Sure, here you go:" +- NO outer code fence + +WRONG (do NOT do this): +\`\`\` +--- + +\`\`\`mdma +type: callout +... +\`\`\` +\`\`\` + +RIGHT (start your response exactly like this): +\`\`\` +\`\`\`mdma +type: callout +... +\`\`\` +\`\`\` +`; + export const MDMA_FIXER_PROMPT_GPT_5_5 = `${MDMA_FIXER_BASE} ${CRITICAL_OUTPUT_LINE} @@ -32,4 +69,6 @@ ${MDMA_FIXER_FORMS} ${MDMA_FIXER_TABLES_CHARTS} ${MDMA_FIXER_FLOW} ${MDMA_FIXER_APPROVAL} -${MDMA_FIXER_EXAMPLES}`; +${MDMA_FIXER_EXAMPLES} + +${NO_LEADING_SEPARATOR_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.ts b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.ts new file mode 100644 index 0000000..97950e6 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/openai/gpt-5.ts @@ -0,0 +1,37 @@ +/** + * MDMA Fixer Prompt — OpenAI GPT-5 variant. + * + * Starting baseline mirroring the other gpt-5.x fixer variants + * (base + CRITICAL_OUTPUT_LINE + all extensions). Add inline framing + * blocks here as failure modes surface during evals. + * + * Routing note: `gpt-5` is a substring of every other gpt-5.x filename, but + * the longest-match rule in `evals/select-prompt.mjs` ensures `gpt-5.5`, + * `gpt-5.4`, etc. still pick their dedicated variants. This file only + * matches the exact model id `gpt-5`. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { CRITICAL_OUTPUT_LINE } from './_shared.js'; + +export const MDMA_FIXER_PROMPT_GPT_5 = `${MDMA_FIXER_BASE} + +${CRITICAL_OUTPUT_LINE} +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/x-ai/_shared.ts b/packages/prompt-pack/src/prompts/mdma-fixer/x-ai/_shared.ts new file mode 100644 index 0000000..aaa3ae8 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/x-ai/_shared.ts @@ -0,0 +1,58 @@ +/** + * Shared content for MDMA-Fixer xAI (Grok) variants. + * + * Format choice: Markdown (`##` headers) rather than XML tags. xAI's + * Grok prompting playbook flags that the model responds unpredictably + * to "pseudo system/persona toggles and long, heavily instrumented + * prompt headers" — published guidance recommends keeping master + * prompts "boring" with a clean hierarchical structure. The cross- + * variant base (`MDMA_FIXER_BASE` and extensions) is already heavily + * Markdown-headed, so Markdown stays consistent. + * + * Sibling of `mdma-fixer/openai/_shared.ts`, + * `mdma-fixer/anthropic/_shared.ts`, and `mdma-fixer/google/_shared.ts`. + * The `_` filename prefix is recognized by `evals/select-prompt.mjs` + * and skipped during variant discovery. + */ + +export const OUTPUT_FORMAT_BLOCK = `## Output Format + +Your output IS the corrected Markdown document — write headings, paragraphs, and \`\`\`mdma blocks directly. Do not wrap your response in \`\`\`markdown fences; the response renders as Markdown automatically.`; + +/** + * Same intent as the openai/anthropic/google siblings — forbid inventing + * surrounding Markdown structure around a bare ```mdma block. Content + * duplicated by hand to keep each vendor folder self-contained. + */ +export const PRESERVE_INPUT_STRUCTURE_BLOCK = `## Preserve Input Structure + +!IMPORTANT: Preserve the structure of the input document exactly. If the input is a bare \`\`\`mdma block with no surrounding Markdown, your output is a bare \`\`\`mdma block with no surrounding Markdown. + +Do NOT invent surrounding context. Specifically, never add: +- A Markdown heading (\`# Contact Form\`, \`## Form\`, etc.) above the block +- A descriptive paragraph above or below the block ("Please tell us how…", "Here is the corrected form:") +- A \`---\` horizontal rule +- A blank line prefix or any leading whitespace before the first \`\`\`mdma fence + +The very first character of your response is the backtick that opens \`\`\`mdma. The very last character is the third backtick of the closing fence. Nothing before, nothing after. + +WRONG (do NOT do this): +\`\`\` +# Contact Form + +Please fill out the form below. + +\`\`\`mdma +type: form +... +\`\`\` +\`\`\` + +RIGHT (start your response exactly like this): +\`\`\` +\`\`\`mdma +type: form +... +\`\`\` +\`\`\` +`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/x-ai/grok-4.20.ts b/packages/prompt-pack/src/prompts/mdma-fixer/x-ai/grok-4.20.ts new file mode 100644 index 0000000..55190f5 --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/x-ai/grok-4.20.ts @@ -0,0 +1,65 @@ +/** + * MDMA Fixer Prompt — xAI Grok 4.20 variant. + * + * Starting baseline. Grok 4.20 is a reasoning model — internal CoT runs + * before the first visible token, so the explicit output contract at + * the top is safe (unlike Grok 4.3 where adding output-format up front + * caused "draft then revise" behavior). + * + * Add inline framing blocks here as failure modes surface during evals. + * If a reasoning-token leak is observed (visible "Thinking:" preamble), + * extend the `isGeminiPro` check in `evals/promptfooconfig.fixer.js` to + * include grok models — same `passthrough.reasoning.exclude` knob works + * for xAI via OpenRouter. + * + * Routing: substring match on `grok-4.20` (9 chars). Beats `grok-4.3` + * (8 chars) for ids containing the `4.20` literal. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; +import { OUTPUT_FORMAT_BLOCK, PRESERVE_INPUT_STRUCTURE_BLOCK } from './_shared.js'; + +/** + * Reinforces rule 1 of MDMA_FIXER_BASE ("Fix every listed issue"). + * Grok 4.20 consistently fixes some-but-not-all reported errors when a + * single component has multiple issues — e.g. on the "kitchen sink" + * employee form it adds \`sensitive: true\` to the email field (one PII + * fix) but leaves the field without the required \`label\` (a separate + * schema-conformance error reported on the same field). Same family of + * failure as gpt-4.1-nano's partial-placeholder fix; the wording here + * generalizes to ANY required field, not just placeholder text. + */ +const FIX_ALL_LISTED_ERRORS_BLOCK = `## Fix Every Listed Error + +!IMPORTANT: The validator may report MULTIPLE errors for the same component (e.g. the same field can have both \`sensitive\` missing AND \`label\` missing). Fix EVERY error, not just the first or most prominent one. + +For each component you emit, walk through every error listed for that component and confirm the fix landed. A common partial-fix mistake on Grok 4.20: addressing a PII flag (\`sensitive: true\`) while forgetting an adjacent missing required field (\`label\`). + +Before emitting your final output, cross-check each error in the input list against the corresponding field in your output. If any error remains unresolved, fix it.`; + +export const MDMA_FIXER_PROMPT_GROK_4_20 = `${OUTPUT_FORMAT_BLOCK} + +${MDMA_FIXER_BASE} + +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES} + +${PRESERVE_INPUT_STRUCTURE_BLOCK} + +${FIX_ALL_LISTED_ERRORS_BLOCK}`; diff --git a/packages/prompt-pack/src/prompts/mdma-fixer/x-ai/grok-4.3.ts b/packages/prompt-pack/src/prompts/mdma-fixer/x-ai/grok-4.3.ts new file mode 100644 index 0000000..76ff0bf --- /dev/null +++ b/packages/prompt-pack/src/prompts/mdma-fixer/x-ai/grok-4.3.ts @@ -0,0 +1,42 @@ +/** + * MDMA Fixer Prompt — xAI Grok 4.3 variant. + * + * Minimal composition by design. The author variant's docblock explains + * the rationale at length: Grok 4.3 regresses when extra framing is + * stacked on top of the base prompt — top-anchored OUTPUT_FORMAT_BLOCK + * caused "draft-then-revise" behavior, and an explicit "no preamble" + * block tripled failures relative to no framing at all. Grok's own + * community guidance: "responds unpredictably to long, heavily + * instrumented prompt headers." + * + * Start with just MDMA_FIXER_BASE + extensions. Only add inline framing + * blocks if a specific failure mode is observed AND empirically benefits + * from the block (regression-check both directions when adding). + * + * Routing: substring match on `grok-4.3` (8 chars). The 4.20 variant + * (`grok-4.20`, 9 chars) wins for ids containing `4.20`; `grok-4.3` + * doesn't substring-match `4.20`, so no collision either way. + */ + +import { + MDMA_FIXER_APPROVAL, + MDMA_FIXER_BASE, + MDMA_FIXER_BINDINGS, + MDMA_FIXER_EXAMPLES, + MDMA_FIXER_FLOW, + MDMA_FIXER_FORMS, + MDMA_FIXER_PII, + MDMA_FIXER_STRUCTURE, + MDMA_FIXER_TABLES_CHARTS, +} from '../_shared.js'; + +export const MDMA_FIXER_PROMPT_GROK_4_3 = `${MDMA_FIXER_BASE} + +${MDMA_FIXER_STRUCTURE} +${MDMA_FIXER_BINDINGS} +${MDMA_FIXER_PII} +${MDMA_FIXER_FORMS} +${MDMA_FIXER_TABLES_CHARTS} +${MDMA_FIXER_FLOW} +${MDMA_FIXER_APPROVAL} +${MDMA_FIXER_EXAMPLES}`; From dc60a02b7d7ee2f62314d22367965faff8ae970d Mon Sep 17 00:00:00 2001 From: gitsad Date: Wed, 20 May 2026 14:22:19 +0200 Subject: [PATCH 15/26] feat: added preview --- README.md | 2 +- demo/src/App.tsx | 10 +- demo/src/HomeView.tsx | 7 ++ demo/src/PreviewView.tsx | 83 ++++++++++++ demo/src/agent/AgentMessage.tsx | 47 ++++--- demo/src/agent/types.ts | 7 ++ demo/src/agent/use-agent.ts | 146 +++++++++++++--------- demo/src/preview/PreviewPanel.tsx | 80 ++++++++++++ demo/src/preview/insurance-backend.ts | 102 +++++++++++++++ demo/src/preview/insurance-flow-prompt.ts | 36 ++++++ demo/src/preview/use-insurance-flow.ts | 139 ++++++++++++++++++++ demo/src/styles.css | 141 +++++++++++++++++++++ 12 files changed, 725 insertions(+), 75 deletions(-) create mode 100644 demo/src/PreviewView.tsx create mode 100644 demo/src/preview/PreviewPanel.tsx create mode 100644 demo/src/preview/insurance-backend.ts create mode 100644 demo/src/preview/insurance-flow-prompt.ts create mode 100644 demo/src/preview/use-insurance-flow.ts diff --git a/README.md b/README.md index c2ba3d3..bd68030 100644 --- a/README.md +++ b/README.md @@ -589,7 +589,7 @@ pnpm eval:view - [x] Multi-model eval coverage (Claude, GPT, Gemini, Grok) - [x] Prompt tuning toolkit — test and compare custom prompts - [x] Agent-friendly SDK — let AI agent generate your MDMA -- [ ] Validator evals +- [x] Validator tests & Fixer evals - [ ] Integrations - [ ] Webhook execution engine (real HTTP calls in production environments) diff --git a/demo/src/App.tsx b/demo/src/App.tsx index aab340f..15c0f2d 100644 --- a/demo/src/App.tsx +++ b/demo/src/App.tsx @@ -5,6 +5,7 @@ import { ChatView } from './ChatView.js'; import { CustomChatView } from './CustomChatView.js'; import { DocsView } from './DocsView.js'; import { HomeView } from './HomeView.js'; +import { PreviewView } from './PreviewView.js'; import { ValidatorView } from './ValidatorView.js'; // ── Routing ────────────────────────────────────────────────────────────────── @@ -25,7 +26,7 @@ function navigate(to: string) { // ── Nav config ─────────────────────────────────────────────────────────────── -type Route = '/' | '/chat' | '/author' | '/custom' | '/validator' | '/docs'; +type Route = '/' | '/chat' | '/preview' | '/author' | '/custom' | '/validator' | '/docs'; interface NavItem { path: Route; @@ -41,7 +42,10 @@ interface NavGroup { const NAV_GROUPS: NavGroup[] = [ { label: 'Agentic', - items: [{ path: '/chat', label: 'Agent Chat', icon: '⚡' }], + items: [ + { path: '/chat', label: 'Agent Chat', icon: '⚡' }, + { path: '/preview', label: 'Insurance Preview', icon: '🛡️' }, + ], }, { label: 'Completions', @@ -184,6 +188,8 @@ export function App() { ) : route === '/author' ? ( + ) : route === '/preview' ? ( + ) : ( )} diff --git a/demo/src/HomeView.tsx b/demo/src/HomeView.tsx index 3e1e6a1..2f4704f 100644 --- a/demo/src/HomeView.tsx +++ b/demo/src/HomeView.tsx @@ -18,6 +18,13 @@ const SECTIONS = [ description: 'Autonomous agent that thinks, plans, and generates interactive MDMA documents via tool calls.', }, + { + path: '/preview', + label: 'Insurance Preview', + icon: '🛡️', + description: + 'Multi-step insurance claim flow demo — chat on the left, live MDMA preview with auto-validation and fixer on the right.', + }, ], }, { diff --git a/demo/src/PreviewView.tsx b/demo/src/PreviewView.tsx new file mode 100644 index 0000000..a14fc3f --- /dev/null +++ b/demo/src/PreviewView.tsx @@ -0,0 +1,83 @@ +import { useRef, useEffect, useCallback } from 'react'; +import { useAgent } from './agent/use-agent.js'; +import { AgentMessage } from './agent/AgentMessage.js'; +import { AgentSettings } from './agent/AgentSettings.js'; +import { ChatInput } from './chat/ChatInput.js'; +import { PreviewPanel } from './preview/PreviewPanel.js'; +import { INSURANCE_FLOW_PROMPT } from './preview/insurance-flow-prompt.js'; +import { useInsuranceFlow } from './preview/use-insurance-flow.js'; + +export function PreviewView() { + const { + turns, + isGenerating, + error, + input, + setInput, + config, + updateConfig, + send, + sendHidden, + stop, + clear, + inputRef, + } = useAgent({ flowPrompt: INSURANCE_FLOW_PROMPT }); + + useInsuranceFlow({ turns, sendHidden, isGenerating }); + + const chatEndRef = useRef(null); + const prevCountRef = useRef(turns.length); + + useEffect(() => { + if (turns.length > prevCountRef.current) { + chatEndRef.current?.scrollIntoView({ behavior: 'smooth' }); + } + prevCountRef.current = turns.length; + }, [turns]); + + const handleClear = useCallback(() => { + clear(); + }, [clear]); + + return ( +

+
+ + +
+ {turns.length === 0 && ( +
+

Insurance Claim Demo

+

+ Ask the agent to start a new insurance claim. It will walk you through name & + birthday, claim details, bank account, and a final confirmation — each step + rendered live in the preview pane on the right. +

+
+ )} + + {turns.map((turn) => ( + + ))} + + {error &&
{error}
} + +
+
+ + 0} + inputRef={inputRef} + /> +
+ + +
+ ); +} diff --git a/demo/src/agent/AgentMessage.tsx b/demo/src/agent/AgentMessage.tsx index 869977c..e44c703 100644 --- a/demo/src/agent/AgentMessage.tsx +++ b/demo/src/agent/AgentMessage.tsx @@ -164,9 +164,9 @@ function TextBlockView({ block }: { block: TextBlock }) { return ; } -function ToolUseBlockView({ block }: { block: ToolUseBlock }) { +function ToolUseBlockView({ block, compact }: { block: ToolUseBlock; compact?: boolean }) { return ( -
+
{block.name} {block.isStreaming && generating…} + {compact && !block.isStreaming && ( + rendered in preview → + )}
-
- {block.isStreaming ? ( -
- -
- ) : block.ast && block.store ? ( - - ) : block.document ? ( - 
{block.document}
- ) : null} -
+ {!compact && ( +
+ {block.isStreaming ? ( +
+ +
+ ) : block.ast && block.store ? ( + + ) : block.document ? ( + 
{block.document}
+ ) : null} +
+ )}
); } // ── Turn renderer ───────────────────────────────────────────────────────────── -export const AgentMessage = memo(function AgentMessage({ turn }: { turn: AgentDisplayTurn }) { +interface AgentMessageProps { + turn: AgentDisplayTurn; + /** + * When true, tool_use blocks render as a compact chip (no inline MDMA + * preview). Used by the Preview page, where the rendered MDMA lives in + * the right-side pane and would be duplicated in the chat otherwise. + */ + compactToolUse?: boolean; +} + +export const AgentMessage = memo(function AgentMessage({ turn, compactToolUse }: AgentMessageProps) { if (turn.role === 'user') { + if (turn.hidden) return null; return (
@@ -230,7 +246,8 @@ export const AgentMessage = memo(function AgentMessage({ turn }: { turn: AgentDi if (block.type === 'thinking') return ; if (block.type === 'text') return ; - if (block.type === 'tool_use') return ; + if (block.type === 'tool_use') + return ; }) )}
diff --git a/demo/src/agent/types.ts b/demo/src/agent/types.ts index 241acd0..07d8987 100644 --- a/demo/src/agent/types.ts +++ b/demo/src/agent/types.ts @@ -34,6 +34,13 @@ export interface UserTurn { id: string; role: 'user'; content: string; + /** + * When true, the turn is not rendered in the chat UI but is still part of + * the API history sent to the agent. Used by the Insurance Preview to + * carry "step N submitted, please continue" signals without exposing + * synthetic prompts (or form data) to the user. + */ + hidden?: boolean; } export interface AssistantTurn { diff --git a/demo/src/agent/use-agent.ts b/demo/src/agent/use-agent.ts index c20477c..b610783 100644 --- a/demo/src/agent/use-agent.ts +++ b/demo/src/agent/use-agent.ts @@ -514,7 +514,16 @@ function patchBlock( // ── Hook ───────────────────────────────────────────────────────────────────── -export function useAgent() { +export interface UseAgentOptions { + /** + * Extra flow-definition text appended to the agent's customPrompt. Used by + * the Insurance Preview to lock the conversation to a specific 4-step + * flow. When omitted, the agent behaves like the regular Agent Chat. + */ + flowPrompt?: string; +} + +export function useAgent(options: UseAgentOptions = {}) { const storedRef = useRef(loadAgentHistory()); const stored = storedRef.current; @@ -570,66 +579,88 @@ export function useAgent() { }); }, []); + const runTurn = useCallback( + async (text: string, hidden: boolean) => { + if (!text || isGenerating) return; + setError(null); + setIsGenerating(true); + + const assistantTurnId = nextId(); + setTurns((prev) => [ + ...prev, + { id: nextId(), role: 'user', content: text, hidden }, + { id: assistantTurnId, role: 'assistant', blocks: [] }, + ]); + + abortRef.current = new AbortController(); + const toolPrompt = getAgentToolPromptVariant(config.systemPromptId).prompt; + const customPrompt = options.flowPrompt + ? `${toolPrompt}\n\n---\n\n${options.flowPrompt}` + : toolPrompt; + const systemPrompt = buildSystemPrompt({ + authorPrompt: getAuthorPromptVariant(config.systemPromptId).prompt, + customPrompt, + }); + + const provider = config.provider ?? 'anthropic'; + + try { + if (provider === 'anthropic') { + const history: ApiMessage[] = [ + ...apiHistoryRef.current, + { role: 'user', content: text }, + ]; + await runAgentLoop( + config, + systemPrompt, + history, + assistantTurnId, + abortRef.current.signal, + setTurns, + setError, + nextId, + ); + apiHistoryRef.current = history; + } else { + const history = [...openaiHistoryRef.current, { role: 'user' as const, content: text }]; + await runOpenAIAgentLoop( + config, + systemPrompt, + history, + assistantTurnId, + abortRef.current.signal, + setTurns, + setError, + nextId, + ); + openaiHistoryRef.current = history; + } + } catch (err) { + if (!(err instanceof DOMException && err.name === 'AbortError')) { + setError(err instanceof Error ? err.message : String(err)); + } + } finally { + setIsGenerating(false); + abortRef.current = null; + inputRef.current?.focus(); + } + }, + [config, isGenerating, nextId, options.flowPrompt], + ); + const send = useCallback(async () => { const text = input.trim(); - if (!text || isGenerating) return; - setError(null); - setIsGenerating(true); + if (!text) return; setInput(''); + await runTurn(text, false); + }, [input, runTurn]); - const assistantTurnId = nextId(); - setTurns((prev) => [ - ...prev, - { id: nextId(), role: 'user', content: text }, - { id: assistantTurnId, role: 'assistant', blocks: [] }, - ]); - - abortRef.current = new AbortController(); - const systemPrompt = buildSystemPrompt({ - authorPrompt: getAuthorPromptVariant(config.systemPromptId).prompt, - customPrompt: getAgentToolPromptVariant(config.systemPromptId).prompt, - }); - - const provider = config.provider ?? 'anthropic'; - - try { - if (provider === 'anthropic') { - const history: ApiMessage[] = [...apiHistoryRef.current, { role: 'user', content: text }]; - await runAgentLoop( - config, - systemPrompt, - history, - assistantTurnId, - abortRef.current.signal, - setTurns, - setError, - nextId, - ); - apiHistoryRef.current = history; - } else { - const history = [...openaiHistoryRef.current, { role: 'user' as const, content: text }]; - await runOpenAIAgentLoop( - config, - systemPrompt, - history, - assistantTurnId, - abortRef.current.signal, - setTurns, - setError, - nextId, - ); - openaiHistoryRef.current = history; - } - } catch (err) { - if (!(err instanceof DOMException && err.name === 'AbortError')) { - setError(err instanceof Error ? err.message : String(err)); - } - } finally { - setIsGenerating(false); - abortRef.current = null; - inputRef.current?.focus(); - } - }, [config, input, isGenerating, nextId]); + const sendHidden = useCallback( + async (text: string) => { + await runTurn(text, true); + }, + [runTurn], + ); const stop = useCallback(() => { abortRef.current?.abort(); @@ -654,6 +685,7 @@ export function useAgent() { config, updateConfig, send, + sendHidden, stop, clear, inputRef, diff --git a/demo/src/preview/PreviewPanel.tsx b/demo/src/preview/PreviewPanel.tsx new file mode 100644 index 0000000..20225c5 --- /dev/null +++ b/demo/src/preview/PreviewPanel.tsx @@ -0,0 +1,80 @@ +import { useMemo } from 'react'; +import { MdmaDocument } from '@mobile-reality/mdma-renderer-react'; +import { customizations } from '../custom-components.js'; +import type { AgentDisplayTurn, AssistantTurn, ToolUseBlock } from '../agent/types.js'; + +interface PreviewPanelProps { + turns: AgentDisplayTurn[]; +} + +interface LatestMdma { + block: ToolUseBlock; + turnId: string; +} + +function findLatestMdmaBlock(turns: AgentDisplayTurn[]): LatestMdma | null { + for (let i = turns.length - 1; i >= 0; i--) { + const turn = turns[i]; + if (turn.role !== 'assistant') continue; + const blocks = (turn as AssistantTurn).blocks; + for (let j = blocks.length - 1; j >= 0; j--) { + const block = blocks[j]; + if (block.type === 'tool_use') return { block, turnId: turn.id }; + } + } + return null; +} + +export function PreviewPanel({ turns }: PreviewPanelProps) { + const latest = useMemo(() => findLatestMdmaBlock(turns), [turns]); + + const status: 'idle' | 'streaming' | 'ready' = !latest + ? 'idle' + : latest.block.isStreaming + ? 'streaming' + : latest.block.ast && latest.block.store + ? 'ready' + : 'streaming'; + + const statusLabel = + status === 'idle' ? 'idle' : status === 'streaming' ? 'generating' : 'ready'; + const statusClass = + status === 'idle' + ? 'preview-pane-status--idle' + : status === 'streaming' + ? 'preview-pane-status--validating' + : 'preview-pane-status--ready'; + + return ( +
+
+ Live MDMA Preview + {statusLabel} +
+
+ {!latest ? ( +
+

Insurance claim flow

+

+ Start the chat on the left. As the agent emits MDMA blocks, they'll be rendered here. +

+
+ ) : latest.block.isStreaming || !latest.block.ast || !latest.block.store ? ( +
+

Generating…

+

+ The agent is still emitting this step. The rendered output will appear when the block + is complete. +

+
+ ) : ( + + )} +
+
+ ); +} diff --git a/demo/src/preview/insurance-backend.ts b/demo/src/preview/insurance-backend.ts new file mode 100644 index 0000000..2aeb554 --- /dev/null +++ b/demo/src/preview/insurance-backend.ts @@ -0,0 +1,102 @@ +/** + * Mock backend for the Insurance Preview demo. Each function pretends to be + * an endpoint of the insurance provider's API: validates a tiny shape, + * waits a few hundred ms, and resolves with a fake server response. No + * data leaves the browser — values land in the in-memory `submissionLog`, + * which the optional debug pane on the right column displays. + */ + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +function maskIban(iban: string): string { + const trimmed = iban.replace(/\s+/g, ''); + if (trimmed.length <= 8) return '••••'; + return `${trimmed.slice(0, 4)} •••• ${trimmed.slice(-4)}`; +} + +export interface SubmissionLogEntry { + step: 'personal-info' | 'claim' | 'bank'; + at: Date; + claimId: string; + /** Display-only summary (sensitive values masked). Never raw user data. */ + summary: string; +} + +const submissionLog: SubmissionLogEntry[] = []; + +export interface PersonalInfoPayload { + 'full-name': string; + birthday: string; +} + +export interface ClaimPayload { + 'claim-description': string; +} + +export interface BankPayload { + iban: string; +} + +export interface PersonalInfoResult { + claimId: string; + accepted: true; +} + +export interface ClaimResult { + accepted: true; +} + +export interface BankResult { + accepted: true; + etaDays: number; +} + +function makeClaimId(): string { + return `clm_${Math.random().toString(36).slice(2, 8)}`; +} + +export const insuranceBackend = { + async collectPersonalInfo(payload: PersonalInfoPayload): Promise { + await delay(700); + const claimId = makeClaimId(); + submissionLog.push({ + step: 'personal-info', + at: new Date(), + claimId, + summary: `${payload['full-name']} (DOB ${payload.birthday})`, + }); + return { claimId, accepted: true }; + }, + + async collectClaim(claimId: string, payload: ClaimPayload): Promise { + await delay(800); + const desc = payload['claim-description']; + const preview = desc.length > 60 ? `${desc.slice(0, 60)}…` : desc; + submissionLog.push({ + step: 'claim', + at: new Date(), + claimId, + summary: `"${preview}"`, + }); + return { accepted: true }; + }, + + async collectBank(claimId: string, payload: BankPayload): Promise { + await delay(700); + submissionLog.push({ + step: 'bank', + at: new Date(), + claimId, + summary: `IBAN ${maskIban(payload.iban)}`, + }); + return { accepted: true, etaDays: 5 }; + }, +}; + +export function getSubmissionLog(): readonly SubmissionLogEntry[] { + return submissionLog; +} + +export function clearSubmissionLog(): void { + submissionLog.length = 0; +} diff --git a/demo/src/preview/insurance-flow-prompt.ts b/demo/src/preview/insurance-flow-prompt.ts new file mode 100644 index 0000000..d99a90b --- /dev/null +++ b/demo/src/preview/insurance-flow-prompt.ts @@ -0,0 +1,36 @@ +/** + * Insurance claim flow — locked custom prompt for the Preview page. + * + * Defines a 4-message conversation: gather personal info, then claim + * description, then bank account for receiving funds, then a final + * confirmation callout. Each interactive step is a single MDMA component + * per assistant turn (one form / one callout) — matches the rules the + * conversation-flow eval enforces. + */ +export const INSURANCE_FLOW_PROMPT = `## Insurance Claim Intake Flow + +You are a friendly claims assistant for **MDMA Mutual Insurance**. Walk the user through filing a new claim across exactly four assistant turns, one interactive MDMA component per turn. Use a warm, plain-language tone. + +### Step 1 — Personal info +First assistant turn. Emit a single \`form\` component with id \`personal-info-form\` and \`onSubmit: collect-personal-info\`. Fields: +- \`full-name\` (text, required, label "Full name") +- \`birthday\` (date, required, label "Date of birth") + +### Step 2 — Claim description +Second assistant turn (after the user submits personal info). Emit a single \`form\` component with id \`claim-description-form\` and \`onSubmit: collect-claim\`. Fields: +- \`claim-description\` (textarea, required, label "What happened?") + +### Step 3 — Bank account +Third assistant turn (after the user submits the claim description). Emit a single \`form\` component with id \`bank-account-form\` and \`onSubmit: collect-bank\`. Fields: +- \`iban\` (text, required, sensitive: true, label "IBAN where we should send the funds") + +### Step 4 — Confirmation +Fourth assistant turn (after the user submits the bank account). Emit a single \`callout\` component with id \`claim-submitted-callout\`, \`variant: success\`, \`title: "Claim received"\`, and a friendly \`content\` explaining the claim will be processed by an insurance specialist within a few business days. No further interactive components — the flow ends here. + +### Rules +- One interactive component (\`form\`) per assistant turn for steps 1–3. Step 4 is a non-interactive \`callout\`. +- Use the **exact** ids and \`onSubmit\` action labels listed above. +- Don't regenerate previously-shown components in later turns. +- Don't add components beyond what each step requires (no extra callouts, buttons, or webhooks). +- It's fine to precede a step's form with a short plain-text intro sentence, but do not emit any other MDMA component types. +`; diff --git a/demo/src/preview/use-insurance-flow.ts b/demo/src/preview/use-insurance-flow.ts new file mode 100644 index 0000000..d58bb64 --- /dev/null +++ b/demo/src/preview/use-insurance-flow.ts @@ -0,0 +1,139 @@ +import { useEffect, useRef } from 'react'; +import type { DocumentStore } from '@mobile-reality/mdma-runtime'; +import type { AgentDisplayTurn, AssistantTurn } from '../agent/types.js'; +import { + insuranceBackend, + type BankPayload, + type ClaimPayload, + type PersonalInfoPayload, +} from './insurance-backend.js'; + +interface UseInsuranceFlowOptions { + turns: AgentDisplayTurn[]; + sendHidden: (message: string) => Promise; + isGenerating: boolean; +} + +const ACTION_IDS = ['collect-personal-info', 'collect-claim', 'collect-bank'] as const; +type ActionId = (typeof ACTION_IDS)[number]; + +function isHandledActionId(id: string): id is ActionId { + return (ACTION_IDS as readonly string[]).includes(id); +} + +/** + * Drives the Insurance Preview flow: + * + * 1. Listens for `ACTION_TRIGGERED` events on the MDMA renderer stores of + * each new assistant turn. + * 2. When an event with one of our known `actionId`s fires, pulls the + * submitted values straight from the store (does NOT include them in + * any message to the agent), calls the mock backend, and waits for the + * success response. + * 3. On success, sends a HIDDEN user message to the agent — never shown + * in the chat — carrying only a "step N complete, please continue" + * signal. The agent uses that to emit the next step naturally. + * + * The claim id returned by step 1 is threaded into steps 2 + 3 via a ref + * so consecutive backend calls reference the same claim. + */ +export function useInsuranceFlow({ turns, sendHidden, isGenerating }: UseInsuranceFlowOptions) { + const subscribedStores = useRef(new Set()); + const handledActions = useRef(new Set()); + const claimIdRef = useRef(null); + const isGeneratingRef = useRef(isGenerating); + isGeneratingRef.current = isGenerating; + const sendHiddenRef = useRef(sendHidden); + sendHiddenRef.current = sendHidden; + + useEffect(() => { + for (const turn of turns) { + if (turn.role !== 'assistant') continue; + const blocks = (turn as AssistantTurn).blocks; + for (const block of blocks) { + if (block.type !== 'tool_use') continue; + const store = block.store; + if (!store || subscribedStores.current.has(store)) continue; + subscribedStores.current.add(store); + + store.getEventBus().on('ACTION_TRIGGERED', (action) => { + if (isGeneratingRef.current) return; + const { actionId, componentId } = action; + if (!isHandledActionId(actionId)) return; + + // De-dupe: one ACTION_TRIGGERED per (componentId, actionId) + const key = `${componentId}:${actionId}`; + if (handledActions.current.has(key)) return; + handledActions.current.add(key); + + const values = (store.getComponentState(componentId)?.values ?? {}) as Record< + string, + unknown + >; + void dispatch(actionId, values).catch((err) => { + handledActions.current.delete(key); + // Surfacing errors to the user is out of scope for now; log and + // let them retry the submission. + console.error('[insurance-flow] backend call failed', err); + }); + }); + } + } + }, [turns]); + + async function dispatch(actionId: ActionId, values: Record) { + if (actionId === 'collect-personal-info') { + const payload: PersonalInfoPayload = { + 'full-name': String(values['full-name'] ?? ''), + birthday: String(values.birthday ?? ''), + }; + const result = await insuranceBackend.collectPersonalInfo(payload); + claimIdRef.current = result.claimId; + await sendHiddenRef.current( + `[system] The user submitted the personal-info form and the backend accepted it (claim id: ${result.claimId}). Proceed to step 2 by emitting the claim description form.`, + ); + return; + } + + if (actionId === 'collect-claim') { + const claimId = claimIdRef.current; + if (!claimId) { + console.warn('[insurance-flow] collect-claim fired before claim id was available'); + return; + } + const payload: ClaimPayload = { + 'claim-description': String(values['claim-description'] ?? ''), + }; + await insuranceBackend.collectClaim(claimId, payload); + await sendHiddenRef.current( + `[system] The user submitted the claim description and the backend accepted it (claim id: ${claimId}). Proceed to step 3 by emitting the bank-account form.`, + ); + return; + } + + if (actionId === 'collect-bank') { + const claimId = claimIdRef.current; + if (!claimId) { + console.warn('[insurance-flow] collect-bank fired before claim id was available'); + return; + } + const payload: BankPayload = { iban: String(values.iban ?? '') }; + const result = await insuranceBackend.collectBank(claimId, payload); + await sendHiddenRef.current( + `[system] The user submitted the bank-account form and the backend accepted it (claim id: ${claimId}, funds ETA: ${result.etaDays} business days). Proceed to step 4 by emitting the final success callout.`, + ); + return; + } + } + + // Reset internal state when the chat is cleared (turns goes from N to 0). + const prevTurnCount = useRef(turns.length); + useEffect(() => { + if (prevTurnCount.current > 0 && turns.length === 0) { + subscribedStores.current.clear(); + handledActions.current.clear(); + claimIdRef.current = null; + } + prevTurnCount.current = turns.length; + }, [turns.length]); +} diff --git a/demo/src/styles.css b/demo/src/styles.css index d31ccea..8fb3fe7 100644 --- a/demo/src/styles.css +++ b/demo/src/styles.css @@ -5421,3 +5421,144 @@ body { .docs-dont h4 { color: #b91c1c; } + +/* ===== Preview Layout (insurance claim demo) ===== */ +/* All rules below are scoped to .preview-layout to keep them isolated + from the other routes (Agent Chat, Author Chat, Validator, Docs). */ + +.preview-layout { + display: flex; + flex-direction: row; + flex: 1; + min-height: 0; + overflow: hidden; +} + +.preview-layout .preview-chat { + display: flex; + flex-direction: column; + flex: 1 1 50%; + min-width: 0; + min-height: 0; + overflow: hidden; + border-right: 1px solid #e5e7eb; +} + +.preview-layout .preview-pane { + display: flex; + flex-direction: column; + flex: 1 1 50%; + min-width: 0; + min-height: 0; + overflow: hidden; + background: #fafafa; +} + +.preview-layout .preview-pane-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + padding: 14px 20px; + border-bottom: 1px solid #e5e7eb; + background: #fff; +} + +.preview-layout .preview-pane-title { + font-size: 14px; + font-weight: 600; + color: #111827; +} + +.preview-layout .preview-pane-status { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 3px 10px; + border-radius: 999px; + font-size: 11px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.04em; +} + +.preview-layout .preview-pane-status--idle { + background: #f3f4f6; + color: #6b7280; +} + +.preview-layout .preview-pane-status--validating, +.preview-layout .preview-pane-status--fixing { + background: #fef3c7; + color: #92400e; +} + +.preview-layout .preview-pane-status--ready { + background: #dcfce7; + color: #15803d; +} + +.preview-layout .preview-pane-status--invalid { + background: #fee2e2; + color: #b91c1c; +} + +.preview-layout .preview-pane-body { + flex: 1; + min-height: 0; + overflow-y: auto; + padding: 20px; +} + +.preview-layout .preview-pane-empty { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + height: 100%; + padding: 40px 24px; + text-align: center; +} + +.preview-layout .preview-pane-empty-title { + margin: 0 0 8px; + font-size: 15px; + font-weight: 600; + color: #374151; +} + +.preview-layout .preview-pane-empty-hint { + margin: 0; + max-width: 360px; + font-size: 13px; + line-height: 1.5; + color: #6b7280; +} + +/* Compact tool_use chip — used by AgentMessage when compactToolUse is true. + Suppresses the inline MDMA preview in the chat so the right-side pane is + the single source of truth for the live render. */ +.preview-layout .agent-tool-call--compact { + padding: 6px 10px; +} +.preview-layout .agent-tool-call--compact .agent-tool-call-header { + margin-bottom: 0; +} +.preview-layout .agent-tool-call--compact .agent-tool-call-body { + display: none; +} + +/* Stack vertically on narrow screens so the preview pane stays usable. */ +@media (max-width: 900px) { + .preview-layout { + flex-direction: column; + } + .preview-layout .preview-chat { + flex: 1 1 50%; + border-right: none; + border-bottom: 1px solid #e5e7eb; + } + .preview-layout .preview-pane { + flex: 1 1 50%; + } +} From 6b536748ddfd84a41aab92362495e7779b19b423 Mon Sep 17 00:00:00 2001 From: gitsad Date: Wed, 20 May 2026 14:46:21 +0200 Subject: [PATCH 16/26] feat: working preview with fixer --- demo/src/PreviewView.tsx | 8 +- demo/src/preview/PreviewPanel.tsx | 105 +++---- demo/src/preview/use-preview-validation.ts | 326 +++++++++++++++++++++ demo/src/styles.css | 29 ++ 4 files changed, 417 insertions(+), 51 deletions(-) create mode 100644 demo/src/preview/use-preview-validation.ts diff --git a/demo/src/PreviewView.tsx b/demo/src/PreviewView.tsx index a14fc3f..a0f7013 100644 --- a/demo/src/PreviewView.tsx +++ b/demo/src/PreviewView.tsx @@ -6,6 +6,7 @@ import { ChatInput } from './chat/ChatInput.js'; import { PreviewPanel } from './preview/PreviewPanel.js'; import { INSURANCE_FLOW_PROMPT } from './preview/insurance-flow-prompt.js'; import { useInsuranceFlow } from './preview/use-insurance-flow.js'; +import { usePreviewValidation } from './preview/use-preview-validation.js'; export function PreviewView() { const { @@ -25,6 +26,11 @@ export function PreviewView() { useInsuranceFlow({ turns, sendHidden, isGenerating }); + const previewState = usePreviewValidation({ + turns, + agentConfig: config, + }); + const chatEndRef = useRef(null); const prevCountRef = useRef(turns.length); @@ -77,7 +83,7 @@ export function PreviewView() { />
- +
); } diff --git a/demo/src/preview/PreviewPanel.tsx b/demo/src/preview/PreviewPanel.tsx index 20225c5..ad9365d 100644 --- a/demo/src/preview/PreviewPanel.tsx +++ b/demo/src/preview/PreviewPanel.tsx @@ -1,78 +1,83 @@ -import { useMemo } from 'react'; import { MdmaDocument } from '@mobile-reality/mdma-renderer-react'; import { customizations } from '../custom-components.js'; -import type { AgentDisplayTurn, AssistantTurn, ToolUseBlock } from '../agent/types.js'; +import type { PreviewState } from './use-preview-validation.js'; interface PreviewPanelProps { - turns: AgentDisplayTurn[]; + state: PreviewState; } -interface LatestMdma { - block: ToolUseBlock; - turnId: string; -} - -function findLatestMdmaBlock(turns: AgentDisplayTurn[]): LatestMdma | null { - for (let i = turns.length - 1; i >= 0; i--) { - const turn = turns[i]; - if (turn.role !== 'assistant') continue; - const blocks = (turn as AssistantTurn).blocks; - for (let j = blocks.length - 1; j >= 0; j--) { - const block = blocks[j]; - if (block.type === 'tool_use') return { block, turnId: turn.id }; - } - } - return null; -} +const STATUS_LABELS: Record = { + idle: 'idle', + validating: 'validating', + fixing: 'fixing', + ready: 'ready', + invalid: 'invalid', +}; -export function PreviewPanel({ turns }: PreviewPanelProps) { - const latest = useMemo(() => findLatestMdmaBlock(turns), [turns]); +const STATUS_CLASS: Record = { + idle: 'preview-pane-status--idle', + validating: 'preview-pane-status--validating', + fixing: 'preview-pane-status--fixing', + ready: 'preview-pane-status--ready', + invalid: 'preview-pane-status--invalid', +}; - const status: 'idle' | 'streaming' | 'ready' = !latest - ? 'idle' - : latest.block.isStreaming - ? 'streaming' - : latest.block.ast && latest.block.store - ? 'ready' - : 'streaming'; - - const statusLabel = - status === 'idle' ? 'idle' : status === 'streaming' ? 'generating' : 'ready'; - const statusClass = - status === 'idle' - ? 'preview-pane-status--idle' - : status === 'streaming' - ? 'preview-pane-status--validating' - : 'preview-pane-status--ready'; +export function PreviewPanel({ state }: PreviewPanelProps) { + const { status, ast, store, unresolvedIssues, wasFixed } = state; + const showRender = ast !== null && store !== null; return (
Live MDMA Preview - {statusLabel} + + {STATUS_LABELS[status]} +
- {!latest ? ( + {status === 'idle' && !showRender ? (

Insurance claim flow

- Start the chat on the left. As the agent emits MDMA blocks, they'll be rendered here. + Start the chat on the left. As the agent emits MDMA blocks, they'll be validated, + auto-fixed if needed, and rendered here.

- ) : latest.block.isStreaming || !latest.block.ast || !latest.block.store ? ( + ) : status === 'validating' || (status === 'fixing' && !showRender) ? (
-

Generating…

+

+ {status === 'validating' ? 'Validating…' : 'Fixing with LLM…'} +

- The agent is still emitting this step. The rendered output will appear when the block - is complete. + {status === 'validating' + ? "Checking the agent's MDMA against the spec." + : "Calling the LLM fixer to repair the agent's output before rendering."}

) : ( - + <> + {wasFixed && status === 'ready' && ( +
+ Auto-fixed before render. +
+ )} + {status === 'invalid' && unresolvedIssues.length > 0 && ( +
+ {unresolvedIssues.length} unresolved issue(s): +
    + {unresolvedIssues.slice(0, 3).map((i, idx) => ( +
  • + {i.ruleId} — {i.message} +
    • + ))} + {unresolvedIssues.length > 3 &&
  • …and {unresolvedIssues.length - 3} more
    • } +
+
+ )} + {showRender && ( + + )} + )}
diff --git a/demo/src/preview/use-preview-validation.ts b/demo/src/preview/use-preview-validation.ts new file mode 100644 index 0000000..b8ff0ca --- /dev/null +++ b/demo/src/preview/use-preview-validation.ts @@ -0,0 +1,326 @@ +import { useEffect, useRef, useState } from 'react'; +import { + validate, + type ValidationIssue, + type ValidationResult, +} from '@mobile-reality/mdma-validator'; +import { + buildFixerPrompt, + buildFixerMessage, + buildSystemPrompt, +} from '@mobile-reality/mdma-prompt-pack'; +import type { MdmaRoot } from '@mobile-reality/mdma-spec'; +import type { DocumentStore } from '@mobile-reality/mdma-runtime'; +import type { AgentDisplayTurn, AssistantTurn, ToolUseBlock } from '../agent/types.js'; +import type { AnthropicConfig } from '../agent/anthropic-client.js'; +import { chatCompletion, type LlmConfig } from '../llm-client.js'; +import { parseMarkdown } from '../chat/parse-markdown.js'; + +export type PreviewStatus = 'idle' | 'validating' | 'fixing' | 'ready' | 'invalid'; + +export interface PreviewState { + status: PreviewStatus; + ast: MdmaRoot | null; + store: DocumentStore | null; + unresolvedIssues: ValidationIssue[]; + wasFixed: boolean; +} + +interface UsePreviewValidationOptions { + turns: AgentDisplayTurn[]; + /** + * Same config the agent uses. The fixer picks its credentials + model + * from this — anthropic provider → haiku via x-api-key, openai → gpt-4.1-mini, + * openrouter → anthropic/claude-haiku-4-5 via openrouter. + */ + agentConfig: AnthropicConfig; +} + +const INITIAL_STATE: PreviewState = { + status: 'idle', + ast: null, + store: null, + unresolvedIssues: [], + wasFixed: false, +}; + +type FixerResolution = + | { + kind: 'anthropic'; + apiKey: string; + model: string; + } + | { + kind: 'openai-compatible'; + apiKey: string; + baseUrl: string; + model: string; + }; + +/** + * Picks the fixer endpoint + model based on the agent's current provider. + * Returns null when the relevant API key isn't configured. + */ +function resolveFixer(config: AnthropicConfig): FixerResolution | null { + const provider = config.provider ?? 'anthropic'; + if (provider === 'anthropic') { + if (!config.apiKey) return null; + return { kind: 'anthropic', apiKey: config.apiKey, model: 'claude-haiku-4-5-20251001' }; + } + if (provider === 'openai') { + if (!config.openaiApiKey) return null; + return { + kind: 'openai-compatible', + apiKey: config.openaiApiKey, + baseUrl: 'https://api.openai.com/v1', + model: 'gpt-4.1-mini', + }; + } + if (provider === 'openrouter') { + if (!config.openrouterApiKey) return null; + return { + kind: 'openai-compatible', + apiKey: config.openrouterApiKey, + baseUrl: 'https://openrouter.ai/api/v1', + model: 'anthropic/claude-haiku-4-5', + }; + } + return null; +} + +/** + * Non-streaming Anthropic Messages API call — used by the fixer when the + * agent provider is anthropic. Reuses the same direct-browser-access + * header the streaming agent client sets. + */ +async function anthropicFix( + apiKey: string, + model: string, + systemPrompt: string, + userMessage: string, + signal: AbortSignal, +): Promise { + const response = await fetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'x-api-key': apiKey, + 'anthropic-version': '2023-06-01', + 'anthropic-dangerous-direct-browser-access': 'true', + }, + body: JSON.stringify({ + model, + max_tokens: 4096, + system: systemPrompt, + messages: [{ role: 'user', content: userMessage }], + }), + signal, + }); + if (!response.ok) { + const body = await response.text(); + throw new Error(`Anthropic fixer failed (${response.status}): ${body}`); + } + const json = (await response.json()) as { content?: Array<{ type: string; text?: string }> }; + const text = (json.content ?? []) + .filter((block): block is { type: 'text'; text: string } => block.type === 'text' && typeof block.text === 'string') + .map((block) => block.text) + .join(''); + return text; +} + +function findLatestToolUseBlock(turns: AgentDisplayTurn[]): ToolUseBlock | null { + for (let i = turns.length - 1; i >= 0; i--) { + const turn = turns[i]; + if (turn.role !== 'assistant') continue; + const blocks = (turn as AssistantTurn).blocks; + for (let j = blocks.length - 1; j >= 0; j--) { + const block = blocks[j]; + if (block.type === 'tool_use') return block; + } + } + return null; +} + +/** + * Validates the latest assistant tool_use block's MDMA document and, if it + * fails validation, runs the LLM fixer (single-block scope) to repair it + * before rendering. The fixer model + credentials are picked from the + * agent's current provider (see resolveFixer). + */ +export function usePreviewValidation({ + turns, + agentConfig, +}: UsePreviewValidationOptions): PreviewState { + const [state, setState] = useState(INITIAL_STATE); + const handledRef = useRef(new Set()); + const inFlightRef = useRef(null); + + useEffect(() => { + const block = findLatestToolUseBlock(turns); + if (!block) { + setState(INITIAL_STATE); + return; + } + + if (block.isStreaming || !block.document) { + setState({ + status: 'validating', + ast: null, + store: null, + unresolvedIssues: [], + wasFixed: false, + }); + return; + } + + const handleKey = `${block.id}:${block.document.length}`; + if (handledRef.current.has(handleKey)) return; + handledRef.current.add(handleKey); + + inFlightRef.current?.abort(); + inFlightRef.current = null; + + const fixer = resolveFixer(agentConfig); + void processBlock(block, fixer, setState, (ctrl) => { + inFlightRef.current = ctrl; + }); + }, [turns, agentConfig]); + + const prevTurnCount = useRef(turns.length); + useEffect(() => { + if (prevTurnCount.current > 0 && turns.length === 0) { + handledRef.current.clear(); + inFlightRef.current?.abort(); + inFlightRef.current = null; + setState(INITIAL_STATE); + } + prevTurnCount.current = turns.length; + }, [turns.length]); + + return state; +} + +async function processBlock( + block: ToolUseBlock, + fixer: FixerResolution | null, + setState: (state: PreviewState) => void, + registerAbort: (ctrl: AbortController) => void, +): Promise { + setState({ + status: 'validating', + ast: null, + store: null, + unresolvedIssues: [], + wasFixed: false, + }); + + const initial: ValidationResult = validate(block.document, { + exclude: ['thinking-block', 'flow-ordering'], + }); + const unfixed = initial.issues.filter( + (i) => !i.fixed && (i.severity === 'error' || i.severity === 'warning'), + ); + + if (unfixed.length === 0) { + const { ast, store } = await parseMarkdown(initial.output); + setState({ + status: 'ready', + ast, + store, + unresolvedIssues: [], + wasFixed: initial.fixCount > 0, + }); + return; + } + + if (!fixer) { + try { + const { ast, store } = await parseMarkdown(initial.output); + setState({ + status: 'invalid', + ast, + store, + unresolvedIssues: unfixed, + wasFixed: false, + }); + } catch { + setState({ + status: 'invalid', + ast: null, + store: null, + unresolvedIssues: unfixed, + wasFixed: false, + }); + } + return; + } + + setState({ + status: 'fixing', + ast: null, + store: null, + unresolvedIssues: unfixed, + wasFixed: false, + }); + + const ctrl = new AbortController(); + registerAbort(ctrl); + try { + const systemPrompt = `${buildSystemPrompt()}\n\n---\n\n${buildFixerPrompt('single-block')}`; + const userMessage = buildFixerMessage(block.document, unfixed); + + let fixed: string; + if (fixer.kind === 'anthropic') { + fixed = await anthropicFix(fixer.apiKey, fixer.model, systemPrompt, userMessage, ctrl.signal); + } else { + const llmConfig: LlmConfig = { + baseUrl: fixer.baseUrl, + apiKey: fixer.apiKey, + model: fixer.model, + }; + fixed = await chatCompletion( + llmConfig, + [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: userMessage }, + ], + ctrl.signal, + ); + } + + const revalidated = validate(fixed, { exclude: ['thinking-block', 'flow-ordering'] }); + const stillUnfixed = revalidated.issues.filter( + (i) => !i.fixed && (i.severity === 'error' || i.severity === 'warning'), + ); + + const { ast, store } = await parseMarkdown(revalidated.output); + setState({ + status: stillUnfixed.length === 0 ? 'ready' : 'invalid', + ast, + store, + unresolvedIssues: stillUnfixed, + wasFixed: true, + }); + } catch (err) { + if (err instanceof DOMException && err.name === 'AbortError') return; + console.error('[preview-validation] fixer failed', err); + try { + const { ast, store } = await parseMarkdown(initial.output); + setState({ + status: 'invalid', + ast, + store, + unresolvedIssues: unfixed, + wasFixed: false, + }); + } catch { + setState({ + status: 'invalid', + ast: null, + store: null, + unresolvedIssues: unfixed, + wasFixed: false, + }); + } + } +} diff --git a/demo/src/styles.css b/demo/src/styles.css index 8fb3fe7..91b9544 100644 --- a/demo/src/styles.css +++ b/demo/src/styles.css @@ -5535,6 +5535,35 @@ body { color: #6b7280; } +/* Validation / fixer status notes shown above the rendered MDMA. */ +.preview-layout .preview-pane-note { + margin-bottom: 14px; + padding: 10px 14px; + border-radius: 8px; + font-size: 12px; + line-height: 1.5; +} +.preview-layout .preview-pane-note--fixed { + background: #fef9c3; + color: #854d0e; + border: 1px solid #fde68a; +} +.preview-layout .preview-pane-note--invalid { + background: #fee2e2; + color: #991b1b; + border: 1px solid #fecaca; +} +.preview-layout .preview-pane-note--invalid ul { + margin: 6px 0 0; + padding-left: 18px; +} +.preview-layout .preview-pane-note--invalid code { + background: rgba(0, 0, 0, 0.06); + padding: 1px 5px; + border-radius: 4px; + font-size: 11px; +} + /* Compact tool_use chip — used by AgentMessage when compactToolUse is true. Suppresses the inline MDMA preview in the chat so the right-side pane is the single source of truth for the live render. */ From 9c42a2be6d2d67bf1c376aeca418e2356e5204fc Mon Sep 17 00:00:00 2001 From: gitsad Date: Wed, 20 May 2026 14:57:25 +0200 Subject: [PATCH 17/26] feat: added backend log --- demo/src/PreviewView.tsx | 8 +- demo/src/preview/BackendLogPane.tsx | 68 +++++++++++++ demo/src/preview/PreviewPanel.tsx | 5 + demo/src/preview/insurance-backend.ts | 64 ++++++++---- demo/src/preview/use-insurance-flow.ts | 96 ++++++++---------- demo/src/preview/use-submission-log.ts | 15 +++ demo/src/styles.css | 131 +++++++++++++++++++++++++ 7 files changed, 309 insertions(+), 78 deletions(-) create mode 100644 demo/src/preview/BackendLogPane.tsx create mode 100644 demo/src/preview/use-submission-log.ts diff --git a/demo/src/PreviewView.tsx b/demo/src/PreviewView.tsx index a0f7013..c2b72a9 100644 --- a/demo/src/PreviewView.tsx +++ b/demo/src/PreviewView.tsx @@ -24,13 +24,17 @@ export function PreviewView() { inputRef, } = useAgent({ flowPrompt: INSURANCE_FLOW_PROMPT }); - useInsuranceFlow({ turns, sendHidden, isGenerating }); - const previewState = usePreviewValidation({ turns, agentConfig: config, }); + useInsuranceFlow({ + currentStore: previewState.store, + sendHidden, + isGenerating, + }); + const chatEndRef = useRef(null); const prevCountRef = useRef(turns.length); diff --git a/demo/src/preview/BackendLogPane.tsx b/demo/src/preview/BackendLogPane.tsx new file mode 100644 index 0000000..1f6be41 --- /dev/null +++ b/demo/src/preview/BackendLogPane.tsx @@ -0,0 +1,68 @@ +import { useState } from 'react'; +import { clearSubmissionLog, type SubmissionLogEntry } from './insurance-backend.js'; + +interface BackendLogPaneProps { + entries: readonly SubmissionLogEntry[]; +} + +const STEP_LABEL: Record = { + 'personal-info': 'POST /claims', + claim: 'POST /claims/:id/description', + bank: 'POST /claims/:id/bank', +}; + +function formatTime(d: Date): string { + return d.toLocaleTimeString(undefined, { + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + }); +} + +export function BackendLogPane({ entries }: BackendLogPaneProps) { + const [open, setOpen] = useState(true); + + return ( +
setOpen((e.target as HTMLDetailsElement).open)}> + + Backend log + {entries.length} + {entries.length > 0 && ( + + )} + + {entries.length === 0 ? ( +

+ No submissions yet. Once the user submits a form, the mock backend response will appear + here. +

+ ) : ( +
    + {entries.map((entry, i) => ( +
  1. +
    + {STEP_LABEL[entry.step]} + 200 OK + {formatTime(entry.at)} +
    +
    + {entry.claimId} + {entry.summary} +
    +
    2. + ))} +
+ )} +
+ ); +} diff --git a/demo/src/preview/PreviewPanel.tsx b/demo/src/preview/PreviewPanel.tsx index ad9365d..40b0924 100644 --- a/demo/src/preview/PreviewPanel.tsx +++ b/demo/src/preview/PreviewPanel.tsx @@ -1,6 +1,8 @@ import { MdmaDocument } from '@mobile-reality/mdma-renderer-react'; import { customizations } from '../custom-components.js'; +import { BackendLogPane } from './BackendLogPane.js'; import type { PreviewState } from './use-preview-validation.js'; +import { useSubmissionLog } from './use-submission-log.js'; interface PreviewPanelProps { state: PreviewState; @@ -25,6 +27,7 @@ const STATUS_CLASS: Record = { export function PreviewPanel({ state }: PreviewPanelProps) { const { status, ast, store, unresolvedIssues, wasFixed } = state; const showRender = ast !== null && store !== null; + const submissionLog = useSubmissionLog(); return (
@@ -79,6 +82,8 @@ export function PreviewPanel({ state }: PreviewPanelProps) { )} )} + +
); diff --git a/demo/src/preview/insurance-backend.ts b/demo/src/preview/insurance-backend.ts index 2aeb554..3c15d17 100644 --- a/demo/src/preview/insurance-backend.ts +++ b/demo/src/preview/insurance-backend.ts @@ -22,7 +22,18 @@ export interface SubmissionLogEntry { summary: string; } -const submissionLog: SubmissionLogEntry[] = []; +let submissionLog: SubmissionLogEntry[] = []; +const listeners = new Set<() => void>(); +function notify() { + for (const fn of listeners) fn(); +} + +export function subscribeSubmissionLog(listener: () => void): () => void { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; +} export interface PersonalInfoPayload { 'full-name': string; @@ -59,12 +70,16 @@ export const insuranceBackend = { async collectPersonalInfo(payload: PersonalInfoPayload): Promise { await delay(700); const claimId = makeClaimId(); - submissionLog.push({ - step: 'personal-info', - at: new Date(), - claimId, - summary: `${payload['full-name']} (DOB ${payload.birthday})`, - }); + submissionLog = [ + ...submissionLog, + { + step: 'personal-info', + at: new Date(), + claimId, + summary: `${payload['full-name']} (DOB ${payload.birthday})`, + }, + ]; + notify(); return { claimId, accepted: true }; }, @@ -72,23 +87,31 @@ export const insuranceBackend = { await delay(800); const desc = payload['claim-description']; const preview = desc.length > 60 ? `${desc.slice(0, 60)}…` : desc; - submissionLog.push({ - step: 'claim', - at: new Date(), - claimId, - summary: `"${preview}"`, - }); + submissionLog = [ + ...submissionLog, + { + step: 'claim', + at: new Date(), + claimId, + summary: `"${preview}"`, + }, + ]; + notify(); return { accepted: true }; }, async collectBank(claimId: string, payload: BankPayload): Promise { await delay(700); - submissionLog.push({ - step: 'bank', - at: new Date(), - claimId, - summary: `IBAN ${maskIban(payload.iban)}`, - }); + submissionLog = [ + ...submissionLog, + { + step: 'bank', + at: new Date(), + claimId, + summary: `IBAN ${maskIban(payload.iban)}`, + }, + ]; + notify(); return { accepted: true, etaDays: 5 }; }, }; @@ -98,5 +121,6 @@ export function getSubmissionLog(): readonly SubmissionLogEntry[] { } export function clearSubmissionLog(): void { - submissionLog.length = 0; + submissionLog = []; + notify(); } diff --git a/demo/src/preview/use-insurance-flow.ts b/demo/src/preview/use-insurance-flow.ts index d58bb64..1895cd6 100644 --- a/demo/src/preview/use-insurance-flow.ts +++ b/demo/src/preview/use-insurance-flow.ts @@ -1,6 +1,5 @@ import { useEffect, useRef } from 'react'; import type { DocumentStore } from '@mobile-reality/mdma-runtime'; -import type { AgentDisplayTurn, AssistantTurn } from '../agent/types.js'; import { insuranceBackend, type BankPayload, @@ -9,7 +8,14 @@ import { } from './insurance-backend.js'; interface UseInsuranceFlowOptions { - turns: AgentDisplayTurn[]; + /** + * The store currently rendered in the preview pane (validated/fixed + * output, NOT the agent's raw block.store). When the user clicks Submit + * in the right pane, the ACTION_TRIGGERED event fires on this store, so + * the hook must subscribe to *this* store — earlier versions subscribed + * to block.store and silently missed every submit. + */ + currentStore: DocumentStore | null; sendHidden: (message: string) => Promise; isGenerating: boolean; } @@ -24,21 +30,21 @@ function isHandledActionId(id: string): id is ActionId { /** * Drives the Insurance Preview flow: * - * 1. Listens for `ACTION_TRIGGERED` events on the MDMA renderer stores of - * each new assistant turn. - * 2. When an event with one of our known `actionId`s fires, pulls the - * submitted values straight from the store (does NOT include them in - * any message to the agent), calls the mock backend, and waits for the - * success response. - * 3. On success, sends a HIDDEN user message to the agent — never shown - * in the chat — carrying only a "step N complete, please continue" - * signal. The agent uses that to emit the next step naturally. + * 1. Subscribes to `ACTION_TRIGGERED` on whatever store is currently being + * rendered in the preview pane. + * 2. When a known `actionId` fires, pulls the submitted values from that + * same store, calls the mock backend, and waits for success. + * 3. On success, sends a HIDDEN user message to the agent — no form data, + * just a "step N done, please continue" signal. * - * The claim id returned by step 1 is threaded into steps 2 + 3 via a ref - * so consecutive backend calls reference the same claim. + * The claim id from step 1 is threaded into steps 2 + 3 via a ref. */ -export function useInsuranceFlow({ turns, sendHidden, isGenerating }: UseInsuranceFlowOptions) { - const subscribedStores = useRef(new Set()); +export function useInsuranceFlow({ + currentStore, + sendHidden, + isGenerating, +}: UseInsuranceFlowOptions) { + const subscribedStores = useRef(new WeakSet()); const handledActions = useRef(new Set()); const claimIdRef = useRef(null); const isGeneratingRef = useRef(isGenerating); @@ -47,39 +53,28 @@ export function useInsuranceFlow({ turns, sendHidden, isGenerating }: UseInsuran sendHiddenRef.current = sendHidden; useEffect(() => { - for (const turn of turns) { - if (turn.role !== 'assistant') continue; - const blocks = (turn as AssistantTurn).blocks; - for (const block of blocks) { - if (block.type !== 'tool_use') continue; - const store = block.store; - if (!store || subscribedStores.current.has(store)) continue; - subscribedStores.current.add(store); + if (!currentStore || subscribedStores.current.has(currentStore)) return; + subscribedStores.current.add(currentStore); - store.getEventBus().on('ACTION_TRIGGERED', (action) => { - if (isGeneratingRef.current) return; - const { actionId, componentId } = action; - if (!isHandledActionId(actionId)) return; + currentStore.getEventBus().on('ACTION_TRIGGERED', (action) => { + if (isGeneratingRef.current) return; + const { actionId, componentId } = action; + if (!isHandledActionId(actionId)) return; - // De-dupe: one ACTION_TRIGGERED per (componentId, actionId) - const key = `${componentId}:${actionId}`; - if (handledActions.current.has(key)) return; - handledActions.current.add(key); + const key = `${componentId}:${actionId}`; + if (handledActions.current.has(key)) return; + handledActions.current.add(key); - const values = (store.getComponentState(componentId)?.values ?? {}) as Record< - string, - unknown - >; - void dispatch(actionId, values).catch((err) => { - handledActions.current.delete(key); - // Surfacing errors to the user is out of scope for now; log and - // let them retry the submission. - console.error('[insurance-flow] backend call failed', err); - }); - }); - } - } - }, [turns]); + const values = (currentStore.getComponentState(componentId)?.values ?? {}) as Record< + string, + unknown + >; + void dispatch(actionId, values).catch((err) => { + handledActions.current.delete(key); + console.error('[insurance-flow] backend call failed', err); + }); + }); + }, [currentStore]); async function dispatch(actionId: ActionId, values: Record) { if (actionId === 'collect-personal-info') { @@ -125,15 +120,4 @@ export function useInsuranceFlow({ turns, sendHidden, isGenerating }: UseInsuran return; } } - - // Reset internal state when the chat is cleared (turns goes from N to 0). - const prevTurnCount = useRef(turns.length); - useEffect(() => { - if (prevTurnCount.current > 0 && turns.length === 0) { - subscribedStores.current.clear(); - handledActions.current.clear(); - claimIdRef.current = null; - } - prevTurnCount.current = turns.length; - }, [turns.length]); } diff --git a/demo/src/preview/use-submission-log.ts b/demo/src/preview/use-submission-log.ts new file mode 100644 index 0000000..b78bfbd --- /dev/null +++ b/demo/src/preview/use-submission-log.ts @@ -0,0 +1,15 @@ +import { useSyncExternalStore } from 'react'; +import { + getSubmissionLog, + subscribeSubmissionLog, + type SubmissionLogEntry, +} from './insurance-backend.js'; + +/** + * Reactive read of the mock backend's submission log. The store lives in + * `insurance-backend.ts` (module-level array + subscriber set); this hook + * re-renders any consumer whenever a new submission is recorded. + */ +export function useSubmissionLog(): readonly SubmissionLogEntry[] { + return useSyncExternalStore(subscribeSubmissionLog, getSubmissionLog, getSubmissionLog); +} diff --git a/demo/src/styles.css b/demo/src/styles.css index 91b9544..c87686c 100644 --- a/demo/src/styles.css +++ b/demo/src/styles.css @@ -5564,6 +5564,137 @@ body { font-size: 11px; } +/* Backend log pane — collapsible "Backend log" beneath the rendered MDMA. */ +.preview-layout .preview-log { + margin-top: 24px; + border: 1px solid #e5e7eb; + border-radius: 8px; + background: #fff; + font-size: 12px; +} +.preview-layout .preview-log-summary { + display: flex; + align-items: center; + gap: 10px; + padding: 10px 14px; + cursor: pointer; + list-style: none; + user-select: none; +} +.preview-layout .preview-log-summary::-webkit-details-marker { + display: none; +} +.preview-layout .preview-log-summary::before { + content: '▸'; + display: inline-block; + font-size: 10px; + color: #6b7280; + transition: transform 0.15s ease; +} +.preview-layout .preview-log[open] .preview-log-summary::before { + transform: rotate(90deg); +} +.preview-layout .preview-log-title { + font-weight: 600; + color: #111827; +} +.preview-layout .preview-log-count { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 22px; + height: 18px; + padding: 0 6px; + border-radius: 999px; + background: #f3f4f6; + color: #374151; + font-size: 11px; + font-weight: 600; +} +.preview-layout .preview-log-clear { + margin-left: auto; + padding: 3px 10px; + border-radius: 6px; + border: 1px solid #d1d5db; + background: #fff; + color: #374151; + font-size: 11px; + font-weight: 500; + cursor: pointer; +} +.preview-layout .preview-log-clear:hover { + background: #f9fafb; +} +.preview-layout .preview-log-empty { + margin: 0; + padding: 0 14px 14px; + color: #6b7280; + font-size: 12px; + line-height: 1.5; +} +.preview-layout .preview-log-list { + margin: 0; + padding: 0 14px 14px; + list-style: none; + display: flex; + flex-direction: column; + gap: 8px; +} +.preview-layout .preview-log-item { + padding: 10px 12px; + border-radius: 6px; + background: #f9fafb; + border: 1px solid #e5e7eb; +} +.preview-layout .preview-log-item-meta { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 6px; +} +.preview-layout .preview-log-item-method { + font-family: ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 11px; + color: #1f2937; +} +.preview-layout .preview-log-item-status { + padding: 1px 8px; + border-radius: 999px; + background: #dcfce7; + color: #15803d; + font-size: 10px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.04em; +} +.preview-layout .preview-log-item-time { + margin-left: auto; + color: #6b7280; + font-size: 11px; +} +.preview-layout .preview-log-item-body { + display: flex; + align-items: center; + gap: 10px; +} +.preview-layout .preview-log-item-claim { + padding: 1px 6px; + border-radius: 4px; + background: #eef2ff; + color: #3730a3; + font-size: 11px; + font-family: ui-monospace, SFMono-Regular, Menlo, monospace; +} +.preview-layout .preview-log-item-summary { + color: #374151; + font-size: 12px; + flex: 1; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + /* Compact tool_use chip — used by AgentMessage when compactToolUse is true. Suppresses the inline MDMA preview in the chat so the right-side pane is the single source of truth for the live render. */ From bd6446048bdaab3c9d32875a6be3efe7a63e4571 Mon Sep 17 00:00:00 2001 From: gitsad Date: Wed, 20 May 2026 15:09:11 +0200 Subject: [PATCH 18/26] chore: changed naming --- demo/src/App.tsx | 2 +- demo/src/HomeView.tsx | 4 +- demo/src/PreviewView.tsx | 3 + demo/src/preview/BackendLogPane.tsx | 120 ++++++++++++++++++---------- demo/src/preview/PreviewPanel.tsx | 5 -- demo/src/styles.css | 116 ++++++++++++++++++++------- 6 files changed, 170 insertions(+), 80 deletions(-) diff --git a/demo/src/App.tsx b/demo/src/App.tsx index 15c0f2d..625941b 100644 --- a/demo/src/App.tsx +++ b/demo/src/App.tsx @@ -44,7 +44,7 @@ const NAV_GROUPS: NavGroup[] = [ label: 'Agentic', items: [ { path: '/chat', label: 'Agent Chat', icon: '⚡' }, - { path: '/preview', label: 'Insurance Preview', icon: '🛡️' }, + { path: '/preview', label: 'Preview', icon: '🛡️' }, ], }, { diff --git a/demo/src/HomeView.tsx b/demo/src/HomeView.tsx index 2f4704f..3d9d6fc 100644 --- a/demo/src/HomeView.tsx +++ b/demo/src/HomeView.tsx @@ -20,10 +20,10 @@ const SECTIONS = [ }, { path: '/preview', - label: 'Insurance Preview', + label: 'Preview', icon: '🛡️', description: - 'Multi-step insurance claim flow demo — chat on the left, live MDMA preview with auto-validation and fixer on the right.', + 'Multi-step flow demo (insurance claim) — chat on the left, live MDMA preview with auto-validation and fixer on the right.', }, ], }, diff --git a/demo/src/PreviewView.tsx b/demo/src/PreviewView.tsx index c2b72a9..3c27d0f 100644 --- a/demo/src/PreviewView.tsx +++ b/demo/src/PreviewView.tsx @@ -3,6 +3,7 @@ import { useAgent } from './agent/use-agent.js'; import { AgentMessage } from './agent/AgentMessage.js'; import { AgentSettings } from './agent/AgentSettings.js'; import { ChatInput } from './chat/ChatInput.js'; +import { BackendLogDrawer } from './preview/BackendLogPane.js'; import { PreviewPanel } from './preview/PreviewPanel.js'; import { INSURANCE_FLOW_PROMPT } from './preview/insurance-flow-prompt.js'; import { useInsuranceFlow } from './preview/use-insurance-flow.js'; @@ -88,6 +89,8 @@ export function PreviewView() { + + ); } diff --git a/demo/src/preview/BackendLogPane.tsx b/demo/src/preview/BackendLogPane.tsx index 1f6be41..695c9a2 100644 --- a/demo/src/preview/BackendLogPane.tsx +++ b/demo/src/preview/BackendLogPane.tsx @@ -1,9 +1,6 @@ import { useState } from 'react'; import { clearSubmissionLog, type SubmissionLogEntry } from './insurance-backend.js'; - -interface BackendLogPaneProps { - entries: readonly SubmissionLogEntry[]; -} +import { useSubmissionLog } from './use-submission-log.js'; const STEP_LABEL: Record = { 'personal-info': 'POST /claims', @@ -19,50 +16,85 @@ function formatTime(d: Date): string { }); } -export function BackendLogPane({ entries }: BackendLogPaneProps) { - const [open, setOpen] = useState(true); +/** + * Floating toggle + slide-out drawer on the right edge of the Preview + * page. Lives at the layout root (not inside `PreviewPanel`) so the log + * doesn't share scroll/space with the rendered MDMA — the demo audience + * can pop it open at any time to see the masked submissions land. + */ +export function BackendLogDrawer() { + const entries = useSubmissionLog(); + const [open, setOpen] = useState(false); return ( -
setOpen((e.target as HTMLDetailsElement).open)}> - - Backend log - {entries.length} - {entries.length > 0 && ( + <> + + + {open && ( + <> - )} - - {entries.length === 0 ? ( -

- No submissions yet. Once the user submits a form, the mock backend response will appear - here. -

- ) : ( -
    - {entries.map((entry, i) => ( -
  1. -
    - {STEP_LABEL[entry.step]} - 200 OK - {formatTime(entry.at)} -
    -
    - {entry.claimId} - {entry.summary} -
    -
    2. - ))} -
+ className="preview-log-backdrop" + onClick={() => setOpen(false)} + aria-label="Close backend log" + /> + + )} -
+ ); } diff --git a/demo/src/preview/PreviewPanel.tsx b/demo/src/preview/PreviewPanel.tsx index 40b0924..ad9365d 100644 --- a/demo/src/preview/PreviewPanel.tsx +++ b/demo/src/preview/PreviewPanel.tsx @@ -1,8 +1,6 @@ import { MdmaDocument } from '@mobile-reality/mdma-renderer-react'; import { customizations } from '../custom-components.js'; -import { BackendLogPane } from './BackendLogPane.js'; import type { PreviewState } from './use-preview-validation.js'; -import { useSubmissionLog } from './use-submission-log.js'; interface PreviewPanelProps { state: PreviewState; @@ -27,7 +25,6 @@ const STATUS_CLASS: Record = { export function PreviewPanel({ state }: PreviewPanelProps) { const { status, ast, store, unresolvedIssues, wasFixed } = state; const showRender = ast !== null && store !== null; - const submissionLog = useSubmissionLog(); return (
@@ -82,8 +79,6 @@ export function PreviewPanel({ state }: PreviewPanelProps) { )} )} - -
); diff --git a/demo/src/styles.css b/demo/src/styles.css index c87686c..8652455 100644 --- a/demo/src/styles.css +++ b/demo/src/styles.css @@ -5564,41 +5564,84 @@ body { font-size: 11px; } -/* Backend log pane — collapsible "Backend log" beneath the rendered MDMA. */ -.preview-layout .preview-log { - margin-top: 24px; - border: 1px solid #e5e7eb; - border-radius: 8px; - background: #fff; - font-size: 12px; -} -.preview-layout .preview-log-summary { +/* Backend log — floating toggle (bottom-right) + slide-out drawer. + Drawer overlays the right edge of the layout so the log is accessible + from any sub-view of the Preview page without sharing space with the + rendered MDMA preview. */ +.preview-layout .preview-log-toggle { + position: fixed; + bottom: 80px; + right: 20px; + z-index: 50; display: flex; align-items: center; - gap: 10px; - padding: 10px 14px; + gap: 8px; + padding: 8px 14px; + font-size: 12px; + font-weight: 600; + border: 1px solid #d1d5db; + border-radius: 20px; + background: #fff; + color: #374151; cursor: pointer; - list-style: none; - user-select: none; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); } -.preview-layout .preview-log-summary::-webkit-details-marker { - display: none; +.preview-layout .preview-log-toggle:hover { + border-color: #6c5ce7; + color: #6c5ce7; } -.preview-layout .preview-log-summary::before { - content: '▸'; - display: inline-block; +.preview-layout .preview-log-toggle-badge { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 18px; + height: 18px; + padding: 0 5px; font-size: 10px; - color: #6b7280; - transition: transform 0.15s ease; + font-weight: 700; + border-radius: 9px; + background: #6c5ce7; + color: #fff; } -.preview-layout .preview-log[open] .preview-log-summary::before { - transform: rotate(90deg); + +.preview-layout .preview-log-backdrop { + position: fixed; + inset: 0; + z-index: 60; + background: rgba(15, 23, 42, 0.25); + border: none; + padding: 0; + cursor: pointer; +} + +.preview-layout .preview-log-drawer { + position: fixed; + top: 0; + right: 0; + bottom: 0; + z-index: 70; + width: min(440px, 100vw); + display: flex; + flex-direction: column; + background: #fff; + border-left: 1px solid #e5e7eb; + box-shadow: -4px 0 16px rgba(0, 0, 0, 0.12); +} + +.preview-layout .preview-log-drawer-header { + display: flex; + align-items: center; + gap: 10px; + padding: 14px 18px; + border-bottom: 1px solid #e5e7eb; + background: #fafafa; } -.preview-layout .preview-log-title { +.preview-layout .preview-log-drawer-title { + font-size: 14px; font-weight: 600; color: #111827; } -.preview-layout .preview-log-count { +.preview-layout .preview-log-drawer-count { display: inline-flex; align-items: center; justify-content: center; @@ -5606,8 +5649,8 @@ body { height: 18px; padding: 0 6px; border-radius: 999px; - background: #f3f4f6; - color: #374151; + background: #eef2ff; + color: #3730a3; font-size: 11px; font-weight: 600; } @@ -5625,16 +5668,33 @@ body { .preview-layout .preview-log-clear:hover { background: #f9fafb; } +.preview-layout .preview-log-drawer-close { + padding: 0 8px; + border: none; + background: transparent; + font-size: 22px; + line-height: 1; + color: #6b7280; + cursor: pointer; +} +.preview-layout .preview-log-drawer-close:hover { + color: #111827; +} +.preview-layout .preview-log-drawer-body { + flex: 1; + min-height: 0; + overflow-y: auto; + padding: 16px; +} .preview-layout .preview-log-empty { margin: 0; - padding: 0 14px 14px; color: #6b7280; font-size: 12px; line-height: 1.5; } .preview-layout .preview-log-list { margin: 0; - padding: 0 14px 14px; + padding: 0; list-style: none; display: flex; flex-direction: column; From 7a998ce020bf01d8276de0bba59ee1fdc40f5af1 Mon Sep 17 00:00:00 2001 From: gitsad Date: Thu, 21 May 2026 08:46:04 +0200 Subject: [PATCH 19/26] fix: improved callout for preview --- demo/src/PreviewView.tsx | 31 ++++- demo/src/agent/AgentMessage.tsx | 71 ++++++++++- demo/src/preview/PreviewPanel.tsx | 21 +++- demo/src/preview/preview-customizations.tsx | 77 ++++++++++++ demo/src/preview/use-preview-validation.ts | 95 ++++++++++++--- demo/src/styles.css | 123 ++++++++++++++++++++ 6 files changed, 392 insertions(+), 26 deletions(-) create mode 100644 demo/src/preview/preview-customizations.tsx diff --git a/demo/src/PreviewView.tsx b/demo/src/PreviewView.tsx index 3c27d0f..67ab28f 100644 --- a/demo/src/PreviewView.tsx +++ b/demo/src/PreviewView.tsx @@ -1,4 +1,4 @@ -import { useRef, useEffect, useCallback } from 'react'; +import { useRef, useEffect, useCallback, useState } from 'react'; import { useAgent } from './agent/use-agent.js'; import { AgentMessage } from './agent/AgentMessage.js'; import { AgentSettings } from './agent/AgentSettings.js'; @@ -25,8 +25,14 @@ export function PreviewView() { inputRef, } = useAgent({ flowPrompt: INSURANCE_FLOW_PROMPT }); + // `selectedBlockId` controls which tool_use block the preview pane renders. + // null = follow the latest one. When the agent emits a new tool_use block, + // we snap back to "latest" so the new step shows automatically. + const [selectedBlockId, setSelectedBlockId] = useState(null); + const previewState = usePreviewValidation({ turns, + selectedBlockId, agentConfig: config, }); @@ -36,6 +42,21 @@ export function PreviewView() { isGenerating, }); + // Snap back to "latest" whenever a new tool_use block appears. Reading + // `turns` inside (not just .length) satisfies the deps lint while still + // only resetting on genuinely new content — selection survives interim + // streaming updates. + const prevToolUseCountRef = useRef(0); + useEffect(() => { + let count = 0; + for (const turn of turns) { + if (turn.role !== 'assistant') continue; + for (const block of turn.blocks) if (block.type === 'tool_use') count++; + } + if (count > prevToolUseCountRef.current) setSelectedBlockId(null); + prevToolUseCountRef.current = count; + }, [turns]); + const chatEndRef = useRef(null); const prevCountRef = useRef(turns.length); @@ -68,7 +89,13 @@ export function PreviewView() { )} {turns.map((turn) => ( - + ))} {error &&
{error}
} diff --git a/demo/src/agent/AgentMessage.tsx b/demo/src/agent/AgentMessage.tsx index e44c703..422f8d9 100644 --- a/demo/src/agent/AgentMessage.tsx +++ b/demo/src/agent/AgentMessage.tsx @@ -164,9 +164,46 @@ function TextBlockView({ block }: { block: TextBlock }) { return ; } -function ToolUseBlockView({ block, compact }: { block: ToolUseBlock; compact?: boolean }) { +function ToolUseBlockView({ + block, + compact, + isActive, + onSelect, +}: { + block: ToolUseBlock; + compact?: boolean; + isActive?: boolean; + onSelect?: () => void; +}) { + const clickable = compact && !block.isStreaming && Boolean(onSelect); + const className = [ + 'agent-tool-call', + compact ? 'agent-tool-call--compact' : '', + clickable ? 'agent-tool-call--clickable' : '', + isActive ? 'agent-tool-call--active' : '', + ] + .filter(Boolean) + .join(' '); + + const handleClick = clickable ? onSelect : undefined; + return ( -
+
{ + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + onSelect?.(); + } + } + : undefined + } + role={clickable ? 'button' : undefined} + tabIndex={clickable ? 0 : undefined} + >
{block.name} {block.isStreaming && generating…} {compact && !block.isStreaming && ( - rendered in preview → + + {isActive ? 'showing in preview' : 'show in preview →'} + )}
@@ -214,9 +253,23 @@ interface AgentMessageProps { * the right-side pane and would be duplicated in the chat otherwise. */ compactToolUse?: boolean; + /** + * Block id currently shown in the Preview pane. Highlighted in the chat. + */ + activeToolUseId?: string | null; + /** + * When set, the compact tool_use chip becomes clickable and calls this + * with the block's id when the user wants to inspect that step. + */ + onSelectToolUse?: (blockId: string) => void; } -export const AgentMessage = memo(function AgentMessage({ turn, compactToolUse }: AgentMessageProps) { +export const AgentMessage = memo(function AgentMessage({ + turn, + compactToolUse, + activeToolUseId, + onSelectToolUse, +}: AgentMessageProps) { if (turn.role === 'user') { if (turn.hidden) return null; return ( @@ -247,7 +300,15 @@ export const AgentMessage = memo(function AgentMessage({ turn, compactToolUse }: return ; if (block.type === 'text') return ; if (block.type === 'tool_use') - return ; + return ( + onSelectToolUse(block.id) : undefined} + /> + ); }) )}
diff --git a/demo/src/preview/PreviewPanel.tsx b/demo/src/preview/PreviewPanel.tsx index ad9365d..b0594fe 100644 --- a/demo/src/preview/PreviewPanel.tsx +++ b/demo/src/preview/PreviewPanel.tsx @@ -1,5 +1,5 @@ import { MdmaDocument } from '@mobile-reality/mdma-renderer-react'; -import { customizations } from '../custom-components.js'; +import { previewCustomizations } from './preview-customizations.js'; import type { PreviewState } from './use-preview-validation.js'; interface PreviewPanelProps { @@ -23,13 +23,16 @@ const STATUS_CLASS: Record = { }; export function PreviewPanel({ state }: PreviewPanelProps) { - const { status, ast, store, unresolvedIssues, wasFixed } = state; + const { status, ast, store, unresolvedIssues, wasFixed, submitted } = state; const showRender = ast !== null && store !== null; return (
Live MDMA Preview + {submitted && ( + submitted + )} {STATUS_LABELS[status]} @@ -74,8 +77,20 @@ export function PreviewPanel({ state }: PreviewPanelProps) {
)} + {submitted && ( +
+ This step has already been submitted. Inputs are read-only — go back to the latest + step from the chat to continue. +
+ )} {showRender && ( - +
+ +
)} )} diff --git a/demo/src/preview/preview-customizations.tsx b/demo/src/preview/preview-customizations.tsx new file mode 100644 index 0000000..8a916a3 --- /dev/null +++ b/demo/src/preview/preview-customizations.tsx @@ -0,0 +1,77 @@ +import { memo } from 'react'; +import type { MdmaBlockRendererProps } from '@mobile-reality/mdma-renderer-react'; +import type { MdmaCustomizations } from '../ChatView.js'; +import { customizations as baseCustomizations } from '../custom-components.js'; + +/** + * Preview-pane-specific callout renderer. The base `CustomCalloutRenderer` + * is used across the demo, so changing it would affect Agent Chat and the + * other views. This renderer emits its own `.preview-callout` markup and + * is wired only via `previewCustomizations` below, keeping the polished + * look local to the Insurance Preview page. + */ + +const VARIANT_ICONS: Record = { + info: 'ℹ️', + warning: '⚠️', + error: '❌', + success: '✅', +}; + +const PreviewCalloutRenderer = memo(function PreviewCalloutRenderer({ + component, + componentState, + dispatch, +}: MdmaBlockRendererProps) { + if (component.type !== 'callout') return null; + if (componentState?.values.dismissed) return null; + + const variant = (component.variant as string | undefined) ?? 'info'; + const icon = VARIANT_ICONS[variant] ?? VARIANT_ICONS.info; + + return ( +
+ +
+ {component.title &&
{component.title}
} + {component.content &&

{component.content}

} +
+ {component.dismissible && ( + + )} +
+ ); +}); + +/** + * Same as the base demo customizations but with the callout swapped out + * for the preview-specific renderer. Forms, buttons, charts, etc. keep + * the existing custom styling. + */ +export const previewCustomizations: MdmaCustomizations = { + ...baseCustomizations, + components: { + ...baseCustomizations.components, + callout: PreviewCalloutRenderer, + }, +}; diff --git a/demo/src/preview/use-preview-validation.ts b/demo/src/preview/use-preview-validation.ts index b8ff0ca..5b5bd08 100644 --- a/demo/src/preview/use-preview-validation.ts +++ b/demo/src/preview/use-preview-validation.ts @@ -24,10 +24,22 @@ export interface PreviewState { store: DocumentStore | null; unresolvedIssues: ValidationIssue[]; wasFixed: boolean; + /** Id of the block currently being rendered (the agent's tool_use block id). */ + blockId: string | null; + /** + * True when the rendered block is from an earlier step than the latest + * one — i.e. it's already been submitted in the flow and re-interacting + * with it shouldn't happen. PreviewPanel uses this to disable inputs. + */ + submitted: boolean; } interface UsePreviewValidationOptions { turns: AgentDisplayTurn[]; + /** + * When set, show this specific tool_use block. When null, show the latest. + */ + selectedBlockId: string | null; /** * Same config the agent uses. The fixer picks its credentials + model * from this — anthropic provider → haiku via x-api-key, openai → gpt-4.1-mini, @@ -42,6 +54,8 @@ const INITIAL_STATE: PreviewState = { store: null, unresolvedIssues: [], wasFixed: false, + blockId: null, + submitted: false, }; type FixerResolution = @@ -128,17 +142,29 @@ async function anthropicFix( return text; } -function findLatestToolUseBlock(turns: AgentDisplayTurn[]): ToolUseBlock | null { - for (let i = turns.length - 1; i >= 0; i--) { - const turn = turns[i]; +function collectToolUseBlocks(turns: AgentDisplayTurn[]): ToolUseBlock[] { + const blocks: ToolUseBlock[] = []; + for (const turn of turns) { if (turn.role !== 'assistant') continue; - const blocks = (turn as AssistantTurn).blocks; - for (let j = blocks.length - 1; j >= 0; j--) { - const block = blocks[j]; - if (block.type === 'tool_use') return block; + for (const block of (turn as AssistantTurn).blocks) { + if (block.type === 'tool_use') blocks.push(block); } } - return null; + return blocks; +} + +function resolveBlock( + turns: AgentDisplayTurn[], + selectedBlockId: string | null, +): { block: ToolUseBlock | null; submitted: boolean } { + const all = collectToolUseBlocks(turns); + if (all.length === 0) return { block: null, submitted: false }; + if (!selectedBlockId) { + return { block: all[all.length - 1], submitted: false }; + } + const idx = all.findIndex((b) => b.id === selectedBlockId); + if (idx === -1) return { block: all[all.length - 1], submitted: false }; + return { block: all[idx], submitted: idx < all.length - 1 }; } /** @@ -149,14 +175,15 @@ function findLatestToolUseBlock(turns: AgentDisplayTurn[]): ToolUseBlock | null */ export function usePreviewValidation({ turns, + selectedBlockId, agentConfig, }: UsePreviewValidationOptions): PreviewState { const [state, setState] = useState(INITIAL_STATE); - const handledRef = useRef(new Set()); + const handledRef = useRef(new Map()); const inFlightRef = useRef(null); useEffect(() => { - const block = findLatestToolUseBlock(turns); + const { block, submitted } = resolveBlock(turns, selectedBlockId); if (!block) { setState(INITIAL_STATE); return; @@ -169,22 +196,42 @@ export function usePreviewValidation({ store: null, unresolvedIssues: [], wasFixed: false, + blockId: block.id, + submitted, }); return; } + // De-dupe on (blockId, doc length) so toggling the selection between + // already-processed blocks re-uses the cached PreviewState instead of + // re-running validation + fixer. const handleKey = `${block.id}:${block.document.length}`; - if (handledRef.current.has(handleKey)) return; - handledRef.current.add(handleKey); + const cached = handledRef.current.get(handleKey); + if (cached) { + setState({ ...cached, submitted }); + return; + } inFlightRef.current?.abort(); inFlightRef.current = null; const fixer = resolveFixer(agentConfig); - void processBlock(block, fixer, setState, (ctrl) => { - inFlightRef.current = ctrl; - }); - }, [turns, agentConfig]); + void processBlock( + block, + fixer, + (next) => { + const withFlags = { ...next, blockId: block.id, submitted }; + // Snapshot terminal states so revisits don't refire the LLM. + if (next.status === 'ready' || next.status === 'invalid') { + handledRef.current.set(handleKey, withFlags); + } + setState(withFlags); + }, + (ctrl) => { + inFlightRef.current = ctrl; + }, + ); + }, [turns, selectedBlockId, agentConfig]); const prevTurnCount = useRef(turns.length); useEffect(() => { @@ -212,6 +259,8 @@ async function processBlock( store: null, unresolvedIssues: [], wasFixed: false, + blockId: block.id, + submitted: false, }); const initial: ValidationResult = validate(block.document, { @@ -229,6 +278,8 @@ async function processBlock( store, unresolvedIssues: [], wasFixed: initial.fixCount > 0, + blockId: block.id, + submitted: false, }); return; } @@ -242,6 +293,8 @@ async function processBlock( store, unresolvedIssues: unfixed, wasFixed: false, + blockId: block.id, + submitted: false, }); } catch { setState({ @@ -250,6 +303,8 @@ async function processBlock( store: null, unresolvedIssues: unfixed, wasFixed: false, + blockId: block.id, + submitted: false, }); } return; @@ -261,6 +316,8 @@ async function processBlock( store: null, unresolvedIssues: unfixed, wasFixed: false, + blockId: block.id, + submitted: false, }); const ctrl = new AbortController(); @@ -300,6 +357,8 @@ async function processBlock( store, unresolvedIssues: stillUnfixed, wasFixed: true, + blockId: block.id, + submitted: false, }); } catch (err) { if (err instanceof DOMException && err.name === 'AbortError') return; @@ -312,6 +371,8 @@ async function processBlock( store, unresolvedIssues: unfixed, wasFixed: false, + blockId: block.id, + submitted: false, }); } catch { setState({ @@ -320,6 +381,8 @@ async function processBlock( store: null, unresolvedIssues: unfixed, wasFixed: false, + blockId: block.id, + submitted: false, }); } } diff --git a/demo/src/styles.css b/demo/src/styles.css index 8652455..ad42306 100644 --- a/demo/src/styles.css +++ b/demo/src/styles.css @@ -5535,6 +5535,129 @@ body { color: #6b7280; } +/* ===== Preview-specific callout component ===== + The agent demo's default callout uses `.custom-callout*` markup which + was already styled across the app. We render a dedicated callout in + the Preview pane (`.preview-callout*`) so we can give it a distinct, + confident look without affecting the other views. */ +.preview-layout .preview-callout { + position: relative; + display: flex; + gap: 14px; + padding: 18px 22px; + margin-bottom: 16px; + border-radius: 12px; + border: 1px solid transparent; + border-left-width: 5px; + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.04); +} +.preview-layout .preview-callout-icon { + font-size: 22px; + line-height: 1.2; + flex-shrink: 0; +} +.preview-layout .preview-callout-body { + flex: 1; + min-width: 0; +} +.preview-layout .preview-callout-title { + font-size: 16px; + font-weight: 700; + margin-bottom: 6px; +} +.preview-layout .preview-callout-content { + margin: 0; + color: #1f2937; + font-size: 14px; + line-height: 1.6; +} +.preview-layout .preview-callout-dismiss { + position: absolute; + top: 10px; + right: 12px; + border: none; + background: transparent; + font-size: 18px; + color: #6b7280; + cursor: pointer; +} +.preview-layout .preview-callout-dismiss:hover { + color: #111827; +} + +.preview-layout .preview-callout--info { + background: linear-gradient(180deg, #eff6ff 0%, #dbeafe 100%); + border-color: #bfdbfe; + border-left-color: #2563eb; +} +.preview-layout .preview-callout--info .preview-callout-title { + color: #1e3a8a; +} +.preview-layout .preview-callout--success { + background: linear-gradient(180deg, #ecfdf5 0%, #d1fae5 100%); + border-color: #a7f3d0; + border-left-color: #059669; +} +.preview-layout .preview-callout--success .preview-callout-title { + color: #065f46; +} +.preview-layout .preview-callout--warning { + background: linear-gradient(180deg, #fffbeb 0%, #fef3c7 100%); + border-color: #fde68a; + border-left-color: #d97706; +} +.preview-layout .preview-callout--warning .preview-callout-title { + color: #92400e; +} +.preview-layout .preview-callout--error { + background: linear-gradient(180deg, #fef2f2 0%, #fee2e2 100%); + border-color: #fecaca; + border-left-color: #dc2626; +} +.preview-layout .preview-callout--error .preview-callout-title { + color: #991b1b; +} + +/* "Submitted" status chip + read-only overlay for revisiting a past step. */ +.preview-layout .preview-pane-status--submitted { + background: #e0e7ff; + color: #3730a3; +} +.preview-layout .preview-pane-note--submitted { + background: #eef2ff; + color: #3730a3; + border: 1px solid #c7d2fe; + margin-bottom: 14px; + padding: 10px 14px; + border-radius: 8px; + font-size: 12px; + line-height: 1.5; +} +.preview-layout .preview-pane-locked { + position: relative; + pointer-events: none; + opacity: 0.62; + filter: grayscale(0.2); +} + +/* Clickable / active variants of the compact tool_use chip in the chat. */ +.preview-layout .agent-tool-call--clickable { + cursor: pointer; + transition: background-color 0.12s ease, border-color 0.12s ease; +} +.preview-layout .agent-tool-call--clickable:hover { + border-color: #6c5ce7; + background: #faf5ff; +} +.preview-layout .agent-tool-call--active { + border-color: #6c5ce7; + background: #ede9fe; +} +.preview-layout .agent-tool-call--active .agent-tool-streaming { + color: #6c5ce7; + font-weight: 600; +} + /* Validation / fixer status notes shown above the rendered MDMA. */ .preview-layout .preview-pane-note { margin-bottom: 14px; From 6173a7f7efc7dc4a5141160e3d1b6728c10637a7 Mon Sep 17 00:00:00 2001 From: gitsad Date: Thu, 21 May 2026 13:12:46 +0200 Subject: [PATCH 20/26] feat: working preview on all models --- demo/src/PreviewView.tsx | 35 +- demo/src/agent/use-agent.ts | 231 +++++++++++-- demo/src/preview/BackendLogPane.tsx | 6 - demo/src/preview/PreviewPanel.tsx | 36 +- demo/src/preview/insurance-backend.ts | 57 +--- demo/src/preview/insurance-flow-prompt.ts | 40 ++- demo/src/preview/preview-customizations.tsx | 13 - demo/src/preview/use-insurance-flow.ts | 152 +++++---- demo/src/preview/use-preview-validation.ts | 351 ++++++++------------ demo/src/preview/use-submission-log.ts | 5 - 10 files changed, 494 insertions(+), 432 deletions(-) diff --git a/demo/src/PreviewView.tsx b/demo/src/PreviewView.tsx index 67ab28f..16d234d 100644 --- a/demo/src/PreviewView.tsx +++ b/demo/src/PreviewView.tsx @@ -5,10 +5,20 @@ import { AgentSettings } from './agent/AgentSettings.js'; import { ChatInput } from './chat/ChatInput.js'; import { BackendLogDrawer } from './preview/BackendLogPane.js'; import { PreviewPanel } from './preview/PreviewPanel.js'; +import { clearSubmissionLog } from './preview/insurance-backend.js'; import { INSURANCE_FLOW_PROMPT } from './preview/insurance-flow-prompt.js'; import { useInsuranceFlow } from './preview/use-insurance-flow.js'; import { usePreviewValidation } from './preview/use-preview-validation.js'; +function countToolUseBlocks(turns: ReturnType['turns']): number { + let count = 0; + for (const turn of turns) { + if (turn.role !== 'assistant') continue; + for (const block of turn.blocks) if (block.type === 'tool_use') count++; + } + return count; +} + export function PreviewView() { const { turns, @@ -23,11 +33,8 @@ export function PreviewView() { stop, clear, inputRef, - } = useAgent({ flowPrompt: INSURANCE_FLOW_PROMPT }); + } = useAgent({ flowPrompt: INSURANCE_FLOW_PROMPT, useAuthorSubAgent: true }); - // `selectedBlockId` controls which tool_use block the preview pane renders. - // null = follow the latest one. When the agent emits a new tool_use block, - // we snap back to "latest" so the new step shows automatically. const [selectedBlockId, setSelectedBlockId] = useState(null); const previewState = usePreviewValidation({ @@ -36,30 +43,23 @@ export function PreviewView() { agentConfig: config, }); - useInsuranceFlow({ + const insuranceFlow = useInsuranceFlow({ currentStore: previewState.store, sendHidden, isGenerating, }); - // Snap back to "latest" whenever a new tool_use block appears. Reading - // `turns` inside (not just .length) satisfies the deps lint while still - // only resetting on genuinely new content — selection survives interim - // streaming updates. + // Snap back to the latest step whenever a new tool_use block appears so + // the user doesn't get stuck viewing the previous step. const prevToolUseCountRef = useRef(0); useEffect(() => { - let count = 0; - for (const turn of turns) { - if (turn.role !== 'assistant') continue; - for (const block of turn.blocks) if (block.type === 'tool_use') count++; - } + const count = countToolUseBlocks(turns); if (count > prevToolUseCountRef.current) setSelectedBlockId(null); prevToolUseCountRef.current = count; }, [turns]); const chatEndRef = useRef(null); const prevCountRef = useRef(turns.length); - useEffect(() => { if (turns.length > prevCountRef.current) { chatEndRef.current?.scrollIntoView({ behavior: 'smooth' }); @@ -69,7 +69,10 @@ export function PreviewView() { const handleClear = useCallback(() => { clear(); - }, [clear]); + setSelectedBlockId(null); + clearSubmissionLog(); + insuranceFlow.reset(); + }, [clear, insuranceFlow]); return (
diff --git a/demo/src/agent/use-agent.ts b/demo/src/agent/use-agent.ts index b610783..77b3662 100644 --- a/demo/src/agent/use-agent.ts +++ b/demo/src/agent/use-agent.ts @@ -20,9 +20,9 @@ import { parseMarkdown } from '../chat/parse-markdown.js'; import { getDefaultPromptVariantForModel } from '../model-prompt-variant.js'; import type { AgentDisplayTurn, AssistantTurn, AgentBlock } from './types.js'; -// ── Tool definition ────────────────────────────────────────────────────────── +// ── Tool definitions ───────────────────────────────────────────────────────── -const GENERATE_MDMA_TOOL = { +const GENERATE_MDMA_TOOL_INLINE = { name: 'generate_mdma', description: 'Generate an MDMA Markdown document to present structured interactive content to the user. ' + @@ -40,6 +40,31 @@ const GENERATE_MDMA_TOOL = { }, }; +// Sub-agent mode: the conversation agent describes the intent; a separate +// author sub-agent (same model + provider) produces the actual MDMA. The +// conversation agent never writes raw MDMA into its visible response. +const GENERATE_MDMA_TOOL_BRIEF = { + name: 'generate_mdma', + description: + 'Request the MDMA Author (a specialised sub-agent) to generate an interactive MDMA component ' + + 'for the user. Provide a clear brief describing what to generate — component type, id, fields, ' + + "labels, action labels (onSubmit etc.), and any constraints. Do NOT write MDMA Markdown yourself; " + + 'the author will produce the final document and render it on the user’s screen.', + input_schema: { + type: 'object' as const, + properties: { + brief: { + type: 'string', + description: + 'A natural-language description of the MDMA component(s) to generate. Include the ' + + 'component type, id, every field with its label/type, required/sensitive flags, ' + + 'onSubmit / onAction labels, and any other constraints. Do not include MDMA syntax.', + }, + }, + required: ['brief'], + }, +}; + // ── Config persistence ─────────────────────────────────────────────────────── const CONFIG_KEY = 'mdma-agent-config'; @@ -133,6 +158,94 @@ interface BlockMeta { partialJson?: string; } +// ── Sub-agent author dispatch ──────────────────────────────────────────────── + +type AuthorSubAgent = (brief: string, signal: AbortSignal) => Promise; + +async function callAuthorAnthropic( + config: AnthropicConfig, + authorPrompt: string, + brief: string, + signal: AbortSignal, +): Promise { + const response = await fetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'x-api-key': config.apiKey, + 'anthropic-version': '2023-06-01', + 'anthropic-dangerous-direct-browser-access': 'true', + }, + body: JSON.stringify({ + model: config.model, + max_tokens: 8192, + system: authorPrompt, + messages: [{ role: 'user', content: brief }], + }), + signal, + }); + if (!response.ok) { + throw new Error(`Author sub-agent failed (${response.status}): ${await response.text()}`); + } + const json = (await response.json()) as { content?: Array<{ type: string; text?: string }> }; + return (json.content ?? []) + .filter( + (b): b is { type: 'text'; text: string } => b.type === 'text' && typeof b.text === 'string', + ) + .map((b) => b.text) + .join(''); +} + +async function callAuthorOpenAI( + config: AnthropicConfig, + authorPrompt: string, + brief: string, + signal: AbortSignal, +): Promise { + const provider = config.provider ?? 'openai'; + const baseUrl = OPENAI_COMPAT_BASE_URLS[provider] ?? OPENAI_COMPAT_BASE_URLS.openai!; + const apiKey = getApiKeyForProvider(config); + const response = await fetch(`${baseUrl}/chat/completions`, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${apiKey}` }, + body: JSON.stringify({ + model: config.model, + messages: [ + { role: 'system', content: authorPrompt }, + { role: 'user', content: brief }, + ], + }), + signal, + }); + if (!response.ok) { + throw new Error(`Author sub-agent failed (${response.status}): ${await response.text()}`); + } + const json = (await response.json()) as { + choices?: Array<{ message?: { content?: string } }>; + }; + return json.choices?.[0]?.message?.content ?? ''; +} + +function makeAuthorSubAgent(config: AnthropicConfig): AuthorSubAgent { + const authorPrompt = getAuthorPromptVariant(config.systemPromptId).prompt; + const provider = config.provider ?? 'anthropic'; + return (brief, signal) => + provider === 'anthropic' + ? callAuthorAnthropic(config, authorPrompt, brief, signal) + : callAuthorOpenAI(config, authorPrompt, brief, signal); +} + +// The author sub-agent occasionally wraps its entire response in an outer +// ```markdown / ```md fence (the "treat the answer as a code block" failure +// mode). If so, peel that outer wrapper off. NEVER strip ```mdma fences — +// those are the document's actual component markers and removing them +// would collapse a multi-block document into a single bare YAML snippet. +function extractDocumentFromBrief(rawText: string): string { + const trimmed = rawText.trim(); + const outer = trimmed.match(/^```(?:markdown|md)\s*\n([\s\S]*)\n```\s*$/); + return outer ? outer[1] : rawText; +} + // ── Anthropic agentic loop ──────────────────────────────────────────────────── async function runAgentLoop( @@ -144,7 +257,9 @@ async function runAgentLoop( setTurns: Dispatch>, onError: (msg: string) => void, nextId: () => string, + subAgent: AuthorSubAgent | null, ): Promise { + const tool = subAgent ? GENERATE_MDMA_TOOL_BRIEF : GENERATE_MDMA_TOOL_INLINE; let continueLoop = true; while (continueLoop && !signal.aborted) { @@ -156,7 +271,7 @@ async function runAgentLoop( config, systemPrompt, history, - [GENERATE_MDMA_TOOL], + [tool], signal, )) { if (ev.type === 'stream_error') { @@ -255,14 +370,28 @@ async function runAgentLoop( if (!meta) continue; if (meta.partialJson !== undefined) { - let document = ''; + let parsedInput: { document?: string; brief?: string } = {}; try { - const parsed = JSON.parse(meta.partialJson) as { document?: string }; - document = parsed.document ?? ''; + parsedInput = JSON.parse(meta.partialJson); } catch { - document = meta.partialJson; + parsedInput = subAgent ? { brief: meta.partialJson } : { document: meta.partialJson }; + } + + let document: string; + if (subAgent) { + const brief = parsedInput.brief ?? ''; + try { + const raw = await subAgent(brief, signal); + document = extractDocumentFromBrief(raw); + } catch (err) { + onError(err instanceof Error ? err.message : String(err)); + document = ''; + } + if (meta.apiBlock.type === 'tool_use') meta.apiBlock.input = { brief }; + } else { + document = parsedInput.document ?? ''; + if (meta.apiBlock.type === 'tool_use') meta.apiBlock.input = { document }; } - if (meta.apiBlock.type === 'tool_use') meta.apiBlock.input = { document }; const parsed = await parseMarkdown(document).catch(() => null); const ast = parsed?.ast ?? null; @@ -331,10 +460,12 @@ async function runOpenAIAgentLoop( setTurns: Dispatch>, onError: (msg: string) => void, nextId: () => string, + subAgent: AuthorSubAgent | null, ): Promise { const baseUrl = OPENAI_COMPAT_BASE_URLS[config.provider ?? 'openai'] ?? OPENAI_COMPAT_BASE_URLS.openai!; const apiKey = getApiKeyForProvider(config); + const tool = subAgent ? GENERATE_MDMA_TOOL_BRIEF : GENERATE_MDMA_TOOL_INLINE; let continueLoop = true; while (continueLoop && !signal.aborted) { @@ -349,7 +480,7 @@ async function runOpenAIAgentLoop( config.model, systemPrompt, history, - [GENERATE_MDMA_TOOL], + [tool], signal, baseUrl, )) { @@ -417,20 +548,41 @@ async function runOpenAIAgentLoop( if (!meta) continue; if (meta.partialJson !== undefined) { - let document = ''; + let parsedInput: { document?: string; brief?: string } = {}; try { - const parsed = JSON.parse(meta.partialJson) as { document?: string }; - document = parsed.document ?? ''; + parsedInput = JSON.parse(meta.partialJson); } catch { - document = meta.partialJson; + parsedInput = subAgent ? { brief: meta.partialJson } : { document: meta.partialJson }; } - if (meta.apiBlock.type === 'tool_use') { - meta.apiBlock.input = { document }; - finishedToolCalls.push({ - id: meta.apiBlock.id, - name: meta.apiBlock.name, - arguments: meta.partialJson, - }); + + let document: string; + if (subAgent) { + const brief = parsedInput.brief ?? ''; + try { + const raw = await subAgent(brief, signal); + document = extractDocumentFromBrief(raw); + } catch (err) { + onError(err instanceof Error ? err.message : String(err)); + document = ''; + } + if (meta.apiBlock.type === 'tool_use') { + meta.apiBlock.input = { brief }; + finishedToolCalls.push({ + id: meta.apiBlock.id, + name: meta.apiBlock.name, + arguments: meta.partialJson, + }); + } + } else { + document = parsedInput.document ?? ''; + if (meta.apiBlock.type === 'tool_use') { + meta.apiBlock.input = { document }; + finishedToolCalls.push({ + id: meta.apiBlock.id, + name: meta.apiBlock.name, + arguments: meta.partialJson, + }); + } } const parsed = await parseMarkdown(document).catch(() => null); @@ -455,9 +607,12 @@ async function runOpenAIAgentLoop( if (signal.aborted) break; // Push OpenAI-formatted assistant message + // OpenAI's Chat Completions endpoint rejects `content: null` even when + // tool_calls are present (despite the spec allowing it). Emit "" so a + // tool-only assistant turn stays valid in the history. const assistantMsg: OpenAIAssistantMessage = { role: 'assistant', - content: finishedTextContent || null, + content: finishedTextContent || '', }; if (finishedToolCalls.length > 0) { assistantMsg.tool_calls = finishedToolCalls.map((tc) => ({ @@ -515,6 +670,13 @@ function patchBlock( // ── Hook ───────────────────────────────────────────────────────────────────── export interface UseAgentOptions { + /** + * When true, the conversation agent never writes raw MDMA. Instead, the + * `generate_mdma` tool takes a high-level `brief` and a separate author + * sub-agent (same model + provider) produces the actual document. Keeps + * MDMA generation out of the chat surface. + */ + useAuthorSubAgent?: boolean; /** * Extra flow-definition text appended to the agent's customPrompt. Used by * the Insurance Preview to lock the conversation to a specific 4-step @@ -594,14 +756,21 @@ export function useAgent(options: UseAgentOptions = {}) { abortRef.current = new AbortController(); const toolPrompt = getAgentToolPromptVariant(config.systemPromptId).prompt; - const customPrompt = options.flowPrompt - ? `${toolPrompt}\n\n---\n\n${options.flowPrompt}` - : toolPrompt; - const systemPrompt = buildSystemPrompt({ - authorPrompt: getAuthorPromptVariant(config.systemPromptId).prompt, - customPrompt, - }); - + // In sub-agent mode the conversation agent never writes MDMA directly, + // so its system prompt omits the author prompt and the buildSystemPrompt + // reminder (both of which would tempt the agent to inline MDMA in chat). + const systemPrompt = options.useAuthorSubAgent + ? options.flowPrompt + ? `${toolPrompt}\n\n---\n\n${options.flowPrompt}` + : toolPrompt + : buildSystemPrompt({ + authorPrompt: getAuthorPromptVariant(config.systemPromptId).prompt, + customPrompt: options.flowPrompt + ? `${toolPrompt}\n\n---\n\n${options.flowPrompt}` + : toolPrompt, + }); + + const subAgent = options.useAuthorSubAgent ? makeAuthorSubAgent(config) : null; const provider = config.provider ?? 'anthropic'; try { @@ -619,6 +788,7 @@ export function useAgent(options: UseAgentOptions = {}) { setTurns, setError, nextId, + subAgent, ); apiHistoryRef.current = history; } else { @@ -632,6 +802,7 @@ export function useAgent(options: UseAgentOptions = {}) { setTurns, setError, nextId, + subAgent, ); openaiHistoryRef.current = history; } @@ -645,7 +816,7 @@ export function useAgent(options: UseAgentOptions = {}) { inputRef.current?.focus(); } }, - [config, isGenerating, nextId, options.flowPrompt], + [config, isGenerating, nextId, options.flowPrompt, options.useAuthorSubAgent], ); const send = useCallback(async () => { diff --git a/demo/src/preview/BackendLogPane.tsx b/demo/src/preview/BackendLogPane.tsx index 695c9a2..bef5545 100644 --- a/demo/src/preview/BackendLogPane.tsx +++ b/demo/src/preview/BackendLogPane.tsx @@ -16,12 +16,6 @@ function formatTime(d: Date): string { }); } -/** - * Floating toggle + slide-out drawer on the right edge of the Preview - * page. Lives at the layout root (not inside `PreviewPanel`) so the log - * doesn't share scroll/space with the rendered MDMA — the demo audience - * can pop it open at any time to see the masked submissions land. - */ export function BackendLogDrawer() { const entries = useSubmissionLog(); const [open, setOpen] = useState(false); diff --git a/demo/src/preview/PreviewPanel.tsx b/demo/src/preview/PreviewPanel.tsx index b0594fe..30566b6 100644 --- a/demo/src/preview/PreviewPanel.tsx +++ b/demo/src/preview/PreviewPanel.tsx @@ -26,6 +26,22 @@ export function PreviewPanel({ state }: PreviewPanelProps) { const { status, ast, store, unresolvedIssues, wasFixed, submitted } = state; const showRender = ast !== null && store !== null; + const placeholder = + !showRender && status === 'idle' + ? { + title: 'Insurance claim flow', + hint: "Start the chat on the left. As the agent emits MDMA blocks, they'll be validated, auto-fixed if needed, and rendered here.", + } + : status === 'validating' || (status === 'fixing' && !showRender) + ? { + title: status === 'validating' ? 'Validating…' : 'Fixing with LLM…', + hint: + status === 'validating' + ? "Checking the agent's MDMA against the spec." + : "Calling the LLM fixer to repair the agent's output before rendering.", + } + : null; + return (
@@ -38,24 +54,10 @@ export function PreviewPanel({ state }: PreviewPanelProps) {
- {status === 'idle' && !showRender ? ( + {placeholder ? (
-

Insurance claim flow

-

- Start the chat on the left. As the agent emits MDMA blocks, they'll be validated, - auto-fixed if needed, and rendered here. -

-
- ) : status === 'validating' || (status === 'fixing' && !showRender) ? ( -
-

- {status === 'validating' ? 'Validating…' : 'Fixing with LLM…'} -

-

- {status === 'validating' - ? "Checking the agent's MDMA against the spec." - : "Calling the LLM fixer to repair the agent's output before rendering."} -

+

{placeholder.title}

+

{placeholder.hint}

) : ( <> diff --git a/demo/src/preview/insurance-backend.ts b/demo/src/preview/insurance-backend.ts index 3c15d17..8526813 100644 --- a/demo/src/preview/insurance-backend.ts +++ b/demo/src/preview/insurance-backend.ts @@ -1,11 +1,3 @@ -/** - * Mock backend for the Insurance Preview demo. Each function pretends to be - * an endpoint of the insurance provider's API: validates a tiny shape, - * waits a few hundred ms, and resolves with a fake server response. No - * data leaves the browser — values land in the in-memory `submissionLog`, - * which the optional debug pane on the right column displays. - */ - const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); function maskIban(iban: string): string { @@ -18,7 +10,6 @@ export interface SubmissionLogEntry { step: 'personal-info' | 'claim' | 'bank'; at: Date; claimId: string; - /** Display-only summary (sensitive values masked). Never raw user data. */ summary: string; } @@ -66,20 +57,21 @@ function makeClaimId(): string { return `clm_${Math.random().toString(36).slice(2, 8)}`; } +function appendEntry(entry: SubmissionLogEntry): void { + submissionLog = [...submissionLog, entry]; + notify(); +} + export const insuranceBackend = { async collectPersonalInfo(payload: PersonalInfoPayload): Promise { await delay(700); const claimId = makeClaimId(); - submissionLog = [ - ...submissionLog, - { - step: 'personal-info', - at: new Date(), - claimId, - summary: `${payload['full-name']} (DOB ${payload.birthday})`, - }, - ]; - notify(); + appendEntry({ + step: 'personal-info', + at: new Date(), + claimId, + summary: `${payload['full-name']} (DOB ${payload.birthday})`, + }); return { claimId, accepted: true }; }, @@ -87,31 +79,18 @@ export const insuranceBackend = { await delay(800); const desc = payload['claim-description']; const preview = desc.length > 60 ? `${desc.slice(0, 60)}…` : desc; - submissionLog = [ - ...submissionLog, - { - step: 'claim', - at: new Date(), - claimId, - summary: `"${preview}"`, - }, - ]; - notify(); + appendEntry({ step: 'claim', at: new Date(), claimId, summary: `"${preview}"` }); return { accepted: true }; }, async collectBank(claimId: string, payload: BankPayload): Promise { await delay(700); - submissionLog = [ - ...submissionLog, - { - step: 'bank', - at: new Date(), - claimId, - summary: `IBAN ${maskIban(payload.iban)}`, - }, - ]; - notify(); + appendEntry({ + step: 'bank', + at: new Date(), + claimId, + summary: `IBAN ${maskIban(payload.iban)}`, + }); return { accepted: true, etaDays: 5 }; }, }; diff --git a/demo/src/preview/insurance-flow-prompt.ts b/demo/src/preview/insurance-flow-prompt.ts index d99a90b..c49d64a 100644 --- a/demo/src/preview/insurance-flow-prompt.ts +++ b/demo/src/preview/insurance-flow-prompt.ts @@ -1,36 +1,34 @@ -/** - * Insurance claim flow — locked custom prompt for the Preview page. - * - * Defines a 4-message conversation: gather personal info, then claim - * description, then bank account for receiving funds, then a final - * confirmation callout. Each interactive step is a single MDMA component - * per assistant turn (one form / one callout) — matches the rules the - * conversation-flow eval enforces. - */ export const INSURANCE_FLOW_PROMPT = `## Insurance Claim Intake Flow -You are a friendly claims assistant for **MDMA Mutual Insurance**. Walk the user through filing a new claim across exactly four assistant turns, one interactive MDMA component per turn. Use a warm, plain-language tone. +You are a friendly claims assistant for **MDMA Mutual Insurance**. Walk the user through filing a new claim across exactly four assistant turns. In each of these four turns you call the \`generate_mdma\` tool **once** to produce that turn's interactive component. Use a warm, plain-language tone in your visible text. ### Step 1 — Personal info -First assistant turn. Emit a single \`form\` component with id \`personal-info-form\` and \`onSubmit: collect-personal-info\`. Fields: +First assistant turn. Call \`generate_mdma\` with a brief that describes a single \`form\` component, id \`personal-info-form\`, \`onSubmit: collect-personal-info\`. Fields: - \`full-name\` (text, required, label "Full name") - \`birthday\` (date, required, label "Date of birth") ### Step 2 — Claim description -Second assistant turn (after the user submits personal info). Emit a single \`form\` component with id \`claim-description-form\` and \`onSubmit: collect-claim\`. Fields: +Second assistant turn (after the user submits personal info). Call \`generate_mdma\` with a brief that describes a single \`form\` component, id \`claim-description-form\`, \`onSubmit: collect-claim\`. Fields: - \`claim-description\` (textarea, required, label "What happened?") ### Step 3 — Bank account -Third assistant turn (after the user submits the claim description). Emit a single \`form\` component with id \`bank-account-form\` and \`onSubmit: collect-bank\`. Fields: +Third assistant turn (after the user submits the claim description). Call \`generate_mdma\` with a brief that describes a single \`form\` component, id \`bank-account-form\`, \`onSubmit: collect-bank\`. Fields: - \`iban\` (text, required, sensitive: true, label "IBAN where we should send the funds") ### Step 4 — Confirmation -Fourth assistant turn (after the user submits the bank account). Emit a single \`callout\` component with id \`claim-submitted-callout\`, \`variant: success\`, \`title: "Claim received"\`, and a friendly \`content\` explaining the claim will be processed by an insurance specialist within a few business days. No further interactive components — the flow ends here. - -### Rules -- One interactive component (\`form\`) per assistant turn for steps 1–3. Step 4 is a non-interactive \`callout\`. -- Use the **exact** ids and \`onSubmit\` action labels listed above. -- Don't regenerate previously-shown components in later turns. -- Don't add components beyond what each step requires (no extra callouts, buttons, or webhooks). -- It's fine to precede a step's form with a short plain-text intro sentence, but do not emit any other MDMA component types. +Fourth assistant turn (after the user submits the bank account). Call \`generate_mdma\` with a brief that describes a single \`callout\` component, id \`claim-submitted-callout\`, \`variant: success\`, \`title: "Claim received"\`, and a friendly \`content\` explaining the claim will be processed by an insurance specialist within a few business days. No further interactive components — the flow ends here. + +### Visible text +Your visible text is plain, warm conversation — a short sentence or two introducing each step or answering the user's question. The interactive component itself is rendered by the \`generate_mdma\` tool; your text just sets the tone alongside it. + +### When to advance to the next step +Step advancement is **driven by system messages**, not by user chat. After the user submits a form, you will receive a message that starts with \`[system]\` (sent on the user's behalf) confirming the submission and naming the next step to emit, e.g.: + +> \`[system] The user submitted the personal-info form and the backend accepted it (claim id: clm_abc123). Proceed to step 2 by emitting the claim description form.\` + +Rules: +- Only call \`generate_mdma\` for step **N+1** after you have seen a \`[system]\` message instructing you to proceed to step N+1. +- The very first assistant turn is the exception — emit step 1 immediately on the first user message, no \`[system]\` message required. +- If the user chats between steps ("is this it?", "what about my address?", "ok thanks", etc.), they are still on the current step. Answer in plain conversation only and **wait** for the \`[system]\` advance message before calling the tool again. +- Use the **exact** ids and \`onSubmit\` action labels listed above. Don't regenerate previously-shown components. Don't add extras (no buttons, webhooks, callouts beyond what each step requires). `; diff --git a/demo/src/preview/preview-customizations.tsx b/demo/src/preview/preview-customizations.tsx index 8a916a3..8166331 100644 --- a/demo/src/preview/preview-customizations.tsx +++ b/demo/src/preview/preview-customizations.tsx @@ -3,14 +3,6 @@ import type { MdmaBlockRendererProps } from '@mobile-reality/mdma-renderer-react import type { MdmaCustomizations } from '../ChatView.js'; import { customizations as baseCustomizations } from '../custom-components.js'; -/** - * Preview-pane-specific callout renderer. The base `CustomCalloutRenderer` - * is used across the demo, so changing it would affect Agent Chat and the - * other views. This renderer emits its own `.preview-callout` markup and - * is wired only via `previewCustomizations` below, keeping the polished - * look local to the Insurance Preview page. - */ - const VARIANT_ICONS: Record = { info: 'ℹ️', warning: '⚠️', @@ -63,11 +55,6 @@ const PreviewCalloutRenderer = memo(function PreviewCalloutRenderer({ ); }); -/** - * Same as the base demo customizations but with the callout swapped out - * for the preview-specific renderer. Forms, buttons, charts, etc. keep - * the existing custom styling. - */ export const previewCustomizations: MdmaCustomizations = { ...baseCustomizations, components: { diff --git a/demo/src/preview/use-insurance-flow.ts b/demo/src/preview/use-insurance-flow.ts index 1895cd6..03b22b1 100644 --- a/demo/src/preview/use-insurance-flow.ts +++ b/demo/src/preview/use-insurance-flow.ts @@ -1,44 +1,65 @@ import { useEffect, useRef } from 'react'; import type { DocumentStore } from '@mobile-reality/mdma-runtime'; -import { - insuranceBackend, - type BankPayload, - type ClaimPayload, - type PersonalInfoPayload, -} from './insurance-backend.js'; +import { insuranceBackend } from './insurance-backend.js'; interface UseInsuranceFlowOptions { - /** - * The store currently rendered in the preview pane (validated/fixed - * output, NOT the agent's raw block.store). When the user clicks Submit - * in the right pane, the ACTION_TRIGGERED event fires on this store, so - * the hook must subscribe to *this* store — earlier versions subscribed - * to block.store and silently missed every submit. - */ currentStore: DocumentStore | null; sendHidden: (message: string) => Promise; isGenerating: boolean; } -const ACTION_IDS = ['collect-personal-info', 'collect-claim', 'collect-bank'] as const; -type ActionId = (typeof ACTION_IDS)[number]; +type ActionId = 'collect-personal-info' | 'collect-claim' | 'collect-bank'; -function isHandledActionId(id: string): id is ActionId { - return (ACTION_IDS as readonly string[]).includes(id); +interface StepDispatcher { + call: ( + values: Record, + claimId: string | null, + ) => Promise<{ claimId?: string; message: string }>; + requiresClaimId: boolean; +} + +const STEPS: Record = { + 'collect-personal-info': { + requiresClaimId: false, + async call(values) { + const result = await insuranceBackend.collectPersonalInfo({ + 'full-name': String(values['full-name'] ?? ''), + birthday: String(values.birthday ?? ''), + }); + return { + claimId: result.claimId, + message: `[system] The user submitted the personal-info form and the backend accepted it (claim id: ${result.claimId}). Proceed to step 2 by emitting the claim description form.`, + }; + }, + }, + 'collect-claim': { + requiresClaimId: true, + async call(values, claimId) { + await insuranceBackend.collectClaim(claimId!, { + 'claim-description': String(values['claim-description'] ?? ''), + }); + return { + message: `[system] The user submitted the claim description and the backend accepted it (claim id: ${claimId}). Proceed to step 3 by emitting the bank-account form.`, + }; + }, + }, + 'collect-bank': { + requiresClaimId: true, + async call(values, claimId) { + const result = await insuranceBackend.collectBank(claimId!, { + iban: String(values.iban ?? ''), + }); + return { + message: `[system] The user submitted the bank-account form and the backend accepted it (claim id: ${claimId}, funds ETA: ${result.etaDays} business days). Proceed to step 4 by emitting the final success callout.`, + }; + }, + }, +}; + +function isActionId(id: string): id is ActionId { + return id in STEPS; } -/** - * Drives the Insurance Preview flow: - * - * 1. Subscribes to `ACTION_TRIGGERED` on whatever store is currently being - * rendered in the preview pane. - * 2. When a known `actionId` fires, pulls the submitted values from that - * same store, calls the mock backend, and waits for success. - * 3. On success, sends a HIDDEN user message to the agent — no form data, - * just a "step N done, please continue" signal. - * - * The claim id from step 1 is threaded into steps 2 + 3 via a ref. - */ export function useInsuranceFlow({ currentStore, sendHidden, @@ -59,65 +80,40 @@ export function useInsuranceFlow({ currentStore.getEventBus().on('ACTION_TRIGGERED', (action) => { if (isGeneratingRef.current) return; const { actionId, componentId } = action; - if (!isHandledActionId(actionId)) return; + if (!isActionId(actionId)) return; const key = `${componentId}:${actionId}`; if (handledActions.current.has(key)) return; handledActions.current.add(key); + const step = STEPS[actionId]; + if (step.requiresClaimId && !claimIdRef.current) { + console.warn(`[insurance-flow] ${actionId} fired before claim id was available`); + return; + } + const values = (currentStore.getComponentState(componentId)?.values ?? {}) as Record< string, unknown >; - void dispatch(actionId, values).catch((err) => { - handledActions.current.delete(key); - console.error('[insurance-flow] backend call failed', err); - }); + + step + .call(values, claimIdRef.current) + .then(async (result) => { + if (result.claimId) claimIdRef.current = result.claimId; + await sendHiddenRef.current(result.message); + }) + .catch((err) => { + handledActions.current.delete(key); + console.error('[insurance-flow] backend call failed', err); + }); }); }, [currentStore]); - async function dispatch(actionId: ActionId, values: Record) { - if (actionId === 'collect-personal-info') { - const payload: PersonalInfoPayload = { - 'full-name': String(values['full-name'] ?? ''), - birthday: String(values.birthday ?? ''), - }; - const result = await insuranceBackend.collectPersonalInfo(payload); - claimIdRef.current = result.claimId; - await sendHiddenRef.current( - `[system] The user submitted the personal-info form and the backend accepted it (claim id: ${result.claimId}). Proceed to step 2 by emitting the claim description form.`, - ); - return; - } - - if (actionId === 'collect-claim') { - const claimId = claimIdRef.current; - if (!claimId) { - console.warn('[insurance-flow] collect-claim fired before claim id was available'); - return; - } - const payload: ClaimPayload = { - 'claim-description': String(values['claim-description'] ?? ''), - }; - await insuranceBackend.collectClaim(claimId, payload); - await sendHiddenRef.current( - `[system] The user submitted the claim description and the backend accepted it (claim id: ${claimId}). Proceed to step 3 by emitting the bank-account form.`, - ); - return; - } - - if (actionId === 'collect-bank') { - const claimId = claimIdRef.current; - if (!claimId) { - console.warn('[insurance-flow] collect-bank fired before claim id was available'); - return; - } - const payload: BankPayload = { iban: String(values.iban ?? '') }; - const result = await insuranceBackend.collectBank(claimId, payload); - await sendHiddenRef.current( - `[system] The user submitted the bank-account form and the backend accepted it (claim id: ${claimId}, funds ETA: ${result.etaDays} business days). Proceed to step 4 by emitting the final success callout.`, - ); - return; - } - } + return { + reset: () => { + handledActions.current.clear(); + claimIdRef.current = null; + }, + }; } diff --git a/demo/src/preview/use-preview-validation.ts b/demo/src/preview/use-preview-validation.ts index 5b5bd08..e552b07 100644 --- a/demo/src/preview/use-preview-validation.ts +++ b/demo/src/preview/use-preview-validation.ts @@ -3,6 +3,7 @@ import { validate, type ValidationIssue, type ValidationResult, + type ValidationRuleId, } from '@mobile-reality/mdma-validator'; import { buildFixerPrompt, @@ -24,27 +25,13 @@ export interface PreviewState { store: DocumentStore | null; unresolvedIssues: ValidationIssue[]; wasFixed: boolean; - /** Id of the block currently being rendered (the agent's tool_use block id). */ blockId: string | null; - /** - * True when the rendered block is from an earlier step than the latest - * one — i.e. it's already been submitted in the flow and re-interacting - * with it shouldn't happen. PreviewPanel uses this to disable inputs. - */ submitted: boolean; } interface UsePreviewValidationOptions { turns: AgentDisplayTurn[]; - /** - * When set, show this specific tool_use block. When null, show the latest. - */ selectedBlockId: string | null; - /** - * Same config the agent uses. The fixer picks its credentials + model - * from this — anthropic provider → haiku via x-api-key, openai → gpt-4.1-mini, - * openrouter → anthropic/claude-haiku-4-5 via openrouter. - */ agentConfig: AnthropicConfig; } @@ -58,23 +45,13 @@ const INITIAL_STATE: PreviewState = { submitted: false, }; +const EXCLUDE_RULES: ValidationRuleId[] = ['thinking-block', 'flow-ordering']; +const VALIDATE_OPTIONS = { exclude: EXCLUDE_RULES }; + type FixerResolution = - | { - kind: 'anthropic'; - apiKey: string; - model: string; - } - | { - kind: 'openai-compatible'; - apiKey: string; - baseUrl: string; - model: string; - }; + | { kind: 'anthropic'; apiKey: string; model: string } + | { kind: 'openai-compatible'; apiKey: string; baseUrl: string; model: string }; -/** - * Picks the fixer endpoint + model based on the agent's current provider. - * Returns null when the relevant API key isn't configured. - */ function resolveFixer(config: AnthropicConfig): FixerResolution | null { const provider = config.provider ?? 'anthropic'; if (provider === 'anthropic') { @@ -102,44 +79,57 @@ function resolveFixer(config: AnthropicConfig): FixerResolution | null { return null; } -/** - * Non-streaming Anthropic Messages API call — used by the fixer when the - * agent provider is anthropic. Reuses the same direct-browser-access - * header the streaming agent client sets. - */ -async function anthropicFix( - apiKey: string, - model: string, - systemPrompt: string, - userMessage: string, +async function callFixer( + fixer: FixerResolution, + document: string, + unfixed: ValidationIssue[], signal: AbortSignal, ): Promise { - const response = await fetch('https://api.anthropic.com/v1/messages', { - method: 'POST', - headers: { - 'content-type': 'application/json', - 'x-api-key': apiKey, - 'anthropic-version': '2023-06-01', - 'anthropic-dangerous-direct-browser-access': 'true', - }, - body: JSON.stringify({ - model, - max_tokens: 4096, - system: systemPrompt, - messages: [{ role: 'user', content: userMessage }], - }), - signal, - }); - if (!response.ok) { - const body = await response.text(); - throw new Error(`Anthropic fixer failed (${response.status}): ${body}`); + const systemPrompt = `${buildSystemPrompt()}\n\n---\n\n${buildFixerPrompt('single-block')}`; + const userMessage = buildFixerMessage(document, unfixed); + + if (fixer.kind === 'anthropic') { + const response = await fetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'x-api-key': fixer.apiKey, + 'anthropic-version': '2023-06-01', + 'anthropic-dangerous-direct-browser-access': 'true', + }, + body: JSON.stringify({ + model: fixer.model, + max_tokens: 4096, + system: systemPrompt, + messages: [{ role: 'user', content: userMessage }], + }), + signal, + }); + if (!response.ok) { + throw new Error(`Anthropic fixer failed (${response.status}): ${await response.text()}`); + } + const json = (await response.json()) as { content?: Array<{ type: string; text?: string }> }; + return (json.content ?? []) + .filter( + (b): b is { type: 'text'; text: string } => b.type === 'text' && typeof b.text === 'string', + ) + .map((b) => b.text) + .join(''); } - const json = (await response.json()) as { content?: Array<{ type: string; text?: string }> }; - const text = (json.content ?? []) - .filter((block): block is { type: 'text'; text: string } => block.type === 'text' && typeof block.text === 'string') - .map((block) => block.text) - .join(''); - return text; + + const llmConfig: LlmConfig = { + baseUrl: fixer.baseUrl, + apiKey: fixer.apiKey, + model: fixer.model, + }; + return chatCompletion( + llmConfig, + [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: userMessage }, + ], + signal, + ); } function collectToolUseBlocks(turns: AgentDisplayTurn[]): ToolUseBlock[] { @@ -159,27 +149,47 @@ function resolveBlock( ): { block: ToolUseBlock | null; submitted: boolean } { const all = collectToolUseBlocks(turns); if (all.length === 0) return { block: null, submitted: false }; - if (!selectedBlockId) { - return { block: all[all.length - 1], submitted: false }; - } + if (!selectedBlockId) return { block: all[all.length - 1], submitted: false }; const idx = all.findIndex((b) => b.id === selectedBlockId); if (idx === -1) return { block: all[all.length - 1], submitted: false }; return { block: all[idx], submitted: idx < all.length - 1 }; } -/** - * Validates the latest assistant tool_use block's MDMA document and, if it - * fails validation, runs the LLM fixer (single-block scope) to repair it - * before rendering. The fixer model + credentials are picked from the - * agent's current provider (see resolveFixer). - */ +function getUnfixedIssues(result: ValidationResult): ValidationIssue[] { + return result.issues.filter( + (i) => !i.fixed && (i.severity === 'error' || i.severity === 'warning'), + ); +} + +function buildState( + blockId: string, + submitted: boolean, + status: PreviewStatus, + ast: MdmaRoot | null = null, + store: DocumentStore | null = null, + unresolvedIssues: ValidationIssue[] = [], + wasFixed = false, +): PreviewState { + return { status, ast, store, unresolvedIssues, wasFixed, blockId, submitted }; +} + +async function tryParse( + markdown: string, +): Promise<{ ast: MdmaRoot; store: DocumentStore } | null> { + try { + return await parseMarkdown(markdown); + } catch { + return null; + } +} + export function usePreviewValidation({ turns, selectedBlockId, agentConfig, }: UsePreviewValidationOptions): PreviewState { const [state, setState] = useState(INITIAL_STATE); - const handledRef = useRef(new Map()); + const cacheRef = useRef(new Map()); const inFlightRef = useRef(null); useEffect(() => { @@ -190,23 +200,12 @@ export function usePreviewValidation({ } if (block.isStreaming || !block.document) { - setState({ - status: 'validating', - ast: null, - store: null, - unresolvedIssues: [], - wasFixed: false, - blockId: block.id, - submitted, - }); + setState(buildState(block.id, submitted, 'validating')); return; } - // De-dupe on (blockId, doc length) so toggling the selection between - // already-processed blocks re-uses the cached PreviewState instead of - // re-running validation + fixer. - const handleKey = `${block.id}:${block.document.length}`; - const cached = handledRef.current.get(handleKey); + const cacheKey = `${block.id}:${block.document.length}`; + const cached = cacheRef.current.get(cacheKey); if (cached) { setState({ ...cached, submitted }); return; @@ -218,14 +217,13 @@ export function usePreviewValidation({ const fixer = resolveFixer(agentConfig); void processBlock( block, + submitted, fixer, (next) => { - const withFlags = { ...next, blockId: block.id, submitted }; - // Snapshot terminal states so revisits don't refire the LLM. if (next.status === 'ready' || next.status === 'invalid') { - handledRef.current.set(handleKey, withFlags); + cacheRef.current.set(cacheKey, next); } - setState(withFlags); + setState(next); }, (ctrl) => { inFlightRef.current = ctrl; @@ -236,7 +234,7 @@ export function usePreviewValidation({ const prevTurnCount = useRef(turns.length); useEffect(() => { if (prevTurnCount.current > 0 && turns.length === 0) { - handledRef.current.clear(); + cacheRef.current.clear(); inFlightRef.current?.abort(); inFlightRef.current = null; setState(INITIAL_STATE); @@ -249,141 +247,80 @@ export function usePreviewValidation({ async function processBlock( block: ToolUseBlock, + submitted: boolean, fixer: FixerResolution | null, setState: (state: PreviewState) => void, registerAbort: (ctrl: AbortController) => void, ): Promise { - setState({ - status: 'validating', - ast: null, - store: null, - unresolvedIssues: [], - wasFixed: false, - blockId: block.id, - submitted: false, - }); + setState(buildState(block.id, submitted, 'validating')); - const initial: ValidationResult = validate(block.document, { - exclude: ['thinking-block', 'flow-ordering'], - }); - const unfixed = initial.issues.filter( - (i) => !i.fixed && (i.severity === 'error' || i.severity === 'warning'), - ); + const initial = validate(block.document, VALIDATE_OPTIONS); + const unfixed = getUnfixedIssues(initial); if (unfixed.length === 0) { - const { ast, store } = await parseMarkdown(initial.output); - setState({ - status: 'ready', - ast, - store, - unresolvedIssues: [], - wasFixed: initial.fixCount > 0, - blockId: block.id, - submitted: false, - }); + const parsed = await tryParse(initial.output); + setState( + buildState( + block.id, + submitted, + 'ready', + parsed?.ast ?? null, + parsed?.store ?? null, + [], + initial.fixCount > 0, + ), + ); return; } if (!fixer) { - try { - const { ast, store } = await parseMarkdown(initial.output); - setState({ - status: 'invalid', - ast, - store, - unresolvedIssues: unfixed, - wasFixed: false, - blockId: block.id, - submitted: false, - }); - } catch { - setState({ - status: 'invalid', - ast: null, - store: null, - unresolvedIssues: unfixed, - wasFixed: false, - blockId: block.id, - submitted: false, - }); - } + const parsed = await tryParse(initial.output); + setState( + buildState( + block.id, + submitted, + 'invalid', + parsed?.ast ?? null, + parsed?.store ?? null, + unfixed, + ), + ); return; } - setState({ - status: 'fixing', - ast: null, - store: null, - unresolvedIssues: unfixed, - wasFixed: false, - blockId: block.id, - submitted: false, - }); + setState(buildState(block.id, submitted, 'fixing', null, null, unfixed)); const ctrl = new AbortController(); registerAbort(ctrl); try { - const systemPrompt = `${buildSystemPrompt()}\n\n---\n\n${buildFixerPrompt('single-block')}`; - const userMessage = buildFixerMessage(block.document, unfixed); - - let fixed: string; - if (fixer.kind === 'anthropic') { - fixed = await anthropicFix(fixer.apiKey, fixer.model, systemPrompt, userMessage, ctrl.signal); - } else { - const llmConfig: LlmConfig = { - baseUrl: fixer.baseUrl, - apiKey: fixer.apiKey, - model: fixer.model, - }; - fixed = await chatCompletion( - llmConfig, - [ - { role: 'system', content: systemPrompt }, - { role: 'user', content: userMessage }, - ], - ctrl.signal, - ); - } - - const revalidated = validate(fixed, { exclude: ['thinking-block', 'flow-ordering'] }); - const stillUnfixed = revalidated.issues.filter( - (i) => !i.fixed && (i.severity === 'error' || i.severity === 'warning'), + const fixed = await callFixer(fixer, block.document, unfixed, ctrl.signal); + const revalidated = validate(fixed, VALIDATE_OPTIONS); + const stillUnfixed = getUnfixedIssues(revalidated); + const parsed = await tryParse(revalidated.output); + setState( + buildState( + block.id, + submitted, + stillUnfixed.length === 0 ? 'ready' : 'invalid', + parsed?.ast ?? null, + parsed?.store ?? null, + stillUnfixed, + true, + ), ); - - const { ast, store } = await parseMarkdown(revalidated.output); - setState({ - status: stillUnfixed.length === 0 ? 'ready' : 'invalid', - ast, - store, - unresolvedIssues: stillUnfixed, - wasFixed: true, - blockId: block.id, - submitted: false, - }); } catch (err) { if (err instanceof DOMException && err.name === 'AbortError') return; console.error('[preview-validation] fixer failed', err); - try { - const { ast, store } = await parseMarkdown(initial.output); - setState({ - status: 'invalid', - ast, - store, - unresolvedIssues: unfixed, - wasFixed: false, - blockId: block.id, - submitted: false, - }); - } catch { - setState({ - status: 'invalid', - ast: null, - store: null, - unresolvedIssues: unfixed, - wasFixed: false, - blockId: block.id, - submitted: false, - }); - } + const parsed = await tryParse(initial.output); + setState( + buildState( + block.id, + submitted, + 'invalid', + parsed?.ast ?? null, + parsed?.store ?? null, + unfixed, + ), + ); } } diff --git a/demo/src/preview/use-submission-log.ts b/demo/src/preview/use-submission-log.ts index b78bfbd..11e6765 100644 --- a/demo/src/preview/use-submission-log.ts +++ b/demo/src/preview/use-submission-log.ts @@ -5,11 +5,6 @@ import { type SubmissionLogEntry, } from './insurance-backend.js'; -/** - * Reactive read of the mock backend's submission log. The store lives in - * `insurance-backend.ts` (module-level array + subscriber set); this hook - * re-renders any consumer whenever a new submission is recorded. - */ export function useSubmissionLog(): readonly SubmissionLogEntry[] { return useSyncExternalStore(subscribeSubmissionLog, getSubmissionLog, getSubmissionLog); } From dae22b07c131ae28bbca61b113f17a2b1716120f Mon Sep 17 00:00:00 2001 From: gitsad Date: Thu, 21 May 2026 13:21:35 +0200 Subject: [PATCH 21/26] chore: switched places --- demo/src/AgentChatView.tsx | 2 +- demo/src/App.tsx | 2 +- demo/src/HomeView.tsx | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/demo/src/AgentChatView.tsx b/demo/src/AgentChatView.tsx index ab1c1f8..658bb0f 100644 --- a/demo/src/AgentChatView.tsx +++ b/demo/src/AgentChatView.tsx @@ -19,7 +19,7 @@ export function AgentChatView() { stop, clear, inputRef, - } = useAgent(); + } = useAgent({ useAuthorSubAgent: true }); const { events, isOpen: logOpen, setIsOpen: setLogOpen, clearEvents } = useAgentActionLog(turns); diff --git a/demo/src/App.tsx b/demo/src/App.tsx index 625941b..30435cc 100644 --- a/demo/src/App.tsx +++ b/demo/src/App.tsx @@ -43,8 +43,8 @@ const NAV_GROUPS: NavGroup[] = [ { label: 'Agentic', items: [ - { path: '/chat', label: 'Agent Chat', icon: '⚡' }, { path: '/preview', label: 'Preview', icon: '🛡️' }, + { path: '/chat', label: 'Agent Chat', icon: '⚡' }, ], }, { diff --git a/demo/src/HomeView.tsx b/demo/src/HomeView.tsx index 3d9d6fc..ce7ab3e 100644 --- a/demo/src/HomeView.tsx +++ b/demo/src/HomeView.tsx @@ -11,13 +11,6 @@ const SECTIONS = [ label: 'Agentic', description: 'Agent with tool use', items: [ - { - path: '/chat', - label: 'Agent Chat', - icon: '⚡', - description: - 'Autonomous agent that thinks, plans, and generates interactive MDMA documents via tool calls.', - }, { path: '/preview', label: 'Preview', @@ -25,6 +18,13 @@ const SECTIONS = [ description: 'Multi-step flow demo (insurance claim) — chat on the left, live MDMA preview with auto-validation and fixer on the right.', }, + { + path: '/chat', + label: 'Agent Chat', + icon: '⚡', + description: + 'Autonomous agent that thinks, plans, and generates interactive MDMA documents via tool calls.', + }, ], }, { From 5bb852965927956717a5fc4c15dd49e5f3e06d86 Mon Sep 17 00:00:00 2001 From: gitsad Date: Thu, 21 May 2026 13:26:00 +0200 Subject: [PATCH 22/26] chore: changeset --- .changeset/clear-windows-see.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changeset/clear-windows-see.md diff --git a/.changeset/clear-windows-see.md b/.changeset/clear-windows-see.md new file mode 100644 index 0000000..e7ad3f1 --- /dev/null +++ b/.changeset/clear-windows-see.md @@ -0,0 +1,8 @@ +--- +"@mobile-reality/mdma-validator": minor +"@mobile-reality/mdma-spec": minor +"@mobile-reality/mdma-prompt-pack": patch +"@mobile-reality/mdma-demo": patch +--- + +Split validator into per-block validate() and multi-message validateConversation(); make form.onSubmit required and rewrite action-label fields as opaque labels (drop the action-references rule); add many model-specific fixer/author/agent-tool prompt variants (gpt-5.x family, Claude opus/sonnet/haiku, Gemini 2.5/3, Grok), promote the conversation-judge prompt out of mdma-fixer/ and rename its export to MDMA_CONVERSATION_JUDGE. From 019778a52ef4dec359b6129e66a001bf8d514c17 Mon Sep 17 00:00:00 2001 From: gitsad Date: Thu, 21 May 2026 13:35:55 +0200 Subject: [PATCH 23/26] chore: update tests --- .changeset/clever-lines-trade.md | 7 +++++++ packages/attachables-core/tests/handlers.test.ts | 1 + packages/cli/tests/commands/validate.test.ts | 1 + packages/parser/tests/fixtures/complex-bindings.md | 1 + packages/parser/tests/fixtures/multi-component.md | 1 + 5 files changed, 11 insertions(+) create mode 100644 .changeset/clever-lines-trade.md diff --git a/.changeset/clever-lines-trade.md b/.changeset/clever-lines-trade.md new file mode 100644 index 0000000..504fb86 --- /dev/null +++ b/.changeset/clever-lines-trade.md @@ -0,0 +1,7 @@ +--- +"@mobile-reality/mdma-attachables-core": patch +"@mobile-reality/mdma-parser": patch +"@mobile-reality/mdma-cli": patch +--- + +Tests update diff --git a/packages/attachables-core/tests/handlers.test.ts b/packages/attachables-core/tests/handlers.test.ts index 752ea61..4588efc 100644 --- a/packages/attachables-core/tests/handlers.test.ts +++ b/packages/attachables-core/tests/handlers.test.ts @@ -36,6 +36,7 @@ describe('formHandler', () => { { name: 'email', type: 'email', label: 'Email' }, { name: 'agree', type: 'checkbox', label: 'Agree' }, ], + onSubmit: 'submit-f1', }); expect(state.values.email).toBe(''); expect(state.values.agree).toBe(false); diff --git a/packages/cli/tests/commands/validate.test.ts b/packages/cli/tests/commands/validate.test.ts index 70ca902..056bfae 100644 --- a/packages/cli/tests/commands/validate.test.ts +++ b/packages/cli/tests/commands/validate.test.ts @@ -16,6 +16,7 @@ fields: label: Email required: true sensitive: true +onSubmit: submit-test \`\`\` `; const result = validate(markdown); diff --git a/packages/parser/tests/fixtures/complex-bindings.md b/packages/parser/tests/fixtures/complex-bindings.md index be3a603..e9166e6 100644 --- a/packages/parser/tests/fixtures/complex-bindings.md +++ b/packages/parser/tests/fixtures/complex-bindings.md @@ -11,6 +11,7 @@ fields: - name: user_name type: text label: Name +onSubmit: submit-data ``` ## Data Table diff --git a/packages/parser/tests/fixtures/multi-component.md b/packages/parser/tests/fixtures/multi-component.md index 828ddde..857a492 100644 --- a/packages/parser/tests/fixtures/multi-component.md +++ b/packages/parser/tests/fixtures/multi-component.md @@ -16,6 +16,7 @@ fields: options: - { label: P1 - Critical, value: P1 } - { label: P2 - High, value: P2 } +onSubmit: submit-triage ``` ## Checklist From dfc8c865d61724ec1788848ce3aec3abb1b1e006 Mon Sep 17 00:00:00 2001 From: gitsad Date: Thu, 21 May 2026 13:38:46 +0200 Subject: [PATCH 24/26] chore: lint adn format --- demo/src/PreviewView.tsx | 4 +- demo/src/agent/AgentSettings.tsx | 3 +- demo/src/agent/use-agent.ts | 15 +---- demo/src/chat/ChatSettings.tsx | 3 +- demo/src/docs/DocsView.tsx | 19 ++++-- .../sections/CustomPromptBestPractices.tsx | 64 ++++++++++--------- .../docs/sections/IntegrationLangchain.tsx | 21 +++--- demo/src/docs/sections/PromptMatrix.tsx | 14 ++-- demo/src/preview/BackendLogPane.tsx | 6 +- demo/src/preview/PreviewPanel.tsx | 6 +- demo/src/preview/use-preview-validation.ts | 4 +- evals/assertions/fixer-contains-component.mjs | 8 ++- evals/prompt-fixer.mjs | 4 +- evals/promptfooconfig.fixer.js | 3 +- .../validator/src/validate-conversation.ts | 6 +- .../single-interactive-component.test.ts | 7 +- 16 files changed, 91 insertions(+), 96 deletions(-) diff --git a/demo/src/PreviewView.tsx b/demo/src/PreviewView.tsx index 16d234d..496294c 100644 --- a/demo/src/PreviewView.tsx +++ b/demo/src/PreviewView.tsx @@ -85,8 +85,8 @@ export function PreviewView() {

Insurance Claim Demo

Ask the agent to start a new insurance claim. It will walk you through name & - birthday, claim details, bank account, and a final confirmation — each step - rendered live in the preview pane on the right. + birthday, claim details, bank account, and a final confirmation — each step rendered + live in the preview pane on the right.

)} diff --git a/demo/src/agent/AgentSettings.tsx b/demo/src/agent/AgentSettings.tsx index a0011d6..37ca983 100644 --- a/demo/src/agent/AgentSettings.tsx +++ b/demo/src/agent/AgentSettings.tsx @@ -188,7 +188,8 @@ export const AgentSettings = memo(function AgentSettings({ config, onUpdate }: A : 'OpenAI-compatible mode uses Chat Completions with function calling. Reasoning is internal and not displayed.'}

- 🔒 Your API key is stored in your browser's localStorage only. It is never sent to any server other than the AI provider you select. + 🔒 Your API key is stored in your browser's localStorage only. It is never sent to + any server other than the AI provider you select.

)} diff --git a/demo/src/agent/use-agent.ts b/demo/src/agent/use-agent.ts index 77b3662..cfaf730 100644 --- a/demo/src/agent/use-agent.ts +++ b/demo/src/agent/use-agent.ts @@ -48,7 +48,7 @@ const GENERATE_MDMA_TOOL_BRIEF = { description: 'Request the MDMA Author (a specialised sub-agent) to generate an interactive MDMA component ' + 'for the user. Provide a clear brief describing what to generate — component type, id, fields, ' + - "labels, action labels (onSubmit etc.), and any constraints. Do NOT write MDMA Markdown yourself; " + + 'labels, action labels (onSubmit etc.), and any constraints. Do NOT write MDMA Markdown yourself; ' + 'the author will produce the final document and render it on the user’s screen.', input_schema: { type: 'object' as const, @@ -267,13 +267,7 @@ async function runAgentLoop( const blockMeta = new Map(); let stopReason = 'end_turn'; - for await (const ev of streamAgentMessages( - config, - systemPrompt, - history, - [tool], - signal, - )) { + for await (const ev of streamAgentMessages(config, systemPrompt, history, [tool], signal)) { if (ev.type === 'stream_error') { onError(ev.message); continueLoop = false; @@ -775,10 +769,7 @@ export function useAgent(options: UseAgentOptions = {}) { try { if (provider === 'anthropic') { - const history: ApiMessage[] = [ - ...apiHistoryRef.current, - { role: 'user', content: text }, - ]; + const history: ApiMessage[] = [...apiHistoryRef.current, { role: 'user', content: text }]; await runAgentLoop( config, systemPrompt, diff --git a/demo/src/chat/ChatSettings.tsx b/demo/src/chat/ChatSettings.tsx index bc70ce5..3789c20 100644 --- a/demo/src/chat/ChatSettings.tsx +++ b/demo/src/chat/ChatSettings.tsx @@ -183,7 +183,8 @@ export const ChatSettings = memo(function ChatSettings({

- 🔒 Your API key is stored in your browser's localStorage only. It is never sent to any server other than the AI provider you select. + 🔒 Your API key is stored in your browser's localStorage only. It is never sent to + any server other than the AI provider you select.

)} diff --git a/demo/src/docs/DocsView.tsx b/demo/src/docs/DocsView.tsx index 31a6079..91d6352 100644 --- a/demo/src/docs/DocsView.tsx +++ b/demo/src/docs/DocsView.tsx @@ -72,18 +72,27 @@ export function DocsView() { const previewEntry = COMPONENTS.find((c) => c.type === selectedComponent) ?? COMPONENTS[0]; const isPackagesActive = active === 'packages' || active.startsWith('packages/'); - const activePackageSlug = active.startsWith('packages/') ? active.slice('packages/'.length) : null; - const activePackage = activePackageSlug ? PACKAGES.find((p) => p.slug === activePackageSlug) : null; + const activePackageSlug = active.startsWith('packages/') + ? active.slice('packages/'.length) + : null; + const activePackage = activePackageSlug + ? PACKAGES.find((p) => p.slug === activePackageSlug) + : null; const isIntegrationsActive = active === 'integrations' || active.startsWith('integrations/'); - const activeIntegrationSlug = active.startsWith('integrations/') ? active.slice('integrations/'.length) : null; - const ActiveIntegration = activeIntegrationSlug ? INTEGRATION_COMPONENTS[activeIntegrationSlug] : null; + const activeIntegrationSlug = active.startsWith('integrations/') + ? active.slice('integrations/'.length) + : null; + const ActiveIntegration = activeIntegrationSlug + ? INTEGRATION_COMPONENTS[activeIntegrationSlug] + : null; const section = SECTIONS.find((s) => s.slug === active); const SectionContent = section?.component ?? null; function renderContent() { - if (showPreview) return ; + if (showPreview) + return ; if (activePackage) return ; if (active === 'packages') return ; if (ActiveIntegration) return ; diff --git a/demo/src/docs/sections/CustomPromptBestPractices.tsx b/demo/src/docs/sections/CustomPromptBestPractices.tsx index e46a17b..843d675 100644 --- a/demo/src/docs/sections/CustomPromptBestPractices.tsx +++ b/demo/src/docs/sections/CustomPromptBestPractices.tsx @@ -6,10 +6,9 @@ export function CustomPromptBestPractices() {

Custom Prompt Best Practices

When you pass a customPrompt to buildSystemPrompt, it sits - alongside the MDMA author rules. The model treats both as authoritative, so wording - choices in the custom prompt strongly influence the output — sometimes overriding MDMA - rules. The patterns below are drawn from eval failures we've fixed across the - prompt matrix. + alongside the MDMA author rules. The model treats both as authoritative, so wording choices + in the custom prompt strongly influence the output — sometimes overriding MDMA rules. The + patterns below are drawn from eval failures we've fixed across the prompt matrix.

1. Frame multi-step workflows as turns, not single-message blueprints

@@ -20,8 +19,8 @@ export function CustomPromptBestPractices() { one-interactive-component-per-response rule.

- Instead, describe the workflow as a sequence of turns. The model then emits only the - first interactive component initially and treats the rest as follow-ups. + Instead, describe the workflow as a sequence of turns. The model then emits only the first + interactive component initially and treats the rest as follow-ups.

@@ -84,10 +83,10 @@ follow-up steps and appear in later turns.`}

2. Always specify an onSubmit handler for forms

The form schema requires onSubmit. When the custom prompt doesn't name - one, the model either omits it (schema violation) or invents a self-referencing handler - (onSubmit: my-form targets itself), both of which fail validation. Always - give the form an explicit handler name in the prompt — it's an opaque string label, - so it doesn't need to correspond to a real component. + one, the model either omits it (schema violation) or invents a self-referencing handler ( + onSubmit: my-form targets itself), both of which fail validation. Always give + the form an explicit handler name in the prompt — it's an opaque string label, so it + doesn't need to correspond to a real component.

@@ -110,9 +109,9 @@ follow-up steps and appear in later turns.`}

3. Avoid special characters in field name descriptions

Slashes, ampersands, and parenthetical alternatives in field names confuse the YAML - generation step. The model occasionally produces malformed YAML keys - (e.g. name:ssn-tax-id instead of name: ssn-tax-id) when it - tries to convert a compound label into a single field name. + generation step. The model occasionally produces malformed YAML keys (e.g.{' '} + name:ssn-tax-id instead of name: ssn-tax-id) when it tries to + convert a compound label into a single field name.

@@ -132,37 +131,40 @@ follow-up steps and appear in later turns.`}

4. Don't materialize action-label targets as sibling components

- Action-label fields like onSubmit, onAction, - onApprove, onDeny, trigger, and - onComplete are opaque string labels — they do not need to match - any other component in the same message. A callout, webhook, or button with an{' '} - id that matches another component's action label is a follow-up step, - not a sibling. + Action-label fields like onSubmit, onAction,onApprove + , onDeny, trigger, and + onComplete are opaque string labels — they do not need to match any + other component in the same message. A callout, webhook, or button with an id{' '} + that matches another component's action label is a follow-up step, not a sibling.

- When your prompt includes such a follow-up component, describe it as part of a later - turn (see pattern 1). Don't instruct the model to render the handler alongside the - action that triggers it. + When your prompt includes such a follow-up component, describe it as part of a later turn + (see pattern 1). Don't instruct the model to render the handler alongside the action + that triggers it.

5. Single-interactive-component constraint

- Every response contains at most one interactive component - (form, button, webhook, - approval-gate, tasklist). Non-interactive components - (callout, chart, table) are unaffected — you can - emit as many as you need. + Every response contains at most one interactive component (form,{' '} + button, webhook,approval-gate, tasklist + ). Non-interactive components (callout, chart, table) + are unaffected — you can emit as many as you need.

Your custom prompt should respect this. If you describe a workflow that needs multiple - interactive components (form + approval + button), structure it as turns (pattern 1) - rather than asking for all of them at once. + interactive components (form + approval + button), structure it as turns (pattern 1) rather + than asking for all of them at once.

Quick checklist

    -
  • Multi-step workflows are described as "Turn 1 / Turn 2 / Turn 3", not as a single batch.
    • -
  • Every form has an explicit onSubmit handler in the prompt.
    • +
  • + Multi-step workflows are described as "Turn 1 / Turn 2 / Turn 3", not as a + single batch. +
    • +
  • + Every form has an explicit onSubmit handler in the prompt. +
  • Field labels avoid slashes, ampersands, and parenthetical alternatives.
  • Follow-up callouts/webhooks/buttons are described as future turns, not siblings.
  • The initial response emits only one interactive component.
    • diff --git a/demo/src/docs/sections/IntegrationLangchain.tsx b/demo/src/docs/sections/IntegrationLangchain.tsx index cf0535c..ef32894 100644 --- a/demo/src/docs/sections/IntegrationLangchain.tsx +++ b/demo/src/docs/sections/IntegrationLangchain.tsx @@ -5,14 +5,18 @@ export function IntegrationLangchain() { <>

    LangChain.js

    - MDMA is framework-agnostic — it doesn't care how you call the LLM. LangChain.js works out - of the box: use mdma-prompt-pack for the system prompt and{' '} - remarkMdma from mdma-parser with a standard{' '} - unified pipeline to process the output. + MDMA is framework-agnostic — it doesn't care how you call the LLM. LangChain.js works out of + the box: use mdma-prompt-pack for the system prompt and remarkMdma{' '} + from mdma-parser with a standard unified pipeline to process the + output.

    Install

    - {'npm install @langchain/anthropic @langchain/core unified remark-parse @mobile-reality/mdma-prompt-pack @mobile-reality/mdma-parser'} + + { + 'npm install @langchain/anthropic @langchain/core unified remark-parse @mobile-reality/mdma-prompt-pack @mobile-reality/mdma-parser' + } +

    Simple completion (MDMA_AUTHOR)

    {`import { ChatAnthropic } from '@langchain/anthropic'; @@ -97,9 +101,9 @@ const result = await executor.invoke({ input: 'I need a project status report fo

    Python LangChain

    - The prompt-pack is a TypeScript package. For Python, copy the prompt string from the - package source or expose it via a small JS service, then use it as the system message in - any Python LangChain chain. + The prompt-pack is a TypeScript package. For Python, copy the prompt string from the package + source or expose it via a small JS service, then use it as the system message in any Python + LangChain chain.

    {`from langchain_anthropic import ChatAnthropic from langchain_core.messages import SystemMessage, HumanMessage @@ -116,7 +120,6 @@ response = model.invoke([ # response.content is an MDMA markdown string # pass it to your frontend or a JS service running mdma-parser`} - ); } diff --git a/demo/src/docs/sections/PromptMatrix.tsx b/demo/src/docs/sections/PromptMatrix.tsx index e170ee7..d93d2a4 100644 --- a/demo/src/docs/sections/PromptMatrix.tsx +++ b/demo/src/docs/sections/PromptMatrix.tsx @@ -47,8 +47,8 @@ export function PromptMatrix() { [i] Noticeably slow response times — single-turn responses commonly take tens of seconds.

    - † gpt-5.4 intermittent duplication bug — passes one-shot evals reliably - but shows non-deterministic output duplication in multi-turn, custom-prompt, and flow evals + † gpt-5.4 intermittent duplication bug — passes one-shot evals reliably but + shows non-deterministic output duplication in multi-turn, custom-prompt, and flow evals (~7–15% of runs). The model generates a correct response then immediately re-emits it verbatim, causing [duplicate-ids] validation errors. This is a known model-level issue unrelated to the prompt variant.{' '} @@ -66,11 +66,11 @@ export function PromptMatrix() { runs, the model emits a chain-of-thought as visible Markdown prose (" **Investigating Production Errors**" repeated 3–5 times) instead of opening a{' '} ```mdma block, producing either{' '} - [yaml-correctness: outside fenced block] or{' '} - [duplicate-ids] errors. Per Google's official Gemini 3 prompting guide, this - is a model-level behavior driven by temperature/sampling choices — prompt-level fixes shift - which test loops rather than eliminating the loops. Prefer gemini-2.5-pro for - production multi-step flows requiring deterministic output. + [yaml-correctness: outside fenced block] or [duplicate-ids]{' '} + errors. Per Google's official Gemini 3 prompting guide, this is a model-level behavior + driven by temperature/sampling choices — prompt-level fixes shift which test loops rather + than eliminating the loops. Prefer gemini-2.5-pro for production multi-step + flows requiring deterministic output.

    MDMA_AGENT Prompt Matrix

    diff --git a/demo/src/preview/BackendLogPane.tsx b/demo/src/preview/BackendLogPane.tsx index bef5545..cedee44 100644 --- a/demo/src/preview/BackendLogPane.tsx +++ b/demo/src/preview/BackendLogPane.tsx @@ -45,11 +45,7 @@ export function BackendLogDrawer() { Backend log {entries.length} {entries.length > 0 && ( - )} diff --git a/demo/src/preview/PreviewPanel.tsx b/demo/src/preview/PreviewPanel.tsx index 30566b6..d7145b9 100644 --- a/demo/src/preview/PreviewPanel.tsx +++ b/demo/src/preview/PreviewPanel.tsx @@ -87,11 +87,7 @@ export function PreviewPanel({ state }: PreviewPanelProps) { )} {showRender && (
    - +
    )} diff --git a/demo/src/preview/use-preview-validation.ts b/demo/src/preview/use-preview-validation.ts index e552b07..72f6bef 100644 --- a/demo/src/preview/use-preview-validation.ts +++ b/demo/src/preview/use-preview-validation.ts @@ -173,9 +173,7 @@ function buildState( return { status, ast, store, unresolvedIssues, wasFixed, blockId, submitted }; } -async function tryParse( - markdown: string, -): Promise<{ ast: MdmaRoot; store: DocumentStore } | null> { +async function tryParse(markdown: string): Promise<{ ast: MdmaRoot; store: DocumentStore } | null> { try { return await parseMarkdown(markdown); } catch { diff --git a/evals/assertions/fixer-contains-component.mjs b/evals/assertions/fixer-contains-component.mjs index 77f4245..927b79f 100644 --- a/evals/assertions/fixer-contains-component.mjs +++ b/evals/assertions/fixer-contains-component.mjs @@ -121,14 +121,18 @@ function compareFields(expected, actual, prefix) { if (typeof expectedVal[i] === 'object' && expectedVal[i] !== null) { failures.push(...compareFields(expectedVal[i], actualVal[i] ?? {}, `${path}[${i}]`)); } else if (expectedVal[i] !== actualVal[i]) { - failures.push(`"${path}[${i}]": expected ${JSON.stringify(expectedVal[i])}, got ${JSON.stringify(actualVal[i])}`); + failures.push( + `"${path}[${i}]": expected ${JSON.stringify(expectedVal[i])}, got ${JSON.stringify(actualVal[i])}`, + ); } } } } else if (typeof expectedVal === 'object') { failures.push(...compareFields(expectedVal, actualVal ?? {}, path)); } else if (actualVal !== expectedVal) { - failures.push(`"${path}": expected ${JSON.stringify(expectedVal)}, got ${JSON.stringify(actualVal)}`); + failures.push( + `"${path}": expected ${JSON.stringify(expectedVal)}, got ${JSON.stringify(actualVal)}`, + ); } } return failures; diff --git a/evals/prompt-fixer.mjs b/evals/prompt-fixer.mjs index 21c4086..9d5c488 100644 --- a/evals/prompt-fixer.mjs +++ b/evals/prompt-fixer.mjs @@ -30,9 +30,7 @@ export default async function ({ vars }) { if (variantKey !== 'flow') exclude.push('flow-ordering'); const result = validate(vars.brokenDocument, { exclude }); - const allIssues = result.issues.filter( - (i) => i.severity === 'error' || i.severity === 'warning', - ); + const allIssues = result.issues.filter((i) => i.severity === 'error' || i.severity === 'warning'); const { prompt: variantPrompt, source: fixerSource } = await selectFixerPrompt(); const fixerPrompt = fixerSource.startsWith('default') diff --git a/evals/promptfooconfig.fixer.js b/evals/promptfooconfig.fixer.js index a5efba6..1c39331 100644 --- a/evals/promptfooconfig.fixer.js +++ b/evals/promptfooconfig.fixer.js @@ -1,7 +1,6 @@ const provider = process.env.EVAL_PROVIDER || 'openai:gpt-4.1-mini'; const leaksReasoningTokens = - (provider.includes('gemini') && provider.includes('pro')) || - provider.includes('grok-4.3'); + (provider.includes('gemini') && provider.includes('pro')) || provider.includes('grok-4.3'); const providerConfig = { max_tokens: 8192, diff --git a/packages/validator/src/validate-conversation.ts b/packages/validator/src/validate-conversation.ts index 5f50a8d..522a1b4 100644 --- a/packages/validator/src/validate-conversation.ts +++ b/packages/validator/src/validate-conversation.ts @@ -132,11 +132,7 @@ export function validateConversation( const expectedTypes = new Set(steps.map((s) => s.type)); for (let msgIdx = 0; msgIdx < assistantMessages.length; msgIdx++) { - const components = extractStepComponents( - assistantMessages[msgIdx], - expectedIds, - expectedTypes, - ); + const components = extractStepComponents(assistantMessages[msgIdx], expectedIds, expectedTypes); if (components.length === 0) continue; // pure-text reply is allowed diff --git a/packages/validator/tests/rules/single-interactive-component.test.ts b/packages/validator/tests/rules/single-interactive-component.test.ts index dcf9c9b..8aeaa74 100644 --- a/packages/validator/tests/rules/single-interactive-component.test.ts +++ b/packages/validator/tests/rules/single-interactive-component.test.ts @@ -1,13 +1,14 @@ import { describe, it, expect } from 'vitest'; import { validate } from '../../src/index.js'; -const doc = (...blocks: string[]) => - blocks.map((b) => `\`\`\`mdma\n${b}\`\`\``).join('\n\n'); +const doc = (...blocks: string[]) => blocks.map((b) => `\`\`\`mdma\n${b}\`\`\``).join('\n\n'); describe('single-interactive-component rule', () => { it('passes for a single form', () => { const result = validate( - doc('type: form\nid: f\nfields:\n - name: x\n type: text\n label: X\nonSubmit: done\n'), + doc( + 'type: form\nid: f\nfields:\n - name: x\n type: text\n label: X\nonSubmit: done\n', + ), ); const issues = result.issues.filter((i) => i.ruleId === 'single-interactive-component'); expect(issues).toHaveLength(0); From 5aaaef875b8603f701262d8bdc99aab39afd7698 Mon Sep 17 00:00:00 2001 From: gitsad Date: Thu, 21 May 2026 13:54:36 +0200 Subject: [PATCH 25/26] chore: updated Readme and Docs with fixer prompt matrix --- README.md | 44 +++++++++++++++++++ demo/src/docs/sections/PromptMatrix.tsx | 56 +++++++++++++++++++++++-- 2 files changed, 97 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bd68030..d0158f3 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,50 @@ Each cell shows the pass rate of the model-specialized MDMA_AUTHOR prompt varian \[i] Noticeably slow response times — single-turn responses commonly take tens of seconds and full eval runs measure in minutes. +## MDMA_FIXER prompt matrix + +Each cell shows the pass rate of the model-specialized MDMA_FIXER prompt variant on the single-block fixer eval (15 tests covering structural fixes, bindings, PII, forms, tables/charts, approvals). The fixer is what powers automatic repair of LLM output that fails `validate()` — every supported model lands at ✅ via model-tailored inline guards (no-leading-separator, preserve-input-structure, table-key-direction, replace-all-placeholders, fix-all-listed-errors, etc.). + +✅ 100% on the single-block fixer eval (15/15). + + +| Variant | single-block fixer | notes | +| :--- | :---: | :--- | +| **OpenAI** | | | +| `gpt-5.5` | ✅ | | +| `gpt-5.4` | ✅ | | +| `gpt-5.4-mini` | ✅ | | +| `gpt-5.4-nano` | ✅ | | +| `gpt-5.2` | ✅ | | +| `gpt-5.1` | ✅ | | +| `gpt-5` | ✅ | | +| `gpt-5-mini` | ✅ \* | | +| `gpt-5-nano` | ✅ \* | | +| `gpt-4.1` | ✅ | | +| `gpt-4.1-mini` | ✅ | | +| `gpt-4.1-nano` | ✅ | | +| **Anthropic** | | | +| `claude-opus-4.7` | ✅ | | +| `claude-opus-4.6` | ✅ | | +| `claude-sonnet` | ✅ | catch-all variant — matches `claude-sonnet-4-5`, `claude-sonnet-4-6`, etc. | +| `claude-haiku` | ✅ | | +| **Google** | | | +| `gemini-3.1-pro-preview` | ✅ ‡ | requires OpenRouter `reasoning.exclude: true` (already wired in `evals/promptfooconfig.fixer.js`) | +| `gemini-3.1-pro-preview-customtools` | ✅ ‡ | same `reasoning.exclude` requirement | +| `gemini-3.1-flash-lite-preview` | ✅ | | +| `gemini-3-flash-preview` | ✅ | | +| `gemini-2.5-pro` | ✅ ‡ | same `reasoning.exclude` requirement | +| `gemini-2.5-flash` | ✅ | | +| `gemini-2.5-flash-lite` | ✅ | | +| **xAI** | | | +| `grok-4.3` | ✅ ‡ | minimal prompt + `reasoning.exclude: true` — extra framing regresses Grok 4.3 | +| `grok-4.20` | ✅ | | + +\* Smaller-tier residual flakiness — `gpt-5-mini` and `gpt-5-nano` occasionally re-emit a leading `---` despite the inline guard (~1/15 on a bad run). Re-runs clear 15/15. Documented in the variant docblocks. + +‡ Reasoning-token leak suppression — for reasoning-flavoured Gemini Pro variants and Grok 4.3, the fixer would otherwise see visible "Thinking: **Topic**" prose prepended to every response. The eval config sets `passthrough.reasoning.exclude: true` (and the demo's `usePreviewValidation` does the same per-provider) to strip reasoning tokens from the response body at the API layer rather than at the prompt layer. + + ## Components 9 built-in component types, all rendered out of the box by `@mobile-reality/mdma-renderer-react`: diff --git a/demo/src/docs/sections/PromptMatrix.tsx b/demo/src/docs/sections/PromptMatrix.tsx index d93d2a4..28ee378 100644 --- a/demo/src/docs/sections/PromptMatrix.tsx +++ b/demo/src/docs/sections/PromptMatrix.tsx @@ -114,14 +114,64 @@ export function PromptMatrix() {

    — Full eval data is being collected for these variants.

    +

    MDMA_FIXER Prompt Matrix

    +

    + Each cell shows the pass rate of the model-specialized MDMA_FIXER prompt + variant on the single-block fixer eval (15 tests covering structural fixes, bindings, PII, + forms, tables/charts, approvals). The fixer is what powers automatic repair of LLM output + that fails validate(). +

    +

    ✅ 100% on the single-block fixer eval (15/15).

    + +

    + * Smaller-tier residual flakiness — gpt-5-mini and gpt-5-nano{' '} + occasionally re-emit a leading --- despite the inline guard (~1/15 on a bad + run). Re-runs clear 15/15. +

    +

    + ‡ Reasoning-token leak suppression — for reasoning-flavoured Gemini Pro + variants and Grok 4.3, the fixer would otherwise see visible "Thinking: **Topic**" prose + prepended to every response. The eval config sets{' '} + passthrough.reasoning.exclude: true (and the demo's{' '} + usePreviewValidation does the same per-provider) to strip reasoning tokens + from the response body at the API layer rather than the prompt layer. +

    +

    In Progress

    - The following prompts exist in mdma-prompt-pack but are still being optimized — - they do not yet have model-specific variants for GPT, Claude, Gemini, or Grok. + The following prompt still ships without model-specific variants and is on the roadmap:

    {[ - { name: 'MDMA_FIXER', description: 'Corrects invalid or malformed MDMA documents.' }, { name: 'MDMA_REVIEWER', description: 'Reviews and critiques MDMA documents for quality and spec conformance.', From 094f046e3011ce204ccc7a6b07204f44411e2d9d Mon Sep 17 00:00:00 2001 From: gitsad Date: Thu, 21 May 2026 13:59:47 +0200 Subject: [PATCH 26/26] chore: updated readme --- README.md | 6 ++---- demo/src/docs/sections/PromptMatrix.tsx | 9 ++------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index d0158f3..ea4dfef 100644 --- a/README.md +++ b/README.md @@ -141,8 +141,8 @@ Each cell shows the pass rate of the model-specialized MDMA_FIXER prompt variant | `gpt-5.2` | ✅ | | | `gpt-5.1` | ✅ | | | `gpt-5` | ✅ | | -| `gpt-5-mini` | ✅ \* | | -| `gpt-5-nano` | ✅ \* | | +| `gpt-5-mini` | ✅ | | +| `gpt-5-nano` | ✅ | | | `gpt-4.1` | ✅ | | | `gpt-4.1-mini` | ✅ | | | `gpt-4.1-nano` | ✅ | | @@ -163,8 +163,6 @@ Each cell shows the pass rate of the model-specialized MDMA_FIXER prompt variant | `grok-4.3` | ✅ ‡ | minimal prompt + `reasoning.exclude: true` — extra framing regresses Grok 4.3 | | `grok-4.20` | ✅ | | -\* Smaller-tier residual flakiness — `gpt-5-mini` and `gpt-5-nano` occasionally re-emit a leading `---` despite the inline guard (~1/15 on a bad run). Re-runs clear 15/15. Documented in the variant docblocks. - ‡ Reasoning-token leak suppression — for reasoning-flavoured Gemini Pro variants and Grok 4.3, the fixer would otherwise see visible "Thinking: **Topic**" prose prepended to every response. The eval config sets `passthrough.reasoning.exclude: true` (and the demo's `usePreviewValidation` does the same per-provider) to strip reasoning tokens from the response body at the API layer rather than at the prompt layer. diff --git a/demo/src/docs/sections/PromptMatrix.tsx b/demo/src/docs/sections/PromptMatrix.tsx index 28ee378..841b490 100644 --- a/demo/src/docs/sections/PromptMatrix.tsx +++ b/demo/src/docs/sections/PromptMatrix.tsx @@ -132,8 +132,8 @@ export function PromptMatrix() { ['gpt-5.2', '✅', ''], ['gpt-5.1', '✅', ''], ['gpt-5', '✅', ''], - ['gpt-5-mini', '✅ *', ''], - ['gpt-5-nano', '✅ *', ''], + ['gpt-5-mini', '✅', ''], + ['gpt-5-nano', '✅', ''], ['gpt-4.1', '✅', ''], ['gpt-4.1-mini', '✅', ''], ['gpt-4.1-nano', '✅', ''], @@ -152,11 +152,6 @@ export function PromptMatrix() { ['grok-4.20', '✅', ''], ]} /> -

    - * Smaller-tier residual flakiness — gpt-5-mini and gpt-5-nano{' '} - occasionally re-emit a leading --- despite the inline guard (~1/15 on a bad - run). Re-runs clear 15/15. -

    ‡ Reasoning-token leak suppression — for reasoning-flavoured Gemini Pro variants and Grok 4.3, the fixer would otherwise see visible "Thinking: **Topic**" prose