Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 19 additions & 14 deletions materializer/src/coverage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
// The collector is deliberately template-agnostic: a new
// ScenarioTemplate added to `configs/<config>/ontology/scenario-templates.json`
// (and emitted by the planner under a new subdirectory of
// `scenarios/templates/`) extends suppression automatically as soon as
// it is wired into `templateOutputDirs`.
// `scenarios/templates/`) extends suppression automatically as soon
// as it is included in the `templates` option passed to `buildCoverage`
// (in production, that's `TEMPLATE_REGISTRY` — see
// `materializer/src/templateRegistry.ts`).
//
// `PrereqChain` steps are intentionally *excluded* from coverage —
// they are scaffolding, not units-under-test. The Invoke/Observe vs
Expand All @@ -22,6 +24,7 @@

import { promises as fs } from 'node:fs';
import path from 'node:path';
import type { TemplateBinding } from './templateRegistry.js';

export interface CoverageEntry {
operationId: string;
Expand All @@ -44,15 +47,16 @@ export interface BuildCoverageOptions {
* to treat every template as suppressing (the default). */
templatesAboxPath: string | undefined;
/**
* Maps `templateName → emitted output directory relative to the
* playwright suite root`. For each scenario file under
* `<templateScenariosRootDir>/<templateName>/<subjectName>.json` the
* The materializer's scenario-template registry — the single source
* of truth for which templates emit a spec and where. For each
* scenario file under
* `<templateScenariosRootDir>/<binding.name>/<subjectName>.json` the
* recorded `emittedSpec` becomes
* `<emittedDir>/<subjectName>.lifecycle.spec.ts`. Templates not
* present in this map are skipped (no spec is emitted, so coverage
* cannot legitimately claim the op).
* `<binding.outputDir>/<subjectName>.lifecycle.spec.ts`. Templates
* not present in this registry are skipped (no spec is emitted on
* disk, so coverage cannot legitimately claim the op).
*/
templateOutputDirs: Record<string, string>;
templates: readonly TemplateBinding[];
}

interface ScenarioStep {
Expand Down Expand Up @@ -91,16 +95,17 @@ export async function buildCoverage(opts: BuildCoverageOptions): Promise<Coverag
}
templateDirs.sort();

// Build a name → outputDir lookup from the registry. Templates the
// registry doesn't know about are skipped (no spec is emitted on disk,
// so coverage cannot legitimately claim the op).
const outputDirByName = new Map(opts.templates.map((t) => [t.name, t.outputDir]));

for (const templateName of templateDirs) {
const templateDir = path.join(opts.templateScenariosRootDir, templateName);
const stat = await fs.stat(templateDir);
if (!stat.isDirectory()) continue;

// Templates not wired to an emitted output dir don't actually
// produce a spec on disk; suppressing the feature spec on the
// strength of a scenario JSON nobody renders would create an
// untested gap. Skip.
const emittedDir = opts.templateOutputDirs[templateName];
const emittedDir = outputDirByName.get(templateName);
if (!emittedDir) continue;

// Default: every well-formed scenario template suppresses. The
Expand Down
95 changes: 28 additions & 67 deletions materializer/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import {
} from './playwright/materialize-support.js';
import { loadRoleBundlesForActiveConfig } from './playwright/roleRenderer.js';
import { emitTemplateSuites } from './playwright/templateEmitter.js';
import { TEMPLATE_REGISTRY } from './templateRegistry.js';

// Built-in emitter registrations. RoleHookProviders are no longer
// registered statically here: every provider lives next to its role
Expand Down Expand Up @@ -470,24 +471,18 @@ async function run() {
if (positional === '--all') {
// #331: scenario-template coverage. Build the suppression set
// from on-disk template scenario JSONs before the feature loop so
// operations covered by a well-formed EdgeLifecycle /
// EntityLifecycle / UpdatedFieldVisibleOnReadBack /
// StateTransitionVisibleAfterAction spec (or any future scenario
// template wired into `templateOutputDirs`) do not also emit a
// structurally weaker feature spec. Suppression only applies to
// emitters that ship the corresponding template suites; for now
// that is Playwright. See materializer/src/coverage.ts and #331.
// operations covered by a well-formed scenario-template spec do
// not also emit a structurally weaker per-endpoint feature spec.
// Suppression only applies to emitters that ship the
// corresponding template suites; for now that is Playwright. The
// canonical list of templates lives in `TEMPLATE_REGISTRY`
// (#333) — see `materializer/src/templateRegistry.ts` and #331.
let coverage: CoverageResult = { suppressedOpIds: new Set(), entries: [] };
if (emitter.id === PlaywrightEmitter.id) {
coverage = await buildCoverage({
templateScenariosRootDir: getTemplateScenariosRootDir(repoRoot),
templatesAboxPath: path.join(configDir, 'ontology', 'scenario-templates.json'),
templateOutputDirs: {
EdgeLifecycle: 'edges',
EntityLifecycle: 'entities',
UpdatedFieldVisibleOnReadBack: 'runtime-entities',
StateTransitionVisibleAfterAction: 'state-transitions',
},
templates: TEMPLATE_REGISTRY,
});
}
let count = 0;
Expand Down Expand Up @@ -538,69 +533,35 @@ async function run() {
}
}
// Template-derived suites (Lift 22 / #270; extended in #280 with
// EntityLifecycle). One Playwright suite per subject under
// `<playwrightSuiteDir>/edges/` (Edge templates) or
// `<playwrightSuiteDir>/entities/` (Entity templates). Only the
// Playwright emitter wires this — other emitters opt in by
// implementing their own template-aware renderer. The scenarios
// EntityLifecycle, #305 with UpdatedFieldVisibleOnReadBack +
// StateTransitionVisibleAfterAction). One Playwright suite per
// subject under `<playwrightSuiteDir>/<template.outputDir>/`.
// Only the Playwright emitter wires this — other emitters opt in
// by implementing their own template-aware renderer. The scenarios
// are produced by the planner (`scenarioTemplateInstantiator.ts`);
// if a directory is missing (older planner runs, configs that
// don't ship the corresponding template), `emitTemplateSuites`
// no-ops.
// no-ops. Adding a new template is one entry in `TEMPLATE_REGISTRY`
// (#333).
let lifecycleCount = 0;
if (emitter.id === PlaywrightEmitter.id) {
const seedsArg = globalContextSeeds.map((s) => ({
binding: s.binding,
seedRule: s.seedRule,
}));
const edgesOutDir = path.join(outDir, 'edges');
// Wipe the per-template subdir for the same reason the parent
// `outDir` is wiped above: stale specs from a previous spec version
// must not survive into the current run.
await fs.rm(edgesOutDir, { recursive: true, force: true });
const edgesWritten = await emitTemplateSuites({
scenariosDir: getTemplateScenariosDir(repoRoot, 'EdgeLifecycle'),
outDir: edgesOutDir,
globalContextSeeds: seedsArg,
});
lifecycleCount += edgesWritten.length;

const entitiesOutDir = path.join(outDir, 'entities');
await fs.rm(entitiesOutDir, { recursive: true, force: true });
const entitiesWritten = await emitTemplateSuites({
scenariosDir: getTemplateScenariosDir(repoRoot, 'EntityLifecycle'),
outDir: entitiesOutDir,
globalContextSeeds: seedsArg,
});
lifecycleCount += entitiesWritten.length;

// #305 Phase 4 — UpdatedFieldVisibleOnReadBack (RuntimeEntity).
// Sibling subdir to edges/ and entities/ for symmetry. Suites
// here are 3-step (prereq → mutate → observe field equality)
// and rendered via `renderReadBackSuite` inside emitTemplateSuites.
const runtimeOutDir = path.join(outDir, 'runtime-entities');
await fs.rm(runtimeOutDir, { recursive: true, force: true });
const runtimeWritten = await emitTemplateSuites({
scenariosDir: getTemplateScenariosDir(repoRoot, 'UpdatedFieldVisibleOnReadBack'),
outDir: runtimeOutDir,
globalContextSeeds: seedsArg,
});
lifecycleCount += runtimeWritten.length;

// #305 Phase 5d / #189 — StateTransitionVisibleAfterAction
// (RuntimeEntity). Sibling subdir to runtime-entities/ so the
// two RuntimeEntity-scoped templates don't fight over file
// names (Incident.resolveIncident is a state-transition;
// future Incident.mutateIncident — if ever — would be a
// readback, separate file in runtime-entities/).
const stateTransitionOutDir = path.join(outDir, 'state-transitions');
await fs.rm(stateTransitionOutDir, { recursive: true, force: true });
const stateTransitionWritten = await emitTemplateSuites({
scenariosDir: getTemplateScenariosDir(repoRoot, 'StateTransitionVisibleAfterAction'),
outDir: stateTransitionOutDir,
globalContextSeeds: seedsArg,
});
lifecycleCount += stateTransitionWritten.length;
for (const template of TEMPLATE_REGISTRY) {
const templateOutDir = path.join(outDir, template.outputDir);
// Wipe the per-template subdir for the same reason the parent
// `outDir` is wiped above: stale specs from a previous spec
// version must not survive into the current run.
await fs.rm(templateOutDir, { recursive: true, force: true });
const written = await emitTemplateSuites({
scenariosDir: getTemplateScenariosDir(repoRoot, template.name),
outDir: templateOutDir,
globalContextSeeds: seedsArg,
});
lifecycleCount += written.length;
}
}
// #331: persist the coverage artefact alongside the suites so it
// is diffable in PRs and consumable by the L3 invariant in
Expand Down
51 changes: 51 additions & 0 deletions materializer/src/templateRegistry.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
// Scenario-template registry (#333).
//
// Single source of truth for the materializer's knowledge of which
// scenario templates exist and where their emitted Playwright suites
// live. Consumed by:
//
// - `materializer/src/index.ts` — the template emission loop reads
// this registry to wire `emitTemplateSuites` per template, instead
// of open-coding one block per template.
// - `materializer/src/coverage.ts` — `buildCoverage` reads this
// registry to map template-name → emitted output directory when
// computing the per-endpoint feature-spec suppression set (#331).
//
// Adding a new ScenarioTemplate normally requires two coordinated edits:
//
// 1. Add an ABox row in `configs/<config>/ontology/scenario-templates.json`.
// 2. Add one entry here.
//
// The TBox in `path-analyser/src/ontology/scenarioTemplateSchema.ts`
// is name-agnostic (template names are just strings), so it only needs
// updating when the ABox row *shape* changes (a new step kind, a new
// `appliesTo` kind, etc.). Likewise, `emitTemplateSuites` only needs
// extending when the new template's rendered shape genuinely differs
// from the lifecycle templates it already handles.
//
// No other site in the orchestrator needs to know about the new
// template. The guard in `tests/codegen/template-registry.test.ts`
// asserts symmetric equality between the active config's ABox and
// this registry so an ABox row without a registry entry (or vice
// versa) fails red.
//
// Step 2 of #333 (deriving the registry directly from the ABox) is
// deferred; that requires teaching the materializer to dispatch
// emitters by name when future templates need shapes beyond the
// universal `emitTemplateSuites` renderer.

export interface TemplateBinding {
/** Template name as declared in `scenarioTemplateSchema.ts` and as
* the subdirectory name under `generated/<config>/scenarios/templates/`. */
name: string;
/** Output subdirectory relative to the Playwright suite root,
* i.e. `generated/<config>/playwright/<outputDir>/`. */
outputDir: string;
}

export const TEMPLATE_REGISTRY: readonly TemplateBinding[] = [
{ name: 'EdgeLifecycle', outputDir: 'edges' },
{ name: 'EntityLifecycle', outputDir: 'entities' },
{ name: 'UpdatedFieldVisibleOnReadBack', outputDir: 'runtime-entities' },
{ name: 'StateTransitionVisibleAfterAction', outputDir: 'state-transitions' },
];
16 changes: 8 additions & 8 deletions tests/codegen/coverage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ describe('buildCoverage (#331)', () => {
const result = await buildCoverage({
templateScenariosRootDir: root,
templatesAboxPath: undefined,
templateOutputDirs: { EntityLifecycle: 'entities' },
templates: [{ name: 'EntityLifecycle', outputDir: 'entities' }],
});

expect([...result.suppressedOpIds].sort()).toEqual(['createUser', 'getUser']);
Expand Down Expand Up @@ -86,10 +86,10 @@ describe('buildCoverage (#331)', () => {
const result = await buildCoverage({
templateScenariosRootDir: root,
templatesAboxPath: aboxPath,
templateOutputDirs: {
EntityLifecycle: 'entities',
SmokeTemplate: 'smoke',
},
templates: [
{ name: 'EntityLifecycle', outputDir: 'entities' },
{ name: 'SmokeTemplate', outputDir: 'smoke' },
],
});

expect(result.suppressedOpIds.has('createUser')).toBe(true);
Expand All @@ -107,14 +107,14 @@ describe('buildCoverage (#331)', () => {
const result = await buildCoverage({
templateScenariosRootDir: root,
templatesAboxPath: undefined,
templateOutputDirs: { EntityLifecycle: 'entities' },
templates: [{ name: 'EntityLifecycle', outputDir: 'entities' }],
});

expect([...result.suppressedOpIds].sort()).toEqual(['createUser', 'getUser']);
expect(result.suppressedOpIds.has('createTenant')).toBe(false);
});

test('templates not wired into templateOutputDirs are skipped (no spec emitted, no coverage claimed)', async () => {
test('templates not present in the buildCoverage templates option are skipped (no spec emitted, no coverage claimed)', async () => {
const root = path.join(tmp, 'scenarios', 'templates');
await writeScenario(root, 'UnwiredTemplate', 'Thing', [
{ kind: 'invoke', operationId: 'doThing' },
Expand All @@ -123,7 +123,7 @@ describe('buildCoverage (#331)', () => {
const result = await buildCoverage({
templateScenariosRootDir: root,
templatesAboxPath: undefined,
templateOutputDirs: {}, // not wired
templates: [], // not wired
});

expect(result.suppressedOpIds.size).toBe(0);
Expand Down
45 changes: 45 additions & 0 deletions tests/codegen/template-registry.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import path from 'node:path';
import { describe, expect, test } from 'vitest';
import { TEMPLATE_REGISTRY } from '../../materializer/src/templateRegistry.ts';
import { loadScenarioTemplatesAbox } from '../../path-analyser/src/ontology/loader.ts';

Comment thread
jwulf marked this conversation as resolved.
const REPO_ROOT = path.resolve(import.meta.dirname, '..', '..');

/**
* Registry / ABox symmetry guard (#333).
*
* Locks in the invariant that the active config's scenario-templates
* ABox and the materializer's `TEMPLATE_REGISTRY` agree on which
* templates exist. Drift in either direction is a real defect:
*
* - Template in ABox but not in registry → planner emits scenario
* JSONs that the orchestrator never renders to a spec, and
* `buildCoverage` silently skips them. Adding a template would
* appear to do nothing.
* - Template in registry but not in ABox → orchestrator wipes and
* re-creates an output dir for a template whose scenarios the
* planner never produced; the dir stays empty and `buildCoverage`
* sees no scenarios. Effectively dead wiring.
*
* The single registry file (`materializer/src/templateRegistry.ts`)
* is the only place to wire a new template into the orchestrator.
*/
describe('TEMPLATE_REGISTRY ↔ scenario-templates ABox symmetry (#333)', () => {
test('every ABox template has a registry entry, and vice versa', () => {
const abox = loadScenarioTemplatesAbox(REPO_ROOT);
if (!abox) {
throw new Error(
'scenario-templates ABox missing for the active config — cannot verify registry symmetry',
);
}
const aboxNames = abox.templates.map((t) => t.name).sort();
const registryNames = TEMPLATE_REGISTRY.map((t) => t.name).sort();
expect(registryNames).toEqual(aboxNames);
});

test('registry outputDirs are distinct (no two templates clobber the same subdir)', () => {
const outputDirs = TEMPLATE_REGISTRY.map((t) => t.outputDir);
const unique = new Set(outputDirs);
expect(unique.size).toBe(outputDirs.length);
});
});