feat: emit coverage summary in coverage.json (v2) and render Markdown report

materializer/src/coverageSummary.ts

@@ -0,0 +1,138 @@
+// Deterministic coverage summary (#335 follow-up).
+//
+// Reads the bundled OpenAPI spec for the active config and folds the
+// emitted-feature opIds, the template-derived suppression set, and the
+// raw coverage entries into a single summary block that ships inside
+// `generated/<config>/playwright/coverage.json` (v2). The summary is
+// the source of truth for:
+//
+//   • the reconciliation math (total spec ops = emitted features +
+//     suppressed-by-template + unmapped),
+//   • per-template aggregates (specs, unique ops, total entries,
+//     invoke / observe step tallies),
+//   • the unmapped-operations list (must be empty on a healthy run).
+//
+// Built once by the materializer so a separate report renderer
+// (`scripts/render-coverage-report.ts`) can transform it without
+// re-walking the planner outputs — guaranteeing the Markdown report
+// and the JSON artefact agree by construction.
+
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import type { CoverageEntry } from './coverage.js';
+
+export interface PerTemplateSummary {
+  name: string;
+  specs: number;
+  uniqueOperations: number;
+  entries: number;
+  invokeSteps: number;
+  observeSteps: number;
+}
+
+export interface CoverageSummary {
+  totalSpecOperations: number;
+  emittedFeatureSpecs: number;
+  suppressedByTemplate: number;
+  variantSpecs: number;
+  lifecycleSpecs: number;
+  unmappedOperations: string[];
+  perTemplate: PerTemplateSummary[];
+}
+
+export interface BuildCoverageSummaryInput {
+  allSpecOpIds: readonly string[];
+  emittedFeatureOpIds: ReadonlySet<string>;
+  suppressedOpIds: ReadonlySet<string>;
+  entries: readonly CoverageEntry[];
+  variantSpecs: number;
+  lifecycleSpecs: number;
+}
+
+interface BundledSpec {
+  paths?: Record<string, Record<string, unknown>>;
+}
+
+/**
+ * Collect every operationId declared in the bundled OpenAPI spec.
+ * Returns `[]` when the spec is missing (e.g. a config that ships no
+ * bundled spec) so the summary still renders deterministically.
+ */
+export async function loadSpecOperationIds(specBundleDir: string): Promise<string[]> {
+  const bundledSpecPath = path.join(specBundleDir, 'rest-api.bundle.json');
+  let raw: string;
+  try {
+    raw = await fs.readFile(bundledSpecPath, 'utf8');
+  } catch (e) {
+    if (typeof e === 'object' && e !== null && 'code' in e && Reflect.get(e, 'code') === 'ENOENT') {
+      return [];
+    }
+    throw e;
+  }
+  let spec: BundledSpec;
+  try {
+    // biome-ignore lint/plugin: runtime contract boundary — bundled OpenAPI spec; only paths[].{method}.operationId is read.
+    spec = JSON.parse(raw) as BundledSpec;
+  } catch (e) {
+    const msg = e instanceof Error ? e.message : String(e);
+    throw new Error(`coverage summary: failed to parse ${bundledSpecPath}: ${msg}`);
+  }
+  // Dedupe via Set: OpenAPI does not technically forbid duplicate
+  // operationIds, and treating them as a multiset would inflate
+  // totalSpecOperations and the unmapped count (cf. PR #337 review).
+  // Other consumers of spec opIds in this repo (e.g.
+  // configs/camunda-oca/regression-invariants.test.ts) also treat them
+  // as a Set, so the reconciliation math stays consistent.
+  const ids = new Set<string>();
+  for (const pathItem of Object.values(spec.paths ?? {})) {
+    if (!pathItem || typeof pathItem !== 'object') continue;
+    for (const op of Object.values(pathItem)) {
+      if (!op || typeof op !== 'object') continue;
+      const opId = Reflect.get(op, 'operationId');
+      if (typeof opId === 'string' && opId.length > 0) ids.add(opId);
+    }
+  }
+  return [...ids].sort();
+}
+
+export function buildCoverageSummary(input: BuildCoverageSummaryInput): CoverageSummary {
+  const accountedFor = new Set<string>([...input.emittedFeatureOpIds, ...input.suppressedOpIds]);
+  const unmappedOperations = input.allSpecOpIds.filter((id) => !accountedFor.has(id)).sort();
+
+  const perTemplateAgg = new Map<
+    string,
+    { specs: Set<string>; opIds: Set<string>; entries: number; invoke: number; observe: number }
+  >();
+  for (const e of input.entries) {
+    let agg = perTemplateAgg.get(e.template);
+    if (!agg) {
+      agg = { specs: new Set(), opIds: new Set(), entries: 0, invoke: 0, observe: 0 };
+      perTemplateAgg.set(e.template, agg);
+    }
+    agg.specs.add(e.emittedSpec);
+    agg.opIds.add(e.operationId);
+    agg.entries++;
+    if (e.stepKind === 'invoke') agg.invoke++;
+    else if (e.stepKind === 'observe') agg.observe++;
+  }
+  const perTemplate: PerTemplateSummary[] = [...perTemplateAgg.entries()]
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([name, agg]) => ({
+      name,
+      specs: agg.specs.size,
+      uniqueOperations: agg.opIds.size,
+      entries: agg.entries,
+      invokeSteps: agg.invoke,
+      observeSteps: agg.observe,
+    }));
+
+  return {
+    totalSpecOperations: input.allSpecOpIds.length,
+    emittedFeatureSpecs: input.emittedFeatureOpIds.size,
+    suppressedByTemplate: input.suppressedOpIds.size,
+    variantSpecs: input.variantSpecs,
+    lifecycleSpecs: input.lifecycleSpecs,
+    unmappedOperations,
+    perTemplate,
+  };
+}

materializer/src/index.ts

@@ -17,6 +17,7 @@ import {
   getActiveConfigName,
   getFeatureOutputDir,
   getPlaywrightSuiteDir,
+  getSpecBundleDir,
   getTemplateScenariosDir,
   getTemplateScenariosRootDir,
   getVariantOutputDir,
@@ -31,6 +32,7 @@ import { getEmitterRoleForOperation } from 'path-analyser/ontology/operationRole
 import type { EndpointScenarioCollection, GlobalContextSeed } from 'path-analyser/types';
 import { parseCliArgs } from './cli-args.js';
 import { buildCoverage, type CoverageResult, templateOutputDir } from './coverage.js';
+import { buildCoverageSummary, loadSpecOperationIds } from './coverageSummary.js';
 import { writeEmitted, writeScaffolded } from './orchestrator.js';
 import { PlaywrightEmitter } from './playwright/emitter.js';
 import {
@@ -496,6 +498,12 @@ async function run() {
     }
     let count = 0;
     let suppressedCount = 0;
+    // #335: track which opIds were emitted as feature specs so the
+    // coverage summary can compute the unmapped set (ops in the spec
+    // that are neither emitted as a feature spec nor suppressed by a
+    // scenario-template lifecycle suite). Should be empty on a healthy
+    // spec; a non-empty set surfaces planner / coverage drift.
+    const emittedFeatureOpIds = new Set<string>();
     for (const f of files) {
       try {
         const content = await fs.readFile(path.join(featureDir, f), 'utf8');
@@ -506,6 +514,7 @@ async function run() {
           continue;
         }
         await writeEmitted(emitter, parsed, buildCtx(parsed.endpoint.operationId, 'feature'));
+        emittedFeatureOpIds.add(parsed.endpoint.operationId);
         count++;
       } catch (e) {
         const msg = e instanceof Error ? e.message : String(e);
@@ -572,6 +581,23 @@ async function run() {
         lifecycleCount += written.length;
       }
     }
+    // #335: build a deterministic coverage summary alongside the raw
+    // suppression set / entries. The summary block answers "how many
+    // operations in the spec are covered, by what kind of suite, and
+    // by which template" without requiring readers to re-walk the
+    // feature-output / template-scenarios directories. The summary is
+    // emitted for every emitter so PR diffs and the
+    // `npm run coverage:report` script see the same shape regardless
+    // of which target the materializer was invoked for.
+    const allSpecOpIds = await loadSpecOperationIds(getSpecBundleDir(repoRoot));
+    const summary = buildCoverageSummary({
+      allSpecOpIds,
+      emittedFeatureOpIds,
+      suppressedOpIds: coverage.suppressedOpIds,
+      entries: coverage.entries,
+      variantSpecs: variantCount,
+      lifecycleSpecs: lifecycleCount,
+    });
     // #331: persist the coverage artefact alongside the suites so it
     // is diffable in PRs and consumable by the L3 invariant in
     // configs/<config>/regression-invariants.test.ts. Written for
@@ -581,7 +607,10 @@ async function run() {
       path.join(outDir, 'coverage.json'),
       `${JSON.stringify(
         {
-          version: 1,
+          version: 2,
+          config: configName,
+          emitter: emitter.id,
+          summary,
           suppressedOpIds: [...coverage.suppressedOpIds].sort(),
           entries: [...coverage.entries].sort((a, b) =>
             a.operationId === b.operationId

package.json

@@ -35,6 +35,7 @@
     "testsuite:observe:run": "npm run codegen:playwright:all && npm run test:pw:path-analyser && npm run observe:aggregate",
     "observe:aggregate": "npm run build:analyser && node path-analyser/dist/src/scripts/aggregate-observations.js",
     "optional-responses": "node optional-responses/report.js",
+    "coverage:report": "tsx scripts/render-coverage-report.ts",
     "pipeline": "npm run fetch-spec && npm run testsuite:generate && npm run generate:request-validation",
     "lint": "biome check",
     "lint:fix": "biome check --write",