feat(e2e): generate scenario fan-out matrix from typed registry

.github/workflows/e2e-scenarios-all.yaml

@@ -1,15 +1,14 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
-# Scenario-based E2E fan-out. Runs every setup scenario from the current
-# migration catalog by calling the single-scenario runner workflow.
+# Scenario-based E2E fan-out. Generates the job matrix from the typed
+# scenario registry (`test/e2e-scenario/scenarios/registry.ts`) so that
+# adding a scenario in `baseline.ts` automatically produces a tile here on
+# the next run -- no workflow edits required.
 #
-# Each child job passes its scenario id through the called workflow's
-# `scenarios` input (comma-separated; one id here per child). The legacy
-# per-call `suite_filter` input was retired when the runner moved to a
-# typed scenario contract, so this fan-out no longer accepts a fan-out-wide
-# suite filter; if per-suite scoping is needed in the future, route it
-# through `e2e-scenarios.yaml` directly with a custom scenario list.
+# The single-scenario runner workflow `e2e-scenarios.yaml` remains the
+# authoritative execution path; this file just fans out one call per
+# scenario id with the runner label resolved by `--emit-matrix`.
 
 name: E2E / Scenario Runner / All
 
@@ -24,51 +23,63 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
-  ubuntu-repo-cloud-openclaw:
-    uses: ./.github/workflows/e2e-scenarios.yaml
-    with:
-      scenarios: ubuntu-repo-cloud-openclaw
-    secrets:
-      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+  generate-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.emit.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
 
-  ubuntu-repo-cloud-hermes:
-    uses: ./.github/workflows/e2e-scenarios.yaml
-    with:
-      scenarios: ubuntu-repo-cloud-hermes
-    secrets:
-      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+      - name: Set up Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
+        with:
+          node-version: 22
+          cache: npm
 
-  gpu-repo-local-ollama-openclaw:
-    uses: ./.github/workflows/e2e-scenarios.yaml
-    with:
-      scenarios: gpu-repo-local-ollama-openclaw
-    secrets:
-      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+      - name: Install root dependencies
+        run: npm ci --ignore-scripts
 
-  macos-repo-cloud-openclaw:
-    uses: ./.github/workflows/e2e-scenarios.yaml
-    with:
-      scenarios: macos-repo-cloud-openclaw
-    secrets:
-      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
-
-  wsl-repo-cloud-openclaw:
-    uses: ./.github/workflows/e2e-scenarios.yaml
-    with:
-      scenarios: wsl-repo-cloud-openclaw
-    secrets:
-      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+      - id: emit
+        name: Emit scenario matrix from typed registry
+        run: |
+          set -euo pipefail
+          matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts --emit-matrix)"
+          # Sanity-check that the output is non-empty JSON before handing it
+          # to GHA, so a registry error fails this job loudly instead of
+          # producing zero matrix tiles.
+          if [ -z "${matrix}" ] || [ "${matrix}" = "[]" ]; then
+            echo "::error::scenario matrix is empty; check typed registry" >&2
+            exit 1
+          fi
+          echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
 
-  brev-launchable-cloud-openclaw:
-    uses: ./.github/workflows/e2e-scenarios.yaml
-    with:
-      scenarios: brev-launchable-cloud-openclaw
-    secrets:
-      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+      - name: Render matrix summary
+        env:
+          MATRIX_JSON: ${{ steps.emit.outputs.matrix }}
+        run: |
+          {
+            echo '## E2E scenario matrix'
+            echo ''
+            echo '| Scenario | Runner | Label |'
+            echo '| --- | --- | --- |'
+            python3 - <<'PY'
+          import json, os
+          for e in json.loads(os.environ["MATRIX_JSON"]):
+              print(f"| `{e['id']}` | {e['runner']} | {e['label']} |")
+          PY
+          } >> "$GITHUB_STEP_SUMMARY"
 
-  ubuntu-no-docker-preflight-negative:
+  run-scenario:
+    needs: generate-matrix
+    name: ${{ matrix.label }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
     uses: ./.github/workflows/e2e-scenarios.yaml
     with:
-      scenarios: ubuntu-no-docker-preflight-negative
+      scenarios: ${{ matrix.id }}
     secrets:
       NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}

test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts

@@ -0,0 +1,105 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { spawnSync } from "node:child_process";
+import path from "node:path";
+
+import { describe, expect, it } from "vitest";
+
+import { buildScenarioMatrix } from "../scenarios/run.ts";
+import { listScenarios } from "../scenarios/registry.ts";
+import { resolveRunnerForScenario } from "../scenarios/runner-routing.ts";
+import { scenario } from "../scenarios/builder.ts";
+
+const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
+const RUN_SCENARIOS = path.join(REPO_ROOT, "test/e2e-scenario/scenarios/run.ts");
+const TSX = path.join(REPO_ROOT, "node_modules/.bin/tsx");
+
+function runEmitMatrix() {
+  return spawnSync(TSX, [RUN_SCENARIOS, "--emit-matrix"], {
+    cwd: REPO_ROOT,
+    encoding: "utf8",
+    timeout: Number(process.env.E2E_SPAWN_TIMEOUT_MS ?? 60_000),
+  });
+}
+
+describe("typed scenario matrix", () => {
+  it("emits one matrix entry per registered scenario", () => {
+    const matrix = buildScenarioMatrix();
+    const ids = listScenarios().map((s) => s.id);
+    expect(matrix.map((entry) => entry.id).sort()).toEqual([...ids].sort());
+  });
+
+  it("resolves a runner label for every scenario", () => {
+    const matrix = buildScenarioMatrix();
+    expect(matrix.length).toBeGreaterThan(0);
+    for (const entry of matrix) {
+      expect(entry.runner, `runner missing for ${entry.id}`).toMatch(/[A-Za-z0-9-]/);
+      expect(entry.label, `label missing for ${entry.id}`).toContain(entry.id);
+    }
+  });
+
+  it("routes platforms to their canonical runners", () => {
+    const byId = new Map(buildScenarioMatrix().map((entry) => [entry.id, entry]));
+    expect(byId.get("ubuntu-repo-cloud-openclaw")?.runner).toBe("ubuntu-latest");
+    expect(byId.get("macos-repo-cloud-openclaw")?.runner).toBe("macos-26");
+    expect(byId.get("wsl-repo-cloud-openclaw")?.runner).toBe("windows-latest");
+    expect(byId.get("gpu-repo-local-ollama-openclaw")?.runner).toBe(
+      "linux-amd64-gpu-rtxpro6000-latest-1",
+    );
+  });
+
+  it("honors an explicit runs-on:<label> requirement override", () => {
+    const custom = scenario("test-runs-on-override")
+      .description("test fixture")
+      .manifest("test/e2e-scenario/manifests/openclaw-nvidia.yaml")
+      .environment({
+        platform: "ubuntu-local",
+        install: "repo-current",
+        runtime: "docker-running",
+        onboarding: "cloud-openclaw",
+      })
+      .expectedState("cloud-openclaw-ready")
+      .onboardingAssertions(["base-installed"])
+      .suites(["smoke"])
+      .runnerRequirements(["runs-on:custom-self-hosted"])
+      .build();
+    expect(resolveRunnerForScenario(custom).runner).toBe("custom-self-hosted");
+  });
+
+  it("fails loudly when a platform has no default runner mapping", () => {
+    const broken = scenario("test-unknown-platform")
+      .description("test fixture")
+      .manifest("test/e2e-scenario/manifests/openclaw-nvidia.yaml")
+      .environment({
+        platform: "made-up-platform",
+        install: "repo-current",
+        runtime: "docker-running",
+        onboarding: "cloud-openclaw",
+      })
+      .expectedState("cloud-openclaw-ready")
+      .onboardingAssertions(["base-installed"])
+      .suites(["smoke"])
+      .build();
+    expect(() => resolveRunnerForScenario(broken)).toThrow(/no default for platform/);
+  });
+
+  it("--emit-matrix prints a single-line JSON array compatible with $GITHUB_OUTPUT", () => {
+    const result = runEmitMatrix();
+    expect(result.status, result.stderr).toBe(0);
+    const lines = result.stdout.trim().split("\n");
+    expect(lines.length, "matrix output must be a single line").toBe(1);
+    const parsed = JSON.parse(lines[0]);
+    expect(Array.isArray(parsed)).toBe(true);
+    expect(parsed.length).toBe(listScenarios().length);
+    for (const entry of parsed) {
+      expect(entry).toMatchObject({
+        id: expect.any(String),
+        runner: expect.any(String),
+        label: expect.any(String),
+        platform: expect.any(String),
+        suites: expect.any(Array),
+      });
+    }
+  });
+});

test/e2e-scenario/scenarios/run.ts

@@ -1,26 +1,57 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import { realpathSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
 import { compileRunPlans, renderPlanText, writePlanArtifacts } from "./compiler.ts";
 import { ScenarioRunner } from "./orchestrators/runner.ts";
 import { listScenarios } from "./registry.ts";
+import { resolveRunnerForScenario } from "./runner-routing.ts";
+import type { ScenarioDefinition } from "./types.ts";
 
 interface Args {
   list: boolean;
   planOnly: boolean;
   dryRun: boolean;
   validateOnly: boolean;
+  emitMatrix: boolean;
   scenarios: string[];
 }
 
+/**
+ * Shape of a single GitHub Actions matrix `include` entry emitted by
+ * `--emit-matrix`. The fields are kept short and JSON-stable so the consuming
+ * workflow can reference them as `${{ matrix.id }}`, `${{ matrix.runner }}`,
+ * etc. without further parsing.
+ */
+export interface ScenarioMatrixEntry {
+  id: string;
+  runner: string;
+  label: string;
+  platform: string;
+  suites: string[];
+}
+
 function parseArgs(argv: string[]): Args {
-  const args: Args = { list: false, planOnly: false, dryRun: false, validateOnly: false, scenarios: [] };
+  const args: Args = {
+    list: false,
+    planOnly: false,
+    dryRun: false,
+    validateOnly: false,
+    emitMatrix: false,
+    scenarios: [],
+  };
   for (let i = 0; i < argv.length; i += 1) {
     const arg = argv[i];
     if (arg === "--list") {
       args.list = true;
       continue;
     }
+    if (arg === "--emit-matrix") {
+      args.emitMatrix = true;
+      continue;
+    }
     if (arg === "--plan-only") {
       args.planOnly = true;
       continue;
@@ -54,12 +85,56 @@ function printList() {
   }
 }
 
+function buildLabel(scenario: ScenarioDefinition): string {
+  const platform = scenario.environment?.platform ?? "unknown-platform";
+  const suites = scenario.suiteIds ?? [];
+  if (scenario.expectedFailure) {
+    const cls = scenario.expectedFailure.errorClass ?? "expected-failure";
+    return `${platform} \u00b7 ${scenario.id} \u00b7 expect-fail:${cls}`;
+  }
+  if (suites.length === 0) {
+    return `${platform} \u00b7 ${scenario.id}`;
+  }
+  if (suites.length <= 3) {
+    return `${platform} \u00b7 ${scenario.id} \u00b7 ${suites.join("+")}`;
+  }
+  return `${platform} \u00b7 ${scenario.id} \u00b7 ${suites.length} suites`;
+}
+
+/**
+ * Build the GitHub Actions matrix for every scenario in the typed registry.
+ * Sorted by id so workflow runs are deterministic and diffable.
+ */
+export function buildScenarioMatrix(): ScenarioMatrixEntry[] {
+  return listScenarios().map((scenario): ScenarioMatrixEntry => {
+    const { runner } = resolveRunnerForScenario(scenario);
+    return {
+      id: scenario.id,
+      runner,
+      label: buildLabel(scenario),
+      platform: scenario.environment?.platform ?? "unknown",
+      suites: scenario.suiteIds ?? [],
+    };
+  });
+}
+
+function emitMatrix() {
+  // Single line so GHA's `$GITHUB_OUTPUT` can consume it via
+  //   echo "matrix=$(npx tsx ... --emit-matrix)" >> "$GITHUB_OUTPUT"
+  // without needing heredoc multi-line output handling.
+  process.stdout.write(`${JSON.stringify(buildScenarioMatrix())}\n`);
+}
+
 async function main() {
   const args = parseArgs(process.argv.slice(2));
   if (args.list) {
     printList();
     return;
   }
+  if (args.emitMatrix) {
+    emitMatrix();
+    return;
+  }
 
   const modeCount = [args.planOnly, args.dryRun, args.validateOnly].filter(Boolean).length;
   if (modeCount !== 1) {
@@ -86,9 +161,25 @@ async function main() {
   }
 }
 
-try {
-  await main();
-} catch (error) {
-  console.error(error instanceof Error ? error.message : String(error));
-  process.exitCode = 1;
+// Only execute when invoked directly as a script. Importing this module from
+// tests (e.g. `buildScenarioMatrix`) must not trigger the CLI side-effects.
+// Compare via realpath so symlinked paths (e.g. `/tmp` -> `/private/tmp` on
+// macOS) still resolve as equal.
+function isInvokedDirectly(): boolean {
+  const entry = process.argv[1];
+  if (!entry) return false;
+  try {
+    return realpathSync(entry) === realpathSync(fileURLToPath(import.meta.url));
+  } catch {
+    return false;
+  }
+}
+
+if (isInvokedDirectly()) {
+  try {
+    await main();
+  } catch (error) {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exitCode = 1;
+  }
 }