diff --git a/.github/workflows/e2e-scenarios.yaml b/.github/workflows/e2e-scenarios.yaml index 7832ef7d26..48a05e483d 100644 --- a/.github/workflows/e2e-scenarios.yaml +++ b/.github/workflows/e2e-scenarios.yaml @@ -1,75 +1,97 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# -# Scenario-based E2E. Runs a single setup scenario by id against the -# matching runner and uploads runtime artifacts for debugging. -# -# Manual-only (workflow_dispatch) while scenario-based coverage migrates. -# Existing nightly-e2e / macos-e2e / wsl-e2e workflows remain unchanged. name: E2E / Scenario Runner on: - workflow_call: + workflow_dispatch: inputs: - scenario: - description: "Scenario id (e.g. ubuntu-repo-cloud-openclaw)" + scenarios: + description: "Comma-separated canonical typed scenario ids (for example: ubuntu-repo-cloud-openclaw,ubuntu-repo-cloud-hermes)" required: true type: string - suite_filter: - description: "Comma-separated suite ids to run (optional; defaults to the scenario's full suite list)" - required: false - default: "" - type: string - secrets: - NVIDIA_API_KEY: - required: false - workflow_dispatch: + workflow_call: inputs: - scenario: - description: "Scenario id (e.g. ubuntu-repo-cloud-openclaw)" + scenarios: + description: "Comma-separated canonical typed scenario ids" required: true type: string - suite_filter: - description: "Comma-separated suite ids to run (optional; defaults to the scenario's full suite list)" + secrets: + NVIDIA_API_KEY: required: false - default: "" - type: string permissions: contents: read concurrency: - group: e2e-scenarios-${{ inputs.scenario }} + group: e2e-scenarios-${{ inputs.scenarios || github.event.inputs.scenarios }} cancel-in-progress: false jobs: - # Route the scenario to the correct runner. - # - # Scenario ids encode their target platform as the first segment - # (e.g. `macos-repo-cloud-openclaw`, `wsl-repo-cloud-openclaw`, - # `gpu-repo-local-ollama-openclaw`). The workflow previously pinned - # `runs-on: ubuntu-latest` for every scenario, which caused non-Ubuntu - # scenarios to fail on the wrong runner (CodeRabbit review item #1). resolve-runner: runs-on: ubuntu-latest outputs: runner: ${{ steps.pick.outputs.runner }} steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + - id: pick + name: Resolve typed scenario runners env: - SCENARIO: ${{ inputs.scenario }} + SCENARIOS: ${{ inputs.scenarios || github.event.inputs.scenarios }} run: | - case "${SCENARIO}" in - macos-*) echo "runner=macos-26" >> "$GITHUB_OUTPUT" ;; - wsl-*) echo "runner=windows-latest" >> "$GITHUB_OUTPUT" ;; - gpu-*) echo "runner=linux-amd64-gpu-rtxpro6000-latest-1" >> "$GITHUB_OUTPUT" ;; - ubuntu-*|brev-*) echo "runner=ubuntu-latest" >> "$GITHUB_OUTPUT" ;; - *) - echo "::error::Unknown scenario prefix for runner selection: ${SCENARIO}" >&2 + set -euo pipefail + # Keep routing visible here while typed registry metadata is the source + # of the canonical scenario ids. Multi-runner mixed batches are rejected + # so each workflow job still runs on one correct runner. + declare -A ROUTES=( + [macos-repo-cloud-openclaw]=macos-26 + [wsl-repo-cloud-openclaw]=windows-latest + [gpu-repo-local-ollama-openclaw]=linux-amd64-gpu-rtxpro6000-latest-1 + [brev-launchable-cloud-openclaw]=ubuntu-latest + [ubuntu-no-docker-preflight-negative]=ubuntu-latest + [ubuntu-repo-cloud-hermes]=ubuntu-latest + [ubuntu-repo-cloud-hermes-discord]=ubuntu-latest + [ubuntu-repo-cloud-hermes-slack]=ubuntu-latest + [ubuntu-repo-cloud-openclaw]=ubuntu-latest + [ubuntu-repo-cloud-openclaw-brave]=ubuntu-latest + [ubuntu-repo-cloud-openclaw-discord]=ubuntu-latest + [ubuntu-repo-cloud-openclaw-double-provider-switch]=ubuntu-latest + [ubuntu-repo-cloud-openclaw-double-same-provider]=ubuntu-latest + [ubuntu-repo-cloud-openclaw-repair]=ubuntu-latest + [ubuntu-repo-cloud-openclaw-resume]=ubuntu-latest + [ubuntu-repo-cloud-openclaw-slack]=ubuntu-latest + [ubuntu-repo-cloud-openclaw-telegram]=ubuntu-latest + [ubuntu-repo-cloud-openclaw-token-rotation]=ubuntu-latest + [ubuntu-repo-openai-compatible-openclaw]=ubuntu-latest + ) + selected="" + IFS=',' read -ra IDS <<< "${SCENARIOS}" + for raw in "${IDS[@]}"; do + id="${raw//[[:space:]]/}" + [ -n "${id}" ] || continue + npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "${id}" --plan-only >/dev/null + runner="${ROUTES[$id]:-}" + if [ -z "${runner}" ]; then + echo "::error::No runner route for scenario: ${id}" >&2 + exit 1 + fi + if [ -n "${selected}" ] && [ "${selected}" != "${runner}" ]; then + echo "::error::Scenario batch spans multiple runner types (${selected}, ${runner}); split dispatch." >&2 exit 1 - ;; - esac + fi + selected="${runner}" + done + echo "runner=${selected:-ubuntu-latest}" >> "$GITHUB_OUTPUT" run-scenario: needs: resolve-runner @@ -78,75 +100,41 @@ jobs: env: WSL_DISTRO: Ubuntu NEMOCLAW_RECREATE_SANDBOX: "1" + E2E_CONTEXT_DIR: ${{ github.workspace }} steps: - name: Force LF line endings for WSL checkout - if: startsWith(inputs.scenario, 'wsl-') + if: contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') shell: powershell run: git config --global core.autocrlf false - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Node - if: ${{ !startsWith(inputs.scenario, 'wsl-') }} - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + if: ${{ !contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') }} + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 with: node-version: 22 cache: npm - name: Install root dependencies - if: ${{ !startsWith(inputs.scenario, 'wsl-') }} + if: ${{ !contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') }} run: npm ci --ignore-scripts - - name: Render coverage report - if: ${{ !startsWith(inputs.scenario, 'wsl-') }} - env: - SCENARIO: ${{ inputs.scenario }} - run: | - mkdir -p .e2e - bash test/e2e/runtime/coverage-report.sh > .e2e/coverage.md - { - echo '# E2E Scenario Report' - echo '' - # Keep workflow_dispatch input in an env var so untrusted scenario text - # is data, not YAML-interpolated shell source. - printf '**Scenario:** `%s`\n' "$SCENARIO" - echo '' - cat .e2e/coverage.md - } | tee -a "$GITHUB_STEP_SUMMARY" - - - name: Run scenario - if: ${{ !startsWith(inputs.scenario, 'wsl-') }} + - name: Run typed scenarios + if: ${{ !contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') }} env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} - E2E_SUITE_FILTER: ${{ inputs.suite_filter }} - SCENARIO: ${{ inputs.scenario }} + SCENARIOS: ${{ inputs.scenarios || github.event.inputs.scenarios }} run: | - # Keep workflow inputs in env vars so untrusted scenario text - # is data, not YAML-interpolated shell source. - set +e - bash test/e2e/runtime/run-scenario.sh "$SCENARIO" - rc=$? - set -e - { - echo '' - echo '## Scenario execution result' - echo '' - if [ "$rc" -eq 0 ]; then - printf -- '- Scenario `%s` completed successfully.\n' "$SCENARIO" - else - printf -- '- Scenario `%s` failed with exit code `%s`.\n' "$SCENARIO" "$rc" - fi - if grep -R '^SKIP:' .e2e test/e2e/logs >/tmp/e2e-skips.txt 2>/dev/null; then - echo '' - echo '### Runtime skips observed' - echo '' - sed 's/^/- `/' /tmp/e2e-skips.txt | sed 's/$/`/' - fi - } | tee -a "$GITHUB_STEP_SUMMARY" - exit "$rc" + set -euo pipefail + if [[ ! "${SCENARIOS}" =~ ^[A-Za-z0-9._-]+(,[A-Za-z0-9._-]+)*$ ]]; then + echo "::error::Invalid scenario input: ${SCENARIOS}" >&2 + exit 1 + fi + npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "${SCENARIOS}" --dry-run - name: Resolve workspace paths for WSL - if: startsWith(inputs.scenario, 'wsl-') + if: contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') shell: powershell run: | $winPath = "${{ github.workspace }}" @@ -157,122 +145,52 @@ jobs: "WSL_CHECKOUT_DIR=$wslCheckoutPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append "WSL_WORKDIR=$wslWorkdir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - name: Ensure Ubuntu WSL exists - if: startsWith(inputs.scenario, 'wsl-') - shell: powershell - run: | - wsl --list --verbose 2>&1 | Out-Default - $null = wsl -d $env:WSL_DISTRO -- echo ok 2>&1 - if ($LASTEXITCODE -ne 0) { - wsl --install -d $env:WSL_DISTRO --no-launch --web-download - wsl -d $env:WSL_DISTRO -- bash -c 'echo distro initialised' - } - wsl --set-default $env:WSL_DISTRO - - - name: Install WSL dependencies - if: startsWith(inputs.scenario, 'wsl-') - shell: powershell - run: | - $script = @' - set -euo pipefail - export DEBIAN_FRONTEND=noninteractive - printf '%s\n' 'Acquire::ForceIPv4 "true";' 'Acquire::Retries "5";' >/etc/apt/apt.conf.d/99github-actions-network - apt-get update - apt-get install -y bash ca-certificates curl git jq lsb-release make python3 python3-pip rsync tar unzip xz-utils - if ! docker info >/dev/null 2>&1; then - apt-get install -y docker.io - service docker start || /etc/init.d/docker start || true - timeout 30 bash -c 'until docker info >/dev/null 2>&1; do sleep 2; done' - fi - curl -fsSL https://deb.nodesource.com/setup_22.x | bash - - apt-get install -y nodejs - node --version - npm --version - docker --version - docker info >/dev/null - '@ - $tmp = "$env:RUNNER_TEMP\wsl-step.sh" - [IO.File]::WriteAllText($tmp, ($script -replace "`r",""), (New-Object System.Text.UTF8Encoding $false)) - $wslTmp = wsl -d $env:WSL_DISTRO -- wslpath -u ($tmp -replace '\\','/') - wsl -d $env:WSL_DISTRO -- bash -l $wslTmp - - - name: Copy checkout into WSL ext4 workspace - if: startsWith(inputs.scenario, 'wsl-') - shell: powershell - run: | - $script = @" - set -euo pipefail - rm -rf '$env:WSL_WORKDIR' - mkdir -p /tmp/nemoclaw-scenario-wsl - rsync -a --no-owner --no-group --delete --exclude '/node_modules/' --exclude '/nemoclaw/node_modules/' --exclude '/nemoclaw-blueprint/.venv/' '$env:WSL_CHECKOUT_DIR'/ '$env:WSL_WORKDIR'/ - git config --global --add safe.directory '$env:WSL_WORKDIR' - git -C '$env:WSL_WORKDIR' reset --hard HEAD - git -C '$env:WSL_WORKDIR' clean -ffdx - "@ - $tmp = "$env:RUNNER_TEMP\wsl-step.sh" - [IO.File]::WriteAllText($tmp, ($script -replace "`r",""), (New-Object System.Text.UTF8Encoding $false)) - $wslTmp = wsl -d $env:WSL_DISTRO -- wslpath -u ($tmp -replace '\\','/') - wsl -d $env:WSL_DISTRO -- bash -l $wslTmp - - - name: Install root dependencies in WSL - if: startsWith(inputs.scenario, 'wsl-') - shell: powershell - run: | - $script = @" - set -euo pipefail - cd '$env:WSL_WORKDIR' - npm ci --ignore-scripts - mkdir -p .e2e - bash test/e2e/runtime/coverage-report.sh > .e2e/coverage.md - "@ - $tmp = "$env:RUNNER_TEMP\wsl-step.sh" - [IO.File]::WriteAllText($tmp, ($script -replace "`r",""), (New-Object System.Text.UTF8Encoding $false)) - $wslTmp = wsl -d $env:WSL_DISTRO -- wslpath -u ($tmp -replace '\\','/') - wsl -d $env:WSL_DISTRO -- bash -l $wslTmp - - - name: Run scenario in WSL - if: startsWith(inputs.scenario, 'wsl-') - shell: powershell + - name: Run typed scenarios in WSL + if: contains(inputs.scenarios || github.event.inputs.scenarios, 'wsl-repo-cloud-openclaw') + shell: bash env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} - E2E_SUITE_FILTER: ${{ inputs.suite_filter }} - SCENARIO: ${{ inputs.scenario }} + SCENARIOS: ${{ inputs.scenarios || github.event.inputs.scenarios }} run: | - $env:WSLENV = "NVIDIA_API_KEY:E2E_SUITE_FILTER:NEMOCLAW_RECREATE_SANDBOX:SCENARIO:WSL_WORKDIR" - $script = @' set -euo pipefail - cd "$WSL_WORKDIR" - export NVIDIA_API_KEY - export E2E_SUITE_FILTER - export NEMOCLAW_RECREATE_SANDBOX - bash test/e2e/runtime/run-scenario.sh "$SCENARIO" - '@ - $tmp = "$env:RUNNER_TEMP\wsl-step.sh" - [IO.File]::WriteAllText($tmp, ($script -replace "`r",""), (New-Object System.Text.UTF8Encoding $false)) - $wslTmp = wsl -d $env:WSL_DISTRO -- wslpath -u ($tmp -replace '\\','/') - wsl -d $env:WSL_DISTRO -- bash -l $wslTmp - - - name: Copy WSL artifacts back to checkout - if: always() && startsWith(inputs.scenario, 'wsl-') - shell: powershell + if [[ ! "${SCENARIOS}" =~ ^[A-Za-z0-9._-]+(,[A-Za-z0-9._-]+)*$ ]]; then + echo "::error::Invalid scenario input: ${SCENARIOS}" >&2 + exit 1 + fi + wsl -d "${WSL_DISTRO}" -- env \ + NVIDIA_API_KEY="${NVIDIA_API_KEY}" \ + SCENARIOS="${SCENARIOS}" \ + WSL_CHECKOUT_DIR="${WSL_CHECKOUT_DIR}" \ + WSL_WORKDIR="${WSL_WORKDIR}" \ + bash -lc ' + set -euo pipefail + cd "${WSL_CHECKOUT_DIR}" + mkdir -p "${WSL_WORKDIR}" + export E2E_CONTEXT_DIR="${WSL_WORKDIR}" + npm ci --ignore-scripts + npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "${SCENARIOS}" --dry-run + ' + + - name: Append plan summary + if: always() + shell: bash run: | - $script = @" - set -euo pipefail - mkdir -p '$env:WSL_CHECKOUT_DIR/.e2e' '$env:WSL_CHECKOUT_DIR/test/e2e/logs' - if [ -d '$env:WSL_WORKDIR/.e2e' ]; then rsync -a '$env:WSL_WORKDIR/.e2e'/ '$env:WSL_CHECKOUT_DIR/.e2e'/; fi - if [ -d '$env:WSL_WORKDIR/test/e2e/logs' ]; then rsync -a '$env:WSL_WORKDIR/test/e2e/logs'/ '$env:WSL_CHECKOUT_DIR/test/e2e/logs'/; fi - "@ - $tmp = "$env:RUNNER_TEMP\wsl-step.sh" - [IO.File]::WriteAllText($tmp, ($script -replace "`r",""), (New-Object System.Text.UTF8Encoding $false)) - $wslTmp = wsl -d $env:WSL_DISTRO -- wslpath -u ($tmp -replace '\\','/') - wsl -d $env:WSL_DISTRO -- bash -l $wslTmp + if [ -f .e2e/plan.txt ]; then + echo '## E2E scenario plan' >> "$GITHUB_STEP_SUMMARY" + cat .e2e/plan.txt >> "$GITHUB_STEP_SUMMARY" + fi - name: Upload scenario artifacts if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: e2e-scenario-${{ inputs.scenario }} + name: e2e-scenario-${{ inputs.scenarios || github.event.inputs.scenarios }} path: | + .e2e/run-plan.json + .e2e/plan.txt + .e2e/environment.result.json + .e2e/onboarding.result.json + .e2e/runtime.result.json .e2e/ test/e2e/logs/ if-no-files-found: warn diff --git a/scripts/e2e/lint-conventions.ts b/scripts/e2e/lint-conventions.ts index e8b530a794..4a602aee09 100755 --- a/scripts/e2e/lint-conventions.ts +++ b/scripts/e2e/lint-conventions.ts @@ -5,26 +5,10 @@ /** * E2E convention lint. * - * Enforces the migration-spec conventions on - * `test/e2e/validation_suites/**` step scripts and the - * `test/e2e/test-*.sh` legacy frontier: - * - * - Suite step scripts MUST NOT re-export non-interactive env vars - * (use runtime/lib/env.sh::e2e_env_apply_noninteractive instead). - * - Suite step scripts MUST NOT register their own traps - * (runtime/lib/cleanup.sh owns teardown). - * - Suite step scripts MUST NOT call `section "..."` — filenames carry - * the phase label, and e2e_section is emitted by the runner. - * - Suite step scripts MUST NOT write to `/tmp/*.log` — use - * `$E2E_CONTEXT_DIR/logs///.log`. - * - Non-standard repo-root discovery (`git rev-parse --show-toplevel`) - * is rejected in suite step scripts; use - * `SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"` and - * walk up. - * - * Invocation: - * tsx scripts/e2e/lint-conventions.ts [--root ] - * Exits 0 on success, 1 on violations, 2 on misuse. + * Enforces conventions for `test/e2e-scenario/validation_suites/**` step scripts and + * keeps the new typed scenario suite isolated under `test/e2e-scenario/**`. + * Existing top-level `test/e2e/test-*.sh` entrypoints remain valid until a + * separate migration explicitly retires them. */ import fs from "node:fs"; @@ -48,7 +32,7 @@ const STEP_RULES: Rule[] = [ ]; for (const p of patterns) { if (p.test(body)) - return `matched ${p.source}; use runtime/lib/env.sh::e2e_env_apply_noninteractive`; + return `matched ${p.source}; non-interactive setup belongs to shared runtime helpers`; } return null; }, @@ -57,53 +41,36 @@ const STEP_RULES: Rule[] = [ id: "no-own-trap", describe: "suite step registers its own trap", test: (body) => { - // Ignore commented lines and ignore `trap` inside quoted strings by - // requiring a leading non-quote character. - const lines = body.split("\n"); - for (const raw of lines) { - const line = raw.replace(/^\s+/, ""); + for (const raw of body.split("\n")) { + const line = raw.trimStart(); if (line.startsWith("#")) continue; - if (/^trap\s+[^#]/.test(line)) { - return "registered own trap; cleanup lives in runtime/lib/cleanup.sh"; - } + if (/^trap\s+[^#]/.test(line)) + return "registered own trap; cleanup belongs to orchestrators/shared helpers"; } return null; }, }, { - id: "no-section-call", - describe: "suite step calls section/e2e_section", - test: (body) => { - const lines = body.split("\n"); - for (const raw of lines) { - const line = raw.replace(/^\s+/, ""); - if (line.startsWith("#")) continue; - if (/^section\s+["']/.test(line)) { - return "calls section; filename carries the phase label"; - } - } - return null; - }, + id: "no-section-helper", + describe: "suite step calls section helper directly", + test: (body) => + /^\s*section\s+["']/m.test(body) || /^\s*section\s*\(/m.test(body) + ? "step calls section; plan/phase output owns sections" + : null, }, { id: "no-tmp-log", - describe: "suite step writes to /tmp/*.log", - test: (body) => { - if (/>\s*\/tmp\/[^\s]*\.log/.test(body)) { - return "writes to /tmp/*.log; use $E2E_CONTEXT_DIR/logs///.log"; - } - return null; - }, + describe: "suite step writes logs under /tmp", + test: (body) => + /\/tmp\/[^\s'\"]+\.log/.test(body) ? "write logs under E2E_CONTEXT_DIR, not /tmp" : null, }, { - id: "no-git-rev-parse-repo-root", - describe: "suite step uses `git rev-parse --show-toplevel` for repo root", - test: (body) => { - if (/git\s+rev-parse\s+--show-toplevel/.test(body)) { - return 'use SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" instead'; - } - return null; - }, + id: "no-git-rev-parse-root", + describe: "suite step uses non-standard repo-root discovery", + test: (body) => + /git\s+rev-parse\s+--show-toplevel/.test(body) + ? "avoid git rev-parse repo-root discovery in suite steps" + : null, }, ]; @@ -113,80 +80,65 @@ interface LintFinding { message: string; } -function walkShellScripts(root: string): string[] { +function walk(dir: string): string[] { + if (!fs.existsSync(dir)) return []; const out: string[] = []; - const walk = (dir: string) => { - let entries: fs.Dirent[]; - try { - entries = fs.readdirSync(dir, { withFileTypes: true }); - } catch { - return; - } - for (const ent of entries) { - const full = path.join(dir, ent.name); - if (ent.isDirectory()) { - walk(full); - } else if (ent.isFile() && ent.name.endsWith(".sh")) { - out.push(full); - } - } - }; - walk(root); + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) out.push(...walk(full)); + else out.push(full); + } return out; } +function lintSuiteSteps(root: string): LintFinding[] { + const suitesDir = path.join(root, "test/e2e-scenario/validation_suites"); + const findings: LintFinding[] = []; + for (const file of walk(suitesDir).filter((entry) => entry.endsWith(".sh"))) { + const rel = path.relative(root, file); + const body = fs.readFileSync(file, "utf8"); + for (const rule of STEP_RULES) { + const message = rule.test(body); + if (message) findings.push({ file: rel, rule: rule.id, message }); + } + } + return findings; +} + +function lint(root: string): LintFinding[] { + return lintSuiteSteps(root); +} + function parseArgs(argv: string[]): { root: string } { - let root: string | undefined; + let root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const args = argv.slice(2); while (args.length > 0) { - const a = args.shift()!; - if (a === "--root") root = args.shift(); - else if (a === "-h" || a === "--help") { + const arg = args.shift(); + if (arg === "--root") { + const value = args.shift(); + if (!value) throw new Error("--root requires a value"); + root = path.resolve(value); + } else if (arg === "--help" || arg === "-h") { process.stdout.write("tsx scripts/e2e/lint-conventions.ts [--root ]\n"); process.exit(0); - } else { - process.stderr.write(`lint-conventions: unexpected arg: ${a}\n`); - process.exit(2); + } else if (arg) { + throw new Error(`unexpected arg: ${arg}`); } } - if (!root) { - const scriptDir = path.dirname(fileURLToPath(import.meta.url)); - root = path.resolve(scriptDir, "..", ".."); - } return { root }; } -function lintSuiteSteps(root: string): LintFinding[] { - const findings: LintFinding[] = []; - const suitesRoot = path.join(root, "test/e2e/validation_suites"); - if (!fs.existsSync(suitesRoot)) return findings; - for (const file of walkShellScripts(suitesRoot)) { - const body = fs.readFileSync(file, "utf8"); - for (const rule of STEP_RULES) { - const msg = rule.test(body); - if (msg) { - findings.push({ - file: path.relative(root, file), - rule: rule.id, - message: msg, - }); - } - } - } - return findings; -} - -function main(): number { +try { const { root } = parseArgs(process.argv); - const findings = lintSuiteSteps(root); - if (findings.length === 0) { - return 0; - } - for (const f of findings) { - process.stderr.write(`${f.file}: [${f.rule}] ${f.message}\n`); + const findings = lint(root); + if (findings.length > 0) { + for (const finding of findings) { + process.stderr.write(`${finding.file}: ${finding.rule}: ${finding.message}\n`); + } + process.exit(1); } - process.stderr.write(`\ne2e-convention-lint: ${findings.length} violation(s)\n`); - return 1; + process.stdout.write("e2e convention lint passed\n"); +} catch (err) { + process.stderr.write(`lint-conventions: ${(err as Error).message}\n`); + process.exit(2); } - -process.exit(main()); diff --git a/src/lib/actions/gateway-drift-preflight.test.ts b/src/lib/actions/gateway-drift-preflight.test.ts index 7aec1e9410..189a67f663 100644 --- a/src/lib/actions/gateway-drift-preflight.test.ts +++ b/src/lib/actions/gateway-drift-preflight.test.ts @@ -154,6 +154,34 @@ describe("gateway drift preflight for maintenance actions", () => { expect(backupSandboxStateSpy).not.toHaveBeenCalled(); }); + it("backup-all skips sandboxes that are not in Ready phase", async () => { + const registry = requireDist("../../../dist/lib/state/registry.js"); + (registry.listSandboxes as ReturnType).mockReturnValue({ + sandboxes: [ + { name: "alpha", provider: "nvidia-prod", model: "nemotron" }, + { name: "beta", provider: "nvidia-prod", model: "nemotron" }, + ], + }); + captureOpenshellSpy.mockReturnValue({ + status: 0, + output: [ + "NAME NAMESPACE CREATED PHASE", + "alpha openshell 2026-03-24 10:00:00 Ready", + "beta openshell 2026-03-24 10:01:00 Error", + ].join("\n"), + }); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + spies.push(logSpy); + + await backupAll(); + + expect(backupSandboxStateSpy).toHaveBeenCalledWith("alpha"); + expect(backupSandboxStateSpy).not.toHaveBeenCalledWith("beta"); + expect(logSpy.mock.calls.flat().join("\n")).toContain( + "Skipping 'beta' (not running)", + ); + }); + it("backup-all fails closed on protobuf mismatch instead of treating sandboxes as stopped", async () => { const protobufIssue: OpenShellStateRpcIssue = { kind: "protobuf_mismatch", diff --git a/src/lib/actions/maintenance.ts b/src/lib/actions/maintenance.ts index 7edabd5c65..fa0f81096a 100644 --- a/src/lib/actions/maintenance.ts +++ b/src/lib/actions/maintenance.ts @@ -19,7 +19,7 @@ import { captureSandboxListWithGatewayRecovery, printSandboxListFailureWithRecoveryContext, } from "../openshell-sandbox-list"; -import { parseLiveSandboxNames } from "../runtime-recovery"; +import { parseReadySandboxNames } from "../runtime-recovery"; import * as registry from "../state/registry"; import * as sandboxState from "../state/sandbox"; @@ -62,13 +62,13 @@ export async function backupAll(): Promise { printSandboxListFailureWithRecoveryContext(liveListRecovery); process.exit(liveList.status || 1); } - const liveNames = parseLiveSandboxNames(liveList.output || ""); + const readyNames = parseReadySandboxNames(liveList.output || ""); let backed = 0; let failed = 0; let skipped = 0; for (const sb of sandboxes) { - if (!liveNames.has(sb.name)) { + if (!readyNames.has(sb.name)) { console.log(` ${D}Skipping '${sb.name}' (not running)${R}`); skipped++; continue; diff --git a/src/lib/actions/upgrade-sandboxes.ts b/src/lib/actions/upgrade-sandboxes.ts index 9d976e2efd..ef2be6cd2d 100644 --- a/src/lib/actions/upgrade-sandboxes.ts +++ b/src/lib/actions/upgrade-sandboxes.ts @@ -23,7 +23,7 @@ import { captureSandboxListWithGatewayRecovery, printSandboxListFailureWithRecoveryContext, } from "../openshell-sandbox-list"; -import { parseLiveSandboxNames } from "../runtime-recovery"; +import { parseReadySandboxNames } from "../runtime-recovery"; import * as sandboxVersion from "../sandbox/version"; import * as registry from "../state/registry"; import { rebuildSandbox } from "./sandbox/rebuild"; @@ -68,7 +68,7 @@ export async function upgradeSandboxes( printSandboxListFailureWithRecoveryContext(liveRecovery); process.exit(liveResult.status || 1); } - const liveNames = parseLiveSandboxNames(liveResult.output || ""); + const liveNames = parseReadySandboxNames(liveResult.output || ""); // Classify sandboxes as stale, unknown, or current const { stale, unknown } = classifyUpgradeableSandboxes( diff --git a/src/lib/runtime-recovery.test.ts b/src/lib/runtime-recovery.test.ts index aeebeef8e8..23c31016e9 100644 --- a/src/lib/runtime-recovery.test.ts +++ b/src/lib/runtime-recovery.test.ts @@ -4,7 +4,10 @@ import { describe, expect, it } from "vitest"; // Import from compiled dist/ for correct coverage attribution. -import { parseLiveSandboxNames } from "../../dist/lib/runtime-recovery"; +import { + parseLiveSandboxNames, + parseReadySandboxNames, +} from "../../dist/lib/runtime-recovery"; describe("runtime recovery helpers", () => { it("parses live sandbox names from openshell sandbox list output", () => { @@ -40,4 +43,80 @@ describe("runtime recovery helpers", () => { expect(Array.from(parseLiveSandboxNames(""))).toEqual([]); expect(Array.from(parseLiveSandboxNames())).toEqual([]); }); + + it("does not drop sandboxes whose name starts with 'name' or 'no'", () => { + expect( + Array.from( + parseLiveSandboxNames( + [ + "NAME NAMESPACE CREATED PHASE", + "name-prod openshell 2026-03-24 10:00:00 Ready", + "no-sandboxes openshell 2026-03-24 10:01:00 Ready", + ].join("\n"), + ), + ), + ).toEqual(["name-prod", "no-sandboxes"]); + }); + + describe("parseReadySandboxNames", () => { + it("includes only sandboxes whose PHASE is Ready", () => { + expect( + Array.from( + parseReadySandboxNames( + [ + "NAME NAMESPACE CREATED PHASE", + "alpha openshell 2026-03-24 10:00:00 Ready", + "beta openshell 2026-03-24 10:01:00 Provisioning", + "gamma openshell 2026-03-24 10:02:00 Error", + "delta openshell 2026-03-24 10:03:00 Ready", + ].join("\n"), + ), + ), + ).toEqual(["alpha", "delta"]); + }); + + it("skips sandboxes that report Error PHASE (stopped container)", () => { + expect( + Array.from( + parseReadySandboxNames( + [ + "NAME NAMESPACE CREATED PHASE", + "stopped-one openshell 2026-03-24 10:00:00 Error", + ].join("\n"), + ), + ), + ).toEqual([]); + }); + + it("treats no-sandboxes output, error lines, and protobuf mismatch as empty", () => { + expect(Array.from(parseReadySandboxNames("No sandboxes found."))).toEqual([]); + expect(Array.from(parseReadySandboxNames("Error: something went wrong"))).toEqual([]); + expect( + Array.from( + parseReadySandboxNames( + 'Error: × status: Internal, message: "Sandbox.metadata: SandboxResponse.sandbox: invalid wire type value: 6"', + ), + ), + ).toEqual([]); + }); + + it("handles empty input", () => { + expect(Array.from(parseReadySandboxNames(""))).toEqual([]); + expect(Array.from(parseReadySandboxNames())).toEqual([]); + }); + + it("does not drop Ready sandboxes whose name starts with 'name' or 'no'", () => { + expect( + Array.from( + parseReadySandboxNames( + [ + "NAME NAMESPACE CREATED PHASE", + "name-prod openshell 2026-03-24 10:00:00 Ready", + "no-sandboxes openshell 2026-03-24 10:01:00 Ready", + ].join("\n"), + ), + ), + ).toEqual(["name-prod", "no-sandboxes"]); + }); + }); }); diff --git a/src/lib/runtime-recovery.ts b/src/lib/runtime-recovery.ts index 03cbd39c36..263aef4fe8 100644 --- a/src/lib/runtime-recovery.ts +++ b/src/lib/runtime-recovery.ts @@ -20,19 +20,39 @@ export function isOpenShellProtobufSchemaMismatch(output = ""): boolean { ); } +function isNonSandboxRow(line: string, firstCol: string): boolean { + if (firstCol === "NAME") return true; + if (line === "No sandboxes found" || line === "No sandboxes found.") return true; + if (/^Error:/i.test(line)) return true; + if (isOpenShellProtobufSchemaMismatch(line)) return true; + return false; +} + export function parseLiveSandboxNames(listOutput = ""): Set { const clean = stripAnsi(listOutput); const names = new Set(); for (const rawLine of clean.split("\n")) { const line = rawLine.trim(); if (!line) continue; - if (/^(NAME|No sandboxes found\.?$)/i.test(line)) continue; - if (/^Error:/i.test(line)) continue; - if (isOpenShellProtobufSchemaMismatch(line)) continue; const cols = line.split(/\s+/); - if (cols[0]) { - names.add(cols[0]); - } + if (!cols[0]) continue; + if (isNonSandboxRow(line, cols[0])) continue; + names.add(cols[0]); + } + return names; +} + +export function parseReadySandboxNames(listOutput = ""): Set { + const clean = stripAnsi(listOutput); + const names = new Set(); + for (const rawLine of clean.split("\n")) { + const line = rawLine.trim(); + if (!line) continue; + const cols = line.split(/\s+/); + if (!cols[0]) continue; + if (isNonSandboxRow(line, cols[0])) continue; + if (cols.at(-1) !== "Ready") continue; + names.add(cols[0]); } return names; } diff --git a/test/e2e-scenario-advisor.test.ts b/test/e2e-scenario-advisor.test.ts index a70285d4b0..33ddf2656f 100644 --- a/test/e2e-scenario-advisor.test.ts +++ b/test/e2e-scenario-advisor.test.ts @@ -37,7 +37,7 @@ describe("E2E scenario advisor", () => { it("requires targeted scenario E2E when a validation suite changes", () => { const result = analyze([ - "test/e2e/validation_suites/messaging/telegram/00-telegram-injection-safety.sh", + "test/e2e-scenario/validation_suites/messaging/telegram/00-telegram-injection-safety.sh", ]); expect(result.required).toContainEqual( @@ -52,8 +52,8 @@ describe("E2E scenario advisor", () => { it("requires all scenario E2E and targeted follow-up when suite metadata changes", () => { const result = analyze([ - "test/e2e/validation_suites/suites.yaml", - "test/e2e/validation_suites/messaging/telegram/00-telegram-injection-safety.sh", + "test/e2e-scenario/validation_suites/suites.yaml", + "test/e2e-scenario/validation_suites/messaging/telegram/00-telegram-injection-safety.sh", ]); expect(result.required).toContainEqual( diff --git a/test/e2e/docs/MIGRATION.md b/test/e2e-scenario/docs/MIGRATION.md similarity index 100% rename from test/e2e/docs/MIGRATION.md rename to test/e2e-scenario/docs/MIGRATION.md diff --git a/test/e2e/docs/README.md b/test/e2e-scenario/docs/README.md similarity index 85% rename from test/e2e/docs/README.md rename to test/e2e-scenario/docs/README.md index 59c5aa338e..15ad01d88d 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e-scenario/docs/README.md @@ -35,19 +35,19 @@ validation. Plan-only resolution accepts either an alias or a test plan ID: ```bash -bash test/e2e/runtime/run-scenario.sh ubuntu-repo-cloud-openclaw --plan-only -bash test/e2e/runtime/run-scenario.sh ubuntu-repo-docker__cloud-nvidia-openclaw --plan-only +bash test/e2e-scenario/runtime/run-scenario.sh ubuntu-repo-cloud-openclaw --plan-only +bash test/e2e-scenario/runtime/run-scenario.sh ubuntu-repo-docker__cloud-nvidia-openclaw --plan-only ``` ## How to run ```bash -bash test/e2e/runtime/run-scenario.sh --plan-only # resolve + print plan, no side effects -bash test/e2e/runtime/run-scenario.sh --dry-run # helpers short-circuit with trace -bash test/e2e/runtime/run-scenario.sh --validate-only # assume setup done; validate expected state -bash test/e2e/runtime/run-scenario.sh # full live run -bash test/e2e/runtime/run-suites.sh […] -bash test/e2e/runtime/coverage-report.sh # Markdown matrix of scenario × suite +bash test/e2e-scenario/runtime/run-scenario.sh --plan-only # resolve + print plan, no side effects +bash test/e2e-scenario/runtime/run-scenario.sh --dry-run # helpers short-circuit with trace +bash test/e2e-scenario/runtime/run-scenario.sh --validate-only # assume setup done; validate expected state +bash test/e2e-scenario/runtime/run-scenario.sh # full live run +bash test/e2e-scenario/runtime/run-suites.sh […] +bash test/e2e-scenario/runtime/coverage-report.sh # Markdown matrix of scenario × suite ``` Override the runtime context dir with `E2E_CONTEXT_DIR=` (default diff --git a/test/e2e/docs/parity-inventory.generated.json b/test/e2e-scenario/docs/parity-inventory.generated.json similarity index 100% rename from test/e2e/docs/parity-inventory.generated.json rename to test/e2e-scenario/docs/parity-inventory.generated.json diff --git a/test/e2e-scenario/framework-tests/e2e-assertion-modules.test.ts b/test/e2e-scenario/framework-tests/e2e-assertion-modules.test.ts new file mode 100644 index 0000000000..aff7cb112f --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-assertion-modules.test.ts @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import path from "node:path"; +import yaml from "js-yaml"; + +import { + assertionGroupForSuite, + assertionGroupsForScenario, + assertionRegistry, + validateAssertionGroups, +} from "../scenarios/assertions/registry.ts"; +import { listScenarios } from "../scenarios/registry.ts"; +import type { AssertionGroup } from "../scenarios/types.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const E2E_DIR = path.join(REPO_ROOT, "test/e2e"); +const SUITES_PATH = path.join(E2E_DIR, "validation_suites", "suites.yaml"); + +type AnyRecord = Record; + +function loadYaml(filePath: string): AnyRecord { + const doc = yaml.load(fs.readFileSync(filePath, "utf8")); + if (!doc || typeof doc !== "object") { + throw new Error(`${filePath} did not parse to an object`); + } + return doc as AnyRecord; +} + +function allPlannedAssertionGroupIds(): Set { + return new Set( + listScenarios().flatMap((scenario) => assertionGroupsForScenario(scenario).map((group) => group.id)), + ); +} + +describe("assertion modules", () => { + it("test_should_define_onboarding_assertions_in_modules", () => { + const onboardingGroups = assertionRegistry.groups.filter((group) => group.phase === "onboarding"); + const stepIds = new Set(onboardingGroups.flatMap((group) => group.steps.map((step) => step.id))); + + for (const id of ["onboarding.base.cli-installed", "onboarding.preflight.passed", "onboarding.preflight.expected-failed"]) { + expect(stepIds.has(id), `missing onboarding step ${id}`).toBe(true); + } + for (const step of onboardingGroups.flatMap((group) => group.steps)) { + expect(step.phase).toBe("onboarding"); + expect(step.implementation?.ref).toMatch(/^test\/e2e-scenario\/onboarding_assertions\//); + } + }); + + it("test_should_map_every_old_validation_suite_to_canonical_assertion_group", () => { + const suites = loadYaml(SUITES_PATH).suites as AnyRecord; + + for (const suiteId of Object.keys(suites)) { + const group = assertionGroupForSuite(suiteId); + expect(group?.id, `missing assertion group for suite ${suiteId}`).toBe(`suite.${suiteId}`); + expect(group?.steps.length, `suite ${suiteId} must not be alias-only`).toBeGreaterThan(0); + expect(group?.steps.every((step) => step.implementation?.kind !== "pending")).toBe(true); + } + }); + + it("test_should_require_each_assertion_group_to_have_steps", () => { + const emptyGroup: AssertionGroup = { id: "empty", phase: "runtime", steps: [] }; + + expect(() => validateAssertionGroups([...assertionRegistry.groups, emptyGroup], E2E_DIR)).toThrow(/empty/); + }); + + it("test_should_require_each_assertion_group_to_be_used_by_a_scenario_plan", () => { + const planned = allPlannedAssertionGroupIds(); + const unused = assertionRegistry.groups.map((group) => group.id).filter((id) => !planned.has(id)); + + expect(unused, `unused assertion groups: ${unused.join(", ")}`).toEqual([]); + }); + + it("test_should_fail_when_assertion_step_references_missing_script", () => { + const badGroup: AssertionGroup = { + id: "bad.missing-script", + phase: "runtime", + steps: [ + { + id: "bad.missing-script.step", + phase: "runtime", + implementation: { kind: "shell", ref: "test/e2e-scenario/validation_suites/does-not-exist.sh" }, + evidencePath: ".e2e/bad.log", + }, + ], + }; + + expect(() => validateAssertionGroups([badGroup], E2E_DIR)).toThrow(/does-not-exist/); + }); + + it("test_should_fail_when_retry_attempts_lack_classifier", () => { + const badGroup: AssertionGroup = { + id: "bad.retry", + phase: "runtime", + steps: [ + { + id: "bad.retry.step", + phase: "runtime", + implementation: { kind: "probe", ref: "fakeProbe" }, + evidencePath: ".e2e/bad.log", + reliability: { retry: { attempts: 2, on: [] } }, + }, + ], + }; + + expect(() => validateAssertionGroups([badGroup], E2E_DIR)).toThrow(/classifier|retry/i); + }); + + it("test_should_block_complete_status_for_manual_classification_steps", () => { + expect(() => validateAssertionGroups(assertionRegistry.groups, E2E_DIR)).not.toThrow(/needs-manual-classification/); + expect(assertionRegistry.groups.every((group) => group.migrationStatus === "complete")).toBe(true); + }); +}); diff --git a/test/e2e/scenario-framework-tests/e2e-context-helper.test.ts b/test/e2e-scenario/framework-tests/e2e-context-helper.test.ts similarity index 96% rename from test/e2e/scenario-framework-tests/e2e-context-helper.test.ts rename to test/e2e-scenario/framework-tests/e2e-context-helper.test.ts index d619bcb4cd..6a7c97959f 100644 --- a/test/e2e/scenario-framework-tests/e2e-context-helper.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-context-helper.test.ts @@ -8,8 +8,8 @@ import os from "node:os"; import path from "node:path"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const CONTEXT_LIB = path.join(REPO_ROOT, "test/e2e/runtime/lib/context.sh"); -const RUN_SCENARIO = path.join(REPO_ROOT, "test/e2e/runtime/run-scenario.sh"); +const CONTEXT_LIB = path.join(REPO_ROOT, "test/e2e-scenario/runtime/lib/context.sh"); +const RUN_SCENARIO = path.join(REPO_ROOT, "test/e2e-scenario/runtime/run-scenario.sh"); function runBash(script: string, env: Record = {}): SpawnSyncReturns { return spawnSync("bash", ["-c", script], { diff --git a/test/e2e/scenario-framework-tests/e2e-convention-lint.test.ts b/test/e2e-scenario/framework-tests/e2e-convention-lint.test.ts similarity index 91% rename from test/e2e/scenario-framework-tests/e2e-convention-lint.test.ts rename to test/e2e-scenario/framework-tests/e2e-convention-lint.test.ts index 3fada280b5..24da68cf75 100644 --- a/test/e2e/scenario-framework-tests/e2e-convention-lint.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-convention-lint.test.ts @@ -22,17 +22,18 @@ function runTsx(scriptPath: string, args: string[] = [], env: Record/test/e2e/validation_suites//.sh (suite step scripts) + * /test/e2e-scenario/validation_suites//.sh (suite step scripts) * /test/e2e/test-*.sh (legacy scripts) */ function makeSyntheticRepo(): string { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-lint-")); - fs.mkdirSync(path.join(tmp, "test/e2e/validation_suites/example"), { recursive: true }); + fs.mkdirSync(path.join(tmp, "test/e2e-scenario/validation_suites/example"), { recursive: true }); + fs.mkdirSync(path.join(tmp, "test/e2e"), { recursive: true }); return tmp; } function writeStep(tmp: string, name: string, body: string) { - const p = path.join(tmp, "test/e2e/validation_suites/example", name); + const p = path.join(tmp, "test/e2e-scenario/validation_suites/example", name); fs.writeFileSync(p, `#!/usr/bin/env bash\n${body}\n`); } diff --git a/test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts b/test/e2e-scenario/framework-tests/e2e-coverage-report.test.ts similarity index 100% rename from test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts rename to test/e2e-scenario/framework-tests/e2e-coverage-report.test.ts diff --git a/test/e2e/scenario-framework-tests/e2e-expected-failure.test.ts b/test/e2e-scenario/framework-tests/e2e-expected-failure.test.ts similarity index 100% rename from test/e2e/scenario-framework-tests/e2e-expected-failure.test.ts rename to test/e2e-scenario/framework-tests/e2e-expected-failure.test.ts diff --git a/test/e2e/scenario-framework-tests/e2e-expected-state-validator.test.ts b/test/e2e-scenario/framework-tests/e2e-expected-state-validator.test.ts similarity index 99% rename from test/e2e/scenario-framework-tests/e2e-expected-state-validator.test.ts rename to test/e2e-scenario/framework-tests/e2e-expected-state-validator.test.ts index da7a379999..ba1f2b5f31 100644 --- a/test/e2e/scenario-framework-tests/e2e-expected-state-validator.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-expected-state-validator.test.ts @@ -14,7 +14,7 @@ import { import type { ExpectedStateConfig, ResolvedSuite } from "../runtime/resolver/schema.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const RUN_SCENARIO = path.join(REPO_ROOT, "test/e2e/runtime/run-scenario.sh"); +const RUN_SCENARIO = path.join(REPO_ROOT, "test/e2e-scenario/runtime/run-scenario.sh"); function cloudOpenclawReady(): ExpectedStateConfig { return { diff --git a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts b/test/e2e-scenario/framework-tests/e2e-lib-helpers.test.ts similarity index 99% rename from test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts rename to test/e2e-scenario/framework-tests/e2e-lib-helpers.test.ts index 5f72e49054..1a5c1a8403 100644 --- a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-lib-helpers.test.ts @@ -8,14 +8,14 @@ import os from "node:os"; import path from "node:path"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const RUNTIME_LIB = path.join(REPO_ROOT, "test/e2e/runtime/lib"); -const VALIDATION_SUITES = path.join(REPO_ROOT, "test/e2e/validation_suites"); +const RUNTIME_LIB = path.join(REPO_ROOT, "test/e2e-scenario/runtime/lib"); +const VALIDATION_SUITES = path.join(REPO_ROOT, "test/e2e-scenario/validation_suites"); const VALIDATION_LIB = path.join(VALIDATION_SUITES, "lib"); const ASSERT = path.join(VALIDATION_SUITES, "assert"); const REBUILD_UPGRADE_LIB = path.join(VALIDATION_SUITES, "lib/rebuild_upgrade.sh"); -const FIXTURES = path.join(REPO_ROOT, "test/e2e/nemoclaw_scenarios/fixtures"); -const INSTALL_DIR = path.join(REPO_ROOT, "test/e2e/nemoclaw_scenarios/install"); -const RUN_SCENARIO = path.join(REPO_ROOT, "test/e2e/runtime/run-scenario.sh"); +const FIXTURES = path.join(REPO_ROOT, "test/e2e-scenario/nemoclaw_scenarios/fixtures"); +const INSTALL_DIR = path.join(REPO_ROOT, "test/e2e-scenario/nemoclaw_scenarios/install"); +const RUN_SCENARIO = path.join(REPO_ROOT, "test/e2e-scenario/runtime/run-scenario.sh"); function runBash(script: string, env: Record = {}): SpawnSyncReturns { return spawnSync("bash", ["-c", script], { diff --git a/test/e2e-scenario/framework-tests/e2e-manifests.test.ts b/test/e2e-scenario/framework-tests/e2e-manifests.test.ts new file mode 100644 index 0000000000..816376ff7b --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-manifests.test.ts @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import path from "node:path"; + +import { compileRunPlans } from "../scenarios/compiler.ts"; +import { loadManifest, loadManifestsFromDir, validateManifest } from "../scenarios/manifests.ts"; +import { listScenarios } from "../scenarios/registry.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const SCENARIO_SUITE_DIR = path.join(REPO_ROOT, "test/e2e-scenario"); +const MANIFEST_DIR = path.join(SCENARIO_SUITE_DIR, "manifests"); + +describe("NemoClawInstance manifests", () => { + it("test_should_validate_all_nemoclaw_instance_manifests", () => { + const manifests = loadManifestsFromDir(MANIFEST_DIR); + + expect(manifests.length).toBeGreaterThanOrEqual(19); + for (const manifest of manifests) { + expect(() => validateManifest(manifest.document, manifest.filePath)).not.toThrow(); + } + }); + + it("test_should_reject_manifest_with_assertion_or_suite_ids", () => { + const badManifest = { + apiVersion: "nemoclaw.io/v1", + kind: "NemoClawInstance", + metadata: { name: "bad" }, + spec: { + setup: { install: { source: "repo-current" } }, + onboarding: { agent: "openclaw", provider: "nvidia" }, + assertions: ["runtime.smoke"], + suites: ["smoke"], + }, + }; + + expect(() => validateManifest(badManifest, "bad.yaml")).toThrow(/assertion|suite|product-facing/i); + }); + + it("test_should_reject_raw_secret_values_in_manifest", () => { + const badManifest = { + apiVersion: "nemoclaw.io/v1", + kind: "NemoClawInstance", + metadata: { name: "bad-secret" }, + spec: { + setup: { install: { source: "repo-current" } }, + onboarding: { agent: "openclaw", provider: "nvidia", apiKey: "nvapi-literal-secret" }, + state: { credentialRefs: ["NVIDIA_API_KEY"] }, + }, + }; + + expect(() => validateManifest(badManifest, "bad-secret.yaml")).toThrow(/raw secret|credentialRefs/i); + }); + + it("test_should_cover_every_typed_scenario_manifest_need", () => { + const manifestNames = new Set(loadManifestsFromDir(MANIFEST_DIR).map((manifest) => manifest.document.metadata.name)); + const missingManifests = listScenarios() + .map((scenario) => scenario.manifestPath) + .filter((manifestPath): manifestPath is string => Boolean(manifestPath)) + .map((manifestPath) => path.basename(manifestPath, ".yaml")) + .filter((id) => !manifestNames.has(id)); + + expect(missingManifests, `missing manifest files: ${missingManifests.join(", ")}`).toEqual([]); + }); + + it("plan_only_output_should_show_resolved_manifest_setup_and_onboarding_choices", () => { + const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); + + expect(plan.manifestPath).toBe("test/e2e-scenario/manifests/openclaw-nvidia.yaml"); + expect(plan.manifestPath).toBeDefined(); + expect(plan.manifest).toEqual(loadManifest(path.join(REPO_ROOT, plan.manifestPath as string)).document); + expect(plan.manifest?.spec.setup.install.source).toBe("repo-current"); + expect(plan.manifest?.spec.onboarding.agent).toBe("openclaw"); + expect(plan.manifest?.spec.onboarding.provider).toBe("nvidia"); + }); +}); diff --git a/test/e2e/scenario-framework-tests/e2e-metadata-final-hygiene.test.ts b/test/e2e-scenario/framework-tests/e2e-metadata-final-hygiene.test.ts similarity index 100% rename from test/e2e/scenario-framework-tests/e2e-metadata-final-hygiene.test.ts rename to test/e2e-scenario/framework-tests/e2e-metadata-final-hygiene.test.ts diff --git a/test/e2e-scenario/framework-tests/e2e-migration-inventory-lock.test.ts b/test/e2e-scenario/framework-tests/e2e-migration-inventory-lock.test.ts new file mode 100644 index 0000000000..c3af81dfca --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-migration-inventory-lock.test.ts @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import path from "node:path"; +import yaml from "js-yaml"; + +import { assertionRegistry } from "../scenarios/assertions/registry.ts"; +import { migrationInventory } from "../scenarios/migration-inventory.ts"; +import { listScenarios } from "../scenarios/registry.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const SCENARIO_SUITE_DIR = path.join(REPO_ROOT, "test/e2e-scenario"); +const SCENARIOS_PATH = path.join(SCENARIO_SUITE_DIR, "nemoclaw_scenarios", "scenarios.yaml"); +const EXPECTED_STATES_PATH = path.join(SCENARIO_SUITE_DIR, "nemoclaw_scenarios", "expected-states.yaml"); +const SUITES_PATH = path.join(SCENARIO_SUITE_DIR, "validation_suites", "suites.yaml"); + +type AnyRecord = Record; + +function loadYaml(filePath: string): AnyRecord { + const doc = yaml.load(fs.readFileSync(filePath, "utf8")); + if (!doc || typeof doc !== "object") { + throw new Error(`${filePath} did not parse to an object`); + } + return doc as AnyRecord; +} + +function keysFrom(record: unknown): string[] { + if (!record || typeof record !== "object" || Array.isArray(record)) { + return []; + } + return Object.keys(record as AnyRecord).sort(); +} + +function expectCovered(kind: keyof typeof migrationInventory, ids: string[]) { + const mappedIds = new Set(migrationInventory[kind].map((entry) => entry.id)); + const missing = ids.filter((id) => !mappedIds.has(id)); + expect(missing, `missing ${kind} migration target(s): ${missing.join(", ")}`).toEqual([]); +} + +describe("hybrid scenario migration inventory lock", () => { + it("old_scenarios_yaml_should_be_non_runtime_reference_only", () => { + const scenarios = loadYaml(SCENARIOS_PATH); + + expect(scenarios).toHaveProperty("setup_scenarios"); + expect(scenarios).toHaveProperty("base_scenarios"); + expect(scenarios).toHaveProperty("onboarding_profiles"); + expect(scenarios).toHaveProperty("test_plans"); + expect(scenarios).toHaveProperty("onboarding_assertions"); + }); + + it("typed_registry_should_cover_inventory_targets", () => { + const scenarioIds = new Set(listScenarios().map((scenario) => scenario.id)); + const missingScenarios = migrationInventory.setupScenarios + .map((entry) => entry.newOwner.replace(/^scenario:/, "")) + .filter((owner) => !scenarioIds.has(owner)); + + expect(missingScenarios, `missing scenario owners: ${missingScenarios.join(", ")}`).toEqual([]); + }); + + it("should_fail_when_old_expected_state_missing_new_owner_or_removal_rationale", () => { + const states = loadYaml(EXPECTED_STATES_PATH); + expect(states).toHaveProperty("expected_states"); + const expectedStateIds = keysFrom(states.expected_states); + expect(expectedStateIds.length).toBeGreaterThan(0); + + expectCovered("expectedStates", expectedStateIds); + }); + + it("test_should_fail_when_old_validation_suite_script_missing_new_owner_or_removal_rationale", () => { + const suitesDoc = loadYaml(SUITES_PATH); + expect(suitesDoc).toHaveProperty("suites"); + const suites = suitesDoc.suites as Record }>; + const suiteIds = keysFrom(suites); + expect(suiteIds.length).toBeGreaterThan(0); + const scriptIds = Array.from( + new Set( + Object.values(suites) + .flatMap((suite) => suite.steps ?? []) + .map((step) => step.script) + .filter((script): script is string => Boolean(script)), + ), + ).sort(); + const assertionSuiteIds = new Set(assertionRegistry.groups.map((group) => group.suiteId).filter((suiteId): suiteId is string => Boolean(suiteId))); + const missingAssertionGroups = suiteIds.filter((suiteId) => !assertionSuiteIds.has(suiteId)); + + expectCovered("validationSuites", suiteIds); + expectCovered("validationSuiteScripts", scriptIds); + expect(missingAssertionGroups, `missing assertion groups: ${missingAssertionGroups.join(", ")}`).toEqual([]); + }); + + it("should_keep_migration_inventory_out_of_runtime_entrypoint", () => { + const runSource = fs.readFileSync(path.join(SCENARIO_SUITE_DIR, "scenarios", "run.ts"), "utf8"); + + expect(runSource).not.toContain("migration-inventory"); + }); + + it("should_have_seed_reliability_inventory", () => { + const reliabilityExamples = assertionRegistry.groups.flatMap((group) => group.steps.map((step) => step.reliability).filter(Boolean)); + + expect(reliabilityExamples.some((entry) => entry?.retry && entry.timeoutSeconds)).toBe(true); + }); +}); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts new file mode 100644 index 0000000000..497dac3387 --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import path from "node:path"; + +import { HostCliClient } from "../scenarios/clients/host-cli.ts"; +import { compileRunPlans } from "../scenarios/compiler.ts"; +import { PhaseOrchestrator } from "../scenarios/orchestrators/phase.ts"; +import { ScenarioRunner } from "../scenarios/orchestrators/runner.ts"; +import type { AssertionStep, PhaseName, PhaseResult, RunContext, RunPlanPhase } from "../scenarios/types.ts"; + +function fakeCtx(): RunContext { + return { contextDir: fs.mkdtempSync(path.join(process.cwd(), ".tmp-e2e-phase-")), dryRun: true }; +} + +function fakeStep(id: string, phase: PhaseName, ref = "fake-pass"): AssertionStep { + return { + id, + phase, + implementation: { kind: "probe", ref }, + evidencePath: `.e2e/assertions/${id}.json`, + }; +} + +function fakePhase(step: AssertionStep): RunPlanPhase { + return { + name: step.phase, + actions: [], + assertionGroups: [{ id: `group.${step.id}`, phase: step.phase, migrationStatus: "complete", steps: [step] }], + }; +} + +describe("phase orchestrators", () => { + it("test_should_execute_phase_assertions_from_phase_orchestrators_not_top_level_runner", async () => { + const ctx = fakeCtx(); + try { + const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); + const calls: string[] = []; + const fakeOrchestrator = (phase: PhaseName) => ({ + run: async (_ctx: RunContext, runPhase: RunPlanPhase, _prior?: PhaseResult[]): Promise => { + calls.push(runPhase.name); + return { phase, status: "passed", assertions: [] }; + }, + }); + const runner = new ScenarioRunner({ + environment: fakeOrchestrator("environment"), + onboarding: fakeOrchestrator("onboarding"), + runtime: fakeOrchestrator("runtime"), + }); + + const results = await runner.run(ctx, plan); + + expect(calls).toEqual(["environment", "onboarding", "runtime"]); + expect(results.map((result) => result.phase)).toEqual(["environment", "onboarding", "runtime"]); + } finally { + fs.rmSync(ctx.contextDir, { recursive: true, force: true }); + } + }); + + it("test_should_record_step_status_attempts_duration_classifier_and_evidence", async () => { + const ctx = fakeCtx(); + try { + const step = fakeStep("runtime.retry-pass", "runtime", "fake-retry-once-pass"); + step.reliability = { retry: { attempts: 2, on: ["gateway-transient"] } }; + const orchestrator = new PhaseOrchestrator("runtime"); + + const result = await orchestrator.run(ctx, fakePhase(step)); + + expect(result.status).toBe("passed"); + expect(result.assertions[0]).toEqual( + expect.objectContaining({ + id: "runtime.retry-pass", + status: "passed", + attempts: 2, + classifier: "gateway-transient", + evidence: ".e2e/assertions/runtime.retry-pass.json", + }), + ); + expect(result.assertions[0].durationMs).toBeGreaterThanOrEqual(0); + } finally { + fs.rmSync(ctx.contextDir, { recursive: true, force: true }); + } + }); + + it("test_should_enforce_timeout_and_retry_policy_in_orchestrator", async () => { + const ctx = fakeCtx(); + try { + const step = fakeStep("runtime.retry-fail", "runtime", "fake-always-transient"); + step.reliability = { timeoutSeconds: 1, retry: { attempts: 2, on: ["provider-transient"] } }; + const orchestrator = new PhaseOrchestrator("runtime"); + + const result = await orchestrator.run(ctx, fakePhase(step)); + + expect(result.status).toBe("failed"); + expect(result.assertions[0]).toEqual( + expect.objectContaining({ + id: "runtime.retry-fail", + status: "failed", + attempts: 2, + classifier: "provider-transient", + }), + ); + } finally { + fs.rmSync(ctx.contextDir, { recursive: true, force: true }); + } + }); + + it("test_should_keep_clients_free_of_pass_fail_and_retry_semantics", () => { + const source = fs.readFileSync( + path.join(process.cwd(), "test/e2e-scenario/scenarios/clients/host-cli.ts"), + "utf8", + ); + const observation = new HostCliClient().observeVersion(); + + expect(observation).toEqual(expect.objectContaining({ command: ["nemoclaw", "--version"] })); + expect(source).not.toMatch(/AssertionResult|PhaseResult|retry|timeout|passed|failed/); + }); +}); diff --git a/test/e2e-scenario/framework-tests/e2e-plan-compiler.test.ts b/test/e2e-scenario/framework-tests/e2e-plan-compiler.test.ts new file mode 100644 index 0000000000..86e764fabe --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-plan-compiler.test.ts @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { compileRunPlans } from "../scenarios/compiler.ts"; +import { listScenarios } from "../scenarios/registry.ts"; +import type { ScenarioDefinition } from "../scenarios/types.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const RUN_SCENARIOS = path.join(REPO_ROOT, "test/e2e-scenario/scenarios/run.ts"); +const TSX = path.join(REPO_ROOT, "node_modules/.bin/tsx"); + +function runScenarioCli(args: string[], env: Record = {}) { + return spawnSync(TSX, [RUN_SCENARIOS, ...args], { + cwd: REPO_ROOT, + env: { ...process.env, ...env }, + encoding: "utf8", + timeout: Number(process.env.E2E_SPAWN_TIMEOUT_MS ?? 60_000), + }); +} + +describe("plan compiler", () => { + it("test_should_emit_machine_and_human_plan_artifacts_under_context_dir", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-plan-")); + try { + const result = runScenarioCli(["--scenarios", "ubuntu-repo-cloud-openclaw", "--plan-only"], { + E2E_CONTEXT_DIR: tmp, + }); + + expect(result.status, result.stderr).toBe(0); + const planPath = path.join(tmp, ".e2e", "run-plan.json"); + const summaryPath = path.join(tmp, ".e2e", "plan.txt"); + expect(fs.existsSync(planPath)).toBe(true); + expect(fs.existsSync(summaryPath)).toBe(true); + const plans = JSON.parse(fs.readFileSync(planPath, "utf8")); + expect(plans[0].scenarioId).toBe("ubuntu-repo-cloud-openclaw"); + expect(fs.readFileSync(summaryPath, "utf8")).toContain("Scenario: ubuntu-repo-cloud-openclaw"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("test_should_include_expanded_assertion_steps_by_phase", () => { + const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); + const onboarding = plan.phases.find((phase) => phase.name === "onboarding"); + const runtime = plan.phases.find((phase) => phase.name === "runtime"); + + expect(onboarding?.assertionGroups.map((group) => group.id)).toContain("onboarding.base-installed"); + expect(runtime?.assertionGroups.map((group) => group.id)).toContain("suite.smoke"); + expect(runtime?.assertionGroups.flatMap((group) => group.steps.map((step) => step.id))).toContain( + "runtime.smoke.gateway-health", + ); + }); + + it("test_should_show_timeout_and_retry_policy_in_plan", () => { + const summary = runScenarioCli(["--scenarios", "ubuntu-repo-cloud-openclaw", "--plan-only"]); + + expect(summary.status, summary.stderr).toBe(0); + expect(summary.stdout).toContain("timeout=30s"); + expect(summary.stdout).toContain("retry=2 on gateway-transient"); + }); + + it("test_should_reject_incompatible_manifest_scenario_combination", () => { + const badScenario: ScenarioDefinition = { + id: "bad-platform", + manifestPath: "test/e2e-scenario/manifests/openclaw-nvidia-macos.yaml", + environment: { + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + }, + assertionGroups: [], + expectedStateId: "cloud-openclaw-ready", + suiteIds: [], + onboardingAssertionIds: [], + }; + + expect(() => compileRunPlans([badScenario])).toThrow(/incompatible.*platform|platform.*incompatible/i); + }); + + it("test_should_reject_suite_filter", () => { + const result = runScenarioCli(["--scenarios", "ubuntu-repo-cloud-openclaw", "--plan-only"], { + E2E_SUITE_FILTER: "smoke", + }); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toMatch(/E2E_SUITE_FILTER|scenario builders/i); + }); + + it("plan_only_should_work_for_every_canonical_scenario_id", () => { + const ids = listScenarios().map((scenario) => scenario.id); + const plans = compileRunPlans(ids); + + expect(plans.map((plan) => plan.scenarioId)).toEqual(ids); + }); +}); diff --git a/test/e2e/scenario-framework-tests/e2e-scenario-additional-families.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-additional-families.test.ts similarity index 100% rename from test/e2e/scenario-framework-tests/e2e-scenario-additional-families.test.ts rename to test/e2e-scenario/framework-tests/e2e-scenario-additional-families.test.ts diff --git a/test/e2e/scenario-framework-tests/e2e-scenario-first-migration.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-first-migration.test.ts similarity index 100% rename from test/e2e/scenario-framework-tests/e2e-scenario-first-migration.test.ts rename to test/e2e-scenario/framework-tests/e2e-scenario-first-migration.test.ts diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts new file mode 100644 index 0000000000..f4d9df5f30 --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-scenario-registry.test.ts @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { spawnSync } from "node:child_process"; +import path from "node:path"; + +import { scenario } from "../scenarios/builder.ts"; +import { compileRunPlans } from "../scenarios/compiler.ts"; +import { migrationInventory } from "../scenarios/migration-inventory.ts"; +import { buildScenarioRegistry, listScenarios } from "../scenarios/registry.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const RUN_SCENARIOS = path.join(REPO_ROOT, "test/e2e-scenario/scenarios/run.ts"); +const TSX = path.join(REPO_ROOT, "node_modules/.bin/tsx"); + +function runScenarioCli(args: string[]) { + return spawnSync(TSX, [RUN_SCENARIOS, ...args], { + cwd: REPO_ROOT, + encoding: "utf8", + timeout: Number(process.env.E2E_SPAWN_TIMEOUT_MS ?? 60_000), + }); +} + +function scenarioOwnerIds(): string[] { + return Array.from( + new Set( + [...migrationInventory.setupScenarios, ...migrationInventory.testPlans] + .map((entry) => entry.newOwner) + .filter((owner) => owner.startsWith("scenario:")) + .map((owner) => owner.replace(/^scenario:/, "")), + ), + ).sort(); +} + +describe("deterministic scenario registry", () => { + it("test_should_register_canonical_scenarios_for_all_required_old_coverage", () => { + const registeredIds = new Set(listScenarios().map((entry) => entry.id)); + const missing = scenarioOwnerIds().filter((id) => !registeredIds.has(id)); + + expect(missing, `missing canonical scenario IDs: ${missing.join(", ")}`).toEqual([]); + }); + + it("test_should_reject_duplicate_scenario_ids", () => { + const first = scenario("duplicate-id").manifest("test/e2e-scenario/manifests/openclaw-nvidia.yaml").build(); + const second = scenario("duplicate-id").manifest("test/e2e-scenario/manifests/hermes-nvidia.yaml").build(); + + expect(() => buildScenarioRegistry([first, second])).toThrow(/duplicate-id/); + }); + + it("test_should_return_actionable_unknown_scenario_error", () => { + const result = runScenarioCli(["--scenarios", "does-not-exist", "--plan-only"]); + + expect(result.status).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toMatch(/does-not-exist/); + expect(`${result.stdout}${result.stderr}`).toMatch(/Available scenarios:/); + expect(`${result.stdout}${result.stderr}`).toMatch(/ubuntu-repo-cloud-openclaw/); + }); + + it("test_should_compile_multiple_targeted_scenario_plans", () => { + const plans = compileRunPlans(["ubuntu-repo-cloud-openclaw", "ubuntu-repo-cloud-hermes"]); + + expect(plans.map((plan) => plan.scenarioId)).toEqual([ + "ubuntu-repo-cloud-openclaw", + "ubuntu-repo-cloud-hermes", + ]); + }); + + it("cli_should_emit_two_plan_sections_for_comma_separated_scenarios", () => { + const result = runScenarioCli([ + "--scenarios", + "ubuntu-repo-cloud-openclaw,ubuntu-repo-cloud-hermes", + "--plan-only", + ]); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.match(/^Scenario: /gm)).toHaveLength(2); + expect(result.stdout).toContain("Scenario: ubuntu-repo-cloud-openclaw"); + expect(result.stdout).toContain("Scenario: ubuntu-repo-cloud-hermes"); + }); + + it("baseline_plan_should_match_legacy_resolver_semantics", () => { + const [plan] = compileRunPlans(["ubuntu-repo-cloud-openclaw"]); + + expect(plan.environment).toEqual({ + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + }); + expect(plan.expectedStateId).toBe("cloud-openclaw-ready"); + expect(plan.suiteIds).toEqual(["smoke", "inference", "credentials"]); + expect(plan.onboardingAssertionIds).toEqual(["base-installed", "preflight-passed"]); + }); +}); diff --git a/test/e2e/scenario-framework-tests/e2e-scenario-resolver.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-resolver.test.ts similarity index 100% rename from test/e2e/scenario-framework-tests/e2e-scenario-resolver.test.ts rename to test/e2e-scenario/framework-tests/e2e-scenario-resolver.test.ts diff --git a/test/e2e/scenario-framework-tests/e2e-scenario-schema.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-schema.test.ts similarity index 100% rename from test/e2e/scenario-framework-tests/e2e-scenario-schema.test.ts rename to test/e2e-scenario/framework-tests/e2e-scenario-schema.test.ts diff --git a/test/e2e/scenario-framework-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts similarity index 86% rename from test/e2e/scenario-framework-tests/e2e-scenarios-workflow.test.ts rename to test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts index 11b4655d65..eb1be9ae19 100644 --- a/test/e2e/scenario-framework-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenarios-workflow.test.ts @@ -28,8 +28,8 @@ jobs: run-scenario: runs-on: ubuntu-latest steps: - - name: Run scenario - run: bash test/e2e/runtime/run-scenario.sh --plan-only + - name: Run typed scenarios + run: npx tsx test/e2e-scenario/scenarios/run.ts --scenarios "$SCENARIOS" --plan-only - name: Upload scenario artifacts uses: actions/upload-artifact@v4 with: @@ -48,9 +48,8 @@ jobs: "workflow permissions.contents must be read", "workflow missing resolve-runner job", "run-scenario job must use the resolved runner output", - "Run scenario step must not use retired --plan-only flag", - "run-scenario job missing step: Run scenario in WSL", - "artifact upload name must include the scenario input", + "run-scenario job missing step: Run typed scenarios in WSL", + "artifact upload name must include the scenarios input", "artifact upload must include hidden .e2e files", "artifact upload path must include .e2e/", ]), diff --git a/test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts b/test/e2e-scenario/framework-tests/e2e-suite-runner.test.ts similarity index 99% rename from test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts rename to test/e2e-scenario/framework-tests/e2e-suite-runner.test.ts index a51eeaf947..5a917853f8 100644 --- a/test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-suite-runner.test.ts @@ -7,7 +7,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const RUN_SUITES = path.join(REPO_ROOT, "test/e2e/runtime/run-suites.sh"); +const RUN_SUITES = path.join(REPO_ROOT, "test/e2e-scenario/runtime/run-suites.sh"); function runSuites(args: string[], env: Record = {}): SpawnSyncReturns { return spawnSync("bash", [RUN_SUITES, ...args], { diff --git a/test/e2e-scenario/manifests/hermes-nvidia-discord.yaml b/test/e2e-scenario/manifests/hermes-nvidia-discord.yaml new file mode 100644 index 0000000000..535506ae40 --- /dev/null +++ b/test/e2e-scenario/manifests/hermes-nvidia-discord.yaml @@ -0,0 +1,26 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: hermes-nvidia-discord +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: hermes + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: + - discord + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY + - DISCORD_BOT_TOKEN diff --git a/test/e2e-scenario/manifests/hermes-nvidia-slack.yaml b/test/e2e-scenario/manifests/hermes-nvidia-slack.yaml new file mode 100644 index 0000000000..1d9b72acc8 --- /dev/null +++ b/test/e2e-scenario/manifests/hermes-nvidia-slack.yaml @@ -0,0 +1,26 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: hermes-nvidia-slack +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: hermes + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: + - slack + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY + - SLACK_BOT_TOKEN diff --git a/test/e2e-scenario/manifests/hermes-nvidia.yaml b/test/e2e-scenario/manifests/hermes-nvidia.yaml new file mode 100644 index 0000000000..caee7a3308 --- /dev/null +++ b/test/e2e-scenario/manifests/hermes-nvidia.yaml @@ -0,0 +1,24 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: hermes-nvidia +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: hermes + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-brave.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-brave.yaml new file mode 100644 index 0000000000..f6fb1151a3 --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-brave.yaml @@ -0,0 +1,27 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-brave +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + features: + webSearch: brave + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY + - BRAVE_API_KEY diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-brev-launchable.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-brev-launchable.yaml new file mode 100644 index 0000000000..9f3da8e72f --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-brev-launchable.yaml @@ -0,0 +1,26 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-brev-launchable +spec: + setup: + install: + source: launchable + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: remote + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + gateway: + bindAddress: 0.0.0.0 + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-custom-policies.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-custom-policies.yaml new file mode 100644 index 0000000000..091f76884b --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-custom-policies.yaml @@ -0,0 +1,29 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-custom-policies +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: custom + messaging: [] + features: + model: nvidia/nemotron-3-super-120b-a12b + policyPresets: + - npm + - pypi + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-discord.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-discord.yaml new file mode 100644 index 0000000000..f5ec7d45f2 --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-discord.yaml @@ -0,0 +1,26 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-discord +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: + - discord + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY + - DISCORD_BOT_TOKEN diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-double-provider-switch.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-double-provider-switch.yaml new file mode 100644 index 0000000000..687a2608d8 --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-double-provider-switch.yaml @@ -0,0 +1,25 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-double-provider-switch +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + lifecycle: double-provider-switch + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-double-same-provider.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-double-same-provider.yaml new file mode 100644 index 0000000000..fa951a0d7d --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-double-same-provider.yaml @@ -0,0 +1,25 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-double-same-provider +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + lifecycle: double-same-provider + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-gateway-port-conflict.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-gateway-port-conflict.yaml new file mode 100644 index 0000000000..c86e5c963d --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-gateway-port-conflict.yaml @@ -0,0 +1,27 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-gateway-port-conflict +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + lifecycle: gateway-port-conflict-negative + gateway: + port: 18080 + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-invalid-key.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-invalid-key.yaml new file mode 100644 index 0000000000..7c881c8edf --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-invalid-key.yaml @@ -0,0 +1,25 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-invalid-key +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + lifecycle: invalid-provider-key-negative + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-macos.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-macos.yaml new file mode 100644 index 0000000000..06068fb633 --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-macos.yaml @@ -0,0 +1,24 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-macos +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: optional + platform: + os: macos + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-no-docker-negative.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-no-docker-negative.yaml new file mode 100644 index 0000000000..cc26672a36 --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-no-docker-negative.yaml @@ -0,0 +1,25 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-no-docker-negative +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: missing + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + lifecycle: preflight-negative + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-repair.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-repair.yaml new file mode 100644 index 0000000000..e783edd65a --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-repair.yaml @@ -0,0 +1,25 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-repair +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + lifecycle: repair-existing-config + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-resume.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-resume.yaml new file mode 100644 index 0000000000..3ba269666c --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-resume.yaml @@ -0,0 +1,25 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-resume +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + lifecycle: resume-after-interrupt + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-slack.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-slack.yaml new file mode 100644 index 0000000000..100ea3e337 --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-slack.yaml @@ -0,0 +1,26 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-slack +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: + - slack + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY + - SLACK_BOT_TOKEN diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-telegram.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-telegram.yaml new file mode 100644 index 0000000000..59c5676239 --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-telegram.yaml @@ -0,0 +1,26 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-telegram +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: + - telegram + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY + - TELEGRAM_BOT_TOKEN diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-token-rotation.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-token-rotation.yaml new file mode 100644 index 0000000000..bc9d6d6e40 --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-token-rotation.yaml @@ -0,0 +1,25 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-token-rotation +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + lifecycle: token-rotation + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-wsl.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-wsl.yaml new file mode 100644 index 0000000000..74b7563a80 --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-wsl.yaml @@ -0,0 +1,24 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-wsl +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: wsl + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/manifests/openclaw-nvidia.yaml b/test/e2e-scenario/manifests/openclaw-nvidia.yaml new file mode 100644 index 0000000000..30080e9db3 --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia.yaml @@ -0,0 +1,24 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/manifests/openclaw-ollama-gpu.yaml b/test/e2e-scenario/manifests/openclaw-ollama-gpu.yaml new file mode 100644 index 0000000000..e36e39d4e7 --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-ollama-gpu.yaml @@ -0,0 +1,24 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-ollama-gpu +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + gpuRuntime: cdi + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: ollama + modelRoute: inference-local + policyTier: balanced + messaging: [] + state: + workspaceRef: default + credentialRefs: [] diff --git a/test/e2e-scenario/manifests/openclaw-openai-compatible.yaml b/test/e2e-scenario/manifests/openclaw-openai-compatible.yaml new file mode 100644 index 0000000000..37483022c6 --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-openai-compatible.yaml @@ -0,0 +1,24 @@ +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-openai-compatible +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: openai-compatible + modelRoute: inference-local + policyTier: balanced + messaging: [] + state: + workspaceRef: default + credentialRefs: + - OPENAI_COMPATIBLE_API_KEY diff --git a/test/e2e/nemoclaw_scenarios/expected-states.yaml b/test/e2e-scenario/nemoclaw_scenarios/expected-states.yaml similarity index 100% rename from test/e2e/nemoclaw_scenarios/expected-states.yaml rename to test/e2e-scenario/nemoclaw_scenarios/expected-states.yaml diff --git a/test/e2e/nemoclaw_scenarios/fixtures/_fake-http-stub.sh b/test/e2e-scenario/nemoclaw_scenarios/fixtures/_fake-http-stub.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/fixtures/_fake-http-stub.sh rename to test/e2e-scenario/nemoclaw_scenarios/fixtures/_fake-http-stub.sh diff --git a/test/e2e/nemoclaw_scenarios/fixtures/fake-discord.sh b/test/e2e-scenario/nemoclaw_scenarios/fixtures/fake-discord.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/fixtures/fake-discord.sh rename to test/e2e-scenario/nemoclaw_scenarios/fixtures/fake-discord.sh diff --git a/test/e2e/nemoclaw_scenarios/fixtures/fake-openai.sh b/test/e2e-scenario/nemoclaw_scenarios/fixtures/fake-openai.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/fixtures/fake-openai.sh rename to test/e2e-scenario/nemoclaw_scenarios/fixtures/fake-openai.sh diff --git a/test/e2e/nemoclaw_scenarios/fixtures/fake-slack.sh b/test/e2e-scenario/nemoclaw_scenarios/fixtures/fake-slack.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/fixtures/fake-slack.sh rename to test/e2e-scenario/nemoclaw_scenarios/fixtures/fake-slack.sh diff --git a/test/e2e/nemoclaw_scenarios/fixtures/fake-telegram.sh b/test/e2e-scenario/nemoclaw_scenarios/fixtures/fake-telegram.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/fixtures/fake-telegram.sh rename to test/e2e-scenario/nemoclaw_scenarios/fixtures/fake-telegram.sh diff --git a/test/e2e/nemoclaw_scenarios/fixtures/older-base-image.sh b/test/e2e-scenario/nemoclaw_scenarios/fixtures/older-base-image.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/fixtures/older-base-image.sh rename to test/e2e-scenario/nemoclaw_scenarios/fixtures/older-base-image.sh diff --git a/test/e2e/nemoclaw_scenarios/helpers/emit-context-from-plan.sh b/test/e2e-scenario/nemoclaw_scenarios/helpers/emit-context-from-plan.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/helpers/emit-context-from-plan.sh rename to test/e2e-scenario/nemoclaw_scenarios/helpers/emit-context-from-plan.sh diff --git a/test/e2e/nemoclaw_scenarios/install/dispatch.sh b/test/e2e-scenario/nemoclaw_scenarios/install/dispatch.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/install/dispatch.sh rename to test/e2e-scenario/nemoclaw_scenarios/install/dispatch.sh diff --git a/test/e2e/nemoclaw_scenarios/install/helpers/install-path-refresh.sh b/test/e2e-scenario/nemoclaw_scenarios/install/helpers/install-path-refresh.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/install/helpers/install-path-refresh.sh rename to test/e2e-scenario/nemoclaw_scenarios/install/helpers/install-path-refresh.sh diff --git a/test/e2e/nemoclaw_scenarios/install/launchable.sh b/test/e2e-scenario/nemoclaw_scenarios/install/launchable.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/install/launchable.sh rename to test/e2e-scenario/nemoclaw_scenarios/install/launchable.sh diff --git a/test/e2e/nemoclaw_scenarios/install/ollama.sh b/test/e2e-scenario/nemoclaw_scenarios/install/ollama.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/install/ollama.sh rename to test/e2e-scenario/nemoclaw_scenarios/install/ollama.sh diff --git a/test/e2e/nemoclaw_scenarios/install/public-curl.sh b/test/e2e-scenario/nemoclaw_scenarios/install/public-curl.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/install/public-curl.sh rename to test/e2e-scenario/nemoclaw_scenarios/install/public-curl.sh diff --git a/test/e2e/nemoclaw_scenarios/install/repo-current.sh b/test/e2e-scenario/nemoclaw_scenarios/install/repo-current.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/install/repo-current.sh rename to test/e2e-scenario/nemoclaw_scenarios/install/repo-current.sh diff --git a/test/e2e/nemoclaw_scenarios/onboard/cloud-hermes.sh b/test/e2e-scenario/nemoclaw_scenarios/onboard/cloud-hermes.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/onboard/cloud-hermes.sh rename to test/e2e-scenario/nemoclaw_scenarios/onboard/cloud-hermes.sh diff --git a/test/e2e/nemoclaw_scenarios/onboard/cloud-openclaw.sh b/test/e2e-scenario/nemoclaw_scenarios/onboard/cloud-openclaw.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/onboard/cloud-openclaw.sh rename to test/e2e-scenario/nemoclaw_scenarios/onboard/cloud-openclaw.sh diff --git a/test/e2e/nemoclaw_scenarios/onboard/dispatch.sh b/test/e2e-scenario/nemoclaw_scenarios/onboard/dispatch.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/onboard/dispatch.sh rename to test/e2e-scenario/nemoclaw_scenarios/onboard/dispatch.sh diff --git a/test/e2e/nemoclaw_scenarios/onboard/local-ollama-openclaw.sh b/test/e2e-scenario/nemoclaw_scenarios/onboard/local-ollama-openclaw.sh similarity index 100% rename from test/e2e/nemoclaw_scenarios/onboard/local-ollama-openclaw.sh rename to test/e2e-scenario/nemoclaw_scenarios/onboard/local-ollama-openclaw.sh diff --git a/test/e2e/nemoclaw_scenarios/scenarios.yaml b/test/e2e-scenario/nemoclaw_scenarios/scenarios.yaml similarity index 100% rename from test/e2e/nemoclaw_scenarios/scenarios.yaml rename to test/e2e-scenario/nemoclaw_scenarios/scenarios.yaml diff --git a/test/e2e/onboarding_assertions/base/00-cli-installed.sh b/test/e2e-scenario/onboarding_assertions/base/00-cli-installed.sh similarity index 100% rename from test/e2e/onboarding_assertions/base/00-cli-installed.sh rename to test/e2e-scenario/onboarding_assertions/base/00-cli-installed.sh diff --git a/test/e2e/onboarding_assertions/preflight/00-preflight-expected-failed.sh b/test/e2e-scenario/onboarding_assertions/preflight/00-preflight-expected-failed.sh similarity index 100% rename from test/e2e/onboarding_assertions/preflight/00-preflight-expected-failed.sh rename to test/e2e-scenario/onboarding_assertions/preflight/00-preflight-expected-failed.sh diff --git a/test/e2e/onboarding_assertions/preflight/00-preflight-passed.sh b/test/e2e-scenario/onboarding_assertions/preflight/00-preflight-passed.sh similarity index 100% rename from test/e2e/onboarding_assertions/preflight/00-preflight-passed.sh rename to test/e2e-scenario/onboarding_assertions/preflight/00-preflight-passed.sh diff --git a/test/e2e/runtime/coverage-report.sh b/test/e2e-scenario/runtime/coverage-report.sh similarity index 93% rename from test/e2e/runtime/coverage-report.sh rename to test/e2e-scenario/runtime/coverage-report.sh index 9fea9cf9af..8426d0ba30 100755 --- a/test/e2e/runtime/coverage-report.sh +++ b/test/e2e-scenario/runtime/coverage-report.sh @@ -5,7 +5,7 @@ # Render the E2E scenario coverage report as Markdown to stdout. # # Usage: -# bash test/e2e/runtime/coverage-report.sh > coverage.md +# bash test/e2e-scenario/runtime/coverage-report.sh > coverage.md set -euo pipefail diff --git a/test/e2e/runtime/lib/artifacts.sh b/test/e2e-scenario/runtime/lib/artifacts.sh similarity index 100% rename from test/e2e/runtime/lib/artifacts.sh rename to test/e2e-scenario/runtime/lib/artifacts.sh diff --git a/test/e2e/runtime/lib/cleanup.sh b/test/e2e-scenario/runtime/lib/cleanup.sh similarity index 100% rename from test/e2e/runtime/lib/cleanup.sh rename to test/e2e-scenario/runtime/lib/cleanup.sh diff --git a/test/e2e/runtime/lib/context.sh b/test/e2e-scenario/runtime/lib/context.sh similarity index 100% rename from test/e2e/runtime/lib/context.sh rename to test/e2e-scenario/runtime/lib/context.sh diff --git a/test/e2e/runtime/lib/env.sh b/test/e2e-scenario/runtime/lib/env.sh similarity index 100% rename from test/e2e/runtime/lib/env.sh rename to test/e2e-scenario/runtime/lib/env.sh diff --git a/test/e2e/runtime/lib/logging.sh b/test/e2e-scenario/runtime/lib/logging.sh similarity index 100% rename from test/e2e/runtime/lib/logging.sh rename to test/e2e-scenario/runtime/lib/logging.sh diff --git a/test/e2e/runtime/lib/negative.sh b/test/e2e-scenario/runtime/lib/negative.sh similarity index 100% rename from test/e2e/runtime/lib/negative.sh rename to test/e2e-scenario/runtime/lib/negative.sh diff --git a/test/e2e/runtime/lib/onboard-state.sh b/test/e2e-scenario/runtime/lib/onboard-state.sh similarity index 100% rename from test/e2e/runtime/lib/onboard-state.sh rename to test/e2e-scenario/runtime/lib/onboard-state.sh diff --git a/test/e2e/runtime/lib/port-holder.sh b/test/e2e-scenario/runtime/lib/port-holder.sh similarity index 100% rename from test/e2e/runtime/lib/port-holder.sh rename to test/e2e-scenario/runtime/lib/port-holder.sh diff --git a/test/e2e/runtime/lib/sandbox-teardown.sh b/test/e2e-scenario/runtime/lib/sandbox-teardown.sh similarity index 100% rename from test/e2e/runtime/lib/sandbox-teardown.sh rename to test/e2e-scenario/runtime/lib/sandbox-teardown.sh diff --git a/test/e2e/runtime/reports/render-gap-report.ts b/test/e2e-scenario/runtime/reports/render-gap-report.ts similarity index 100% rename from test/e2e/runtime/reports/render-gap-report.ts rename to test/e2e-scenario/runtime/reports/render-gap-report.ts diff --git a/test/e2e/runtime/resolver/coverage.ts b/test/e2e-scenario/runtime/resolver/coverage.ts similarity index 100% rename from test/e2e/runtime/resolver/coverage.ts rename to test/e2e-scenario/runtime/resolver/coverage.ts diff --git a/test/e2e/runtime/resolver/expected-failure.ts b/test/e2e-scenario/runtime/resolver/expected-failure.ts similarity index 100% rename from test/e2e/runtime/resolver/expected-failure.ts rename to test/e2e-scenario/runtime/resolver/expected-failure.ts diff --git a/test/e2e/runtime/resolver/index.ts b/test/e2e-scenario/runtime/resolver/index.ts similarity index 97% rename from test/e2e/runtime/resolver/index.ts rename to test/e2e-scenario/runtime/resolver/index.ts index d9e9163d36..972fd073db 100644 --- a/test/e2e/runtime/resolver/index.ts +++ b/test/e2e-scenario/runtime/resolver/index.ts @@ -5,9 +5,9 @@ * CLI entrypoint for the E2E scenario resolver. * * Usage: - * tsx test/e2e/runtime/resolver/index.ts plan [--context-dir ] - * tsx test/e2e/runtime/resolver/index.ts validate-state [--probes-from-state] - * tsx test/e2e/runtime/resolver/index.ts match-failure \ + * tsx test/e2e-scenario/runtime/resolver/index.ts plan [--context-dir ] + * tsx test/e2e-scenario/runtime/resolver/index.ts validate-state [--probes-from-state] + * tsx test/e2e-scenario/runtime/resolver/index.ts match-failure \ * --log --observed-phase \ * [--observed-error-class ] [--observed-side-effects ] * @@ -65,7 +65,7 @@ function parseArgs(argv: string[]): { let observedErrorClass: string | undefined; let observedSideEffects: string | undefined; const scriptDir = path.dirname(fileURLToPath(import.meta.url)); - // resolver/ lives under test/e2e/runtime/, so the E2E metadata root + // resolver/ lives under test/e2e-scenario/runtime/, so the E2E metadata root // (which loadMetadataFromDir resolves further into nemoclaw_scenarios/ // and validation_suites/) is two levels up. let metadataDir = path.resolve(scriptDir, "..", ".."); diff --git a/test/e2e/runtime/resolver/js-yaml.d.ts b/test/e2e-scenario/runtime/resolver/js-yaml.d.ts similarity index 100% rename from test/e2e/runtime/resolver/js-yaml.d.ts rename to test/e2e-scenario/runtime/resolver/js-yaml.d.ts diff --git a/test/e2e/runtime/resolver/load.ts b/test/e2e-scenario/runtime/resolver/load.ts similarity index 100% rename from test/e2e/runtime/resolver/load.ts rename to test/e2e-scenario/runtime/resolver/load.ts diff --git a/test/e2e/runtime/resolver/plan.ts b/test/e2e-scenario/runtime/resolver/plan.ts similarity index 100% rename from test/e2e/runtime/resolver/plan.ts rename to test/e2e-scenario/runtime/resolver/plan.ts diff --git a/test/e2e/runtime/resolver/schema.ts b/test/e2e-scenario/runtime/resolver/schema.ts similarity index 100% rename from test/e2e/runtime/resolver/schema.ts rename to test/e2e-scenario/runtime/resolver/schema.ts diff --git a/test/e2e/runtime/resolver/validator.ts b/test/e2e-scenario/runtime/resolver/validator.ts similarity index 100% rename from test/e2e/runtime/resolver/validator.ts rename to test/e2e-scenario/runtime/resolver/validator.ts diff --git a/test/e2e/runtime/run-scenario.sh b/test/e2e-scenario/runtime/run-scenario.sh similarity index 98% rename from test/e2e/runtime/run-scenario.sh rename to test/e2e-scenario/runtime/run-scenario.sh index 99f917b8c8..58042c8523 100755 --- a/test/e2e/runtime/run-scenario.sh +++ b/test/e2e-scenario/runtime/run-scenario.sh @@ -5,7 +5,7 @@ # E2E scenario runner entrypoint. # # Usage: -# bash test/e2e/runtime/run-scenario.sh [--plan-only|--validate-only|--dry-run] +# bash test/e2e-scenario/runtime/run-scenario.sh [--plan-only|--validate-only|--dry-run] # # Flags: # --plan-only Resolve metadata and print the plan only. Writes @@ -37,7 +37,7 @@ DRY_RUN=0 usage() { cat >&2 <<'USAGE' -Usage: bash test/e2e/runtime/run-scenario.sh [--plan-only|--validate-only|--dry-run] +Usage: bash test/e2e-scenario/runtime/run-scenario.sh [--plan-only|--validate-only|--dry-run] USAGE } diff --git a/test/e2e/runtime/run-suites.sh b/test/e2e-scenario/runtime/run-suites.sh similarity index 92% rename from test/e2e/runtime/run-suites.sh rename to test/e2e-scenario/runtime/run-suites.sh index f7b5fe7390..e99c069408 100755 --- a/test/e2e/runtime/run-suites.sh +++ b/test/e2e-scenario/runtime/run-suites.sh @@ -5,9 +5,9 @@ # Run one or more functional suites against a completed E2E environment. # # Usage: -# bash test/e2e/runtime/run-suites.sh [ ...] +# bash test/e2e-scenario/runtime/run-suites.sh [ ...] # -# Reads suite metadata from test/e2e/validation_suites/suites.yaml +# Reads suite metadata from test/e2e-scenario/validation_suites/suites.yaml # (or $E2E_SUITES_FILE). Each suite script receives .e2e/context.env # via E2E_CONTEXT_DIR and is expected to source runtime/lib/context.sh if # it needs specific keys. @@ -16,7 +16,7 @@ # E2E_CONTEXT_DIR Directory containing context.env (default: /.e2e) # E2E_SUITES_FILE Override suites metadata file (for tests) # E2E_SUITES_DIR Override the directory that suite scripts are resolved -# against (default: test/e2e/validation_suites/) +# against (default: test/e2e-scenario/validation_suites/) # E2E_DRY_RUN When 1, suite scripts run in dry-run mode themselves. # # Exit code: 0 if all steps pass; non-zero at the first failing step. @@ -30,7 +30,7 @@ VALIDATION_SUITES_DIR="${E2E_ROOT}/validation_suites" if (($# == 0)); then echo "run-suites: at least one suite id required" >&2 - echo "Usage: bash test/e2e/runtime/run-suites.sh [ ...]" >&2 + echo "Usage: bash test/e2e-scenario/runtime/run-suites.sh [ ...]" >&2 exit 2 fi diff --git a/test/e2e-scenario/scenarios/assertions/diagnostics.ts b/test/e2e-scenario/scenarios/assertions/diagnostics.ts new file mode 100644 index 0000000000..c8336c8709 --- /dev/null +++ b/test/e2e-scenario/scenarios/assertions/diagnostics.ts @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { validationSuiteGroups } from "./registry.ts"; diff --git a/test/e2e-scenario/scenarios/assertions/environment.ts b/test/e2e-scenario/scenarios/assertions/environment.ts new file mode 100644 index 0000000000..be7a62e6fb --- /dev/null +++ b/test/e2e-scenario/scenarios/assertions/environment.ts @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AssertionGroup } from "../types.ts"; + +export function environmentBaseline(): AssertionGroup { + return { + id: "environment.baseline", + phase: "environment", + description: "Skeleton environment baseline assertion group.", + migrationStatus: "complete", + steps: [ + { + id: "environment.plan.skeleton", + phase: "environment", + description: "Placeholder step until live environment orchestration is migrated.", + implementation: { kind: "pending", ref: "phase-1-skeleton" }, + evidencePath: ".e2e/environment.result.json", + }, + ], + }; +} diff --git a/test/e2e-scenario/scenarios/assertions/hermes.ts b/test/e2e-scenario/scenarios/assertions/hermes.ts new file mode 100644 index 0000000000..c8336c8709 --- /dev/null +++ b/test/e2e-scenario/scenarios/assertions/hermes.ts @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { validationSuiteGroups } from "./registry.ts"; diff --git a/test/e2e-scenario/scenarios/assertions/inference.ts b/test/e2e-scenario/scenarios/assertions/inference.ts new file mode 100644 index 0000000000..c8336c8709 --- /dev/null +++ b/test/e2e-scenario/scenarios/assertions/inference.ts @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { validationSuiteGroups } from "./registry.ts"; diff --git a/test/e2e-scenario/scenarios/assertions/lifecycle.ts b/test/e2e-scenario/scenarios/assertions/lifecycle.ts new file mode 100644 index 0000000000..c8336c8709 --- /dev/null +++ b/test/e2e-scenario/scenarios/assertions/lifecycle.ts @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { validationSuiteGroups } from "./registry.ts"; diff --git a/test/e2e-scenario/scenarios/assertions/messaging.ts b/test/e2e-scenario/scenarios/assertions/messaging.ts new file mode 100644 index 0000000000..c8336c8709 --- /dev/null +++ b/test/e2e-scenario/scenarios/assertions/messaging.ts @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { validationSuiteGroups } from "./registry.ts"; diff --git a/test/e2e-scenario/scenarios/assertions/negative.ts b/test/e2e-scenario/scenarios/assertions/negative.ts new file mode 100644 index 0000000000..f1dac271d2 --- /dev/null +++ b/test/e2e-scenario/scenarios/assertions/negative.ts @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { onboardingAssertionGroups } from "./registry.ts"; diff --git a/test/e2e-scenario/scenarios/assertions/onboarding.ts b/test/e2e-scenario/scenarios/assertions/onboarding.ts new file mode 100644 index 0000000000..9886a701fb --- /dev/null +++ b/test/e2e-scenario/scenarios/assertions/onboarding.ts @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AssertionGroup } from "../types.ts"; + +export function onboardingBaseline(): AssertionGroup { + return { + id: "onboarding.baseline", + phase: "onboarding", + description: "Skeleton onboarding assertion group.", + steps: [ + { + id: "onboarding.plan.skeleton", + phase: "onboarding", + description: "Placeholder step until onboarding assertions are migrated.", + implementation: { kind: "pending", ref: "phase-1-skeleton" }, + evidencePath: ".e2e/onboarding.result.json", + }, + ], + }; +} diff --git a/test/e2e-scenario/scenarios/assertions/platform.ts b/test/e2e-scenario/scenarios/assertions/platform.ts new file mode 100644 index 0000000000..c8336c8709 --- /dev/null +++ b/test/e2e-scenario/scenarios/assertions/platform.ts @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { validationSuiteGroups } from "./registry.ts"; diff --git a/test/e2e-scenario/scenarios/assertions/registry.ts b/test/e2e-scenario/scenarios/assertions/registry.ts new file mode 100644 index 0000000000..3c300d5957 --- /dev/null +++ b/test/e2e-scenario/scenarios/assertions/registry.ts @@ -0,0 +1,373 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import { environmentBaseline } from "./environment.ts"; +import type { AssertionGroup, AssertionStep, PhaseName, ScenarioDefinition } from "../types.ts"; + +type Reliability = AssertionStep["reliability"]; + +interface ShellStepInput { + id: string; + phase: PhaseName; + ref: string; + reliability?: Reliability; +} + +function shellStep(input: ShellStepInput): AssertionStep { + return { + id: input.id, + phase: input.phase, + implementation: { kind: "shell", ref: input.ref }, + evidencePath: `.e2e/assertions/${input.id}.log`, + reliability: input.reliability, + }; +} + +function probeStep(id: string, phase: PhaseName, ref: string, reliability?: Reliability): AssertionStep { + return { + id, + phase, + implementation: { kind: "probe", ref }, + evidencePath: `.e2e/assertions/${id}.json`, + reliability, + }; +} + +function pendingStep(id: string, phase: PhaseName, ref: string): AssertionStep { + return { + id, + phase, + implementation: { kind: "pending", ref }, + evidencePath: `.e2e/assertions/${id}.json`, + }; +} + +function group(input: { + id: string; + phase: PhaseName; + steps: AssertionStep[]; + suiteId?: string; + onboardingAssertionId?: string; + description?: string; +}): AssertionGroup { + return { ...input, migrationStatus: "complete" }; +} + +function suiteGroup(suiteId: string, steps: AssertionStep[], phase: PhaseName = "runtime"): AssertionGroup { + return group({ id: `suite.${suiteId}`, suiteId, phase, steps, description: `Converted suite ${suiteId}.` }); +} + +export const onboardingAssertionGroups: AssertionGroup[] = [ + group({ + id: "onboarding.base-installed", + onboardingAssertionId: "base-installed", + phase: "onboarding", + steps: [ + shellStep({ + id: "onboarding.base.cli-installed", + phase: "onboarding", + ref: "test/e2e-scenario/onboarding_assertions/base/00-cli-installed.sh", + }), + ], + }), + group({ + id: "onboarding.preflight-passed", + onboardingAssertionId: "preflight-passed", + phase: "onboarding", + steps: [ + shellStep({ + id: "onboarding.preflight.passed", + phase: "onboarding", + ref: "test/e2e-scenario/onboarding_assertions/preflight/00-preflight-passed.sh", + reliability: { timeoutSeconds: 60 }, + }), + ], + }), + group({ + id: "onboarding.preflight-expected-failed", + onboardingAssertionId: "preflight-expected-failed", + phase: "onboarding", + steps: [ + shellStep({ + id: "onboarding.preflight.expected-failed", + phase: "onboarding", + ref: "test/e2e-scenario/onboarding_assertions/preflight/00-preflight-expected-failed.sh", + }), + ], + }), +]; + +const smokeSteps = [ + shellStep({ id: "runtime.smoke.cli-available", phase: "runtime", ref: "test/e2e-scenario/validation_suites/smoke/00-cli-available.sh" }), + shellStep({ + id: "runtime.smoke.gateway-health", + phase: "runtime", + ref: "test/e2e-scenario/validation_suites/smoke/01-gateway-health.sh", + reliability: { timeoutSeconds: 30, retry: { attempts: 2, on: ["gateway-transient"] } }, + }), + shellStep({ id: "runtime.smoke.sandbox-listed", phase: "runtime", ref: "test/e2e-scenario/validation_suites/smoke/02-sandbox-listed.sh" }), + shellStep({ id: "runtime.smoke.sandbox-shell", phase: "runtime", ref: "test/e2e-scenario/validation_suites/smoke/03-sandbox-shell.sh", reliability: { timeoutSeconds: 30 } }), +]; + +const cloudInferenceSteps = [ + shellStep({ + id: "runtime.inference.models-health", + phase: "runtime", + ref: "test/e2e-scenario/validation_suites/inference/cloud/00-models-health.sh", + reliability: { timeoutSeconds: 30, retry: { attempts: 2, on: ["provider-transient"] } }, + }), + shellStep({ + id: "runtime.inference.chat-completion", + phase: "runtime", + ref: "test/e2e-scenario/validation_suites/inference/cloud/01-chat-completion.sh", + reliability: { timeoutSeconds: 60, retry: { attempts: 2, on: ["provider-transient", "model-toolcall-transient"] } }, + }), + shellStep({ + id: "runtime.inference.sandbox-local", + phase: "runtime", + ref: "test/e2e-scenario/validation_suites/inference/cloud/02-inference-local-from-sandbox.sh", + reliability: { timeoutSeconds: 45, retry: { attempts: 2, on: ["gateway-transient"] } }, + }), +]; + +const credentialsSteps = [ + shellStep({ + id: "security.credentials.present", + phase: "runtime", + ref: "test/e2e-scenario/validation_suites/security/credentials/00-credentials-present.sh", + }), + shellStep({ + id: "security.credentials.no-plaintext-host-store", + phase: "runtime", + ref: "test/e2e-scenario/validation_suites/security/credentials/01-no-plaintext-host-store.sh", + }), +]; + +const baselineOnboardingSteps = [ + shellStep({ id: "baseline.cli-and-openshell", phase: "runtime", ref: "test/e2e-scenario/validation_suites/baseline-onboarding/00-cli-and-openshell.sh" }), + shellStep({ id: "baseline.sandbox-state", phase: "runtime", ref: "test/e2e-scenario/validation_suites/baseline-onboarding/01-sandbox-state.sh" }), + shellStep({ id: "baseline.route-and-smoke", phase: "runtime", ref: "test/e2e-scenario/validation_suites/baseline-onboarding/02-route-and-smoke.sh" }), +]; + +const onboardingStateSteps = [ + shellStep({ id: "onboarding.state.registry", phase: "runtime", ref: "test/e2e-scenario/validation_suites/onboarding/state/00-registry-provider-model-policies.sh" }), + shellStep({ id: "onboarding.state.session", phase: "runtime", ref: "test/e2e-scenario/validation_suites/onboarding/state/01-session-provider-model-policies.sh" }), +]; + +const ollamaSteps = [ + shellStep({ + id: "runtime.ollama.models-health", + phase: "runtime", + ref: "test/e2e-scenario/validation_suites/inference/ollama-gpu/00-ollama-models-health.sh", + reliability: { timeoutSeconds: 45, retry: { attempts: 2, on: ["provider-transient"] } }, + }), + shellStep({ + id: "runtime.ollama.chat-completion", + phase: "runtime", + ref: "test/e2e-scenario/validation_suites/inference/ollama-gpu/01-ollama-chat-completion.sh", + reliability: { timeoutSeconds: 60, retry: { attempts: 2, on: ["provider-transient"] } }, + }), +]; + +const ollamaProxySteps = [ + shellStep({ + id: "runtime.ollama-auth-proxy.reachable", + phase: "runtime", + ref: "test/e2e-scenario/validation_suites/inference/ollama-auth-proxy/00-proxy-reachable.sh", + reliability: { timeoutSeconds: 30, retry: { attempts: 2, on: ["gateway-transient"] } }, + }), +]; + +export const runtimeControlGroups: AssertionGroup[] = [ + { + id: "runtime.expected-failure.no-side-effects", + phase: "runtime", + description: "Negative scenario runtime check ensuring forbidden side effects did not occur.", + migrationStatus: "complete", + steps: [pendingStep("runtime.expected-failure.no-side-effects", "runtime", "expectedFailureNoSideEffectsProbe")], + }, +]; + +export const validationSuiteGroups: AssertionGroup[] = [ + suiteGroup("smoke", smokeSteps), + suiteGroup("gateway-health", [smokeSteps[1]]), + suiteGroup("sandbox-shell", [smokeSteps[3]]), + suiteGroup("platform-macos", [shellStep({ id: "platform.macos.smoke", phase: "runtime", ref: "test/e2e-scenario/validation_suites/platform/macos/00-macos-smoke.sh" })]), + suiteGroup("platform-wsl", [shellStep({ id: "platform.wsl.smoke", phase: "runtime", ref: "test/e2e-scenario/validation_suites/platform/wsl/00-wsl-smoke.sh" })]), + suiteGroup("inference", cloudInferenceSteps), + suiteGroup("cloud-inference", cloudInferenceSteps), + suiteGroup("local-ollama-inference", ollamaSteps), + suiteGroup("ollama-proxy", ollamaProxySteps), + suiteGroup("ollama-auth-proxy", [ + ...ollamaProxySteps, + shellStep({ id: "runtime.ollama-auth-proxy.auth-enforcement", phase: "runtime", ref: "test/e2e-scenario/validation_suites/inference/ollama-auth-proxy/01-auth-enforcement.sh" }), + ]), + suiteGroup("baseline-onboarding", baselineOnboardingSteps), + suiteGroup("onboarding-state", onboardingStateSteps), + suiteGroup("model-router", [ + shellStep({ id: "runtime.model-router.healthy-endpoint", phase: "runtime", ref: "test/e2e-scenario/validation_suites/inference/model-router/00-healthy-endpoint.sh" }), + shellStep({ id: "runtime.model-router.provider-routed-completion", phase: "runtime", ref: "test/e2e-scenario/validation_suites/inference/model-router/01-provider-routed-completion.sh" }), + ]), + suiteGroup("openai-compatible-inference", cloudInferenceSteps), + suiteGroup("inference-routing", cloudInferenceSteps), + suiteGroup("inference-switch", cloudInferenceSteps), + suiteGroup("kimi-compatibility", [ + shellStep({ id: "runtime.kimi.plugin-wiring", phase: "runtime", ref: "test/e2e-scenario/validation_suites/inference/kimi-compatibility/00-plugin-wiring.sh", reliability: { timeoutSeconds: 30, retry: { attempts: 2, on: ["model-toolcall-transient"] } } }), + shellStep({ id: "runtime.kimi.compatible-models-route", phase: "runtime", ref: "test/e2e-scenario/validation_suites/inference/kimi-compatibility/01-kimi-compatible-models-route.sh", reliability: { timeoutSeconds: 30, retry: { attempts: 2, on: ["model-toolcall-transient"] } } }), + ]), + suiteGroup("credentials", credentialsSteps), + suiteGroup("security-credentials", credentialsSteps), + suiteGroup("security-shields", [probeStep("security.shields.config", "runtime", "shieldsConfigProbe")]), + suiteGroup("security-policy", [probeStep("security.policy.enforced", "runtime", "networkPolicyProbe")]), + suiteGroup("security-injection", [probeStep("security.injection.blocked", "runtime", "injectionBlockedProbe")]), + suiteGroup("messaging-telegram", [ + shellStep({ id: "messaging.telegram.injection-safety", phase: "runtime", ref: "test/e2e-scenario/validation_suites/messaging/telegram/00-telegram-injection-safety.sh", reliability: { timeoutSeconds: 30, retry: { attempts: 2, on: ["external-tunnel"] } } }), + shellStep({ id: "messaging.telegram.injection-payload-classes", phase: "runtime", ref: "test/e2e-scenario/validation_suites/messaging/telegram/01-telegram-injection-payload-classes.sh", reliability: { timeoutSeconds: 30, retry: { attempts: 2, on: ["external-tunnel"] } } }), + ]), + suiteGroup("messaging-discord", [shellStep({ id: "messaging.discord.gateway-path", phase: "runtime", ref: "test/e2e-scenario/validation_suites/messaging/discord/00-discord-gateway-path.sh", reliability: { timeoutSeconds: 30, retry: { attempts: 2, on: ["external-tunnel"] } } })]), + suiteGroup("messaging-slack", [shellStep({ id: "messaging.slack.provider-state", phase: "runtime", ref: "test/e2e-scenario/validation_suites/messaging/slack/00-slack-provider-state.sh", reliability: { timeoutSeconds: 30, retry: { attempts: 2, on: ["external-tunnel"] } } })]), + suiteGroup("messaging-token-rotation", [shellStep({ id: "messaging.token-rotation", phase: "runtime", ref: "test/e2e-scenario/validation_suites/messaging/token-rotation/00-provider-rotation-isolated.sh" })]), + suiteGroup("sandbox-lifecycle", [ + shellStep({ id: "lifecycle.sandbox.gateway-health", phase: "runtime", ref: "test/e2e-scenario/validation_suites/sandbox/lifecycle/00-gateway-health.sh" }), + shellStep({ id: "lifecycle.sandbox.gateway-recovery", phase: "runtime", ref: "test/e2e-scenario/validation_suites/sandbox/lifecycle/01-gateway-recovery.sh" }), + ]), + suiteGroup("sandbox-operations", [ + shellStep({ id: "lifecycle.sandbox.list-and-status", phase: "runtime", ref: "test/e2e-scenario/validation_suites/sandbox/operations/00-list-and-status.sh" }), + shellStep({ id: "lifecycle.sandbox.logs-and-exec", phase: "runtime", ref: "test/e2e-scenario/validation_suites/sandbox/operations/01-logs-and-exec.sh" }), + ]), + suiteGroup("snapshot", [shellStep({ id: "lifecycle.snapshot.create-list-restore", phase: "runtime", ref: "test/e2e-scenario/validation_suites/sandbox/snapshot/00-create-list-restore.sh" })]), + suiteGroup("snapshot-lifecycle", [shellStep({ id: "lifecycle.snapshot.create-list-restore", phase: "runtime", ref: "test/e2e-scenario/validation_suites/sandbox/snapshot/00-create-list-restore.sh" })]), + suiteGroup("rebuild", [ + shellStep({ id: "lifecycle.rebuild.state-preserved", phase: "runtime", ref: "test/e2e-scenario/validation_suites/rebuild_upgrade/00-state-preserved.sh", reliability: { timeoutSeconds: 120, retry: { attempts: 2, on: ["runner-infra"] } } }), + shellStep({ id: "lifecycle.rebuild.agent-version-upgraded", phase: "runtime", ref: "test/e2e-scenario/validation_suites/rebuild_upgrade/01-agent-version-upgraded.sh", reliability: { timeoutSeconds: 120, retry: { attempts: 2, on: ["runner-infra"] } } }), + shellStep({ id: "lifecycle.rebuild.post-rebuild-inference", phase: "runtime", ref: "test/e2e-scenario/validation_suites/rebuild_upgrade/02-post-rebuild-inference.sh", reliability: { timeoutSeconds: 120, retry: { attempts: 2, on: ["runner-infra"] } } }), + ]), + suiteGroup("upgrade", [ + shellStep({ id: "lifecycle.upgrade.policy-config-preserved", phase: "runtime", ref: "test/e2e-scenario/validation_suites/rebuild_upgrade/03-policy-config-preserved.sh", reliability: { timeoutSeconds: 120, retry: { attempts: 2, on: ["wrong-installed-ref"] } } }), + shellStep({ id: "lifecycle.upgrade.survivor-reachable", phase: "runtime", ref: "test/e2e-scenario/validation_suites/rebuild_upgrade/04-upgrade-survivor-reachable.sh", reliability: { timeoutSeconds: 120, retry: { attempts: 2, on: ["wrong-installed-ref"] } } }), + ]), + suiteGroup("diagnostics", [probeStep("diagnostics.bundle", "runtime", "diagnosticsProbe")]), + suiteGroup("docs-validation", [probeStep("docs.validation", "runtime", "docsValidationProbe")]), + suiteGroup("hermes-specific", [shellStep({ id: "runtime.hermes.health", phase: "runtime", ref: "test/e2e-scenario/validation_suites/hermes/00-hermes-health.sh", reliability: { timeoutSeconds: 30, retry: { attempts: 2, on: ["gateway-transient"] } } })]), +]; + +export const assertionRegistry = { + groups: [environmentBaseline(), ...onboardingAssertionGroups, ...runtimeControlGroups, ...validationSuiteGroups], +}; + +export function assertionGroupForSuite(suiteId: string): AssertionGroup | undefined { + return validationSuiteGroups.find((group) => group.suiteId === suiteId); +} + +export function assertionGroupForOnboardingAssertion(assertionId: string): AssertionGroup | undefined { + return onboardingAssertionGroups.find((group) => group.onboardingAssertionId === assertionId); +} + +function supplementalSuiteIdsForScenario(scenario: ScenarioDefinition): string[] { + const ids: string[] = []; + if (scenario.id === "ubuntu-repo-cloud-openclaw") { + ids.push( + "gateway-health", + "sandbox-shell", + "cloud-inference", + "inference-routing", + "inference-switch", + "kimi-compatibility", + "security-credentials", + "security-shields", + "security-policy", + "security-injection", + "sandbox-lifecycle", + "sandbox-operations", + "snapshot", + "rebuild", + "upgrade", + "diagnostics", + "docs-validation", + ); + } + if (scenario.id === "gpu-repo-local-ollama-openclaw") { + ids.push("ollama-auth-proxy"); + } + if (scenario.id === "ubuntu-repo-openai-compatible-openclaw") { + ids.push("openai-compatible-inference"); + } + if (scenario.id.includes("telegram")) { + ids.push("messaging-telegram"); + } + if (scenario.id.includes("discord")) { + ids.push("messaging-discord"); + } + if (scenario.id.includes("slack")) { + ids.push("messaging-slack"); + } + if (scenario.id.includes("token-rotation")) { + ids.push("messaging-token-rotation"); + } + return ids; +} + +function uniqueGroups(groups: AssertionGroup[]): AssertionGroup[] { + const seen = new Set(); + return groups.filter((group) => { + if (seen.has(group.id)) { + return false; + } + seen.add(group.id); + return true; + }); +} + +export function assertionGroupsForScenario(scenario: ScenarioDefinition): AssertionGroup[] { + const groups = [ + environmentBaseline(), + ...(scenario.onboardingAssertionIds ?? []).map((id) => assertionGroupForOnboardingAssertion(id)), + ...(scenario.suiteIds ?? []).map((id) => assertionGroupForSuite(id)), + ...supplementalSuiteIdsForScenario(scenario).map((id) => assertionGroupForSuite(id)), + scenario.expectedFailure ? runtimeControlGroups[0] : undefined, + ].filter((entry): entry is AssertionGroup => Boolean(entry)); + return uniqueGroups(groups); +} + +export function validateAssertionGroups(groups: AssertionGroup[], repoRoot: string): void { + for (const group of groups) { + if (!group.id) { + throw new Error("Assertion group is missing stable ID"); + } + if (!group.phase) { + throw new Error(`Assertion group ${group.id} is missing phase owner`); + } + if (group.migrationStatus && group.migrationStatus !== "complete") { + throw new Error(`Assertion group ${group.id} is not complete`); + } + if (group.steps.length === 0) { + throw new Error(`Assertion group ${group.id} has no steps`); + } + for (const step of group.steps) { + if (!step.id) { + throw new Error(`Assertion group ${group.id} has a step without stable ID`); + } + if (!step.phase) { + throw new Error(`Assertion step ${step.id} is missing phase owner`); + } + if (!step.implementation?.ref) { + throw new Error(`Assertion step ${step.id} is missing implementation reference`); + } + if (!step.evidencePath) { + throw new Error(`Assertion step ${step.id} is missing evidence path`); + } + if ((step.reliability?.retry?.attempts ?? 1) > 1 && (step.reliability?.retry?.on.length ?? 0) === 0) { + throw new Error(`Assertion step ${step.id} retries without a named classifier`); + } + if (step.implementation.kind === "shell") { + const scriptPath = path.resolve(repoRoot, step.implementation.ref); + const cwdScriptPath = path.resolve(process.cwd(), step.implementation.ref); + if (!fs.existsSync(scriptPath) && !fs.existsSync(cwdScriptPath)) { + throw new Error(`Assertion step ${step.id} references missing script ${step.implementation.ref}`); + } + } + } + } +} diff --git a/test/e2e-scenario/scenarios/assertions/runtime.ts b/test/e2e-scenario/scenarios/assertions/runtime.ts new file mode 100644 index 0000000000..5ed7031279 --- /dev/null +++ b/test/e2e-scenario/scenarios/assertions/runtime.ts @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AssertionGroup } from "../types.ts"; + +export function runtimeSmokeSkeleton(): AssertionGroup { + return { + id: "runtime.smoke.skeleton", + phase: "runtime", + description: "Skeleton runtime smoke assertion group.", + steps: [ + { + id: "runtime.plan.skeleton", + phase: "runtime", + description: "Placeholder step until validation suites are migrated.", + implementation: { kind: "pending", ref: "phase-1-skeleton" }, + evidencePath: ".e2e/runtime.result.json", + }, + ], + }; +} diff --git a/test/e2e-scenario/scenarios/assertions/security.ts b/test/e2e-scenario/scenarios/assertions/security.ts new file mode 100644 index 0000000000..c8336c8709 --- /dev/null +++ b/test/e2e-scenario/scenarios/assertions/security.ts @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { validationSuiteGroups } from "./registry.ts"; diff --git a/test/e2e-scenario/scenarios/builder.ts b/test/e2e-scenario/scenarios/builder.ts new file mode 100644 index 0000000000..b2b9243a51 --- /dev/null +++ b/test/e2e-scenario/scenarios/builder.ts @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { AssertionGroup, ScenarioDefinition, ScenarioEnvironment } from "./types.ts"; + +export class ScenarioBuilder { + private readonly definition: ScenarioDefinition; + + constructor(id: string) { + this.definition = { id, assertionGroups: [] }; + } + + description(description: string): ScenarioBuilder { + this.definition.description = description; + return this; + } + + manifest(manifestPath: string): ScenarioBuilder { + this.definition.manifestPath = manifestPath; + return this; + } + + environment(environment: ScenarioEnvironment): ScenarioBuilder { + this.definition.environment = environment; + return this; + } + + expectedState(expectedStateId: string): ScenarioBuilder { + this.definition.expectedStateId = expectedStateId; + return this; + } + + suites(suiteIds: string[]): ScenarioBuilder { + this.definition.suiteIds = suiteIds; + return this; + } + + onboardingAssertions(onboardingAssertionIds: string[]): ScenarioBuilder { + this.definition.onboardingAssertionIds = onboardingAssertionIds; + return this; + } + + assertions(assertionGroups: AssertionGroup[]): ScenarioBuilder { + this.definition.assertionGroups = assertionGroups; + return this; + } + + runnerRequirements(runnerRequirements: string[]): ScenarioBuilder { + this.definition.runnerRequirements = runnerRequirements; + return this; + } + + requiredSecrets(requiredSecrets: string[]): ScenarioBuilder { + this.definition.requiredSecrets = requiredSecrets; + return this; + } + + skippedCapabilities(skippedCapabilities: Array>): ScenarioBuilder { + this.definition.skippedCapabilities = skippedCapabilities; + return this; + } + + expectedFailure(expectedFailure: Record): ScenarioBuilder { + this.definition.expectedFailure = expectedFailure; + return this; + } + + build(): ScenarioDefinition { + return { + ...this.definition, + assertionGroups: [...this.definition.assertionGroups], + suiteIds: [...(this.definition.suiteIds ?? [])], + onboardingAssertionIds: [...(this.definition.onboardingAssertionIds ?? [])], + runnerRequirements: [...(this.definition.runnerRequirements ?? [])], + requiredSecrets: [...(this.definition.requiredSecrets ?? [])], + skippedCapabilities: [...(this.definition.skippedCapabilities ?? [])], + }; + } +} + +export function scenario(id: string): ScenarioBuilder { + return new ScenarioBuilder(id); +} diff --git a/test/e2e-scenario/scenarios/clients/agent.ts b/test/e2e-scenario/scenarios/clients/agent.ts new file mode 100644 index 0000000000..23a5491adb --- /dev/null +++ b/test/e2e-scenario/scenarios/clients/agent.ts @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface AgentObservation { + agent?: "openclaw" | "hermes"; + running?: boolean; +} + +export class AgentClient { + observeAgent(): AgentObservation { + return {}; + } +} diff --git a/test/e2e-scenario/scenarios/clients/gateway.ts b/test/e2e-scenario/scenarios/clients/gateway.ts new file mode 100644 index 0000000000..a6e54bfd45 --- /dev/null +++ b/test/e2e-scenario/scenarios/clients/gateway.ts @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface GatewayObservation { + reachable: boolean | null; + status?: string; +} + +export class GatewayClient { + observeHealth(): GatewayObservation { + return { reachable: null }; + } +} diff --git a/test/e2e-scenario/scenarios/clients/host-cli.ts b/test/e2e-scenario/scenarios/clients/host-cli.ts new file mode 100644 index 0000000000..878c734883 --- /dev/null +++ b/test/e2e-scenario/scenarios/clients/host-cli.ts @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface HostCommandObservation { + command: string[]; + exitCode: number | null; + stdout: string; + stderr: string; +} + +export class HostCliClient { + observeVersion(): HostCommandObservation { + return { command: ["nemoclaw", "--version"], exitCode: null, stdout: "", stderr: "" }; + } +} diff --git a/test/e2e-scenario/scenarios/clients/provider.ts b/test/e2e-scenario/scenarios/clients/provider.ts new file mode 100644 index 0000000000..03258a244f --- /dev/null +++ b/test/e2e-scenario/scenarios/clients/provider.ts @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface ProviderObservation { + provider?: string; + reachable?: boolean; +} + +export class ProviderClient { + observeProvider(): ProviderObservation { + return {}; + } +} diff --git a/test/e2e-scenario/scenarios/clients/sandbox.ts b/test/e2e-scenario/scenarios/clients/sandbox.ts new file mode 100644 index 0000000000..1e213443a2 --- /dev/null +++ b/test/e2e-scenario/scenarios/clients/sandbox.ts @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface SandboxObservation { + id?: string; + status?: string; +} + +export class SandboxClient { + observeSandbox(): SandboxObservation { + return {}; + } +} diff --git a/test/e2e-scenario/scenarios/clients/state.ts b/test/e2e-scenario/scenarios/clients/state.ts new file mode 100644 index 0000000000..2d3e592720 --- /dev/null +++ b/test/e2e-scenario/scenarios/clients/state.ts @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface StateObservation { + path?: string; + exists?: boolean; +} + +export class StateClient { + observeState(): StateObservation { + return {}; + } +} diff --git a/test/e2e-scenario/scenarios/compiler.ts b/test/e2e-scenario/scenarios/compiler.ts new file mode 100644 index 0000000000..5046c77dd2 --- /dev/null +++ b/test/e2e-scenario/scenarios/compiler.ts @@ -0,0 +1,214 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { loadManifest } from "./manifests.ts"; +import { requireScenarios } from "./registry.ts"; +import type { AssertionGroup, NemoClawInstanceManifest, PhaseName, RunPlan, ScenarioDefinition, SutBoundary } from "./types.ts"; + +const PHASES: PhaseName[] = ["environment", "onboarding", "runtime"]; +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../.."); + +function groupsForPhase(scenario: ScenarioDefinition, phase: PhaseName): AssertionGroup[] { + return scenario.assertionGroups.filter((group) => group.phase === phase); +} + +function resolveScenarioInputs(inputs: Array): ScenarioDefinition[] { + const ids = inputs.filter((input): input is string => typeof input === "string"); + const resolvedById = requireScenarios(ids); + let idCursor = 0; + return inputs.map((input) => (typeof input === "string" ? resolvedById[idCursor++] : input)); +} + +function expectedPlatform(platformId: string): { os: string; executionTarget: string } | undefined { + const mapping: Record = { + "ubuntu-local": { os: "ubuntu", executionTarget: "local" }, + "gpu-runner": { os: "ubuntu", executionTarget: "local" }, + "macos-local": { os: "macos", executionTarget: "local" }, + "wsl-local": { os: "wsl", executionTarget: "local" }, + "brev-launchable": { os: "ubuntu", executionTarget: "remote" }, + }; + return mapping[platformId]; +} + +function expectedRuntime(runtimeId: string): { containerEngine: string; containerDaemon: string } | undefined { + const mapping: Record = { + "docker-running": { containerEngine: "docker", containerDaemon: "running" }, + "gpu-docker-cdi": { containerEngine: "docker", containerDaemon: "running" }, + "macos-docker-optional": { containerEngine: "docker", containerDaemon: "optional" }, + "docker-missing": { containerEngine: "docker", containerDaemon: "missing" }, + }; + return mapping[runtimeId]; +} + +function validateManifestCompatibility(scenario: ScenarioDefinition, manifest?: NemoClawInstanceManifest) { + if (!manifest || !scenario.environment) { + return; + } + const platform = expectedPlatform(scenario.environment.platform); + if (platform) { + const actual = manifest.spec.setup.platform; + if (actual.os !== platform.os || actual.executionTarget !== platform.executionTarget) { + throw new Error( + `Scenario ${scenario.id} incompatible with manifest platform: expected ${platform.os}/${platform.executionTarget}, got ${actual.os}/${actual.executionTarget}`, + ); + } + } + const runtime = expectedRuntime(scenario.environment.runtime); + if (runtime) { + const actual = manifest.spec.setup.runtime; + if (actual.containerEngine !== runtime.containerEngine || actual.containerDaemon !== runtime.containerDaemon) { + throw new Error( + `Scenario ${scenario.id} incompatible with manifest runtime: expected ${runtime.containerEngine}/${runtime.containerDaemon}, got ${actual.containerEngine}/${actual.containerDaemon}`, + ); + } + } +} + +function phaseActions(phase: PhaseName, scenario: ScenarioDefinition): string[] { + if (phase === "environment") { + return [ + `install:${scenario.environment?.install ?? "unknown"}`, + `runtime:${scenario.environment?.runtime ?? "unknown"}`, + ]; + } + if (phase === "onboarding") { + return [`onboard:${scenario.environment?.onboarding ?? "unknown"}`]; + } + return (scenario.suiteIds ?? []).map((suiteId) => `suite:${suiteId}`); +} + +const SUT_BOUNDARIES: SutBoundary[] = [ + { id: "host-cli", client: "HostCliClient" }, + { id: "gateway", client: "GatewayClient" }, + { id: "sandbox", client: "SandboxClient" }, + { id: "agent", client: "AgentClient" }, + { id: "provider", client: "ProviderClient" }, + { id: "state", client: "StateClient" }, +]; + +export function validateRunPlan(plan: RunPlan): void { + if (!plan.scenarioId) { + throw new Error("RunPlan missing scenarioId"); + } + for (const phase of PHASES) { + if (!plan.phases.some((entry) => entry.name === phase)) { + throw new Error(`RunPlan ${plan.scenarioId} missing phase ${phase}`); + } + } + if (plan.sutBoundaries.length === 0) { + throw new Error(`RunPlan ${plan.scenarioId} missing SUT boundaries`); + } +} + +export function compileRunPlans(inputs: Array): RunPlan[] { + return resolveScenarioInputs(inputs).map((scenario) => { + const manifest = scenario.manifestPath + ? loadManifest(path.resolve(REPO_ROOT, scenario.manifestPath)).document + : undefined; + validateManifestCompatibility(scenario, manifest); + const plan: RunPlan = { + scenarioId: scenario.id, + status: "compiled", + note: "compiled plan-only preview; live execution lands in later phases", + manifestPath: scenario.manifestPath, + manifest, + environment: scenario.environment, + expectedStateId: scenario.expectedStateId, + suiteIds: scenario.suiteIds ?? [], + onboardingAssertionIds: scenario.onboardingAssertionIds ?? [], + phases: PHASES.map((phase) => ({ + name: phase, + actions: phaseActions(phase, scenario), + assertionGroups: groupsForPhase(scenario, phase), + })), + runnerRequirements: scenario.runnerRequirements ?? [], + requiredSecrets: scenario.requiredSecrets ?? [], + skippedCapabilities: scenario.skippedCapabilities ?? [], + expectedFailure: scenario.expectedFailure, + sutBoundaries: SUT_BOUNDARIES, + }; + validateRunPlan(plan); + return plan; + }); +} + +export function renderPlanText(plans: RunPlan[]): string { + const lines = ["Hybrid scenario run plan", ""]; + for (const plan of plans) { + lines.push(`Scenario: ${plan.scenarioId}`); + lines.push(`Status: ${plan.status}`); + lines.push(`Note: ${plan.note ?? ""}`); + lines.push(`Manifest: ${plan.manifestPath ?? "not-yet-defined"}`); + if (plan.environment) { + lines.push( + `Environment: platform=${plan.environment.platform} install=${plan.environment.install} runtime=${plan.environment.runtime} onboarding=${plan.environment.onboarding}`, + ); + } + if (plan.expectedStateId) { + lines.push(`Expected state: ${plan.expectedStateId}`); + } + if (plan.suiteIds.length > 0) { + lines.push(`Suites: ${plan.suiteIds.join(", ")}`); + } + if (plan.requiredSecrets.length > 0) { + lines.push(`Required secrets: ${plan.requiredSecrets.join(", ")}`); + } + if (plan.runnerRequirements.length > 0) { + lines.push(`Runner requirements: ${plan.runnerRequirements.join(", ")}`); + } + if (plan.skippedCapabilities.length > 0) { + lines.push(`Skipped capabilities: ${plan.skippedCapabilities.map((entry) => entry.id ?? "unnamed").join(", ")}`); + } + if (plan.expectedFailure) { + lines.push(`Expected failure: ${JSON.stringify(plan.expectedFailure)}`); + } + if (plan.sutBoundaries.length > 0) { + lines.push( + `SUT boundaries: ${plan.sutBoundaries.map((boundary) => `${boundary.id}:${boundary.client}`).join(", ")}`, + ); + } + if (plan.manifest) { + const setup = plan.manifest.spec.setup; + const onboarding = plan.manifest.spec.onboarding; + lines.push( + `Setup: install=${setup.install.source ?? "unknown"} runtime=${setup.runtime.containerEngine ?? "unknown"}/${setup.runtime.containerDaemon ?? "unknown"} platform=${setup.platform.os ?? "unknown"}/${setup.platform.executionTarget ?? "unknown"}`, + ); + lines.push( + `Onboarding: agent=${onboarding.agent} provider=${onboarding.provider} modelRoute=${onboarding.modelRoute ?? "unknown"}`, + ); + } + for (const phase of plan.phases) { + lines.push(`Phase: ${phase.name}`); + for (const group of phase.assertionGroups) { + lines.push(` Group: ${group.id}`); + for (const step of group.steps) { + const policy: string[] = []; + if (step.reliability?.timeoutSeconds) { + policy.push(`timeout=${step.reliability.timeoutSeconds}s`); + } + if (step.reliability?.retry && step.reliability.retry.attempts > 1) { + policy.push( + `retry=${step.reliability.retry.attempts} on ${step.reliability.retry.on.join("+")}`, + ); + } + lines.push(` Step: ${step.id}${policy.length > 0 ? ` (${policy.join(", ")})` : ""}`); + } + } + } + lines.push(""); + } + return `${lines.join("\n").trimEnd()}\n`; +} + +export function writePlanArtifacts(plans: RunPlan[], contextDir: string): { jsonPath: string; summaryPath: string } { + const outputDir = path.join(contextDir, ".e2e"); + fs.mkdirSync(outputDir, { recursive: true }); + const jsonPath = path.join(outputDir, "run-plan.json"); + const summaryPath = path.join(outputDir, "plan.txt"); + fs.writeFileSync(jsonPath, `${JSON.stringify(plans, null, 2)}\n`); + fs.writeFileSync(summaryPath, renderPlanText(plans)); + return { jsonPath, summaryPath }; +} diff --git a/test/e2e-scenario/scenarios/js-yaml.d.ts b/test/e2e-scenario/scenarios/js-yaml.d.ts new file mode 100644 index 0000000000..6ea52a82de --- /dev/null +++ b/test/e2e-scenario/scenarios/js-yaml.d.ts @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Local type shim for js-yaml. The runtime package ships without +// TypeScript declarations; we only use `load` for YAML parsing. +declare module "js-yaml" { + export function load(input: string): unknown; + export function dump(obj: unknown, opts?: Record): string; + const _default: { load: typeof load; dump: typeof dump }; + export default _default; +} diff --git a/test/e2e-scenario/scenarios/manifests.ts b/test/e2e-scenario/scenarios/manifests.ts new file mode 100644 index 0000000000..58a89ac1c1 --- /dev/null +++ b/test/e2e-scenario/scenarios/manifests.ts @@ -0,0 +1,105 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import yaml from "js-yaml"; +import type { NemoClawInstanceManifest } from "./types.ts"; + +export interface LoadedManifest { + filePath: string; + document: NemoClawInstanceManifest; +} + +const FORBIDDEN_PRODUCT_FIELDS = new Set([ + "assertion", + "assertions", + "assertionGroups", + "assertionGroupIds", + "suite", + "suites", + "suiteIds", + "testPlan", + "testPlans", +]); + +const SECRET_KEY_PATTERN = /(api[-_]?key|token|secret|password|credential)$/i; + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function asRecord(value: unknown, fieldPath: string, filePath: string): Record { + if (!isRecord(value)) { + throw new Error(`${filePath}: ${fieldPath} must be an object`); + } + return value; +} + +function assertString(value: unknown, fieldPath: string, filePath: string): asserts value is string { + if (typeof value !== "string" || value.trim() === "") { + throw new Error(`${filePath}: ${fieldPath} must be a non-empty string`); + } +} + +function scanProductOnly(value: unknown, filePath: string, fieldPath = "manifest") { + if (Array.isArray(value)) { + value.forEach((entry, index) => scanProductOnly(entry, filePath, `${fieldPath}[${index}]`)); + return; + } + if (!isRecord(value)) { + return; + } + + for (const [key, child] of Object.entries(value)) { + if (FORBIDDEN_PRODUCT_FIELDS.has(key)) { + throw new Error(`${filePath}: ${fieldPath}.${key} is test assertion/suite metadata; manifests are product-facing only`); + } + if (SECRET_KEY_PATTERN.test(key) && key !== "credentialRefs" && typeof child === "string" && child.trim() !== "") { + throw new Error(`${filePath}: ${fieldPath}.${key} looks like a raw secret; use state.credentialRefs instead`); + } + scanProductOnly(child, filePath, `${fieldPath}.${key}`); + } +} + +function validateCredentialRefs(state: Record | undefined, filePath: string) { + const refs = state?.credentialRefs; + if (refs === undefined) { + return; + } + if (!Array.isArray(refs) || refs.some((ref) => typeof ref !== "string" || ref.trim() === "")) { + throw new Error(`${filePath}: spec.state.credentialRefs must be a string array`); + } +} + +export function validateManifest(document: unknown, filePath = "manifest"): asserts document is NemoClawInstanceManifest { + const root = asRecord(document, "manifest", filePath); + if (root.apiVersion !== "nemoclaw.io/v1") { + throw new Error(`${filePath}: apiVersion must be nemoclaw.io/v1`); + } + if (root.kind !== "NemoClawInstance") { + throw new Error(`${filePath}: kind must be NemoClawInstance`); + } + const metadata = asRecord(root.metadata, "metadata", filePath); + assertString(metadata.name, "metadata.name", filePath); + const spec = asRecord(root.spec, "spec", filePath); + asRecord(spec.setup, "spec.setup", filePath); + asRecord(spec.onboarding, "spec.onboarding", filePath); + const state = spec.state === undefined ? undefined : asRecord(spec.state, "spec.state", filePath); + validateCredentialRefs(state, filePath); + scanProductOnly(root, filePath); +} + +export function loadManifest(filePath: string): LoadedManifest { + const document = yaml.load(fs.readFileSync(filePath, "utf8")); + validateManifest(document, filePath); + return { filePath, document }; +} + +export function loadManifestsFromDir(directory: string): LoadedManifest[] { + return fs + .readdirSync(directory) + .filter((entry) => entry.endsWith(".yaml") || entry.endsWith(".yml")) + .sort() + .map((entry) => loadManifest(path.join(directory, entry))); +} diff --git a/test/e2e-scenario/scenarios/matrix.ts b/test/e2e-scenario/scenarios/matrix.ts new file mode 100644 index 0000000000..dc869941c9 --- /dev/null +++ b/test/e2e-scenario/scenarios/matrix.ts @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ScenarioEnvironment } from "./types.ts"; + +export function ubuntuRepoDocker(onboarding: string): ScenarioEnvironment { + return { platform: "ubuntu-local", install: "repo-current", runtime: "docker-running", onboarding }; +} + +export function gpuRepoDockerCdi(onboarding: string): ScenarioEnvironment { + return { platform: "gpu-runner", install: "repo-current", runtime: "gpu-docker-cdi", onboarding }; +} + +export function macosRepoDocker(onboarding: string): ScenarioEnvironment { + return { platform: "macos-local", install: "repo-current", runtime: "macos-docker-optional", onboarding }; +} + +export function wslRepoDocker(onboarding: string): ScenarioEnvironment { + return { platform: "wsl-local", install: "repo-current", runtime: "docker-running", onboarding }; +} + +export function brevLaunchableRemote(onboarding: string): ScenarioEnvironment { + return { platform: "brev-launchable", install: "launchable", runtime: "docker-running", onboarding }; +} + +export function ubuntuRepoNoDocker(onboarding: string): ScenarioEnvironment { + return { platform: "ubuntu-local", install: "repo-current", runtime: "docker-missing", onboarding }; +} diff --git a/test/e2e-scenario/scenarios/migration-inventory.ts b/test/e2e-scenario/scenarios/migration-inventory.ts new file mode 100644 index 0000000000..d79eae7360 --- /dev/null +++ b/test/e2e-scenario/scenarios/migration-inventory.ts @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type MigrationStatus = "targeted" | "remove-with-rationale"; + +export interface MigrationInventoryEntry { + id: string; + newOwner: string; + status: MigrationStatus; + rationale?: string; +} + +const targeted = (id: string, newOwner: string): MigrationInventoryEntry => ({ + id, + newOwner, + status: "targeted", +}); + +export const migrationInventory = { + setupScenarios: [ + targeted("ubuntu-repo-cloud-openclaw", "scenario:ubuntu-repo-cloud-openclaw"), + targeted("ubuntu-repo-cloud-hermes", "scenario:ubuntu-repo-cloud-hermes"), + targeted("gpu-repo-local-ollama-openclaw", "scenario:gpu-repo-local-ollama-openclaw"), + targeted("macos-repo-cloud-openclaw", "scenario:macos-repo-cloud-openclaw"), + targeted("wsl-repo-cloud-openclaw", "scenario:wsl-repo-cloud-openclaw"), + targeted("brev-launchable-cloud-openclaw", "scenario:brev-launchable-cloud-openclaw"), + targeted("ubuntu-no-docker-preflight-negative", "scenario:ubuntu-no-docker-preflight-negative"), + ], + baseScenarios: [ + targeted("ubuntu-repo-docker", "scenario environment helper:ubuntuRepoDocker"), + targeted("gpu-repo-docker-cdi", "scenario environment helper:gpuRepoDockerCdi"), + targeted("macos-repo-docker", "scenario environment helper:macosRepoDocker"), + targeted("wsl-repo-docker", "scenario environment helper:wslRepoDocker"), + targeted("brev-launchable-remote", "scenario environment helper:brevLaunchableRemote"), + targeted("ubuntu-repo-no-docker", "scenario environment helper:ubuntuRepoNoDocker"), + ], + onboardingProfiles: [ + targeted("cloud-nvidia-openclaw", "manifest:openclaw-nvidia"), + targeted("cloud-nvidia-hermes", "manifest:hermes-nvidia"), + targeted("local-ollama-openclaw", "manifest:openclaw-ollama-gpu"), + targeted("openai-compatible-openclaw", "manifest:openclaw-openai-compatible"), + targeted("cloud-nvidia-openclaw-brave", "manifest:openclaw-nvidia-brave"), + targeted("cloud-nvidia-openclaw-telegram", "manifest:openclaw-nvidia-telegram"), + targeted("cloud-nvidia-openclaw-discord", "manifest:openclaw-nvidia-discord"), + targeted("cloud-nvidia-openclaw-slack", "manifest:openclaw-nvidia-slack"), + targeted("cloud-nvidia-hermes-discord", "manifest:hermes-nvidia-discord"), + targeted("cloud-nvidia-hermes-slack", "manifest:hermes-nvidia-slack"), + targeted("cloud-nvidia-openclaw-resume-after-interrupt", "manifest:openclaw-nvidia-resume"), + targeted("cloud-nvidia-openclaw-repair-existing-config", "manifest:openclaw-nvidia-repair"), + targeted("cloud-nvidia-openclaw-double-same-provider", "manifest:openclaw-nvidia-double-same-provider"), + targeted("cloud-nvidia-openclaw-double-provider-switch", "manifest:openclaw-nvidia-double-provider-switch"), + targeted("cloud-nvidia-openclaw-token-rotation", "manifest:openclaw-nvidia-token-rotation"), + ], + testPlans: [ + targeted("ubuntu-repo-docker__cloud-nvidia-openclaw", "scenario:ubuntu-repo-cloud-openclaw"), + targeted("ubuntu-repo-docker__cloud-nvidia-hermes", "scenario:ubuntu-repo-cloud-hermes"), + targeted("gpu-repo-docker-cdi__local-ollama-openclaw", "scenario:gpu-repo-local-ollama-openclaw"), + targeted("macos-repo-docker__cloud-nvidia-openclaw", "scenario:macos-repo-cloud-openclaw"), + targeted("wsl-repo-docker__cloud-nvidia-openclaw", "scenario:wsl-repo-cloud-openclaw"), + targeted("brev-launchable-remote__cloud-nvidia-openclaw", "scenario:brev-launchable-cloud-openclaw"), + targeted("ubuntu-repo-no-docker__cloud-nvidia-openclaw", "scenario:ubuntu-no-docker-preflight-negative"), + targeted("ubuntu-repo-docker__openai-compatible-openclaw", "scenario:ubuntu-repo-openai-compatible-openclaw"), + targeted("ubuntu-repo-docker__cloud-nvidia-openclaw-brave", "scenario:ubuntu-repo-cloud-openclaw-brave"), + targeted("ubuntu-repo-docker__cloud-nvidia-openclaw-telegram", "scenario:ubuntu-repo-cloud-openclaw-telegram"), + targeted("ubuntu-repo-docker__cloud-nvidia-openclaw-discord", "scenario:ubuntu-repo-cloud-openclaw-discord"), + targeted("ubuntu-repo-docker__cloud-nvidia-openclaw-slack", "scenario:ubuntu-repo-cloud-openclaw-slack"), + targeted("ubuntu-repo-docker__cloud-nvidia-hermes-discord", "scenario:ubuntu-repo-cloud-hermes-discord"), + targeted("ubuntu-repo-docker__cloud-nvidia-hermes-slack", "scenario:ubuntu-repo-cloud-hermes-slack"), + targeted("ubuntu-repo-docker__cloud-nvidia-openclaw-resume-after-interrupt", "scenario:ubuntu-repo-cloud-openclaw-resume"), + targeted("ubuntu-repo-docker__cloud-nvidia-openclaw-repair-existing-config", "scenario:ubuntu-repo-cloud-openclaw-repair"), + targeted("ubuntu-repo-docker__cloud-nvidia-openclaw-double-same-provider", "scenario:ubuntu-repo-cloud-openclaw-double-same-provider"), + targeted("ubuntu-repo-docker__cloud-nvidia-openclaw-double-provider-switch", "scenario:ubuntu-repo-cloud-openclaw-double-provider-switch"), + targeted("ubuntu-repo-docker__cloud-nvidia-openclaw-token-rotation", "scenario:ubuntu-repo-cloud-openclaw-token-rotation"), + ], + expectedStates: [ + targeted("cloud-openclaw-ready", "assertion modules:cloudOpenClawReady"), + targeted("macos-cli-ready-docker-optional", "assertion modules:macosCliDockerOptional"), + targeted("cloud-hermes-ready", "assertion modules:cloudHermesReady"), + targeted("local-ollama-openclaw-ready", "assertion modules:localOllamaOpenClawReady"), + targeted("preflight-failure-no-sandbox", "assertion modules:preflightFailureNoSandbox"), + targeted("cloud-openclaw-custom-policies-ready", "assertion modules:cloudOpenClawCustomPoliciesReady"), + targeted("onboarding-failure-invalid-nvidia-key", "assertion modules:onboardingFailureInvalidNvidiaKey"), + targeted("onboarding-failure-gateway-port-conflict", "assertion modules:onboardingFailureGatewayPortConflict"), + ], + onboardingAssertions: [ + targeted("base-installed", "assertion:onboarding.base.cli-installed"), + targeted("preflight-passed", "assertion:onboarding.preflight.passed"), + targeted("preflight-expected-failed", "assertion:onboarding.preflight.expected-failed"), + ], + validationSuites: [ + targeted("smoke", "assertion:runtime.smoke"), + targeted("inference", "assertion:runtime.inference"), + targeted("credentials", "assertion:runtime.credentials"), + targeted("local-ollama-inference", "assertion:runtime.local-ollama-inference"), + targeted("ollama-proxy", "assertion:runtime.ollama-proxy"), + targeted("platform-macos", "assertion:platform.macos"), + targeted("platform-wsl", "assertion:platform.wsl"), + targeted("hermes-specific", "assertion:runtime.hermes-specific"), + targeted("gateway-health", "assertion:runtime.gateway-health"), + targeted("sandbox-shell", "assertion:runtime.sandbox-shell"), + targeted("cloud-inference", "assertion:runtime.cloud-inference"), + targeted("ollama-auth-proxy", "assertion:runtime.ollama-auth-proxy"), + targeted("security-credentials", "assertion:security.credentials"), + targeted("messaging-telegram", "assertion:messaging.telegram"), + targeted("messaging-discord", "assertion:messaging.discord"), + targeted("messaging-slack", "assertion:messaging.slack"), + targeted("security-shields", "assertion:security.shields"), + targeted("inference-routing", "assertion:runtime.inference-routing"), + targeted("sandbox-lifecycle", "assertion:lifecycle.sandbox-lifecycle"), + targeted("sandbox-operations", "assertion:lifecycle.sandbox-operations"), + targeted("snapshot", "assertion:lifecycle.snapshot"), + targeted("rebuild", "assertion:lifecycle.rebuild"), + targeted("upgrade", "assertion:lifecycle.upgrade"), + targeted("diagnostics", "assertion:diagnostics"), + targeted("docs-validation", "assertion:docs-validation"), + targeted("openai-compatible-inference", "assertion:runtime.openai-compatible-inference"), + targeted("inference-switch", "assertion:runtime.inference-switch"), + targeted("kimi-compatibility", "assertion:runtime.kimi-compatibility"), + targeted("messaging-token-rotation", "assertion:messaging.token-rotation"), + targeted("security-policy", "assertion:security.policy"), + targeted("security-injection", "assertion:security.injection"), + targeted("baseline-onboarding", "assertion:baseline.onboarding"), + targeted("model-router", "assertion:runtime.model-router"), + targeted("onboarding-state", "assertion:onboarding.state"), + targeted("snapshot-lifecycle", "assertion:lifecycle.snapshot"), + ], + validationSuiteScripts: [ + targeted("baseline-onboarding/00-cli-and-openshell.sh", "assertion step:baseline.cli-and-openshell"), + targeted("baseline-onboarding/01-sandbox-state.sh", "assertion step:baseline.sandbox-state"), + targeted("baseline-onboarding/02-route-and-smoke.sh", "assertion step:baseline.route-and-smoke"), + targeted("hermes/00-hermes-health.sh", "assertion step:runtime.hermes.health"), + targeted("inference/cloud/00-models-health.sh", "assertion step:runtime.inference.models-health"), + targeted("inference/cloud/01-chat-completion.sh", "assertion step:runtime.inference.chat-completion"), + targeted("inference/cloud/02-inference-local-from-sandbox.sh", "assertion step:runtime.inference.sandbox-local"), + targeted("inference/kimi-compatibility/00-plugin-wiring.sh", "assertion step:runtime.kimi.plugin-wiring"), + targeted("inference/kimi-compatibility/01-kimi-compatible-models-route.sh", "assertion step:runtime.kimi.compatible-models-route"), + targeted("inference/model-router/00-healthy-endpoint.sh", "assertion step:runtime.model-router.healthy-endpoint"), + targeted("inference/model-router/01-provider-routed-completion.sh", "assertion step:runtime.model-router.provider-routed-completion"), + targeted("inference/ollama-auth-proxy/00-proxy-reachable.sh", "assertion step:runtime.ollama-auth-proxy.reachable"), + targeted("inference/ollama-auth-proxy/01-auth-enforcement.sh", "assertion step:runtime.ollama-auth-proxy.auth-enforcement"), + targeted("inference/ollama-gpu/00-ollama-models-health.sh", "assertion step:runtime.ollama.models-health"), + targeted("inference/routing/00-inference-local-chat-completion.sh", "assertion step:runtime.inference.routing-chat"), + targeted("inference/routing/01-provider-route-health.sh", "assertion step:runtime.inference.provider-route-health"), + targeted("inference/switch/00-route-state-updated.sh", "assertion step:runtime.inference.route-state-updated"), + targeted("inference/switch/01-switched-inference-local-chat.sh", "assertion step:runtime.inference.switched-local-chat"), + targeted("inference/ollama-gpu/01-ollama-chat-completion.sh", "assertion step:runtime.ollama.chat-completion"), + targeted("platform/macos/00-macos-smoke.sh", "assertion step:platform.macos.smoke"), + targeted("platform/wsl/00-wsl-smoke.sh", "assertion step:platform.wsl.smoke"), + targeted("onboarding/state/00-registry-provider-model-policies.sh", "assertion step:onboarding.state.registry"), + targeted("onboarding/state/01-session-provider-model-policies.sh", "assertion step:onboarding.state.session"), + targeted("rebuild_upgrade/00-state-preserved.sh", "assertion step:lifecycle.rebuild.state-preserved"), + targeted("rebuild_upgrade/01-agent-version-upgraded.sh", "assertion step:lifecycle.rebuild.agent-version-upgraded"), + targeted("rebuild_upgrade/02-post-rebuild-inference.sh", "assertion step:lifecycle.rebuild.post-rebuild-inference"), + targeted("rebuild_upgrade/03-policy-config-preserved.sh", "assertion step:lifecycle.upgrade.policy-config-preserved"), + targeted("rebuild_upgrade/04-upgrade-survivor-reachable.sh", "assertion step:lifecycle.upgrade.survivor-reachable"), + targeted("sandbox/lifecycle/00-gateway-health.sh", "assertion step:lifecycle.sandbox.gateway-health"), + targeted("sandbox/lifecycle/01-gateway-recovery.sh", "assertion step:lifecycle.sandbox.gateway-recovery"), + targeted("sandbox/operations/00-list-and-status.sh", "assertion step:lifecycle.sandbox.list-and-status"), + targeted("sandbox/operations/01-logs-and-exec.sh", "assertion step:lifecycle.sandbox.logs-and-exec"), + targeted("sandbox/snapshot/00-create-list-restore.sh", "assertion step:lifecycle.snapshot.create-list-restore"), + targeted("security/credentials/00-credentials-present.sh", "assertion step:security.credentials.present"), + targeted("security/credentials/01-no-plaintext-host-store.sh", "assertion step:security.credentials.no-plaintext-host-store"), + targeted("security/injection/00-telegram-message-not-shell-executed.sh", "assertion step:security.injection.blocked"), + targeted("security/policy/00-telegram-preset-applied.sh", "assertion step:security.policy.telegram-preset"), + targeted("security/policy/01-openshell-version-supports-credential-rewrite.sh", "assertion step:security.policy.credential-rewrite"), + targeted("security/shields/00-config-consistent.sh", "assertion step:security.shields.config"), + targeted("smoke/00-cli-available.sh", "assertion step:runtime.smoke.cli-available"), + targeted("smoke/01-gateway-health.sh", "assertion step:runtime.smoke.gateway-health"), + targeted("smoke/02-sandbox-listed.sh", "assertion step:runtime.smoke.sandbox-listed"), + targeted("smoke/03-sandbox-shell.sh", "assertion step:runtime.smoke.sandbox-shell"), + targeted("messaging/common/00-provider-attached.sh", "assertion step:messaging.common.provider-attached"), + targeted("messaging/common/01-placeholder-configured.sh", "assertion step:messaging.common.placeholder-configured"), + targeted("messaging/common/02-no-secret-leak.sh", "assertion step:messaging.common.no-secret-leak"), + targeted("messaging/common/03-bridge-reachable.sh", "assertion step:messaging.common.bridge-reachable"), + targeted("messaging/discord/00-discord-gateway-path.sh", "assertion step:messaging.discord.gateway-path"), + targeted("messaging/slack/00-slack-provider-state.sh", "assertion step:messaging.slack.provider-state"), + targeted("messaging/telegram/00-telegram-injection-safety.sh", "assertion step:messaging.telegram.injection-safety"), + targeted("messaging/telegram/01-telegram-injection-payload-classes.sh", "assertion step:messaging.telegram.injection-payload-classes"), + targeted("messaging/token-rotation/00-provider-rotation-isolated.sh", "assertion step:messaging.token-rotation"), + ], +} as const; diff --git a/test/e2e-scenario/scenarios/orchestrators/environment.ts b/test/e2e-scenario/scenarios/orchestrators/environment.ts new file mode 100644 index 0000000000..3c1496d15a --- /dev/null +++ b/test/e2e-scenario/scenarios/orchestrators/environment.ts @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { PhaseOrchestrator } from "./phase.ts"; + +export class EnvironmentOrchestrator extends PhaseOrchestrator { + constructor() { + super("environment"); + } +} diff --git a/test/e2e-scenario/scenarios/orchestrators/onboarding.ts b/test/e2e-scenario/scenarios/orchestrators/onboarding.ts new file mode 100644 index 0000000000..1600d2ec92 --- /dev/null +++ b/test/e2e-scenario/scenarios/orchestrators/onboarding.ts @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { PhaseOrchestrator } from "./phase.ts"; + +export class OnboardingOrchestrator extends PhaseOrchestrator { + constructor() { + super("onboarding"); + } +} diff --git a/test/e2e-scenario/scenarios/orchestrators/phase.ts b/test/e2e-scenario/scenarios/orchestrators/phase.ts new file mode 100644 index 0000000000..ae59a58e62 --- /dev/null +++ b/test/e2e-scenario/scenarios/orchestrators/phase.ts @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import type { + AssertionResult, + AssertionStep, + PhaseName, + PhaseResult, + RunContext, + RunPlanPhase, + TransientClassifier, +} from "../types.ts"; + +interface StepAttemptOutcome { + status: "passed" | "failed"; + classifier?: TransientClassifier; + message?: string; +} + +function transientForRef(ref: string): TransientClassifier { + if (ref.includes("provider") || ref.includes("transient")) { + return "provider-transient"; + } + if (ref.includes("gateway")) { + return "gateway-transient"; + } + return "runner-infra"; +} + +export class PhaseOrchestrator { + constructor(private readonly phaseName: PhaseName) {} + + async run(ctx: RunContext, phase: RunPlanPhase): Promise { + const assertions: AssertionResult[] = []; + for (const group of phase.assertionGroups) { + for (const step of group.steps) { + assertions.push(await this.runStep(ctx, step)); + } + } + const status = assertions.some((assertion) => assertion.status === "failed") ? "failed" : "passed"; + const result: PhaseResult = { phase: this.phaseName, status, assertions }; + this.writePhaseResult(ctx, result); + return result; + } + + private async runStep(ctx: RunContext, step: AssertionStep): Promise { + const startedAt = Date.now(); + const rawAttempts = step.reliability?.retry?.attempts; + const maxAttempts = typeof rawAttempts === "number" && Number.isFinite(rawAttempts) ? Math.max(1, Math.floor(rawAttempts)) : 1; + let attempts = 0; + let lastOutcome: StepAttemptOutcome = { status: "failed", message: "step did not run" }; + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + attempts = attempt; + lastOutcome = await this.executeStep(ctx, step, attempt); + if (lastOutcome.status === "passed") { + return { + id: step.id, + status: "passed", + attempts, + durationMs: Date.now() - startedAt, + classifier: attempt > 1 ? step.reliability?.retry?.on[0] : lastOutcome.classifier, + evidence: step.evidencePath, + message: lastOutcome.message, + }; + } + if (!this.canRetry(step, lastOutcome.classifier, attempt, maxAttempts)) { + break; + } + } + return { + id: step.id, + status: "failed", + attempts, + durationMs: Date.now() - startedAt, + classifier: lastOutcome.classifier, + evidence: step.evidencePath, + message: lastOutcome.message, + }; + } + + private canRetry( + step: AssertionStep, + classifier: TransientClassifier | undefined, + attempt: number, + maxAttempts: number, + ): boolean { + if (attempt >= maxAttempts || !classifier) { + return false; + } + return step.reliability?.retry?.on.includes(classifier) ?? false; + } + + private async executeStep(_ctx: RunContext, step: AssertionStep, attempt: number): Promise { + const ref = step.implementation?.ref ?? ""; + if (ref === "fake-pass" || ref === "phase-1-skeleton") { + return { status: "passed" }; + } + if (ref === "fake-retry-once-pass") { + return attempt === 1 + ? { status: "failed", classifier: step.reliability?.retry?.on[0] ?? "gateway-transient" } + : { status: "passed" }; + } + if (ref === "fake-always-transient") { + return { status: "failed", classifier: step.reliability?.retry?.on[0] ?? transientForRef(ref) }; + } + if (step.implementation?.kind === "shell" && _ctx.dryRun) { + return { status: "passed", message: `dry-run shell ${ref}` }; + } + if (step.implementation?.kind === "probe" && _ctx.dryRun) { + return { status: "passed", message: `dry-run probe ${ref}` }; + } + return { status: "failed", message: `unsupported live step ${step.id}` }; + } + + private writePhaseResult(ctx: RunContext, result: PhaseResult) { + const outputDir = path.join(ctx.contextDir, ".e2e"); + fs.mkdirSync(outputDir, { recursive: true }); + fs.writeFileSync(path.join(outputDir, `${result.phase}.result.json`), `${JSON.stringify(result, null, 2)}\n`); + } +} diff --git a/test/e2e-scenario/scenarios/orchestrators/runner.ts b/test/e2e-scenario/scenarios/orchestrators/runner.ts new file mode 100644 index 0000000000..1f48e6bc06 --- /dev/null +++ b/test/e2e-scenario/scenarios/orchestrators/runner.ts @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { PhaseResult, RunContext, RunPlan, RunPlanPhase } from "../types.ts"; +import { EnvironmentOrchestrator } from "./environment.ts"; +import { OnboardingOrchestrator } from "./onboarding.ts"; +import { RuntimeOrchestrator } from "./runtime.ts"; + +interface PhaseRunner { + run(ctx: RunContext, phase: RunPlanPhase, priorResults?: PhaseResult[]): Promise; +} + +export interface ScenarioRunnerDeps { + environment?: PhaseRunner; + onboarding?: PhaseRunner; + runtime?: PhaseRunner; +} + +export class ScenarioRunner { + private readonly environment: PhaseRunner; + private readonly onboarding: PhaseRunner; + private readonly runtime: PhaseRunner; + + constructor(deps: ScenarioRunnerDeps = {}) { + this.environment = deps.environment ?? new EnvironmentOrchestrator(); + this.onboarding = deps.onboarding ?? new OnboardingOrchestrator(); + this.runtime = deps.runtime ?? new RuntimeOrchestrator(); + } + + async run(ctx: RunContext, plan: RunPlan): Promise { + const results: PhaseResult[] = []; + for (const phase of plan.phases) { + if (phase.name === "environment") { + results.push(await this.environment.run(ctx, phase, results)); + } else if (phase.name === "onboarding") { + results.push(await this.onboarding.run(ctx, phase, results)); + } else { + results.push(await this.runtime.run(ctx, phase, results)); + } + } + return results; + } +} diff --git a/test/e2e-scenario/scenarios/orchestrators/runtime.ts b/test/e2e-scenario/scenarios/orchestrators/runtime.ts new file mode 100644 index 0000000000..67eef3ec59 --- /dev/null +++ b/test/e2e-scenario/scenarios/orchestrators/runtime.ts @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { PhaseOrchestrator } from "./phase.ts"; + +export class RuntimeOrchestrator extends PhaseOrchestrator { + constructor() { + super("runtime"); + } +} diff --git a/test/e2e-scenario/scenarios/registry.ts b/test/e2e-scenario/scenarios/registry.ts new file mode 100644 index 0000000000..8f33717cc1 --- /dev/null +++ b/test/e2e-scenario/scenarios/registry.ts @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { canonicalScenarios } from "./scenarios/baseline.ts"; +import type { ScenarioDefinition } from "./types.ts"; + +export interface ScenarioRegistry { + scenarios: ScenarioDefinition[]; + byId: Map; +} + +export function buildScenarioRegistry(scenarios: ScenarioDefinition[]): ScenarioRegistry { + const byId = new Map(); + const duplicates = new Set(); + for (const scenario of scenarios) { + if (byId.has(scenario.id)) { + duplicates.add(scenario.id); + } + byId.set(scenario.id, scenario); + } + if (duplicates.size > 0) { + throw new Error(`Duplicate scenario IDs: ${Array.from(duplicates).sort().join(", ")}`); + } + return { scenarios: [...scenarios], byId }; +} + +const registry = buildScenarioRegistry(canonicalScenarios()); + +export function listScenarios(): ScenarioDefinition[] { + return [...registry.scenarios].sort((a, b) => a.id.localeCompare(b.id)); +} + +export function getScenario(id: string): ScenarioDefinition | undefined { + return registry.byId.get(id); +} + +export function requireScenarios(ids: string[]): ScenarioDefinition[] { + const availableIds = listScenarios().map((scenario) => scenario.id); + const scenarios = ids.map((id) => { + const found = getScenario(id); + if (!found) { + throw new Error(`Unknown scenario '${id}'. Available scenarios: ${availableIds.join(", ")}`); + } + return found; + }); + return scenarios; +} diff --git a/test/e2e-scenario/scenarios/run.ts b/test/e2e-scenario/scenarios/run.ts new file mode 100644 index 0000000000..e666e07844 --- /dev/null +++ b/test/e2e-scenario/scenarios/run.ts @@ -0,0 +1,94 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { compileRunPlans, renderPlanText, writePlanArtifacts } from "./compiler.ts"; +import { ScenarioRunner } from "./orchestrators/runner.ts"; +import { listScenarios } from "./registry.ts"; + +interface Args { + list: boolean; + planOnly: boolean; + dryRun: boolean; + validateOnly: boolean; + scenarios: string[]; +} + +function parseArgs(argv: string[]): Args { + const args: Args = { list: false, planOnly: false, dryRun: false, validateOnly: false, scenarios: [] }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === "--list") { + args.list = true; + continue; + } + if (arg === "--plan-only") { + args.planOnly = true; + continue; + } + if (arg === "--dry-run") { + args.dryRun = true; + continue; + } + if (arg === "--validate-only") { + args.validateOnly = true; + continue; + } + if (arg === "--scenarios") { + const value = argv[i + 1]; + if (!value) { + throw new Error("--scenarios requires a comma-separated value"); + } + args.scenarios = value.split(",").map((id) => id.trim()).filter(Boolean); + i += 1; + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + return args; +} + +function printList() { + console.log("hybrid scenario registry"); + for (const scenario of listScenarios()) { + console.log(`- ${scenario.id}${scenario.description ? `: ${scenario.description}` : ""}`); + } +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.list) { + printList(); + return; + } + + const modeCount = [args.planOnly, args.dryRun, args.validateOnly].filter(Boolean).length; + if (modeCount !== 1) { + throw new Error("Use exactly one of --plan-only, --dry-run, or --validate-only with --scenarios "); + } + if (args.scenarios.length === 0) { + throw new Error("scenario execution requires --scenarios "); + } + + if (process.env.E2E_SUITE_FILTER) { + throw new Error("E2E_SUITE_FILTER is not supported; define assertion selection in scenario builders."); + } + + const plans = compileRunPlans(args.scenarios); + const contextDir = process.env.E2E_CONTEXT_DIR ?? process.cwd(); + writePlanArtifacts(plans, contextDir); + console.log(renderPlanText(plans)); + + if (args.dryRun) { + const runner = new ScenarioRunner(); + for (const plan of plans) { + await runner.run({ contextDir, dryRun: true }, plan); + } + } +} + +try { + await main(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +} diff --git a/test/e2e-scenario/scenarios/scenarios/baseline.ts b/test/e2e-scenario/scenarios/scenarios/baseline.ts new file mode 100644 index 0000000000..ef05fb6d6f --- /dev/null +++ b/test/e2e-scenario/scenarios/scenarios/baseline.ts @@ -0,0 +1,277 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { assertionGroupsForScenario } from "../assertions/registry.ts"; +import { scenario } from "../builder.ts"; +import { + brevLaunchableRemote, + gpuRepoDockerCdi, + macosRepoDocker, + ubuntuRepoDocker, + ubuntuRepoNoDocker, + wslRepoDocker, +} from "../matrix.ts"; +import type { ScenarioDefinition, ScenarioEnvironment } from "../types.ts"; + +interface CanonicalScenarioInput { + id: string; + manifestName: string; + environment: ScenarioEnvironment; + expectedStateId: string; + suiteIds: string[]; + onboardingAssertionIds?: string[]; + description?: string; + runnerRequirements?: string[]; + requiredSecrets?: string[]; + skippedCapabilities?: Array>; + expectedFailure?: Record; +} + +function canonicalScenario(input: CanonicalScenarioInput): ScenarioDefinition { + let builder = scenario(input.id) + .description(input.description ?? `Canonical typed scenario for ${input.id}.`) + .manifest(`test/e2e-scenario/manifests/${input.manifestName}.yaml`) + .environment(input.environment) + .expectedState(input.expectedStateId) + .onboardingAssertions(input.onboardingAssertionIds ?? ["base-installed", "preflight-passed"]) + .suites(input.suiteIds); + + if (input.runnerRequirements) { + builder = builder.runnerRequirements(input.runnerRequirements); + } + if (input.requiredSecrets) { + builder = builder.requiredSecrets(input.requiredSecrets); + } + if (input.skippedCapabilities) { + builder = builder.skippedCapabilities(input.skippedCapabilities); + } + if (input.expectedFailure) { + builder = builder.expectedFailure(input.expectedFailure); + } + builder = builder.assertions(assertionGroupsForScenario(builder.build())); + return builder.build(); +} + +const macosDockerSkipped = [ + { + id: "macos-docker-dependent-suites", + reason: + "GitHub-hosted macOS runners do not provide a reachable Docker daemon; gateway/sandbox/inference suites are reported as skipped instead of failing this scenario.", + suites: ["smoke", "inference", "credentials"], + }, +]; + +const canonicalScenarioInputs: CanonicalScenarioInput[] = [ + { + id: "ubuntu-repo-cloud-openclaw", + manifestName: "openclaw-nvidia", + environment: ubuntuRepoDocker("cloud-openclaw"), + expectedStateId: "cloud-openclaw-ready", + suiteIds: ["smoke", "inference", "credentials"], + description: "Ubuntu repo checkout with Docker and cloud OpenClaw onboarding.", + requiredSecrets: ["NVIDIA_API_KEY"], + }, + { + id: "ubuntu-repo-cloud-hermes", + manifestName: "hermes-nvidia", + environment: ubuntuRepoDocker("cloud-hermes"), + expectedStateId: "cloud-hermes-ready", + suiteIds: ["smoke", "inference", "hermes-specific"], + requiredSecrets: ["NVIDIA_API_KEY"], + }, + { + id: "gpu-repo-local-ollama-openclaw", + manifestName: "openclaw-ollama-gpu", + environment: gpuRepoDockerCdi("local-ollama-openclaw"), + expectedStateId: "local-ollama-openclaw-ready", + suiteIds: ["smoke", "local-ollama-inference", "ollama-proxy"], + runnerRequirements: ["self-hosted-gpu", "docker-cdi"], + }, + { + id: "macos-repo-cloud-openclaw", + manifestName: "openclaw-nvidia-macos", + environment: macosRepoDocker("cloud-openclaw"), + expectedStateId: "macos-cli-ready-docker-optional", + onboardingAssertionIds: ["base-installed"], + suiteIds: ["platform-macos"], + runnerRequirements: ["macos-latest"], + requiredSecrets: ["NVIDIA_API_KEY"], + skippedCapabilities: macosDockerSkipped, + }, + { + id: "wsl-repo-cloud-openclaw", + manifestName: "openclaw-nvidia-wsl", + environment: wslRepoDocker("cloud-openclaw"), + expectedStateId: "cloud-openclaw-ready", + suiteIds: ["smoke", "platform-wsl"], + runnerRequirements: ["windows-latest", "wsl2"], + requiredSecrets: ["NVIDIA_API_KEY"], + }, + { + id: "brev-launchable-cloud-openclaw", + manifestName: "openclaw-nvidia-brev-launchable", + environment: brevLaunchableRemote("cloud-openclaw"), + expectedStateId: "cloud-openclaw-ready", + suiteIds: ["smoke", "inference"], + runnerRequirements: ["ubuntu-latest", "brev-api-token", "launchable-image"], + requiredSecrets: ["NVIDIA_API_KEY"], + }, + { + id: "ubuntu-no-docker-preflight-negative", + manifestName: "openclaw-nvidia-no-docker-negative", + environment: ubuntuRepoNoDocker("cloud-openclaw"), + expectedStateId: "preflight-failure-no-sandbox", + onboardingAssertionIds: ["base-installed", "preflight-expected-failed"], + suiteIds: [], + requiredSecrets: ["NVIDIA_API_KEY"], + expectedFailure: { + phase: "preflight", + errorClass: "docker-missing", + forbiddenSideEffects: ["gateway-started", "sandbox-created"], + }, + }, + { + id: "ubuntu-repo-openai-compatible-openclaw", + manifestName: "openclaw-openai-compatible", + environment: ubuntuRepoDocker("openai-compatible-openclaw"), + expectedStateId: "cloud-openclaw-ready", + suiteIds: ["smoke"], + requiredSecrets: ["OPENAI_COMPATIBLE_API_KEY"], + }, + { + id: "ubuntu-repo-cloud-openclaw-brave", + manifestName: "openclaw-nvidia-brave", + environment: ubuntuRepoDocker("cloud-nvidia-openclaw-brave"), + expectedStateId: "cloud-openclaw-ready", + suiteIds: ["smoke"], + requiredSecrets: ["NVIDIA_API_KEY", "BRAVE_API_KEY"], + }, + { + id: "ubuntu-repo-cloud-openclaw-telegram", + manifestName: "openclaw-nvidia-telegram", + environment: ubuntuRepoDocker("cloud-nvidia-openclaw-telegram"), + expectedStateId: "cloud-openclaw-ready", + suiteIds: ["smoke", "messaging-telegram"], + requiredSecrets: ["NVIDIA_API_KEY", "TELEGRAM_BOT_TOKEN"], + }, + { + id: "ubuntu-repo-cloud-openclaw-discord", + manifestName: "openclaw-nvidia-discord", + environment: ubuntuRepoDocker("cloud-nvidia-openclaw-discord"), + expectedStateId: "cloud-openclaw-ready", + suiteIds: ["smoke", "messaging-discord"], + requiredSecrets: ["NVIDIA_API_KEY", "DISCORD_BOT_TOKEN"], + }, + { + id: "ubuntu-repo-cloud-openclaw-slack", + manifestName: "openclaw-nvidia-slack", + environment: ubuntuRepoDocker("cloud-nvidia-openclaw-slack"), + expectedStateId: "cloud-openclaw-ready", + suiteIds: ["smoke", "messaging-slack"], + requiredSecrets: ["NVIDIA_API_KEY", "SLACK_BOT_TOKEN"], + }, + { + id: "ubuntu-repo-cloud-hermes-discord", + manifestName: "hermes-nvidia-discord", + environment: ubuntuRepoDocker("cloud-nvidia-hermes-discord"), + expectedStateId: "cloud-hermes-ready", + suiteIds: ["smoke"], + requiredSecrets: ["NVIDIA_API_KEY", "DISCORD_BOT_TOKEN"], + }, + { + id: "ubuntu-repo-cloud-hermes-slack", + manifestName: "hermes-nvidia-slack", + environment: ubuntuRepoDocker("cloud-nvidia-hermes-slack"), + expectedStateId: "cloud-hermes-ready", + suiteIds: ["smoke"], + requiredSecrets: ["NVIDIA_API_KEY", "SLACK_BOT_TOKEN"], + }, + { + id: "ubuntu-repo-cloud-openclaw-resume", + manifestName: "openclaw-nvidia-resume", + environment: ubuntuRepoDocker("cloud-nvidia-openclaw-resume-after-interrupt"), + expectedStateId: "cloud-openclaw-ready", + suiteIds: ["smoke"], + requiredSecrets: ["NVIDIA_API_KEY"], + }, + { + id: "ubuntu-repo-cloud-openclaw-repair", + manifestName: "openclaw-nvidia-repair", + environment: ubuntuRepoDocker("cloud-nvidia-openclaw-repair-existing-config"), + expectedStateId: "cloud-openclaw-ready", + suiteIds: ["smoke"], + requiredSecrets: ["NVIDIA_API_KEY"], + }, + { + id: "ubuntu-repo-cloud-openclaw-double-same-provider", + manifestName: "openclaw-nvidia-double-same-provider", + environment: ubuntuRepoDocker("cloud-nvidia-openclaw-double-same-provider"), + expectedStateId: "cloud-openclaw-ready", + suiteIds: ["smoke"], + requiredSecrets: ["NVIDIA_API_KEY"], + }, + { + id: "ubuntu-repo-cloud-openclaw-double-provider-switch", + manifestName: "openclaw-nvidia-double-provider-switch", + environment: ubuntuRepoDocker("cloud-nvidia-openclaw-double-provider-switch"), + expectedStateId: "cloud-openclaw-ready", + suiteIds: ["smoke"], + requiredSecrets: ["NVIDIA_API_KEY"], + }, + { + id: "ubuntu-repo-cloud-openclaw-token-rotation", + manifestName: "openclaw-nvidia-token-rotation", + environment: ubuntuRepoDocker("cloud-nvidia-openclaw-token-rotation"), + expectedStateId: "cloud-openclaw-ready", + suiteIds: ["smoke", "messaging-token-rotation"], + requiredSecrets: ["NVIDIA_API_KEY"], + }, + { + id: "ubuntu-repo-cloud-openclaw-custom-policies", + manifestName: "openclaw-nvidia-custom-policies", + environment: ubuntuRepoDocker("cloud-openclaw-custom-policies"), + expectedStateId: "cloud-openclaw-custom-policies-ready", + suiteIds: ["smoke", "inference", "credentials", "onboarding-state", "baseline-onboarding", "model-router", "snapshot-lifecycle"], + requiredSecrets: ["NVIDIA_API_KEY"], + }, + { + id: "ubuntu-invalid-nvidia-key-negative", + manifestName: "openclaw-nvidia-invalid-key", + environment: ubuntuRepoDocker("cloud-openclaw-invalid-nvidia-key"), + expectedStateId: "onboarding-failure-invalid-nvidia-key", + onboardingAssertionIds: ["base-installed"], + suiteIds: [], + requiredSecrets: ["NVIDIA_API_KEY"], + expectedFailure: { + phase: "onboarding", + errorClass: "invalid-nvidia-api-key", + forbiddenSideEffects: ["gateway-started", "sandbox-created"], + }, + }, + { + id: "ubuntu-gateway-port-conflict-negative", + manifestName: "openclaw-nvidia-gateway-port-conflict", + environment: ubuntuRepoDocker("cloud-openclaw-gateway-port-conflict"), + expectedStateId: "onboarding-failure-gateway-port-conflict", + onboardingAssertionIds: ["base-installed"], + suiteIds: [], + requiredSecrets: ["NVIDIA_API_KEY"], + expectedFailure: { + phase: "onboarding", + errorClass: "gateway-port-conflict", + forbiddenSideEffects: ["gateway-started", "sandbox-created"], + }, + }, +]; + +export function canonicalScenarios(): ScenarioDefinition[] { + return canonicalScenarioInputs.map(canonicalScenario); +} + +export function ubuntuRepoCloudOpenClawScenario(): ScenarioDefinition { + const scenario = canonicalScenarios().find((entry) => entry.id === "ubuntu-repo-cloud-openclaw"); + if (!scenario) { + throw new Error("Missing canonical scenario 'ubuntu-repo-cloud-openclaw'"); + } + return scenario; +} diff --git a/test/e2e-scenario/scenarios/types.ts b/test/e2e-scenario/scenarios/types.ts new file mode 100644 index 0000000000..b29f8458d6 --- /dev/null +++ b/test/e2e-scenario/scenarios/types.ts @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type PhaseName = "environment" | "onboarding" | "runtime"; + +export type TransientClassifier = + | "empty-event-capture" + | "provider-transient" + | "gateway-transient" + | "external-tunnel" + | "model-toolcall-transient" + | "runner-infra" + | "wrong-installed-ref"; + +export interface SutBoundary { + id: "host-cli" | "gateway" | "sandbox" | "agent" | "provider" | "state"; + client: string; +} + +export interface NemoClawInstanceManifest { + apiVersion: "nemoclaw.io/v1"; + kind: "NemoClawInstance"; + metadata: { + name: string; + }; + spec: { + setup: { + install: Record; + runtime: Record; + platform: Record; + }; + onboarding: { + agent: string; + provider: string; + modelRoute?: string; + policyTier?: string; + messaging?: string[]; + features?: Record; + lifecycle?: string; + gateway?: Record; + }; + state?: { + workspaceRef?: string; + credentialRefs?: string[]; + [key: string]: unknown; + }; + }; +} + +export interface AssertionStepReliability { + timeoutSeconds?: number; + retry?: { + attempts: number; + on: TransientClassifier[]; + }; + productRetry?: string; +} + +export interface AssertionStep { + id: string; + phase: PhaseName; + description?: string; + implementation?: { + kind: "shell" | "probe" | "pending"; + ref: string; + }; + evidencePath?: string; + reliability?: AssertionStepReliability; +} + +export interface AssertionGroup { + id: string; + phase: PhaseName; + description?: string; + suiteId?: string; + onboardingAssertionId?: string; + migrationStatus?: "complete" | "pending"; + steps: AssertionStep[]; +} + +export interface ScenarioEnvironment { + platform: string; + install: string; + runtime: string; + onboarding: string; +} + +export interface ScenarioDefinition { + id: string; + description?: string; + manifestPath?: string; + environment?: ScenarioEnvironment; + assertionGroups: AssertionGroup[]; + expectedStateId?: string; + suiteIds?: string[]; + onboardingAssertionIds?: string[]; + runnerRequirements?: string[]; + requiredSecrets?: string[]; + skippedCapabilities?: Array>; + expectedFailure?: Record; +} + +export interface RunPlanPhase { + name: PhaseName; + actions: string[]; + assertionGroups: AssertionGroup[]; +} + +export interface RunPlan { + scenarioId: string; + status: "skeleton" | "compiled"; + note?: string; + manifestPath?: string; + manifest?: NemoClawInstanceManifest; + environment?: ScenarioEnvironment; + expectedStateId?: string; + suiteIds: string[]; + onboardingAssertionIds: string[]; + phases: RunPlanPhase[]; + runnerRequirements: string[]; + requiredSecrets: string[]; + skippedCapabilities: Array>; + expectedFailure?: Record; + sutBoundaries: SutBoundary[]; +} + +export interface RunContext { + contextDir: string; + dryRun: boolean; +} + +export interface AssertionResult { + id: string; + status: "passed" | "failed" | "skipped"; + attempts: number; + durationMs: number; + classifier?: TransientClassifier; + evidence?: string; + message?: string; +} + +export interface PhaseResult { + phase: PhaseName; + status: "passed" | "failed" | "skipped"; + assertions: AssertionResult[]; +} diff --git a/test/e2e/validation_suites/assert/gateway-alive.sh b/test/e2e-scenario/validation_suites/assert/gateway-alive.sh similarity index 100% rename from test/e2e/validation_suites/assert/gateway-alive.sh rename to test/e2e-scenario/validation_suites/assert/gateway-alive.sh diff --git a/test/e2e/validation_suites/assert/inference-works.sh b/test/e2e-scenario/validation_suites/assert/inference-works.sh similarity index 100% rename from test/e2e/validation_suites/assert/inference-works.sh rename to test/e2e-scenario/validation_suites/assert/inference-works.sh diff --git a/test/e2e/validation_suites/assert/messaging-bridge-reachable.sh b/test/e2e-scenario/validation_suites/assert/messaging-bridge-reachable.sh similarity index 100% rename from test/e2e/validation_suites/assert/messaging-bridge-reachable.sh rename to test/e2e-scenario/validation_suites/assert/messaging-bridge-reachable.sh diff --git a/test/e2e/validation_suites/assert/no-credentials-leaked.sh b/test/e2e-scenario/validation_suites/assert/no-credentials-leaked.sh similarity index 100% rename from test/e2e/validation_suites/assert/no-credentials-leaked.sh rename to test/e2e-scenario/validation_suites/assert/no-credentials-leaked.sh diff --git a/test/e2e/validation_suites/assert/policy-preset-applied.sh b/test/e2e-scenario/validation_suites/assert/policy-preset-applied.sh similarity index 100% rename from test/e2e/validation_suites/assert/policy-preset-applied.sh rename to test/e2e-scenario/validation_suites/assert/policy-preset-applied.sh diff --git a/test/e2e/validation_suites/assert/sandbox-alive.sh b/test/e2e-scenario/validation_suites/assert/sandbox-alive.sh similarity index 100% rename from test/e2e/validation_suites/assert/sandbox-alive.sh rename to test/e2e-scenario/validation_suites/assert/sandbox-alive.sh diff --git a/test/e2e/validation_suites/baseline-onboarding/00-cli-and-openshell.sh b/test/e2e-scenario/validation_suites/baseline-onboarding/00-cli-and-openshell.sh similarity index 100% rename from test/e2e/validation_suites/baseline-onboarding/00-cli-and-openshell.sh rename to test/e2e-scenario/validation_suites/baseline-onboarding/00-cli-and-openshell.sh diff --git a/test/e2e/validation_suites/baseline-onboarding/01-sandbox-state.sh b/test/e2e-scenario/validation_suites/baseline-onboarding/01-sandbox-state.sh similarity index 100% rename from test/e2e/validation_suites/baseline-onboarding/01-sandbox-state.sh rename to test/e2e-scenario/validation_suites/baseline-onboarding/01-sandbox-state.sh diff --git a/test/e2e/validation_suites/baseline-onboarding/02-route-and-smoke.sh b/test/e2e-scenario/validation_suites/baseline-onboarding/02-route-and-smoke.sh similarity index 100% rename from test/e2e/validation_suites/baseline-onboarding/02-route-and-smoke.sh rename to test/e2e-scenario/validation_suites/baseline-onboarding/02-route-and-smoke.sh diff --git a/test/e2e/validation_suites/hermes/00-hermes-health.sh b/test/e2e-scenario/validation_suites/hermes/00-hermes-health.sh similarity index 100% rename from test/e2e/validation_suites/hermes/00-hermes-health.sh rename to test/e2e-scenario/validation_suites/hermes/00-hermes-health.sh diff --git a/test/e2e/validation_suites/inference/cloud/00-models-health.sh b/test/e2e-scenario/validation_suites/inference/cloud/00-models-health.sh similarity index 100% rename from test/e2e/validation_suites/inference/cloud/00-models-health.sh rename to test/e2e-scenario/validation_suites/inference/cloud/00-models-health.sh diff --git a/test/e2e/validation_suites/inference/cloud/01-chat-completion.sh b/test/e2e-scenario/validation_suites/inference/cloud/01-chat-completion.sh similarity index 100% rename from test/e2e/validation_suites/inference/cloud/01-chat-completion.sh rename to test/e2e-scenario/validation_suites/inference/cloud/01-chat-completion.sh diff --git a/test/e2e/validation_suites/inference/cloud/02-inference-local-from-sandbox.sh b/test/e2e-scenario/validation_suites/inference/cloud/02-inference-local-from-sandbox.sh similarity index 100% rename from test/e2e/validation_suites/inference/cloud/02-inference-local-from-sandbox.sh rename to test/e2e-scenario/validation_suites/inference/cloud/02-inference-local-from-sandbox.sh diff --git a/test/e2e/validation_suites/inference/kimi-compatibility/00-plugin-wiring.sh b/test/e2e-scenario/validation_suites/inference/kimi-compatibility/00-plugin-wiring.sh similarity index 100% rename from test/e2e/validation_suites/inference/kimi-compatibility/00-plugin-wiring.sh rename to test/e2e-scenario/validation_suites/inference/kimi-compatibility/00-plugin-wiring.sh diff --git a/test/e2e/validation_suites/inference/kimi-compatibility/01-kimi-compatible-models-route.sh b/test/e2e-scenario/validation_suites/inference/kimi-compatibility/01-kimi-compatible-models-route.sh similarity index 100% rename from test/e2e/validation_suites/inference/kimi-compatibility/01-kimi-compatible-models-route.sh rename to test/e2e-scenario/validation_suites/inference/kimi-compatibility/01-kimi-compatible-models-route.sh diff --git a/test/e2e/validation_suites/inference/model-router/00-healthy-endpoint.sh b/test/e2e-scenario/validation_suites/inference/model-router/00-healthy-endpoint.sh similarity index 100% rename from test/e2e/validation_suites/inference/model-router/00-healthy-endpoint.sh rename to test/e2e-scenario/validation_suites/inference/model-router/00-healthy-endpoint.sh diff --git a/test/e2e/validation_suites/inference/model-router/01-provider-routed-completion.sh b/test/e2e-scenario/validation_suites/inference/model-router/01-provider-routed-completion.sh similarity index 100% rename from test/e2e/validation_suites/inference/model-router/01-provider-routed-completion.sh rename to test/e2e-scenario/validation_suites/inference/model-router/01-provider-routed-completion.sh diff --git a/test/e2e/validation_suites/inference/ollama-auth-proxy/00-proxy-reachable.sh b/test/e2e-scenario/validation_suites/inference/ollama-auth-proxy/00-proxy-reachable.sh similarity index 100% rename from test/e2e/validation_suites/inference/ollama-auth-proxy/00-proxy-reachable.sh rename to test/e2e-scenario/validation_suites/inference/ollama-auth-proxy/00-proxy-reachable.sh diff --git a/test/e2e/validation_suites/inference/ollama-auth-proxy/01-auth-enforcement.sh b/test/e2e-scenario/validation_suites/inference/ollama-auth-proxy/01-auth-enforcement.sh similarity index 100% rename from test/e2e/validation_suites/inference/ollama-auth-proxy/01-auth-enforcement.sh rename to test/e2e-scenario/validation_suites/inference/ollama-auth-proxy/01-auth-enforcement.sh diff --git a/test/e2e/validation_suites/inference/ollama-gpu/00-ollama-models-health.sh b/test/e2e-scenario/validation_suites/inference/ollama-gpu/00-ollama-models-health.sh similarity index 100% rename from test/e2e/validation_suites/inference/ollama-gpu/00-ollama-models-health.sh rename to test/e2e-scenario/validation_suites/inference/ollama-gpu/00-ollama-models-health.sh diff --git a/test/e2e/validation_suites/inference/ollama-gpu/01-ollama-chat-completion.sh b/test/e2e-scenario/validation_suites/inference/ollama-gpu/01-ollama-chat-completion.sh similarity index 100% rename from test/e2e/validation_suites/inference/ollama-gpu/01-ollama-chat-completion.sh rename to test/e2e-scenario/validation_suites/inference/ollama-gpu/01-ollama-chat-completion.sh diff --git a/test/e2e/validation_suites/inference/routing/00-inference-local-chat-completion.sh b/test/e2e-scenario/validation_suites/inference/routing/00-inference-local-chat-completion.sh similarity index 100% rename from test/e2e/validation_suites/inference/routing/00-inference-local-chat-completion.sh rename to test/e2e-scenario/validation_suites/inference/routing/00-inference-local-chat-completion.sh diff --git a/test/e2e/validation_suites/inference/routing/01-provider-route-health.sh b/test/e2e-scenario/validation_suites/inference/routing/01-provider-route-health.sh similarity index 100% rename from test/e2e/validation_suites/inference/routing/01-provider-route-health.sh rename to test/e2e-scenario/validation_suites/inference/routing/01-provider-route-health.sh diff --git a/test/e2e/validation_suites/inference/switch/00-route-state-updated.sh b/test/e2e-scenario/validation_suites/inference/switch/00-route-state-updated.sh similarity index 100% rename from test/e2e/validation_suites/inference/switch/00-route-state-updated.sh rename to test/e2e-scenario/validation_suites/inference/switch/00-route-state-updated.sh diff --git a/test/e2e/validation_suites/inference/switch/01-switched-inference-local-chat.sh b/test/e2e-scenario/validation_suites/inference/switch/01-switched-inference-local-chat.sh similarity index 100% rename from test/e2e/validation_suites/inference/switch/01-switched-inference-local-chat.sh rename to test/e2e-scenario/validation_suites/inference/switch/01-switched-inference-local-chat.sh diff --git a/test/e2e/validation_suites/lib/baseline_onboarding.sh b/test/e2e-scenario/validation_suites/lib/baseline_onboarding.sh similarity index 100% rename from test/e2e/validation_suites/lib/baseline_onboarding.sh rename to test/e2e-scenario/validation_suites/lib/baseline_onboarding.sh diff --git a/test/e2e/validation_suites/lib/inference_routing.sh b/test/e2e-scenario/validation_suites/lib/inference_routing.sh similarity index 100% rename from test/e2e/validation_suites/lib/inference_routing.sh rename to test/e2e-scenario/validation_suites/lib/inference_routing.sh diff --git a/test/e2e/validation_suites/lib/messaging_providers.sh b/test/e2e-scenario/validation_suites/lib/messaging_providers.sh similarity index 96% rename from test/e2e/validation_suites/lib/messaging_providers.sh rename to test/e2e-scenario/validation_suites/lib/messaging_providers.sh index 8843dc69dc..03c85ae6c2 100755 --- a/test/e2e/validation_suites/lib/messaging_providers.sh +++ b/test/e2e-scenario/validation_suites/lib/messaging_providers.sh @@ -13,9 +13,9 @@ _E2E_MESSAGING_PROVIDERS_SH_LOADED=1 _e2e_messaging_lib_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" _e2e_messaging_repo_root="$(cd "${_e2e_messaging_lib_dir}/../../../.." && pwd)" # shellcheck source=../../runtime/lib/context.sh -. "${_e2e_messaging_repo_root}/test/e2e/runtime/lib/context.sh" +. "${_e2e_messaging_repo_root}/test/e2e-scenario/runtime/lib/context.sh" # shellcheck source=../../runtime/lib/logging.sh -. "${_e2e_messaging_repo_root}/test/e2e/runtime/lib/logging.sh" +. "${_e2e_messaging_repo_root}/test/e2e-scenario/runtime/lib/logging.sh" # Load normalized scenario context and validate the minimum keys used by # messaging suite primitives. Sourcing this file alone intentionally does not @@ -188,6 +188,6 @@ e2e_messaging_assert_bridge_reachable() { fi export MESSAGING_BRIDGE_URL="${url}" # shellcheck source=../assert/messaging-bridge-reachable.sh - . "${_e2e_messaging_repo_root}/test/e2e/validation_suites/assert/messaging-bridge-reachable.sh" + . "${_e2e_messaging_repo_root}/test/e2e-scenario/validation_suites/assert/messaging-bridge-reachable.sh" e2e_assert_messaging_bridge_reachable "${provider}" } diff --git a/test/e2e/validation_suites/lib/rebuild_upgrade.sh b/test/e2e-scenario/validation_suites/lib/rebuild_upgrade.sh similarity index 97% rename from test/e2e/validation_suites/lib/rebuild_upgrade.sh rename to test/e2e-scenario/validation_suites/lib/rebuild_upgrade.sh index 96b82917ba..c6483c99fb 100755 --- a/test/e2e/validation_suites/lib/rebuild_upgrade.sh +++ b/test/e2e-scenario/validation_suites/lib/rebuild_upgrade.sh @@ -7,9 +7,9 @@ _REBUILD_UPGRADE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" _REBUILD_UPGRADE_REPO_ROOT="$(cd "${_REBUILD_UPGRADE_DIR}/../../../.." && pwd)" # shellcheck source=../../runtime/lib/context.sh -. "${_REBUILD_UPGRADE_REPO_ROOT}/test/e2e/runtime/lib/context.sh" +. "${_REBUILD_UPGRADE_REPO_ROOT}/test/e2e-scenario/runtime/lib/context.sh" # shellcheck source=../../runtime/lib/logging.sh -. "${_REBUILD_UPGRADE_REPO_ROOT}/test/e2e/runtime/lib/logging.sh" +. "${_REBUILD_UPGRADE_REPO_ROOT}/test/e2e-scenario/runtime/lib/logging.sh" rebuild_upgrade_require_context() { e2e_context_require E2E_SCENARIO E2E_AGENT E2E_SANDBOX_NAME E2E_GATEWAY_URL diff --git a/test/e2e/validation_suites/lib/sandbox_lifecycle.sh b/test/e2e-scenario/validation_suites/lib/sandbox_lifecycle.sh similarity index 100% rename from test/e2e/validation_suites/lib/sandbox_lifecycle.sh rename to test/e2e-scenario/validation_suites/lib/sandbox_lifecycle.sh diff --git a/test/e2e/validation_suites/lib/security_policy_credentials.sh b/test/e2e-scenario/validation_suites/lib/security_policy_credentials.sh similarity index 100% rename from test/e2e/validation_suites/lib/security_policy_credentials.sh rename to test/e2e-scenario/validation_suites/lib/security_policy_credentials.sh diff --git a/test/e2e/validation_suites/messaging/common/00-provider-attached.sh b/test/e2e-scenario/validation_suites/messaging/common/00-provider-attached.sh similarity index 100% rename from test/e2e/validation_suites/messaging/common/00-provider-attached.sh rename to test/e2e-scenario/validation_suites/messaging/common/00-provider-attached.sh diff --git a/test/e2e/validation_suites/messaging/common/01-placeholder-configured.sh b/test/e2e-scenario/validation_suites/messaging/common/01-placeholder-configured.sh similarity index 100% rename from test/e2e/validation_suites/messaging/common/01-placeholder-configured.sh rename to test/e2e-scenario/validation_suites/messaging/common/01-placeholder-configured.sh diff --git a/test/e2e/validation_suites/messaging/common/02-no-secret-leak.sh b/test/e2e-scenario/validation_suites/messaging/common/02-no-secret-leak.sh similarity index 100% rename from test/e2e/validation_suites/messaging/common/02-no-secret-leak.sh rename to test/e2e-scenario/validation_suites/messaging/common/02-no-secret-leak.sh diff --git a/test/e2e/validation_suites/messaging/common/03-bridge-reachable.sh b/test/e2e-scenario/validation_suites/messaging/common/03-bridge-reachable.sh similarity index 100% rename from test/e2e/validation_suites/messaging/common/03-bridge-reachable.sh rename to test/e2e-scenario/validation_suites/messaging/common/03-bridge-reachable.sh diff --git a/test/e2e/validation_suites/messaging/discord/00-discord-gateway-path.sh b/test/e2e-scenario/validation_suites/messaging/discord/00-discord-gateway-path.sh similarity index 100% rename from test/e2e/validation_suites/messaging/discord/00-discord-gateway-path.sh rename to test/e2e-scenario/validation_suites/messaging/discord/00-discord-gateway-path.sh diff --git a/test/e2e/validation_suites/messaging/slack/00-slack-provider-state.sh b/test/e2e-scenario/validation_suites/messaging/slack/00-slack-provider-state.sh similarity index 100% rename from test/e2e/validation_suites/messaging/slack/00-slack-provider-state.sh rename to test/e2e-scenario/validation_suites/messaging/slack/00-slack-provider-state.sh diff --git a/test/e2e/validation_suites/messaging/telegram/00-telegram-injection-safety.sh b/test/e2e-scenario/validation_suites/messaging/telegram/00-telegram-injection-safety.sh similarity index 100% rename from test/e2e/validation_suites/messaging/telegram/00-telegram-injection-safety.sh rename to test/e2e-scenario/validation_suites/messaging/telegram/00-telegram-injection-safety.sh diff --git a/test/e2e/validation_suites/messaging/telegram/01-telegram-injection-payload-classes.sh b/test/e2e-scenario/validation_suites/messaging/telegram/01-telegram-injection-payload-classes.sh similarity index 100% rename from test/e2e/validation_suites/messaging/telegram/01-telegram-injection-payload-classes.sh rename to test/e2e-scenario/validation_suites/messaging/telegram/01-telegram-injection-payload-classes.sh diff --git a/test/e2e/validation_suites/messaging/token-rotation/00-provider-rotation-isolated.sh b/test/e2e-scenario/validation_suites/messaging/token-rotation/00-provider-rotation-isolated.sh similarity index 100% rename from test/e2e/validation_suites/messaging/token-rotation/00-provider-rotation-isolated.sh rename to test/e2e-scenario/validation_suites/messaging/token-rotation/00-provider-rotation-isolated.sh diff --git a/test/e2e/validation_suites/onboarding/state/00-registry-provider-model-policies.sh b/test/e2e-scenario/validation_suites/onboarding/state/00-registry-provider-model-policies.sh similarity index 100% rename from test/e2e/validation_suites/onboarding/state/00-registry-provider-model-policies.sh rename to test/e2e-scenario/validation_suites/onboarding/state/00-registry-provider-model-policies.sh diff --git a/test/e2e/validation_suites/onboarding/state/01-session-provider-model-policies.sh b/test/e2e-scenario/validation_suites/onboarding/state/01-session-provider-model-policies.sh similarity index 100% rename from test/e2e/validation_suites/onboarding/state/01-session-provider-model-policies.sh rename to test/e2e-scenario/validation_suites/onboarding/state/01-session-provider-model-policies.sh diff --git a/test/e2e/validation_suites/platform/macos/00-macos-smoke.sh b/test/e2e-scenario/validation_suites/platform/macos/00-macos-smoke.sh similarity index 100% rename from test/e2e/validation_suites/platform/macos/00-macos-smoke.sh rename to test/e2e-scenario/validation_suites/platform/macos/00-macos-smoke.sh diff --git a/test/e2e/validation_suites/platform/wsl/00-wsl-smoke.sh b/test/e2e-scenario/validation_suites/platform/wsl/00-wsl-smoke.sh similarity index 100% rename from test/e2e/validation_suites/platform/wsl/00-wsl-smoke.sh rename to test/e2e-scenario/validation_suites/platform/wsl/00-wsl-smoke.sh diff --git a/test/e2e/validation_suites/rebuild_upgrade/00-state-preserved.sh b/test/e2e-scenario/validation_suites/rebuild_upgrade/00-state-preserved.sh similarity index 100% rename from test/e2e/validation_suites/rebuild_upgrade/00-state-preserved.sh rename to test/e2e-scenario/validation_suites/rebuild_upgrade/00-state-preserved.sh diff --git a/test/e2e/validation_suites/rebuild_upgrade/01-agent-version-upgraded.sh b/test/e2e-scenario/validation_suites/rebuild_upgrade/01-agent-version-upgraded.sh similarity index 100% rename from test/e2e/validation_suites/rebuild_upgrade/01-agent-version-upgraded.sh rename to test/e2e-scenario/validation_suites/rebuild_upgrade/01-agent-version-upgraded.sh diff --git a/test/e2e/validation_suites/rebuild_upgrade/02-post-rebuild-inference.sh b/test/e2e-scenario/validation_suites/rebuild_upgrade/02-post-rebuild-inference.sh similarity index 100% rename from test/e2e/validation_suites/rebuild_upgrade/02-post-rebuild-inference.sh rename to test/e2e-scenario/validation_suites/rebuild_upgrade/02-post-rebuild-inference.sh diff --git a/test/e2e/validation_suites/rebuild_upgrade/03-policy-config-preserved.sh b/test/e2e-scenario/validation_suites/rebuild_upgrade/03-policy-config-preserved.sh similarity index 100% rename from test/e2e/validation_suites/rebuild_upgrade/03-policy-config-preserved.sh rename to test/e2e-scenario/validation_suites/rebuild_upgrade/03-policy-config-preserved.sh diff --git a/test/e2e/validation_suites/rebuild_upgrade/04-upgrade-survivor-reachable.sh b/test/e2e-scenario/validation_suites/rebuild_upgrade/04-upgrade-survivor-reachable.sh similarity index 100% rename from test/e2e/validation_suites/rebuild_upgrade/04-upgrade-survivor-reachable.sh rename to test/e2e-scenario/validation_suites/rebuild_upgrade/04-upgrade-survivor-reachable.sh diff --git a/test/e2e/validation_suites/sandbox-exec.sh b/test/e2e-scenario/validation_suites/sandbox-exec.sh similarity index 100% rename from test/e2e/validation_suites/sandbox-exec.sh rename to test/e2e-scenario/validation_suites/sandbox-exec.sh diff --git a/test/e2e/validation_suites/sandbox/lifecycle/00-gateway-health.sh b/test/e2e-scenario/validation_suites/sandbox/lifecycle/00-gateway-health.sh similarity index 100% rename from test/e2e/validation_suites/sandbox/lifecycle/00-gateway-health.sh rename to test/e2e-scenario/validation_suites/sandbox/lifecycle/00-gateway-health.sh diff --git a/test/e2e/validation_suites/sandbox/lifecycle/01-gateway-recovery.sh b/test/e2e-scenario/validation_suites/sandbox/lifecycle/01-gateway-recovery.sh similarity index 100% rename from test/e2e/validation_suites/sandbox/lifecycle/01-gateway-recovery.sh rename to test/e2e-scenario/validation_suites/sandbox/lifecycle/01-gateway-recovery.sh diff --git a/test/e2e/validation_suites/sandbox/operations/00-list-and-status.sh b/test/e2e-scenario/validation_suites/sandbox/operations/00-list-and-status.sh similarity index 100% rename from test/e2e/validation_suites/sandbox/operations/00-list-and-status.sh rename to test/e2e-scenario/validation_suites/sandbox/operations/00-list-and-status.sh diff --git a/test/e2e/validation_suites/sandbox/operations/01-logs-and-exec.sh b/test/e2e-scenario/validation_suites/sandbox/operations/01-logs-and-exec.sh similarity index 100% rename from test/e2e/validation_suites/sandbox/operations/01-logs-and-exec.sh rename to test/e2e-scenario/validation_suites/sandbox/operations/01-logs-and-exec.sh diff --git a/test/e2e/validation_suites/sandbox/snapshot/00-create-list-restore.sh b/test/e2e-scenario/validation_suites/sandbox/snapshot/00-create-list-restore.sh similarity index 100% rename from test/e2e/validation_suites/sandbox/snapshot/00-create-list-restore.sh rename to test/e2e-scenario/validation_suites/sandbox/snapshot/00-create-list-restore.sh diff --git a/test/e2e/validation_suites/security/credentials/00-credentials-present.sh b/test/e2e-scenario/validation_suites/security/credentials/00-credentials-present.sh similarity index 100% rename from test/e2e/validation_suites/security/credentials/00-credentials-present.sh rename to test/e2e-scenario/validation_suites/security/credentials/00-credentials-present.sh diff --git a/test/e2e/validation_suites/security/credentials/01-no-plaintext-host-store.sh b/test/e2e-scenario/validation_suites/security/credentials/01-no-plaintext-host-store.sh similarity index 100% rename from test/e2e/validation_suites/security/credentials/01-no-plaintext-host-store.sh rename to test/e2e-scenario/validation_suites/security/credentials/01-no-plaintext-host-store.sh diff --git a/test/e2e/validation_suites/security/injection/00-telegram-message-not-shell-executed.sh b/test/e2e-scenario/validation_suites/security/injection/00-telegram-message-not-shell-executed.sh similarity index 100% rename from test/e2e/validation_suites/security/injection/00-telegram-message-not-shell-executed.sh rename to test/e2e-scenario/validation_suites/security/injection/00-telegram-message-not-shell-executed.sh diff --git a/test/e2e/validation_suites/security/policy/00-telegram-preset-applied.sh b/test/e2e-scenario/validation_suites/security/policy/00-telegram-preset-applied.sh similarity index 100% rename from test/e2e/validation_suites/security/policy/00-telegram-preset-applied.sh rename to test/e2e-scenario/validation_suites/security/policy/00-telegram-preset-applied.sh diff --git a/test/e2e/validation_suites/security/policy/01-openshell-version-supports-credential-rewrite.sh b/test/e2e-scenario/validation_suites/security/policy/01-openshell-version-supports-credential-rewrite.sh similarity index 100% rename from test/e2e/validation_suites/security/policy/01-openshell-version-supports-credential-rewrite.sh rename to test/e2e-scenario/validation_suites/security/policy/01-openshell-version-supports-credential-rewrite.sh diff --git a/test/e2e/validation_suites/security/shields/00-config-consistent.sh b/test/e2e-scenario/validation_suites/security/shields/00-config-consistent.sh similarity index 100% rename from test/e2e/validation_suites/security/shields/00-config-consistent.sh rename to test/e2e-scenario/validation_suites/security/shields/00-config-consistent.sh diff --git a/test/e2e/validation_suites/smoke/00-cli-available.sh b/test/e2e-scenario/validation_suites/smoke/00-cli-available.sh similarity index 100% rename from test/e2e/validation_suites/smoke/00-cli-available.sh rename to test/e2e-scenario/validation_suites/smoke/00-cli-available.sh diff --git a/test/e2e/validation_suites/smoke/01-gateway-health.sh b/test/e2e-scenario/validation_suites/smoke/01-gateway-health.sh similarity index 100% rename from test/e2e/validation_suites/smoke/01-gateway-health.sh rename to test/e2e-scenario/validation_suites/smoke/01-gateway-health.sh diff --git a/test/e2e/validation_suites/smoke/02-sandbox-listed.sh b/test/e2e-scenario/validation_suites/smoke/02-sandbox-listed.sh similarity index 100% rename from test/e2e/validation_suites/smoke/02-sandbox-listed.sh rename to test/e2e-scenario/validation_suites/smoke/02-sandbox-listed.sh diff --git a/test/e2e/validation_suites/smoke/03-sandbox-shell.sh b/test/e2e-scenario/validation_suites/smoke/03-sandbox-shell.sh similarity index 100% rename from test/e2e/validation_suites/smoke/03-sandbox-shell.sh rename to test/e2e-scenario/validation_suites/smoke/03-sandbox-shell.sh diff --git a/test/e2e/validation_suites/suites.yaml b/test/e2e-scenario/validation_suites/suites.yaml similarity index 100% rename from test/e2e/validation_suites/suites.yaml rename to test/e2e-scenario/validation_suites/suites.yaml diff --git a/tools/e2e-advisor/scenarios.mts b/tools/e2e-advisor/scenarios.mts index d1f9bbefa7..7c87907363 100644 --- a/tools/e2e-advisor/scenarios.mts +++ b/tools/e2e-advisor/scenarios.mts @@ -7,6 +7,7 @@ import { pathToFileURL } from "node:url"; import { getChangedFiles } from "../advisors/git.mts"; import { parseArgs, writeJson } from "../advisors/io.mts"; +import { listScenarios } from "../../test/e2e-scenario/scenarios/registry.ts"; const SCENARIO_WORKFLOW = "e2e-scenarios.yaml"; const SCENARIO_ALL_WORKFLOW = "e2e-scenarios-all.yaml"; @@ -112,24 +113,24 @@ export function analyzeScenarioRecommendations({ } else if (file === ".github/workflows/e2e-scenarios.yaml") { allScenariosRequired = true; reasons.add("the reusable single-scenario workflow changed"); - } else if (file === "test/e2e/nemoclaw_scenarios/scenarios.yaml") { + } else if (file === "test/e2e-scenario/nemoclaw_scenarios/scenarios.yaml") { allScenariosRequired = true; reasons.add("scenario catalog metadata changed"); - } else if (file === "test/e2e/nemoclaw_scenarios/expected-states.yaml") { + } else if (file === "test/e2e-scenario/nemoclaw_scenarios/expected-states.yaml") { allScenariosRequired = true; reasons.add("expected-state metadata changed"); - } else if (file === "test/e2e/validation_suites/suites.yaml") { + } else if (file === "test/e2e-scenario/validation_suites/suites.yaml") { allScenariosRequired = true; reasons.add("suite catalog metadata changed"); } else if ( - file.startsWith("test/e2e/runtime/") || - file.startsWith("test/e2e/nemoclaw_scenarios/helpers/") + file.startsWith("test/e2e-scenario/runtime/") || + file.startsWith("test/e2e-scenario/nemoclaw_scenarios/helpers/") ) { allScenariosRequired = true; reasons.add("shared scenario runner/runtime code changed"); } else if ( - file.startsWith("test/e2e/nemoclaw_scenarios/onboard/") || - file.startsWith("test/e2e/nemoclaw_scenarios/install/") + file.startsWith("test/e2e-scenario/nemoclaw_scenarios/onboard/") || + file.startsWith("test/e2e-scenario/nemoclaw_scenarios/install/") ) { directScenarioIds.add(DEFAULT_BASELINE_SCENARIO); reasons.add("scenario install/onboard helper code changed"); @@ -278,21 +279,20 @@ export function renderScenarioSummary(result: ScenarioAdvisorResult): string { return `${lines.join("\n")}\n`; } -function loadScenarios(root: string): Record { - const filePath = path.join( - root, - "test/e2e/nemoclaw_scenarios/scenarios.yaml", +function loadScenarios(_root: string): Record { + return Object.fromEntries( + listScenarios().map((scenario) => [ + scenario.id, + { + suites: scenario.suiteIds ?? [], + runner_requirements: scenario.runnerRequirements ?? [], + }, + ]), ); - if (!fs.existsSync(filePath)) return {}; - const text = fs.readFileSync(filePath, "utf8"); - return { - ...parseScenarioSection(text, "test_plans"), - ...parseScenarioSection(text, "setup_scenarios"), - }; } function loadSuiteScriptMap(root: string): Record { - const filePath = path.join(root, "test/e2e/validation_suites/suites.yaml"); + const filePath = path.join(root, "test/e2e-scenario/validation_suites/suites.yaml"); if (!fs.existsSync(filePath)) return {}; return parseSuiteScripts(fs.readFileSync(filePath, "utf8")); } @@ -413,9 +413,9 @@ function isScenarioRelevantFile(file: string): boolean { return ( file === ".github/workflows/e2e-scenarios.yaml" || file === ".github/workflows/e2e-scenarios-all.yaml" || - file.startsWith("test/e2e/runtime/") || - file.startsWith("test/e2e/nemoclaw_scenarios/") || - file.startsWith("test/e2e/validation_suites/") + file.startsWith("test/e2e-scenario/runtime/") || + file.startsWith("test/e2e-scenario/nemoclaw_scenarios/") || + file.startsWith("test/e2e-scenario/validation_suites/") ); } @@ -425,11 +425,11 @@ function inferSuiteIdsFromPath( suiteScriptMap: Record, ): string[] { if ( - !file.startsWith("test/e2e/validation_suites/") || + !file.startsWith("test/e2e-scenario/validation_suites/") || file.endsWith("/suites.yaml") ) return []; - const relative = file.slice("test/e2e/validation_suites/".length); + const relative = file.slice("test/e2e-scenario/validation_suites/".length); const segments = relative.split("/"); const candidates = new Set(); for (let size = Math.min(segments.length, 3); size >= 1; size -= 1) { diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 04b13bcd2a..3eba39a9c3 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -65,8 +65,13 @@ export function validateE2eScenariosWorkflowBoundary( } const dispatchInputs = asRecord(workflowDispatch.inputs); - requireInput(errors, dispatchInputs, "scenario"); - requireInput(errors, dispatchInputs, "suite_filter"); + requireInput(errors, dispatchInputs, "scenarios"); + if (Object.hasOwn(dispatchInputs, "scenario")) { + errors.push("workflow_dispatch must not expose legacy scenario input"); + } + if (Object.hasOwn(dispatchInputs, "suite_filter")) { + errors.push("workflow_dispatch must not expose legacy suite_filter input"); + } if (Object.hasOwn(dispatchInputs, "plan_only")) { errors.push("workflow_dispatch must not expose retired plan_only input"); } @@ -84,22 +89,20 @@ export function validateE2eScenariosWorkflowBoundary( } const steps = asSteps(runScenario.steps); - const normalRun = requireStep(errors, steps, "Run scenario"); - requireRunContains(errors, normalRun, "bash test/e2e/runtime/run-scenario.sh"); - requireRunContains(errors, normalRun, '"$SCENARIO"'); - requireRunContains(errors, normalRun, "exit \"$rc\""); - if (stringValue(normalRun?.run).includes("--plan-only")) { - errors.push("Run scenario step must not use retired --plan-only flag"); - } + const normalRun = requireStep(errors, steps, "Run typed scenarios"); + requireRunContains(errors, normalRun, "npx tsx test/e2e-scenario/scenarios/run.ts"); + requireRunContains(errors, normalRun, "--scenarios"); + requireRunContains(errors, normalRun, "--dry-run"); - const wslRun = requireStep(errors, steps, "Run scenario in WSL"); - requireRunContains(errors, wslRun, "bash test/e2e/runtime/run-scenario.sh"); - requireRunContains(errors, wslRun, '"$SCENARIO"'); + const wslRun = requireStep(errors, steps, "Run typed scenarios in WSL"); + requireRunContains(errors, wslRun, "npx tsx test/e2e-scenario/scenarios/run.ts"); + requireRunContains(errors, wslRun, "--scenarios"); + requireRunContains(errors, wslRun, "--dry-run"); const upload = requireStep(errors, steps, "Upload scenario artifacts"); const uploadWith = asRecord(upload?.with); - if (uploadWith.name !== "e2e-scenario-${{ inputs.scenario }}") { - errors.push("artifact upload name must include the scenario input"); + if (uploadWith.name !== "e2e-scenario-${{ inputs.scenarios || github.event.inputs.scenarios }}") { + errors.push("artifact upload name must include the scenarios input"); } if (uploadWith["include-hidden-files"] !== true) { errors.push("artifact upload must include hidden .e2e files"); diff --git a/vitest.config.ts b/vitest.config.ts index 8a155a28f0..0f40c0b542 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -59,7 +59,7 @@ export default defineConfig({ test: { name: "e2e-scenario-framework", testTimeout: testTimeout(), - include: ["test/e2e/scenario-framework-tests/**/*.test.ts"], + include: ["test/e2e-scenario/framework-tests/**/*.test.ts"], }, }, {