Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
38da01a
Simplify inference routing coverage spec
jyaunches May 20, 2026
99d7fbe
Add test specification for inference routing coverage
jyaunches May 20, 2026
4c3b8a6
Add validation plan for inference routing coverage
jyaunches May 20, 2026
383a732
Approve validation plan for 2026-05-20_inference-routing-provider-cov…
jyaunches May 20, 2026
120671f
Apply design review for inference routing coverage spec
jyaunches May 20, 2026
6d66658
Apply implementation review for inference routing coverage spec
jyaunches May 20, 2026
e3002b4
test: Add failing tests for Phase 1
jyaunches May 20, 2026
f011049
Mark Phase 1 as completed [e3002b46f]
jyaunches May 20, 2026
d8b955a
test: Add failing tests for Phase 2
jyaunches May 20, 2026
6c96e7d
feat: Implement Phase 2 - inference routing primitives
jyaunches May 20, 2026
9663659
Mark Phase 2 as completed [6c96e7d3a]
jyaunches May 20, 2026
9cb5678
test: Add failing tests for Phase 3
jyaunches May 20, 2026
4b03eef
feat: Implement Phase 3 - domain inference suites
jyaunches May 20, 2026
a9c17cc
Mark Phase 3 as completed [4b03eefc7]
jyaunches May 20, 2026
b252a09
test: Add failing tests for Phase 4
jyaunches May 20, 2026
fc960f1
feat: Implement Phase 4 - parity coverage summary
jyaunches May 20, 2026
8b0b763
Mark Phase 4 as completed [fc960f166]
jyaunches May 20, 2026
a6d4d39
Mark Phase 5 as completed [fc960f166]
jyaunches May 20, 2026
5f08f8f
test(e2e): mark inference routing validation results
jyaunches May 20, 2026
1581b64
chore(spec): normalize validation markdown
jyaunches May 20, 2026
1e6c959
chore: apply hook formatting
jyaunches May 20, 2026
78d3a50
Merge remote-tracking branch 'origin/main' into issue-3812-migrate-in…
jyaunches May 20, 2026
356bca5
Merge remote-tracking branch 'origin/main' into issue-3812-migrate-in…
jyaunches May 20, 2026
59366c8
fix(ci): remove ignored specs from PR
jyaunches May 20, 2026
c35842d
fix(e2e): address inference review feedback
jyaunches May 20, 2026
7d9f8b1
merge: update branch with main
jyaunches May 21, 2026
442fa48
Merge remote-tracking branch 'origin/main' into issue-3812-migrate-in…
jyaunches May 21, 2026
3d4a3b3
Merge remote-tracking branch 'origin/main' into issue-3812-migrate-in…
jyaunches May 21, 2026
dff1ce0
Merge remote-tracking branch 'origin/main' into issue-3812-migrate-in…
jyaunches May 21, 2026
9f7f3af
Merge remote-tracking branch 'origin/main' into issue-3812-migrate-in…
jyaunches May 21, 2026
3ac440e
Merge remote-tracking branch 'origin/main' into issue-3812-migrate-in…
jyaunches May 21, 2026
074eeea
Merge remote-tracking branch 'origin/main' into issue-3812-migrate-in…
jyaunches May 21, 2026
b9d0f7c
Merge remote-tracking branch 'origin/main' into issue-3812-migrate-in…
jyaunches May 21, 2026
91e0950
Merge remote-tracking branch 'origin/main' into issue-3812-migrate-in…
jyaunches May 21, 2026
9bcd4d7
Merge remote-tracking branch 'origin/main' into issue-3812-migrate-in…
jyaunches May 21, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions test/e2e/docs/parity-map.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4540,7 +4540,7 @@ scripts:
test-inference-routing.sh:
scenario: ubuntu-repo-cloud-openclaw
status: migrated
bucket: providers-messaging
bucket: inference-routing-provider
assertions:
- legacy: 'TC-INF-05: Setup'
status: deferred
Expand Down Expand Up @@ -5078,7 +5078,7 @@ scripts:
test-kimi-inference-compat.sh:
scenario: ubuntu-repo-cloud-openclaw
status: migrated
bucket: providers-messaging
bucket: inference-routing-provider
assertions:
- legacy: 'K1: source CLI/OpenShell preparation failed (exit $prep_exit)'
status: deferred
Expand Down Expand Up @@ -6989,7 +6989,7 @@ scripts:
test-ollama-auth-proxy-e2e.sh:
scenario: gpu-repo-local-ollama-openclaw
status: migrated
bucket: providers-messaging
bucket: inference-routing-provider
assertions:
- legacy: Node.js not found
status: deferred
Expand Down Expand Up @@ -7807,7 +7807,7 @@ scripts:
test-openclaw-inference-switch.sh:
scenario: ubuntu-repo-cloud-openclaw
status: migrated
bucket: providers-messaging
bucket: inference-routing-provider
assertions:
- legacy: 'OpenShell inference get failed: ${output:0:240}'
status: deferred
Expand Down Expand Up @@ -11639,7 +11639,7 @@ scripts:
test-model-router-provider-routed-inference.sh:
scenario: ubuntu-repo-cloud-openclaw
status: migrated
bucket: providers-messaging
bucket: inference-routing-provider
assertions:
- legacy: Docker is running
status: deferred
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/runtime/resolver/coverage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ function renderLegacyParitySummary(meta: ResolverInput): string[] {
a.localeCompare(b),
)) {
lines.push(
`| ${bucket} | ${row.scripts.size} | ${row.mapped} | ${row.deferred} | ${row.retired} | ${row.unmapped} |`,
`| ${bucket} | ${[...row.scripts].sort().join(", ")} | ${row.mapped} | ${row.deferred} | ${row.retired} | ${row.unmapped} |`,
);
}
lines.push("");
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/runtime/run-scenario.sh
Original file line number Diff line number Diff line change
Expand Up @@ -464,7 +464,7 @@ if [[ "${DOCKER_OPTIONAL_UNAVAILABLE}" -eq 1 ]]; then
FILTERED_SUITE_IDS=()
for suite_id in "${SUITE_IDS[@]}"; do
case "${suite_id}" in
smoke | inference | credentials | hermes-specific | local-ollama-inference | ollama-proxy | gateway-health | sandbox-shell | cloud-inference | ollama-auth-proxy | security-credentials | messaging-telegram | messaging-discord | messaging-slack | security-shields | inference-routing | sandbox-lifecycle | sandbox-operations | snapshot | rebuild | upgrade | diagnostics | docs-validation | openai-compatible-inference | inference-switch | kimi-compatibility | messaging-token-rotation | security-policy | security-injection)
smoke | inference | credentials | hermes-specific | local-ollama-inference | ollama-proxy | gateway-health | sandbox-shell | cloud-inference | ollama-auth-proxy | security-credentials | messaging-telegram | messaging-discord | messaging-slack | security-shields | inference-routing | sandbox-lifecycle | sandbox-operations | snapshot | rebuild | upgrade | diagnostics | docs-validation | openai-compatible-inference | inference-switch | kimi-compatibility | messaging-token-rotation | security-policy | security-injection | model-router)
echo "SKIP: suite.${suite_id} skipped because optional Docker runtime ${RUNTIME_ID} is unavailable"
;;
*)
Expand Down
11 changes: 11 additions & 0 deletions test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,17 @@ describe("coverage report", () => {
}
});

it("test_should_report_issue_3812_domain_coverage_summary", () => {
const meta = loadMetadataFromDir(E2E_DIR);
const md = renderCoverageReport(meta);

expect(md).toMatch(/inference-routing-provider/);
expect(md).toMatch(/test-inference-routing\.sh/);
expect(md).toMatch(/test-openclaw-inference-switch\.sh/);
expect(md).toMatch(/test-kimi-inference-compat\.sh/);
expect(md).toMatch(/test-ollama-auth-proxy-e2e\.sh/);
expect(md).toMatch(/test-model-router-provider-routed-inference\.sh/);
});

it("test_should_report_scoped_lifecycle_parity_sections", () => {
const meta = loadMetadataFromDir(E2E_DIR);
Expand Down
76 changes: 75 additions & 1 deletion test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,80 @@ function runBash(script: string, env: Record<string, string> = {}): SpawnSyncRet
// ──────────────────────────────────────────────────────────────────────────

describe("E2E shell helpers", () => {
it("test_should_source_inference_routing_helpers_under_strict_shell_mode", () => {
const r = runBash(`
set -euo pipefail
. "${VALIDATION_SUITES}/lib/inference_routing.sh"
declare -F e2e_inference_routing_assert_chat_completion
`);
expect(r.status, r.stderr).toBe(0);
});

it("test_should_fail_clearly_when_required_context_is_missing", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-inf-missing-"));
try {
const r = runBash(
`
set -euo pipefail
. "${RUNTIME_LIB}/context.sh"
. "${VALIDATION_SUITES}/lib/inference_routing.sh"
e2e_context_init
e2e_inference_routing_assert_chat_completion "post-onboard.inference-routing.inference-local-chat-completion"
`,
{ E2E_CONTEXT_DIR: tmp },
);
expect(r.status).not.toBe(0);
expect(r.stderr).toMatch(/E2E_SANDBOX_NAME|E2E_CONTEXT_DIR|context/i);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
});

it("test_should_emit_plan_only_checks_without_live_infrastructure", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-inf-plan-"));
try {
const r = runBash(
`
set -euo pipefail
. "${RUNTIME_LIB}/context.sh"
. "${VALIDATION_SUITES}/lib/inference_routing.sh"
e2e_context_init
e2e_context_set E2E_SANDBOX_NAME sandbox-1
e2e_inference_routing_assert_chat_completion "post-onboard.inference-routing.inference-local-chat-completion"
`,
{ E2E_CONTEXT_DIR: tmp, E2E_DRY_RUN: "1" },
);
expect(r.status, r.stderr).toBe(0);
expect(r.stdout).toContain("post-onboard.inference-routing.inference-local-chat-completion");
expect(r.stdout).toMatch(/dry-run|plan/i);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
});

it("test_should_not_print_secret_values_in_helper_output", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-inf-secret-"));
try {
const r = runBash(
`
set -euo pipefail
. "${RUNTIME_LIB}/context.sh"
. "${VALIDATION_SUITES}/lib/inference_routing.sh"
e2e_context_init
e2e_context_set E2E_SANDBOX_NAME sandbox-1
e2e_context_set E2E_PROVIDER_API_KEY super-secret-test-token
e2e_inference_routing_assert_auth_proxy "post-onboard.ollama-auth-proxy.authenticated-request-accepted" "valid"
`,
{ E2E_CONTEXT_DIR: tmp, E2E_DRY_RUN: "1" },
);
expect(r.status, r.stderr).toBe(0);
expect(r.stdout + r.stderr).not.toContain("super-secret-test-token");
expect(r.stdout + r.stderr).toMatch(/REDACTED|dry-run|plan/i);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
});

it("security_policy_credentials_helper_should_load_with_context_library", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-context-"));
try {
Expand Down Expand Up @@ -177,7 +251,7 @@ exit 2
fs.writeFileSync(
path.join(fakeBin, "nemoclaw"),
`#!/usr/bin/env bash
echo " ○ telegram — Telegram bridge egress"
echo " slack — Slack bridge egress"
exit 0
`,
{ mode: 0o755 },
Expand Down
41 changes: 41 additions & 0 deletions test/e2e/scenario-framework-tests/e2e-parity-map.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,20 @@ function runCheck(root: string, args: string[] = []) {
});
}

const ISSUE_3812_TARGET_SCRIPTS = [
"test-inference-routing.sh",
"test-openclaw-inference-switch.sh",
"test-kimi-inference-compat.sh",
"test-ollama-auth-proxy-e2e.sh",
"test-model-router-provider-routed-inference.sh",
];

function loadRealParityMap(): { scripts?: Record<string, unknown> } {
return yaml.load(fs.readFileSync(path.join(REPO_ROOT, "test/e2e/docs/parity-map.yaml"), "utf8")) as {
scripts?: Record<string, unknown>;
};
}

describe("rebuild/upgrade parity map records", () => {
it("parity_map_should_classify_all_rebuild_upgrade_legacy_assertions", () => {
const inventory = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, "test/e2e/docs/parity-inventory.generated.json"), "utf8")) as {
Expand Down Expand Up @@ -180,6 +194,33 @@ scripts:
expect(missingStatus.stdout + missingStatus.stderr).toMatch(/status/);
});

it("test_should_include_all_issue_3812_target_scripts_in_parity_map", () => {
const parityMap = loadRealParityMap();

for (const script of ISSUE_3812_TARGET_SCRIPTS) {
expect(parityMap.scripts, script).toHaveProperty(script);
}
});

it("test_should_reject_unknown_target_assertion_status", () => {
writeMap(
tmp,
`
scripts:
test-new.sh:
scenario: ubuntu-repo-cloud-openclaw
assertions:
- legacy: "CLI ready"
status: planned
`,
);
const r = runCheck(tmp);
expect(r.status).not.toBe(0);
expect(r.stdout + r.stderr).toMatch(/test-new\.sh/);
expect(r.stdout + r.stderr).toMatch(/assertions\[0\]/);
expect(r.stdout + r.stderr).toMatch(/status/i);
});

it("check_parity_map_should_reject_unknown_legacy_assertion_strings", () => {
writeMap(
tmp,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import fs from "node:fs";
import os from "node:os";
import path from "node:path";

import yaml from "js-yaml";
import { loadMetadataFromDir } from "../runtime/resolver/load.ts";
import { resolveScenario } from "../runtime/resolver/plan.ts";

Expand Down Expand Up @@ -42,6 +43,20 @@ function planOnly(scenarioId: string): { stdout: string; stderr: string; status:
}
}

describe("Issue 3812: inference/provider suite families", () => {
it("test_should_route_inference_suite_families_to_domain_specific_steps", () => {
const suites = yaml.load(fs.readFileSync(path.join(E2E_DIR, "validation_suites/suites.yaml"), "utf8")) as {
suites: Record<string, { steps?: { script?: string }[] }>;
};
for (const family of ["inference-routing", "inference-switch", "kimi-compatibility", "ollama-auth-proxy", "model-router"]) {
const scripts = suites.suites[family]?.steps?.map((step) => step.script ?? "") ?? [];
expect(scripts.length, family).toBeGreaterThan(0);
expect(scripts.every((script) => script.startsWith("inference/")), family).toBe(true);
expect(scripts.some((script) => !script.startsWith("inference/cloud/")), family).toBe(true);
}
});
});

describe("Phase 9: additional scenario families - metadata", () => {
it("resolver should resolve all new scenarios", () => {
const meta = loadMetadataFromDir(E2E_DIR);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=../../lib/inference_routing.sh
. "${SCRIPT_DIR}/../../lib/inference_routing.sh"
e2e_inference_routing_assert_health "post-onboard.kimi-compatibility.plugin-wired"
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=../../lib/inference_routing.sh
. "${SCRIPT_DIR}/../../lib/inference_routing.sh"
e2e_inference_routing_assert_health "post-onboard.kimi-compatibility.models-route-reachable"
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=../../lib/inference_routing.sh
. "${SCRIPT_DIR}/../../lib/inference_routing.sh"
e2e_inference_routing_assert_health "post-onboard.model-router.healthy-endpoint-reported"
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=../../lib/inference_routing.sh
. "${SCRIPT_DIR}/../../lib/inference_routing.sh"
e2e_inference_routing_assert_chat_completion "post-onboard.model-router.provider-routed-completion"
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=../../lib/inference_routing.sh
. "${SCRIPT_DIR}/../../lib/inference_routing.sh"
e2e_inference_routing_assert_auth_proxy "post-onboard.ollama-auth-proxy.unauthenticated-request-rejected" "unauthenticated"
e2e_inference_routing_assert_auth_proxy "post-onboard.ollama-auth-proxy.authenticated-request-accepted" "valid"
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=../../lib/inference_routing.sh
. "${SCRIPT_DIR}/../../lib/inference_routing.sh"
e2e_inference_routing_assert_chat_completion "post-onboard.inference-routing.inference-local-chat-completion"
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=../../lib/inference_routing.sh
. "${SCRIPT_DIR}/../../lib/inference_routing.sh"
e2e_inference_routing_assert_health "post-onboard.inference-routing.provider-route-healthy"
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=../../lib/inference_routing.sh
. "${SCRIPT_DIR}/../../lib/inference_routing.sh"
e2e_inference_routing_assert_health "post-onboard.inference-switch.route-state-updated"
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=../../lib/inference_routing.sh
. "${SCRIPT_DIR}/../../lib/inference_routing.sh"
e2e_inference_routing_assert_chat_completion "post-onboard.inference-switch.switched-chat-completion"
Loading
Loading