diff --git a/test/e2e/docs/parity-map.yaml b/test/e2e/docs/parity-map.yaml index b65e96c10d..b15e66823c 100644 --- a/test/e2e/docs/parity-map.yaml +++ b/test/e2e/docs/parity-map.yaml @@ -4540,7 +4540,7 @@ scripts: test-inference-routing.sh: scenario: ubuntu-repo-cloud-openclaw status: migrated - bucket: providers-messaging + bucket: inference-routing-provider assertions: - legacy: 'TC-INF-05: Setup' status: deferred @@ -5078,7 +5078,7 @@ scripts: test-kimi-inference-compat.sh: scenario: ubuntu-repo-cloud-openclaw status: migrated - bucket: providers-messaging + bucket: inference-routing-provider assertions: - legacy: 'K1: source CLI/OpenShell preparation failed (exit $prep_exit)' status: deferred @@ -6989,7 +6989,7 @@ scripts: test-ollama-auth-proxy-e2e.sh: scenario: gpu-repo-local-ollama-openclaw status: migrated - bucket: providers-messaging + bucket: inference-routing-provider assertions: - legacy: Node.js not found status: deferred @@ -7807,7 +7807,7 @@ scripts: test-openclaw-inference-switch.sh: scenario: ubuntu-repo-cloud-openclaw status: migrated - bucket: providers-messaging + bucket: inference-routing-provider assertions: - legacy: 'OpenShell inference get failed: ${output:0:240}' status: deferred @@ -11639,7 +11639,7 @@ scripts: test-model-router-provider-routed-inference.sh: scenario: ubuntu-repo-cloud-openclaw status: migrated - bucket: providers-messaging + bucket: inference-routing-provider assertions: - legacy: Docker is running status: deferred diff --git a/test/e2e/runtime/resolver/coverage.ts b/test/e2e/runtime/resolver/coverage.ts index d3544e0338..ce6f7519bf 100644 --- a/test/e2e/runtime/resolver/coverage.ts +++ b/test/e2e/runtime/resolver/coverage.ts @@ -99,7 +99,7 @@ function renderLegacyParitySummary(meta: ResolverInput): string[] { a.localeCompare(b), )) { lines.push( - `| ${bucket} | ${row.scripts.size} | ${row.mapped} | ${row.deferred} | ${row.retired} | ${row.unmapped} |`, + `| ${bucket} | ${[...row.scripts].sort().join(", ")} | ${row.mapped} | ${row.deferred} | ${row.retired} | ${row.unmapped} |`, ); } lines.push(""); diff --git a/test/e2e/runtime/run-scenario.sh b/test/e2e/runtime/run-scenario.sh index 03721e0e7a..99f917b8c8 100755 --- a/test/e2e/runtime/run-scenario.sh +++ b/test/e2e/runtime/run-scenario.sh @@ -464,7 +464,7 @@ if [[ "${DOCKER_OPTIONAL_UNAVAILABLE}" -eq 1 ]]; then FILTERED_SUITE_IDS=() for suite_id in "${SUITE_IDS[@]}"; do case "${suite_id}" in - smoke | inference | credentials | hermes-specific | local-ollama-inference | ollama-proxy | gateway-health | sandbox-shell | cloud-inference | ollama-auth-proxy | security-credentials | messaging-telegram | messaging-discord | messaging-slack | security-shields | inference-routing | sandbox-lifecycle | sandbox-operations | snapshot | rebuild | upgrade | diagnostics | docs-validation | openai-compatible-inference | inference-switch | kimi-compatibility | messaging-token-rotation | security-policy | security-injection) + smoke | inference | credentials | hermes-specific | local-ollama-inference | ollama-proxy | gateway-health | sandbox-shell | cloud-inference | ollama-auth-proxy | security-credentials | messaging-telegram | messaging-discord | messaging-slack | security-shields | inference-routing | sandbox-lifecycle | sandbox-operations | snapshot | rebuild | upgrade | diagnostics | docs-validation | openai-compatible-inference | inference-switch | kimi-compatibility | messaging-token-rotation | security-policy | security-injection | model-router) echo "SKIP: suite.${suite_id} skipped because optional Docker runtime ${RUNTIME_ID} is unavailable" ;; *) diff --git a/test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts b/test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts index 2d79ccd6d6..0cd97885df 100644 --- a/test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts @@ -123,6 +123,17 @@ describe("coverage report", () => { } }); + it("test_should_report_issue_3812_domain_coverage_summary", () => { + const meta = loadMetadataFromDir(E2E_DIR); + const md = renderCoverageReport(meta); + + expect(md).toMatch(/inference-routing-provider/); + expect(md).toMatch(/test-inference-routing\.sh/); + expect(md).toMatch(/test-openclaw-inference-switch\.sh/); + expect(md).toMatch(/test-kimi-inference-compat\.sh/); + expect(md).toMatch(/test-ollama-auth-proxy-e2e\.sh/); + expect(md).toMatch(/test-model-router-provider-routed-inference\.sh/); + }); it("test_should_report_scoped_lifecycle_parity_sections", () => { const meta = loadMetadataFromDir(E2E_DIR); diff --git a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts b/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts index d5318cccef..56624ec1af 100644 --- a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts @@ -32,6 +32,80 @@ function runBash(script: string, env: Record = {}): SpawnSyncRet // ────────────────────────────────────────────────────────────────────────── describe("E2E shell helpers", () => { + it("test_should_source_inference_routing_helpers_under_strict_shell_mode", () => { + const r = runBash(` + set -euo pipefail + . "${VALIDATION_SUITES}/lib/inference_routing.sh" + declare -F e2e_inference_routing_assert_chat_completion + `); + expect(r.status, r.stderr).toBe(0); + }); + + it("test_should_fail_clearly_when_required_context_is_missing", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-inf-missing-")); + try { + const r = runBash( + ` + set -euo pipefail + . "${RUNTIME_LIB}/context.sh" + . "${VALIDATION_SUITES}/lib/inference_routing.sh" + e2e_context_init + e2e_inference_routing_assert_chat_completion "post-onboard.inference-routing.inference-local-chat-completion" + `, + { E2E_CONTEXT_DIR: tmp }, + ); + expect(r.status).not.toBe(0); + expect(r.stderr).toMatch(/E2E_SANDBOX_NAME|E2E_CONTEXT_DIR|context/i); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("test_should_emit_plan_only_checks_without_live_infrastructure", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-inf-plan-")); + try { + const r = runBash( + ` + set -euo pipefail + . "${RUNTIME_LIB}/context.sh" + . "${VALIDATION_SUITES}/lib/inference_routing.sh" + e2e_context_init + e2e_context_set E2E_SANDBOX_NAME sandbox-1 + e2e_inference_routing_assert_chat_completion "post-onboard.inference-routing.inference-local-chat-completion" + `, + { E2E_CONTEXT_DIR: tmp, E2E_DRY_RUN: "1" }, + ); + expect(r.status, r.stderr).toBe(0); + expect(r.stdout).toContain("post-onboard.inference-routing.inference-local-chat-completion"); + expect(r.stdout).toMatch(/dry-run|plan/i); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("test_should_not_print_secret_values_in_helper_output", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-inf-secret-")); + try { + const r = runBash( + ` + set -euo pipefail + . "${RUNTIME_LIB}/context.sh" + . "${VALIDATION_SUITES}/lib/inference_routing.sh" + e2e_context_init + e2e_context_set E2E_SANDBOX_NAME sandbox-1 + e2e_context_set E2E_PROVIDER_API_KEY super-secret-test-token + e2e_inference_routing_assert_auth_proxy "post-onboard.ollama-auth-proxy.authenticated-request-accepted" "valid" + `, + { E2E_CONTEXT_DIR: tmp, E2E_DRY_RUN: "1" }, + ); + expect(r.status, r.stderr).toBe(0); + expect(r.stdout + r.stderr).not.toContain("super-secret-test-token"); + expect(r.stdout + r.stderr).toMatch(/REDACTED|dry-run|plan/i); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("security_policy_credentials_helper_should_load_with_context_library", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-context-")); try { @@ -177,7 +251,7 @@ exit 2 fs.writeFileSync( path.join(fakeBin, "nemoclaw"), `#!/usr/bin/env bash -echo " ○ telegram — Telegram bridge egress" +echo " slack — Slack bridge egress" exit 0 `, { mode: 0o755 }, diff --git a/test/e2e/scenario-framework-tests/e2e-parity-map.test.ts b/test/e2e/scenario-framework-tests/e2e-parity-map.test.ts index d151616b7f..10f1460463 100644 --- a/test/e2e/scenario-framework-tests/e2e-parity-map.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-parity-map.test.ts @@ -50,6 +50,20 @@ function runCheck(root: string, args: string[] = []) { }); } +const ISSUE_3812_TARGET_SCRIPTS = [ + "test-inference-routing.sh", + "test-openclaw-inference-switch.sh", + "test-kimi-inference-compat.sh", + "test-ollama-auth-proxy-e2e.sh", + "test-model-router-provider-routed-inference.sh", +]; + +function loadRealParityMap(): { scripts?: Record } { + return yaml.load(fs.readFileSync(path.join(REPO_ROOT, "test/e2e/docs/parity-map.yaml"), "utf8")) as { + scripts?: Record; + }; +} + describe("rebuild/upgrade parity map records", () => { it("parity_map_should_classify_all_rebuild_upgrade_legacy_assertions", () => { const inventory = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, "test/e2e/docs/parity-inventory.generated.json"), "utf8")) as { @@ -180,6 +194,33 @@ scripts: expect(missingStatus.stdout + missingStatus.stderr).toMatch(/status/); }); + it("test_should_include_all_issue_3812_target_scripts_in_parity_map", () => { + const parityMap = loadRealParityMap(); + + for (const script of ISSUE_3812_TARGET_SCRIPTS) { + expect(parityMap.scripts, script).toHaveProperty(script); + } + }); + + it("test_should_reject_unknown_target_assertion_status", () => { + writeMap( + tmp, + ` +scripts: + test-new.sh: + scenario: ubuntu-repo-cloud-openclaw + assertions: + - legacy: "CLI ready" + status: planned +`, + ); + const r = runCheck(tmp); + expect(r.status).not.toBe(0); + expect(r.stdout + r.stderr).toMatch(/test-new\.sh/); + expect(r.stdout + r.stderr).toMatch(/assertions\[0\]/); + expect(r.stdout + r.stderr).toMatch(/status/i); + }); + it("check_parity_map_should_reject_unknown_legacy_assertion_strings", () => { writeMap( tmp, diff --git a/test/e2e/scenario-framework-tests/e2e-scenario-additional-families.test.ts b/test/e2e/scenario-framework-tests/e2e-scenario-additional-families.test.ts index db424686d2..0bf5cdca2f 100644 --- a/test/e2e/scenario-framework-tests/e2e-scenario-additional-families.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-scenario-additional-families.test.ts @@ -15,6 +15,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import yaml from "js-yaml"; import { loadMetadataFromDir } from "../runtime/resolver/load.ts"; import { resolveScenario } from "../runtime/resolver/plan.ts"; @@ -42,6 +43,20 @@ function planOnly(scenarioId: string): { stdout: string; stderr: string; status: } } +describe("Issue 3812: inference/provider suite families", () => { + it("test_should_route_inference_suite_families_to_domain_specific_steps", () => { + const suites = yaml.load(fs.readFileSync(path.join(E2E_DIR, "validation_suites/suites.yaml"), "utf8")) as { + suites: Record; + }; + for (const family of ["inference-routing", "inference-switch", "kimi-compatibility", "ollama-auth-proxy", "model-router"]) { + const scripts = suites.suites[family]?.steps?.map((step) => step.script ?? "") ?? []; + expect(scripts.length, family).toBeGreaterThan(0); + expect(scripts.every((script) => script.startsWith("inference/")), family).toBe(true); + expect(scripts.some((script) => !script.startsWith("inference/cloud/")), family).toBe(true); + } + }); +}); + describe("Phase 9: additional scenario families - metadata", () => { it("resolver should resolve all new scenarios", () => { const meta = loadMetadataFromDir(E2E_DIR); diff --git a/test/e2e/validation_suites/inference/kimi-compatibility/00-plugin-wiring.sh b/test/e2e/validation_suites/inference/kimi-compatibility/00-plugin-wiring.sh new file mode 100755 index 0000000000..d2a2bdff7a --- /dev/null +++ b/test/e2e/validation_suites/inference/kimi-compatibility/00-plugin-wiring.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../../lib/inference_routing.sh +. "${SCRIPT_DIR}/../../lib/inference_routing.sh" +e2e_inference_routing_assert_health "post-onboard.kimi-compatibility.plugin-wired" diff --git a/test/e2e/validation_suites/inference/kimi-compatibility/01-kimi-compatible-models-route.sh b/test/e2e/validation_suites/inference/kimi-compatibility/01-kimi-compatible-models-route.sh new file mode 100755 index 0000000000..aeda33b4ce --- /dev/null +++ b/test/e2e/validation_suites/inference/kimi-compatibility/01-kimi-compatible-models-route.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../../lib/inference_routing.sh +. "${SCRIPT_DIR}/../../lib/inference_routing.sh" +e2e_inference_routing_assert_health "post-onboard.kimi-compatibility.models-route-reachable" diff --git a/test/e2e/validation_suites/inference/model-router/00-healthy-endpoint.sh b/test/e2e/validation_suites/inference/model-router/00-healthy-endpoint.sh new file mode 100755 index 0000000000..eb9f74ff13 --- /dev/null +++ b/test/e2e/validation_suites/inference/model-router/00-healthy-endpoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../../lib/inference_routing.sh +. "${SCRIPT_DIR}/../../lib/inference_routing.sh" +e2e_inference_routing_assert_health "post-onboard.model-router.healthy-endpoint-reported" diff --git a/test/e2e/validation_suites/inference/model-router/01-provider-routed-completion.sh b/test/e2e/validation_suites/inference/model-router/01-provider-routed-completion.sh new file mode 100755 index 0000000000..537fe3c551 --- /dev/null +++ b/test/e2e/validation_suites/inference/model-router/01-provider-routed-completion.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../../lib/inference_routing.sh +. "${SCRIPT_DIR}/../../lib/inference_routing.sh" +e2e_inference_routing_assert_chat_completion "post-onboard.model-router.provider-routed-completion" diff --git a/test/e2e/validation_suites/inference/ollama-auth-proxy/01-auth-enforcement.sh b/test/e2e/validation_suites/inference/ollama-auth-proxy/01-auth-enforcement.sh new file mode 100755 index 0000000000..90bce8092a --- /dev/null +++ b/test/e2e/validation_suites/inference/ollama-auth-proxy/01-auth-enforcement.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../../lib/inference_routing.sh +. "${SCRIPT_DIR}/../../lib/inference_routing.sh" +e2e_inference_routing_assert_auth_proxy "post-onboard.ollama-auth-proxy.unauthenticated-request-rejected" "unauthenticated" +e2e_inference_routing_assert_auth_proxy "post-onboard.ollama-auth-proxy.authenticated-request-accepted" "valid" diff --git a/test/e2e/validation_suites/inference/routing/00-inference-local-chat-completion.sh b/test/e2e/validation_suites/inference/routing/00-inference-local-chat-completion.sh new file mode 100755 index 0000000000..b2060bffff --- /dev/null +++ b/test/e2e/validation_suites/inference/routing/00-inference-local-chat-completion.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../../lib/inference_routing.sh +. "${SCRIPT_DIR}/../../lib/inference_routing.sh" +e2e_inference_routing_assert_chat_completion "post-onboard.inference-routing.inference-local-chat-completion" diff --git a/test/e2e/validation_suites/inference/routing/01-provider-route-health.sh b/test/e2e/validation_suites/inference/routing/01-provider-route-health.sh new file mode 100755 index 0000000000..307b0f4ef3 --- /dev/null +++ b/test/e2e/validation_suites/inference/routing/01-provider-route-health.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../../lib/inference_routing.sh +. "${SCRIPT_DIR}/../../lib/inference_routing.sh" +e2e_inference_routing_assert_health "post-onboard.inference-routing.provider-route-healthy" diff --git a/test/e2e/validation_suites/inference/switch/00-route-state-updated.sh b/test/e2e/validation_suites/inference/switch/00-route-state-updated.sh new file mode 100755 index 0000000000..1c287abd7f --- /dev/null +++ b/test/e2e/validation_suites/inference/switch/00-route-state-updated.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../../lib/inference_routing.sh +. "${SCRIPT_DIR}/../../lib/inference_routing.sh" +e2e_inference_routing_assert_health "post-onboard.inference-switch.route-state-updated" diff --git a/test/e2e/validation_suites/inference/switch/01-switched-inference-local-chat.sh b/test/e2e/validation_suites/inference/switch/01-switched-inference-local-chat.sh new file mode 100755 index 0000000000..9e03d6cb1b --- /dev/null +++ b/test/e2e/validation_suites/inference/switch/01-switched-inference-local-chat.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../../lib/inference_routing.sh +. "${SCRIPT_DIR}/../../lib/inference_routing.sh" +e2e_inference_routing_assert_chat_completion "post-onboard.inference-switch.switched-chat-completion" diff --git a/test/e2e/validation_suites/lib/inference_routing.sh b/test/e2e/validation_suites/lib/inference_routing.sh new file mode 100755 index 0000000000..b4f4c1d63f --- /dev/null +++ b/test/e2e/validation_suites/lib/inference_routing.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Inference/provider validation primitives for scenario-suite steps. + +_E2E_INF_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +_E2E_INF_RUNTIME_LIB_DIR="$(cd "${_E2E_INF_LIB_DIR}/../../runtime/lib" && pwd)" +_E2E_INF_VALIDATION_DIR="$(cd "${_E2E_INF_LIB_DIR}/.." && pwd)" +# shellcheck source=../../runtime/lib/env.sh +. "${_E2E_INF_RUNTIME_LIB_DIR}/env.sh" +# shellcheck source=../../runtime/lib/context.sh +. "${_E2E_INF_RUNTIME_LIB_DIR}/context.sh" +# shellcheck source=../sandbox-exec.sh +. "${_E2E_INF_VALIDATION_DIR}/sandbox-exec.sh" + +_e2e_inference_assertion() { + local assertion_id="${1:-}" + if [[ -z "${assertion_id}" ]]; then + echo "e2e_inference_routing: missing assertion id" >&2 + return 2 + fi + e2e_section "${assertion_id}" +} + +_e2e_inference_require_sandbox() { + e2e_context_require E2E_SANDBOX_NAME +} + +_e2e_inference_sandbox_name() { + e2e_context_get E2E_SANDBOX_NAME +} + +_e2e_inference_plan() { + local assertion_id="${1:-}" + local detail="${2:-planned inference/provider check}" + e2e_env_trace "inference:plan" "${assertion_id} ${detail}" + echo "[dry-run] ${assertion_id}: ${detail}" + if [[ -f "$(e2e_context_path)" ]]; then + e2e_context_dump | sed -E 's/(TOKEN|SECRET|API_KEY|APIKEY|CREDENTIAL|PASSWORD)([^=]*)=.*/\1\2=REDACTED/' + fi +} + +_e2e_inference_curl_json() { + local sandbox="$1" + local url="$2" + local payload="${3:-}" + if [[ -n "${payload}" ]]; then + printf '%s' "${payload}" | e2e_sandbox_exec_stdin "${sandbox}" -- curl --silent --show-error --fail --max-time 20 \ + -H 'content-type: application/json' -d @- "${url}" + else + e2e_sandbox_exec "${sandbox}" -- curl --silent --show-error --fail --max-time 20 "${url}" + fi +} + +_e2e_inference_status() { + local sandbox="$1" + local url="$2" + shift 2 + e2e_sandbox_exec "${sandbox}" -- curl --silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 20 "$@" "${url}" +} + +e2e_inference_routing_assert_chat_completion() { + local assertion_id="${1:-post-onboard.inference-routing.inference-local-chat-completion}" + _e2e_inference_assertion "${assertion_id}" + _e2e_inference_require_sandbox + if e2e_env_is_dry_run; then + _e2e_inference_plan "${assertion_id}" "POST https://inference.local/v1/chat/completions with bounded curl" + return 0 + fi + local sandbox payload output + sandbox="$(_e2e_inference_sandbox_name)" + payload='{"model":"default","messages":[{"role":"user","content":"Say ok"}],"max_tokens":8}' + output="$(_e2e_inference_curl_json "${sandbox}" "https://inference.local/v1/chat/completions" "${payload}")" + if [[ "${output}" != *choices* && "${output}" != *content* ]]; then + echo "e2e_inference_routing: chat completion response missing choices/content" >&2 + return 1 + fi + e2e_pass "${assertion_id}" +} + +e2e_inference_routing_assert_health() { + local assertion_id="${1:-post-onboard.inference-routing.provider-route-healthy}" + local url="${2:-https://inference.local/v1/models}" + _e2e_inference_assertion "${assertion_id}" + _e2e_inference_require_sandbox + if e2e_env_is_dry_run; then + _e2e_inference_plan "${assertion_id}" "GET ${url} with bounded curl" + return 0 + fi + local sandbox status + sandbox="$(_e2e_inference_sandbox_name)" + status="$(_e2e_inference_status "${sandbox}" "${url}")" + [[ "${status}" =~ ^2[0-9][0-9]$ ]] || { + echo "e2e_inference_routing: ${url} returned HTTP ${status}" >&2 + return 1 + } + e2e_pass "${assertion_id}" +} + +e2e_inference_routing_assert_auth_proxy() { + local assertion_id="${1:-post-onboard.ollama-auth-proxy.authenticated-request-accepted}" + local mode="${2:-valid}" + _e2e_inference_assertion "${assertion_id}" + _e2e_inference_require_sandbox + if e2e_env_is_dry_run; then + _e2e_inference_plan "${assertion_id}" "auth-proxy ${mode} request; sensitive context redacted" + return 0 + fi + local sandbox status token + sandbox="$(_e2e_inference_sandbox_name)" + case "${mode}" in + unauthenticated) + status="$(_e2e_inference_status "${sandbox}" "https://inference.local/v1/models")" + [[ "${status}" =~ ^(401|403)$ ]] || return 1 + ;; + invalid) + status="$(_e2e_inference_status "${sandbox}" "https://inference.local/v1/models" -H 'Authorization: Bearer invalid-token')" + [[ "${status}" =~ ^(401|403)$ ]] || return 1 + ;; + valid) + e2e_context_require E2E_OLLAMA_AUTH_TOKEN + token="$(e2e_context_get E2E_OLLAMA_AUTH_TOKEN)" + status="$(_e2e_inference_status "${sandbox}" "https://inference.local/v1/models" -H "Authorization: Bearer ${token}")" + [[ "${status}" =~ ^2[0-9][0-9]$ ]] || return 1 + ;; + *) + echo "e2e_inference_routing: unknown auth proxy mode ${mode}" >&2 + return 2 + ;; + esac + e2e_pass "${assertion_id}" +} diff --git a/test/e2e/validation_suites/suites.yaml b/test/e2e/validation_suites/suites.yaml index 7152feeec4..3da49da83f 100644 --- a/test/e2e/validation_suites/suites.yaml +++ b/test/e2e/validation_suites/suites.yaml @@ -104,7 +104,11 @@ suites: steps: *id004 ollama-auth-proxy: requires_state: *id005 - steps: *id006 + steps: + - id: proxy-reachable + script: inference/ollama-auth-proxy/00-proxy-reachable.sh + - id: auth-enforcement + script: inference/ollama-auth-proxy/01-auth-enforcement.sh security-credentials: requires_state: credentials.expected: present @@ -162,7 +166,11 @@ suites: script: security/shields/00-config-consistent.sh inference-routing: requires_state: *id003 - steps: *id004 + steps: + - id: inference-local-chat-completion + script: inference/routing/00-inference-local-chat-completion.sh + - id: provider-route-health + script: inference/routing/01-provider-route-health.sh sandbox-lifecycle: requires_state: gateway.health: healthy @@ -222,10 +230,25 @@ suites: steps: *id004 inference-switch: requires_state: *id003 - steps: *id004 + steps: + - id: route-state-updated + script: inference/switch/00-route-state-updated.sh + - id: switched-inference-local-chat + script: inference/switch/01-switched-inference-local-chat.sh kimi-compatibility: requires_state: *id003 - steps: *id004 + steps: + - id: plugin-wiring + script: inference/kimi-compatibility/00-plugin-wiring.sh + - id: kimi-compatible-models-route + script: inference/kimi-compatibility/01-kimi-compatible-models-route.sh + model-router: + requires_state: *id003 + steps: + - id: healthy-endpoint + script: inference/model-router/00-healthy-endpoint.sh + - id: provider-routed-completion + script: inference/model-router/01-provider-routed-completion.sh messaging-token-rotation: requires_state: *id001 steps: