test(agent): live smoke through real gateway to real Anthropic

crates/agent/tests/common/mod.rs

@@ -1,13 +1,40 @@
-//! Shared test helpers: a mock model server speaking Anthropic SSE.
+//! Shared test helpers: a mock model server speaking Anthropic SSE, port helpers, and a locator for
+//! the gateway binary.
 #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic, dead_code)]
 
 use std::io::{Read, Write};
 use std::net::{TcpListener, TcpStream};
+use std::path::PathBuf;
+use std::process::Command;
 use std::sync::{Arc, Mutex};
 use std::thread;
 
 use serde_json::{Value, json};
 
+/// Deterministic dev signing public key (standard base64), for a gateway `[signing_keys] 1 = …`.
+pub const DEV_PUBKEY_B64: &str = "6kpsY+KcUgq+9VB7Ey7F+ZVHdq6+vnuSQh7qaRRG0iw=";
+/// The matching dev `bai_v1` token (tenant 1 / vpc 1, kid 1).
+pub const DEV_TOKEN: &str = "bai_v1.1.AQAAAAAAAAABAAAAAAAAAA.WrWcPbklu91PS-4WuR6GnBNF3h4nROpH0EQQlfJf06f7_lEnlQOCSBimhH2JMwXFJgw40BniTB7-yIdFnpldDw";
+
+/// Locate the gateway binary (built beside the agent binary); build it on demand if absent.
+pub fn gateway_bin() -> PathBuf {
+    let agent = PathBuf::from(env!("CARGO_BIN_EXE_beyond-ai-agent"));
+    let dir = agent.parent().unwrap();
+    let gw = dir.join("beyond-ai");
+    if !gw.exists() {
+        let mut args = vec!["build", "-q", "-p", "beyond-ai", "--bin", "beyond-ai"];
+        if agent.to_string_lossy().contains("/release/") {
+            args.push("--release");
+        }
+        let status = Command::new(env!("CARGO"))
+            .args(&args)
+            .status()
+            .expect("build gateway");
+        assert!(status.success(), "failed to build the gateway binary");
+    }
+    gw
+}
+
 /// An Anthropic SSE turn that calls one tool with the given JSON-argument string.
 pub fn turn_tool_use(id: &str, name: &str, args_json: &str) -> String {
     sse(&[

crates/agent/tests/gateway_e2e.rs

@@ -12,36 +12,14 @@
 
 mod common;
 
-use std::path::PathBuf;
 use std::process::{Command, Stdio};
 
-use common::{free_port, spawn_model_server, turn_text, turn_tool_use, wait_for_port};
+use common::{
+    DEV_PUBKEY_B64, DEV_TOKEN, free_port, gateway_bin, spawn_model_server, turn_text,
+    turn_tool_use, wait_for_port,
+};
 use serde_json::json;
 
-/// Deterministic dev signing public key (standard base64), for `[signing_keys] 1 = …`.
-const DEV_PUBKEY_B64: &str = "6kpsY+KcUgq+9VB7Ey7F+ZVHdq6+vnuSQh7qaRRG0iw=";
-/// The matching dev `bai_v1` token (tenant 1 / vpc 1, kid 1).
-const DEV_TOKEN: &str = "bai_v1.1.AQAAAAAAAAABAAAAAAAAAA.WrWcPbklu91PS-4WuR6GnBNF3h4nROpH0EQQlfJf06f7_lEnlQOCSBimhH2JMwXFJgw40BniTB7-yIdFnpldDw";
-
-/// Locate the gateway binary (built beside the agent binary); build it on demand if absent.
-fn gateway_bin() -> PathBuf {
-    let agent = PathBuf::from(env!("CARGO_BIN_EXE_beyond-ai-agent"));
-    let dir = agent.parent().unwrap();
-    let gw = dir.join("beyond-ai");
-    if !gw.exists() {
-        let mut args = vec!["build", "-q", "-p", "beyond-ai", "--bin", "beyond-ai"];
-        if agent.to_string_lossy().contains("/release/") {
-            args.push("--release");
-        }
-        let status = Command::new(env!("CARGO"))
-            .args(&args)
-            .status()
-            .expect("build gateway");
-        assert!(status.success(), "failed to build the gateway binary");
-    }
-    gw
-}
-
 #[test]
 fn agent_through_real_gateway_to_mock_upstream() {
     let dir = tempfile::tempdir().unwrap();

crates/agent/tests/smoke.rs

@@ -0,0 +1,98 @@
+//! Live smoke: the real `beyond-ai-agent` binary → the real `beyond-ai` gateway → **real Anthropic**.
+//!
+//! Ignored by default (bills a tiny real request). Run with a key present:
+//!
+//!   ANTHROPIC_API_KEY=sk-ant-… mise run test:smoke:agent
+//!   ANTHROPIC_API_KEY=sk-ant-… cargo test -p beyond-ai-agent --test smoke -- --ignored --nocapture
+//!
+//! This is the one test that validates the dialect decoder against a *real* provider's SSE: the
+//! gateway boots with the caller's Anthropic key as the managed pool key (and the dev signing key),
+//! then the agent — holding only a `bai_v1` virtual key — drives a real tool round-trip through it.
+//! A model-not-found is a stale model id, not a harness bug (adjust `MODEL`).
+#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
+
+mod common;
+
+use std::process::{Command, Stdio};
+
+use common::{DEV_PUBKEY_B64, DEV_TOKEN, free_port, gateway_bin, wait_for_port};
+
+/// Cheapest small Anthropic model (matches the gateway's own smoke test).
+const MODEL: &str = "claude-haiku-4-5";
+
+fn env_key(name: &str) -> Option<String> {
+    std::env::var(name).ok().filter(|v| !v.trim().is_empty())
+}
+
+#[test]
+#[ignore = "live provider smoke; run via `mise run test:smoke:agent` with ANTHROPIC_API_KEY set"]
+fn smoke_agent_through_gateway_to_real_anthropic() {
+    let Some(anthropic_key) = env_key("ANTHROPIC_API_KEY") else {
+        eprintln!("smoke[agent]: ANTHROPIC_API_KEY unset — skipping");
+        return;
+    };
+
+    // A file the agent must read with its `read` tool, then echo back — proves a live tool round-trip.
+    let dir = tempfile::tempdir().unwrap();
+    let token = "PINEAPPLE-7493";
+    std::fs::write(dir.path().join("marker.txt"), format!("{token}\n")).unwrap();
+
+    // Gateway → REAL Anthropic (no authority override, real TLS upstream). Managed pool key = the
+    // caller's real key; the agent presents only the dev virtual key.
+    let gw_port = free_port();
+    let metrics_port = free_port();
+    let config = format!(
+        "listen = \"127.0.0.1:{gw_port}\"\n\
+         metrics_listen = \"127.0.0.1:{metrics_port}\"\n\
+         nats_url = \"nats://127.0.0.1:59321\"\n\
+         config_bucket = \"ai-gateway\"\n\
+         upstream_tls = true\n\
+         \n[pool_keys]\nanthropic = \"{anthropic_key}\"\n\
+         \n[signing_keys]\n1 = \"{DEV_PUBKEY_B64}\"\n"
+    );
+    let config_path = dir.path().join("gateway.toml");
+    std::fs::write(&config_path, config).unwrap();
+
+    let mut gateway = Command::new(gateway_bin())
+        .arg("run")
+        .arg("-c")
+        .arg(&config_path)
+        .env("AI_LOG", "warn")
+        .stdout(Stdio::null())
+        .stderr(Stdio::inherit())
+        .spawn()
+        .expect("spawn gateway");
+    wait_for_port(gw_port);
+
+    let output = Command::new(env!("CARGO_BIN_EXE_beyond-ai-agent"))
+        .args([
+            "run",
+            "Use the read tool to read the file marker.txt in the current directory, then reply with ONLY the exact token it contains.",
+            "--gateway-url",
+            &format!("http://127.0.0.1:{gw_port}"),
+            "--key",
+            DEV_TOKEN,
+            "--model",
+            MODEL,
+            "--max-steps",
+            "6",
+        ])
+        .current_dir(dir.path())
+        .output()
+        .expect("spawn agent");
+
+    let _ = gateway.kill();
+    let _ = gateway.wait();
+
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    let stderr = String::from_utf8_lossy(&output.stderr);
+    eprintln!("--- agent stdout ---\n{stdout}\n--- agent stderr ---\n{stderr}");
+    assert!(
+        output.status.success(),
+        "agent failed against real Anthropic.\nstdout: {stdout}\nstderr: {stderr}"
+    );
+    assert!(
+        stdout.contains(token),
+        "real Claude should have read the file via the tool and echoed `{token}`.\nstdout: {stdout}"
+    );
+}

mise.toml

@@ -54,6 +54,13 @@ if [ -f .env ]; then set -a; . ./.env; set +a; fi
 cargo test -p beyond-ai --test smoke -- --ignored --nocapture
 """
 
+[tasks."test:smoke:agent"]
+description = "Live agent smoke: the real agent binary → real gateway → REAL Anthropic. Auto-loads .env; needs ANTHROPIC_API_KEY. Skips if unset. Bills a tiny real request."
+run = """
+if [ -f .env ]; then set -a; . ./.env; set +a; fi
+cargo test -p beyond-ai-agent --test smoke -- --ignored --nocapture
+"""
+
 [tasks."bench:unit"]
 description = "divan micro-benchmarks of the IO-free hot paths (key/peek/usage/route/deny): timing + native allocation counts."
 run = "cargo bench --bench unit"