diff --git a/crates/agent/tests/common/mod.rs b/crates/agent/tests/common/mod.rs index cf23548..8ac3640 100644 --- a/crates/agent/tests/common/mod.rs +++ b/crates/agent/tests/common/mod.rs @@ -1,13 +1,40 @@ -//! Shared test helpers: a mock model server speaking Anthropic SSE. +//! Shared test helpers: a mock model server speaking Anthropic SSE, port helpers, and a locator for +//! the gateway binary. #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic, dead_code)] use std::io::{Read, Write}; use std::net::{TcpListener, TcpStream}; +use std::path::PathBuf; +use std::process::Command; use std::sync::{Arc, Mutex}; use std::thread; use serde_json::{Value, json}; +/// Deterministic dev signing public key (standard base64), for a gateway `[signing_keys] 1 = …`. +pub const DEV_PUBKEY_B64: &str = "6kpsY+KcUgq+9VB7Ey7F+ZVHdq6+vnuSQh7qaRRG0iw="; +/// The matching dev `bai_v1` token (tenant 1 / vpc 1, kid 1). +pub const DEV_TOKEN: &str = "bai_v1.1.AQAAAAAAAAABAAAAAAAAAA.WrWcPbklu91PS-4WuR6GnBNF3h4nROpH0EQQlfJf06f7_lEnlQOCSBimhH2JMwXFJgw40BniTB7-yIdFnpldDw"; + +/// Locate the gateway binary (built beside the agent binary); build it on demand if absent. +pub fn gateway_bin() -> PathBuf { + let agent = PathBuf::from(env!("CARGO_BIN_EXE_beyond-ai-agent")); + let dir = agent.parent().unwrap(); + let gw = dir.join("beyond-ai"); + if !gw.exists() { + let mut args = vec!["build", "-q", "-p", "beyond-ai", "--bin", "beyond-ai"]; + if agent.to_string_lossy().contains("/release/") { + args.push("--release"); + } + let status = Command::new(env!("CARGO")) + .args(&args) + .status() + .expect("build gateway"); + assert!(status.success(), "failed to build the gateway binary"); + } + gw +} + /// An Anthropic SSE turn that calls one tool with the given JSON-argument string. pub fn turn_tool_use(id: &str, name: &str, args_json: &str) -> String { sse(&[ diff --git a/crates/agent/tests/gateway_e2e.rs b/crates/agent/tests/gateway_e2e.rs index 2fbfe72..088f64d 100644 --- a/crates/agent/tests/gateway_e2e.rs +++ b/crates/agent/tests/gateway_e2e.rs @@ -12,36 +12,14 @@ mod common; -use std::path::PathBuf; use std::process::{Command, Stdio}; -use common::{free_port, spawn_model_server, turn_text, turn_tool_use, wait_for_port}; +use common::{ + DEV_PUBKEY_B64, DEV_TOKEN, free_port, gateway_bin, spawn_model_server, turn_text, + turn_tool_use, wait_for_port, +}; use serde_json::json; -/// Deterministic dev signing public key (standard base64), for `[signing_keys] 1 = …`. -const DEV_PUBKEY_B64: &str = "6kpsY+KcUgq+9VB7Ey7F+ZVHdq6+vnuSQh7qaRRG0iw="; -/// The matching dev `bai_v1` token (tenant 1 / vpc 1, kid 1). -const DEV_TOKEN: &str = "bai_v1.1.AQAAAAAAAAABAAAAAAAAAA.WrWcPbklu91PS-4WuR6GnBNF3h4nROpH0EQQlfJf06f7_lEnlQOCSBimhH2JMwXFJgw40BniTB7-yIdFnpldDw"; - -/// Locate the gateway binary (built beside the agent binary); build it on demand if absent. -fn gateway_bin() -> PathBuf { - let agent = PathBuf::from(env!("CARGO_BIN_EXE_beyond-ai-agent")); - let dir = agent.parent().unwrap(); - let gw = dir.join("beyond-ai"); - if !gw.exists() { - let mut args = vec!["build", "-q", "-p", "beyond-ai", "--bin", "beyond-ai"]; - if agent.to_string_lossy().contains("/release/") { - args.push("--release"); - } - let status = Command::new(env!("CARGO")) - .args(&args) - .status() - .expect("build gateway"); - assert!(status.success(), "failed to build the gateway binary"); - } - gw -} - #[test] fn agent_through_real_gateway_to_mock_upstream() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/agent/tests/smoke.rs b/crates/agent/tests/smoke.rs new file mode 100644 index 0000000..f4c0592 --- /dev/null +++ b/crates/agent/tests/smoke.rs @@ -0,0 +1,98 @@ +//! Live smoke: the real `beyond-ai-agent` binary → the real `beyond-ai` gateway → **real Anthropic**. +//! +//! Ignored by default (bills a tiny real request). Run with a key present: +//! +//! ANTHROPIC_API_KEY=sk-ant-… mise run test:smoke:agent +//! ANTHROPIC_API_KEY=sk-ant-… cargo test -p beyond-ai-agent --test smoke -- --ignored --nocapture +//! +//! This is the one test that validates the dialect decoder against a *real* provider's SSE: the +//! gateway boots with the caller's Anthropic key as the managed pool key (and the dev signing key), +//! then the agent — holding only a `bai_v1` virtual key — drives a real tool round-trip through it. +//! A model-not-found is a stale model id, not a harness bug (adjust `MODEL`). +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +mod common; + +use std::process::{Command, Stdio}; + +use common::{DEV_PUBKEY_B64, DEV_TOKEN, free_port, gateway_bin, wait_for_port}; + +/// Cheapest small Anthropic model (matches the gateway's own smoke test). +const MODEL: &str = "claude-haiku-4-5"; + +fn env_key(name: &str) -> Option { + std::env::var(name).ok().filter(|v| !v.trim().is_empty()) +} + +#[test] +#[ignore = "live provider smoke; run via `mise run test:smoke:agent` with ANTHROPIC_API_KEY set"] +fn smoke_agent_through_gateway_to_real_anthropic() { + let Some(anthropic_key) = env_key("ANTHROPIC_API_KEY") else { + eprintln!("smoke[agent]: ANTHROPIC_API_KEY unset — skipping"); + return; + }; + + // A file the agent must read with its `read` tool, then echo back — proves a live tool round-trip. + let dir = tempfile::tempdir().unwrap(); + let token = "PINEAPPLE-7493"; + std::fs::write(dir.path().join("marker.txt"), format!("{token}\n")).unwrap(); + + // Gateway → REAL Anthropic (no authority override, real TLS upstream). Managed pool key = the + // caller's real key; the agent presents only the dev virtual key. + let gw_port = free_port(); + let metrics_port = free_port(); + let config = format!( + "listen = \"127.0.0.1:{gw_port}\"\n\ + metrics_listen = \"127.0.0.1:{metrics_port}\"\n\ + nats_url = \"nats://127.0.0.1:59321\"\n\ + config_bucket = \"ai-gateway\"\n\ + upstream_tls = true\n\ + \n[pool_keys]\nanthropic = \"{anthropic_key}\"\n\ + \n[signing_keys]\n1 = \"{DEV_PUBKEY_B64}\"\n" + ); + let config_path = dir.path().join("gateway.toml"); + std::fs::write(&config_path, config).unwrap(); + + let mut gateway = Command::new(gateway_bin()) + .arg("run") + .arg("-c") + .arg(&config_path) + .env("AI_LOG", "warn") + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .expect("spawn gateway"); + wait_for_port(gw_port); + + let output = Command::new(env!("CARGO_BIN_EXE_beyond-ai-agent")) + .args([ + "run", + "Use the read tool to read the file marker.txt in the current directory, then reply with ONLY the exact token it contains.", + "--gateway-url", + &format!("http://127.0.0.1:{gw_port}"), + "--key", + DEV_TOKEN, + "--model", + MODEL, + "--max-steps", + "6", + ]) + .current_dir(dir.path()) + .output() + .expect("spawn agent"); + + let _ = gateway.kill(); + let _ = gateway.wait(); + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + eprintln!("--- agent stdout ---\n{stdout}\n--- agent stderr ---\n{stderr}"); + assert!( + output.status.success(), + "agent failed against real Anthropic.\nstdout: {stdout}\nstderr: {stderr}" + ); + assert!( + stdout.contains(token), + "real Claude should have read the file via the tool and echoed `{token}`.\nstdout: {stdout}" + ); +} diff --git a/mise.toml b/mise.toml index 463007e..50f0fdc 100644 --- a/mise.toml +++ b/mise.toml @@ -54,6 +54,13 @@ if [ -f .env ]; then set -a; . ./.env; set +a; fi cargo test -p beyond-ai --test smoke -- --ignored --nocapture """ +[tasks."test:smoke:agent"] +description = "Live agent smoke: the real agent binary → real gateway → REAL Anthropic. Auto-loads .env; needs ANTHROPIC_API_KEY. Skips if unset. Bills a tiny real request." +run = """ +if [ -f .env ]; then set -a; . ./.env; set +a; fi +cargo test -p beyond-ai-agent --test smoke -- --ignored --nocapture +""" + [tasks."bench:unit"] description = "divan micro-benchmarks of the IO-free hot paths (key/peek/usage/route/deny): timing + native allocation counts." run = "cargo bench --bench unit"