diff --git a/.env.example b/.env.example index 727672a7..26688dcc 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,17 @@ -OPENAI_API_KEY=sk-YOUR-API-KEY-HERE +# Instance auth (optional): set INSTANCE_AUTH=true to gate /services/* on the +# api_key callers send, looked up in lightning_clients via POSTGRES_URL. See +# services/_instance_auth/README.md. +# INSTANCE_AUTH=true + +# Optional at-rest encryption for stored client Anthropic keys. Base64 of 32 bytes +# (openssl rand -base64 32). See services/_instance_auth/README.md. +# APOLLO_ENC_KEY= + ANTHROPIC_API_KEY=sk-YOUR-API-KEY-HERE + +OPENAI_API_KEY=sk-YOUR-API-KEY-HERE PINECONE_KEY=YOUR-API-KEY-HERE -POSTGRES_URL=POSTGRES_URL=postgresql://localhost:5432/apollo_dev +POSTGRES_URL=postgresql://localhost:5432/apollo_dev SENTRY_DSN=YOUR-API-KEY-HERE GITHUB_TOKEN=KEY diff --git a/CLAUDE.md b/CLAUDE.md index 53210e4d..a8383861 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,6 +45,32 @@ TypeScript) service modules. - **Service discovery**: `platform/src/util/describe-modules.ts` - Auto-mounts any `services//` directory not starting with `_`. Detects service type by checking for `.py` (Python) or `.ts` (TypeScript) index file. +- **Instance auth** (`platform/src/middleware/auth.ts`): `/services/*` is gated + when the `INSTANCE_AUTH` env var is set (opt-in; otherwise open). The + credential is the `api_key` the caller (Lightning) already sends in the request + body — there is no bearer token and no Lightning-side change. Its SHA-256 is + looked up in the `lightning_clients` table via `POSTGRES_URL`; a missing or + unknown key is rejected with `401`. The inbound `api_key` is treated purely as + a credential and is **never** forwarded to the LLM: on a match it is replaced + with the matched client's stored `anthropic_api_key`, or stripped (falling back + to the global `ANTHROPIC_API_KEY`) when that column is `NULL`. The stored + `anthropic_api_key` may be plaintext or AES-256-GCM-encrypted (`enc:v1:` + values, decrypted with `APOLLO_ENC_KEY`; see + `platform/src/util/instance-key-crypto.ts` and + `services/_instance_auth/encrypt_key.ts`). Lookups are + cached in memory (~60s TTL), so the DB is hit at most once per minute per + process, not per request. The refresh is single-flight with + stale-while-revalidate, so a burst of requests at the TTL boundary shares one + DB read (cold start awaits it; a warm-but-stale cache is served while one + background refresh runs) rather than stampeding the DB. If auth is enabled but + the table can't be reached, the gate fails closed (rejects all external + callers). The gate is scoped to + `/services/*`, so health/root endpoints outside that group are unaffected. + Internal Apollo-to-Apollo `apollo()` calls are exempt via a per-process + internal token (`APOLLO_INTERNAL_TOKEN`, minted at startup, injected into the + env and echoed back by `services/util.py`); this replaces the old loopback + exemption so a co-located Lightning is still required to authenticate. + Provisioning lives in `services/_instance_auth/`. ### Services Architecture diff --git a/README.md b/README.md index 61424545..84a7ab2c 100644 --- a/README.md +++ b/README.md @@ -175,12 +175,56 @@ in your json, keep it inside a tmp dir and it'll remain safe and secret. The Apollo server uses bunjs with the Elysia framework. -It is a very lightweight server, with at the time of writing no authentication -or governance included. +It is a very lightweight server. By default it includes no authentication, but +instance auth can be enabled (see below). Python services are hosted at `/services/`. Each service expects a POST request with a JSON body, and will return JSON. +## Database + +First, make sure you've configured your desired `POSTGRES_URL` in your `.env` +file. + +### Create the DB + +Create a Postgres DB matching your POSTGRES_URL from the `.env` file + +### To reset the DB + +`set -a; . ./.env; set +a; psql "$POSTGRES_URL" -c "DROP TABLE IF EXISTS lightning_clients, adaptor_function_docs CASCADE;"` + +### Run the "migrations" (apply both schemas): + +`set -a; . ./.env; set +a; psql "$POSTGRES_URL" -f services/_instance_auth/schema.sql && psql "$POSTGRES_URL" -f services/load_adaptor_docs/schema.sql` + +### Instance authentication (optional) + +`/services/*` can be gated so that only known clients (e.g. specific Lightning +instances) may call it, with Apollo using **each client's own Anthropic API +key** for that client's requests. + +- It is **opt-in and backward compatible**: auth is active only when the + `INSTANCE_AUTH` environment variable is set (e.g. `INSTANCE_AUTH=true`). + Otherwise the server stays fully open as before. Tokens are looked up in the + `lightning_clients` table via `POSTGRES_URL`; if auth is enabled but that + table can't be reached, the gate **fails closed** (rejects all external + callers) rather than silently opening up. +- The credential is the **`api_key` the caller already sends in the request + body** — there is no bearer token, no `Authorization` header, and no + Lightning-side change. Apollo stores only a SHA-256 hash of it; an + unknown/missing key gets `401 { "code": 401, "type": "UNAUTHORIZED" }`. +- On a match, the inbound `api_key` is treated purely as a credential and is + **never** forwarded to the LLM: it is replaced with the client's stored + Anthropic key (so LLM usage bills to that client), or stripped — falling back + to the global `ANTHROPIC_API_KEY` — if the client has no stored key. +- Health/root endpoints (`/livez`, `/status`, `/`) are outside `/services/*` and + never gated. Internal Apollo-to-Apollo `apollo()` calls are exempt via a + per-process internal token (`APOLLO_INTERNAL_TOKEN`), not by network position. + +To enable it and provision clients, see +[`services/_instance_auth/`](services/_instance_auth/README.md). + There is very little standard for formality in the JSON structures to date. The server may soon establish some conventions for better interopability with the CLI. diff --git a/platform/src/middleware/auth.ts b/platform/src/middleware/auth.ts new file mode 100644 index 00000000..b6742d14 --- /dev/null +++ b/platform/src/middleware/auth.ts @@ -0,0 +1,308 @@ +import { SQL } from "bun"; +import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; +import type { ApolloError } from "../util/errors"; +import { ENC_PREFIX, decryptKey, parseEncKey } from "../util/instance-key-crypto"; + +type Client = { name: string; anthropicKey: string | null }; +type Lookup = (hash: string) => Promise | Client | null; + +// Opt-in via INSTANCE_AUTH (see initAuth); when off, /services/* is open as before. +let enabled = false; + +// False until the lightning_clients lookup is usable. Auth-on but unusable DB => +// the gate fails CLOSED (rejects every external caller) rather than opening up. +let dbReady = false; + +let lookupOverride: Lookup | null = null; +let sql: SQL | null = null; + +// In-memory client cache (keyed by token hash), refreshed on a TTL so DB changes +// are picked up within CACHE_TTL_MS. +const CACHE_TTL_MS = 60_000; +let clientCache: Map | null = null; +let cacheTs = 0; + +// Single-flight handle: concurrent refreshes share this one promise, so a refresh +// can never trigger more than one DB read no matter the concurrency. +let refreshInFlight: Promise> | null = null; + +// Master key for decrypting stored anthropic_api_key values (APOLLO_ENC_KEY). Null +// when unset — plaintext rows still work, but "enc:v1:" rows can't be decrypted. +let encKey: Buffer | null = null; + +let loaderOverride: (() => Promise>) | null = null; + +// Per-process secret identifying genuine Apollo-to-Apollo calls. The bridge spawns +// Python children that inherit this env, so services/util.py apollo() echoes it back +// via the internal header; authGate exempts requests carrying it without trusting +// network position. Honour an operator-provided value, else mint a random one. +// +// MULTI-PROCESS: when unset, each process mints its OWN token. apollo() self-calls +// hit 127.0.0.1:{port} and normally land on the same process, but if processes +// share a port (SO_REUSEPORT / clustering) a self-call can hit a sibling and 401. +// Set APOLLO_INTERNAL_TOKEN to the SAME value across processes in that case. +export const INTERNAL_HEADER = "x-apollo-internal"; +const INTERNAL_TOKEN = + process.env.APOLLO_INTERNAL_TOKEN ?? randomBytes(32).toString("hex"); +process.env.APOLLO_INTERNAL_TOKEN = INTERNAL_TOKEN; + +/** Constant-time string compare, length-guarded. */ +function safeEqual(a: string, b: string): boolean { + const ab = Buffer.from(a); + const bb = Buffer.from(b); + return ab.length === bb.length && timingSafeEqual(ab, bb); +} + +export function internalAuthHeader(): Record { + return { [INTERNAL_HEADER]: INTERNAL_TOKEN }; +} + +/** SHA-256 hex of a client credential. Must match services/_instance_auth/hash_token.py. */ +export function hashToken(token: string): string { + return createHash("sha256").update(token).digest("hex"); +} + +// Returned when an encrypted key can't be decrypted: that client is omitted from +// the cache (fail closed) rather than falling back to the global key, which would +// mis-bill its usage. +const DECRYPT_FAILED = Symbol("decrypt-failed"); + +// null => global env key; "enc:v1:…" => AES-256-GCM decrypt (DECRYPT_FAILED on +// error); anything else => legacy plaintext. +function decryptStoredKey( + stored: string | null, + clientName: string +): string | null | typeof DECRYPT_FAILED { + if (stored === null) return null; + if (!stored.startsWith(ENC_PREFIX)) return stored; + if (!encKey) { + console.error( + `Apollo instance auth: client "${clientName}" has an encrypted anthropic_api_key but APOLLO_ENC_KEY is unset/invalid — omitting this client (fail closed).` + ); + return DECRYPT_FAILED; + } + try { + return decryptKey(stored, encKey); + } catch (err) { + console.error( + `Apollo instance auth: could not decrypt anthropic_api_key for client "${clientName}" — omitting this client (fail closed).`, + err + ); + return DECRYPT_FAILED; + } +} + +/** Build the hash→client map, dropping any client whose encrypted key can't be + * decrypted (fail closed). Exported for tests. */ +export function buildClientMap( + rows: Array<{ + name: string; + auth_token_hash: string; + anthropic_api_key: string | null; + }> +): Map { + const map = new Map(); + for (const row of rows) { + const key = decryptStoredKey(row.anthropic_api_key, row.name); + if (key === DECRYPT_FAILED) continue; + map.set(row.auth_token_hash, { name: row.name, anthropicKey: key }); + } + return map; +} + +async function loadClients(): Promise> { + const rows = (await sql!` + SELECT name, auth_token_hash, anthropic_api_key FROM lightning_clients + `) as Array<{ + name: string; + auth_token_hash: string; + anthropic_api_key: string | null; + }>; + return buildClientMap(rows); +} + +// Single-flight refresh; the in-flight handle is cleared in finally so a failed +// refresh never wedges the cache (the next caller retries). +function refreshClients(): Promise> { + if (refreshInFlight) return refreshInFlight; + refreshInFlight = (loaderOverride ?? loadClients)() + .then((map) => { + clientCache = map; + cacheTs = Date.now(); + return map; + }) + .finally(() => { + refreshInFlight = null; + }); + return refreshInFlight; +} + +async function dbLookup(hash: string): Promise { + if (!dbReady) return null; // auth on but lookup never came up -> fail closed + if (!loaderOverride && !sql) return null; + + const fresh = clientCache !== null && Date.now() - cacheTs <= CACHE_TTL_MS; + if (!fresh) { + if (clientCache !== null) { + // Warm but stale: serve the current map now and refresh once in the + // background. Errors are swallowed so a DB blip keeps serving stale (cacheTs + // only advances on success, so the next request retries). + void refreshClients().catch(() => {}); + } else { + // Cold start: every concurrent caller awaits the one shared load. On failure + // leave the cache null and fail closed. + try { + await refreshClients(); + } catch { + return null; + } + } + } + return clientCache?.get(hash) ?? null; +} + +function authOptIn(): boolean { + const v = (process.env.INSTANCE_AUTH ?? "").trim().toLowerCase(); + return v === "1" || v === "true" || v === "yes" || v === "on"; +} + +// Decide once at startup whether auth is active. When INSTANCE_AUTH is set but the +// lightning_clients table can't be reached, auth stays on and the gate fails CLOSED +// — an explicit opt-in must never silently fall back to open. +export async function initAuth(): Promise { + enabled = authOptIn(); + if (!enabled) { + dbReady = false; + console.warn( + "Apollo instance auth DISABLED: INSTANCE_AUTH not set — /services/* is open to all callers." + ); + return; + } + + encKey = parseEncKey(process.env.APOLLO_ENC_KEY); + if (process.env.APOLLO_ENC_KEY && !encKey) { + console.error( + "Apollo instance auth: APOLLO_ENC_KEY is set but is not valid base64 of 32 bytes — encrypted client keys cannot be decrypted and those clients will be REJECTED." + ); + } + + const url = process.env.POSTGRES_URL; + if (!url) { + dbReady = false; + console.error( + "Apollo instance auth ENABLED but POSTGRES_URL is not set — clients cannot be looked up. /services/* will REJECT all external callers until this is fixed." + ); + return; + } + + try { + sql = new SQL(url); + const rows = (await sql` + SELECT to_regclass('public.lightning_clients') AS t + `) as Array<{ t: string | null }>; + if (rows?.[0]?.t) { + dbReady = true; + clientCache = null; // force a fresh load on the first request + console.log( + "Apollo instance auth ENABLED (INSTANCE_AUTH set; lightning_clients table present)." + ); + } else { + dbReady = false; + console.error( + "Apollo instance auth ENABLED but the lightning_clients table was not found — /services/* will REJECT all external callers. Run services/_instance_auth/schema.sql to provision it." + ); + } + } catch (err) { + dbReady = false; + console.error( + "Apollo instance auth ENABLED but the lightning_clients table could not be verified — /services/* will REJECT all external callers.", + err + ); + } +} + +function unauthorized(ctx: any): ApolloError { + ctx.set.status = 401; + return { code: 401, type: "UNAUTHORIZED", message: "Missing or invalid API key" }; +} + +// Elysia onBeforeHandle hook for the /services group. Returning a value +// short-circuits the request with that body; on success it stashes the resolved +// client on the context for apiKeyOverride. +export async function authGate(ctx: any): Promise { + if (!enabled) return; + + // Apollo calling itself: Python children echo back the internal token (see + // services/util.py apollo()), so such calls skip the api_key check. External + // callers can't forge it — it's a per-process secret never sent to clients. + const internal = ctx.request?.headers?.get?.(INTERNAL_HEADER) ?? ""; + if (internal && safeEqual(internal, INTERNAL_TOKEN)) { + // Flag so apiKeyOverride leaves the forwarded api_key untouched rather than + // stripping it to the global key — which would mis-bill a per-client key + // passed down an apollo() hop. + ctx.internalCall = true; + return; + } + + // The credential is the api_key the caller sends in the body; hash it and look + // for a matching client. Absent or unknown -> 401. + const apiKey = + typeof ctx.body?.api_key === "string" ? ctx.body.api_key.trim() : ""; + if (!apiKey) return unauthorized(ctx); + + const lookup = lookupOverride ?? dbLookup; + const client = await lookup(hashToken(apiKey)); + if (!client) return unauthorized(ctx); + + ctx.lightningClient = client; +} + +/** + * Resolve the api_key for the outgoing payload (merged LAST so it wins). Auth off, + * or an internal self-call: return {} (passthrough). Otherwise the inbound + * credential is NEVER forwarded to the LLM — it's replaced with the client's stored + * key, or undefined (dropped on serialise) so Apollo falls back to its global key. + */ +export function apiKeyOverride(ctx: any): { api_key?: string } { + if (!enabled || ctx?.internalCall) return {}; + const client = ctx?.lightningClient as Client | undefined; + return { api_key: client?.anthropicKey ?? undefined }; +} + +// --- Test seams --- + +export function __setAuthForTest(provider: Lookup | null): void { + if (provider) { + enabled = true; + lookupOverride = provider; + } else { + enabled = false; + lookupOverride = null; + } +} + +// Drives the real dbLookup (single-flight + stale-while-revalidate) with a fake +// loader instead of Postgres; null tears it back down. +export function __setLoaderForTest( + loader: (() => Promise>) | null +): void { + loaderOverride = loader; + clientCache = null; + cacheTs = 0; + refreshInFlight = null; + if (loader) { + enabled = true; + dbReady = true; + lookupOverride = null; + } else { + dbReady = false; + } +} + +export function __expireCacheForTest(): void { + cacheTs = 0; +} + +export function __setEncKeyForTest(key: Buffer | null): void { + encKey = key; +} diff --git a/platform/src/middleware/services.ts b/platform/src/middleware/services.ts index 125e425e..0dbc1b2b 100644 --- a/platform/src/middleware/services.ts +++ b/platform/src/middleware/services.ts @@ -7,6 +7,7 @@ import describeModules, { type ModuleDescription, } from "../util/describe-modules"; import { isApolloError } from "../util/errors"; +import { authGate, apiKeyOverride } from "./auth"; const callService = ( m: ModuleDescription, @@ -27,6 +28,10 @@ export default async (app: Elysia, port: number) => { console.log("Loading routes:"); const modules = await describeModules(path.resolve("./services")); app.group("/services", (app) => { + // Gate every /services/* route on a valid instance token (no-op when + // instance auth is disabled). + app.onBeforeHandle(authGate); + modules.forEach((m) => { const { name, readme } = m; console.log(" - mounted /services/" + name); @@ -37,6 +42,7 @@ export default async (app: Elysia, port: number) => { const payload = { ...(ctx.body ?? {}), session_id: ctx.uuid, + ...apiKeyOverride(ctx), }; const result = await callService(m, port, payload as any); @@ -58,6 +64,7 @@ export default async (app: Elysia, port: number) => { const payload = { ...(ctx.body ?? {}), session_id: ctx.uuid, + ...apiKeyOverride(ctx), }; const stream = new ReadableStream({ @@ -133,6 +140,10 @@ export default async (app: Elysia, port: number) => { // TODO in the web socket API, does it make more sense to open a socket at root // and then pick the service you want? So you'd connect to /ws an send { call: 'echo', payload: {} } app.ws(name, { + // Gate the WS upgrade too. It carries no body, so it has no api_key to + // validate; when auth is on, WS upgrades are rejected. Fine — Lightning + // uses the POST and /stream transports, where the credential lives. + beforeHandle: authGate, open() { console.log(`Websocket connected at /services/${name}`); }, diff --git a/platform/src/server.ts b/platform/src/server.ts index 0d334ef0..9bb47e89 100644 --- a/platform/src/server.ts +++ b/platform/src/server.ts @@ -5,6 +5,7 @@ import setupHealthcheck from "./middleware/healthcheck"; import setupServices from "./middleware/services"; import { html } from "@elysiajs/html"; import logRequest from "./util/log-request"; +import { initAuth } from "./middleware/auth"; import { randomUUID } from "node:crypto"; export default async (port: number | string = 3000) => { @@ -19,6 +20,8 @@ export default async (port: number | string = 3000) => { await setupDir(app); await setupServices(app, +port); + await initAuth(); + console.log("Apollo Server listening on ", port); app.listen(port); diff --git a/platform/src/util/instance-key-crypto.ts b/platform/src/util/instance-key-crypto.ts new file mode 100644 index 00000000..6cab7669 --- /dev/null +++ b/platform/src/util/instance-key-crypto.ts @@ -0,0 +1,46 @@ +// AES-256-GCM helpers for the per-client anthropic_api_key in lightning_clients. +// Shared by the auth middleware (decrypt) and encrypt_key.ts (encrypt) so the byte +// format can't drift. Stored format: "enc:v1:"; master key is APOLLO_ENC_KEY (base64 of 32 bytes). Values without +// the prefix are treated as legacy plaintext elsewhere, so encryption is opt-in. +import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; + +export const ENC_PREFIX = "enc:v1:"; +const IV_BYTES = 12; // GCM nonce +const TAG_BYTES = 16; // GCM auth tag + +/** Decode APOLLO_ENC_KEY (base64 of exactly 32 bytes) into a key Buffer, or null if absent/malformed. */ +export function parseEncKey(raw: string | undefined | null): Buffer | null { + if (!raw) return null; + let buf: Buffer; + try { + buf = Buffer.from(raw.trim(), "base64"); + } catch { + return null; + } + return buf.length === 32 ? buf : null; +} + +export function encryptKey(plaintext: string, key: Buffer): string { + const iv = randomBytes(IV_BYTES); + const cipher = createCipheriv("aes-256-gcm", key, iv); + const ciphertext = Buffer.concat([ + cipher.update(plaintext, "utf8"), + cipher.final(), + ]); + const tag = cipher.getAuthTag(); + return ENC_PREFIX + Buffer.concat([iv, tag, ciphertext]).toString("base64"); +} + +/** Decrypt an "enc:v1:…" value; throws on wrong key, corrupt value, or failed auth tag. */ +export function decryptKey(stored: string, key: Buffer): string { + const blob = Buffer.from(stored.slice(ENC_PREFIX.length), "base64"); + const iv = blob.subarray(0, IV_BYTES); + const tag = blob.subarray(IV_BYTES, IV_BYTES + TAG_BYTES); + const ciphertext = blob.subarray(IV_BYTES + TAG_BYTES); + const decipher = createDecipheriv("aes-256-gcm", key, iv); + decipher.setAuthTag(tag); + return Buffer.concat([decipher.update(ciphertext), decipher.final()]).toString( + "utf8" + ); +} diff --git a/platform/test/server.test.ts b/platform/test/server.test.ts index d6ec22bc..78268856 100644 --- a/platform/test/server.test.ts +++ b/platform/test/server.test.ts @@ -1,5 +1,17 @@ -import { describe, expect, it } from "bun:test"; +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { randomBytes } from "node:crypto"; import setup from "../src/server"; +import { + __expireCacheForTest, + __setAuthForTest, + __setEncKeyForTest, + __setLoaderForTest, + authGate, + buildClientMap, + hashToken, + internalAuthHeader, +} from "../src/middleware/auth"; +import { encryptKey } from "../src/util/instance-key-crypto"; const port = 9865; @@ -7,6 +19,11 @@ const baseUrl = `http://localhost:${port}`; const app = await setup(port); +// setup() runs initAuth(), which would enable auth if the dev's .env sets +// INSTANCE_AUTH. Force it off so the suite is deterministic; the auth block below +// opts back in via the test seam. +__setAuthForTest(null); + const get = (path: string) => { return new Request(`${baseUrl}/${path}`); }; @@ -124,9 +141,271 @@ describe("Python Services", () => { ); expect(response.status).toBe(200); - + const body = await response.json(); expect(body).toEqual({ success: true }); }); }); }); + +describe("Instance authentication", () => { + // No real DB — the seam keys clients by SHA-256 of the api_key they send. ALPHA + // has a stored Anthropic key (swapped in); BETA has none (credential stripped). + const ALPHA = "lightning-cred-alpha"; + const BETA = "lightning-cred-beta"; + const clients: Record = { + [hashToken(ALPHA)]: { name: "alpha", anthropicKey: "sk-ant-stored-alpha" }, + [hashToken(BETA)]: { name: "beta", anthropicKey: null }, + }; + + const postKey = (path: string, data: any, apiKey?: string) => + post(path, { ...data, ...(apiKey ? { api_key: apiKey } : {}) }); + + afterEach(() => { + __setAuthForTest(null); + }); + + it("stays open when auth is disabled", async () => { + __setAuthForTest(null); + const res = await app.handle(post("services/echo", { x: 1 })); + expect(res.status).toBe(200); + }); + + describe("when enabled", () => { + beforeEach(() => { + __setAuthForTest((hash) => clients[hash] ?? null); + }); + + it("accepts a known credential and swaps in the client's stored key", async () => { + const res = await app.handle(postKey("services/echo", { x: 1 }, ALPHA)); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.x).toBe(1); + expect(body.api_key).toBe("sk-ant-stored-alpha"); + expect(body.api_key).not.toBe(ALPHA); + }); + + it("strips the credential when the client has no stored key", async () => { + const res = await app.handle(postKey("services/echo", { x: 2 }, BETA)); + expect(res.status).toBe(200); + const body = await res.json(); + // No stored key → api_key dropped entirely (Apollo uses its global key). + expect(body.api_key).toBeUndefined(); + }); + + it("rejects a missing api_key with 401 UNAUTHORIZED", async () => { + const res = await app.handle(post("services/echo", { x: 1 })); + expect(res.status).toBe(401); + const body = await res.json(); + expect(body.code).toBe(401); + expect(body.type).toBe("UNAUTHORIZED"); + }); + + it("rejects an unknown api_key with 401", async () => { + const res = await app.handle(postKey("services/echo", { x: 1 }, "sk-ant-nope")); + expect(res.status).toBe(401); + }); + + it("leaves health and root endpoints open", async () => { + expect((await app.handle(get("livez"))).status).toBe(200); + expect((await app.handle(get(""))).status).toBe(200); + }); + + it("exempts internal apollo() self-calls carrying the internal token", async () => { + const req = new Request(`${baseUrl}/services/echo`, { + method: "POST", + body: JSON.stringify({ x: 9 }), + headers: { "Content-Type": "application/json", ...internalAuthHeader() }, + }); + const res = await app.handle(req); + expect(res.status).toBe(200); + }); + + it("still rejects a request bearing a bogus internal token", async () => { + const req = new Request(`${baseUrl}/services/echo`, { + method: "POST", + body: JSON.stringify({ x: 9 }), + headers: { "Content-Type": "application/json", "x-apollo-internal": "nope" }, + }); + const res = await app.handle(req); + expect(res.status).toBe(401); + }); + + it("passes a forwarded api_key through on internal self-calls untouched", async () => { + // Already authenticated upstream, so a forwarded api_key must survive into + // the payload rather than being stripped to the global key. + const req = new Request(`${baseUrl}/services/echo`, { + method: "POST", + body: JSON.stringify({ x: 9, api_key: "sk-ant-forwarded" }), + headers: { "Content-Type": "application/json", ...internalAuthHeader() }, + }); + const res = await app.handle(req); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.api_key).toBe("sk-ant-forwarded"); + }); + + it("rejects an unauthenticated WebSocket upgrade", async () => { + // The WS upgrade carries no body, so no api_key to validate; the gate must + // reject it rather than let it bypass auth. + const req = new Request(`${baseUrl}/services/echo`, { + method: "GET", + headers: { + Connection: "Upgrade", + Upgrade: "websocket", + "Sec-WebSocket-Key": "dGhlIHNhbXBsZSBub25jZQ==", + "Sec-WebSocket-Version": "13", + }, + }); + const res = await app.handle(req); + expect(res.status).toBe(401); + }); + }); +}); + +describe("Instance auth cache refresh", () => { + // Drive the real dbLookup with a fake loader so we can count DB reads per burst. + // authGate is called directly with a minimal ctx (no echo service spawned). + const ALPHA = "lightning-cred-alpha"; + const mapWith = (anthropicKey: string | null) => + new Map([[hashToken(ALPHA), { name: "alpha", anthropicKey }]]); + const fakeCtx = (apiKey?: string) => + ({ + request: { headers: { get: () => null } }, + body: apiKey ? { api_key: apiKey } : {}, + set: { status: 200 }, + }) as any; + const tick = () => new Promise((r) => setTimeout(r, 10)); + const settle = () => new Promise((r) => setTimeout(r, 40)); + + afterEach(() => { + __setLoaderForTest(null); + __setAuthForTest(null); + }); + + it("collapses a cold-start burst into a single DB read", async () => { + let calls = 0; + __setLoaderForTest(async () => { + calls++; + await tick(); + return mapWith("sk-ant-stored-alpha"); + }); + + const ctxs = Array.from({ length: 50 }, () => fakeCtx(ALPHA)); + await Promise.all(ctxs.map((c) => authGate(c))); + + expect(calls).toBe(1); + for (const c of ctxs) { + expect(c.lightningClient?.anthropicKey).toBe("sk-ant-stored-alpha"); + } + }); + + it("serves the stale list while one background refresh runs", async () => { + let calls = 0; + let current = mapWith("sk-ant-v1"); + __setLoaderForTest(async () => { + calls++; + await tick(); + return current; + }); + + // Cold start awaits the one load and warms the cache with v1. + const warm = fakeCtx(ALPHA); + await authGate(warm); + expect(calls).toBe(1); + expect(warm.lightningClient?.anthropicKey).toBe("sk-ant-v1"); + + // New data lands in the DB; mark the cache stale. + current = mapWith("sk-ant-v2"); + __expireCacheForTest(); + + // The burst is served immediately from the stale v1 map and triggers + // exactly one background refresh (not one per request). + const ctxs = Array.from({ length: 25 }, () => fakeCtx(ALPHA)); + await Promise.all(ctxs.map((c) => authGate(c))); + expect(calls).toBe(2); + for (const c of ctxs) { + expect(c.lightningClient?.anthropicKey).toBe("sk-ant-v1"); + } + + // Once the background refresh settles, the new value is visible — with no + // further DB reads. + await settle(); + const after = fakeCtx(ALPHA); + await authGate(after); + expect(after.lightningClient?.anthropicKey).toBe("sk-ant-v2"); + expect(calls).toBe(2); + }); + + it("keeps serving stale when the refresh fails, then recovers", async () => { + let calls = 0; + let fail = false; + __setLoaderForTest(async () => { + calls++; + if (fail) throw new Error("db down"); + return mapWith("sk-ant-v1"); + }); + + const warm = fakeCtx(ALPHA); + await authGate(warm); + expect(warm.lightningClient?.anthropicKey).toBe("sk-ant-v1"); + + // Refresh now fails; stale callers are still authenticated (no 500/empty). + fail = true; + __expireCacheForTest(); + const ctxs = Array.from({ length: 10 }, () => fakeCtx(ALPHA)); + await Promise.all(ctxs.map((c) => authGate(c))); + for (const c of ctxs) { + expect(c.lightningClient?.anthropicKey).toBe("sk-ant-v1"); + } + + // Recover once the DB is back. + fail = false; + __expireCacheForTest(); + const ok = fakeCtx(ALPHA); + await authGate(ok); + expect(ok.lightningClient?.anthropicKey).toBe("sk-ant-v1"); + }); +}); + +describe("Instance auth key encryption", () => { + afterEach(() => __setEncKeyForTest(null)); + + it("round-trips encrypted, plaintext, and null keys through buildClientMap", () => { + const key = randomBytes(32); + __setEncKeyForTest(key); + const enc = encryptKey("sk-ant-secret", key); + + const map = buildClientMap([ + { name: "enc", auth_token_hash: "h-enc", anthropic_api_key: enc }, + { name: "plain", auth_token_hash: "h-plain", anthropic_api_key: "sk-ant-plain" }, + { name: "none", auth_token_hash: "h-none", anthropic_api_key: null }, + ]); + + expect(map.get("h-enc")?.anthropicKey).toBe("sk-ant-secret"); + expect(map.get("h-plain")?.anthropicKey).toBe("sk-ant-plain"); + expect(map.get("h-none")?.anthropicKey).toBeNull(); + }); + + it("omits a client whose encrypted key can't be decrypted (wrong key)", () => { + const enc = encryptKey("sk-ant-secret", randomBytes(32)); // encrypted with key A + __setEncKeyForTest(randomBytes(32)); // server holds a different key + + const map = buildClientMap([ + { name: "bad", auth_token_hash: "h-bad", anthropic_api_key: enc }, + ]); + + expect(map.has("h-bad")).toBe(false); + }); + + it("omits an encrypted key when APOLLO_ENC_KEY is not configured", () => { + __setEncKeyForTest(null); + const enc = encryptKey("sk-ant-secret", randomBytes(32)); + + const map = buildClientMap([ + { name: "bad", auth_token_hash: "h-bad", anthropic_api_key: enc }, + ]); + + expect(map.has("h-bad")).toBe(false); + }); +}); diff --git a/services/_instance_auth/README.md b/services/_instance_auth/README.md new file mode 100644 index 00000000..860f573a --- /dev/null +++ b/services/_instance_auth/README.md @@ -0,0 +1,117 @@ +# Instance auth + +Gates Apollo's `/services/*` endpoints so that only known Lightning instances can +call them, and makes Apollo use **its own per-client Anthropic API key** for each +request rather than trusting anything the caller sends. + +This directory is not a mounted service (the leading `_` keeps it off the HTTP +router). It holds the schema and provisioning scripts; the actual gate lives in +`platform/src/middleware/auth.ts`. + +## How it works + +- The credential is the **`api_key` the caller already sends in the request + body** — the same field Lightning sends today. There is no bearer token, no + `Authorization` header, and **no change required on the Lightning side**. +- A single Postgres table, `lightning_clients`, is the allow-list. Each row has a + `name`, the **SHA-256 hash** of that client's `api_key` (never the plaintext), + and an optional `anthropic_api_key`. +- On every `/services/*` request the server reads `api_key` from the body, + hashes it, and looks for a matching row. Missing or no match → `401 + UNAUTHORIZED`. +- The inbound `api_key` is treated **purely as a credential and is never + forwarded to the LLM**. On a match it is replaced with the client's stored + `anthropic_api_key`, so all LLM usage for that request bills to the key Apollo + controls. If the column is `NULL`, the inbound key is **stripped** and Apollo + falls back to its global `ANTHROPIC_API_KEY`. Either way the caller's key + cannot pass through. +- **Performance:** the client list is cached in memory and refreshed on a ~60s + TTL, so the database is queried at most once per minute per process, never on + the per-request path to Anthropic. The per-request cost is a hash plus a map + lookup. The trade-off is that revocations/rotations take up to ~60s to take + effect (see Managing clients). +- **Opt-in / backward compatible:** auth is active only when the `INSTANCE_AUTH` + environment variable is set (e.g. `INSTANCE_AUTH=true`). Otherwise the server + stays open exactly as before and the caller's `api_key` passes through + untouched. When auth is enabled but this table can't be reached, the gate + **fails closed** (every external caller gets `401`) rather than silently + opening up. +- The health endpoints (`/livez`, `/status`, `/`) sit outside `/services/*` and + are never gated. Internal Apollo-to-Apollo `apollo()` calls are exempt via a + per-process internal token (`APOLLO_INTERNAL_TOKEN`), not by network position. + +## Setting up a client + +`provision_client.ts` is the canonical way to add a Lightning client: it mints the +credential, hashes it, and encrypts the client's Anthropic key in one step. + +1. Create the table (once): + + ```sh + set -a; . ./.env; set +a + psql "$POSTGRES_URL" -f services/_instance_auth/schema.sql + ``` + +2. Set a master encryption key in `.env` (once) — `provision_client.ts` uses it to + encrypt each client's Anthropic key at rest: + + ```sh + echo "APOLLO_ENC_KEY=$(openssl rand -base64 32)" >> .env + ``` + +3. Provision the client with a name and the Anthropic key Apollo should use for it: + + ```sh + set -a; . ./.env; set +a + bun services/_instance_auth/provision_client.ts + ``` + + This prints the `api_key` to give the Lightning instance and a ready-to-run + `psql` `INSERT`. Run that `INSERT` to add the row. + +4. Enable auth: set `INSTANCE_AUTH=true` in Apollo's environment and restart. The + startup log shows `Apollo instance auth ENABLED`. (If `INSTANCE_AUTH` is set but + the table is missing, the log warns and every external caller is rejected.) + +5. Give the printed `api_key` to the Lightning instance. It keeps sending it as + `api_key` exactly as it does today — no other Lightning-side change. + +## Managing clients + +Manage rows directly in the DB. Changes are picked up within ~60s (the server +caches the client list briefly); restart Apollo to apply a revocation immediately. + +- **Revoke:** `DELETE FROM lightning_clients WHERE name = '...';` +- **Rotate a credential / change the Anthropic key:** re-run `provision_client.ts` + and apply its output as an `UPDATE` for the existing `name` (it prints an + `INSERT` — swap the verb). + +### Lower-level scripts + +`provision_client.ts` is built from two primitives, occasionally useful on their +own — e.g. to add a client whose `anthropic_api_key` is `NULL` (so it uses Apollo's +global `ANTHROPIC_API_KEY`), which `provision_client.ts` doesn't cover: + +- `hash_token.py ` — hash (or, with no argument, mint) just the credential. +- `encrypt_key.ts ` — encrypt just an Anthropic key to an `enc:v1:…` + value. + +## At-rest encryption + +`anthropic_api_key` is stored encrypted (AES-256-GCM) when written via +`provision_client.ts`/`encrypt_key.ts`; plaintext rows are still accepted for +backward compatibility. + +- **Fail closed.** If an `enc:v1:` row can't be decrypted (wrong/missing + `APOLLO_ENC_KEY` or corrupt value), that client is dropped from the allow-list + and its requests get `401` — Apollo never falls back to the global key for an + encrypted-but-undecryptable row. A `NULL` key still means "use the global key". +- **Rotation** is manual: re-encrypt every `enc:v1:` row with the new key, then + swap `APOLLO_ENC_KEY` and restart. +- **What it protects.** The ciphertext is useless without `APOLLO_ENC_KEY`, so this + guards DB dumps, backups, read replicas, and accidental `SELECT`s in logs. It + does **not** protect a full Apollo host/process compromise: the running process + necessarily holds both the key and the decrypted values in memory. Protect the + table at rest (restricted access, DB encryption) regardless. + +The clients' `api_key` credentials are only ever stored and compared as hashes. diff --git a/services/_instance_auth/encrypt_key.ts b/services/_instance_auth/encrypt_key.ts new file mode 100644 index 00000000..82db7ac2 --- /dev/null +++ b/services/_instance_auth/encrypt_key.ts @@ -0,0 +1,37 @@ +// Encrypt an Anthropic API key for lightning_clients.anthropic_api_key. Run from +// the repo root so Bun auto-loads .env (where APOLLO_ENC_KEY lives): +// bun services/_instance_auth/encrypt_key.ts +// Prints the "enc:v1:…" value to store plus a ready-to-edit INSERT. +import { encryptKey, parseEncKey } from "../../platform/src/util/instance-key-crypto"; + +const plaintext = process.argv[2]; +if (!plaintext) { + console.error( + "Usage: bun services/_instance_auth/encrypt_key.ts \n" + + "(run from the repo root so APOLLO_ENC_KEY is read from .env)" + ); + process.exit(1); +} + +const key = parseEncKey(process.env.APOLLO_ENC_KEY); +if (!key) { + console.error( + "APOLLO_ENC_KEY not found (needs to be base64 of exactly 32 bytes — it\n" + + "encrypts the Anthropic key).\n\n" + + "Run this from the repo root so Bun picks up .env. If it's still missing,\n" + + "the key isn't in .env yet — add one with:\n\n" + + ' echo "APOLLO_ENC_KEY=$(openssl rand -base64 32)" >> .env\n\n' + + "then re-run (and restart Apollo so it can decrypt at runtime)." + ); + process.exit(1); +} + +const enc = encryptKey(plaintext, key); + +console.log(enc); +console.log(); +console.log("Example INSERT (set name and the hash from hash_token.py):\n"); +console.log( + " INSERT INTO lightning_clients (name, auth_token_hash, anthropic_api_key)\n" + + ` VALUES ('my-instance', '', '${enc}');` +); diff --git a/services/_instance_auth/hash_token.py b/services/_instance_auth/hash_token.py new file mode 100644 index 00000000..69a81767 --- /dev/null +++ b/services/_instance_auth/hash_token.py @@ -0,0 +1,47 @@ +"""Hash (or mint) a Lightning client's api_key credential for lightning_clients. + + # Hash the api_key a Lightning instance already sends: + poetry run python services/_instance_auth/hash_token.py + + # Or mint a fresh credential + print its hash and a ready-to-edit INSERT: + poetry run python services/_instance_auth/hash_token.py + +Only the SHA-256 hash is stored. It must match the hash Apollo computes in +platform/src/middleware/auth.ts (sha256 over UTF-8 bytes, lowercase hex). +""" + +import hashlib +import secrets +import sys + + +def hash_token(token: str) -> str: + return hashlib.sha256(token.encode()).hexdigest() + + +def main() -> None: + if len(sys.argv) > 1: + token = sys.argv[1] + generated = False + else: + token = secrets.token_urlsafe(32) + generated = True + + digest = hash_token(token) + + lines = [] + if generated: + lines.append("Generated a new client credential.\n") + lines.append(f" api_key (the instance sends this): {token}") + lines.append(f" auth_token_hash (store in the DB): {digest}\n") + lines.append("Example INSERT (set name and the client's Anthropic key):\n") + lines.append( + " INSERT INTO lightning_clients (name, auth_token_hash, anthropic_api_key)\n" + f" VALUES ('my-lightning-instance', '{digest}', 'sk-ant-...');", + ) + + print("\n".join(lines)) # noqa: T201 + + +if __name__ == "__main__": + main() diff --git a/services/_instance_auth/provision_client.ts b/services/_instance_auth/provision_client.ts new file mode 100644 index 00000000..3d51d495 --- /dev/null +++ b/services/_instance_auth/provision_client.ts @@ -0,0 +1,54 @@ +// Provision a Lightning client in one step: mint an api_key credential, hash it +// (auth_token_hash), and encrypt the Anthropic key (enc:v1:…). Prints all three +// plus a ready-to-run INSERT. Run from the repo root so Bun loads .env (APOLLO_ENC_KEY): +// bun services/_instance_auth/provision_client.ts +import { createHash, randomBytes } from "node:crypto"; +import { encryptKey, parseEncKey } from "../../platform/src/util/instance-key-crypto"; + +const name = process.argv[2]; +const anthropicKey = process.argv[3]; + +if (!name || !anthropicKey) { + console.error( + "Usage: APOLLO_ENC_KEY= \\\n" + + " bun services/_instance_auth/provision_client.ts \n\n" + + "Tip: `set -a; . ./.env; set +a;` first to load APOLLO_ENC_KEY from .env." + ); + process.exit(1); +} + +const encKey = parseEncKey(process.env.APOLLO_ENC_KEY); +if (!encKey) { + console.error( + "APOLLO_ENC_KEY not found (needs to be base64 of exactly 32 bytes — it\n" + + "encrypts the Anthropic key).\n\n" + + "Run this from the repo root so Bun picks up .env. If it's still missing,\n" + + "the key isn't in .env yet — add one with:\n\n" + + ' echo "APOLLO_ENC_KEY=$(openssl rand -base64 32)" >> .env\n\n' + + "then re-run (and restart Apollo so it can decrypt at runtime)." + ); + process.exit(1); +} + +const apiKey = randomBytes(32).toString("base64url"); +const authTokenHash = createHash("sha256").update(apiKey).digest("hex"); +const encAnthropic = encryptKey(anthropicKey, encKey); + +const sqlName = name.replace(/'/g, "''"); +const insert = + "INSERT INTO lightning_clients (name, auth_token_hash, anthropic_api_key)\n" + + ` VALUES ('${sqlName}', '${authTokenHash}', '${encAnthropic}');`; + +console.log(`\n✅ Provisioned client "${name}"\n`); +console.log("1. Lightning api_key — give this to the instance (it sends it as `api_key`):"); +console.log(` ${apiKey}\n`); +console.log("2. auth_token_hash — stored in the DB:"); +console.log(` ${authTokenHash}\n`); +console.log("3. anthropic_api_key (encrypted) — stored in the DB:"); +console.log(` ${encAnthropic}\n`); +console.log("Run this to insert the row (from the repo root):\n"); +console.log( + ` set -a; . ./.env; set +a; psql "$POSTGRES_URL" -c "${insert.replace(/\n\s*/g, " ")}"\n` +); +console.log("Or paste the INSERT directly:\n"); +console.log(` ${insert}\n`); diff --git a/services/_instance_auth/schema.sql b/services/_instance_auth/schema.sql new file mode 100644 index 00000000..eaf9b5f8 --- /dev/null +++ b/services/_instance_auth/schema.sql @@ -0,0 +1,12 @@ +-- Instance auth allow-list of Lightning clients permitted to call Apollo. Opt-in +-- via INSTANCE_AUTH; when set, /services/* requests are gated on the api_key the +-- caller sends, looked up here by hash. Rows are managed by hand. See README.md. + +CREATE TABLE IF NOT EXISTS lightning_clients ( + id SERIAL PRIMARY KEY, + name TEXT UNIQUE NOT NULL, -- human identifier for the client + auth_token_hash VARCHAR(64) UNIQUE NOT NULL, -- sha256 hex of the client's api_key credential (never the plaintext) + anthropic_api_key TEXT -- Anthropic key Apollo uses for this client; NULL => global env key. + -- Plaintext, OR an "enc:v1:..." value from encrypt_key.ts when + -- APOLLO_ENC_KEY is set (see README.md). Both are accepted. +); diff --git a/services/util.py b/services/util.py index bdce2d24..96fc2b75 100644 --- a/services/util.py +++ b/services/util.py @@ -96,7 +96,14 @@ def apollo(name: str, payload: dict) -> dict: :return: JSON response. """ url = f"http://127.0.0.1:{apollo_port}/services/{name}" - r = requests.post(url, json=payload) + # Mark internal Apollo-to-Apollo calls so they bypass instance auth (see + # platform/src/middleware/auth.ts). The server injects the token into the env + # and this child inherits it; absent (auth off) the header is omitted. + headers = {} + internal_token = os.environ.get("APOLLO_INTERNAL_TOKEN") + if internal_token: + headers["X-Apollo-Internal"] = internal_token + r = requests.post(url, json=payload, headers=headers) return r.json()