From ff2d091fa00cb2e6a5efb9384d99476bd7ff8896 Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Wed, 17 Jun 2026 18:51:37 +0200 Subject: [PATCH 01/10] concept --- .env.example | 5 +- CLAUDE.md | 5 + README.md | 25 ++++- platform/src/middleware/auth.ts | 153 ++++++++++++++++++++++++++ platform/src/middleware/services.ts | 12 ++ platform/src/server.ts | 4 + platform/test/server.test.ts | 80 +++++++++++++- services/_instance_auth/README.md | 72 ++++++++++++ services/_instance_auth/hash_token.py | 51 +++++++++ services/_instance_auth/schema.sql | 16 +++ 10 files changed, 418 insertions(+), 5 deletions(-) create mode 100644 platform/src/middleware/auth.ts create mode 100644 services/_instance_auth/README.md create mode 100644 services/_instance_auth/hash_token.py create mode 100644 services/_instance_auth/schema.sql diff --git a/.env.example b/.env.example index 727672a7..25cb8b06 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,10 @@ OPENAI_API_KEY=sk-YOUR-API-KEY-HERE ANTHROPIC_API_KEY=sk-YOUR-API-KEY-HERE PINECONE_KEY=YOUR-API-KEY-HERE -POSTGRES_URL=POSTGRES_URL=postgresql://localhost:5432/apollo_dev +# Setting POSTGRES_URL and creating the lightning_clients table (see +# services/_instance_auth/) enables instance auth: /services/* then requires a +# bearer token. No new var is needed to turn it on. +POSTGRES_URL=postgresql://localhost:5432/apollo_dev SENTRY_DSN=YOUR-API-KEY-HERE GITHUB_TOKEN=KEY diff --git a/CLAUDE.md b/CLAUDE.md index 53210e4d..3c90b9cd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,6 +45,11 @@ TypeScript) service modules. - **Service discovery**: `platform/src/util/describe-modules.ts` - Auto-mounts any `services//` directory not starting with `_`. Detects service type by checking for `.py` (Python) or `.ts` (TypeScript) index file. +- **Instance auth** (`platform/src/middleware/auth.ts`): `/services/*` is gated by + a bearer token only when `POSTGRES_URL` is set AND the `lightning_clients` table + exists (opt-in; otherwise open). A matched client's stored Anthropic key is + injected into the payload as `api_key`. Health endpoints and loopback/internal + `apollo()` calls are exempt. Provisioning lives in `services/_instance_auth/`. ### Services Architecture diff --git a/README.md b/README.md index 61424545..4bb1e007 100644 --- a/README.md +++ b/README.md @@ -175,12 +175,33 @@ in your json, keep it inside a tmp dir and it'll remain safe and secret. The Apollo server uses bunjs with the Elysia framework. -It is a very lightweight server, with at the time of writing no authentication -or governance included. +It is a very lightweight server. By default it includes no authentication, but +instance auth can be enabled (see below). Python services are hosted at `/services/`. Each service expects a POST request with a JSON body, and will return JSON. +### Instance authentication (optional) + +`/services/*` can be gated so that only known clients (e.g. specific Lightning +instances) may call it, with Apollo using **each client's own Anthropic API key** +for that client's requests. + +- It is **opt-in and backward compatible**: auth is active only when + `POSTGRES_URL` is set **and** the `lightning_clients` table exists. Otherwise + the server stays fully open as before. +- Clients authenticate with `Authorization: Bearer `. Apollo stores only a + SHA-256 hash of the token; an unknown/missing token gets + `401 { "code": 401, "type": "UNAUTHORIZED" }`. +- On a match, the client's stored Anthropic key is injected into the request, so + LLM usage bills to that client (falls back to the global `ANTHROPIC_API_KEY` if + the client has no key). +- Health/root endpoints (`/livez`, `/status`, `/`) and loopback/internal + service-to-service calls are exempt. + +To enable it and provision clients, see +[`services/_instance_auth/`](services/_instance_auth/README.md). + There is very little standard for formality in the JSON structures to date. The server may soon establish some conventions for better interopability with the CLI. diff --git a/platform/src/middleware/auth.ts b/platform/src/middleware/auth.ts new file mode 100644 index 00000000..3fd17192 --- /dev/null +++ b/platform/src/middleware/auth.ts @@ -0,0 +1,153 @@ +import { SQL } from "bun"; +import { createHash } from "node:crypto"; +import type { ApolloError } from "../util/errors"; + +// A row from the lightning_clients table, as used at runtime. +type Client = { name: string; anthropicKey: string | null }; + +// A lookup resolves a token hash to a client, or null if unknown. +type Lookup = (hash: string) => Promise | Client | null; + +// Instance auth is opt-in: enabled only when POSTGRES_URL is set AND the +// lightning_clients table exists (see initAuth). With it disabled, /services/* +// is open exactly as before. +let enabled = false; + +// When set (by tests), this replaces the DB-backed lookup so the suite can +// enable auth with a known token set without a real Postgres. +let lookupOverride: Lookup | null = null; + +// Bun's native Postgres client (Bun >= 1.2). Created once in initAuth(). +let sql: SQL | null = null; + +// In-memory cache of all clients keyed by token hash, refreshed on a TTL so +// rows added/revoked directly in the DB are picked up within CACHE_TTL_MS. +const CACHE_TTL_MS = 60_000; +let clientCache: Map | null = null; +let cacheTs = 0; + +const LOOPBACK = new Set(["127.0.0.1", "::1", "::ffff:127.0.0.1"]); + +/** SHA-256 hex of a bearer token. Must match services/_instance_auth/hash_token.py. */ +export function hashToken(token: string): string { + return createHash("sha256").update(token).digest("hex"); +} + +async function loadClients(): Promise> { + const rows = (await sql!` + SELECT name, auth_token_hash, anthropic_api_key FROM lightning_clients + `) as Array<{ + name: string; + auth_token_hash: string; + anthropic_api_key: string | null; + }>; + const map = new Map(); + for (const row of rows) { + map.set(row.auth_token_hash, { + name: row.name, + anthropicKey: row.anthropic_api_key, + }); + } + return map; +} + +async function dbLookup(hash: string): Promise { + const now = Date.now(); + if (!clientCache || now - cacheTs > CACHE_TTL_MS) { + clientCache = await loadClients(); + cacheTs = now; + } + return clientCache.get(hash) ?? null; +} + +/** + * Decide once at startup whether instance auth is active. Enabled iff + * POSTGRES_URL is set and the lightning_clients table exists. Any failure + * leaves auth disabled (fail-open) so a misconfigured DB never bricks Apollo. + */ +export async function initAuth(): Promise { + const url = process.env.POSTGRES_URL; + if (!url) { + enabled = false; + console.warn( + "Apollo instance auth DISABLED: POSTGRES_URL not set — /services/* is open to all callers." + ); + return; + } + try { + sql = new SQL(url); + const rows = (await sql` + SELECT to_regclass('public.lightning_clients') AS t + `) as Array<{ t: string | null }>; + if (rows?.[0]?.t) { + enabled = true; + clientCache = null; // force a fresh load on the first request + console.log( + "Apollo instance auth ENABLED (lightning_clients table present)." + ); + } else { + enabled = false; + console.warn( + "Apollo instance auth DISABLED: lightning_clients table not found — /services/* is open. Run services/_instance_auth/schema.sql to enable." + ); + } + } catch (err) { + enabled = false; + console.warn( + "Apollo instance auth DISABLED: could not verify lightning_clients table — /services/* is open.", + err + ); + } +} + +function unauthorized(ctx: any): ApolloError { + ctx.set.status = 401; + return { code: 401, type: "UNAUTHORIZED", message: "Missing or invalid API token" }; +} + +/** + * Elysia onBeforeHandle hook scoped to the /services group. Returning a value + * short-circuits the request with that value as the response body. + * On success it stashes the resolved client on the context for apiKeyOverride. + */ +export async function authGate(ctx: any): Promise { + if (!enabled) return; + + // Exempt loopback callers — internal service-to-service apollo() calls hit + // 127.0.0.1. Synthetic requests (app.handle) have no peer address, so the + // gate is enforced for them (which is what the test suite relies on). + const address = ctx.server?.requestIP?.(ctx.request)?.address; + if (address && LOOPBACK.has(address)) return; + + const header = ctx.request?.headers?.get?.("authorization") ?? ""; + const token = header.startsWith("Bearer ") ? header.slice(7).trim() : ""; + if (!token) return unauthorized(ctx); + + const lookup = lookupOverride ?? dbLookup; + const client = await lookup(hashToken(token)); + if (!client) return unauthorized(ctx); + + ctx.lightningClient = client; +} + +/** + * Returns an { api_key } override to merge into a service payload so Apollo + * uses the client's own Anthropic key. Returns {} when auth is off or the + * client has no stored key, leaving the payload untouched (Python then falls + * back to the global ANTHROPIC_API_KEY). + */ +export function apiKeyOverride(ctx: any): { api_key?: string } { + const client = ctx?.lightningClient as Client | undefined; + return client?.anthropicKey ? { api_key: client.anthropicKey } : {}; +} + +/** Test seam: pass a fake lookup to enable auth without a DB, or null to disable. */ +export function __setAuthForTest(provider: Lookup | null): void { + if (provider) { + enabled = true; + lookupOverride = provider; + } else { + enabled = false; + lookupOverride = null; + } +} diff --git a/platform/src/middleware/services.ts b/platform/src/middleware/services.ts index 125e425e..7c12256c 100644 --- a/platform/src/middleware/services.ts +++ b/platform/src/middleware/services.ts @@ -7,6 +7,7 @@ import describeModules, { type ModuleDescription, } from "../util/describe-modules"; import { isApolloError } from "../util/errors"; +import { authGate, apiKeyOverride } from "./auth"; const callService = ( m: ModuleDescription, @@ -27,6 +28,10 @@ export default async (app: Elysia, port: number) => { console.log("Loading routes:"); const modules = await describeModules(path.resolve("./services")); app.group("/services", (app) => { + // Gate every /services/* route on a valid instance token (no-op when + // instance auth is disabled). Loopback/internal calls are exempted. + app.onBeforeHandle(authGate); + modules.forEach((m) => { const { name, readme } = m; console.log(" - mounted /services/" + name); @@ -37,6 +42,7 @@ export default async (app: Elysia, port: number) => { const payload = { ...(ctx.body ?? {}), session_id: ctx.uuid, + ...apiKeyOverride(ctx), }; const result = await callService(m, port, payload as any); @@ -58,6 +64,7 @@ export default async (app: Elysia, port: number) => { const payload = { ...(ctx.body ?? {}), session_id: ctx.uuid, + ...apiKeyOverride(ctx), }; const stream = new ReadableStream({ @@ -133,6 +140,11 @@ export default async (app: Elysia, port: number) => { // TODO in the web socket API, does it make more sense to open a socket at root // and then pick the service you want? So you'd connect to /ws an send { call: 'echo', payload: {} } app.ws(name, { + // Gate the WS upgrade too, so it can't be used to bypass auth. Note: + // per-client Anthropic key injection is only wired into the POST and + // /stream paths (the chat transports Lightning uses); WS payloads pass + // through as-is. + beforeHandle: authGate, open() { console.log(`Websocket connected at /services/${name}`); }, diff --git a/platform/src/server.ts b/platform/src/server.ts index 0d334ef0..cbd82476 100644 --- a/platform/src/server.ts +++ b/platform/src/server.ts @@ -5,6 +5,7 @@ import setupHealthcheck from "./middleware/healthcheck"; import setupServices from "./middleware/services"; import { html } from "@elysiajs/html"; import logRequest from "./util/log-request"; +import { initAuth } from "./middleware/auth"; import { randomUUID } from "node:crypto"; export default async (port: number | string = 3000) => { @@ -19,6 +20,9 @@ export default async (port: number | string = 3000) => { await setupDir(app); await setupServices(app, +port); + // Decide whether /services/* is gated by an instance token (see auth.ts). + await initAuth(); + console.log("Apollo Server listening on ", port); app.listen(port); diff --git a/platform/test/server.test.ts b/platform/test/server.test.ts index d6ec22bc..00c212d7 100644 --- a/platform/test/server.test.ts +++ b/platform/test/server.test.ts @@ -1,5 +1,6 @@ -import { describe, expect, it } from "bun:test"; +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; import setup from "../src/server"; +import { __setAuthForTest, hashToken } from "../src/middleware/auth"; const port = 9865; @@ -7,6 +8,11 @@ const baseUrl = `http://localhost:${port}`; const app = await setup(port); +// setup() runs initAuth(), which may enable auth if the dev's .env points at a +// DB with the lightning_clients table. Force auth off so the suite is +// deterministic; the auth block below opts back in via the test seam. +__setAuthForTest(null); + const get = (path: string) => { return new Request(`${baseUrl}/${path}`); }; @@ -124,9 +130,79 @@ describe("Python Services", () => { ); expect(response.status).toBe(200); - + const body = await response.json(); expect(body).toEqual({ success: true }); }); }); }); + +describe("Instance authentication", () => { + // Two known clients, keyed by token hash (no real DB — this is the seam). + const ALPHA = "alpha-token"; + const BETA = "beta-token"; + const clients: Record = { + [hashToken(ALPHA)]: { name: "alpha", anthropicKey: "sk-ant-alpha" }, + [hashToken(BETA)]: { name: "beta", anthropicKey: "sk-ant-beta" }, + }; + + const postWith = (path: string, data: any, token?: string) => + new Request(`${baseUrl}/${path}`, { + method: "POST", + body: JSON.stringify(data), + headers: { + "Content-Type": "application/json", + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }, + }); + + afterEach(() => { + // Restore the open state the rest of the suite expects. + __setAuthForTest(null); + }); + + it("stays open when auth is disabled", async () => { + __setAuthForTest(null); + const res = await app.handle(post("services/echo", { x: 1 })); + expect(res.status).toBe(200); + }); + + describe("when enabled", () => { + beforeEach(() => { + __setAuthForTest((hash) => clients[hash] ?? null); + }); + + it("allows a valid token and injects the client's anthropic key", async () => { + const res = await app.handle(postWith("services/echo", { x: 1 }, ALPHA)); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.x).toBe(1); + expect(body.api_key).toBe("sk-ant-alpha"); + }); + + it("allows a second client's token", async () => { + const res = await app.handle(postWith("services/echo", { x: 2 }, BETA)); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.api_key).toBe("sk-ant-beta"); + }); + + it("rejects a missing token with 401 UNAUTHORIZED", async () => { + const res = await app.handle(post("services/echo", { x: 1 })); + expect(res.status).toBe(401); + const body = await res.json(); + expect(body.code).toBe(401); + expect(body.type).toBe("UNAUTHORIZED"); + }); + + it("rejects a wrong token with 401", async () => { + const res = await app.handle(postWith("services/echo", { x: 1 }, "nope")); + expect(res.status).toBe(401); + }); + + it("leaves health and root endpoints open", async () => { + expect((await app.handle(get("livez"))).status).toBe(200); + expect((await app.handle(get(""))).status).toBe(200); + }); + }); +}); diff --git a/services/_instance_auth/README.md b/services/_instance_auth/README.md new file mode 100644 index 00000000..6e3f6cd3 --- /dev/null +++ b/services/_instance_auth/README.md @@ -0,0 +1,72 @@ +# Instance auth + +Gates Apollo's `/services/*` endpoints so that only known Lightning instances can +call them, and makes Apollo use **each client's own Anthropic API key** for that +client's requests. + +This directory is not a mounted service (the leading `_` keeps it off the HTTP +router). It is just the schema and a helper script; the actual gate lives in +`platform/src/middleware/auth.ts`. + +## How it works + +- A single Postgres table, `lightning_clients`, lists the allowed clients. Each row + has a `name`, the **SHA-256 hash** of its bearer token (never the plaintext), and + an optional `anthropic_api_key`. +- On every `/services/*` request the server reads `Authorization: Bearer `, + hashes it, and looks for a matching row. No match → `401 UNAUTHORIZED`. +- On a match, the client's `anthropic_api_key` (if set) is injected into the + request payload as `api_key`, so all LLM usage for that request bills to the + client. If the column is `NULL`, Apollo falls back to its global + `ANTHROPIC_API_KEY`. +- **Opt-in / backward compatible:** auth is active only when `POSTGRES_URL` is set + *and* this table exists. Otherwise the server stays open exactly as before. +- Loopback callers (`127.0.0.1`/`::1`) and the health endpoints (`/livez`, + `/status`, `/`) are exempt. + +## Enabling it + +1. Make sure `POSTGRES_URL` is set, then create the table: + + ```sh + psql "$POSTGRES_URL" -f services/_instance_auth/schema.sql + ``` + +2. Mint a token for a Lightning instance and get its hash: + + ```sh + poetry run python services/_instance_auth/hash_token.py + ``` + + This prints a plaintext token (give it to the Lightning instance), the hash to + store, and a ready-to-edit `INSERT`. + +3. Insert the row (set `name` and the client's Anthropic key): + + ```sql + INSERT INTO lightning_clients (name, auth_token_hash, anthropic_api_key) + VALUES ('my-lightning-instance', '', 'sk-ant-...'); + ``` + +4. Configure that Lightning instance to send `Authorization: Bearer ` on + its Apollo requests. + +5. Restart Apollo. The startup log shows `Apollo instance auth ENABLED`. + +## Managing clients + +There is no CLI — manage rows directly in the DB: + +- **Revoke a client:** `DELETE FROM lightning_clients WHERE name = '...';` +- **Rotate a token:** mint a new one with `hash_token.py` and + `UPDATE lightning_clients SET auth_token_hash = '' WHERE name = '...';` +- **Change a client's key:** + `UPDATE lightning_clients SET anthropic_api_key = '...' WHERE name = '...';` + +Changes are picked up within ~60s (the server caches the client list briefly). + +## Note + +The `anthropic_api_key` is stored as plaintext because Apollo must be able to use +it. Protect this table at rest (DB encryption, restricted access) accordingly. The +bearer tokens themselves are only ever stored as hashes. diff --git a/services/_instance_auth/hash_token.py b/services/_instance_auth/hash_token.py new file mode 100644 index 00000000..e817fb01 --- /dev/null +++ b/services/_instance_auth/hash_token.py @@ -0,0 +1,51 @@ +"""Mint and/or hash a Lightning client bearer token for the lightning_clients table. + +Usage (run from the repo root): + + # Generate a fresh token and print its hash + a ready-to-edit INSERT: + poetry run python services/_instance_auth/hash_token.py + + # Hash a token you already have: + poetry run python services/_instance_auth/hash_token.py + +The plaintext token goes into the Lightning instance config as +`Authorization: Bearer `. Only the SHA-256 hash is stored in the DB. The +hash here must match the one Apollo computes in platform/src/middleware/auth.ts +(sha256 over the UTF-8 bytes, lowercase hex) -- it does. +""" + +import hashlib +import secrets +import sys + + +def hash_token(token: str) -> str: + return hashlib.sha256(token.encode()).hexdigest() + + +def main() -> None: + if len(sys.argv) > 1: + token = sys.argv[1] + generated = False + else: + token = secrets.token_urlsafe(32) + generated = True + + digest = hash_token(token) + + lines = [] + if generated: + lines.append("Generated a new client token.\n") + lines.append(f" token (give to the Lightning instance): {token}") + lines.append(f" auth_token_hash (store in the DB): {digest}\n") + lines.append("Example INSERT (set name and the client's Anthropic key):\n") + lines.append( + " INSERT INTO lightning_clients (name, auth_token_hash, anthropic_api_key)\n" + f" VALUES ('my-lightning-instance', '{digest}', 'sk-ant-...');", + ) + + print("\n".join(lines)) # noqa: T201 + + +if __name__ == "__main__": + main() diff --git a/services/_instance_auth/schema.sql b/services/_instance_auth/schema.sql new file mode 100644 index 00000000..42f06ff1 --- /dev/null +++ b/services/_instance_auth/schema.sql @@ -0,0 +1,16 @@ +-- Instance auth: the allow-list of Lightning clients permitted to call Apollo. +-- +-- Creating this table is the opt-in for instance auth. When it exists (and +-- POSTGRES_URL is set), the Apollo server gates every /services/* request on a +-- valid bearer token. Without it, the server stays open as before. +-- +-- Rows are managed by hand. Use hash_token.py to mint a token + its hash, then +-- INSERT a row and configure the Lightning instance with the plaintext token as +-- `Authorization: Bearer `. See README.md. + +CREATE TABLE IF NOT EXISTS lightning_clients ( + id SERIAL PRIMARY KEY, + name TEXT UNIQUE NOT NULL, -- human identifier for the client + auth_token_hash VARCHAR(64) UNIQUE NOT NULL, -- sha256 hex of the bearer token (never the plaintext) + anthropic_api_key TEXT -- Anthropic key Apollo uses for this client; NULL => global env key +); From 9c08d9a7a0cfaf4b163803162f8b94313d6da441 Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Wed, 17 Jun 2026 21:42:44 +0200 Subject: [PATCH 02/10] instance auth --- .env.example | 9 +++-- CLAUDE.md | 8 ++-- README.md | 9 +++-- platform/src/middleware/auth.ts | 60 +++++++++++++++++++++--------- platform/test/server.test.ts | 6 +-- services/_instance_auth/README.md | 11 ++++-- services/_instance_auth/schema.sql | 9 +++-- 7 files changed, 77 insertions(+), 35 deletions(-) diff --git a/.env.example b/.env.example index 25cb8b06..8515a74f 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,16 @@ OPENAI_API_KEY=sk-YOUR-API-KEY-HERE ANTHROPIC_API_KEY=sk-YOUR-API-KEY-HERE PINECONE_KEY=YOUR-API-KEY-HERE -# Setting POSTGRES_URL and creating the lightning_clients table (see -# services/_instance_auth/) enables instance auth: /services/* then requires a -# bearer token. No new var is needed to turn it on. POSTGRES_URL=postgresql://localhost:5432/apollo_dev SENTRY_DSN=YOUR-API-KEY-HERE GITHUB_TOKEN=KEY +# Instance auth (optional): set INSTANCE_AUTH=true to require a bearer token on +# /services/*. When enabled, Apollo looks tokens up in the lightning_clients +# table via POSTGRES_URL (see services/_instance_auth/). Leave unset/false to +# keep the server open as before. +INSTANCE_AUTH=false + # Langfuse observability LANGFUSE_SECRET_KEY=sk-lf-... LANGFUSE_PUBLIC_KEY=pk-lf-... diff --git a/CLAUDE.md b/CLAUDE.md index 3c90b9cd..967a8e7c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -46,9 +46,11 @@ TypeScript) service modules. any `services//` directory not starting with `_`. Detects service type by checking for `.py` (Python) or `.ts` (TypeScript) index file. - **Instance auth** (`platform/src/middleware/auth.ts`): `/services/*` is gated by - a bearer token only when `POSTGRES_URL` is set AND the `lightning_clients` table - exists (opt-in; otherwise open). A matched client's stored Anthropic key is - injected into the payload as `api_key`. Health endpoints and loopback/internal + a bearer token when the `INSTANCE_AUTH` env var is set (opt-in; otherwise open). + Tokens are looked up in the `lightning_clients` table via `POSTGRES_URL`; a + matched client's stored Anthropic key is injected into the payload as `api_key`. + If auth is enabled but the table can't be reached, the gate fails closed + (rejects all external callers). Health endpoints and loopback/internal `apollo()` calls are exempt. Provisioning lives in `services/_instance_auth/`. ### Services Architecture diff --git a/README.md b/README.md index 4bb1e007..cf510673 100644 --- a/README.md +++ b/README.md @@ -187,9 +187,12 @@ request with a JSON body, and will return JSON. instances) may call it, with Apollo using **each client's own Anthropic API key** for that client's requests. -- It is **opt-in and backward compatible**: auth is active only when - `POSTGRES_URL` is set **and** the `lightning_clients` table exists. Otherwise - the server stays fully open as before. +- It is **opt-in and backward compatible**: auth is active only when the + `INSTANCE_AUTH` environment variable is set (e.g. `INSTANCE_AUTH=true`). + Otherwise the server stays fully open as before. Tokens are looked up in the + `lightning_clients` table via `POSTGRES_URL`; if auth is enabled but that table + can't be reached, the gate **fails closed** (rejects all external callers) + rather than silently opening up. - Clients authenticate with `Authorization: Bearer `. Apollo stores only a SHA-256 hash of the token; an unknown/missing token gets `401 { "code": 401, "type": "UNAUTHORIZED" }`. diff --git a/platform/src/middleware/auth.ts b/platform/src/middleware/auth.ts index 3fd17192..dae1bbc2 100644 --- a/platform/src/middleware/auth.ts +++ b/platform/src/middleware/auth.ts @@ -8,11 +8,16 @@ type Client = { name: string; anthropicKey: string | null }; // A lookup resolves a token hash to a client, or null if unknown. type Lookup = (hash: string) => Promise | Client | null; -// Instance auth is opt-in: enabled only when POSTGRES_URL is set AND the -// lightning_clients table exists (see initAuth). With it disabled, /services/* -// is open exactly as before. +// Instance auth is opt-in via the INSTANCE_AUTH env var (see initAuth). When it +// is unset/falsey, /services/* is open exactly as before. When set, every +// /services/* request requires a valid bearer token. let enabled = false; +// True once the lightning_clients lookup is actually usable. When auth is +// enabled but the DB/table can't be reached, this stays false and the gate +// fails CLOSED (rejects every external caller) rather than silently opening up. +let dbReady = false; + // When set (by tests), this replaces the DB-backed lookup so the suite can // enable auth with a known token set without a real Postgres. let lookupOverride: Lookup | null = null; @@ -52,6 +57,8 @@ async function loadClients(): Promise> { } async function dbLookup(hash: string): Promise { + // Auth is on but the lookup never came up — reject (fail closed). + if (!dbReady || !sql) return null; const now = Date.now(); if (!clientCache || now - cacheTs > CACHE_TTL_MS) { clientCache = await loadClients(); @@ -60,41 +67,60 @@ async function dbLookup(hash: string): Promise { return clientCache.get(hash) ?? null; } +/** Whether the INSTANCE_AUTH env var opts this instance into auth. */ +function authOptIn(): boolean { + const v = (process.env.INSTANCE_AUTH ?? "").trim().toLowerCase(); + return v === "1" || v === "true" || v === "yes" || v === "on"; +} + /** - * Decide once at startup whether instance auth is active. Enabled iff - * POSTGRES_URL is set and the lightning_clients table exists. Any failure - * leaves auth disabled (fail-open) so a misconfigured DB never bricks Apollo. + * Decide once at startup whether instance auth is active. The INSTANCE_AUTH env + * var is the master switch: unset/falsey leaves /services/* open as before. + * When set, auth is enabled and tokens are looked up in the lightning_clients + * table (via POSTGRES_URL). If that table can't be reached, auth stays enabled + * but the gate fails CLOSED (rejects every external caller) — an explicit + * opt-in must never silently fall back to open. */ export async function initAuth(): Promise { + enabled = authOptIn(); + if (!enabled) { + dbReady = false; + console.warn( + "Apollo instance auth DISABLED: INSTANCE_AUTH not set — /services/* is open to all callers." + ); + return; + } + const url = process.env.POSTGRES_URL; if (!url) { - enabled = false; - console.warn( - "Apollo instance auth DISABLED: POSTGRES_URL not set — /services/* is open to all callers." + dbReady = false; + console.error( + "Apollo instance auth ENABLED but POSTGRES_URL is not set — clients cannot be looked up. /services/* will REJECT all external callers until this is fixed." ); return; } + try { sql = new SQL(url); const rows = (await sql` SELECT to_regclass('public.lightning_clients') AS t `) as Array<{ t: string | null }>; if (rows?.[0]?.t) { - enabled = true; + dbReady = true; clientCache = null; // force a fresh load on the first request console.log( - "Apollo instance auth ENABLED (lightning_clients table present)." + "Apollo instance auth ENABLED (INSTANCE_AUTH set; lightning_clients table present)." ); } else { - enabled = false; - console.warn( - "Apollo instance auth DISABLED: lightning_clients table not found — /services/* is open. Run services/_instance_auth/schema.sql to enable." + dbReady = false; + console.error( + "Apollo instance auth ENABLED but the lightning_clients table was not found — /services/* will REJECT all external callers. Run services/_instance_auth/schema.sql to provision it." ); } } catch (err) { - enabled = false; - console.warn( - "Apollo instance auth DISABLED: could not verify lightning_clients table — /services/* is open.", + dbReady = false; + console.error( + "Apollo instance auth ENABLED but the lightning_clients table could not be verified — /services/* will REJECT all external callers.", err ); } diff --git a/platform/test/server.test.ts b/platform/test/server.test.ts index 00c212d7..61d54d1b 100644 --- a/platform/test/server.test.ts +++ b/platform/test/server.test.ts @@ -8,9 +8,9 @@ const baseUrl = `http://localhost:${port}`; const app = await setup(port); -// setup() runs initAuth(), which may enable auth if the dev's .env points at a -// DB with the lightning_clients table. Force auth off so the suite is -// deterministic; the auth block below opts back in via the test seam. +// setup() runs initAuth(), which enables auth if the dev's .env sets +// INSTANCE_AUTH. Force auth off so the suite is deterministic; the auth block +// below opts back in via the test seam. __setAuthForTest(null); const get = (path: string) => { diff --git a/services/_instance_auth/README.md b/services/_instance_auth/README.md index 6e3f6cd3..8627d604 100644 --- a/services/_instance_auth/README.md +++ b/services/_instance_auth/README.md @@ -19,8 +19,11 @@ router). It is just the schema and a helper script; the actual gate lives in request payload as `api_key`, so all LLM usage for that request bills to the client. If the column is `NULL`, Apollo falls back to its global `ANTHROPIC_API_KEY`. -- **Opt-in / backward compatible:** auth is active only when `POSTGRES_URL` is set - *and* this table exists. Otherwise the server stays open exactly as before. +- **Opt-in / backward compatible:** auth is active only when the `INSTANCE_AUTH` + environment variable is set (e.g. `INSTANCE_AUTH=true`). Otherwise the server + stays open exactly as before. When auth is enabled but this table can't be + reached, the gate **fails closed** (every external caller gets `401`) rather + than silently opening up. - Loopback callers (`127.0.0.1`/`::1`) and the health endpoints (`/livez`, `/status`, `/`) are exempt. @@ -51,7 +54,9 @@ router). It is just the schema and a helper script; the actual gate lives in 4. Configure that Lightning instance to send `Authorization: Bearer ` on its Apollo requests. -5. Restart Apollo. The startup log shows `Apollo instance auth ENABLED`. +5. Set `INSTANCE_AUTH=true` in Apollo's environment and restart. The startup log + shows `Apollo instance auth ENABLED`. (If `INSTANCE_AUTH` is set but the table + is missing, the log warns and every external caller is rejected.) ## Managing clients diff --git a/services/_instance_auth/schema.sql b/services/_instance_auth/schema.sql index 42f06ff1..bdbd48c3 100644 --- a/services/_instance_auth/schema.sql +++ b/services/_instance_auth/schema.sql @@ -1,8 +1,11 @@ -- Instance auth: the allow-list of Lightning clients permitted to call Apollo. -- --- Creating this table is the opt-in for instance auth. When it exists (and --- POSTGRES_URL is set), the Apollo server gates every /services/* request on a --- valid bearer token. Without it, the server stays open as before. +-- Instance auth is opt-in via the INSTANCE_AUTH env var. When it is set, the +-- Apollo server gates every /services/* request on a valid bearer token and +-- looks tokens up in this table (via POSTGRES_URL). This table must exist for +-- auth to work — if INSTANCE_AUTH is set but the table is missing, the gate +-- fails closed and rejects every external caller. Without INSTANCE_AUTH, the +-- server stays open as before. -- -- Rows are managed by hand. Use hash_token.py to mint a token + its hash, then -- INSERT a row and configure the Lightning instance with the plaintext token as From de51e0869189c313f50d8f7b5e1cbbcb11eddcf7 Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Thu, 18 Jun 2026 09:57:45 +0200 Subject: [PATCH 03/10] fix --- .env.example | 12 ++- CLAUDE.md | 25 ++++-- platform/src/middleware/auth.ts | 125 ++++++++++++++++++++++---- platform/src/middleware/services.ts | 11 +-- platform/test/server.test.ts | 72 ++++++++++----- services/_instance_auth/README.md | 77 +++++++++------- services/_instance_auth/hash_token.py | 22 ++--- services/_instance_auth/schema.sql | 18 ++-- services/util.py | 10 ++- 9 files changed, 262 insertions(+), 110 deletions(-) diff --git a/.env.example b/.env.example index 8515a74f..82d70d85 100644 --- a/.env.example +++ b/.env.example @@ -5,11 +5,15 @@ POSTGRES_URL=postgresql://localhost:5432/apollo_dev SENTRY_DSN=YOUR-API-KEY-HERE GITHUB_TOKEN=KEY -# Instance auth (optional): set INSTANCE_AUTH=true to require a bearer token on -# /services/*. When enabled, Apollo looks tokens up in the lightning_clients -# table via POSTGRES_URL (see services/_instance_auth/). Leave unset/false to -# keep the server open as before. +# Instance auth (optional): set INSTANCE_AUTH=true to require that callers send a +# known api_key on /services/*. When enabled, Apollo hashes the request's api_key +# and looks it up in the lightning_clients table via POSTGRES_URL (see +# services/_instance_auth/). Leave unset/false to keep the server open as before. INSTANCE_AUTH=false +# Debug aid: set APOLLO_AUTH_DEBUG=true to log the shape (method, path, peer IP, +# redacted headers, body keys) of every /services/* request. Secrets are never +# printed raw; the api_key is shown as its sha256 hash. Leave off normally. +# APOLLO_AUTH_DEBUG=false # Langfuse observability LANGFUSE_SECRET_KEY=sk-lf-... diff --git a/CLAUDE.md b/CLAUDE.md index 967a8e7c..679a836d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,13 +45,24 @@ TypeScript) service modules. - **Service discovery**: `platform/src/util/describe-modules.ts` - Auto-mounts any `services//` directory not starting with `_`. Detects service type by checking for `.py` (Python) or `.ts` (TypeScript) index file. -- **Instance auth** (`platform/src/middleware/auth.ts`): `/services/*` is gated by - a bearer token when the `INSTANCE_AUTH` env var is set (opt-in; otherwise open). - Tokens are looked up in the `lightning_clients` table via `POSTGRES_URL`; a - matched client's stored Anthropic key is injected into the payload as `api_key`. - If auth is enabled but the table can't be reached, the gate fails closed - (rejects all external callers). Health endpoints and loopback/internal - `apollo()` calls are exempt. Provisioning lives in `services/_instance_auth/`. +- **Instance auth** (`platform/src/middleware/auth.ts`): `/services/*` is gated + when the `INSTANCE_AUTH` env var is set (opt-in; otherwise open). The + credential is the `api_key` the caller (Lightning) already sends in the request + body — there is no bearer token and no Lightning-side change. Its SHA-256 is + looked up in the `lightning_clients` table via `POSTGRES_URL`; a missing or + unknown key is rejected with `401`. The inbound `api_key` is treated purely as + a credential and is **never** forwarded to the LLM: on a match it is replaced + with the matched client's stored `anthropic_api_key`, or stripped (falling back + to the global `ANTHROPIC_API_KEY`) when that column is `NULL`. Lookups are + cached in memory (~60s TTL), so the DB is hit at most once per minute per + process, not per request. If auth is enabled but the table can't be reached, + the gate fails closed (rejects all external callers). The gate is scoped to + `/services/*`, so health/root endpoints outside that group are unaffected. + Internal Apollo-to-Apollo `apollo()` calls are exempt via a per-process + internal token (`APOLLO_INTERNAL_TOKEN`, minted at startup, injected into the + env and echoed back by `services/util.py`); this replaces the old loopback + exemption so a co-located Lightning is still required to authenticate. + Provisioning lives in `services/_instance_auth/`. ### Services Architecture diff --git a/platform/src/middleware/auth.ts b/platform/src/middleware/auth.ts index dae1bbc2..85a7edc6 100644 --- a/platform/src/middleware/auth.ts +++ b/platform/src/middleware/auth.ts @@ -1,5 +1,5 @@ import { SQL } from "bun"; -import { createHash } from "node:crypto"; +import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; import type { ApolloError } from "../util/errors"; // A row from the lightning_clients table, as used at runtime. @@ -10,7 +10,8 @@ type Lookup = (hash: string) => Promise | Client | null; // Instance auth is opt-in via the INSTANCE_AUTH env var (see initAuth). When it // is unset/falsey, /services/* is open exactly as before. When set, every -// /services/* request requires a valid bearer token. +// /services/* request must carry an api_key (in the body) that matches a known +// client. let enabled = false; // True once the lightning_clients lookup is actually usable. When auth is @@ -31,9 +32,31 @@ const CACHE_TTL_MS = 60_000; let clientCache: Map | null = null; let cacheTs = 0; -const LOOPBACK = new Set(["127.0.0.1", "::1", "::ffff:127.0.0.1"]); +// A per-process secret that identifies genuine Apollo-to-Apollo calls. The +// bridge spawns Python services as child processes that inherit this process's +// env, so exporting the token here lets services/util.py apollo() echo it back +// via the X-Apollo-Internal header. authGate exempts any request carrying it, +// so "Apollo calling itself" skips auth without trusting network position (the +// old loopback exemption trusted any local caller, including Lightning). +// Honour an operator-provided value; otherwise mint a fresh random one. +export const INTERNAL_HEADER = "x-apollo-internal"; +const INTERNAL_TOKEN = + process.env.APOLLO_INTERNAL_TOKEN ?? randomBytes(32).toString("hex"); +process.env.APOLLO_INTERNAL_TOKEN = INTERNAL_TOKEN; -/** SHA-256 hex of a bearer token. Must match services/_instance_auth/hash_token.py. */ +/** Constant-time string compare, length-guarded. */ +function safeEqual(a: string, b: string): boolean { + const ab = Buffer.from(a); + const bb = Buffer.from(b); + return ab.length === bb.length && timingSafeEqual(ab, bb); +} + +/** Header set marking a request as an internal apollo() self-call (used by tests). */ +export function internalAuthHeader(): Record { + return { [INTERNAL_HEADER]: INTERNAL_TOKEN }; +} + +/** SHA-256 hex of a client credential (the api_key). Must match services/_instance_auth/hash_token.py. */ export function hashToken(token: string): string { return createHash("sha256").update(token).digest("hex"); } @@ -73,6 +96,55 @@ function authOptIn(): boolean { return v === "1" || v === "true" || v === "yes" || v === "on"; } +/** Whether APOLLO_AUTH_DEBUG opts into per-request logging (read live each call). */ +function debugEnabled(): boolean { + const v = (process.env.APOLLO_AUTH_DEBUG ?? "").trim().toLowerCase(); + return v === "1" || v === "true" || v === "yes" || v === "on"; +} + +/** + * Opt-in logging of the shape of every /services/* request, for debugging what + * callers (e.g. Lightning) actually send. Enable with APOLLO_AUTH_DEBUG=true. + * Secrets are never printed raw: the inbound api_key (the credential) is shown + * as its SHA-256 hash — exactly the value lightning_clients.auth_token_hash + * stores, so you can paste it straight into the table to allow-list a client. + * The internal token is shown only as present/absent and body values are + * omitted (just the top-level key names). Wrapped so it can never break a + * request if something is unexpectedly shaped. + */ +function debugLogRequest(ctx: any): void { + if (!debugEnabled()) return; + try { + const req = ctx.request; + const ip = ctx.server?.requestIP?.(req)?.address ?? "(no peer address)"; + const headers: Record = {}; + req?.headers?.forEach?.((value: string, key: string) => { + if (key === "authorization") { + const token = value.startsWith("Bearer ") ? value.slice(7).trim() : ""; + headers[key] = token ? `Bearer ` : ""; + } else if (key === INTERNAL_HEADER) { + headers[key] = ""; + } else { + headers[key] = value; + } + }); + const bodyKeys = + ctx.body && typeof ctx.body === "object" ? Object.keys(ctx.body) : []; + const rawKey = + typeof ctx.body?.api_key === "string" ? ctx.body.api_key.trim() : ""; + const apiKeyHash = rawKey ? `sha256:${hashToken(rawKey)}` : "(none)"; + const pathname = req?.url ? new URL(req.url).pathname : "(no url)"; + console.log( + `[auth-debug] ${req?.method} ${pathname} from ${ip} | auth ${enabled ? "ON" : "OFF"}\n` + + `[auth-debug] headers: ${JSON.stringify(headers)}\n` + + `[auth-debug] body keys: ${JSON.stringify(bodyKeys)}\n` + + `[auth-debug] api_key (credential): ${apiKeyHash}` + ); + } catch (err) { + console.log("[auth-debug] failed to log request:", err); + } +} + /** * Decide once at startup whether instance auth is active. The INSTANCE_AUTH env * var is the master switch: unset/falsey leaves /services/* open as before. @@ -128,7 +200,7 @@ export async function initAuth(): Promise { function unauthorized(ctx: any): ApolloError { ctx.set.status = 401; - return { code: 401, type: "UNAUTHORIZED", message: "Missing or invalid API token" }; + return { code: 401, type: "UNAUTHORIZED", message: "Missing or invalid API key" }; } /** @@ -137,34 +209,49 @@ function unauthorized(ctx: any): ApolloError { * On success it stashes the resolved client on the context for apiKeyOverride. */ export async function authGate(ctx: any): Promise { + debugLogRequest(ctx); if (!enabled) return; - // Exempt loopback callers — internal service-to-service apollo() calls hit - // 127.0.0.1. Synthetic requests (app.handle) have no peer address, so the - // gate is enforced for them (which is what the test suite relies on). - const address = ctx.server?.requestIP?.(ctx.request)?.address; - if (address && LOOPBACK.has(address)) return; + // Apollo calling itself: the bridge's Python children echo back the internal + // token (see services/util.py apollo()), so such calls skip the api_key + // check. External callers can't forge this — it's a per-process secret that + // is never sent to clients. + const internal = ctx.request?.headers?.get?.(INTERNAL_HEADER) ?? ""; + if (internal && safeEqual(internal, INTERNAL_TOKEN)) return; - const header = ctx.request?.headers?.get?.("authorization") ?? ""; - const token = header.startsWith("Bearer ") ? header.slice(7).trim() : ""; - if (!token) return unauthorized(ctx); + // The credential is the api_key the client (Lightning) already sends in the + // request body — there is no separate bearer token and no Lightning-side + // change. Hash it and look for a matching row in lightning_clients; an absent + // or unknown key is rejected. + const apiKey = + typeof ctx.body?.api_key === "string" ? ctx.body.api_key.trim() : ""; + if (!apiKey) return unauthorized(ctx); const lookup = lookupOverride ?? dbLookup; - const client = await lookup(hashToken(token)); + const client = await lookup(hashToken(apiKey)); if (!client) return unauthorized(ctx); ctx.lightningClient = client; } /** - * Returns an { api_key } override to merge into a service payload so Apollo - * uses the client's own Anthropic key. Returns {} when auth is off or the - * client has no stored key, leaving the payload untouched (Python then falls - * back to the global ANTHROPIC_API_KEY). + * Resolve the api_key for the outgoing service payload. Merged LAST over the + * request body so it always wins. + * + * When auth is OFF, returns {} — the payload is left exactly as the caller sent + * it (backward compatible; the legacy "client supplies its own key" behaviour). + * + * When auth is ON, the api_key the client sent is ONLY an auth credential and is + * NEVER passed through to the LLM. It is always overwritten: with the matched + * client's stored anthropic_api_key, or — if that client has no stored key — + * with `undefined`, which drops the field on serialisation so Apollo falls back + * to its global ANTHROPIC_API_KEY. Either way the inbound credential cannot + * survive into the payload. */ export function apiKeyOverride(ctx: any): { api_key?: string } { + if (!enabled) return {}; const client = ctx?.lightningClient as Client | undefined; - return client?.anthropicKey ? { api_key: client.anthropicKey } : {}; + return { api_key: client?.anthropicKey ?? undefined }; } /** Test seam: pass a fake lookup to enable auth without a DB, or null to disable. */ diff --git a/platform/src/middleware/services.ts b/platform/src/middleware/services.ts index 7c12256c..6e08b418 100644 --- a/platform/src/middleware/services.ts +++ b/platform/src/middleware/services.ts @@ -29,7 +29,7 @@ export default async (app: Elysia, port: number) => { const modules = await describeModules(path.resolve("./services")); app.group("/services", (app) => { // Gate every /services/* route on a valid instance token (no-op when - // instance auth is disabled). Loopback/internal calls are exempted. + // instance auth is disabled). app.onBeforeHandle(authGate); modules.forEach((m) => { @@ -140,10 +140,11 @@ export default async (app: Elysia, port: number) => { // TODO in the web socket API, does it make more sense to open a socket at root // and then pick the service you want? So you'd connect to /ws an send { call: 'echo', payload: {} } app.ws(name, { - // Gate the WS upgrade too, so it can't be used to bypass auth. Note: - // per-client Anthropic key injection is only wired into the POST and - // /stream paths (the chat transports Lightning uses); WS payloads pass - // through as-is. + // Gate the WS upgrade too, so it can't be used to bypass auth. The + // credential lives in the request body (api_key), which a WS upgrade + // doesn't carry — so when auth is on, WS upgrades are rejected. That's + // fine: Lightning uses the POST and /stream transports, which is also + // where the api_key is validated and the per-client key swapped in. beforeHandle: authGate, open() { console.log(`Websocket connected at /services/${name}`); diff --git a/platform/test/server.test.ts b/platform/test/server.test.ts index 61d54d1b..d0164872 100644 --- a/platform/test/server.test.ts +++ b/platform/test/server.test.ts @@ -1,6 +1,10 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test"; import setup from "../src/server"; -import { __setAuthForTest, hashToken } from "../src/middleware/auth"; +import { + __setAuthForTest, + hashToken, + internalAuthHeader, +} from "../src/middleware/auth"; const port = 9865; @@ -138,23 +142,20 @@ describe("Python Services", () => { }); describe("Instance authentication", () => { - // Two known clients, keyed by token hash (no real DB — this is the seam). - const ALPHA = "alpha-token"; - const BETA = "beta-token"; + // The credential is the api_key the client sends in the body. Clients are + // keyed by the SHA-256 of that credential (no real DB — this is the seam). + // ALPHA has a stored Anthropic key (Apollo swaps it in); BETA has none (the + // credential is stripped and Apollo falls back to its global key). Neither + // credential may ever pass through to the LLM call. + const ALPHA = "lightning-cred-alpha"; + const BETA = "lightning-cred-beta"; const clients: Record = { - [hashToken(ALPHA)]: { name: "alpha", anthropicKey: "sk-ant-alpha" }, - [hashToken(BETA)]: { name: "beta", anthropicKey: "sk-ant-beta" }, + [hashToken(ALPHA)]: { name: "alpha", anthropicKey: "sk-ant-stored-alpha" }, + [hashToken(BETA)]: { name: "beta", anthropicKey: null }, }; - const postWith = (path: string, data: any, token?: string) => - new Request(`${baseUrl}/${path}`, { - method: "POST", - body: JSON.stringify(data), - headers: { - "Content-Type": "application/json", - ...(token ? { Authorization: `Bearer ${token}` } : {}), - }, - }); + const postKey = (path: string, data: any, apiKey?: string) => + post(path, { ...data, ...(apiKey ? { api_key: apiKey } : {}) }); afterEach(() => { // Restore the open state the rest of the suite expects. @@ -172,22 +173,25 @@ describe("Instance authentication", () => { __setAuthForTest((hash) => clients[hash] ?? null); }); - it("allows a valid token and injects the client's anthropic key", async () => { - const res = await app.handle(postWith("services/echo", { x: 1 }, ALPHA)); + it("accepts a known credential and swaps in the client's stored key", async () => { + const res = await app.handle(postKey("services/echo", { x: 1 }, ALPHA)); expect(res.status).toBe(200); const body = await res.json(); expect(body.x).toBe(1); - expect(body.api_key).toBe("sk-ant-alpha"); + // The stored key replaces the credential; the credential never passes through. + expect(body.api_key).toBe("sk-ant-stored-alpha"); + expect(body.api_key).not.toBe(ALPHA); }); - it("allows a second client's token", async () => { - const res = await app.handle(postWith("services/echo", { x: 2 }, BETA)); + it("strips the credential when the client has no stored key", async () => { + const res = await app.handle(postKey("services/echo", { x: 2 }, BETA)); expect(res.status).toBe(200); const body = await res.json(); - expect(body.api_key).toBe("sk-ant-beta"); + // No stored key → api_key dropped entirely (Apollo uses its global key). + expect(body.api_key).toBeUndefined(); }); - it("rejects a missing token with 401 UNAUTHORIZED", async () => { + it("rejects a missing api_key with 401 UNAUTHORIZED", async () => { const res = await app.handle(post("services/echo", { x: 1 })); expect(res.status).toBe(401); const body = await res.json(); @@ -195,8 +199,8 @@ describe("Instance authentication", () => { expect(body.type).toBe("UNAUTHORIZED"); }); - it("rejects a wrong token with 401", async () => { - const res = await app.handle(postWith("services/echo", { x: 1 }, "nope")); + it("rejects an unknown api_key with 401", async () => { + const res = await app.handle(postKey("services/echo", { x: 1 }, "sk-ant-nope")); expect(res.status).toBe(401); }); @@ -204,5 +208,25 @@ describe("Instance authentication", () => { expect((await app.handle(get("livez"))).status).toBe(200); expect((await app.handle(get(""))).status).toBe(200); }); + + it("exempts internal apollo() self-calls carrying the internal token", async () => { + const req = new Request(`${baseUrl}/services/echo`, { + method: "POST", + body: JSON.stringify({ x: 9 }), + headers: { "Content-Type": "application/json", ...internalAuthHeader() }, + }); + const res = await app.handle(req); + expect(res.status).toBe(200); + }); + + it("still rejects a request bearing a bogus internal token", async () => { + const req = new Request(`${baseUrl}/services/echo`, { + method: "POST", + body: JSON.stringify({ x: 9 }), + headers: { "Content-Type": "application/json", "x-apollo-internal": "nope" }, + }); + const res = await app.handle(req); + expect(res.status).toBe(401); + }); }); }); diff --git a/services/_instance_auth/README.md b/services/_instance_auth/README.md index 8627d604..bd8dd9f4 100644 --- a/services/_instance_auth/README.md +++ b/services/_instance_auth/README.md @@ -1,8 +1,8 @@ # Instance auth Gates Apollo's `/services/*` endpoints so that only known Lightning instances can -call them, and makes Apollo use **each client's own Anthropic API key** for that -client's requests. +call them, and makes Apollo use **its own per-client Anthropic API key** for each +request rather than trusting anything the caller sends. This directory is not a mounted service (the leading `_` keeps it off the HTTP router). It is just the schema and a helper script; the actual gate lives in @@ -10,22 +10,35 @@ router). It is just the schema and a helper script; the actual gate lives in ## How it works -- A single Postgres table, `lightning_clients`, lists the allowed clients. Each row - has a `name`, the **SHA-256 hash** of its bearer token (never the plaintext), and - an optional `anthropic_api_key`. -- On every `/services/*` request the server reads `Authorization: Bearer `, - hashes it, and looks for a matching row. No match → `401 UNAUTHORIZED`. -- On a match, the client's `anthropic_api_key` (if set) is injected into the - request payload as `api_key`, so all LLM usage for that request bills to the - client. If the column is `NULL`, Apollo falls back to its global - `ANTHROPIC_API_KEY`. +- The credential is the **`api_key` the caller already sends in the request + body** — the same field Lightning sends today. There is no bearer token, no + `Authorization` header, and **no change required on the Lightning side**. +- A single Postgres table, `lightning_clients`, is the allow-list. Each row has a + `name`, the **SHA-256 hash** of that client's `api_key` (never the plaintext), + and an optional `anthropic_api_key`. +- On every `/services/*` request the server reads `api_key` from the body, + hashes it, and looks for a matching row. Missing or no match → `401 + UNAUTHORIZED`. +- The inbound `api_key` is treated **purely as a credential and is never + forwarded to the LLM**. On a match it is replaced with the client's stored + `anthropic_api_key`, so all LLM usage for that request bills to the key Apollo + controls. If the column is `NULL`, the inbound key is **stripped** and Apollo + falls back to its global `ANTHROPIC_API_KEY`. Either way the caller's key + cannot pass through. +- **Performance:** the client list is cached in memory and refreshed on a ~60s + TTL, so the database is queried at most once per minute per process, never on + the per-request path to Anthropic. The per-request cost is a hash plus a map + lookup. The trade-off is that revocations/rotations take up to ~60s to take + effect (see Managing clients). - **Opt-in / backward compatible:** auth is active only when the `INSTANCE_AUTH` environment variable is set (e.g. `INSTANCE_AUTH=true`). Otherwise the server - stays open exactly as before. When auth is enabled but this table can't be - reached, the gate **fails closed** (every external caller gets `401`) rather - than silently opening up. -- Loopback callers (`127.0.0.1`/`::1`) and the health endpoints (`/livez`, - `/status`, `/`) are exempt. + stays open exactly as before and the caller's `api_key` passes through + untouched. When auth is enabled but this table can't be reached, the gate + **fails closed** (every external caller gets `401`) rather than silently + opening up. +- The health endpoints (`/livez`, `/status`, `/`) sit outside `/services/*` and + are never gated. Internal Apollo-to-Apollo `apollo()` calls are exempt via a + per-process internal token (`APOLLO_INTERNAL_TOKEN`), not by network position. ## Enabling it @@ -35,43 +48,47 @@ router). It is just the schema and a helper script; the actual gate lives in psql "$POSTGRES_URL" -f services/_instance_auth/schema.sql ``` -2. Mint a token for a Lightning instance and get its hash: +2. Get the SHA-256 hash of the `api_key` a given Lightning instance sends: ```sh - poetry run python services/_instance_auth/hash_token.py + poetry run python services/_instance_auth/hash_token.py ``` - This prints a plaintext token (give it to the Lightning instance), the hash to - store, and a ready-to-edit `INSERT`. + (Run with no argument to instead mint a fresh credential to hand to the + instance.) This prints the hash to store and a ready-to-edit `INSERT`. -3. Insert the row (set `name` and the client's Anthropic key): +3. Insert the row. Set `name`, the hash, and the Anthropic key Apollo should use + for this client (leave `anthropic_api_key` as `NULL` to use Apollo's global + key): ```sql INSERT INTO lightning_clients (name, auth_token_hash, anthropic_api_key) VALUES ('my-lightning-instance', '', 'sk-ant-...'); ``` -4. Configure that Lightning instance to send `Authorization: Bearer ` on - its Apollo requests. - -5. Set `INSTANCE_AUTH=true` in Apollo's environment and restart. The startup log +4. Set `INSTANCE_AUTH=true` in Apollo's environment and restart. The startup log shows `Apollo instance auth ENABLED`. (If `INSTANCE_AUTH` is set but the table is missing, the log warns and every external caller is rejected.) +No Lightning-side configuration is needed: it keeps sending its `api_key` exactly +as it does now. + ## Managing clients There is no CLI — manage rows directly in the DB: - **Revoke a client:** `DELETE FROM lightning_clients WHERE name = '...';` -- **Rotate a token:** mint a new one with `hash_token.py` and +- **Rotate a credential:** hash the new `api_key` with `hash_token.py` and `UPDATE lightning_clients SET auth_token_hash = '' WHERE name = '...';` -- **Change a client's key:** +- **Change the Anthropic key Apollo uses for a client:** `UPDATE lightning_clients SET anthropic_api_key = '...' WHERE name = '...';` -Changes are picked up within ~60s (the server caches the client list briefly). +Changes are picked up within ~60s (the server caches the client list briefly). If +you need a revocation to take effect immediately, restart Apollo. ## Note The `anthropic_api_key` is stored as plaintext because Apollo must be able to use -it. Protect this table at rest (DB encryption, restricted access) accordingly. The -bearer tokens themselves are only ever stored as hashes. +it, and it is held in the in-memory cache while the server runs. Protect this +table at rest (DB encryption, restricted access) accordingly. The clients' +`api_key` credentials are only ever stored and compared as hashes. diff --git a/services/_instance_auth/hash_token.py b/services/_instance_auth/hash_token.py index e817fb01..9a53cc80 100644 --- a/services/_instance_auth/hash_token.py +++ b/services/_instance_auth/hash_token.py @@ -1,16 +1,16 @@ -"""Mint and/or hash a Lightning client bearer token for the lightning_clients table. +"""Hash (or mint) a Lightning client's api_key credential for the lightning_clients table. Usage (run from the repo root): - # Generate a fresh token and print its hash + a ready-to-edit INSERT: - poetry run python services/_instance_auth/hash_token.py + # Hash the api_key a Lightning instance already sends: + poetry run python services/_instance_auth/hash_token.py - # Hash a token you already have: - poetry run python services/_instance_auth/hash_token.py + # Or mint a fresh credential and print its hash + a ready-to-edit INSERT: + poetry run python services/_instance_auth/hash_token.py -The plaintext token goes into the Lightning instance config as -`Authorization: Bearer `. Only the SHA-256 hash is stored in the DB. The -hash here must match the one Apollo computes in platform/src/middleware/auth.ts +The credential is whatever the client sends as `api_key` in the request body -- +there is no bearer token. Only its SHA-256 hash is stored in the DB. The hash +here must match the one Apollo computes in platform/src/middleware/auth.ts (sha256 over the UTF-8 bytes, lowercase hex) -- it does. """ @@ -35,9 +35,9 @@ def main() -> None: lines = [] if generated: - lines.append("Generated a new client token.\n") - lines.append(f" token (give to the Lightning instance): {token}") - lines.append(f" auth_token_hash (store in the DB): {digest}\n") + lines.append("Generated a new client credential.\n") + lines.append(f" api_key (the instance sends this): {token}") + lines.append(f" auth_token_hash (store in the DB): {digest}\n") lines.append("Example INSERT (set name and the client's Anthropic key):\n") lines.append( " INSERT INTO lightning_clients (name, auth_token_hash, anthropic_api_key)\n" diff --git a/services/_instance_auth/schema.sql b/services/_instance_auth/schema.sql index bdbd48c3..9e6acebf 100644 --- a/services/_instance_auth/schema.sql +++ b/services/_instance_auth/schema.sql @@ -1,19 +1,19 @@ -- Instance auth: the allow-list of Lightning clients permitted to call Apollo. -- -- Instance auth is opt-in via the INSTANCE_AUTH env var. When it is set, the --- Apollo server gates every /services/* request on a valid bearer token and --- looks tokens up in this table (via POSTGRES_URL). This table must exist for --- auth to work — if INSTANCE_AUTH is set but the table is missing, the gate --- fails closed and rejects every external caller. Without INSTANCE_AUTH, the --- server stays open as before. +-- Apollo server gates every /services/* request on the api_key the caller sends +-- in the request body, looking its hash up in this table (via POSTGRES_URL). +-- This table must exist for auth to work — if INSTANCE_AUTH is set but the table +-- is missing, the gate fails closed and rejects every external caller. Without +-- INSTANCE_AUTH, the server stays open as before. -- --- Rows are managed by hand. Use hash_token.py to mint a token + its hash, then --- INSERT a row and configure the Lightning instance with the plaintext token as --- `Authorization: Bearer `. See README.md. +-- Rows are managed by hand. Use hash_token.py to hash a client's api_key, then +-- INSERT a row. The inbound api_key is only a credential; Apollo never forwards +-- it to the LLM, using anthropic_api_key (below) instead. See README.md. CREATE TABLE IF NOT EXISTS lightning_clients ( id SERIAL PRIMARY KEY, name TEXT UNIQUE NOT NULL, -- human identifier for the client - auth_token_hash VARCHAR(64) UNIQUE NOT NULL, -- sha256 hex of the bearer token (never the plaintext) + auth_token_hash VARCHAR(64) UNIQUE NOT NULL, -- sha256 hex of the client's api_key credential (never the plaintext) anthropic_api_key TEXT -- Anthropic key Apollo uses for this client; NULL => global env key ); diff --git a/services/util.py b/services/util.py index bdce2d24..d8985cb9 100644 --- a/services/util.py +++ b/services/util.py @@ -96,7 +96,15 @@ def apollo(name: str, payload: dict) -> dict: :return: JSON response. """ url = f"http://127.0.0.1:{apollo_port}/services/{name}" - r = requests.post(url, json=payload) + # Mark this as an internal Apollo-to-Apollo call so it bypasses instance auth + # (see platform/src/middleware/auth.ts). The server process injects the token + # into the environment and this child process inherits it; absent (auth off + # or token unset) the header is simply omitted. + headers = {} + internal_token = os.environ.get("APOLLO_INTERNAL_TOKEN") + if internal_token: + headers["X-Apollo-Internal"] = internal_token + r = requests.post(url, json=payload, headers=headers) return r.json() From 75447e30c216ce295ade334084d6cb9a560ef047 Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Thu, 18 Jun 2026 14:16:42 +0200 Subject: [PATCH 04/10] clean --- .env.example | 4 --- README.md | 19 +++++++------ platform/src/middleware/auth.ts | 50 --------------------------------- 3 files changed, 11 insertions(+), 62 deletions(-) diff --git a/.env.example b/.env.example index 82d70d85..178321ae 100644 --- a/.env.example +++ b/.env.example @@ -10,10 +10,6 @@ GITHUB_TOKEN=KEY # and looks it up in the lightning_clients table via POSTGRES_URL (see # services/_instance_auth/). Leave unset/false to keep the server open as before. INSTANCE_AUTH=false -# Debug aid: set APOLLO_AUTH_DEBUG=true to log the shape (method, path, peer IP, -# redacted headers, body keys) of every /services/* request. Secrets are never -# printed raw; the api_key is shown as its sha256 hash. Leave off normally. -# APOLLO_AUTH_DEBUG=false # Langfuse observability LANGFUSE_SECRET_KEY=sk-lf-... diff --git a/README.md b/README.md index cf510673..1cc78f11 100644 --- a/README.md +++ b/README.md @@ -193,14 +193,17 @@ for that client's requests. `lightning_clients` table via `POSTGRES_URL`; if auth is enabled but that table can't be reached, the gate **fails closed** (rejects all external callers) rather than silently opening up. -- Clients authenticate with `Authorization: Bearer `. Apollo stores only a - SHA-256 hash of the token; an unknown/missing token gets - `401 { "code": 401, "type": "UNAUTHORIZED" }`. -- On a match, the client's stored Anthropic key is injected into the request, so - LLM usage bills to that client (falls back to the global `ANTHROPIC_API_KEY` if - the client has no key). -- Health/root endpoints (`/livez`, `/status`, `/`) and loopback/internal - service-to-service calls are exempt. +- The credential is the **`api_key` the caller already sends in the request + body** — there is no bearer token, no `Authorization` header, and no + Lightning-side change. Apollo stores only a SHA-256 hash of it; an + unknown/missing key gets `401 { "code": 401, "type": "UNAUTHORIZED" }`. +- On a match, the inbound `api_key` is treated purely as a credential and is + **never** forwarded to the LLM: it is replaced with the client's stored + Anthropic key (so LLM usage bills to that client), or stripped — falling back + to the global `ANTHROPIC_API_KEY` — if the client has no stored key. +- Health/root endpoints (`/livez`, `/status`, `/`) are outside `/services/*` and + never gated. Internal Apollo-to-Apollo `apollo()` calls are exempt via a + per-process internal token (`APOLLO_INTERNAL_TOKEN`), not by network position. To enable it and provision clients, see [`services/_instance_auth/`](services/_instance_auth/README.md). diff --git a/platform/src/middleware/auth.ts b/platform/src/middleware/auth.ts index 85a7edc6..3d359998 100644 --- a/platform/src/middleware/auth.ts +++ b/platform/src/middleware/auth.ts @@ -96,55 +96,6 @@ function authOptIn(): boolean { return v === "1" || v === "true" || v === "yes" || v === "on"; } -/** Whether APOLLO_AUTH_DEBUG opts into per-request logging (read live each call). */ -function debugEnabled(): boolean { - const v = (process.env.APOLLO_AUTH_DEBUG ?? "").trim().toLowerCase(); - return v === "1" || v === "true" || v === "yes" || v === "on"; -} - -/** - * Opt-in logging of the shape of every /services/* request, for debugging what - * callers (e.g. Lightning) actually send. Enable with APOLLO_AUTH_DEBUG=true. - * Secrets are never printed raw: the inbound api_key (the credential) is shown - * as its SHA-256 hash — exactly the value lightning_clients.auth_token_hash - * stores, so you can paste it straight into the table to allow-list a client. - * The internal token is shown only as present/absent and body values are - * omitted (just the top-level key names). Wrapped so it can never break a - * request if something is unexpectedly shaped. - */ -function debugLogRequest(ctx: any): void { - if (!debugEnabled()) return; - try { - const req = ctx.request; - const ip = ctx.server?.requestIP?.(req)?.address ?? "(no peer address)"; - const headers: Record = {}; - req?.headers?.forEach?.((value: string, key: string) => { - if (key === "authorization") { - const token = value.startsWith("Bearer ") ? value.slice(7).trim() : ""; - headers[key] = token ? `Bearer ` : ""; - } else if (key === INTERNAL_HEADER) { - headers[key] = ""; - } else { - headers[key] = value; - } - }); - const bodyKeys = - ctx.body && typeof ctx.body === "object" ? Object.keys(ctx.body) : []; - const rawKey = - typeof ctx.body?.api_key === "string" ? ctx.body.api_key.trim() : ""; - const apiKeyHash = rawKey ? `sha256:${hashToken(rawKey)}` : "(none)"; - const pathname = req?.url ? new URL(req.url).pathname : "(no url)"; - console.log( - `[auth-debug] ${req?.method} ${pathname} from ${ip} | auth ${enabled ? "ON" : "OFF"}\n` + - `[auth-debug] headers: ${JSON.stringify(headers)}\n` + - `[auth-debug] body keys: ${JSON.stringify(bodyKeys)}\n` + - `[auth-debug] api_key (credential): ${apiKeyHash}` - ); - } catch (err) { - console.log("[auth-debug] failed to log request:", err); - } -} - /** * Decide once at startup whether instance auth is active. The INSTANCE_AUTH env * var is the master switch: unset/falsey leaves /services/* open as before. @@ -209,7 +160,6 @@ function unauthorized(ctx: any): ApolloError { * On success it stashes the resolved client on the context for apiKeyOverride. */ export async function authGate(ctx: any): Promise { - debugLogRequest(ctx); if (!enabled) return; // Apollo calling itself: the bridge's Python children echo back the internal From 67d1bc863930bb231fe2ba3018782570b490753b Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Thu, 18 Jun 2026 15:09:49 +0200 Subject: [PATCH 05/10] handle db stampede, encrypt at rest --- .env.example | 7 + CLAUDE.md | 14 +- platform/src/middleware/auth.ts | 163 +++++++++++++++++++++-- platform/src/util/instance-key-crypto.ts | 58 ++++++++ platform/test/server.test.ts | 156 ++++++++++++++++++++++ services/_instance_auth/README.md | 56 +++++++- services/_instance_auth/encrypt_key.ts | 37 +++++ services/_instance_auth/schema.sql | 4 +- 8 files changed, 473 insertions(+), 22 deletions(-) create mode 100644 platform/src/util/instance-key-crypto.ts create mode 100644 services/_instance_auth/encrypt_key.ts diff --git a/.env.example b/.env.example index 178321ae..35b90b00 100644 --- a/.env.example +++ b/.env.example @@ -11,6 +11,13 @@ GITHUB_TOKEN=KEY # services/_instance_auth/). Leave unset/false to keep the server open as before. INSTANCE_AUTH=false +# Optional at-rest encryption for the per-client Anthropic keys stored in +# lightning_clients.anthropic_api_key. Base64 of 32 random bytes +# (openssl rand -base64 32). When set, encrypt keys with +# services/_instance_auth/encrypt_key.ts before INSERT; plaintext rows still work +# when unset. See services/_instance_auth/README.md. +# APOLLO_ENC_KEY= + # Langfuse observability LANGFUSE_SECRET_KEY=sk-lf-... LANGFUSE_PUBLIC_KEY=pk-lf-... diff --git a/CLAUDE.md b/CLAUDE.md index 679a836d..a8383861 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -53,10 +53,18 @@ TypeScript) service modules. unknown key is rejected with `401`. The inbound `api_key` is treated purely as a credential and is **never** forwarded to the LLM: on a match it is replaced with the matched client's stored `anthropic_api_key`, or stripped (falling back - to the global `ANTHROPIC_API_KEY`) when that column is `NULL`. Lookups are + to the global `ANTHROPIC_API_KEY`) when that column is `NULL`. The stored + `anthropic_api_key` may be plaintext or AES-256-GCM-encrypted (`enc:v1:` + values, decrypted with `APOLLO_ENC_KEY`; see + `platform/src/util/instance-key-crypto.ts` and + `services/_instance_auth/encrypt_key.ts`). Lookups are cached in memory (~60s TTL), so the DB is hit at most once per minute per - process, not per request. If auth is enabled but the table can't be reached, - the gate fails closed (rejects all external callers). The gate is scoped to + process, not per request. The refresh is single-flight with + stale-while-revalidate, so a burst of requests at the TTL boundary shares one + DB read (cold start awaits it; a warm-but-stale cache is served while one + background refresh runs) rather than stampeding the DB. If auth is enabled but + the table can't be reached, the gate fails closed (rejects all external + callers). The gate is scoped to `/services/*`, so health/root endpoints outside that group are unaffected. Internal Apollo-to-Apollo `apollo()` calls are exempt via a per-process internal token (`APOLLO_INTERNAL_TOKEN`, minted at startup, injected into the diff --git a/platform/src/middleware/auth.ts b/platform/src/middleware/auth.ts index 3d359998..9c36eeba 100644 --- a/platform/src/middleware/auth.ts +++ b/platform/src/middleware/auth.ts @@ -1,6 +1,7 @@ import { SQL } from "bun"; import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; import type { ApolloError } from "../util/errors"; +import { ENC_PREFIX, decryptKey, parseEncKey } from "../util/instance-key-crypto"; // A row from the lightning_clients table, as used at runtime. type Client = { name: string; anthropicKey: string | null }; @@ -32,6 +33,20 @@ const CACHE_TTL_MS = 60_000; let clientCache: Map | null = null; let cacheTs = 0; +// Single-flight handle for the cache refresh. Concurrent refreshes (e.g. a burst +// of requests hitting the TTL boundary at once) all share THIS one promise, so a +// refresh can never trigger more than one DB read no matter the concurrency. +let refreshInFlight: Promise> | null = null; + +// Master key (32 bytes) for decrypting stored anthropic_api_key values, parsed +// from APOLLO_ENC_KEY in initAuth. Null when unset — plaintext rows still work, +// but any "enc:v1:" row can't be decrypted and its client is omitted. +let encKey: Buffer | null = null; + +// When set (by tests), this replaces loadClients in the refresh path so the +// single-flight/stale-while-revalidate behaviour can be exercised without a DB. +let loaderOverride: (() => Promise>) | null = null; + // A per-process secret that identifies genuine Apollo-to-Apollo calls. The // bridge spawns Python services as child processes that inherit this process's // env, so exporting the token here lets services/util.py apollo() echo it back @@ -61,6 +76,58 @@ export function hashToken(token: string): string { return createHash("sha256").update(token).digest("hex"); } +// Sentinel returned by decryptStoredKey when an encrypted value can't be +// decrypted. Such a client is omitted from the cache (fail closed) rather than +// silently falling back to the global key, which would mis-bill its usage. +const DECRYPT_FAILED = Symbol("decrypt-failed"); + +/** + * Resolve the anthropic_api_key column to the plaintext key Apollo should use. + * - null → null (intentional: this client uses the global env key) + * - "enc:v1:…" → AES-256-GCM decrypt with encKey; DECRYPT_FAILED on error + * - anything else → legacy plaintext, used as-is (backward compatible) + */ +function decryptStoredKey( + stored: string | null, + clientName: string +): string | null | typeof DECRYPT_FAILED { + if (stored === null) return null; + if (!stored.startsWith(ENC_PREFIX)) return stored; + if (!encKey) { + console.error( + `Apollo instance auth: client "${clientName}" has an encrypted anthropic_api_key but APOLLO_ENC_KEY is unset/invalid — omitting this client (fail closed).` + ); + return DECRYPT_FAILED; + } + try { + return decryptKey(stored, encKey); + } catch (err) { + console.error( + `Apollo instance auth: could not decrypt anthropic_api_key for client "${clientName}" — omitting this client (fail closed).`, + err + ); + return DECRYPT_FAILED; + } +} + +/** Build the hash→client map from raw rows, decrypting keys and dropping any + * client whose encrypted key can't be decrypted. Exported for tests. */ +export function buildClientMap( + rows: Array<{ + name: string; + auth_token_hash: string; + anthropic_api_key: string | null; + }> +): Map { + const map = new Map(); + for (const row of rows) { + const key = decryptStoredKey(row.anthropic_api_key, row.name); + if (key === DECRYPT_FAILED) continue; + map.set(row.auth_token_hash, { name: row.name, anthropicKey: key }); + } + return map; +} + async function loadClients(): Promise> { const rows = (await sql!` SELECT name, auth_token_hash, anthropic_api_key FROM lightning_clients @@ -69,25 +136,53 @@ async function loadClients(): Promise> { auth_token_hash: string; anthropic_api_key: string | null; }>; - const map = new Map(); - for (const row of rows) { - map.set(row.auth_token_hash, { - name: row.name, - anthropicKey: row.anthropic_api_key, + return buildClientMap(rows); +} + +/** + * Refresh the client cache, collapsing concurrent callers onto a single DB read + * (single-flight). The in-flight promise is cleared in finally so a failed + * refresh never wedges the cache — the next caller retries. + */ +function refreshClients(): Promise> { + if (refreshInFlight) return refreshInFlight; + refreshInFlight = (loaderOverride ?? loadClients)() + .then((map) => { + clientCache = map; + cacheTs = Date.now(); + return map; + }) + .finally(() => { + refreshInFlight = null; }); - } - return map; + return refreshInFlight; } async function dbLookup(hash: string): Promise { // Auth is on but the lookup never came up — reject (fail closed). - if (!dbReady || !sql) return null; - const now = Date.now(); - if (!clientCache || now - cacheTs > CACHE_TTL_MS) { - clientCache = await loadClients(); - cacheTs = now; + if (!dbReady) return null; + if (!loaderOverride && !sql) return null; + + const fresh = clientCache !== null && Date.now() - cacheTs <= CACHE_TTL_MS; + if (!fresh) { + if (clientCache !== null) { + // Warm but stale: serve the current map now and refresh once in the + // background. Errors are swallowed so a DB blip keeps serving stale (and + // retries next request, since cacheTs only advances on success) instead + // of 500ing every caller. The single-flight handle guarantees one read. + void refreshClients().catch(() => {}); + } else { + // Cold start: nothing to serve yet. Every concurrent caller awaits the + // ONE shared load rather than each firing its own. On failure leave the + // cache null and reject (fail closed). + try { + await refreshClients(); + } catch { + return null; + } + } } - return clientCache.get(hash) ?? null; + return clientCache?.get(hash) ?? null; } /** Whether the INSTANCE_AUTH env var opts this instance into auth. */ @@ -114,6 +209,16 @@ export async function initAuth(): Promise { return; } + // Parse the at-rest encryption key for stored anthropic_api_key values. + // Optional: when unset, plaintext rows keep working; only "enc:v1:" rows need + // it. When set but malformed, warn loudly — encrypted clients will be rejected. + encKey = parseEncKey(process.env.APOLLO_ENC_KEY); + if (process.env.APOLLO_ENC_KEY && !encKey) { + console.error( + "Apollo instance auth: APOLLO_ENC_KEY is set but is not valid base64 of 32 bytes — encrypted client keys cannot be decrypted and those clients will be REJECTED." + ); + } + const url = process.env.POSTGRES_URL; if (!url) { dbReady = false; @@ -214,3 +319,35 @@ export function __setAuthForTest(provider: Lookup | null): void { lookupOverride = null; } } + +/** + * Test seam: drive the real dbLookup (single-flight + stale-while-revalidate) + * with a fake table loader instead of Postgres. Passing a loader enables auth on + * the dbLookup path (lookupOverride cleared) and resets cache state; null tears + * it back down. + */ +export function __setLoaderForTest( + loader: (() => Promise>) | null +): void { + loaderOverride = loader; + clientCache = null; + cacheTs = 0; + refreshInFlight = null; + if (loader) { + enabled = true; + dbReady = true; + lookupOverride = null; + } else { + dbReady = false; + } +} + +/** Test seam: mark the cache stale without touching its contents. */ +export function __expireCacheForTest(): void { + cacheTs = 0; +} + +/** Test seam: set the at-rest decryption key (or null) without running initAuth. */ +export function __setEncKeyForTest(key: Buffer | null): void { + encKey = key; +} diff --git a/platform/src/util/instance-key-crypto.ts b/platform/src/util/instance-key-crypto.ts new file mode 100644 index 00000000..39888c3b --- /dev/null +++ b/platform/src/util/instance-key-crypto.ts @@ -0,0 +1,58 @@ +// AES-256-GCM helpers for the per-client anthropic_api_key stored in +// lightning_clients. Used by the auth middleware (to decrypt on cache load) and +// by services/_instance_auth/encrypt_key.ts (to produce values to INSERT). Kept +// in one module so the byte format can never drift between the two sides. +// +// Stored format: "enc:v1:". The master +// key is APOLLO_ENC_KEY (base64 of 32 bytes). Anything NOT prefixed with the +// tag is treated as legacy plaintext elsewhere, so encryption is opt-in and +// backward compatible. +import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; + +export const ENC_PREFIX = "enc:v1:"; +const IV_BYTES = 12; // GCM standard nonce length +const TAG_BYTES = 16; // GCM auth tag length + +/** + * Decode APOLLO_ENC_KEY (base64 of exactly 32 bytes) into a key Buffer, or null + * if it is absent or malformed. Callers treat null as "no key configured". + */ +export function parseEncKey(raw: string | undefined | null): Buffer | null { + if (!raw) return null; + let buf: Buffer; + try { + buf = Buffer.from(raw.trim(), "base64"); + } catch { + return null; + } + return buf.length === 32 ? buf : null; +} + +/** Encrypt plaintext into an "enc:v1:…" value using AES-256-GCM. */ +export function encryptKey(plaintext: string, key: Buffer): string { + const iv = randomBytes(IV_BYTES); + const cipher = createCipheriv("aes-256-gcm", key, iv); + const ciphertext = Buffer.concat([ + cipher.update(plaintext, "utf8"), + cipher.final(), + ]); + const tag = cipher.getAuthTag(); + return ENC_PREFIX + Buffer.concat([iv, tag, ciphertext]).toString("base64"); +} + +/** + * Decrypt an "enc:v1:…" value. Throws if the key is wrong, the value is + * corrupt, or the auth tag fails — callers decide how to handle the failure + * (the auth middleware omits the client and fails closed). + */ +export function decryptKey(stored: string, key: Buffer): string { + const blob = Buffer.from(stored.slice(ENC_PREFIX.length), "base64"); + const iv = blob.subarray(0, IV_BYTES); + const tag = blob.subarray(IV_BYTES, IV_BYTES + TAG_BYTES); + const ciphertext = blob.subarray(IV_BYTES + TAG_BYTES); + const decipher = createDecipheriv("aes-256-gcm", key, iv); + decipher.setAuthTag(tag); + return Buffer.concat([decipher.update(ciphertext), decipher.final()]).toString( + "utf8" + ); +} diff --git a/platform/test/server.test.ts b/platform/test/server.test.ts index d0164872..82099ce7 100644 --- a/platform/test/server.test.ts +++ b/platform/test/server.test.ts @@ -1,10 +1,17 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { randomBytes } from "node:crypto"; import setup from "../src/server"; import { + __expireCacheForTest, __setAuthForTest, + __setEncKeyForTest, + __setLoaderForTest, + authGate, + buildClientMap, hashToken, internalAuthHeader, } from "../src/middleware/auth"; +import { encryptKey } from "../src/util/instance-key-crypto"; const port = 9865; @@ -230,3 +237,152 @@ describe("Instance authentication", () => { }); }); }); + +describe("Instance auth cache refresh", () => { + // Drive the real dbLookup (single-flight + stale-while-revalidate) with a fake + // table loader so we can assert how many DB reads a request burst causes. + // authGate is called directly with a minimal ctx to avoid spawning the echo + // service for every request in the burst. + const ALPHA = "lightning-cred-alpha"; + const mapWith = (anthropicKey: string | null) => + new Map([[hashToken(ALPHA), { name: "alpha", anthropicKey }]]); + const fakeCtx = (apiKey?: string) => + ({ + request: { headers: { get: () => null } }, + body: apiKey ? { api_key: apiKey } : {}, + set: { status: 200 }, + }) as any; + const tick = () => new Promise((r) => setTimeout(r, 10)); + const settle = () => new Promise((r) => setTimeout(r, 40)); + + afterEach(() => { + __setLoaderForTest(null); + __setAuthForTest(null); + }); + + it("collapses a cold-start burst into a single DB read", async () => { + let calls = 0; + __setLoaderForTest(async () => { + calls++; + await tick(); + return mapWith("sk-ant-stored-alpha"); + }); + + const ctxs = Array.from({ length: 50 }, () => fakeCtx(ALPHA)); + await Promise.all(ctxs.map((c) => authGate(c))); + + expect(calls).toBe(1); + for (const c of ctxs) { + expect(c.lightningClient?.anthropicKey).toBe("sk-ant-stored-alpha"); + } + }); + + it("serves the stale list while one background refresh runs", async () => { + let calls = 0; + let current = mapWith("sk-ant-v1"); + __setLoaderForTest(async () => { + calls++; + await tick(); + return current; + }); + + // Cold start awaits the one load and warms the cache with v1. + const warm = fakeCtx(ALPHA); + await authGate(warm); + expect(calls).toBe(1); + expect(warm.lightningClient?.anthropicKey).toBe("sk-ant-v1"); + + // New data lands in the DB; mark the cache stale. + current = mapWith("sk-ant-v2"); + __expireCacheForTest(); + + // The burst is served immediately from the stale v1 map and triggers + // exactly one background refresh (not one per request). + const ctxs = Array.from({ length: 25 }, () => fakeCtx(ALPHA)); + await Promise.all(ctxs.map((c) => authGate(c))); + expect(calls).toBe(2); + for (const c of ctxs) { + expect(c.lightningClient?.anthropicKey).toBe("sk-ant-v1"); + } + + // Once the background refresh settles, the new value is visible — with no + // further DB reads. + await settle(); + const after = fakeCtx(ALPHA); + await authGate(after); + expect(after.lightningClient?.anthropicKey).toBe("sk-ant-v2"); + expect(calls).toBe(2); + }); + + it("keeps serving stale when the refresh fails, then recovers", async () => { + let calls = 0; + let fail = false; + __setLoaderForTest(async () => { + calls++; + if (fail) throw new Error("db down"); + return mapWith("sk-ant-v1"); + }); + + const warm = fakeCtx(ALPHA); + await authGate(warm); + expect(warm.lightningClient?.anthropicKey).toBe("sk-ant-v1"); + + // Refresh now fails; stale callers are still authenticated (no 500/empty). + fail = true; + __expireCacheForTest(); + const ctxs = Array.from({ length: 10 }, () => fakeCtx(ALPHA)); + await Promise.all(ctxs.map((c) => authGate(c))); + for (const c of ctxs) { + expect(c.lightningClient?.anthropicKey).toBe("sk-ant-v1"); + } + + // Recover once the DB is back. + fail = false; + __expireCacheForTest(); + const ok = fakeCtx(ALPHA); + await authGate(ok); + expect(ok.lightningClient?.anthropicKey).toBe("sk-ant-v1"); + }); +}); + +describe("Instance auth key encryption", () => { + afterEach(() => __setEncKeyForTest(null)); + + it("round-trips encrypted, plaintext, and null keys through buildClientMap", () => { + const key = randomBytes(32); + __setEncKeyForTest(key); + const enc = encryptKey("sk-ant-secret", key); + + const map = buildClientMap([ + { name: "enc", auth_token_hash: "h-enc", anthropic_api_key: enc }, + { name: "plain", auth_token_hash: "h-plain", anthropic_api_key: "sk-ant-plain" }, + { name: "none", auth_token_hash: "h-none", anthropic_api_key: null }, + ]); + + expect(map.get("h-enc")?.anthropicKey).toBe("sk-ant-secret"); + expect(map.get("h-plain")?.anthropicKey).toBe("sk-ant-plain"); + expect(map.get("h-none")?.anthropicKey).toBeNull(); + }); + + it("omits a client whose encrypted key can't be decrypted (wrong key)", () => { + const enc = encryptKey("sk-ant-secret", randomBytes(32)); // encrypted with key A + __setEncKeyForTest(randomBytes(32)); // server holds a different key + + const map = buildClientMap([ + { name: "bad", auth_token_hash: "h-bad", anthropic_api_key: enc }, + ]); + + expect(map.has("h-bad")).toBe(false); + }); + + it("omits an encrypted key when APOLLO_ENC_KEY is not configured", () => { + __setEncKeyForTest(null); + const enc = encryptKey("sk-ant-secret", randomBytes(32)); + + const map = buildClientMap([ + { name: "bad", auth_token_hash: "h-bad", anthropic_api_key: enc }, + ]); + + expect(map.has("h-bad")).toBe(false); + }); +}); diff --git a/services/_instance_auth/README.md b/services/_instance_auth/README.md index bd8dd9f4..17a73d14 100644 --- a/services/_instance_auth/README.md +++ b/services/_instance_auth/README.md @@ -66,6 +66,10 @@ router). It is just the schema and a helper script; the actual gate lives in VALUES ('my-lightning-instance', '', 'sk-ant-...'); ``` + To store the Anthropic key encrypted instead of in the clear, see + [Encrypting the stored Anthropic keys](#encrypting-the-stored-anthropic-keys-optional) + below and paste the `enc:v1:…` value in place of `'sk-ant-...'`. + 4. Set `INSTANCE_AUTH=true` in Apollo's environment and restart. The startup log shows `Apollo instance auth ENABLED`. (If `INSTANCE_AUTH` is set but the table is missing, the log warns and every external caller is rejected.) @@ -86,9 +90,51 @@ There is no CLI — manage rows directly in the DB: Changes are picked up within ~60s (the server caches the client list briefly). If you need a revocation to take effect immediately, restart Apollo. -## Note +## Encrypting the stored Anthropic keys (optional) + +By default `anthropic_api_key` is stored as plaintext. You can instead encrypt it +at rest with AES-256-GCM so a DB dump/backup/replica leak doesn't expose live +keys: + +1. Generate a 32-byte master key and set it in Apollo's environment: + + ```sh + openssl rand -base64 32 # → put the output in APOLLO_ENC_KEY + ``` + +2. Encrypt a key to get the value to store: + + ```sh + APOLLO_ENC_KEY= \ + bun services/_instance_auth/encrypt_key.ts sk-ant-... + ``` + + This prints an `enc:v1:…` blob (and a ready-to-edit `INSERT`). Use it in place + of the plaintext key: + + ```sql + UPDATE lightning_clients SET anthropic_api_key = 'enc:v1:...' WHERE name = '...'; + ``` -The `anthropic_api_key` is stored as plaintext because Apollo must be able to use -it, and it is held in the in-memory cache while the server runs. Protect this -table at rest (DB encryption, restricted access) accordingly. The clients' -`api_key` credentials are only ever stored and compared as hashes. +3. Restart Apollo with `APOLLO_ENC_KEY` set. It decrypts each key once per ~60s + cache refresh. + +Notes: + +- **Backward compatible / opt-in.** Plaintext rows keep working; only `enc:v1:` + rows need `APOLLO_ENC_KEY`. You can migrate one client at a time. +- **Fail closed.** If an `enc:v1:` row can't be decrypted (wrong/missing + `APOLLO_ENC_KEY` or corrupt value), that client is dropped from the allow-list + and its requests get `401` — Apollo never silently falls back to the global key + for an encrypted-but-undecryptable row. A `NULL` key still means "use the + global key" as before. +- **Rotation** is manual: decrypt and re-encrypt every `enc:v1:` row with the new + key, then swap `APOLLO_ENC_KEY` and restart. +- **What it protects.** The ciphertext is useless without `APOLLO_ENC_KEY`, so + this guards DB dumps, backups, read replicas, and accidental `SELECT`s in logs. + It does **not** protect a full Apollo host/process compromise: the running + process necessarily holds both the key and the decrypted values in memory + (Apollo must use the key to call Anthropic). Continue to protect the table at + rest (restricted access, DB encryption) regardless. + +The clients' `api_key` credentials are only ever stored and compared as hashes. diff --git a/services/_instance_auth/encrypt_key.ts b/services/_instance_auth/encrypt_key.ts new file mode 100644 index 00000000..6f7c36b9 --- /dev/null +++ b/services/_instance_auth/encrypt_key.ts @@ -0,0 +1,37 @@ +// Encrypt an Anthropic API key for the lightning_clients.anthropic_api_key +// column, so the key is not stored in the clear. Run from the repo root: +// +// APOLLO_ENC_KEY= \ +// bun services/_instance_auth/encrypt_key.ts +// +// Prints the "enc:v1:…" value to store plus a ready-to-edit INSERT. Apollo +// decrypts it on cache load using the same APOLLO_ENC_KEY (see +// platform/src/middleware/auth.ts). Generate a key with: openssl rand -base64 32 +import { encryptKey, parseEncKey } from "../../platform/src/util/instance-key-crypto"; + +const plaintext = process.argv[2]; +if (!plaintext) { + console.error( + "Usage: APOLLO_ENC_KEY= bun services/_instance_auth/encrypt_key.ts " + ); + process.exit(1); +} + +const key = parseEncKey(process.env.APOLLO_ENC_KEY); +if (!key) { + console.error( + "APOLLO_ENC_KEY must be set to base64 of exactly 32 bytes.\n" + + "Generate one with: openssl rand -base64 32" + ); + process.exit(1); +} + +const enc = encryptKey(plaintext, key); + +console.log(enc); +console.log(); +console.log("Example INSERT (set name and the hash from hash_token.py):\n"); +console.log( + " INSERT INTO lightning_clients (name, auth_token_hash, anthropic_api_key)\n" + + ` VALUES ('my-instance', '', '${enc}');` +); diff --git a/services/_instance_auth/schema.sql b/services/_instance_auth/schema.sql index 9e6acebf..3aa5361b 100644 --- a/services/_instance_auth/schema.sql +++ b/services/_instance_auth/schema.sql @@ -15,5 +15,7 @@ CREATE TABLE IF NOT EXISTS lightning_clients ( id SERIAL PRIMARY KEY, name TEXT UNIQUE NOT NULL, -- human identifier for the client auth_token_hash VARCHAR(64) UNIQUE NOT NULL, -- sha256 hex of the client's api_key credential (never the plaintext) - anthropic_api_key TEXT -- Anthropic key Apollo uses for this client; NULL => global env key + anthropic_api_key TEXT -- Anthropic key Apollo uses for this client; NULL => global env key. + -- Plaintext, OR an "enc:v1:..." value from encrypt_key.ts when + -- APOLLO_ENC_KEY is set (see README.md). Both are accepted. ); From cc1a4af47d1e8d8a7077c9a8af52fde18af07458 Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Thu, 18 Jun 2026 15:58:20 +0200 Subject: [PATCH 06/10] review --- platform/src/middleware/auth.ts | 34 +++++++++++++++++++++++++-------- platform/test/server.test.ts | 33 ++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/platform/src/middleware/auth.ts b/platform/src/middleware/auth.ts index 9c36eeba..ba0e85ec 100644 --- a/platform/src/middleware/auth.ts +++ b/platform/src/middleware/auth.ts @@ -54,6 +54,13 @@ let loaderOverride: (() => Promise>) | null = null; // so "Apollo calling itself" skips auth without trusting network position (the // old loopback exemption trusted any local caller, including Lightning). // Honour an operator-provided value; otherwise mint a fresh random one. +// +// MULTI-PROCESS DEPLOYMENTS: when unset, each process mints its OWN random +// token. apollo() self-calls target 127.0.0.1:{port}, so they normally hit the +// same process and the minted token matches. But if several processes share a +// port (e.g. SO_REUSEPORT / clustering), a self-call can land on a sibling +// whose token differs and get a spurious 401. Set APOLLO_INTERNAL_TOKEN to the +// SAME value across all processes in that case so they recognise each other. export const INTERNAL_HEADER = "x-apollo-internal"; const INTERNAL_TOKEN = process.env.APOLLO_INTERNAL_TOKEN ?? randomBytes(32).toString("hex"); @@ -272,7 +279,14 @@ export async function authGate(ctx: any): Promise { // check. External callers can't forge this — it's a per-process secret that // is never sent to clients. const internal = ctx.request?.headers?.get?.(INTERNAL_HEADER) ?? ""; - if (internal && safeEqual(internal, INTERNAL_TOKEN)) return; + if (internal && safeEqual(internal, INTERNAL_TOKEN)) { + // Flag the exemption so apiKeyOverride leaves the payload untouched. Without + // this, an internal call (which never resolves a lightningClient) would have + // any api_key it forwards stripped to the global key — mis-billing a service + // that legitimately passes the per-client key down an apollo() hop. + ctx.internalCall = true; + return; + } // The credential is the api_key the client (Lightning) already sends in the // request body — there is no separate bearer token and no Lightning-side @@ -296,15 +310,19 @@ export async function authGate(ctx: any): Promise { * When auth is OFF, returns {} — the payload is left exactly as the caller sent * it (backward compatible; the legacy "client supplies its own key" behaviour). * - * When auth is ON, the api_key the client sent is ONLY an auth credential and is - * NEVER passed through to the LLM. It is always overwritten: with the matched - * client's stored anthropic_api_key, or — if that client has no stored key — - * with `undefined`, which drops the field on serialisation so Apollo falls back - * to its global ANTHROPIC_API_KEY. Either way the inbound credential cannot - * survive into the payload. + * Internal apollo() self-calls (flagged by authGate) also return {} — they were + * already authenticated upstream, so whatever api_key the calling service chose + * to forward must pass through unchanged rather than being stripped here. + * + * Otherwise (auth ON, external caller) the api_key the client sent is ONLY an + * auth credential and is NEVER passed through to the LLM. It is always + * overwritten: with the matched client's stored anthropic_api_key, or — if that + * client has no stored key — with `undefined`, which drops the field on + * serialisation so Apollo falls back to its global ANTHROPIC_API_KEY. Either way + * the inbound credential cannot survive into the payload. */ export function apiKeyOverride(ctx: any): { api_key?: string } { - if (!enabled) return {}; + if (!enabled || ctx?.internalCall) return {}; const client = ctx?.lightningClient as Client | undefined; return { api_key: client?.anthropicKey ?? undefined }; } diff --git a/platform/test/server.test.ts b/platform/test/server.test.ts index 82099ce7..7da7c80c 100644 --- a/platform/test/server.test.ts +++ b/platform/test/server.test.ts @@ -235,6 +235,39 @@ describe("Instance authentication", () => { const res = await app.handle(req); expect(res.status).toBe(401); }); + + it("passes a forwarded api_key through on internal self-calls untouched", async () => { + // An internal apollo() call was already authenticated upstream, so any + // api_key the calling service forwards (e.g. the resolved per-client key) + // must survive into the payload rather than being stripped to the global + // key. Echo back what the service actually received. + const req = new Request(`${baseUrl}/services/echo`, { + method: "POST", + body: JSON.stringify({ x: 9, api_key: "sk-ant-forwarded" }), + headers: { "Content-Type": "application/json", ...internalAuthHeader() }, + }); + const res = await app.handle(req); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.api_key).toBe("sk-ant-forwarded"); + }); + + it("rejects an unauthenticated WebSocket upgrade", async () => { + // The WS upgrade carries no body, so it has no api_key to validate; the + // beforeHandle gate must reject it rather than let it bypass auth. (Real + // clients use POST/stream, which is where the credential lives.) + const req = new Request(`${baseUrl}/services/echo`, { + method: "GET", + headers: { + Connection: "Upgrade", + Upgrade: "websocket", + "Sec-WebSocket-Key": "dGhlIHNhbXBsZSBub25jZQ==", + "Sec-WebSocket-Version": "13", + }, + }); + const res = await app.handle(req); + expect(res.status).toBe(401); + }); }); }); From 4aa9b798f08eccc057160f89875257930c3895c6 Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Fri, 19 Jun 2026 14:02:02 +0200 Subject: [PATCH 07/10] updates --- README.md | 36 +++++++++-- services/_instance_auth/encrypt_key.ts | 22 +++++-- services/_instance_auth/provision_client.ts | 71 +++++++++++++++++++++ 3 files changed, 117 insertions(+), 12 deletions(-) create mode 100644 services/_instance_auth/provision_client.ts diff --git a/README.md b/README.md index 1cc78f11..c47efccc 100644 --- a/README.md +++ b/README.md @@ -181,18 +181,44 @@ instance auth can be enabled (see below). Python services are hosted at `/services/`. Each service expects a POST request with a JSON body, and will return JSON. +## Database + +First, make sure you've configured your desired `POSTGRES_URL` in your `.env` +file. + +### Create the DB + +``` + set -a; . ./.env; set +a + u="${POSTGRES_URL#*://}"; cred="${u%%@*}"; hostpart="${u#*@}" + PGUSER="${cred%%:*}"; PGPASSWORD="${cred#*:}" + PGPORT="${hostpart#*:}"; PGPORT="${PGPORT%%/*}" + PGDB="${hostpart##*/}"; PGDB="${PGDB%%\?*}" + docker run -d --name apollo-pg -p "${PGPORT}:5432" \ + -e POSTGRES_USER="$PGUSER" -e POSTGRES_PASSWORD="$PGPASSWORD" -e POSTGRES_DB="$PGDB" \ + -v apollo-pg-data:/var/lib/postgresql/data postgres:16 +``` + +### To reset the DB + +`set -a; . ./.env; set +a; psql "$POSTGRES_URL" -c "DROP TABLE IF EXISTS lightning_clients, adaptor_function_docs CASCADE;"` + +### Run the "migrations" (apply both schemas): + +`set -a; . ./.env; set +a; psql "$POSTGRES_URL" -f services/_instance_auth/schema.sql && psql "$POSTGRES_URL" -f services/load_adaptor_docs/schema.sql` + ### Instance authentication (optional) `/services/*` can be gated so that only known clients (e.g. specific Lightning -instances) may call it, with Apollo using **each client's own Anthropic API key** -for that client's requests. +instances) may call it, with Apollo using **each client's own Anthropic API +key** for that client's requests. - It is **opt-in and backward compatible**: auth is active only when the `INSTANCE_AUTH` environment variable is set (e.g. `INSTANCE_AUTH=true`). Otherwise the server stays fully open as before. Tokens are looked up in the - `lightning_clients` table via `POSTGRES_URL`; if auth is enabled but that table - can't be reached, the gate **fails closed** (rejects all external callers) - rather than silently opening up. + `lightning_clients` table via `POSTGRES_URL`; if auth is enabled but that + table can't be reached, the gate **fails closed** (rejects all external + callers) rather than silently opening up. - The credential is the **`api_key` the caller already sends in the request body** — there is no bearer token, no `Authorization` header, and no Lightning-side change. Apollo stores only a SHA-256 hash of it; an diff --git a/services/_instance_auth/encrypt_key.ts b/services/_instance_auth/encrypt_key.ts index 6f7c36b9..74707450 100644 --- a/services/_instance_auth/encrypt_key.ts +++ b/services/_instance_auth/encrypt_key.ts @@ -1,27 +1,35 @@ // Encrypt an Anthropic API key for the lightning_clients.anthropic_api_key -// column, so the key is not stored in the clear. Run from the repo root: +// column, so the key is not stored in the clear. Run from the repo root so Bun +// auto-loads .env (that's where APOLLO_ENC_KEY lives); running from anywhere else +// won't pick up .env and the script will say so: // -// APOLLO_ENC_KEY= \ -// bun services/_instance_auth/encrypt_key.ts +// bun services/_instance_auth/encrypt_key.ts // // Prints the "enc:v1:…" value to store plus a ready-to-edit INSERT. Apollo // decrypts it on cache load using the same APOLLO_ENC_KEY (see -// platform/src/middleware/auth.ts). Generate a key with: openssl rand -base64 32 +// platform/src/middleware/auth.ts). import { encryptKey, parseEncKey } from "../../platform/src/util/instance-key-crypto"; const plaintext = process.argv[2]; if (!plaintext) { console.error( - "Usage: APOLLO_ENC_KEY= bun services/_instance_auth/encrypt_key.ts " + "Usage: bun services/_instance_auth/encrypt_key.ts \n" + + "(run from the repo root so APOLLO_ENC_KEY is read from .env)" ); process.exit(1); } +// Bun auto-loads .env, but only from the directory it's run in. If the key is +// missing it's almost always because this wasn't run from the repo root. const key = parseEncKey(process.env.APOLLO_ENC_KEY); if (!key) { console.error( - "APOLLO_ENC_KEY must be set to base64 of exactly 32 bytes.\n" + - "Generate one with: openssl rand -base64 32" + "APOLLO_ENC_KEY not found (needs to be base64 of exactly 32 bytes — it\n" + + "encrypts the Anthropic key).\n\n" + + "Run this from the repo root so Bun picks up .env. If it's still missing,\n" + + "the key isn't in .env yet — add one with:\n\n" + + ' echo "APOLLO_ENC_KEY=$(openssl rand -base64 32)" >> .env\n\n' + + "then re-run (and restart Apollo so it can decrypt at runtime)." ); process.exit(1); } diff --git a/services/_instance_auth/provision_client.ts b/services/_instance_auth/provision_client.ts new file mode 100644 index 00000000..85ded7bb --- /dev/null +++ b/services/_instance_auth/provision_client.ts @@ -0,0 +1,71 @@ +// Provision a Lightning client for the lightning_clients allow-list in ONE step. +// Given a client name and an Anthropic API key, this: +// 1. mints a fresh api_key credential for the Lightning instance to send, +// 2. computes its SHA-256 hash (what gets stored as auth_token_hash), and +// 3. encrypts the Anthropic key into an "enc:v1:…" value (AES-256-GCM). +// It then prints all three plus a ready-to-run psql INSERT. +// +// Run from the repo root so Bun auto-loads .env (that's where APOLLO_ENC_KEY +// lives). Running from anywhere else won't pick up .env and the script will say so: +// +// bun services/_instance_auth/provision_client.ts +// +// The minted api_key goes to the Lightning instance (it sends it as `api_key`). +// Everything else goes into the DB. The hash here matches the one Apollo computes +// in platform/src/middleware/auth.ts, and the encryption reuses the same crypto +// module Apollo decrypts with, so the formats can never drift. +import { createHash, randomBytes } from "node:crypto"; +import { encryptKey, parseEncKey } from "../../platform/src/util/instance-key-crypto"; + +const name = process.argv[2]; +const anthropicKey = process.argv[3]; + +if (!name || !anthropicKey) { + console.error( + "Usage: APOLLO_ENC_KEY= \\\n" + + " bun services/_instance_auth/provision_client.ts \n\n" + + "Tip: `set -a; . ./.env; set +a;` first to load APOLLO_ENC_KEY from .env." + ); + process.exit(1); +} + +// Bun auto-loads .env, but only from the directory it's run in. If the key is +// missing it's almost always because this wasn't run from the repo root. +const encKey = parseEncKey(process.env.APOLLO_ENC_KEY); +if (!encKey) { + console.error( + "APOLLO_ENC_KEY not found (needs to be base64 of exactly 32 bytes — it\n" + + "encrypts the Anthropic key).\n\n" + + "Run this from the repo root so Bun picks up .env. If it's still missing,\n" + + "the key isn't in .env yet — add one with:\n\n" + + ' echo "APOLLO_ENC_KEY=$(openssl rand -base64 32)" >> .env\n\n' + + "then re-run (and restart Apollo so it can decrypt at runtime)." + ); + process.exit(1); +} + +// 1. The credential Lightning sends as `api_key`. URL-safe so it travels cleanly. +const apiKey = randomBytes(32).toString("base64url"); +// 2. What we store: the SHA-256 hash (lowercase hex over UTF-8 bytes). +const authTokenHash = createHash("sha256").update(apiKey).digest("hex"); +// 3. The Anthropic key, encrypted at rest. +const encAnthropic = encryptKey(anthropicKey, encKey); + +const sqlName = name.replace(/'/g, "''"); +const insert = + "INSERT INTO lightning_clients (name, auth_token_hash, anthropic_api_key)\n" + + ` VALUES ('${sqlName}', '${authTokenHash}', '${encAnthropic}');`; + +console.log(`\n✅ Provisioned client "${name}"\n`); +console.log("1. Lightning api_key — give this to the instance (it sends it as `api_key`):"); +console.log(` ${apiKey}\n`); +console.log("2. auth_token_hash — stored in the DB:"); +console.log(` ${authTokenHash}\n`); +console.log("3. anthropic_api_key (encrypted) — stored in the DB:"); +console.log(` ${encAnthropic}\n`); +console.log("Run this to insert the row (from the repo root):\n"); +console.log( + ` set -a; . ./.env; set +a; psql "$POSTGRES_URL" -c "${insert.replace(/\n\s*/g, " ")}"\n` +); +console.log("Or paste the INSERT directly:\n"); +console.log(` ${insert}\n`); From f04367c50728015aa1a5c0c8b01ae5f4c5bc8d51 Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Fri, 19 Jun 2026 14:09:56 +0200 Subject: [PATCH 08/10] env --- .env.example | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/.env.example b/.env.example index 35b90b00..fb16c8c8 100644 --- a/.env.example +++ b/.env.example @@ -1,22 +1,25 @@ -OPENAI_API_KEY=sk-YOUR-API-KEY-HERE -ANTHROPIC_API_KEY=sk-YOUR-API-KEY-HERE -PINECONE_KEY=YOUR-API-KEY-HERE -POSTGRES_URL=postgresql://localhost:5432/apollo_dev -SENTRY_DSN=YOUR-API-KEY-HERE -GITHUB_TOKEN=KEY +# At-rest encryption for the per-client AI service keys stored in +# lightning_clients.anthropic_api_key. Base64 of 32 random bytes +# (openssl rand -base64 32). When set, encrypt keys with +# services/_instance_auth/encrypt_key.ts before INSERT; plaintext rows still work +# when unset. See services/_instance_auth/README.md. +# APOLLO_ENC_KEY= # Instance auth (optional): set INSTANCE_AUTH=true to require that callers send a # known api_key on /services/*. When enabled, Apollo hashes the request's api_key # and looks it up in the lightning_clients table via POSTGRES_URL (see # services/_instance_auth/). Leave unset/false to keep the server open as before. -INSTANCE_AUTH=false +# INSTANCE_AUTH=true -# Optional at-rest encryption for the per-client Anthropic keys stored in -# lightning_clients.anthropic_api_key. Base64 of 32 random bytes -# (openssl rand -base64 32). When set, encrypt keys with -# services/_instance_auth/encrypt_key.ts before INSERT; plaintext rows still work -# when unset. See services/_instance_auth/README.md. -# APOLLO_ENC_KEY= +# If no authentication is enabled and you want to use a single token for the whole +# Apollo instance... +ANTHROPIC_API_KEY=sk-YOUR-API-KEY-HERE + +OPENAI_API_KEY=sk-YOUR-API-KEY-HERE +PINECONE_KEY=YOUR-API-KEY-HERE +POSTGRES_URL=postgresql://localhost:5432/apollo_dev +SENTRY_DSN=YOUR-API-KEY-HERE +GITHUB_TOKEN=KEY # Langfuse observability LANGFUSE_SECRET_KEY=sk-lf-... From f91e1b25b7a0c2d601abe9f97ead9491d0cd5f52 Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Fri, 19 Jun 2026 14:24:26 +0200 Subject: [PATCH 09/10] slimmer readme --- README.md | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/README.md b/README.md index c47efccc..84a7ab2c 100644 --- a/README.md +++ b/README.md @@ -188,16 +188,7 @@ file. ### Create the DB -``` - set -a; . ./.env; set +a - u="${POSTGRES_URL#*://}"; cred="${u%%@*}"; hostpart="${u#*@}" - PGUSER="${cred%%:*}"; PGPASSWORD="${cred#*:}" - PGPORT="${hostpart#*:}"; PGPORT="${PGPORT%%/*}" - PGDB="${hostpart##*/}"; PGDB="${PGDB%%\?*}" - docker run -d --name apollo-pg -p "${PGPORT}:5432" \ - -e POSTGRES_USER="$PGUSER" -e POSTGRES_PASSWORD="$PGPASSWORD" -e POSTGRES_DB="$PGDB" \ - -v apollo-pg-data:/var/lib/postgresql/data postgres:16 -``` +Create a Postgres DB matching your POSTGRES_URL from the `.env` file ### To reset the DB From 1254aa5ea70deee9a4f2a722156dd91081aa41ed Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Fri, 19 Jun 2026 14:45:07 +0200 Subject: [PATCH 10/10] trim --- .env.example | 20 +-- platform/src/middleware/auth.ts | 171 +++++++------------- platform/src/middleware/services.ts | 8 +- platform/src/server.ts | 1 - platform/src/util/instance-key-crypto.ts | 30 ++-- platform/test/server.test.ts | 32 ++-- services/_instance_auth/README.md | 117 ++++++-------- services/_instance_auth/encrypt_key.ts | 14 +- services/_instance_auth/hash_token.py | 12 +- services/_instance_auth/provision_client.ts | 23 +-- services/_instance_auth/schema.sql | 15 +- services/util.py | 7 +- 12 files changed, 147 insertions(+), 303 deletions(-) diff --git a/.env.example b/.env.example index fb16c8c8..26688dcc 100644 --- a/.env.example +++ b/.env.example @@ -1,18 +1,12 @@ -# At-rest encryption for the per-client AI service keys stored in -# lightning_clients.anthropic_api_key. Base64 of 32 random bytes -# (openssl rand -base64 32). When set, encrypt keys with -# services/_instance_auth/encrypt_key.ts before INSERT; plaintext rows still work -# when unset. See services/_instance_auth/README.md. -# APOLLO_ENC_KEY= - -# Instance auth (optional): set INSTANCE_AUTH=true to require that callers send a -# known api_key on /services/*. When enabled, Apollo hashes the request's api_key -# and looks it up in the lightning_clients table via POSTGRES_URL (see -# services/_instance_auth/). Leave unset/false to keep the server open as before. +# Instance auth (optional): set INSTANCE_AUTH=true to gate /services/* on the +# api_key callers send, looked up in lightning_clients via POSTGRES_URL. See +# services/_instance_auth/README.md. # INSTANCE_AUTH=true -# If no authentication is enabled and you want to use a single token for the whole -# Apollo instance... +# Optional at-rest encryption for stored client Anthropic keys. Base64 of 32 bytes +# (openssl rand -base64 32). See services/_instance_auth/README.md. +# APOLLO_ENC_KEY= + ANTHROPIC_API_KEY=sk-YOUR-API-KEY-HERE OPENAI_API_KEY=sk-YOUR-API-KEY-HERE diff --git a/platform/src/middleware/auth.ts b/platform/src/middleware/auth.ts index ba0e85ec..b6742d14 100644 --- a/platform/src/middleware/auth.ts +++ b/platform/src/middleware/auth.ts @@ -3,64 +3,44 @@ import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; import type { ApolloError } from "../util/errors"; import { ENC_PREFIX, decryptKey, parseEncKey } from "../util/instance-key-crypto"; -// A row from the lightning_clients table, as used at runtime. type Client = { name: string; anthropicKey: string | null }; - -// A lookup resolves a token hash to a client, or null if unknown. type Lookup = (hash: string) => Promise | Client | null; -// Instance auth is opt-in via the INSTANCE_AUTH env var (see initAuth). When it -// is unset/falsey, /services/* is open exactly as before. When set, every -// /services/* request must carry an api_key (in the body) that matches a known -// client. +// Opt-in via INSTANCE_AUTH (see initAuth); when off, /services/* is open as before. let enabled = false; -// True once the lightning_clients lookup is actually usable. When auth is -// enabled but the DB/table can't be reached, this stays false and the gate -// fails CLOSED (rejects every external caller) rather than silently opening up. +// False until the lightning_clients lookup is usable. Auth-on but unusable DB => +// the gate fails CLOSED (rejects every external caller) rather than opening up. let dbReady = false; -// When set (by tests), this replaces the DB-backed lookup so the suite can -// enable auth with a known token set without a real Postgres. let lookupOverride: Lookup | null = null; - -// Bun's native Postgres client (Bun >= 1.2). Created once in initAuth(). let sql: SQL | null = null; -// In-memory cache of all clients keyed by token hash, refreshed on a TTL so -// rows added/revoked directly in the DB are picked up within CACHE_TTL_MS. +// In-memory client cache (keyed by token hash), refreshed on a TTL so DB changes +// are picked up within CACHE_TTL_MS. const CACHE_TTL_MS = 60_000; let clientCache: Map | null = null; let cacheTs = 0; -// Single-flight handle for the cache refresh. Concurrent refreshes (e.g. a burst -// of requests hitting the TTL boundary at once) all share THIS one promise, so a -// refresh can never trigger more than one DB read no matter the concurrency. +// Single-flight handle: concurrent refreshes share this one promise, so a refresh +// can never trigger more than one DB read no matter the concurrency. let refreshInFlight: Promise> | null = null; -// Master key (32 bytes) for decrypting stored anthropic_api_key values, parsed -// from APOLLO_ENC_KEY in initAuth. Null when unset — plaintext rows still work, -// but any "enc:v1:" row can't be decrypted and its client is omitted. +// Master key for decrypting stored anthropic_api_key values (APOLLO_ENC_KEY). Null +// when unset — plaintext rows still work, but "enc:v1:" rows can't be decrypted. let encKey: Buffer | null = null; -// When set (by tests), this replaces loadClients in the refresh path so the -// single-flight/stale-while-revalidate behaviour can be exercised without a DB. let loaderOverride: (() => Promise>) | null = null; -// A per-process secret that identifies genuine Apollo-to-Apollo calls. The -// bridge spawns Python services as child processes that inherit this process's -// env, so exporting the token here lets services/util.py apollo() echo it back -// via the X-Apollo-Internal header. authGate exempts any request carrying it, -// so "Apollo calling itself" skips auth without trusting network position (the -// old loopback exemption trusted any local caller, including Lightning). -// Honour an operator-provided value; otherwise mint a fresh random one. +// Per-process secret identifying genuine Apollo-to-Apollo calls. The bridge spawns +// Python children that inherit this env, so services/util.py apollo() echoes it back +// via the internal header; authGate exempts requests carrying it without trusting +// network position. Honour an operator-provided value, else mint a random one. // -// MULTI-PROCESS DEPLOYMENTS: when unset, each process mints its OWN random -// token. apollo() self-calls target 127.0.0.1:{port}, so they normally hit the -// same process and the minted token matches. But if several processes share a -// port (e.g. SO_REUSEPORT / clustering), a self-call can land on a sibling -// whose token differs and get a spurious 401. Set APOLLO_INTERNAL_TOKEN to the -// SAME value across all processes in that case so they recognise each other. +// MULTI-PROCESS: when unset, each process mints its OWN token. apollo() self-calls +// hit 127.0.0.1:{port} and normally land on the same process, but if processes +// share a port (SO_REUSEPORT / clustering) a self-call can hit a sibling and 401. +// Set APOLLO_INTERNAL_TOKEN to the SAME value across processes in that case. export const INTERNAL_HEADER = "x-apollo-internal"; const INTERNAL_TOKEN = process.env.APOLLO_INTERNAL_TOKEN ?? randomBytes(32).toString("hex"); @@ -73,27 +53,22 @@ function safeEqual(a: string, b: string): boolean { return ab.length === bb.length && timingSafeEqual(ab, bb); } -/** Header set marking a request as an internal apollo() self-call (used by tests). */ export function internalAuthHeader(): Record { return { [INTERNAL_HEADER]: INTERNAL_TOKEN }; } -/** SHA-256 hex of a client credential (the api_key). Must match services/_instance_auth/hash_token.py. */ +/** SHA-256 hex of a client credential. Must match services/_instance_auth/hash_token.py. */ export function hashToken(token: string): string { return createHash("sha256").update(token).digest("hex"); } -// Sentinel returned by decryptStoredKey when an encrypted value can't be -// decrypted. Such a client is omitted from the cache (fail closed) rather than -// silently falling back to the global key, which would mis-bill its usage. +// Returned when an encrypted key can't be decrypted: that client is omitted from +// the cache (fail closed) rather than falling back to the global key, which would +// mis-bill its usage. const DECRYPT_FAILED = Symbol("decrypt-failed"); -/** - * Resolve the anthropic_api_key column to the plaintext key Apollo should use. - * - null → null (intentional: this client uses the global env key) - * - "enc:v1:…" → AES-256-GCM decrypt with encKey; DECRYPT_FAILED on error - * - anything else → legacy plaintext, used as-is (backward compatible) - */ +// null => global env key; "enc:v1:…" => AES-256-GCM decrypt (DECRYPT_FAILED on +// error); anything else => legacy plaintext. function decryptStoredKey( stored: string | null, clientName: string @@ -117,8 +92,8 @@ function decryptStoredKey( } } -/** Build the hash→client map from raw rows, decrypting keys and dropping any - * client whose encrypted key can't be decrypted. Exported for tests. */ +/** Build the hash→client map, dropping any client whose encrypted key can't be + * decrypted (fail closed). Exported for tests. */ export function buildClientMap( rows: Array<{ name: string; @@ -146,11 +121,8 @@ async function loadClients(): Promise> { return buildClientMap(rows); } -/** - * Refresh the client cache, collapsing concurrent callers onto a single DB read - * (single-flight). The in-flight promise is cleared in finally so a failed - * refresh never wedges the cache — the next caller retries. - */ +// Single-flight refresh; the in-flight handle is cleared in finally so a failed +// refresh never wedges the cache (the next caller retries). function refreshClients(): Promise> { if (refreshInFlight) return refreshInFlight; refreshInFlight = (loaderOverride ?? loadClients)() @@ -166,22 +138,19 @@ function refreshClients(): Promise> { } async function dbLookup(hash: string): Promise { - // Auth is on but the lookup never came up — reject (fail closed). - if (!dbReady) return null; + if (!dbReady) return null; // auth on but lookup never came up -> fail closed if (!loaderOverride && !sql) return null; const fresh = clientCache !== null && Date.now() - cacheTs <= CACHE_TTL_MS; if (!fresh) { if (clientCache !== null) { // Warm but stale: serve the current map now and refresh once in the - // background. Errors are swallowed so a DB blip keeps serving stale (and - // retries next request, since cacheTs only advances on success) instead - // of 500ing every caller. The single-flight handle guarantees one read. + // background. Errors are swallowed so a DB blip keeps serving stale (cacheTs + // only advances on success, so the next request retries). void refreshClients().catch(() => {}); } else { - // Cold start: nothing to serve yet. Every concurrent caller awaits the - // ONE shared load rather than each firing its own. On failure leave the - // cache null and reject (fail closed). + // Cold start: every concurrent caller awaits the one shared load. On failure + // leave the cache null and fail closed. try { await refreshClients(); } catch { @@ -192,20 +161,14 @@ async function dbLookup(hash: string): Promise { return clientCache?.get(hash) ?? null; } -/** Whether the INSTANCE_AUTH env var opts this instance into auth. */ function authOptIn(): boolean { const v = (process.env.INSTANCE_AUTH ?? "").trim().toLowerCase(); return v === "1" || v === "true" || v === "yes" || v === "on"; } -/** - * Decide once at startup whether instance auth is active. The INSTANCE_AUTH env - * var is the master switch: unset/falsey leaves /services/* open as before. - * When set, auth is enabled and tokens are looked up in the lightning_clients - * table (via POSTGRES_URL). If that table can't be reached, auth stays enabled - * but the gate fails CLOSED (rejects every external caller) — an explicit - * opt-in must never silently fall back to open. - */ +// Decide once at startup whether auth is active. When INSTANCE_AUTH is set but the +// lightning_clients table can't be reached, auth stays on and the gate fails CLOSED +// — an explicit opt-in must never silently fall back to open. export async function initAuth(): Promise { enabled = authOptIn(); if (!enabled) { @@ -216,9 +179,6 @@ export async function initAuth(): Promise { return; } - // Parse the at-rest encryption key for stored anthropic_api_key values. - // Optional: when unset, plaintext rows keep working; only "enc:v1:" rows need - // it. When set but malformed, warn loudly — encrypted clients will be rejected. encKey = parseEncKey(process.env.APOLLO_ENC_KEY); if (process.env.APOLLO_ENC_KEY && !encKey) { console.error( @@ -266,32 +226,26 @@ function unauthorized(ctx: any): ApolloError { return { code: 401, type: "UNAUTHORIZED", message: "Missing or invalid API key" }; } -/** - * Elysia onBeforeHandle hook scoped to the /services group. Returning a value - * short-circuits the request with that value as the response body. - * On success it stashes the resolved client on the context for apiKeyOverride. - */ +// Elysia onBeforeHandle hook for the /services group. Returning a value +// short-circuits the request with that body; on success it stashes the resolved +// client on the context for apiKeyOverride. export async function authGate(ctx: any): Promise { if (!enabled) return; - // Apollo calling itself: the bridge's Python children echo back the internal - // token (see services/util.py apollo()), so such calls skip the api_key - // check. External callers can't forge this — it's a per-process secret that - // is never sent to clients. + // Apollo calling itself: Python children echo back the internal token (see + // services/util.py apollo()), so such calls skip the api_key check. External + // callers can't forge it — it's a per-process secret never sent to clients. const internal = ctx.request?.headers?.get?.(INTERNAL_HEADER) ?? ""; if (internal && safeEqual(internal, INTERNAL_TOKEN)) { - // Flag the exemption so apiKeyOverride leaves the payload untouched. Without - // this, an internal call (which never resolves a lightningClient) would have - // any api_key it forwards stripped to the global key — mis-billing a service - // that legitimately passes the per-client key down an apollo() hop. + // Flag so apiKeyOverride leaves the forwarded api_key untouched rather than + // stripping it to the global key — which would mis-bill a per-client key + // passed down an apollo() hop. ctx.internalCall = true; return; } - // The credential is the api_key the client (Lightning) already sends in the - // request body — there is no separate bearer token and no Lightning-side - // change. Hash it and look for a matching row in lightning_clients; an absent - // or unknown key is rejected. + // The credential is the api_key the caller sends in the body; hash it and look + // for a matching client. Absent or unknown -> 401. const apiKey = typeof ctx.body?.api_key === "string" ? ctx.body.api_key.trim() : ""; if (!apiKey) return unauthorized(ctx); @@ -304,22 +258,10 @@ export async function authGate(ctx: any): Promise { } /** - * Resolve the api_key for the outgoing service payload. Merged LAST over the - * request body so it always wins. - * - * When auth is OFF, returns {} — the payload is left exactly as the caller sent - * it (backward compatible; the legacy "client supplies its own key" behaviour). - * - * Internal apollo() self-calls (flagged by authGate) also return {} — they were - * already authenticated upstream, so whatever api_key the calling service chose - * to forward must pass through unchanged rather than being stripped here. - * - * Otherwise (auth ON, external caller) the api_key the client sent is ONLY an - * auth credential and is NEVER passed through to the LLM. It is always - * overwritten: with the matched client's stored anthropic_api_key, or — if that - * client has no stored key — with `undefined`, which drops the field on - * serialisation so Apollo falls back to its global ANTHROPIC_API_KEY. Either way - * the inbound credential cannot survive into the payload. + * Resolve the api_key for the outgoing payload (merged LAST so it wins). Auth off, + * or an internal self-call: return {} (passthrough). Otherwise the inbound + * credential is NEVER forwarded to the LLM — it's replaced with the client's stored + * key, or undefined (dropped on serialise) so Apollo falls back to its global key. */ export function apiKeyOverride(ctx: any): { api_key?: string } { if (!enabled || ctx?.internalCall) return {}; @@ -327,7 +269,8 @@ export function apiKeyOverride(ctx: any): { api_key?: string } { return { api_key: client?.anthropicKey ?? undefined }; } -/** Test seam: pass a fake lookup to enable auth without a DB, or null to disable. */ +// --- Test seams --- + export function __setAuthForTest(provider: Lookup | null): void { if (provider) { enabled = true; @@ -338,12 +281,8 @@ export function __setAuthForTest(provider: Lookup | null): void { } } -/** - * Test seam: drive the real dbLookup (single-flight + stale-while-revalidate) - * with a fake table loader instead of Postgres. Passing a loader enables auth on - * the dbLookup path (lookupOverride cleared) and resets cache state; null tears - * it back down. - */ +// Drives the real dbLookup (single-flight + stale-while-revalidate) with a fake +// loader instead of Postgres; null tears it back down. export function __setLoaderForTest( loader: (() => Promise>) | null ): void { @@ -360,12 +299,10 @@ export function __setLoaderForTest( } } -/** Test seam: mark the cache stale without touching its contents. */ export function __expireCacheForTest(): void { cacheTs = 0; } -/** Test seam: set the at-rest decryption key (or null) without running initAuth. */ export function __setEncKeyForTest(key: Buffer | null): void { encKey = key; } diff --git a/platform/src/middleware/services.ts b/platform/src/middleware/services.ts index 6e08b418..0dbc1b2b 100644 --- a/platform/src/middleware/services.ts +++ b/platform/src/middleware/services.ts @@ -140,11 +140,9 @@ export default async (app: Elysia, port: number) => { // TODO in the web socket API, does it make more sense to open a socket at root // and then pick the service you want? So you'd connect to /ws an send { call: 'echo', payload: {} } app.ws(name, { - // Gate the WS upgrade too, so it can't be used to bypass auth. The - // credential lives in the request body (api_key), which a WS upgrade - // doesn't carry — so when auth is on, WS upgrades are rejected. That's - // fine: Lightning uses the POST and /stream transports, which is also - // where the api_key is validated and the per-client key swapped in. + // Gate the WS upgrade too. It carries no body, so it has no api_key to + // validate; when auth is on, WS upgrades are rejected. Fine — Lightning + // uses the POST and /stream transports, where the credential lives. beforeHandle: authGate, open() { console.log(`Websocket connected at /services/${name}`); diff --git a/platform/src/server.ts b/platform/src/server.ts index cbd82476..9bb47e89 100644 --- a/platform/src/server.ts +++ b/platform/src/server.ts @@ -20,7 +20,6 @@ export default async (port: number | string = 3000) => { await setupDir(app); await setupServices(app, +port); - // Decide whether /services/* is gated by an instance token (see auth.ts). await initAuth(); console.log("Apollo Server listening on ", port); diff --git a/platform/src/util/instance-key-crypto.ts b/platform/src/util/instance-key-crypto.ts index 39888c3b..6cab7669 100644 --- a/platform/src/util/instance-key-crypto.ts +++ b/platform/src/util/instance-key-crypto.ts @@ -1,22 +1,15 @@ -// AES-256-GCM helpers for the per-client anthropic_api_key stored in -// lightning_clients. Used by the auth middleware (to decrypt on cache load) and -// by services/_instance_auth/encrypt_key.ts (to produce values to INSERT). Kept -// in one module so the byte format can never drift between the two sides. -// -// Stored format: "enc:v1:". The master -// key is APOLLO_ENC_KEY (base64 of 32 bytes). Anything NOT prefixed with the -// tag is treated as legacy plaintext elsewhere, so encryption is opt-in and -// backward compatible. +// AES-256-GCM helpers for the per-client anthropic_api_key in lightning_clients. +// Shared by the auth middleware (decrypt) and encrypt_key.ts (encrypt) so the byte +// format can't drift. Stored format: "enc:v1:"; master key is APOLLO_ENC_KEY (base64 of 32 bytes). Values without +// the prefix are treated as legacy plaintext elsewhere, so encryption is opt-in. import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; export const ENC_PREFIX = "enc:v1:"; -const IV_BYTES = 12; // GCM standard nonce length -const TAG_BYTES = 16; // GCM auth tag length +const IV_BYTES = 12; // GCM nonce +const TAG_BYTES = 16; // GCM auth tag -/** - * Decode APOLLO_ENC_KEY (base64 of exactly 32 bytes) into a key Buffer, or null - * if it is absent or malformed. Callers treat null as "no key configured". - */ +/** Decode APOLLO_ENC_KEY (base64 of exactly 32 bytes) into a key Buffer, or null if absent/malformed. */ export function parseEncKey(raw: string | undefined | null): Buffer | null { if (!raw) return null; let buf: Buffer; @@ -28,7 +21,6 @@ export function parseEncKey(raw: string | undefined | null): Buffer | null { return buf.length === 32 ? buf : null; } -/** Encrypt plaintext into an "enc:v1:…" value using AES-256-GCM. */ export function encryptKey(plaintext: string, key: Buffer): string { const iv = randomBytes(IV_BYTES); const cipher = createCipheriv("aes-256-gcm", key, iv); @@ -40,11 +32,7 @@ export function encryptKey(plaintext: string, key: Buffer): string { return ENC_PREFIX + Buffer.concat([iv, tag, ciphertext]).toString("base64"); } -/** - * Decrypt an "enc:v1:…" value. Throws if the key is wrong, the value is - * corrupt, or the auth tag fails — callers decide how to handle the failure - * (the auth middleware omits the client and fails closed). - */ +/** Decrypt an "enc:v1:…" value; throws on wrong key, corrupt value, or failed auth tag. */ export function decryptKey(stored: string, key: Buffer): string { const blob = Buffer.from(stored.slice(ENC_PREFIX.length), "base64"); const iv = blob.subarray(0, IV_BYTES); diff --git a/platform/test/server.test.ts b/platform/test/server.test.ts index 7da7c80c..78268856 100644 --- a/platform/test/server.test.ts +++ b/platform/test/server.test.ts @@ -19,9 +19,9 @@ const baseUrl = `http://localhost:${port}`; const app = await setup(port); -// setup() runs initAuth(), which enables auth if the dev's .env sets -// INSTANCE_AUTH. Force auth off so the suite is deterministic; the auth block -// below opts back in via the test seam. +// setup() runs initAuth(), which would enable auth if the dev's .env sets +// INSTANCE_AUTH. Force it off so the suite is deterministic; the auth block below +// opts back in via the test seam. __setAuthForTest(null); const get = (path: string) => { @@ -149,11 +149,8 @@ describe("Python Services", () => { }); describe("Instance authentication", () => { - // The credential is the api_key the client sends in the body. Clients are - // keyed by the SHA-256 of that credential (no real DB — this is the seam). - // ALPHA has a stored Anthropic key (Apollo swaps it in); BETA has none (the - // credential is stripped and Apollo falls back to its global key). Neither - // credential may ever pass through to the LLM call. + // No real DB — the seam keys clients by SHA-256 of the api_key they send. ALPHA + // has a stored Anthropic key (swapped in); BETA has none (credential stripped). const ALPHA = "lightning-cred-alpha"; const BETA = "lightning-cred-beta"; const clients: Record = { @@ -165,7 +162,6 @@ describe("Instance authentication", () => { post(path, { ...data, ...(apiKey ? { api_key: apiKey } : {}) }); afterEach(() => { - // Restore the open state the rest of the suite expects. __setAuthForTest(null); }); @@ -185,7 +181,6 @@ describe("Instance authentication", () => { expect(res.status).toBe(200); const body = await res.json(); expect(body.x).toBe(1); - // The stored key replaces the credential; the credential never passes through. expect(body.api_key).toBe("sk-ant-stored-alpha"); expect(body.api_key).not.toBe(ALPHA); }); @@ -237,10 +232,8 @@ describe("Instance authentication", () => { }); it("passes a forwarded api_key through on internal self-calls untouched", async () => { - // An internal apollo() call was already authenticated upstream, so any - // api_key the calling service forwards (e.g. the resolved per-client key) - // must survive into the payload rather than being stripped to the global - // key. Echo back what the service actually received. + // Already authenticated upstream, so a forwarded api_key must survive into + // the payload rather than being stripped to the global key. const req = new Request(`${baseUrl}/services/echo`, { method: "POST", body: JSON.stringify({ x: 9, api_key: "sk-ant-forwarded" }), @@ -253,9 +246,8 @@ describe("Instance authentication", () => { }); it("rejects an unauthenticated WebSocket upgrade", async () => { - // The WS upgrade carries no body, so it has no api_key to validate; the - // beforeHandle gate must reject it rather than let it bypass auth. (Real - // clients use POST/stream, which is where the credential lives.) + // The WS upgrade carries no body, so no api_key to validate; the gate must + // reject it rather than let it bypass auth. const req = new Request(`${baseUrl}/services/echo`, { method: "GET", headers: { @@ -272,10 +264,8 @@ describe("Instance authentication", () => { }); describe("Instance auth cache refresh", () => { - // Drive the real dbLookup (single-flight + stale-while-revalidate) with a fake - // table loader so we can assert how many DB reads a request burst causes. - // authGate is called directly with a minimal ctx to avoid spawning the echo - // service for every request in the burst. + // Drive the real dbLookup with a fake loader so we can count DB reads per burst. + // authGate is called directly with a minimal ctx (no echo service spawned). const ALPHA = "lightning-cred-alpha"; const mapWith = (anthropicKey: string | null) => new Map([[hashToken(ALPHA), { name: "alpha", anthropicKey }]]); diff --git a/services/_instance_auth/README.md b/services/_instance_auth/README.md index 17a73d14..860f573a 100644 --- a/services/_instance_auth/README.md +++ b/services/_instance_auth/README.md @@ -5,7 +5,7 @@ call them, and makes Apollo use **its own per-client Anthropic API key** for eac request rather than trusting anything the caller sends. This directory is not a mounted service (the leading `_` keeps it off the HTTP -router). It is just the schema and a helper script; the actual gate lives in +router). It holds the schema and provisioning scripts; the actual gate lives in `platform/src/middleware/auth.ts`. ## How it works @@ -40,101 +40,78 @@ router). It is just the schema and a helper script; the actual gate lives in are never gated. Internal Apollo-to-Apollo `apollo()` calls are exempt via a per-process internal token (`APOLLO_INTERNAL_TOKEN`), not by network position. -## Enabling it +## Setting up a client -1. Make sure `POSTGRES_URL` is set, then create the table: +`provision_client.ts` is the canonical way to add a Lightning client: it mints the +credential, hashes it, and encrypts the client's Anthropic key in one step. + +1. Create the table (once): ```sh + set -a; . ./.env; set +a psql "$POSTGRES_URL" -f services/_instance_auth/schema.sql ``` -2. Get the SHA-256 hash of the `api_key` a given Lightning instance sends: +2. Set a master encryption key in `.env` (once) — `provision_client.ts` uses it to + encrypt each client's Anthropic key at rest: ```sh - poetry run python services/_instance_auth/hash_token.py + echo "APOLLO_ENC_KEY=$(openssl rand -base64 32)" >> .env ``` - (Run with no argument to instead mint a fresh credential to hand to the - instance.) This prints the hash to store and a ready-to-edit `INSERT`. - -3. Insert the row. Set `name`, the hash, and the Anthropic key Apollo should use - for this client (leave `anthropic_api_key` as `NULL` to use Apollo's global - key): +3. Provision the client with a name and the Anthropic key Apollo should use for it: - ```sql - INSERT INTO lightning_clients (name, auth_token_hash, anthropic_api_key) - VALUES ('my-lightning-instance', '', 'sk-ant-...'); + ```sh + set -a; . ./.env; set +a + bun services/_instance_auth/provision_client.ts ``` - To store the Anthropic key encrypted instead of in the clear, see - [Encrypting the stored Anthropic keys](#encrypting-the-stored-anthropic-keys-optional) - below and paste the `enc:v1:…` value in place of `'sk-ant-...'`. + This prints the `api_key` to give the Lightning instance and a ready-to-run + `psql` `INSERT`. Run that `INSERT` to add the row. -4. Set `INSTANCE_AUTH=true` in Apollo's environment and restart. The startup log - shows `Apollo instance auth ENABLED`. (If `INSTANCE_AUTH` is set but the table - is missing, the log warns and every external caller is rejected.) +4. Enable auth: set `INSTANCE_AUTH=true` in Apollo's environment and restart. The + startup log shows `Apollo instance auth ENABLED`. (If `INSTANCE_AUTH` is set but + the table is missing, the log warns and every external caller is rejected.) -No Lightning-side configuration is needed: it keeps sending its `api_key` exactly -as it does now. +5. Give the printed `api_key` to the Lightning instance. It keeps sending it as + `api_key` exactly as it does today — no other Lightning-side change. ## Managing clients -There is no CLI — manage rows directly in the DB: - -- **Revoke a client:** `DELETE FROM lightning_clients WHERE name = '...';` -- **Rotate a credential:** hash the new `api_key` with `hash_token.py` and - `UPDATE lightning_clients SET auth_token_hash = '' WHERE name = '...';` -- **Change the Anthropic key Apollo uses for a client:** - `UPDATE lightning_clients SET anthropic_api_key = '...' WHERE name = '...';` - -Changes are picked up within ~60s (the server caches the client list briefly). If -you need a revocation to take effect immediately, restart Apollo. - -## Encrypting the stored Anthropic keys (optional) - -By default `anthropic_api_key` is stored as plaintext. You can instead encrypt it -at rest with AES-256-GCM so a DB dump/backup/replica leak doesn't expose live -keys: +Manage rows directly in the DB. Changes are picked up within ~60s (the server +caches the client list briefly); restart Apollo to apply a revocation immediately. -1. Generate a 32-byte master key and set it in Apollo's environment: +- **Revoke:** `DELETE FROM lightning_clients WHERE name = '...';` +- **Rotate a credential / change the Anthropic key:** re-run `provision_client.ts` + and apply its output as an `UPDATE` for the existing `name` (it prints an + `INSERT` — swap the verb). - ```sh - openssl rand -base64 32 # → put the output in APOLLO_ENC_KEY - ``` +### Lower-level scripts -2. Encrypt a key to get the value to store: +`provision_client.ts` is built from two primitives, occasionally useful on their +own — e.g. to add a client whose `anthropic_api_key` is `NULL` (so it uses Apollo's +global `ANTHROPIC_API_KEY`), which `provision_client.ts` doesn't cover: - ```sh - APOLLO_ENC_KEY= \ - bun services/_instance_auth/encrypt_key.ts sk-ant-... - ``` - - This prints an `enc:v1:…` blob (and a ready-to-edit `INSERT`). Use it in place - of the plaintext key: - - ```sql - UPDATE lightning_clients SET anthropic_api_key = 'enc:v1:...' WHERE name = '...'; - ``` +- `hash_token.py ` — hash (or, with no argument, mint) just the credential. +- `encrypt_key.ts ` — encrypt just an Anthropic key to an `enc:v1:…` + value. -3. Restart Apollo with `APOLLO_ENC_KEY` set. It decrypts each key once per ~60s - cache refresh. +## At-rest encryption -Notes: +`anthropic_api_key` is stored encrypted (AES-256-GCM) when written via +`provision_client.ts`/`encrypt_key.ts`; plaintext rows are still accepted for +backward compatibility. -- **Backward compatible / opt-in.** Plaintext rows keep working; only `enc:v1:` - rows need `APOLLO_ENC_KEY`. You can migrate one client at a time. - **Fail closed.** If an `enc:v1:` row can't be decrypted (wrong/missing `APOLLO_ENC_KEY` or corrupt value), that client is dropped from the allow-list - and its requests get `401` — Apollo never silently falls back to the global key - for an encrypted-but-undecryptable row. A `NULL` key still means "use the - global key" as before. -- **Rotation** is manual: decrypt and re-encrypt every `enc:v1:` row with the new - key, then swap `APOLLO_ENC_KEY` and restart. -- **What it protects.** The ciphertext is useless without `APOLLO_ENC_KEY`, so - this guards DB dumps, backups, read replicas, and accidental `SELECT`s in logs. - It does **not** protect a full Apollo host/process compromise: the running - process necessarily holds both the key and the decrypted values in memory - (Apollo must use the key to call Anthropic). Continue to protect the table at - rest (restricted access, DB encryption) regardless. + and its requests get `401` — Apollo never falls back to the global key for an + encrypted-but-undecryptable row. A `NULL` key still means "use the global key". +- **Rotation** is manual: re-encrypt every `enc:v1:` row with the new key, then + swap `APOLLO_ENC_KEY` and restart. +- **What it protects.** The ciphertext is useless without `APOLLO_ENC_KEY`, so this + guards DB dumps, backups, read replicas, and accidental `SELECT`s in logs. It + does **not** protect a full Apollo host/process compromise: the running process + necessarily holds both the key and the decrypted values in memory. Protect the + table at rest (restricted access, DB encryption) regardless. The clients' `api_key` credentials are only ever stored and compared as hashes. diff --git a/services/_instance_auth/encrypt_key.ts b/services/_instance_auth/encrypt_key.ts index 74707450..82db7ac2 100644 --- a/services/_instance_auth/encrypt_key.ts +++ b/services/_instance_auth/encrypt_key.ts @@ -1,13 +1,7 @@ -// Encrypt an Anthropic API key for the lightning_clients.anthropic_api_key -// column, so the key is not stored in the clear. Run from the repo root so Bun -// auto-loads .env (that's where APOLLO_ENC_KEY lives); running from anywhere else -// won't pick up .env and the script will say so: -// +// Encrypt an Anthropic API key for lightning_clients.anthropic_api_key. Run from +// the repo root so Bun auto-loads .env (where APOLLO_ENC_KEY lives): // bun services/_instance_auth/encrypt_key.ts -// -// Prints the "enc:v1:…" value to store plus a ready-to-edit INSERT. Apollo -// decrypts it on cache load using the same APOLLO_ENC_KEY (see -// platform/src/middleware/auth.ts). +// Prints the "enc:v1:…" value to store plus a ready-to-edit INSERT. import { encryptKey, parseEncKey } from "../../platform/src/util/instance-key-crypto"; const plaintext = process.argv[2]; @@ -19,8 +13,6 @@ if (!plaintext) { process.exit(1); } -// Bun auto-loads .env, but only from the directory it's run in. If the key is -// missing it's almost always because this wasn't run from the repo root. const key = parseEncKey(process.env.APOLLO_ENC_KEY); if (!key) { console.error( diff --git a/services/_instance_auth/hash_token.py b/services/_instance_auth/hash_token.py index 9a53cc80..69a81767 100644 --- a/services/_instance_auth/hash_token.py +++ b/services/_instance_auth/hash_token.py @@ -1,17 +1,13 @@ -"""Hash (or mint) a Lightning client's api_key credential for the lightning_clients table. - -Usage (run from the repo root): +"""Hash (or mint) a Lightning client's api_key credential for lightning_clients. # Hash the api_key a Lightning instance already sends: poetry run python services/_instance_auth/hash_token.py - # Or mint a fresh credential and print its hash + a ready-to-edit INSERT: + # Or mint a fresh credential + print its hash and a ready-to-edit INSERT: poetry run python services/_instance_auth/hash_token.py -The credential is whatever the client sends as `api_key` in the request body -- -there is no bearer token. Only its SHA-256 hash is stored in the DB. The hash -here must match the one Apollo computes in platform/src/middleware/auth.ts -(sha256 over the UTF-8 bytes, lowercase hex) -- it does. +Only the SHA-256 hash is stored. It must match the hash Apollo computes in +platform/src/middleware/auth.ts (sha256 over UTF-8 bytes, lowercase hex). """ import hashlib diff --git a/services/_instance_auth/provision_client.ts b/services/_instance_auth/provision_client.ts index 85ded7bb..3d51d495 100644 --- a/services/_instance_auth/provision_client.ts +++ b/services/_instance_auth/provision_client.ts @@ -1,19 +1,7 @@ -// Provision a Lightning client for the lightning_clients allow-list in ONE step. -// Given a client name and an Anthropic API key, this: -// 1. mints a fresh api_key credential for the Lightning instance to send, -// 2. computes its SHA-256 hash (what gets stored as auth_token_hash), and -// 3. encrypts the Anthropic key into an "enc:v1:…" value (AES-256-GCM). -// It then prints all three plus a ready-to-run psql INSERT. -// -// Run from the repo root so Bun auto-loads .env (that's where APOLLO_ENC_KEY -// lives). Running from anywhere else won't pick up .env and the script will say so: -// +// Provision a Lightning client in one step: mint an api_key credential, hash it +// (auth_token_hash), and encrypt the Anthropic key (enc:v1:…). Prints all three +// plus a ready-to-run INSERT. Run from the repo root so Bun loads .env (APOLLO_ENC_KEY): // bun services/_instance_auth/provision_client.ts -// -// The minted api_key goes to the Lightning instance (it sends it as `api_key`). -// Everything else goes into the DB. The hash here matches the one Apollo computes -// in platform/src/middleware/auth.ts, and the encryption reuses the same crypto -// module Apollo decrypts with, so the formats can never drift. import { createHash, randomBytes } from "node:crypto"; import { encryptKey, parseEncKey } from "../../platform/src/util/instance-key-crypto"; @@ -29,8 +17,6 @@ if (!name || !anthropicKey) { process.exit(1); } -// Bun auto-loads .env, but only from the directory it's run in. If the key is -// missing it's almost always because this wasn't run from the repo root. const encKey = parseEncKey(process.env.APOLLO_ENC_KEY); if (!encKey) { console.error( @@ -44,11 +30,8 @@ if (!encKey) { process.exit(1); } -// 1. The credential Lightning sends as `api_key`. URL-safe so it travels cleanly. const apiKey = randomBytes(32).toString("base64url"); -// 2. What we store: the SHA-256 hash (lowercase hex over UTF-8 bytes). const authTokenHash = createHash("sha256").update(apiKey).digest("hex"); -// 3. The Anthropic key, encrypted at rest. const encAnthropic = encryptKey(anthropicKey, encKey); const sqlName = name.replace(/'/g, "''"); diff --git a/services/_instance_auth/schema.sql b/services/_instance_auth/schema.sql index 3aa5361b..eaf9b5f8 100644 --- a/services/_instance_auth/schema.sql +++ b/services/_instance_auth/schema.sql @@ -1,15 +1,6 @@ --- Instance auth: the allow-list of Lightning clients permitted to call Apollo. --- --- Instance auth is opt-in via the INSTANCE_AUTH env var. When it is set, the --- Apollo server gates every /services/* request on the api_key the caller sends --- in the request body, looking its hash up in this table (via POSTGRES_URL). --- This table must exist for auth to work — if INSTANCE_AUTH is set but the table --- is missing, the gate fails closed and rejects every external caller. Without --- INSTANCE_AUTH, the server stays open as before. --- --- Rows are managed by hand. Use hash_token.py to hash a client's api_key, then --- INSERT a row. The inbound api_key is only a credential; Apollo never forwards --- it to the LLM, using anthropic_api_key (below) instead. See README.md. +-- Instance auth allow-list of Lightning clients permitted to call Apollo. Opt-in +-- via INSTANCE_AUTH; when set, /services/* requests are gated on the api_key the +-- caller sends, looked up here by hash. Rows are managed by hand. See README.md. CREATE TABLE IF NOT EXISTS lightning_clients ( id SERIAL PRIMARY KEY, diff --git a/services/util.py b/services/util.py index d8985cb9..96fc2b75 100644 --- a/services/util.py +++ b/services/util.py @@ -96,10 +96,9 @@ def apollo(name: str, payload: dict) -> dict: :return: JSON response. """ url = f"http://127.0.0.1:{apollo_port}/services/{name}" - # Mark this as an internal Apollo-to-Apollo call so it bypasses instance auth - # (see platform/src/middleware/auth.ts). The server process injects the token - # into the environment and this child process inherits it; absent (auth off - # or token unset) the header is simply omitted. + # Mark internal Apollo-to-Apollo calls so they bypass instance auth (see + # platform/src/middleware/auth.ts). The server injects the token into the env + # and this child inherits it; absent (auth off) the header is omitted. headers = {} internal_token = os.environ.get("APOLLO_INTERNAL_TOKEN") if internal_token: