From bd11fb530d33661315deeb902d4c84706894cfc2 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 10 Jun 2026 17:59:47 +0100 Subject: [PATCH 01/39] Add streaming tool call argument support --- .changeset/streaming-tool-call-args.md | 5 + packages/agents-runtime/src/entity-schema.ts | 43 +++++- .../agents-runtime/src/entity-timeline.ts | 24 ++- .../agents-runtime/src/outbound-bridge.ts | 143 +++++++++++++++--- packages/agents-runtime/src/pi-adapter.ts | 82 +++++++++- packages/agents-runtime/src/types.ts | 16 +- .../test/outbound-bridge.test.ts | 56 ++++++- .../agents-runtime/test/pi-adapter.test.ts | 135 +++++++++++++++++ 8 files changed, 477 insertions(+), 27 deletions(-) create mode 100644 .changeset/streaming-tool-call-args.md diff --git a/.changeset/streaming-tool-call-args.md b/.changeset/streaming-tool-call-args.md new file mode 100644 index 0000000000..88d56049b4 --- /dev/null +++ b/.changeset/streaming-tool-call-args.md @@ -0,0 +1,5 @@ +--- +"@electric-ax/agents-runtime": minor +--- + +Add runtime support for streaming tool call arguments from Pi model events. diff --git a/packages/agents-runtime/src/entity-schema.ts b/packages/agents-runtime/src/entity-schema.ts index d94b99eac9..dc2b47a23d 100644 --- a/packages/agents-runtime/src/entity-schema.ts +++ b/packages/agents-runtime/src/entity-schema.ts @@ -182,12 +182,28 @@ type ToolCallValue = { run_id?: string tool_call_id?: string tool_name: string - status: `started` | `args_complete` | `executing` | `completed` | `failed` + status: + | `started` + | `args_streaming` + | `args_complete` + | `executing` + | `completed` + | `failed` args?: unknown + args_preview?: unknown result?: unknown error?: string duration_ms?: number } +type ToolArgDeltaValue = { + key?: string + tool_call_key: string + tool_call_id?: string + run_id?: string + seq: number + delta: string + content_index?: number +} type ReasoningValue = { key?: string run_id?: string @@ -553,18 +569,33 @@ function createToolCallSchema(): Schema { tool_name: z.string(), status: z.enum([ `started`, + `args_streaming`, `args_complete`, `executing`, `completed`, `failed`, ]), args: z.unknown().optional(), + args_preview: z.unknown().optional(), result: z.unknown().optional(), error: z.string().optional(), duration_ms: z.number().int().optional(), }) } +function createToolArgDeltaSchema(): Schema { + return z.object({ + key: z.string().optional(), + ...timelineOrderField, + tool_call_key: z.string(), + tool_call_id: z.string().optional(), + run_id: z.string().optional(), + seq: z.number().int(), + delta: z.string(), + content_index: z.number().int().optional(), + }) +} + function createReasoningSchema(): Schema { return z.object({ key: z.string().optional(), @@ -927,6 +958,7 @@ export type Step = SequencedPersistedRow export type Text = SequencedPersistedRow export type TextDelta = SequencedPersistedRow export type ToolCall = SequencedPersistedRow +export type ToolArgDelta = SequencedPersistedRow export type Reasoning = SequencedPersistedRow export type ReasoningDelta = SequencedPersistedRow export type ErrorEvent = SequencedPersistedRow @@ -1050,6 +1082,8 @@ export const BUILT_IN_EVENT_SCHEMAS = { text_delta: createTextDeltaSchema() as unknown as BuiltInEntitySchema, tool_call: createToolCallSchema() as unknown as BuiltInEntitySchema, + tool_arg_delta: + createToolArgDeltaSchema() as unknown as BuiltInEntitySchema, reasoning: createReasoningSchema() as unknown as BuiltInEntitySchema, reasoning_delta: @@ -1088,6 +1122,7 @@ type EntityCollectionsDefinition = { texts: CollectionDefinition textDeltas: CollectionDefinition toolCalls: CollectionDefinition + toolArgDeltas: CollectionDefinition reasoning: CollectionDefinition reasoningDeltas: CollectionDefinition errors: CollectionDefinition @@ -1137,6 +1172,12 @@ export const builtInCollections: EntityCollectionsDefinition = { type: `tool_call`, primaryKey: `key`, }, + toolArgDeltas: { + schema: + BUILT_IN_EVENT_SCHEMAS.tool_arg_delta as StandardSchemaV1, + type: `tool_arg_delta`, + primaryKey: `key`, + }, reasoning: { schema: BUILT_IN_EVENT_SCHEMAS.reasoning as StandardSchemaV1, type: `reasoning`, diff --git a/packages/agents-runtime/src/entity-timeline.ts b/packages/agents-runtime/src/entity-timeline.ts index 171c85338f..1bff013418 100644 --- a/packages/agents-runtime/src/entity-timeline.ts +++ b/packages/agents-runtime/src/entity-timeline.ts @@ -44,7 +44,13 @@ export type EntityTimelineContentItem = toolCallId: string toolName: string args: Record - status: `started` | `args_complete` | `executing` | `completed` | `failed` + status: + | `started` + | `args_streaming` + | `args_complete` + | `executing` + | `completed` + | `failed` result?: string error?: string isError: boolean @@ -113,7 +119,13 @@ export interface IncludesToolCall { run_id: string order: TimelineOrder tool_name: string - status: `started` | `args_complete` | `executing` | `completed` | `failed` + status: + | `started` + | `args_streaming` + | `args_complete` + | `executing` + | `completed` + | `failed` args?: unknown result?: unknown error?: string @@ -243,7 +255,13 @@ export interface EntityTimelineToolCallItem { order: TimelineOrder tool_call_id?: string tool_name: string - status: `started` | `args_complete` | `executing` | `completed` | `failed` + status: + | `started` + | `args_streaming` + | `args_complete` + | `executing` + | `completed` + | `failed` args?: unknown result?: unknown error?: string diff --git a/packages/agents-runtime/src/outbound-bridge.ts b/packages/agents-runtime/src/outbound-bridge.ts index 87c902df93..9ad12a0074 100644 --- a/packages/agents-runtime/src/outbound-bridge.ts +++ b/packages/agents-runtime/src/outbound-bridge.ts @@ -178,6 +178,18 @@ export interface OutboundBridge { onReasoningStart: () => void onReasoningDelta: (delta: string) => void onReasoningEnd: (opts?: { encrypted?: string; summaryTitle?: string }) => void + onToolCallArgsStart( + toolCallId: string, + name: string, + argsPreview?: unknown + ): void + onToolCallArgsDelta( + toolCallId: string, + name: string, + delta: string, + opts?: { contentIndex?: number; argsPreview?: unknown } + ): void + onToolCallArgsEnd(toolCallId: string, name: string, args: unknown): void onToolCallStart(toolCallId: string, name: string, args: unknown): void onToolCallStart(name: string, args: unknown): void onToolCallEnd( @@ -244,7 +256,7 @@ export function createOutboundBridge( let currentReasoningRunKey: string | null = null const toolCallsById = new Map< string, - { key: string; runKey: string; args: unknown } + { key: string; runKey: string; args: unknown; argSeq: number } >() const legacyToolCallIdsByName = new Map>() const requireActiveRun = (action: string): string => { @@ -255,6 +267,65 @@ export function createOutboundBridge( } return currentRunKey } + const ensureToolCall = ( + toolCallId: string, + name: string, + opts?: { + args?: unknown + argsPreview?: unknown + status?: `started` | `args_streaming` | `args_complete` | `executing` + } + ): { key: string; runKey: string; args: unknown; argSeq: number } => { + const runKey = requireActiveRun(`ensureToolCall`) + const existing = toolCallsById.get(toolCallId) + if (existing) { + if (opts && (`args` in opts || `argsPreview` in opts || opts.status)) { + const nextArgs = `args` in opts ? opts.args : existing.args + if (`args` in opts) existing.args = opts.args + writeEvent( + entityStateSchema.toolCalls.update({ + key: existing.key, + value: { + tool_call_id: toolCallId, + tool_name: name, + status: opts.status ?? `args_streaming`, + args: nextArgs, + ...(opts.argsPreview !== undefined && { + args_preview: opts.argsPreview, + }), + run_id: existing.runKey, + } as never, + }) as ChangeEvent + ) + } + return existing + } + const key = `tc-${counters.tc++}` + persistSeed() + const created = { + key, + runKey, + args: opts && `args` in opts ? opts.args : undefined, + argSeq: 0, + } + toolCallsById.set(toolCallId, created) + writeEvent( + entityStateSchema.toolCalls.insert({ + key, + value: { + tool_call_id: toolCallId, + tool_name: name, + status: opts?.status ?? `started`, + args: created.args, + ...(opts?.argsPreview !== undefined && { + args_preview: opts.argsPreview, + }), + run_id: runKey, + } as never, + }) as ChangeEvent + ) + return created + } return { onRunStart() { @@ -444,15 +515,61 @@ export function createOutboundBridge( currentReasoningRunKey = null }, + onToolCallArgsStart( + toolCallId: string, + name: string, + argsPreview?: unknown + ) { + ensureToolCall(toolCallId, name, { + status: `args_streaming`, + argsPreview, + }) + }, + + onToolCallArgsDelta( + toolCallId: string, + name: string, + delta: string, + opts?: { contentIndex?: number; argsPreview?: unknown } + ) { + const toolCall = + toolCallsById.get(toolCallId) ?? + ensureToolCall(toolCallId, name, { + status: `args_streaming`, + argsPreview: opts?.argsPreview, + }) + const seq = toolCall.argSeq++ + writeEvent( + entityStateSchema.toolArgDeltas.insert({ + key: `${toolCall.key}:args-${seq}`, + value: { + tool_call_key: toolCall.key, + tool_call_id: toolCallId, + run_id: toolCall.runKey, + seq, + delta, + ...(opts?.contentIndex !== undefined && { + content_index: opts.contentIndex, + }), + } as never, + }) as ChangeEvent + ) + }, + + onToolCallArgsEnd(toolCallId: string, name: string, args: unknown) { + ensureToolCall(toolCallId, name, { + status: `args_complete`, + args, + }) + }, + onToolCallStart( toolCallIdOrName: string, nameOrArgs: string | unknown, maybeArgs?: unknown ) { - const runKey = requireActiveRun(`onToolCallStart`) - const key = `tc-${counters.tc++}` const legacyCall = maybeArgs === undefined - const toolCallId = legacyCall ? key : toolCallIdOrName + const toolCallId = legacyCall ? `tc-${counters.tc}` : toolCallIdOrName const name = legacyCall ? toolCallIdOrName : (nameOrArgs as string) const args = legacyCall ? nameOrArgs : maybeArgs if (legacyCall) { @@ -460,20 +577,10 @@ export function createOutboundBridge( ids.push(toolCallId) legacyToolCallIdsByName.set(name, ids) } - persistSeed() - toolCallsById.set(toolCallId, { key, runKey, args }) - writeEvent( - entityStateSchema.toolCalls.insert({ - key, - value: { - tool_call_id: toolCallId, - tool_name: name, - status: `started`, - args, - run_id: runKey, - } as never, - }) as ChangeEvent - ) + ensureToolCall(toolCallId, name, { + status: `executing`, + args, + }) }, onToolCallEnd( diff --git a/packages/agents-runtime/src/pi-adapter.ts b/packages/agents-runtime/src/pi-adapter.ts index 269805a1cb..a4c19aa4a7 100644 --- a/packages/agents-runtime/src/pi-adapter.ts +++ b/packages/agents-runtime/src/pi-adapter.ts @@ -21,7 +21,6 @@ import type { ChangeEvent } from '@durable-streams/state' import type { AgentEvent, AgentMessage, - AgentTool, StreamFn, } from '@mariozechner/pi-agent-core' import type { @@ -30,7 +29,12 @@ import type { Provider, SimpleStreamOptions, } from '@mariozechner/pi-ai' -import type { LLMContentBlock, LLMMessage, LLMMessageContent } from './types' +import type { + AgentTool, + LLMContentBlock, + LLMMessage, + LLMMessageContent, +} from './types' /** * Split a streamed reasoning blob into `{ title, body }`. @@ -347,7 +351,24 @@ export function createPiAgentAdapter( case `message_update`: { const assistantEvent = (event as Record) .assistantMessageEvent as - | { type: string; delta?: string } + | { + type: string + contentIndex?: number + delta?: string + toolCall?: { + id?: string + name?: string + arguments?: Record + } + partial?: { + content?: Array<{ + type?: string + id?: string + name?: string + arguments?: Record + }> + } + } | undefined if (assistantEvent?.type === `text_delta`) { if (!textStarted) { @@ -392,6 +413,61 @@ export function createPiAgentAdapter( reasoningStarted = false reasoningAccum = `` } + } else if ( + assistantEvent?.type === `toolcall_start` || + assistantEvent?.type === `toolcall_delta` || + assistantEvent?.type === `toolcall_end` + ) { + const contentIndex = assistantEvent.contentIndex + const partialToolCall = + typeof contentIndex === `number` + ? assistantEvent.partial?.content?.[contentIndex] + : undefined + const toolCall = assistantEvent.toolCall ?? partialToolCall + const toolCallId = toolCall?.id + const toolName = toolCall?.name + const argsPreview = toolCall?.arguments + if (toolCallId && toolName) { + if (assistantEvent.type === `toolcall_start`) { + bridge.onToolCallArgsStart( + toolCallId, + toolName, + argsPreview + ) + } else if (assistantEvent.type === `toolcall_delta`) { + const delta = assistantEvent.delta ?? `` + bridge.onToolCallArgsDelta(toolCallId, toolName, delta, { + contentIndex, + argsPreview, + }) + const tool = opts.tools.find( + (candidate) => candidate.name === toolName + ) + if (tool?.onArgsDelta) { + void Promise.resolve( + tool.onArgsDelta({ + toolCallId, + toolName, + contentIndex, + delta, + argsPreview, + }) + ).catch((error) => { + runtimeLog.warn( + logPrefix, + `streaming tool arg hook failed for ${toolName}:`, + error + ) + }) + } + } else { + bridge.onToolCallArgsEnd( + toolCallId, + toolName, + argsPreview + ) + } + } } else { runtimeLog.debug( logPrefix, diff --git a/packages/agents-runtime/src/types.ts b/packages/agents-runtime/src/types.ts index 3ef89c41b2..00af8449e8 100644 --- a/packages/agents-runtime/src/types.ts +++ b/packages/agents-runtime/src/types.ts @@ -410,6 +410,7 @@ export type TimelineItem = error: string | null status: | `started` + | `args_streaming` | `args_complete` | `executing` | `completed` @@ -947,7 +948,20 @@ export type AgentRunResult = { usage: { tokens: number; duration: number } } -export type AgentTool = PiAgentTool +export interface ToolArgumentDeltaContext { + toolCallId: string + toolName: string + contentIndex?: number + delta: string + argsPreview?: unknown +} + +export type AgentTool = PiAgentTool & { + onArgsDelta?: ( + context: ToolArgumentDeltaContext, + signal?: AbortSignal + ) => Promise | void +} export type AgentModel = string | Model export interface AgentConfig { diff --git a/packages/agents-runtime/test/outbound-bridge.test.ts b/packages/agents-runtime/test/outbound-bridge.test.ts index 62fb5d75b3..fc2ef698b0 100644 --- a/packages/agents-runtime/test/outbound-bridge.test.ts +++ b/packages/agents-runtime/test/outbound-bridge.test.ts @@ -97,10 +97,64 @@ describe(`createOutboundBridge`, () => { expect((writes[1]!.value as Record).tool_name).toBe( `search` ) - expect((writes[1]!.value as Record).status).toBe(`started`) + expect((writes[1]!.value as Record).status).toBe( + `executing` + ) expect((writes[1]!.value as Record).run_id).toBe(`run-0`) }) + it(`persists streaming tool call argument deltas`, () => { + const writes: Array = [] + const bridge = createOutboundBridge([], (e) => { + writes.push(e) + }) + + bridge.onRunStart() + bridge.onToolCallArgsStart(`call-draft`, `draft`, { text: `He` }) + bridge.onToolCallArgsDelta(`call-draft`, `draft`, `llo`, { + contentIndex: 1, + argsPreview: { text: `Hello` }, + }) + bridge.onToolCallArgsEnd(`call-draft`, `draft`, { text: `Hello` }) + + expect(writes[1]).toMatchObject({ + type: `tool_call`, + key: `tc-0`, + headers: { operation: `insert` }, + value: { + tool_call_id: `call-draft`, + tool_name: `draft`, + status: `args_streaming`, + args_preview: { text: `He` }, + run_id: `run-0`, + }, + }) + expect(writes[2]).toMatchObject({ + type: `tool_arg_delta`, + key: `tc-0:args-0`, + value: { + tool_call_key: `tc-0`, + tool_call_id: `call-draft`, + run_id: `run-0`, + seq: 0, + delta: `llo`, + content_index: 1, + }, + }) + expect(writes[3]).toMatchObject({ + type: `tool_call`, + key: `tc-0`, + headers: { operation: `update` }, + value: { + tool_call_id: `call-draft`, + tool_name: `draft`, + status: `args_complete`, + args: { text: `Hello` }, + run_id: `run-0`, + }, + }) + }) + it(`maps tool_call_end to tool_call update with result`, () => { const writes: Array = [] const bridge = createOutboundBridge([], (e) => { diff --git a/packages/agents-runtime/test/pi-adapter.test.ts b/packages/agents-runtime/test/pi-adapter.test.ts index 1424d7e29c..c3c2627b82 100644 --- a/packages/agents-runtime/test/pi-adapter.test.ts +++ b/packages/agents-runtime/test/pi-adapter.test.ts @@ -570,6 +570,141 @@ describe(`createPiAgentAdapter`, () => { ) }) + it(`dispatches streamed tool call arguments to the bridge and tool hook`, async () => { + let streamReadyResolve: + | ((stream: ReturnType) => void) + | null = null + const streamReady = new Promise< + ReturnType + >((resolve) => { + streamReadyResolve = resolve + }) + const partialMessage: AssistantMessage = { + role: `assistant`, + content: [ + { + type: `toolCall`, + id: `call-draft`, + name: `draft`, + arguments: { text: `Hello` }, + }, + ], + api: `anthropic-messages`, + provider: `anthropic`, + model: `claude-sonnet-4-5-20250929`, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + total: 0, + }, + }, + stopReason: `toolUse`, + timestamp: Date.now(), + } + const completedMessage: AssistantMessage = { + ...partialMessage, + content: [{ type: `text`, text: `` }], + stopReason: `stop`, + } + const argDeltas: Array = [] + const events: Array = [] + const factory = createPiAgentAdapter({ + systemPrompt: `Test system prompt`, + model: `claude-sonnet-4-5-20250929`, + tools: [ + { + name: `draft`, + label: `Draft`, + description: `Draft text`, + parameters: { + type: `object`, + properties: { text: { type: `string` } }, + required: [`text`], + } as never, + onArgsDelta: (context) => { + argDeltas.push(context) + }, + execute: async () => ({ + content: [{ type: `text`, text: `ok` }], + details: null, + }), + }, + ], + streamFn: () => { + const stream = createAssistantMessageEventStream() + streamReadyResolve?.(stream) + return stream + }, + }) + const handle = factory({ + entityUrl: `test/entity-1`, + epoch: 1, + messages: [], + outboundIdSeed: { run: 0, step: 0, msg: 0, tc: 0 }, + writeEvent: (event: ChangeEvent) => { + events.push(event) + }, + }) + + const runPromise = handle.run(`hello`) + const stream = await streamReady + stream.push({ + type: `start`, + partial: partialMessage, + }) + stream.push({ + type: `toolcall_start`, + contentIndex: 0, + partial: partialMessage, + }) + stream.push({ + type: `toolcall_delta`, + contentIndex: 0, + delta: `"Hello"`, + partial: partialMessage, + }) + stream.push({ + type: `toolcall_end`, + contentIndex: 0, + toolCall: partialMessage.content[0] as never, + partial: partialMessage, + }) + stream.push({ + type: `done`, + reason: `stop`, + message: completedMessage, + }) + await runPromise + + expect(argDeltas).toEqual([ + { + toolCallId: `call-draft`, + toolName: `draft`, + contentIndex: 0, + delta: `"Hello"`, + argsPreview: { text: `Hello` }, + }, + ]) + expect(events).toContainEqual( + expect.objectContaining({ + type: `tool_arg_delta`, + value: expect.objectContaining({ + tool_call_id: `call-draft`, + delta: `"Hello"`, + content_index: 0, + }), + }) + ) + }) + it(`isRunning returns false initially`, () => { const factory = createPiAgentAdapter({ systemPrompt: `Test system prompt`, From 28b15f004a7a394516efa8c41a8a26eb7d8285f7 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 10 Jun 2026 19:00:01 +0100 Subject: [PATCH 02/39] Preserve legacy tool call start status --- packages/agents-runtime/src/outbound-bridge.ts | 3 ++- packages/agents-runtime/test/outbound-bridge.test.ts | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/agents-runtime/src/outbound-bridge.ts b/packages/agents-runtime/src/outbound-bridge.ts index 9ad12a0074..40e260266e 100644 --- a/packages/agents-runtime/src/outbound-bridge.ts +++ b/packages/agents-runtime/src/outbound-bridge.ts @@ -577,8 +577,9 @@ export function createOutboundBridge( ids.push(toolCallId) legacyToolCallIdsByName.set(name, ids) } + const existing = toolCallsById.has(toolCallId) ensureToolCall(toolCallId, name, { - status: `executing`, + status: existing ? `executing` : `started`, args, }) }, diff --git a/packages/agents-runtime/test/outbound-bridge.test.ts b/packages/agents-runtime/test/outbound-bridge.test.ts index fc2ef698b0..c978e1bf45 100644 --- a/packages/agents-runtime/test/outbound-bridge.test.ts +++ b/packages/agents-runtime/test/outbound-bridge.test.ts @@ -97,9 +97,7 @@ describe(`createOutboundBridge`, () => { expect((writes[1]!.value as Record).tool_name).toBe( `search` ) - expect((writes[1]!.value as Record).status).toBe( - `executing` - ) + expect((writes[1]!.value as Record).status).toBe(`started`) expect((writes[1]!.value as Record).run_id).toBe(`run-0`) }) From a4e79ee992972f63b38c98917d406dc8a355d991 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 17 Jun 2026 10:59:45 +0100 Subject: [PATCH 03/39] Address streaming tool call review feedback --- packages/agents-runtime/src/entity-schema.ts | 3 + .../agents-runtime/src/outbound-bridge.ts | 16 +- packages/agents-runtime/src/pi-adapter.ts | 80 +++++- .../test/outbound-bridge.test.ts | 114 +++++++- .../agents-runtime/test/pi-adapter.test.ts | 255 +++++++++++++++++- 5 files changed, 445 insertions(+), 23 deletions(-) diff --git a/packages/agents-runtime/src/entity-schema.ts b/packages/agents-runtime/src/entity-schema.ts index dc2b47a23d..0f5d8c4304 100644 --- a/packages/agents-runtime/src/entity-schema.ts +++ b/packages/agents-runtime/src/entity-schema.ts @@ -195,6 +195,9 @@ type ToolCallValue = { error?: string duration_ms?: number } +// Tool argument deltas intentionally mirror text deltas: every streamed chunk is +// retained for replay/inspection, while the final parsed args are still stored +// on the tool_call row for the normal result lifecycle. type ToolArgDeltaValue = { key?: string tool_call_key: string diff --git a/packages/agents-runtime/src/outbound-bridge.ts b/packages/agents-runtime/src/outbound-bridge.ts index 40e260266e..6d8600b98b 100644 --- a/packages/agents-runtime/src/outbound-bridge.ts +++ b/packages/agents-runtime/src/outbound-bridge.ts @@ -521,7 +521,7 @@ export function createOutboundBridge( argsPreview?: unknown ) { ensureToolCall(toolCallId, name, { - status: `args_streaming`, + status: `started`, argsPreview, }) }, @@ -532,12 +532,18 @@ export function createOutboundBridge( delta: string, opts?: { contentIndex?: number; argsPreview?: unknown } ) { - const toolCall = - toolCallsById.get(toolCallId) ?? + let toolCall = toolCallsById.get(toolCallId) + if (toolCall) { ensureToolCall(toolCallId, name, { status: `args_streaming`, argsPreview: opts?.argsPreview, }) + } else { + toolCall = ensureToolCall(toolCallId, name, { + status: `args_streaming`, + argsPreview: opts?.argsPreview, + }) + } const seq = toolCall.argSeq++ writeEvent( entityStateSchema.toolArgDeltas.insert({ @@ -569,7 +575,9 @@ export function createOutboundBridge( maybeArgs?: unknown ) { const legacyCall = maybeArgs === undefined - const toolCallId = legacyCall ? `tc-${counters.tc}` : toolCallIdOrName + const toolCallId = legacyCall + ? `legacy-tc-${counters.tc}` + : toolCallIdOrName const name = legacyCall ? toolCallIdOrName : (nameOrArgs as string) const args = legacyCall ? nameOrArgs : maybeArgs if (legacyCall) { diff --git a/packages/agents-runtime/src/pi-adapter.ts b/packages/agents-runtime/src/pi-adapter.ts index a4c19aa4a7..a1f55691b1 100644 --- a/packages/agents-runtime/src/pi-adapter.ts +++ b/packages/agents-runtime/src/pi-adapter.ts @@ -273,6 +273,8 @@ export function createPiAgentAdapter( let reasoningStarted = false let reasoningAccum = `` let abortedRun = false + let activeRunSignal: AbortSignal | undefined + const pendingToolArgDeltaHooks = new Map>() const model = resolvePiModel({ model: opts.model, @@ -288,11 +290,61 @@ export function createPiAgentAdapter( timeoutMs: modelTimeoutMs, maxRetries: modelMaxRetries, }) + const awaitToolArgDeltaHooks = async ( + toolCallId: string | undefined + ): Promise => { + if (!toolCallId) return + await pendingToolArgDeltaHooks.get(toolCallId) + } + const enqueueToolArgDeltaHook = ( + tool: AgentTool, + context: { + toolCallId: string + toolName: string + contentIndex?: number + delta: string + argsPreview?: unknown + }, + logPrefix: string + ): void => { + if (!tool.onArgsDelta) return + const previous = + pendingToolArgDeltaHooks.get(context.toolCallId) ?? Promise.resolve() + const next = previous + .catch(() => undefined) + .then(async () => { + try { + await tool.onArgsDelta?.(context, activeRunSignal) + } catch (error) { + runtimeLog.warn( + logPrefix, + `streaming tool arg hook failed for ${context.toolName}:`, + error + ) + } + }) + pendingToolArgDeltaHooks.set(context.toolCallId, next) + void next.finally(() => { + if (pendingToolArgDeltaHooks.get(context.toolCallId) === next) { + pendingToolArgDeltaHooks.delete(context.toolCallId) + } + }) + } + const agentTools = opts.tools.map( + (tool): AgentTool => ({ + ...tool, + execute: (async (...args: Parameters) => { + const toolCallId = typeof args[0] === `string` ? args[0] : undefined + await awaitToolArgDeltaHooks(toolCallId) + return tool.execute(...args) + }) as AgentTool[`execute`], + }) + ) const agentOptions = { initialState: { systemPrompt: opts.systemPrompt, - tools: opts.tools as Array, + tools: agentTools as Array, messages: history as Array, model, }, @@ -443,22 +495,18 @@ export function createPiAgentAdapter( const tool = opts.tools.find( (candidate) => candidate.name === toolName ) - if (tool?.onArgsDelta) { - void Promise.resolve( - tool.onArgsDelta({ + if (tool) { + enqueueToolArgDeltaHook( + tool, + { toolCallId, toolName, contentIndex, delta, argsPreview, - }) - ).catch((error) => { - runtimeLog.warn( - logPrefix, - `streaming tool arg hook failed for ${toolName}:`, - error - ) - }) + }, + logPrefix + ) } } else { bridge.onToolCallArgsEnd( @@ -467,6 +515,11 @@ export function createPiAgentAdapter( argsPreview ) } + } else { + runtimeLog.debug( + logPrefix, + `pi-adapter message_update missing tool call identity type=${assistantEvent.type}` + ) } } else { runtimeLog.debug( @@ -702,6 +755,7 @@ export function createPiAgentAdapter( async run(input?: string, abortSignal?: AbortSignal): Promise { running = true abortedRun = false + activeRunSignal = abortSignal bridge.onRunStart() @@ -719,6 +773,7 @@ export function createPiAgentAdapter( settled = true clearAbortFallback() running = false + activeRunSignal = undefined abortSignal?.removeEventListener(`abort`, abortRun) unsubscribe() bridge.onRunEnd({ finishReason }) @@ -752,6 +807,7 @@ export function createPiAgentAdapter( if (settled) return settled = true running = false + activeRunSignal = undefined clearAbortFallback() abortSignal?.removeEventListener(`abort`, abortRun) unsubscribe() diff --git a/packages/agents-runtime/test/outbound-bridge.test.ts b/packages/agents-runtime/test/outbound-bridge.test.ts index c978e1bf45..a999e16d5f 100644 --- a/packages/agents-runtime/test/outbound-bridge.test.ts +++ b/packages/agents-runtime/test/outbound-bridge.test.ts @@ -122,12 +122,24 @@ describe(`createOutboundBridge`, () => { value: { tool_call_id: `call-draft`, tool_name: `draft`, - status: `args_streaming`, + status: `started`, args_preview: { text: `He` }, run_id: `run-0`, }, }) expect(writes[2]).toMatchObject({ + type: `tool_call`, + key: `tc-0`, + headers: { operation: `update` }, + value: { + tool_call_id: `call-draft`, + tool_name: `draft`, + status: `args_streaming`, + args_preview: { text: `Hello` }, + run_id: `run-0`, + }, + }) + expect(writes[3]).toMatchObject({ type: `tool_arg_delta`, key: `tc-0:args-0`, value: { @@ -139,7 +151,7 @@ describe(`createOutboundBridge`, () => { content_index: 1, }, }) - expect(writes[3]).toMatchObject({ + expect(writes[4]).toMatchObject({ type: `tool_call`, key: `tc-0`, headers: { operation: `update` }, @@ -153,6 +165,104 @@ describe(`createOutboundBridge`, () => { }) }) + it(`transitions a streamed tool call to executing before completion`, () => { + const writes: Array = [] + const bridge = createOutboundBridge([], (e) => { + writes.push(e) + }) + + bridge.onRunStart() + bridge.onToolCallArgsStart(`call-draft`, `draft`, { text: `He` }) + bridge.onToolCallArgsDelta(`call-draft`, `draft`, `llo`, { + argsPreview: { text: `Hello` }, + }) + bridge.onToolCallArgsEnd(`call-draft`, `draft`, { text: `Hello` }) + bridge.onToolCallStart(`call-draft`, `draft`, { text: `Hello` }) + bridge.onToolCallEnd(`call-draft`, `draft`, `ok`, false) + + expect(writes[5]).toMatchObject({ + type: `tool_call`, + key: `tc-0`, + headers: { operation: `update` }, + value: { + tool_call_id: `call-draft`, + tool_name: `draft`, + status: `executing`, + args: { text: `Hello` }, + run_id: `run-0`, + }, + }) + expect(writes[6]).toMatchObject({ + type: `tool_call`, + key: `tc-0`, + value: { + status: `completed`, + args: { text: `Hello` }, + result: `ok`, + }, + }) + }) + + it(`creates a streaming tool call when a delta arrives before start`, () => { + const writes: Array = [] + const bridge = createOutboundBridge([], (e) => { + writes.push(e) + }) + + bridge.onRunStart() + bridge.onToolCallArgsDelta(`call-draft`, `draft`, `He`, { + argsPreview: { text: `He` }, + }) + + expect(writes[1]).toMatchObject({ + type: `tool_call`, + key: `tc-0`, + headers: { operation: `insert` }, + value: { + tool_call_id: `call-draft`, + tool_name: `draft`, + status: `args_streaming`, + args_preview: { text: `He` }, + }, + }) + expect(writes[2]).toMatchObject({ + type: `tool_arg_delta`, + key: `tc-0:args-0`, + value: { + tool_call_key: `tc-0`, + tool_call_id: `call-draft`, + seq: 0, + delta: `He`, + }, + }) + }) + + it(`keeps legacy synthetic tool ids distinct from provider ids`, () => { + const writes: Array = [] + const bridge = createOutboundBridge([], (e) => { + writes.push(e) + }) + + bridge.onRunStart() + bridge.onToolCallArgsStart(`tc-0`, `provider`, {}) + bridge.onToolCallStart(`legacy`, {}) + + expect(writes[1]).toMatchObject({ + key: `tc-0`, + value: { + tool_call_id: `tc-0`, + tool_name: `provider`, + }, + }) + expect(writes[2]).toMatchObject({ + key: `tc-1`, + value: { + tool_call_id: `legacy-tc-1`, + tool_name: `legacy`, + }, + }) + }) + it(`maps tool_call_end to tool_call update with result`, () => { const writes: Array = [] const bridge = createOutboundBridge([], (e) => { diff --git a/packages/agents-runtime/test/pi-adapter.test.ts b/packages/agents-runtime/test/pi-adapter.test.ts index c3c2627b82..6fa8e04ca4 100644 --- a/packages/agents-runtime/test/pi-adapter.test.ts +++ b/packages/agents-runtime/test/pi-adapter.test.ts @@ -7,9 +7,8 @@ import { import { createAssistantMessageEventStream } from '@mariozechner/pi-ai' import { Type } from '@sinclair/typebox' import type { OutboundIdSeed } from '../src/outbound-bridge' -import type { LLMMessage } from '../src/types' +import type { AgentTool, LLMMessage } from '../src/types' import type { ChangeEvent } from '@durable-streams/state' -import type { AgentTool } from '@mariozechner/pi-agent-core' import type { AssistantMessage, Model, @@ -615,7 +614,9 @@ describe(`createPiAgentAdapter`, () => { stopReason: `stop`, } const argDeltas: Array = [] + const signals: Array = [] const events: Array = [] + const controller = new AbortController() const factory = createPiAgentAdapter({ systemPrompt: `Test system prompt`, model: `claude-sonnet-4-5-20250929`, @@ -629,8 +630,9 @@ describe(`createPiAgentAdapter`, () => { properties: { text: { type: `string` } }, required: [`text`], } as never, - onArgsDelta: (context) => { + onArgsDelta: (context, signal) => { argDeltas.push(context) + signals.push(signal) }, execute: async () => ({ content: [{ type: `text`, text: `ok` }], @@ -648,13 +650,13 @@ describe(`createPiAgentAdapter`, () => { entityUrl: `test/entity-1`, epoch: 1, messages: [], - outboundIdSeed: { run: 0, step: 0, msg: 0, tc: 0 }, + outboundIdSeed: { run: 0, step: 0, msg: 0, tc: 0, reasoning: 0 }, writeEvent: (event: ChangeEvent) => { events.push(event) }, }) - const runPromise = handle.run(`hello`) + const runPromise = handle.run(`hello`, controller.signal) const stream = await streamReady stream.push({ type: `start`, @@ -693,6 +695,7 @@ describe(`createPiAgentAdapter`, () => { argsPreview: { text: `Hello` }, }, ]) + expect(signals).toEqual([controller.signal]) expect(events).toContainEqual( expect.objectContaining({ type: `tool_arg_delta`, @@ -705,6 +708,248 @@ describe(`createPiAgentAdapter`, () => { ) }) + it(`serializes streamed tool argument hooks before tool execution`, async () => { + let streamReadyResolve: + | ((stream: ReturnType) => void) + | null = null + const streamReady = new Promise< + ReturnType + >((resolve) => { + streamReadyResolve = resolve + }) + let releaseFirstDelta!: () => void + const firstDeltaBarrier = new Promise((resolve) => { + releaseFirstDelta = resolve + }) + const usage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + total: 0, + }, + } + const toolCallMessage: AssistantMessage = { + role: `assistant`, + content: [ + { + type: `toolCall`, + id: `call-draft`, + name: `draft`, + arguments: { text: `AB` }, + }, + ], + api: `anthropic-messages`, + provider: `anthropic`, + model: `claude-sonnet-4-5-20250929`, + usage, + stopReason: `toolUse`, + timestamp: Date.now(), + } + const completedMessage: AssistantMessage = { + ...toolCallMessage, + content: [{ type: `text`, text: `done` }], + stopReason: `stop`, + } + const order: Array = [] + let streamCount = 0 + const factory = createPiAgentAdapter({ + systemPrompt: `Test system prompt`, + model: `claude-sonnet-4-5-20250929`, + tools: [ + { + name: `draft`, + label: `Draft`, + description: `Draft text`, + parameters: Type.Object({ text: Type.String() }), + onArgsDelta: async ({ delta }) => { + order.push(`start:${delta}`) + if (delta === `A`) { + await firstDeltaBarrier + } + order.push(`end:${delta}`) + }, + execute: async () => { + order.push(`execute`) + return { + content: [{ type: `text`, text: `ok` }], + details: null, + } + }, + }, + ], + streamFn: () => { + const stream = createAssistantMessageEventStream() + streamCount++ + if (streamCount === 1) { + streamReadyResolve?.(stream) + } else { + queueMicrotask(() => stream.end(completedMessage)) + } + return stream + }, + }) + const handle = factory({ + entityUrl: `test/entity-1`, + epoch: 1, + messages: [], + outboundIdSeed: { run: 0, step: 0, msg: 0, tc: 0, reasoning: 0 }, + writeEvent: (_event: ChangeEvent) => {}, + }) + + const runPromise = handle.run(`hello`) + const stream = await streamReady + stream.push({ type: `start`, partial: toolCallMessage }) + stream.push({ + type: `toolcall_start`, + contentIndex: 0, + partial: toolCallMessage, + }) + stream.push({ + type: `toolcall_delta`, + contentIndex: 0, + delta: `A`, + partial: toolCallMessage, + }) + stream.push({ + type: `toolcall_delta`, + contentIndex: 0, + delta: `B`, + partial: toolCallMessage, + }) + stream.push({ + type: `toolcall_end`, + contentIndex: 0, + toolCall: toolCallMessage.content[0] as never, + partial: toolCallMessage, + }) + stream.push({ + type: `done`, + reason: `toolUse`, + message: toolCallMessage, + }) + + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(order).toEqual([`start:A`]) + + releaseFirstDelta() + await runPromise + + expect(order).toEqual([`start:A`, `end:A`, `start:B`, `end:B`, `execute`]) + }) + + it(`continues tool execution after a streamed argument hook rejects`, async () => { + let streamReadyResolve: + | ((stream: ReturnType) => void) + | null = null + const streamReady = new Promise< + ReturnType + >((resolve) => { + streamReadyResolve = resolve + }) + const usage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + total: 0, + }, + } + const toolCallMessage: AssistantMessage = { + role: `assistant`, + content: [ + { + type: `toolCall`, + id: `call-draft`, + name: `draft`, + arguments: { text: `A` }, + }, + ], + api: `anthropic-messages`, + provider: `anthropic`, + model: `claude-sonnet-4-5-20250929`, + usage, + stopReason: `toolUse`, + timestamp: Date.now(), + } + const completedMessage: AssistantMessage = { + ...toolCallMessage, + content: [{ type: `text`, text: `done` }], + stopReason: `stop`, + } + let executed = false + let streamCount = 0 + const factory = createPiAgentAdapter({ + systemPrompt: `Test system prompt`, + model: `claude-sonnet-4-5-20250929`, + tools: [ + { + name: `draft`, + label: `Draft`, + description: `Draft text`, + parameters: Type.Object({ text: Type.String() }), + onArgsDelta: async () => { + throw new Error(`hook failed`) + }, + execute: async () => { + executed = true + return { + content: [{ type: `text`, text: `ok` }], + details: null, + } + }, + }, + ], + streamFn: () => { + const stream = createAssistantMessageEventStream() + streamCount++ + if (streamCount === 1) { + streamReadyResolve?.(stream) + } else { + queueMicrotask(() => stream.end(completedMessage)) + } + return stream + }, + }) + const handle = factory({ + entityUrl: `test/entity-1`, + epoch: 1, + messages: [], + outboundIdSeed: { run: 0, step: 0, msg: 0, tc: 0, reasoning: 0 }, + writeEvent: (_event: ChangeEvent) => {}, + }) + + const runPromise = handle.run(`hello`) + const stream = await streamReady + stream.push({ type: `start`, partial: toolCallMessage }) + stream.push({ + type: `toolcall_delta`, + contentIndex: 0, + delta: `A`, + partial: toolCallMessage, + }) + stream.push({ + type: `done`, + reason: `toolUse`, + message: toolCallMessage, + }) + + await expect(runPromise).resolves.toBeUndefined() + expect(executed).toBe(true) + }) + it(`isRunning returns false initially`, () => { const factory = createPiAgentAdapter({ systemPrompt: `Test system prompt`, From 69b7333ac4cf6f0ff37bd459ce84ab201483fbfa Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 10 Jun 2026 18:17:14 +0100 Subject: [PATCH 04/39] Add collaborative markdown documents --- .changeset/quiet-markdown-docs.md | 10 + AGENTS_MARKDOWN_DOCS_PLAN.md | 770 ++++++++++++ packages/agents-runtime/package.json | 4 + packages/agents-runtime/src/client.ts | 5 + packages/agents-runtime/src/create-handler.ts | 25 + packages/agents-runtime/src/entity-schema.ts | 50 + packages/agents-runtime/src/index.ts | 6 + .../agents-runtime/src/manifest-helpers.ts | 28 + .../src/markdown-document-session.ts | 163 +++ packages/agents-runtime/src/markdown-yjs.ts | 256 ++++ packages/agents-runtime/src/process-wake.ts | 25 + .../src/runtime-server-client.ts | 158 +++ packages/agents-runtime/src/tools.ts | 1 + .../agents-runtime/src/tools/markdown-docs.ts | 1064 +++++++++++++++++ packages/agents-runtime/src/types.ts | 30 + .../test/markdown-docs-tools.test.ts | 573 +++++++++ packages/agents-server-ui/package.json | 9 + .../src/components/EntityContextDrawer.tsx | 27 + .../src/components/EntityTimeline.tsx | 76 +- .../views/MarkdownDocumentView.module.css | 297 +++++ .../views/MarkdownDocumentView.test.ts | 73 ++ .../components/views/MarkdownDocumentView.tsx | 397 ++++++ .../src/components/workspace/Workspace.tsx | 35 +- .../src/lib/workspace/registerViews.ts | 20 +- .../lib/workspace/workspaceReducer.test.ts | 27 + packages/agents-server/package.json | 5 +- packages/agents-server/src/entity-manager.ts | 257 +++- .../agents-server/src/markdown-documents.ts | 193 +++ .../src/routing/durable-streams-router.ts | 153 +++ .../src/routing/entities-router.ts | 69 ++ packages/agents-server/src/stream-client.ts | 43 + ...ic-agents-manager-write-validation.test.ts | 107 ++ .../test/electric-agents-routes.test.ts | 254 +++- .../test/electric-agents-status.test.ts | 40 +- packages/agents/skills/markdown-docs.md | 122 ++ packages/agents/src/agents/horton.ts | 15 +- packages/agents/src/agents/worker.ts | 27 +- packages/agents/src/bootstrap.ts | 2 + packages/agents/src/tools/spawn-worker.ts | 111 +- .../agents/test/horton-system-prompt.test.ts | 21 + .../test/horton-tool-composition.test.ts | 53 +- .../agents/test/spawn-worker-tool.test.ts | 74 +- pnpm-lock.yaml | 341 ++++-- 43 files changed, 5874 insertions(+), 142 deletions(-) create mode 100644 .changeset/quiet-markdown-docs.md create mode 100644 AGENTS_MARKDOWN_DOCS_PLAN.md create mode 100644 packages/agents-runtime/src/markdown-document-session.ts create mode 100644 packages/agents-runtime/src/markdown-yjs.ts create mode 100644 packages/agents-runtime/src/tools/markdown-docs.ts create mode 100644 packages/agents-runtime/test/markdown-docs-tools.test.ts create mode 100644 packages/agents-server-ui/src/components/views/MarkdownDocumentView.module.css create mode 100644 packages/agents-server-ui/src/components/views/MarkdownDocumentView.test.ts create mode 100644 packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx create mode 100644 packages/agents-server/src/markdown-documents.ts create mode 100644 packages/agents/skills/markdown-docs.md diff --git a/.changeset/quiet-markdown-docs.md b/.changeset/quiet-markdown-docs.md new file mode 100644 index 0000000000..6b4f6ee90a --- /dev/null +++ b/.changeset/quiet-markdown-docs.md @@ -0,0 +1,10 @@ +--- +"@electric-ax/agents": patch +"@electric-ax/agents-runtime": patch +"@electric-ax/agents-server": patch +"@electric-ax/agents-server-ui": patch +--- + +Add collaborative markdown document tools backed by Yjs durable streams. + +Horton can create, read, replace, edit, and stream inserts into markdown documents by mutating a wake-local Y.Doc and appending binary Yjs updates to the document stream. The server now keeps markdown document handling thin by creating document streams and serving manifest metadata while document content changes flow through the Yjs stream. diff --git a/AGENTS_MARKDOWN_DOCS_PLAN.md b/AGENTS_MARKDOWN_DOCS_PLAN.md new file mode 100644 index 0000000000..ffee9b0977 --- /dev/null +++ b/AGENTS_MARKDOWN_DOCS_PLAN.md @@ -0,0 +1,770 @@ +# Agents Markdown Docs Implementation Plan + +## Goal + +Add first-class collaborative markdown documents to Electric Agents. + +Agents should be able to create a markdown document, add it to the entity +manifest, read it, and edit it with file-like replacement tools. Users should be +able to click the manifest entry, open a CodeMirror markdown editor in the +workspace, edit concurrently with other users, and see agent/user presence. + +The first implementation intentionally does not require streaming tool calls or +runtime-level interception of assistant text. Streaming edits can be added after +the document model, auth, UI, and non-streaming tools are working. + +## MVP Scope + +### In scope + +- A new manifest entry kind for collaborative markdown documents. +- One durable Yjs document stream per document, using + `@durable-streams/y-durable-streams`. +- A CodeMirror markdown editor bound to `Y.Text`. +- User presence through Yjs awareness. +- Agent presence during document tools, including status and edit location. +- Agent tools for create/read/write/exact text replacement. +- Unified diff results from write/edit tools, matching the current file tool + behavior. +- Explicit server auth for document Yjs stream paths. +- Forking support so forked entities receive forked document streams. + +### Out of scope for MVP + +- Token-by-token agent edits. +- Streaming tool arguments. +- Runtime routing of assistant text into documents. +- Rich-text CRDTs such as ProseMirror fragments. +- Markdown preview/render mode. +- Comments or suggestions inside docs. +- Document history UI beyond the Yjs/Durable Streams backing log. + +## Core Design + +### Manifest Entry + +Add a new manifest entry kind rather than encoding docs as attachments. + +Attachments are immutable, closed streams with byte length and sha256 semantics. +Markdown docs are mutable CRDT-backed resources, so they should be first-class +manifest entries with their own lifecycle and fork/auth behavior. + +Proposed manifest shape: + +```ts +type ManifestDocumentEntry = { + key?: string + kind: 'document' + id: string + title: string + provider: 'y-durable-streams' + docId: string + docPath: string + streamPath: string + contentMimeType: 'text/markdown' + transportMimeType: 'application/vnd.electric-agents.markdown-yjs' + yTextName: 'markdown' + createdAt: string + createdBy?: string + updatedAt?: string + meta?: Record +} +``` + +Recommended manifest key: + +```ts +document:${id} +``` + +Recommended stream path: + +```ts +/docs/agents/${entityType}/${instanceId}/documents/${id} +``` + +`docId` is the value passed to `YjsProvider`. + +`docPath` is the provider-facing stable document path and should not have a +leading slash: + +```ts +agents/${entityType}/${instanceId}/documents/${id} +``` + +`streamPath` is the Durable Streams document stream path used for auth, forking, +and debugging: + +```ts +/docs/${docPath} +``` + +This shape follows the `y-durable-streams` URL contract. The provider requests: + +```ts +{baseUrl}/docs/{docPath}?{queryParams} +``` + +For the agents server, use: + +```ts +baseUrl = agentsServerUrl +docId = docPath +``` + +Do not set `baseUrl` to the raw `streamPath`; the provider appends `/docs/...` +itself. + +### Yjs Document Model + +Use a plain Yjs text type: + +```ts +const ytext = ydoc.getText('markdown') +``` + +This keeps the MVP simple: + +- The stored CRDT is binary Yjs updates. +- The logical document content is markdown text. +- CodeMirror can bind directly to `Y.Text`. +- Agent tools can operate on `ytext.toString()` and commit Yjs transactions. + +### Mime Types + +Use two concepts: + +- `contentMimeType: 'text/markdown'` for what users and tools are editing. +- `transportMimeType: 'application/vnd.electric-agents.markdown-yjs'` for what + is stored in the durable stream. + +Do not label the durable stream itself as `text/markdown`; its bytes are Yjs +updates/snapshots. + +## Implementation Areas + +### 1. Runtime Types and Manifest Schema + +Files: + +- `packages/agents-runtime/src/entity-schema.ts` +- `packages/agents-runtime/src/types.ts` +- `packages/agents-runtime/src/manifest-helpers.ts` +- `packages/agents-server-ui/src/lib/ElectricAgentsProvider.tsx` + +Tasks: + +- Add `ManifestDocumentEntryValue`. +- Extend the manifest zod union with `kind: 'document'`. +- Export `ManifestDocumentEntry`. +- Add `manifestDocumentKey(id: string)`. +- Update UI-side manifest parsing/types to accept `kind: 'document'`. + +Acceptance: + +- Entity state can contain a `manifest` event with `kind: 'document'`. +- Document manifest entries use the strict shape above; older draft document + manifest shapes are not supported. + +### 2. Server Document API + +Files: + +- `packages/agents-server/src/entity-manager.ts` +- `packages/agents-server/src/routing/entities-router.ts` +- `packages/agents-runtime/src/runtime-server-client.ts` +- `packages/agents-runtime/src/types.ts` + +Tasks: + +- Add document validation helpers: + - document id cannot be empty, start with `.`, or contain `/`. + - title should be non-empty and bounded. +- Add `createDocument(entityUrl, req)`: + - create durable Yjs backing stream if needed. + - initialize the Yjs document with optional markdown text. + - write the document manifest entry. + - return `{ txid, document }`. +- Add `getDocument(entityUrl, id)`. +- Add `readDocument(entityUrl, id)` returning current markdown text. +- Add `writeDocument(entityUrl, id, content)` replacing the whole `Y.Text`. +- Add `editDocument(entityUrl, id, old_string, new_string, replace_all?)`. +- Add HTTP routes under entity API: + - `POST /:type/:instanceId/documents` + - `GET /:type/:instanceId/documents/:documentId` + - `GET /:type/:instanceId/documents/:documentId/content` + - `PUT /:type/:instanceId/documents/:documentId/content` + - `PATCH /:type/:instanceId/documents/:documentId/content` + +Open implementation choice: + +- Preferred for MVP: put create/read/write/edit on the server API and expose + them through `RuntimeServerClient`. This keeps auth, fork locks, and manifest + writes in one place. +- Avoid direct runtime-tool writes to `YjsProvider` in the first cut. That is + faster to prototype, but it spreads auth, fork locks, and stream path rules + into the runtime. + +Acceptance: + +- Creating a doc appends a manifest row. +- Reading returns markdown text from the Yjs doc. +- Writing/editing produces Yjs updates, not manifest content mutations. +- Server rejects operations when the entity is stopped or fork-write-locked. + +### 3. Durable Stream Yjs Integration + +Files: + +- `packages/agents-server/package.json` +- `packages/agents-server-ui/package.json` +- `packages/agents-runtime/package.json` if runtime tools manipulate Yjs locally. + +Dependencies: + +- `@durable-streams/y-durable-streams` +- `yjs` +- `y-protocols` +- `lib0` +- UI only: + - `codemirror` + - `@codemirror/state` + - `@codemirror/view` + - `@codemirror/lang-markdown` + - `y-codemirror.next` + +Tasks: + +- Use `YjsProvider` for browser/editor connections. +- On server create/write/edit, either: + - use `YjsProvider` server-side and wait for sync, or + - use the y-durable-streams server utilities if exposed by the package. +- Always destroy providers after tool/server operations. +- For initial content, create a `Y.Doc`, set `getText('markdown')`, and persist + through the provider. +- Keep the Yjs mount constants in one shared server module: + - `docPathForDocument(entityUrl, documentId)` + - `documentStreamPathForDocPath(docPath)` + - `entityUrlFromYjsDocumentPath(path)` + - `entityUrlFromYjsAwarenessPath(path)` + +Acceptance: + +- A browser editor and server operation converge on the same markdown text. +- New editor clients load through snapshot discovery and then live updates. + +### 4. Durable Stream Auth + +Files: + +- `packages/agents-server/src/routing/durable-streams-router.ts` +- `packages/agents-server/src/routing/stream-append.ts` + +Tasks: + +- Add document path recognition for provider document requests: + +```ts +function entityUrlFromYjsDocumentPath(path: string): string | null { + const match = path.match( + /^\/docs\/agents\/([^/]+)\/([^/]+)\/documents\/[^/]+(?:\/.*)?$/ + ) + if (!match) return null + return `/${match[1]}/${match[2]}` +} +``` + +- Authorize `GET`/`HEAD` document stream access with entity read permission. +- Authorize `POST`/`PUT` document stream writes with entity write/manage rules + or a dedicated document write permission rule. +- Inspect the installed `@durable-streams/y-durable-streams` package and add an + equivalent `entityUrlFromYjsAwarenessPath(path)` for the exact awareness URL + pattern used by the provider. +- Add route tests using real provider URL shapes for: + - snapshot discovery and snapshot load. + - live update reads. + - local edit writes. + - awareness reads/writes. +- Reject direct writes to document paths during fork locks. + +Important: + +The current durable-stream proxy explicitly guards entity streams, attachment +streams, and shared-state streams. Unknown paths intentionally pass through. +Document and document-awareness paths must not remain in that pass-through +bucket. + +Acceptance: + +- Unauthorized users cannot read, write, or observe awareness for document + streams. +- Authorized users can edit through CodeMirror. +- Fork locks prevent concurrent writes while the subtree is being forked. + +### 5. Forking + +Files: + +- `packages/agents-server/src/entity-manager.ts` + +Tasks: + +- Collect document stream paths from document manifest entries during fork + snapshot reads. +- Lock document stream paths during fork, like shared-state streams. +- Fork each document durable stream from source to fork destination. +- Remap document manifest entries: + - `streamPath` + - `docPath` + - `docId` + - possibly `key` if document ids are rewritten. +- Keep document ids stable within a fork unless collisions require suffixing. + +Acceptance: + +- Forked entity opens an independent copy of each document. +- Editing a forked doc does not change the source entity's doc. +- Pointer forks include only document manifest entries visible at the fork point. + +### 6. Runtime Tool Surface + +Files: + +- `packages/agents-runtime/src/tools/documents.ts` +- `packages/agents-runtime/src/tools.ts` +- `packages/agents-runtime/src/types.ts` +- `packages/agents-runtime/src/process-wake.ts` +- `packages/agents/src/bootstrap.ts` + +Tasks: + +- Add framework document tool factory. +- Extend `ProcessWakeConfig.createElectricTools` context with + `principal?: RuntimePrincipal`, and pass `config.principal` through from + `processWake`. Document tools need this for agent awareness state. +- Extend `ProcessWakeConfig.createElectricTools` context with document methods + backed by `RuntimeServerClient`: + - `createMarkdownDocument` + - `readMarkdownDocument` + - `writeMarkdownDocument` + - `editMarkdownDocument` +- Add default built-in tools in `packages/agents/src/bootstrap.ts`, alongside + event-source tools. +- Keep worker exposure explicit if desired. Horton already includes + `ctx.electricTools`; Worker currently gets only selected tools. + +Tool shapes: + +```ts +create_markdown_doc({ + title: string, + content?: string +}) +``` + +```ts +read_markdown_doc({ + docId: string, +}) +``` + +```ts +write_markdown_doc({ + docId: string, + content: string, +}) +``` + +```ts +edit_markdown_doc({ + docId: string, + old_string: string, + new_string: string, + replace_all?: boolean +}) +``` + +Tool behavior should mirror file tools: + +- `read_markdown_doc`, `create_markdown_doc`, and `write_markdown_doc` mark the + document as read in a per-wake read set. +- `edit_markdown_doc` must reject edits unless the document has been read or + written in the same wake. +- `old_string` must occur exactly once unless `replace_all` is true. +- Return a useful error when not found or ambiguous. +- Return `details.diff` using `createTwoFilesPatch`. +- Return replacement counts and byte/char counts. + +Acceptance: + +- An agent can create a doc and then read/edit it with file-like tools. +- Tool call UI shows a diff for document edits without special casing if + possible. + +### 7. Agent Presence During Tools + +Files: + +- `packages/agents-runtime/src/tools/documents.ts` +- server-side document service module, if split from `entity-manager.ts` + +Tasks: + +- When a document tool edits content: + - connect to the Yjs provider with an `Awareness` instance. + - set local awareness state from the principal passed through + `createElectricTools`, or from an agent principal derived by the server: + +```ts +{ + user: { + principalUrl, + role: 'agent', + name, + color, + status: 'editing' + } +} +``` + +- Before applying a replacement, set the agent selection/cursor near the + replacement range. +- Apply the Yjs transaction. +- Move cursor to the end of the replacement. +- Set status back to `idle` or destroy provider so awareness removal is + broadcast. + +Acceptance: + +- While an agent edit tool is running, open editors see the agent presence. +- For quick edits this may be brief; that is acceptable for MVP. + +### 8. UI: Document Manifest Rows + +Files: + +- `packages/agents-server-ui/src/components/EntityTimeline.tsx` +- `packages/agents-server-ui/src/lib/attachments.ts` or a new + `documents.ts` + +Tasks: + +- Add `isDocumentManifest`. +- Display document rows as `Document`. +- Use the title as primary text. +- Show `text/markdown`, provider, and created metadata. +- Add an open action. +- Use workspace helper: + +```ts +workspace.helpers.openEntity(entityUrl, { + viewId: 'markdown-doc', + viewParams: { doc: manifest.id }, +}) +``` + +Acceptance: + +- Document manifests are not hidden as attachments. +- Clicking a document opens the editor view. + +### 9. UI: CodeMirror Markdown Editor View + +Files: + +- `packages/agents-server-ui/src/lib/workspace/registerViews.ts` +- `packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx` +- new CSS module for the editor view. + +Tasks: + +- Register entity view: + +```ts +registerView({ + kind: 'entity', + id: 'markdown-doc', + label: 'Docs', + icon: FileText, + Component: MarkdownDocumentView, +}) +``` + +- Resolve `doc` from `viewParams`. +- Find document manifest from entity DB. +- Construct `Y.Doc`, `Awareness`, and `YjsProvider`. +- Use `baseUrl` pointing at the agents server durable-stream proxy. +- Bind CodeMirror to `ydoc.getText('markdown')`. +- Set local user awareness from `useCurrentPrincipal()`. +- Pass configured auth/principal headers to `YjsProvider.headers`, matching the + rest of the agents UI request path. +- Render presence bar from awareness states. +- Destroy CodeMirror view/provider on unmount. + +Acceptance: + +- Two browser windows can concurrently edit one doc. +- Remote cursor/presence appears. +- Agent tool edits appear live in open editors. +- The editor survives tile split/open/close cycles. + +### 10. Tests + +Unit and integration tests should be added at the layer being changed. + +Runtime: + +- Manifest schema accepts document entries. +- Document tool exact replacement behavior matches file edit behavior. +- Diff details are returned. + +Server: + +- Create document writes manifest. +- Read/write/edit round trip through Yjs. +- Unauthorized durable stream document access is rejected. +- Forked docs are independent. + +UI: + +- Manifest row labels and open action. +- View registration. +- Editor view mounts with missing/invalid doc id states. + +## Deferred Streaming Edit Work + +The repo already has enough evidence for a later streaming path: + +- `@mariozechner/pi-ai` emits `toolcall_start`, `toolcall_delta`, and + `toolcall_end` provider events. +- `@mariozechner/pi-agent-core` forwards those as `message_update` while the + assistant message is streaming. +- `packages/agents-runtime/src/pi-adapter.ts` currently only handles + `text_delta` in `message_update`. +- `packages/agents-runtime/src/outbound-bridge.ts` currently persists tool calls + only at `tool_execution_start` and final completion. + +Later streaming options: + +1. Surface tool argument deltas through the outbound bridge and persist partial + args in the `toolCalls` collection. +2. Add a streaming document insertion tool whose string argument can be consumed + incrementally. +3. Or add a runtime-level text routing mode. This is more invasive and should + remain separate from the MVP. + +This plan intentionally chooses non-streaming exact replacements first because +it avoids changing agent execution semantics. + +## Open Questions + +Resolve these inside the single PR before enabling the feature: + +- Should document tools be enabled for all built-in agents by default, or only + for Horton initially? +- Should workers be able to receive document tools by name in their spawn args? +- Should document stream write permission be tied to entity `manage`, entity + `write`, or a new permission? +- Should document ids remain stable across forks, or be suffixed like shared + state ids? + +## Single PR Implementation Phases + +Implement this as one PR. The phases below are sequencing for development and +review inside the branch, not separate merge boundaries. The PR should not be +merged with document creation/editing enabled until schema, server API, +auth, forking, tools, UI, presence, and tests are all complete. + +### Phase 0: Provider Path Spike + +Goal: remove uncertainty before changing product code. + +Tasks: + +- Inspect the installed `@durable-streams/y-durable-streams` package. +- Confirm the exact document request URLs for: + - snapshot discovery. + - snapshot load. + - live update reads. + - local edit writes. +- Confirm the exact awareness request URLs and methods. +- Capture helper names and URL examples in code comments/tests, not as + free-floating assumptions. + +Exit criteria: + +- The implementation has concrete helpers for document and awareness path + recognition. +- Route tests use real provider-shaped URLs. + +### Phase 1: Schema and Shared Types + +Tasks: + +- Add `ManifestDocumentEntryValue`. +- Extend the manifest schema union. +- Export document manifest types. +- Add `manifestDocumentKey(id)`. +- Add shared document path helpers. +- Update UI manifest parsing/types. + +Exit criteria: + +- Existing entity streams still load. +- A synthetic document manifest row parses in runtime and UI tests. + +### Phase 2: Server Document Service + +Tasks: + +- Add document id/title validation. +- Add create/get/read/write/edit document methods. +- Store initial markdown as `Y.Text('markdown')`. +- Return unified diffs from write/edit operations. +- Add entity API routes and `RuntimeServerClient` methods. +- Keep document writes server-mediated for MVP. + +Exit criteria: + +- Server tests can create, read, write, and exact-replace a markdown doc. +- Edit errors match the file edit tool behavior for missing/ambiguous strings. + +### Phase 3: Auth and Fork Safety + +Tasks: + +- Authorize `/docs/agents/...` document paths. +- Authorize the matching y-durable-streams awareness paths. +- Reject unauthorized document reads/writes/presence. +- Lock document streams during fork work. +- Clone document streams during fork. +- Remap `streamPath`, `docPath`, and `docId` in forked manifest entries. + +Exit criteria: + +- Unauthorized users cannot read/write doc streams or awareness streams. +- Forked entities edit independent document streams. +- Pointer forks include only document manifests visible at the fork point. + +### Phase 4: Runtime Tools + +Tasks: + +- Add document methods and `principal` to `createElectricTools` context. +- Add `create_markdown_doc`, `read_markdown_doc`, `write_markdown_doc`, and + `edit_markdown_doc`. +- Maintain a per-wake read set. +- Require read/write/create before exact edit in the same wake. +- Add document tools to the built-in electric tool bundle. +- Decide and document Worker exposure in the same PR. + +Exit criteria: + +- Horton can create/read/write/edit a doc through tools. +- Tool results include `details.diff`. +- Tool behavior mirrors file tools closely enough that the existing tool UI is + usable. + +### Phase 5: UI Manifest and Editor + +Tasks: + +- Add document manifest row rendering. +- Add open action using `viewId: 'markdown-doc'` and + `viewParams: { doc }`. +- Register the `markdown-doc` entity view. +- Add CodeMirror markdown editor bound to `ydoc.getText('markdown')`. +- Pass auth/principal headers to `YjsProvider`. +- Handle missing/invalid doc ids and provider errors. +- Destroy CodeMirror/Yjs resources on unmount. + +Exit criteria: + +- Clicking a document manifest opens the editor. +- Two editor tiles/windows can edit the same doc concurrently. +- Agent tool edits appear in open editors. + +### Phase 6: Presence + +Tasks: + +- Set user awareness from `useCurrentPrincipal()`. +- Render presence states in the editor. +- Set agent awareness while document tools are running. +- Show agent status and cursor/edit location for replacements. + +Exit criteria: + +- Users see other active users in the document. +- Users see agent presence while an agent edit tool is applying a change. + +### Phase 7: Verification + +Tasks: + +- Run runtime tests. +- Run server tests. +- Run UI tests. +- Run package typechecks. +- Manually verify the desktop flow: + - agent creates a document. + - manifest entry appears. + - user opens it in a tile. + - user edits it. + - agent edits it with exact replacement. + - two windows/tiles see concurrent updates and presence. +- forked entity receives an independent document. + +Suggested commands after `pnpm install` from repo root: + +```sh +pnpm --filter @electric-ax/agents-runtime test +pnpm --filter @electric-ax/agents-server test +pnpm --filter @electric-ax/agents-server-ui test +pnpm --filter @electric-ax/agents-runtime typecheck +pnpm --filter @electric-ax/agents-server typecheck +pnpm --filter @electric-ax/agents-server-ui typecheck +``` + +Streaming edits should be a later design/implementation after the single PR +lands and the non-streaming collaborative document workflow is stable. + +## Current Implementation Status + +Implemented in `samwillis/agents-markdown-docs`: + +- Strict document manifest metadata: + - `provider: 'y-durable-streams'` + - `docId` + - `docPath` + - `streamPath` + - `transportMimeType: 'application/vnd.electric-agents.markdown-yjs'` + - `contentMimeType: 'text/markdown'` + - `yTextName: 'markdown'` +- Server-mediated create/read/write/edit document API backed by framed Yjs + updates. +- Runtime markdown document tools with file-like read-before-edit behavior and + diff details. +- Public Yjs document routes and private backing stream routes guarded by entity + permissions, including awareness streams. +- Fork handling for document update streams and remapped `docPath`, `docId`, + and `streamPath` manifest fields. +- CodeMirror markdown editor view backed by `YjsProvider` and `Y.Text`. +- Manifest and context-drawer open/split-right actions for document entries. +- User awareness in the editor and server-published agent awareness around + create/write/edit tools, including status and cursor/edit range. +- Focused runtime, server, and UI tests plus package typechecks for the touched + packages. + +Deferred: + +- Snapshot discovery/compaction implementation beyond the current MVP redirect + behavior. +- Token-by-token/streaming document edits. + +Manual verification note: + +- Automated tests cover the new server, runtime, fork, auth, and UI helper + behavior. A full desktop two-window/two-tile manual pass still needs a running + agents server plus desktop UI; the in-app browser check against the only + detected local UI port was blocked by browser policy after the tab crashed. diff --git a/packages/agents-runtime/package.json b/packages/agents-runtime/package.json index a12caa5275..690a17d2f0 100644 --- a/packages/agents-runtime/package.json +++ b/packages/agents-runtime/package.json @@ -110,6 +110,7 @@ "@anthropic-ai/sdk": "^0.78.0", "@durable-streams/client": "^0.2.6", "@durable-streams/state": "^0.3.1", + "@durable-streams/y-durable-streams": "0.2.7", "@electric-ax/agents-mcp": "workspace:*", "@mariozechner/pi-agent-core": "^0.70.2", "@mariozechner/pi-ai": "^0.70.2", @@ -121,11 +122,14 @@ "cron-parser": "^5.5.0", "diff": "^9.0.0", "jsdom": "^28.1.0", + "lib0": "^0.2.99", "pino": "^10.3.1", "pino-pretty": "^13.0.0", "turndown": "^7.2.2", "turndown-plugin-gfm": "^1.0.2", "xstate": "^5.32.0", + "y-protocols": "^1.0.6", + "yjs": "^13.6.26", "zod": "^4.3.6", "zod-to-json-schema": "^3.25.2" }, diff --git a/packages/agents-runtime/src/client.ts b/packages/agents-runtime/src/client.ts index 55306cc03a..2c8b2feb9b 100644 --- a/packages/agents-runtime/src/client.ts +++ b/packages/agents-runtime/src/client.ts @@ -29,7 +29,10 @@ export { export { appendPathToUrl } from './url' export { getEntityAttachmentStreamPath, + getEntityMarkdownDocumentPath, + getEntityMarkdownDocumentUrlPath, manifestAttachmentKey, + manifestMarkdownDocumentKey, } from './manifest-helpers' export { buildSections, buildTimelineEntries } from './use-chat' export { @@ -47,6 +50,7 @@ export { export { isGoalCommandText, parseGoalCommand } from './goal-command' export { formatTokenCount } from './token-budget' export type { GoalCommand } from './goal-command' +export { MARKDOWN_DOCUMENT_AGENT_PRESENCE_TTL_MS } from './markdown-yjs' export type { EntityStreamDB, @@ -68,6 +72,7 @@ export type { GoalStatus, Manifest, ManifestAttachmentEntry, + ManifestDocumentEntry, ManifestGoalEntry, } from './entity-schema' export type { diff --git a/packages/agents-runtime/src/create-handler.ts b/packages/agents-runtime/src/create-handler.ts index 1aab7893ed..aae4a12e62 100644 --- a/packages/agents-runtime/src/create-handler.ts +++ b/packages/agents-runtime/src/create-handler.ts @@ -20,7 +20,10 @@ import type { AnyEntityDefinition, EntityStreamDBWithActions, HeadersProvider, + ManifestDocumentEntry, + MarkdownDocumentConnection, ProcessWakeConfig, + RuntimePrincipal, WakeNotification, WebhookNotification, } from './types' @@ -73,6 +76,7 @@ export interface RuntimeRouterConfig { createElectricTools?: (context: { entityUrl: string entityType: string + principal?: RuntimePrincipal args: Readonly> db: EntityStreamDBWithActions events: Array @@ -99,6 +103,27 @@ export interface RuntimeRouterConfig { unsubscribeFromWebhookSource: (opts: { id: string }) => Promise<{ txid: string }> + createMarkdownDocument: (opts: { + id?: string + title: string + meta?: Record + }) => Promise<{ txid: string; document: ManifestDocumentEntry }> + getMarkdownDocumentConnection: ( + streamPath: string + ) => Promise + readMarkdownDocumentStream: ( + streamPath: string, + opts?: { offset?: string } + ) => Promise<{ bytes: Uint8Array; offset?: string }> + appendMarkdownDocumentUpdate: ( + streamPath: string, + update: Uint8Array + ) => Promise<{ offset?: string }> + appendMarkdownDocumentAwareness: ( + streamPath: string, + update: Uint8Array + ) => Promise<{ offset?: string }> + registerCleanup: (cleanup: () => void | Promise) => void }) => Array | Promise> /** * Optional observer for background wake failures. Return true to mark the diff --git a/packages/agents-runtime/src/entity-schema.ts b/packages/agents-runtime/src/entity-schema.ts index 0f5d8c4304..9f91cf295a 100644 --- a/packages/agents-runtime/src/entity-schema.ts +++ b/packages/agents-runtime/src/entity-schema.ts @@ -358,6 +358,23 @@ type ManifestAttachmentEntryValue = { error?: string meta?: Record } +type ManifestDocumentEntryValue = { + key?: string + kind: `document` + id: string + provider: `y-durable-streams` + docId: string + docPath: string + streamPath: string + transportMimeType: `application/vnd.electric-agents.markdown-yjs` + contentMimeType: `text/markdown` + yTextName: `markdown` + title: string + createdAt: string + createdBy?: string + updatedAt?: string + meta?: Record +} type ContextEntryAttrsValue = Record type ManifestContextEntryValue = { key?: string @@ -811,6 +828,7 @@ function createManifestSchema(): Schema< | ManifestSharedStateEntryValue | ManifestEffectEntryValue | ManifestAttachmentEntryValue + | ManifestDocumentEntryValue | ManifestContextEntryValue | ManifestCronScheduleEntryValue | ManifestFutureSendScheduleEntryValue @@ -877,6 +895,26 @@ function createManifestSchema(): Schema< error: z.string().optional(), meta: createAttachmentMetaSchema().optional(), }), + z.object({ + key: z.string().optional(), + ...timelineOrderField, + kind: z.literal(`document`), + id: z.string(), + provider: z.literal(`y-durable-streams`), + docId: z.string(), + docPath: z.string(), + streamPath: z.string(), + transportMimeType: z.literal( + `application/vnd.electric-agents.markdown-yjs` + ), + contentMimeType: z.literal(`text/markdown`), + yTextName: z.literal(`markdown`), + title: z.string(), + createdAt: z.string(), + createdBy: z.string().optional(), + updatedAt: z.string().optional(), + meta: createAttachmentMetaSchema().optional(), + }), z.object({ key: z.string().optional(), ...timelineOrderField, @@ -936,6 +974,7 @@ function createManifestSchema(): Schema< | ManifestSharedStateEntryValue | ManifestEffectEntryValue | ManifestAttachmentEntryValue + | ManifestDocumentEntryValue | ManifestContextEntryValue | ManifestCronScheduleEntryValue | ManifestFutureSendScheduleEntryValue @@ -990,6 +1029,8 @@ export type AttachmentRole = AttachmentRoleValue export type AttachmentSubject = AttachmentSubjectValue export type ManifestAttachmentEntry = SequencedPersistedRow +export type ManifestDocumentEntry = + SequencedPersistedRow export type ManifestContextEntry = SequencedPersistedRow export type ManifestCronScheduleEntry = @@ -1004,6 +1045,7 @@ type ManifestUnion = | ManifestSharedStateEntry | ManifestEffectEntry | ManifestAttachmentEntry + | ManifestDocumentEntry | ManifestContextEntry | ManifestCronScheduleEntry | ManifestFutureSendScheduleEntry @@ -1028,6 +1070,14 @@ export type Manifest = ManifestUnion & { createdBy?: string error?: string meta?: Record + provider?: `y-durable-streams` + docId?: string + docPath?: string + transportMimeType?: `application/vnd.electric-agents.markdown-yjs` + contentMimeType?: `text/markdown` + yTextName?: `markdown` + title?: string + updatedAt?: string name?: string attrs?: ContextEntryAttrs content?: string diff --git a/packages/agents-runtime/src/index.ts b/packages/agents-runtime/src/index.ts index 61093c87b4..5b4e1419cc 100644 --- a/packages/agents-runtime/src/index.ts +++ b/packages/agents-runtime/src/index.ts @@ -7,6 +7,7 @@ export type { ManifestAttachmentEntry, ManifestChildEntry, ManifestContextEntry, + ManifestDocumentEntry, ManifestEntry, ManifestEffectEntry, ManifestSourceEntry, @@ -74,6 +75,7 @@ export type { GeneratedStateActions, HandlerActions, ManifestContextEntry as ManifestContextRow, + ManifestDocumentEntry as ManifestDocumentRow, SchemaInput, SchemaOutput, SourceConfig, @@ -117,6 +119,7 @@ export type { AttachmentSubject, AttachmentSubjectType, ManifestContextEntry as ManifestContextEntryRow, + ManifestDocumentEntry as ManifestDocumentEntryRow, ReplayWatermark, WakeConfigValue, } from './entity-schema' @@ -124,7 +127,10 @@ export type { export { createEntityStreamDB } from './entity-stream-db' export { getEntityAttachmentStreamPath, + getEntityMarkdownDocumentPath, + getEntityMarkdownDocumentUrlPath, manifestAttachmentKey, + manifestMarkdownDocumentKey, } from './manifest-helpers' export { COMPOSER_INPUT_MESSAGE_TYPE, diff --git a/packages/agents-runtime/src/manifest-helpers.ts b/packages/agents-runtime/src/manifest-helpers.ts index 599cd6fa38..bce492b98c 100644 --- a/packages/agents-runtime/src/manifest-helpers.ts +++ b/packages/agents-runtime/src/manifest-helpers.ts @@ -16,9 +16,37 @@ export function manifestAttachmentKey(id: string): string { return `attachment:${id}` } +export function manifestMarkdownDocumentKey(id: string): string { + return `document:${id}` +} + export function getEntityAttachmentStreamPath( entityUrl: string, attachmentId: string ): string { return `${entityUrl.replace(/\/+$/, ``)}/attachments/${attachmentId}` } + +export function getEntityMarkdownDocumentPath( + entityUrl: string, + documentId: string +): string { + const segments = entityUrl.replace(/^\/+|\/+$/g, ``).split(`/`) + if (segments.length !== 2 || !segments[0] || !segments[1]) { + throw new Error( + `Invalid entity URL for markdown document path: ${entityUrl}` + ) + } + return `agents/${segments[0]}/${segments[1]}/documents/${documentId}` +} + +export function getEntityMarkdownDocumentUrlPath( + service: string, + entityUrl: string, + documentId: string +): string { + return `/v1/yjs/${encodeURIComponent(service)}/docs/${getEntityMarkdownDocumentPath( + entityUrl, + documentId + )}` +} diff --git a/packages/agents-runtime/src/markdown-document-session.ts b/packages/agents-runtime/src/markdown-document-session.ts new file mode 100644 index 0000000000..f1879fb823 --- /dev/null +++ b/packages/agents-runtime/src/markdown-document-session.ts @@ -0,0 +1,163 @@ +import { YjsProvider } from '@durable-streams/y-durable-streams' +import { Awareness } from 'y-protocols/awareness' +import * as Y from 'yjs' +import { + MARKDOWN_DOCUMENT_AGENT_PRESENCE_TTL_MS, + MARKDOWN_DOCUMENT_TEXT_NAME, + markdownText, +} from './markdown-yjs' +import type { + ManifestDocumentEntry, + MarkdownDocumentConnection, + RuntimePrincipal, +} from './types' + +export type MarkdownDocumentPresence = { + anchor?: number + head?: number + clear?: boolean +} + +export type MarkdownDocumentSession = { + readonly document: ManifestDocumentEntry + readonly doc: Y.Doc + readonly text: Y.Text + readonly textName: string + content: () => string + setPresence: (opts: MarkdownDocumentPresence) => Promise + flush: () => Promise + close: () => Promise +} + +export async function openMarkdownDocumentSession(opts: { + document: ManifestDocumentEntry + connection: MarkdownDocumentConnection + entityUrl: string + principal?: RuntimePrincipal +}): Promise { + const doc = new Y.Doc() + const textName = opts.document.yTextName || MARKDOWN_DOCUMENT_TEXT_NAME + const text = markdownText(doc, textName) + const awareness = new Awareness(doc) + const provider = new YjsProvider({ + doc, + baseUrl: opts.connection.baseUrl, + docId: opts.connection.docId, + awareness, + headers: opts.connection.headers, + liveMode: `sse`, + connect: false, + }) + const principalUrl = + opts.principal?.url ?? + `/principal/entity:${encodeURIComponent(opts.entityUrl)}` + const color = principalColor(principalUrl) + + await provider.connect() + + const content = (): string => text.toString() + + const setPresence = async ({ + anchor, + head, + clear, + }: MarkdownDocumentPresence): Promise => { + if (clear) { + awareness.setLocalState(null) + await settleAwarenessUpdate() + return + } + const boundedAnchor = boundIndex(anchor ?? text.length, text.length) + const boundedHead = boundIndex(head ?? boundedAnchor, text.length) + const now = Date.now() + awareness.setLocalState({ + user: { + name: principalDisplayName(principalUrl), + principalUrl, + role: principalRole(principalUrl), + status: `editing`, + updatedAt: now, + expiresAt: now + MARKDOWN_DOCUMENT_AGENT_PRESENCE_TTL_MS, + color: color.color, + colorLight: color.colorLight, + }, + cursor: { + anchor: Y.createRelativePositionFromTypeIndex(text, boundedAnchor), + head: Y.createRelativePositionFromTypeIndex(text, boundedHead), + }, + }) + await settleAwarenessUpdate() + } + + return { + document: opts.document, + doc, + text, + textName, + content, + setPresence, + flush: () => provider.flush(), + close: async () => { + awareness.setLocalState(null) + await settleAwarenessUpdate() + await provider.flush() + await provider.disconnect() + awareness.destroy() + provider.destroy() + doc.destroy() + }, + } +} + +function boundIndex(value: number, length: number): number { + return Math.max(0, Math.min(Math.floor(value), length)) +} + +function settleAwarenessUpdate(): Promise { + return new Promise((resolve) => setTimeout(resolve, 0)) +} + +function principalDisplayName(principalUrl: string): string { + const raw = principalUrl.split(`/principal/`).at(-1) ?? principalUrl + let decoded = raw + try { + decoded = decodeURIComponent(raw) + } catch { + // Keep the raw value when the URL segment is not URI encoded. + } + const withoutPrefix = decoded.replace(/^(user|agent|entity|system):/, ``) + return withoutPrefix || decoded || principalUrl +} + +function principalRole(principalUrl: string): `agent` | `user` | `system` { + const raw = principalUrl.split(`/principal/`).at(-1) ?? principalUrl + let decoded = raw + try { + decoded = decodeURIComponent(raw) + } catch { + // Keep the raw value when the URL segment is not URI encoded. + } + if (decoded.startsWith(`user:`)) return `user` + if (decoded.startsWith(`system:`)) return `system` + return `agent` +} + +function principalColor(principalUrl: string): { + color: string + colorLight: string +} { + const colors = [ + [`#2563eb`, `#2563eb33`], + [`#059669`, `#05966933`], + [`#dc2626`, `#dc262633`], + [`#7c3aed`, `#7c3aed33`], + [`#c2410c`, `#c2410c33`], + [`#0f766e`, `#0f766e33`], + ] as const + let hash = 0 + for (let i = 0; i < principalUrl.length; i += 1) { + hash = (hash * 31 + principalUrl.charCodeAt(i)) >>> 0 + } + const [color, colorLight] = colors[hash % colors.length]! + return { color, colorLight } +} diff --git a/packages/agents-runtime/src/markdown-yjs.ts b/packages/agents-runtime/src/markdown-yjs.ts new file mode 100644 index 0000000000..0309508028 --- /dev/null +++ b/packages/agents-runtime/src/markdown-yjs.ts @@ -0,0 +1,256 @@ +import * as decoding from 'lib0/decoding' +import * as encoding from 'lib0/encoding' +import { Awareness, encodeAwarenessUpdate } from 'y-protocols/awareness' +import * as Y from 'yjs' + +export const MARKDOWN_DOCUMENT_TEXT_NAME = `markdown` as const +export const MARKDOWN_DOCUMENT_AGENT_PRESENCE_TTL_MS = 45_000 + +export function frameYjsUpdate(update: Uint8Array): Uint8Array { + const encoder = encoding.createEncoder() + encoding.writeVarUint8Array(encoder, update) + return encoding.toUint8Array(encoder) +} + +export function applyFramedYjsUpdates(doc: Y.Doc, data: Uint8Array): void { + if (data.length === 0) return + const decoder = decoding.createDecoder(data) + while (decoding.hasContent(decoder)) { + Y.applyUpdate(doc, decoding.readVarUint8Array(decoder), `agent`) + } +} + +export function markdownText( + doc: Y.Doc, + name: string = MARKDOWN_DOCUMENT_TEXT_NAME +): Y.Text { + return doc.getText(name) +} + +export function createMarkdownYDoc(data: Uint8Array): Y.Doc { + const doc = new Y.Doc() + applyFramedYjsUpdates(doc, data) + return doc +} + +export function replaceMarkdownText( + doc: Y.Doc, + content: string, + textName: string = MARKDOWN_DOCUMENT_TEXT_NAME +): Uint8Array { + const before = Y.encodeStateVector(doc) + const text = markdownText(doc, textName) + doc.transact(() => { + text.delete(0, text.length) + if (content.length > 0) text.insert(0, content) + }, `agent`) + return Y.encodeStateAsUpdate(doc, before) +} + +export function editMarkdownText( + doc: Y.Doc, + oldString: string, + newString: string, + replaceAll: boolean | undefined, + textName: string = MARKDOWN_DOCUMENT_TEXT_NAME +): { + update: Uint8Array + content: string + replacements: number + cursorIndex?: number +} { + const text = markdownText(doc, textName) + const beforeContent = text.toString() + const matches = beforeContent.split(oldString).length - 1 + if (matches === 0 || (!replaceAll && matches > 1)) { + return { + update: new Uint8Array(), + content: beforeContent, + replacements: matches, + } + } + + const before = Y.encodeStateVector(doc) + let cursorIndex = 0 + if (replaceAll) { + let cursor = 0 + doc.transact(() => { + while (true) { + const index = text.toString().indexOf(oldString, cursor) + if (index < 0) break + text.delete(index, oldString.length) + text.insert(index, newString) + cursor = index + newString.length + cursorIndex = cursor + } + }, `agent`) + } else { + const index = beforeContent.indexOf(oldString) + doc.transact(() => { + text.delete(index, oldString.length) + text.insert(index, newString) + }, `agent`) + cursorIndex = index + newString.length + } + return { + update: Y.encodeStateAsUpdate(doc, before), + content: text.toString(), + replacements: matches, + cursorIndex, + } +} + +export function insertMarkdownText( + doc: Y.Doc, + content: string, + opts?: { + index?: number + position?: Y.RelativePosition + textName?: string + } +): { + update: Uint8Array + index: number + nextIndex: number + nextPosition: Y.RelativePosition +} { + const text = markdownText(doc, opts?.textName) + const absolute = opts?.position + ? Y.createAbsolutePositionFromRelativePosition(opts.position, doc) + : null + const index = + absolute && absolute.type === text + ? Math.max(0, Math.min(absolute.index, text.length)) + : Math.max(0, Math.min(opts?.index ?? text.length, text.length)) + const before = Y.encodeStateVector(doc) + if (content.length > 0) { + doc.transact(() => { + text.insert(index, content) + }, `agent`) + } + const nextIndex = index + content.length + return { + update: Y.encodeStateAsUpdate(doc, before), + index, + nextIndex, + nextPosition: Y.createRelativePositionFromTypeIndex(text, nextIndex), + } +} + +export function deleteMarkdownTextRange( + doc: Y.Doc, + index: number, + length: number, + textName: string = MARKDOWN_DOCUMENT_TEXT_NAME +): { + update: Uint8Array + index: number + length: number + position: Y.RelativePosition +} { + const text = markdownText(doc, textName) + const boundedIndex = Math.max(0, Math.min(index, text.length)) + const boundedLength = Math.max( + 0, + Math.min(length, text.length - boundedIndex) + ) + const before = Y.encodeStateVector(doc) + if (boundedLength > 0) { + doc.transact(() => { + text.delete(boundedIndex, boundedLength) + }, `agent`) + } + return { + update: Y.encodeStateAsUpdate(doc, before), + index: boundedIndex, + length: boundedLength, + position: Y.createRelativePositionFromTypeIndex(text, boundedIndex), + } +} + +export function relativePositionAtMarkdownIndex( + doc: Y.Doc, + index: number, + textName: string = MARKDOWN_DOCUMENT_TEXT_NAME +): Y.RelativePosition { + const text = markdownText(doc, textName) + const boundedIndex = Math.max(0, Math.min(index, text.length)) + return Y.createRelativePositionFromTypeIndex(text, boundedIndex) +} + +export function markdownIndexFromRelativePosition( + doc: Y.Doc, + position: Y.RelativePosition, + textName: string = MARKDOWN_DOCUMENT_TEXT_NAME +): number | undefined { + const text = markdownText(doc, textName) + const absolute = Y.createAbsolutePositionFromRelativePosition(position, doc) + if (!absolute || absolute.type !== text) return undefined + return Math.max(0, Math.min(absolute.index, text.length)) +} + +export function encodeMarkdownAwarenessUpdate(opts: { + doc: Y.Doc + docPath: string + principalUrl: string + clientKey?: string + name: string + role: `agent` | `user` | `system` + status?: `editing` + anchor?: number + head?: number + color: string + colorLight: string + clear?: boolean + textName?: string +}): Uint8Array { + const awarenessDoc = new Y.Doc() + ;(awarenessDoc as { clientID: number }).clientID = + markdownDocumentPresenceClientId( + opts.docPath, + opts.clientKey ?? opts.principalUrl + ) + const awareness = new Awareness(awarenessDoc) + if (opts.clear) { + awareness.setLocalState(null) + } else { + const text = markdownText(opts.doc, opts.textName) + const anchor = Math.max( + 0, + Math.min(opts.anchor ?? text.length, text.length) + ) + const head = Math.max(0, Math.min(opts.head ?? anchor, text.length)) + const now = Date.now() + awareness.setLocalState({ + user: { + name: opts.name, + principalUrl: opts.principalUrl, + role: opts.role, + status: opts.status ?? `editing`, + updatedAt: now, + expiresAt: now + MARKDOWN_DOCUMENT_AGENT_PRESENCE_TTL_MS, + color: opts.color, + colorLight: opts.colorLight, + }, + cursor: { + anchor: Y.createRelativePositionFromTypeIndex(text, anchor), + head: Y.createRelativePositionFromTypeIndex(text, head), + }, + }) + } + return frameYjsUpdate(encodeAwarenessUpdate(awareness, [awareness.clientID])) +} + +function markdownDocumentPresenceClientId( + docPath: string, + principalUrl: string +): number { + let hash = 2166136261 + const input = `${docPath}\0${principalUrl}` + for (let i = 0; i < input.length; i += 1) { + hash ^= input.charCodeAt(i) + hash = Math.imul(hash, 16777619) + } + const id = hash >>> 0 + return id === 0 ? 1 : id +} diff --git a/packages/agents-runtime/src/process-wake.ts b/packages/agents-runtime/src/process-wake.ts index edb7a850a8..2cc8512cc2 100644 --- a/packages/agents-runtime/src/process-wake.ts +++ b/packages/agents-runtime/src/process-wake.ts @@ -584,6 +584,7 @@ export async function processWake( detachWrites?: () => Promise close: () => void }> = [] + const electricToolCleanups: Array<() => void | Promise> = [] let liveProcessError: Error | null = null let acceptLiveInputs = false const handledSignalKeys = new Set() @@ -2078,6 +2079,7 @@ export async function processWake( ? await config.createElectricTools({ entityUrl, entityType: typeName, + principal: notification.principal, args: entityArgs, db, events: currentWakeEvents, @@ -2107,6 +2109,22 @@ export async function processWake( entityUrl, ...opts, }), + createMarkdownDocument: (opts) => + serverClient.createMarkdownDocument({ + entityUrl, + ...opts, + }), + getMarkdownDocumentConnection: (streamPath) => + serverClient.getMarkdownDocumentConnection(streamPath), + readMarkdownDocumentStream: (streamPath, opts) => + serverClient.readMarkdownDocumentStream(streamPath, opts), + appendMarkdownDocumentUpdate: (streamPath, update) => + serverClient.appendMarkdownDocumentUpdate(streamPath, update), + appendMarkdownDocumentAwareness: (streamPath, update) => + serverClient.appendMarkdownDocumentAwareness(streamPath, update), + registerCleanup: (cleanup) => { + electricToolCleanups.push(cleanup) + }, }) : [] @@ -2374,6 +2392,13 @@ export async function processWake( } catch (err) { cleanupErrors.push(toError(err)) } + for (const cleanup of electricToolCleanups.splice(0).reverse()) { + try { + await cleanup() + } catch (err) { + cleanupErrors.push(toError(err)) + } + } // Updated by the handler-error path before control reaches this async cleanup. if (ackCurrentWakeOnFailure && cleanupErrors.length === 0) { diff --git a/packages/agents-runtime/src/runtime-server-client.ts b/packages/agents-runtime/src/runtime-server-client.ts index ec7f713166..613d4148c5 100644 --- a/packages/agents-runtime/src/runtime-server-client.ts +++ b/packages/agents-runtime/src/runtime-server-client.ts @@ -9,7 +9,9 @@ import type { AttachmentCreateInput, ClaimTokenHeader, HeadersProvider, + MarkdownDocumentConnection, ManifestAttachmentEntry, + ManifestDocumentEntry, } from './types' import type { EntitySignal } from './entity-schema' import type { @@ -21,6 +23,13 @@ export type { EntitySignal } from './entity-schema' const ELECTRIC_PRINCIPAL_HEADER = `electric-principal` +function bytesBody(bytes: Uint8Array): ArrayBuffer { + return bytes.buffer.slice( + bytes.byteOffset, + bytes.byteOffset + bytes.byteLength + ) as ArrayBuffer +} + export interface RuntimeServerClientConfig { baseUrl: string fetch?: typeof globalThis.fetch @@ -128,6 +137,27 @@ export interface RuntimeServerClient { entityUrl: string id: string }) => Promise + createMarkdownDocument: (options: { + entityUrl: string + id?: string + title: string + meta?: Record + }) => Promise<{ txid: string; document: ManifestDocumentEntry }> + getMarkdownDocumentConnection: ( + streamPath: string + ) => Promise + readMarkdownDocumentStream: ( + streamPath: string, + opts?: { offset?: string } + ) => Promise<{ bytes: Uint8Array; offset?: string }> + appendMarkdownDocumentUpdate: ( + streamPath: string, + update: Uint8Array + ) => Promise<{ offset?: string }> + appendMarkdownDocumentAwareness: ( + streamPath: string, + update: Uint8Array + ) => Promise<{ offset?: string }> spawnEntity: (options: SpawnEntityOptions) => Promise /** * Fork an entity at the server-resolved `latest_completed_run` anchor. @@ -457,6 +487,129 @@ export function createRuntimeServerClient( return new Uint8Array(await response.arrayBuffer()) } + const createMarkdownDocument = async ({ + entityUrl, + id, + title, + meta, + }: { + entityUrl: string + id?: string + title: string + meta?: Record + }): Promise<{ txid: string; document: ManifestDocumentEntry }> => { + const response = await request(`${entityRpcPath(entityUrl)}/documents`, { + method: `POST`, + headers: { 'content-type': `application/json` }, + body: JSON.stringify({ id, title, meta }), + }) + if (!response.ok) { + throw new Error( + `create markdown document on ${entityUrl} failed (${response.status}): ${await readErrorText(response)}` + ) + } + return (await response.json()) as { + txid: string + document: ManifestDocumentEntry + } + } + + const getMarkdownDocumentConnection = async ( + streamPath: string + ): Promise => { + const docsIndex = streamPath.indexOf(`/docs/`) + if (docsIndex < 0) { + throw new Error( + `markdown document stream path is missing /docs/: ${streamPath}` + ) + } + const prefix = streamPath.slice(0, docsIndex) + const docId = streamPath.slice(docsIndex + `/docs/`.length) + const headers = await resolveHeaders() + if (config.principalKey) { + headers.set(ELECTRIC_PRINCIPAL_HEADER, config.principalKey) + } + const headerRecord: Record = {} + headers.forEach((value, key) => { + headerRecord[key] = value + }) + return { + baseUrl: appendPathToUrl(config.baseUrl, prefix).replace(/\/+$/, ``), + docId, + headers: headerRecord, + } + } + + const readMarkdownDocumentStream = async ( + streamPath: string, + opts?: { offset?: string } + ): Promise<{ bytes: Uint8Array; offset?: string }> => { + const url = new URL(streamPath, `http://agent-runtime.local`) + if (opts?.offset !== undefined) { + url.searchParams.set(`offset`, opts.offset) + } + const path = `${url.pathname}${url.search}` + const response = await request(path, { method: `GET` }) + if (!response.ok) { + throw new Error( + `read markdown document stream ${path} failed (${response.status}): ${await readErrorText(response)}` + ) + } + return { + bytes: new Uint8Array(await response.arrayBuffer()), + offset: response.headers.get(`stream-next-offset`) ?? undefined, + } + } + + const appendMarkdownDocumentUpdate = async ( + streamPath: string, + update: Uint8Array + ): Promise<{ offset?: string }> => { + const response = await request(streamPath, { + method: `POST`, + headers: { 'content-type': `application/octet-stream` }, + body: bytesBody(update), + }) + if (!response.ok) { + throw new Error( + `append markdown document update ${streamPath} failed (${response.status}): ${await readErrorText(response)}` + ) + } + return { + offset: response.headers.get(`stream-next-offset`) ?? undefined, + } + } + + const appendMarkdownDocumentAwareness = async ( + streamPath: string, + update: Uint8Array + ): Promise<{ offset?: string }> => { + const awarenessPath = `${streamPath}?awareness=default` + const append = () => + request(awarenessPath, { + method: `POST`, + headers: { 'content-type': `application/octet-stream` }, + body: bytesBody(update), + }) + let response = await append() + if (response.status === 404) { + response = await request(awarenessPath, { + method: `PUT`, + headers: { 'content-type': `application/octet-stream` }, + body: bytesBody(update), + }) + if (response.status === 409) response = await append() + } + if (!response.ok) { + throw new Error( + `append markdown document awareness ${streamPath} failed (${response.status}): ${await readErrorText(response)}` + ) + } + return { + offset: response.headers.get(`stream-next-offset`) ?? undefined, + } + } + const getEntity = async (entityUrl: string): Promise => { const response = await request(entityRpcPath(entityUrl), { method: `GET` }) if (!response.ok) { @@ -942,6 +1095,11 @@ export function createRuntimeServerClient( sendEntityMessage, createAttachment, readAttachment, + createMarkdownDocument, + getMarkdownDocumentConnection, + readMarkdownDocumentStream, + appendMarkdownDocumentUpdate, + appendMarkdownDocumentAwareness, spawnEntity, forkEntity, getEntity, diff --git a/packages/agents-runtime/src/tools.ts b/packages/agents-runtime/src/tools.ts index e0026c6bf9..a8132d3c4a 100644 --- a/packages/agents-runtime/src/tools.ts +++ b/packages/agents-runtime/src/tools.ts @@ -8,3 +8,4 @@ export { createScheduleTools } from './tools/schedules' export { createWebhookSourceTools } from './tools/webhook-sources' export { createSendTool } from './tools/send' export { createMarkGoalCompleteTool } from './tools/goal-tools' +export { createMarkdownDocumentTools } from './tools/markdown-docs' diff --git a/packages/agents-runtime/src/tools/markdown-docs.ts b/packages/agents-runtime/src/tools/markdown-docs.ts new file mode 100644 index 0000000000..d9354db5fc --- /dev/null +++ b/packages/agents-runtime/src/tools/markdown-docs.ts @@ -0,0 +1,1064 @@ +import { createTwoFilesPatch } from 'diff' +import { Type } from '@sinclair/typebox' +import { + markdownIndexFromRelativePosition, + relativePositionAtMarkdownIndex, +} from '../markdown-yjs' +import { + openMarkdownDocumentSession, + type MarkdownDocumentSession, +} from '../markdown-document-session' +import type { AgentTool, ProcessWakeConfig } from '../types' +import type { ManifestDocumentEntry } from '../entity-schema' +import * as Y from 'yjs' + +type ElectricToolContextBase = Parameters< + NonNullable +>[0] + +type ElectricToolContext = ElectricToolContextBase & { + openMarkdownDocumentSession?: (opts: { + document: ManifestDocumentEntry + entityUrl: string + principal: ElectricToolContextBase[`principal`] + }) => Promise +} + +function docLabel(id: string): string { + return `markdown-doc:${id}` +} + +type InsertMarkdownArgs = { + id: string + content: string + index?: number +} + +type ReplaceMarkdownArgs = { + id: string + content: string + old_string?: string + occurrence?: number + index?: number + length?: number +} + +type SetCursorArgs = { + id: string + index?: number + before?: string + after?: string + occurrence?: number +} + +type InsertSession = { + id?: string + inserted: string + nextIndex?: number + nextPosition?: Y.RelativePosition + seq: number + streamed: boolean + pending: Promise + error?: unknown +} + +type ReplaceSession = InsertSession & { + prepared?: boolean + deleted?: string + deleteIndex?: number + deleteLength?: number + beforeContent?: string +} + +type MaterializedMarkdownDocument = { + document: ManifestDocumentEntry + session: MarkdownDocumentSession + textName: string +} + +function injectedMarkdownDocuments( + args: Readonly> +): Array { + const docs = args.markdownDocs + if (!Array.isArray(docs)) return [] + return docs.filter(isManifestDocumentEntry) +} + +function isManifestDocumentEntry( + value: unknown +): value is ManifestDocumentEntry { + if (!value || typeof value !== `object`) return false + const entry = value as Partial + return ( + entry.kind === `document` && + typeof entry.id === `string` && + entry.provider === `y-durable-streams` && + typeof entry.docPath === `string` && + typeof entry.streamPath === `string` && + entry.transportMimeType === + `application/vnd.electric-agents.markdown-yjs` && + entry.contentMimeType === `text/markdown` && + entry.yTextName === `markdown` && + typeof entry.title === `string` + ) +} + +function asInsertArgs(value: unknown): Partial { + if (!value || typeof value !== `object`) return {} + const input = value as Record + return { + ...(typeof input.id === `string` && { id: input.id }), + ...(typeof input.content === `string` && { content: input.content }), + ...(typeof input.index === `number` && Number.isFinite(input.index) + ? { index: input.index } + : {}), + } +} + +function asReplaceArgs(value: unknown): Partial { + if (!value || typeof value !== `object`) return {} + const input = value as Record + return { + ...(typeof input.id === `string` && { id: input.id }), + ...(typeof input.content === `string` && { content: input.content }), + ...(typeof input.old_string === `string` && { + old_string: input.old_string, + }), + ...(typeof input.occurrence === `number` && + Number.isFinite(input.occurrence) + ? { occurrence: input.occurrence } + : {}), + ...(typeof input.index === `number` && Number.isFinite(input.index) + ? { index: input.index } + : {}), + ...(typeof input.length === `number` && Number.isFinite(input.length) + ? { length: input.length } + : {}), + } +} + +export function createMarkdownDocumentTools( + context: ElectricToolContext +): Array { + const readDocs = new Map() + const insertSessions = new Map() + const replaceSessions = new Map() + const materializedDocs = new Map() + const cursorPositions = new Map() + + const findManifestDocument = ( + id: string + ): ManifestDocumentEntry | undefined => { + const manifests = context.db.collections.manifests?.toArray as + | Array + | undefined + return ( + manifests?.find( + (entry): entry is ManifestDocumentEntry => + isManifestDocumentEntry(entry) && entry.id === id + ) ?? + injectedMarkdownDocuments(context.args).find( + (entry): entry is ManifestDocumentEntry => + isManifestDocumentEntry(entry) && entry.id === id + ) + ) + } + + context.registerCleanup(async () => { + const sessions = Array.from(materializedDocs.values()) + materializedDocs.clear() + await Promise.all( + sessions.map((materialized) => materialized.session.close()) + ) + }) + + const openDocumentSession = async ( + document: ManifestDocumentEntry + ): Promise => { + const cached = materializedDocs.get(document.id) + if (cached) return cached + const session = context.openMarkdownDocumentSession + ? await context.openMarkdownDocumentSession({ + document, + entityUrl: context.entityUrl, + principal: context.principal, + }) + : await openMarkdownDocumentSession({ + document, + connection: await context.getMarkdownDocumentConnection( + document.streamPath + ), + entityUrl: context.entityUrl, + principal: context.principal, + }) + const materialized = { + document, + session, + textName: document.yTextName, + } + materializedDocs.set(document.id, materialized) + readDocs.set(document.id, contentOf(materialized)) + return materialized + } + + const materializeDocument = async ( + id: string + ): Promise => { + const cached = materializedDocs.get(id) + if (cached) return cached + const document = findManifestDocument(id) + if (!document) { + throw new Error( + `Markdown document ${JSON.stringify( + id + )} is not in this entity's manifest or injected document refs. Create it with create_markdown_doc first or pass the document ref to this worker.` + ) + } + return openDocumentSession(document) + } + + const contentOf = (materialized: MaterializedMarkdownDocument): string => + materialized.session.content() + + const appendPresence = async ( + materialized: MaterializedMarkdownDocument, + opts: { anchor?: number; head?: number; clear?: boolean } + ): Promise => { + await materialized.session.setPresence(opts).catch(() => undefined) + } + + const applyInsertChunk = async ( + id: string, + chunk: string, + session: InsertSession, + index?: number + ): Promise => { + const materialized = await materializeDocument(id) + const text = materialized.session.text + const position = + session.nextPosition ?? + (index === undefined ? cursorPositions.get(id) : undefined) + const absolute = position + ? Y.createAbsolutePositionFromRelativePosition( + position, + materialized.session.doc + ) + : null + const insertIndex = + absolute && absolute.type === text + ? Math.max(0, Math.min(absolute.index, text.length)) + : Math.max( + 0, + Math.min( + session.nextIndex ?? (index !== undefined ? index : text.length), + text.length + ) + ) + if (chunk.length > 0) { + materialized.session.doc.transact(() => { + text.insert(insertIndex, chunk) + }, `agent`) + } + const nextIndex = insertIndex + chunk.length + const nextPosition = Y.createRelativePositionFromTypeIndex(text, nextIndex) + await appendPresence(materialized, { + anchor: nextIndex, + head: nextIndex, + }) + session.nextIndex = nextIndex + session.nextPosition = nextPosition + cursorPositions.set(id, nextPosition) + session.streamed = true + readDocs.set(id, contentOf(materialized)) + } + + const setCursor = async ( + id: string, + index: number + ): Promise<{ materialized: MaterializedMarkdownDocument; index: number }> => { + const materialized = await materializeDocument(id) + const text = materialized.session.text + const boundedIndex = Math.max(0, Math.min(index, text.length)) + const position = relativePositionAtMarkdownIndex( + materialized.session.doc, + boundedIndex, + materialized.textName + ) + cursorPositions.set(id, position) + return { materialized, index: boundedIndex } + } + + const resolveCursorIndex = ( + content: string, + args: SetCursorArgs + ): { index?: number; error?: string } => { + const locatorCount = + (args.index !== undefined ? 1 : 0) + + (args.before !== undefined ? 1 : 0) + + (args.after !== undefined ? 1 : 0) + if (locatorCount > 1) { + return { error: `Pass only one of index, before, or after.` } + } + if (args.index !== undefined) return { index: args.index } + const needle = args.before ?? args.after + if (needle === undefined) return { index: content.length } + if (needle.length === 0) { + return { error: `before/after must not be empty.` } + } + const occurrence = Math.max(1, Math.floor(args.occurrence ?? 1)) + let from = 0 + let found = -1 + for (let count = 0; count < occurrence; count += 1) { + found = content.indexOf(needle, from) + if (found < 0) { + return { + error: `Could not find occurrence ${occurrence} of ${JSON.stringify( + needle + )}.`, + } + } + from = found + needle.length + } + return { index: args.after !== undefined ? found + needle.length : found } + } + + const resolveReplaceRange = ( + content: string, + args: Omit + ): { index?: number; length?: number; deleted?: string; error?: string } => { + const hasOldString = args.old_string !== undefined + const hasRange = args.index !== undefined || args.length !== undefined + if (hasOldString && hasRange) { + return { error: `Pass either old_string or index/length, not both.` } + } + if (!hasOldString && !hasRange) { + return { error: `Pass old_string or index/length to choose a range.` } + } + if (hasOldString) { + const oldString = args.old_string! + if (oldString.length === 0) { + return { error: `old_string must not be empty.` } + } + const occurrence = + args.occurrence === undefined + ? undefined + : Math.max(1, Math.floor(args.occurrence)) + let from = 0 + let found = -1 + let count = 0 + while (true) { + const index = content.indexOf(oldString, from) + if (index < 0) break + count += 1 + if (occurrence === undefined || count === occurrence) { + found = index + if (occurrence !== undefined) break + } + from = index + oldString.length + } + if (found < 0) { + return { + error: + occurrence === undefined + ? `old_string not found.` + : `Could not find occurrence ${occurrence} of old_string.`, + } + } + if (occurrence === undefined && count > 1) { + return { + error: `found ${count} matches for old_string; pass occurrence to choose one or provide a more specific old_string.`, + } + } + return { + index: found, + length: oldString.length, + deleted: oldString, + } + } + + if (args.index === undefined || args.length === undefined) { + return { error: `Pass both index and length for explicit ranges.` } + } + const index = Math.max(0, Math.min(Math.floor(args.index), content.length)) + const length = Math.max( + 0, + Math.min(Math.floor(args.length), content.length - index) + ) + if (length === 0) { + return { error: `Replacement range length must be greater than zero.` } + } + return { + index, + length, + deleted: content.slice(index, index + length), + } + } + + const prepareReplaceSession = async ( + session: ReplaceSession, + args: Omit + ): Promise => { + const materialized = await materializeDocument(args.id) + if (session.prepared) return materialized + + const before = contentOf(materialized) + const range = resolveReplaceRange(before, args) + if ( + range.error || + range.index === undefined || + range.length === undefined + ) { + throw new Error(range.error ?? `Could not resolve replacement range.`) + } + + await appendPresence(materialized, { + anchor: range.index, + head: range.index + range.length, + }) + const text = materialized.session.text + const deleteIndex = Math.max(0, Math.min(range.index, text.length)) + const deleteLength = Math.max( + 0, + Math.min(range.length, text.length - deleteIndex) + ) + if (deleteLength > 0) { + materialized.session.doc.transact(() => { + text.delete(deleteIndex, deleteLength) + }, `agent`) + } + const deletePosition = Y.createRelativePositionFromTypeIndex( + text, + deleteIndex + ) + await appendPresence(materialized, { + anchor: deleteIndex, + head: deleteIndex, + }) + + session.id = args.id + session.nextIndex = deleteIndex + session.nextPosition = deletePosition + session.prepared = true + session.deleted = + range.deleted ?? before.slice(range.index, range.index + range.length) + session.deleteIndex = deleteIndex + session.deleteLength = deleteLength + session.beforeContent = before + cursorPositions.set(args.id, deletePosition) + readDocs.set(args.id, contentOf(materialized)) + return materialized + } + + const enqueueInsert = ( + toolCallId: string, + action: (session: InsertSession) => Promise + ): void => { + const session = + insertSessions.get(toolCallId) ?? + ({ + inserted: ``, + seq: 0, + streamed: false, + pending: Promise.resolve(), + } satisfies InsertSession) + insertSessions.set(toolCallId, session) + session.pending = session.pending + .then(() => action(session)) + .catch((error) => { + session.error = error + }) + } + + const awaitInsertSession = async ( + toolCallId: string + ): Promise => { + const session = insertSessions.get(toolCallId) + if (!session) return undefined + await session.pending + if (session.error) throw session.error + return session + } + + const enqueueReplace = ( + toolCallId: string, + action: (session: ReplaceSession) => Promise + ): void => { + const session = + replaceSessions.get(toolCallId) ?? + ({ + inserted: ``, + seq: 0, + streamed: false, + pending: Promise.resolve(), + } satisfies ReplaceSession) + replaceSessions.set(toolCallId, session) + session.pending = session.pending + .then(() => action(session)) + .catch((error) => { + session.error = error + }) + } + + const awaitReplaceSession = async ( + toolCallId: string + ): Promise => { + const session = replaceSessions.get(toolCallId) + if (!session) return undefined + await session.pending + if (session.error) throw session.error + return session + } + + return [ + { + name: `create_markdown_doc`, + label: `Create Markdown Doc`, + description: `Create a collaborative markdown document, persist it as Yjs updates, and add it to this entity's manifest so users can open it in the app. This is not a filesystem file.`, + parameters: Type.Object({ + title: Type.String({ description: `Document title shown in the UI.` }), + content: Type.Optional( + Type.String({ description: `Initial markdown content.` }) + ), + id: Type.Optional( + Type.String({ + description: `Optional stable document id. Use letters, numbers, hyphens, or underscores.`, + }) + ), + }), + execute: async (_toolCallId, params) => { + const { id, title, content } = params as { + id?: string + title: string + content?: string + } + const result = await context.createMarkdownDocument({ + id, + title, + }) + const materialized = await openDocumentSession(result.document) + if (content && content.length > 0) { + await appendPresence(materialized, { anchor: 0, head: 0 }) + materialized.session.doc.transact(() => { + materialized.session.text.delete( + 0, + materialized.session.text.length + ) + materialized.session.text.insert(0, content) + }, `agent`) + await appendPresence(materialized, { + anchor: content.length, + head: content.length, + }) + await materialized.session.flush() + await appendPresence(materialized, { clear: true }) + readDocs.set(result.document.id, contentOf(materialized)) + } + return { + content: [ + { + type: `text` as const, + text: `Created markdown document ${result.document.id}: ${result.document.title}`, + }, + ], + details: { document: result.document, txid: result.txid }, + } + }, + }, + { + name: `set_markdown_doc_cursor`, + label: `Set Markdown Doc Cursor`, + description: `Set the stateful insertion cursor for a collaborative markdown document. The cursor is stored as a Yjs relative position for this wake, so later insert_markdown_doc calls can stream at that position even if the document changes around it. Pass exactly one of index, before, or after; omit all three to place the cursor at the end.`, + parameters: Type.Object({ + id: Type.String({ description: `Document id.` }), + index: Type.Optional( + Type.Number({ + description: `Optional UTF-16 text offset for the cursor.`, + }) + ), + before: Type.Optional( + Type.String({ + description: `Place the cursor before this literal markdown text.`, + }) + ), + after: Type.Optional( + Type.String({ + description: `Place the cursor after this literal markdown text.`, + }) + ), + occurrence: Type.Optional( + Type.Number({ + description: `1-based occurrence for before/after matching. Defaults to 1.`, + }) + ), + }), + execute: async (_toolCallId, params) => { + const args = params as SetCursorArgs + const materialized = await materializeDocument(args.id) + const content = contentOf(materialized) + const resolved = resolveCursorIndex(content, args) + if (resolved.error || resolved.index === undefined) { + return { + content: [ + { + type: `text` as const, + text: `Error: ${resolved.error ?? `could not resolve cursor`}`, + }, + ], + details: { cursorSet: false }, + } + } + const result = await setCursor(args.id, resolved.index) + return { + content: [ + { + type: `text` as const, + text: `Set markdown document ${args.id} cursor at index ${result.index}`, + }, + ], + details: { + document: result.materialized.document, + cursorSet: true, + index: result.index, + }, + } + }, + }, + { + name: `insert_markdown_doc`, + label: `Insert Markdown Doc`, + description: `Insert markdown into a collaborative app document. When the model streams the content argument, the insertion is applied incrementally to the wake-local Yjs document and appended to the document stream so open editors can watch it appear. Put id and optional index before content in the tool arguments. If index is omitted, the current set_markdown_doc_cursor position is used; if no cursor is set, content is appended.`, + parameters: Type.Object({ + id: Type.String({ description: `Document id.` }), + index: Type.Optional( + Type.Number({ + description: `Optional UTF-16 text offset. Omit to append to the end of the current document.`, + }) + ), + content: Type.String({ description: `Markdown content to insert.` }), + }), + onArgsDelta: ({ toolCallId, argsPreview }) => { + const args = asInsertArgs(argsPreview) + if (!args.id || typeof args.content !== `string`) return + enqueueInsert(toolCallId, async (session) => { + session.id = args.id + if (session.nextIndex === undefined && args.index !== undefined) { + session.nextIndex = args.index + } + if (!args.content!.startsWith(session.inserted)) return + const chunk = args.content!.slice(session.inserted.length) + if (chunk.length === 0) return + session.inserted = args.content! + await applyInsertChunk(args.id!, chunk, session, args.index) + session.seq++ + }) + }, + execute: async (toolCallId, params) => { + const { id, content, index } = params as InsertMarkdownArgs + const session = await awaitInsertSession(toolCallId) + let inserted = session?.inserted ?? `` + let streamed = session?.streamed ?? false + let nextIndex = session?.nextIndex ?? index + + if (content !== inserted) { + if (inserted.length === 0 || content.startsWith(inserted)) { + const remaining = + inserted.length === 0 ? content : content.slice(inserted.length) + if (remaining.length > 0) { + const finalSession = + session ?? + ({ + inserted: ``, + seq: 0, + streamed: false, + pending: Promise.resolve(), + } satisfies InsertSession) + await applyInsertChunk(id, remaining, finalSession, nextIndex) + nextIndex = finalSession.nextIndex + inserted = content + streamed = streamed || remaining.length !== content.length + } + } else { + const materialized = materializedDocs.get(id) + if (materialized) { + await appendPresence(materialized, { clear: true }) + } + insertSessions.delete(toolCallId) + return { + content: [ + { + type: `text` as const, + text: `Error: streamed content diverged from final insert content; no final reconciliation was applied.`, + }, + ], + details: { inserted: inserted.length, expected: content.length }, + } + } + } + + const materialized = await materializeDocument(id) + await materialized.session.flush() + await appendPresence(materialized, { clear: true }) + const finalContent = contentOf(materialized) + readDocs.set(id, finalContent) + insertSessions.delete(toolCallId) + return { + content: [ + { + type: `text` as const, + text: `Inserted ${content.length} characters into markdown document ${id}`, + }, + ], + details: { + document: materialized.document, + streamed, + insertedBytes: new TextEncoder().encode(content).length, + nextIndex, + }, + } + }, + executionMode: `sequential`, + }, + { + name: `replace_markdown_doc_range`, + label: `Replace Markdown Doc Range`, + description: `Delete one range from a collaborative app markdown document, then stream or insert replacement markdown at that location. Use old_string for a unique literal match, old_string plus occurrence for repeated text, or index plus length for an explicit UTF-16 range. The agent cursor follows the end of streamed replacement text.`, + parameters: Type.Object({ + id: Type.String({ description: `Document id.` }), + old_string: Type.Optional( + Type.String({ + description: `Literal markdown text to replace. Must be unique unless occurrence is provided.`, + }) + ), + occurrence: Type.Optional( + Type.Number({ + description: `1-based occurrence to replace when old_string appears multiple times.`, + }) + ), + index: Type.Optional( + Type.Number({ + description: `Optional UTF-16 start offset for an explicit replacement range.`, + }) + ), + length: Type.Optional( + Type.Number({ + description: `UTF-16 length for an explicit replacement range. Required when index is used.`, + }) + ), + content: Type.String({ + description: `Replacement markdown content to stream into the deleted range.`, + }), + }), + onArgsDelta: ({ toolCallId, argsPreview }) => { + const args = asReplaceArgs(argsPreview) + if (!args.id || typeof args.content !== `string`) return + if ( + args.old_string === undefined && + (args.index === undefined || args.length === undefined) + ) { + return + } + enqueueReplace(toolCallId, async (session) => { + await prepareReplaceSession(session, { + id: args.id!, + old_string: args.old_string, + occurrence: args.occurrence, + index: args.index, + length: args.length, + }) + if (!args.content!.startsWith(session.inserted)) return + const chunk = args.content!.slice(session.inserted.length) + if (chunk.length === 0) return + session.inserted = args.content! + await applyInsertChunk(args.id!, chunk, session) + session.seq++ + }) + }, + execute: async (toolCallId, params) => { + const args = params as ReplaceMarkdownArgs + const session = + (await awaitReplaceSession(toolCallId)) ?? + ({ + inserted: ``, + seq: 0, + streamed: false, + pending: Promise.resolve(), + } satisfies ReplaceSession) + + try { + await prepareReplaceSession(session, { + id: args.id, + old_string: args.old_string, + occurrence: args.occurrence, + index: args.index, + length: args.length, + }) + } catch (error) { + replaceSessions.delete(toolCallId) + return { + content: [ + { + type: `text` as const, + text: `Error: ${error instanceof Error ? error.message : `could not prepare replacement`}`, + }, + ], + details: { replaced: false }, + } + } + + let inserted = session.inserted + let streamed = session.streamed + let nextIndex = session.nextIndex + if (args.content !== inserted) { + if (inserted.length === 0 || args.content.startsWith(inserted)) { + const remaining = + inserted.length === 0 + ? args.content + : args.content.slice(inserted.length) + if (remaining.length > 0) { + await applyInsertChunk(args.id, remaining, session) + nextIndex = session.nextIndex + inserted = args.content + streamed = streamed || remaining.length !== args.content.length + } + } else { + const materialized = materializedDocs.get(args.id) + if (materialized) { + await appendPresence(materialized, { clear: true }) + } + replaceSessions.delete(toolCallId) + return { + content: [ + { + type: `text` as const, + text: `Error: streamed replacement content diverged from final content; no final reconciliation was applied.`, + }, + ], + details: { + replaced: false, + inserted: inserted.length, + expected: args.content.length, + }, + } + } + } + + const materialized = await materializeDocument(args.id) + await materialized.session.flush() + await appendPresence(materialized, { clear: true }) + const finalContent = contentOf(materialized) + readDocs.set(args.id, finalContent) + replaceSessions.delete(toolCallId) + const diff = createTwoFilesPatch( + docLabel(args.id), + docLabel(args.id), + session.beforeContent ?? finalContent, + finalContent, + undefined, + undefined, + { context: 3 } + ) + return { + content: [ + { + type: `text` as const, + text: `Replaced ${session.deleteLength ?? 0} characters in markdown document ${args.id}`, + }, + ], + details: { + document: materialized.document, + replaced: true, + deleted: session.deleted, + deleteIndex: session.deleteIndex, + deleteLength: session.deleteLength, + streamed, + insertedBytes: new TextEncoder().encode(args.content).length, + nextIndex, + diff, + }, + } + }, + executionMode: `sequential`, + }, + { + name: `read_markdown_doc`, + label: `Read Markdown Doc`, + description: `Read the current plain markdown content from a collaborative app document, not from the filesystem.`, + parameters: Type.Object({ + id: Type.String({ description: `Document id.` }), + }), + execute: async (_toolCallId, params) => { + const { id } = params as { id: string } + const materialized = await materializeDocument(id) + const content = contentOf(materialized) + const cursorIndex = cursorPositions.has(id) + ? markdownIndexFromRelativePosition( + materialized.session.doc, + cursorPositions.get(id)!, + materialized.textName + ) + : undefined + readDocs.set(id, content) + return { + content: [ + { + type: `text` as const, + text: content, + }, + ], + details: { + document: materialized.document, + bytes: new TextEncoder().encode(content).length, + cursorIndex, + }, + } + }, + }, + { + name: `write_markdown_doc`, + label: `Write Markdown Doc`, + description: `Replace the full content of a collaborative app markdown document. This does not write a filesystem file.`, + parameters: Type.Object({ + id: Type.String({ description: `Document id.` }), + content: Type.String({ description: `Full markdown content.` }), + }), + execute: async (_toolCallId, params) => { + const { id, content } = params as { id: string; content: string } + const materialized = await materializeDocument(id) + const before = contentOf(materialized) + await appendPresence(materialized, { anchor: 0, head: 0 }) + materialized.session.doc.transact(() => { + materialized.session.text.delete(0, materialized.session.text.length) + if (content.length > 0) materialized.session.text.insert(0, content) + }, `agent`) + await appendPresence(materialized, { + anchor: content.length, + head: content.length, + }) + await materialized.session.flush() + await appendPresence(materialized, { clear: true }) + readDocs.set(id, content) + const diff = createTwoFilesPatch( + docLabel(id), + docLabel(id), + before, + content, + undefined, + undefined, + { context: 3 } + ) + return { + content: [ + { + type: `text` as const, + text: `Wrote markdown document ${id}`, + }, + ], + details: { document: materialized.document, diff }, + } + }, + executionMode: `sequential`, + }, + { + name: `edit_markdown_doc`, + label: `Edit Markdown Doc`, + description: `Replace text in a collaborative app markdown document by appending a Yjs update, not by writing a filesystem file. Read the document first when you need to inspect current content. By default old_string must occur exactly once; set replace_all to true to replace every occurrence.`, + parameters: Type.Object({ + id: Type.String({ description: `Document id.` }), + old_string: Type.String({ + description: `Literal markdown text to find. Must be unique unless replace_all is true.`, + }), + new_string: Type.String({ description: `Replacement markdown text.` }), + replace_all: Type.Optional( + Type.Boolean({ description: `Replace every occurrence.` }) + ), + }), + execute: async (_toolCallId, params) => { + const { id, old_string, new_string, replace_all } = params as { + id: string + old_string: string + new_string: string + replace_all?: boolean + } + const materialized = await materializeDocument(id) + const before = contentOf(materialized) + + const matches = before.split(old_string).length - 1 + if (matches === 0) { + return { + content: [ + { type: `text` as const, text: `Error: old_string not found` }, + ], + details: { replacements: 0 }, + } + } + if (!replace_all && matches > 1) { + return { + content: [ + { + type: `text` as const, + text: `Error: found ${matches} matches for old_string; pass replace_all=true or provide a more specific old_string.`, + }, + ], + details: { replacements: 0 }, + } + } + + const index = before.indexOf(old_string) + await appendPresence(materialized, { anchor: index, head: index }) + let cursorIndex = index + materialized.session.doc.transact(() => { + if (replace_all) { + let cursor = 0 + while (true) { + const nextIndex = materialized.session.text + .toString() + .indexOf(old_string, cursor) + if (nextIndex < 0) break + materialized.session.text.delete(nextIndex, old_string.length) + materialized.session.text.insert(nextIndex, new_string) + cursor = nextIndex + new_string.length + cursorIndex = cursor + } + } else { + materialized.session.text.delete(index, old_string.length) + materialized.session.text.insert(index, new_string) + cursorIndex = index + new_string.length + } + }, `agent`) + const resultContent = contentOf(materialized) + await appendPresence(materialized, { + anchor: cursorIndex, + head: cursorIndex, + }) + await materialized.session.flush() + await appendPresence(materialized, { clear: true }) + readDocs.set(id, resultContent) + const diff = createTwoFilesPatch( + docLabel(id), + docLabel(id), + before, + resultContent, + undefined, + undefined, + { context: 3 } + ) + return { + content: [ + { + type: `text` as const, + text: `Edited markdown document ${id}: ${matches} replacement${ + matches === 1 ? `` : `s` + }`, + }, + ], + details: { + replacements: matches, + document: materialized.document, + diff, + }, + } + }, + executionMode: `sequential`, + }, + ] +} diff --git a/packages/agents-runtime/src/types.ts b/packages/agents-runtime/src/types.ts index 00af8449e8..c3c0a783b1 100644 --- a/packages/agents-runtime/src/types.ts +++ b/packages/agents-runtime/src/types.ts @@ -41,6 +41,7 @@ import type { ManifestAttachmentEntry as EntityManifestAttachmentEntry, ManifestChildEntry as EntityManifestChildEntry, ManifestContextEntry as EntityManifestContextEntry, + ManifestDocumentEntry as EntityManifestDocumentEntry, ManifestCronScheduleEntry as EntityManifestCronScheduleEntry, ManifestEffectEntry as EntityManifestEffectEntry, ManifestFutureSendScheduleEntry as EntityManifestFutureSendScheduleEntry, @@ -81,6 +82,12 @@ export type EntitiesObservationHandle = ObservationHandle & { db: ObservationStreamDB } +export type MarkdownDocumentConnection = { + baseUrl: string + docId: string + headers?: Record +} + export type JsonValue = | string | number @@ -319,6 +326,7 @@ export type ManifestEntry = EntityManifest export type ManifestAttachmentEntry = EntityManifestAttachmentEntry export type ManifestChildEntry = EntityManifestChildEntry export type ManifestContextEntry = EntityManifestContextEntry +export type ManifestDocumentEntry = EntityManifestDocumentEntry export type ManifestCronScheduleEntry = EntityManifestCronScheduleEntry export type ManifestEffectEntry = EntityManifestEffectEntry export type ManifestFutureSendScheduleEntry = @@ -760,6 +768,7 @@ export interface ProcessWakeConfig { createElectricTools?: (context: { entityUrl: string entityType: string + principal?: RuntimePrincipal args: Readonly> db: EntityStreamDBWithActions events: Array @@ -786,6 +795,27 @@ export interface ProcessWakeConfig { unsubscribeFromWebhookSource: (opts: { id: string }) => Promise<{ txid: string }> + createMarkdownDocument: (opts: { + id?: string + title: string + meta?: Record + }) => Promise<{ txid: string; document: ManifestDocumentEntry }> + getMarkdownDocumentConnection: ( + streamPath: string + ) => Promise + readMarkdownDocumentStream: ( + streamPath: string, + opts?: { offset?: string } + ) => Promise<{ bytes: Uint8Array; offset?: string }> + appendMarkdownDocumentUpdate: ( + streamPath: string, + update: Uint8Array + ) => Promise<{ offset?: string }> + appendMarkdownDocumentAwareness: ( + streamPath: string, + update: Uint8Array + ) => Promise<{ offset?: string }> + registerCleanup: (cleanup: () => void | Promise) => void }) => Array | Promise> /** Optional shutdown signal to end idle waits during host teardown. */ shutdownSignal?: AbortSignal diff --git a/packages/agents-runtime/test/markdown-docs-tools.test.ts b/packages/agents-runtime/test/markdown-docs-tools.test.ts new file mode 100644 index 0000000000..3d244e02f8 --- /dev/null +++ b/packages/agents-runtime/test/markdown-docs-tools.test.ts @@ -0,0 +1,573 @@ +import { describe, expect, it, vi } from 'vitest' +import * as decoding from 'lib0/decoding' +import { Awareness, applyAwarenessUpdate } from 'y-protocols/awareness' +import * as Y from 'yjs' +import { + createMarkdownYDoc, + encodeMarkdownAwarenessUpdate, + frameYjsUpdate, + markdownText, +} from '../src/markdown-yjs' +import { createMarkdownDocumentTools } from '../src/tools/markdown-docs' + +function concatBytes(a: Uint8Array, b: Uint8Array): Uint8Array { + const next = new Uint8Array(a.length + b.length) + next.set(a, 0) + next.set(b, a.length) + return next +} + +function concatFrames(frames: Array): Uint8Array { + return frames.reduce( + (bytes, frame) => concatBytes(bytes, frame), + new Uint8Array() + ) +} + +function streamBytesFromContent(content: string): Uint8Array { + const doc = new Y.Doc() + markdownText(doc).insert(0, content) + return frameYjsUpdate(Y.encodeStateAsUpdate(doc)) +} + +function contentFromStream(streamBytes: Uint8Array): string { + return markdownText(createMarkdownYDoc(streamBytes)).toString() +} + +async function waitForCondition( + predicate: () => boolean, + message: string +): Promise { + for (let i = 0; i < 20; i += 1) { + if (predicate()) return + await new Promise((resolve) => setTimeout(resolve, 0)) + } + throw new Error(message) +} + +function applyFramedAwarenessUpdate( + awareness: Awareness, + data: Uint8Array +): void { + const decoder = decoding.createDecoder(data) + while (decoding.hasContent(decoder)) { + applyAwarenessUpdate(awareness, decoding.readVarUint8Array(decoder), `test`) + } +} + +function cursorHeadIndexFromAwarenessFrame( + doc: Y.Doc, + frame: Uint8Array +): number | undefined { + const awareness = new Awareness(new Y.Doc()) + applyFramedAwarenessUpdate(awareness, frame) + for (const state of awareness.getStates().values()) { + const cursor = ( + state as { + cursor?: { head?: Y.RelativePosition; anchor?: Y.RelativePosition } + } + ).cursor + if (!cursor?.head) continue + const absolute = Y.createAbsolutePositionFromRelativePosition( + cursor.head, + doc + ) + return absolute?.index + } + return undefined +} + +function createToolContext( + opts: { + manifestDocuments?: Array + markdownDocs?: Array + entityUrl?: string + principalUrl?: string + } = {} +) { + const document = { + key: `document:notes`, + kind: `document`, + id: `notes`, + provider: `y-durable-streams`, + docId: `agents/chat/session/documents/notes`, + docPath: `agents/chat/session/documents/notes`, + streamPath: `/v1/yjs/default/docs/agents/chat/session/documents/notes`, + transportMimeType: `application/vnd.electric-agents.markdown-yjs`, + contentMimeType: `text/markdown`, + yTextName: `markdown`, + title: `Notes`, + createdAt: `2026-06-07T00:00:00.000Z`, + } as const + let streamFrames = [streamBytesFromContent(`# Notes\n\nFirst line\n`)] + const awarenessFrames: Array = [] + const openSessions: Array<{ + doc: Y.Doc + off: () => void + }> = [] + const cleanupCallbacks: Array<() => void | Promise> = [] + const context: any = { + entityUrl: opts.entityUrl ?? `/chat/session`, + entityType: `chat`, + principal: { + url: opts.principalUrl ?? `/principal/agent:horton`, + kind: `agent`, + }, + args: { + ...(opts.markdownDocs ? { markdownDocs: opts.markdownDocs } : {}), + }, + db: { + collections: { + manifests: { toArray: opts.manifestDocuments ?? [document] }, + }, + }, + events: [], + createMarkdownDocument: vi.fn( + async (opts: { id?: string; title: string }) => { + streamFrames = [] + return { + txid: `tx-create`, + document: { + ...document, + id: opts.id ?? document.id, + title: opts.title, + }, + } + } + ), + getMarkdownDocumentConnection: vi.fn(async () => ({ + baseUrl: `http://test.local/v1/yjs/default`, + docId: document.docId, + headers: {}, + })), + openMarkdownDocumentSession: vi.fn( + async ({ + document, + entityUrl, + principal, + }: { + document: any + entityUrl: string + principal?: { url?: string } + }) => { + const ydoc = createMarkdownYDoc(concatFrames(streamFrames)) + const text = markdownText(ydoc, document.yTextName) + const onUpdate = (update: Uint8Array, origin: unknown): void => { + if (origin === `server`) return + void context.appendMarkdownDocumentUpdate( + document.streamPath, + frameYjsUpdate(update) + ) + } + ydoc.on(`update`, onUpdate) + openSessions.push({ + doc: ydoc, + off: () => ydoc.off(`update`, onUpdate), + }) + const principalUrl = + principal?.url ?? `/principal/entity:${encodeURIComponent(entityUrl)}` + return { + document, + doc: ydoc, + text, + textName: document.yTextName, + content: () => text.toString(), + setPresence: vi.fn( + async (presence: { + anchor?: number + head?: number + clear?: boolean + }) => { + void context.appendMarkdownDocumentAwareness( + document.streamPath, + encodeMarkdownAwarenessUpdate({ + doc: ydoc, + docPath: document.docPath, + principalUrl, + clientKey: `${principalUrl}\0${entityUrl}`, + name: principalUrl, + role: `agent`, + anchor: presence.anchor, + head: presence.head, + clear: presence.clear, + color: `#000000`, + colorLight: `#00000033`, + textName: document.yTextName, + }) + ) + } + ), + flush: vi.fn(async () => {}), + close: vi.fn(async () => { + ydoc.off(`update`, onUpdate) + ydoc.destroy() + }), + } + } + ), + readMarkdownDocumentStream: vi.fn( + async (_streamPath: string, opts?: { offset?: string }) => { + const offset = + opts?.offset !== undefined ? Number.parseInt(opts.offset, 10) : 0 + const start = Number.isFinite(offset) && offset >= 0 ? offset : 0 + return { + bytes: concatFrames(streamFrames.slice(start)), + offset: String(streamFrames.length), + } + } + ), + appendMarkdownDocumentUpdate: vi.fn( + async (_streamPath: string, update: Uint8Array) => { + streamFrames.push(update) + return { offset: String(streamFrames.length) } + } + ), + appendMarkdownDocumentAwareness: vi.fn( + async (_streamPath: string, update: Uint8Array) => { + awarenessFrames.push(update) + return {} + } + ), + registerCleanup: vi.fn((cleanup: () => void | Promise) => { + cleanupCallbacks.push(cleanup) + }), + upsertCronSchedule: vi.fn(), + upsertFutureSendSchedule: vi.fn(), + deleteSchedule: vi.fn(), + listEventSources: vi.fn(), + subscribeToEventSource: vi.fn(), + unsubscribeFromEventSource: vi.fn(), + } + return { + context, + getContent: () => contentFromStream(concatFrames(streamFrames)), + getDoc: () => createMarkdownYDoc(concatFrames(streamFrames)), + getAwarenessFrames: () => awarenessFrames, + appendExternalText: (text: string) => { + const streamBytes = concatFrames(streamFrames) + const doc = createMarkdownYDoc(streamBytes) + const yText = markdownText(doc) + const before = Y.encodeStateVector(doc) + yText.insert(yText.length, text) + const update = Y.encodeStateAsUpdate(doc, before) + streamFrames.push(frameYjsUpdate(update)) + for (const session of openSessions) { + Y.applyUpdate(session.doc, update, `server`) + } + doc.destroy() + }, + cleanup: async () => { + for (const cleanup of cleanupCallbacks) await cleanup() + for (const session of openSessions) session.off() + }, + document, + } +} + +describe(`markdown document tools`, () => { + it(`uses the optional awareness client key to distinguish same-principal editors`, () => { + const doc = new Y.Doc() + markdownText(doc).insert(0, `hello`) + const awareness = new Awareness(new Y.Doc()) + + applyFramedAwarenessUpdate( + awareness, + encodeMarkdownAwarenessUpdate({ + doc, + docPath: `agents/chat/session/documents/notes`, + principalUrl: `/principal/agent:horton`, + clientKey: `/principal/agent:horton\0/chat/session`, + name: `horton`, + role: `agent`, + color: `#000000`, + colorLight: `#00000033`, + }) + ) + applyFramedAwarenessUpdate( + awareness, + encodeMarkdownAwarenessUpdate({ + doc, + docPath: `agents/chat/session/documents/notes`, + principalUrl: `/principal/agent:horton`, + clientKey: `/principal/agent:horton\0/worker/one`, + name: `worker`, + role: `agent`, + color: `#111111`, + colorLight: `#11111133`, + }) + ) + + const remoteStates = Array.from(awareness.getStates()).filter( + ([clientId]) => clientId !== awareness.clientID + ) + expect(remoteStates).toHaveLength(2) + }) + + it(`creates the server document empty and appends initial content as a Yjs update`, async () => { + const { context, getContent } = createToolContext() + const create = createMarkdownDocumentTools(context).find( + (tool) => tool.name === `create_markdown_doc` + )! + + await create.execute(`tool-create`, { + id: `notes`, + title: `Notes`, + content: `# Created\n\nInitial content`, + }) + + expect(context.createMarkdownDocument).toHaveBeenCalledWith({ + id: `notes`, + title: `Notes`, + }) + expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(1) + expect(getContent()).toBe(`# Created\n\nInitial content`) + }) + + it(`materializes and edits markdown documents through Yjs stream updates`, async () => { + const { context, getContent } = createToolContext() + const edit = createMarkdownDocumentTools(context).find( + (tool) => tool.name === `edit_markdown_doc` + )! + + const result = await edit.execute(`tool-edit`, { + id: `notes`, + old_string: `First`, + new_string: `Second`, + }) + + expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(1) + expect(context.appendMarkdownDocumentAwareness).toHaveBeenCalled() + expect(getContent()).toContain(`Second line`) + expect(result.details).toMatchObject({ replacements: 1 }) + }) + + it(`reads injected markdown document refs without a local manifest entry`, async () => { + const base = createToolContext() + const { context } = createToolContext({ + manifestDocuments: [], + markdownDocs: [base.document], + entityUrl: `/worker/subagent`, + }) + const read = createMarkdownDocumentTools(context).find( + (tool) => tool.name === `read_markdown_doc` + )! + + const result = await read.execute(`tool-read-injected`, { id: `notes` }) + + expect(context.openMarkdownDocumentSession).toHaveBeenCalledWith( + expect.objectContaining({ + document: base.document, + entityUrl: `/worker/subagent`, + }) + ) + expect((result.content[0] as { text: string }).text).toContain(`# Notes`) + }) + + it(`edits a read document and returns a diff`, async () => { + const { context, getContent } = createToolContext() + const tools = createMarkdownDocumentTools(context) + const read = tools.find((tool) => tool.name === `read_markdown_doc`)! + const edit = tools.find((tool) => tool.name === `edit_markdown_doc`)! + + await read.execute(`tool-read`, { id: `notes` }) + const result = await edit.execute(`tool-edit`, { + id: `notes`, + old_string: `First line`, + new_string: `Second line`, + }) + + expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(1) + expect(getContent()).toContain(`Second line`) + expect(result.details).toMatchObject({ replacements: 1 }) + expect(String((result.details as any).diff)).toContain(`Second line`) + }) + + it(`streams insert_markdown_doc content deltas before final execution`, async () => { + const { context, getContent, getDoc, getAwarenessFrames } = + createToolContext() + const insert = createMarkdownDocumentTools(context).find( + (tool) => tool.name === `insert_markdown_doc` + )! + + await insert.onArgsDelta?.({ + toolCallId: `tool-insert`, + toolName: `insert_markdown_doc`, + delta: `"Hello`, + argsPreview: { id: `notes`, content: `Hello` }, + }) + await waitForCondition( + () => context.appendMarkdownDocumentAwareness.mock.calls.length === 1, + `expected first streamed insert presence update` + ) + expect(context.appendMarkdownDocumentAwareness).toHaveBeenCalledTimes(1) + expect( + cursorHeadIndexFromAwarenessFrame(getDoc(), getAwarenessFrames().at(-1)!) + ).toBe(getContent().length) + + await insert.onArgsDelta?.({ + toolCallId: `tool-insert`, + toolName: `insert_markdown_doc`, + delta: ` world"`, + argsPreview: { id: `notes`, content: `Hello world` }, + }) + await waitForCondition( + () => context.appendMarkdownDocumentAwareness.mock.calls.length === 2, + `expected second streamed insert presence update` + ) + expect(context.appendMarkdownDocumentAwareness).toHaveBeenCalledTimes(2) + expect( + cursorHeadIndexFromAwarenessFrame(getDoc(), getAwarenessFrames().at(-1)!) + ).toBe(getContent().length) + + const result = await insert.execute(`tool-insert`, { + id: `notes`, + content: `Hello world`, + }) + + expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(2) + expect(context.appendMarkdownDocumentAwareness).toHaveBeenCalledTimes(3) + expect(getContent()).toContain(`Hello world`) + expect(result.details).toMatchObject({ streamed: true }) + }) + + it(`streams insert_markdown_doc at a saved Yjs-relative cursor`, async () => { + const { context, getContent } = createToolContext() + const tools = createMarkdownDocumentTools(context) + const setCursor = tools.find( + (tool) => tool.name === `set_markdown_doc_cursor` + )! + const insert = tools.find((tool) => tool.name === `insert_markdown_doc`)! + + const cursorResult = await setCursor.execute(`tool-cursor`, { + id: `notes`, + after: `# Notes\n`, + }) + expect(cursorResult.details).toMatchObject({ + cursorSet: true, + index: `# Notes\n`.length, + }) + + await insert.onArgsDelta?.({ + toolCallId: `tool-insert-cursor`, + toolName: `insert_markdown_doc`, + delta: `"Inserted`, + argsPreview: { id: `notes`, content: `Inserted` }, + }) + await insert.onArgsDelta?.({ + toolCallId: `tool-insert-cursor`, + toolName: `insert_markdown_doc`, + delta: ` text\n"`, + argsPreview: { id: `notes`, content: `Inserted text\n` }, + }) + + await insert.execute(`tool-insert-cursor`, { + id: `notes`, + content: `Inserted text\n`, + }) + + expect(getContent()).toBe(`# Notes\nInserted text\n\nFirst line\n`) + expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(2) + }) + + it(`replaces a markdown range with one delete update and one insert update`, async () => { + const { context, getContent } = createToolContext() + const replace = createMarkdownDocumentTools(context).find( + (tool) => tool.name === `replace_markdown_doc_range` + )! + + const result = await replace.execute(`tool-replace`, { + id: `notes`, + old_string: `First line`, + content: `Replacement line`, + }) + + expect(getContent()).toBe(`# Notes\n\nReplacement line\n`) + expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(2) + expect(context.appendMarkdownDocumentAwareness).toHaveBeenCalledTimes(4) + expect(result.details).toMatchObject({ + replaced: true, + deleted: `First line`, + streamed: false, + }) + }) + + it(`streams replace_markdown_doc_range replacement content at the deleted range`, async () => { + const { context, getContent, getDoc, getAwarenessFrames } = + createToolContext() + const replace = createMarkdownDocumentTools(context).find( + (tool) => tool.name === `replace_markdown_doc_range` + )! + + await replace.onArgsDelta?.({ + toolCallId: `tool-stream-replace`, + toolName: `replace_markdown_doc_range`, + delta: `"Replacement`, + argsPreview: { + id: `notes`, + old_string: `First line`, + content: `Replacement`, + }, + }) + await waitForCondition( + () => context.appendMarkdownDocumentAwareness.mock.calls.length === 3, + `expected replacement delete and first streamed insert presence updates` + ) + expect(getContent()).toBe(`# Notes\n\nReplacement\n`) + expect( + cursorHeadIndexFromAwarenessFrame(getDoc(), getAwarenessFrames().at(-1)!) + ).toBe(getContent().length - 1) + + await replace.onArgsDelta?.({ + toolCallId: `tool-stream-replace`, + toolName: `replace_markdown_doc_range`, + delta: ` line"`, + argsPreview: { + id: `notes`, + old_string: `First line`, + content: `Replacement line`, + }, + }) + await waitForCondition( + () => context.appendMarkdownDocumentAwareness.mock.calls.length === 4, + `expected second streamed replacement presence update` + ) + expect(getContent()).toBe(`# Notes\n\nReplacement line\n`) + expect( + cursorHeadIndexFromAwarenessFrame(getDoc(), getAwarenessFrames().at(-1)!) + ).toBe(getContent().length - 1) + + const result = await replace.execute(`tool-stream-replace`, { + id: `notes`, + old_string: `First line`, + content: `Replacement line`, + }) + + expect(context.appendMarkdownDocumentUpdate).toHaveBeenCalledTimes(3) + expect(context.appendMarkdownDocumentAwareness).toHaveBeenCalledTimes(5) + expect(result.details).toMatchObject({ + replaced: true, + streamed: true, + deleted: `First line`, + }) + }) + + it(`refreshes a cached Yjs document from the stream before editing`, async () => { + const { context, getContent, appendExternalText } = createToolContext() + const tools = createMarkdownDocumentTools(context) + const read = tools.find((tool) => tool.name === `read_markdown_doc`)! + const edit = tools.find((tool) => tool.name === `edit_markdown_doc`)! + + await read.execute(`tool-read`, { id: `notes` }) + appendExternalText(`External line\n`) + + await edit.execute(`tool-edit`, { + id: `notes`, + old_string: `External line`, + new_string: `Refreshed line`, + }) + + expect(getContent()).toContain(`Refreshed line`) + expect(context.readMarkdownDocumentStream).not.toHaveBeenCalled() + expect(context.openMarkdownDocumentSession).toHaveBeenCalledTimes(1) + }) +}) diff --git a/packages/agents-server-ui/package.json b/packages/agents-server-ui/package.json index 170fbb2854..78fe977656 100644 --- a/packages/agents-server-ui/package.json +++ b/packages/agents-server-ui/package.json @@ -15,8 +15,12 @@ }, "dependencies": { "@base-ui/react": "^1.4.1", + "@codemirror/lang-markdown": "^6.5.0", + "@codemirror/state": "^6.6.0", + "@codemirror/view": "^6.43.0", "@durable-streams/client": "^0.2.6", "@durable-streams/state": "^0.3.1", + "@durable-streams/y-durable-streams": "0.2.7", "@electric-ax/agents-runtime": "workspace:*", "@handlewithcare/react-prosemirror": "^3.0.6", "@streamdown/math": "^1.0.2", @@ -26,8 +30,10 @@ "@tanstack/react-router": "^1.167.4", "@tanstack/react-table": "^8.21.3", "@tanstack/react-virtual": "^3.13.23", + "codemirror": "^6.0.1", "fractional-indexing": "^3.2.0", "katex": "^0.16.45", + "lib0": "^0.2.99", "lucide-react": "^0.561.0", "mermaid": "^11.14.0", "nanoid": "^3.3.11", @@ -41,6 +47,9 @@ "react-reconciler": "0.32.0", "shiki": "^4.0.2", "streamdown": "^2.5.0", + "y-codemirror.next": "0.3.5", + "y-protocols": "^1.0.6", + "yjs": "^13.6.26", "zod": "^3.25.76" }, "devDependencies": { diff --git a/packages/agents-server-ui/src/components/EntityContextDrawer.tsx b/packages/agents-server-ui/src/components/EntityContextDrawer.tsx index 292b147c89..22c7b4bb13 100644 --- a/packages/agents-server-ui/src/components/EntityContextDrawer.tsx +++ b/packages/agents-server-ui/src/components/EntityContextDrawer.tsx @@ -50,6 +50,7 @@ type DrawerEntry = action: | { kind: `entity`; url: string } | { kind: `state`; sourceId: string } + | { kind: `document`; id: string } | { kind: `inspect` } entity: DrawerEntity | null } @@ -218,11 +219,21 @@ export function EntityContextDrawer({ }) } + const openDocument = (documentId: string, side = false): void => { + helpers.openEntity(entity.url, { + viewId: `markdown-doc`, + viewParams: { doc: documentId }, + ...(side ? { target: { tileId, position: `split-right` as const } } : {}), + }) + } + const handleEntry = (entry: DrawerEntry): void => { if (entry.action.kind === `entity`) { openEntity(entry.action.url) } else if (entry.action.kind === `state`) { openStateInspector(entry.action.sourceId) + } else if (entry.action.kind === `document`) { + openDocument(entry.action.id) } else { setInspectTarget({ title: entry.title, value: entry.manifest }) } @@ -233,6 +244,8 @@ export function EntityContextDrawer({ openEntity(entry.action.url, true) } else if (entry.action.kind === `state`) { openStateInspector(entry.action.sourceId, true) + } else if (entry.action.kind === `document`) { + openDocument(entry.action.id, true) } } @@ -565,6 +578,8 @@ function manifestKindLabel(manifest: Manifest): string { return `Effect` case `attachment`: return `Attachment` + case `document`: + return `Markdown document` case `context`: return `Context` case `schedule`: @@ -684,6 +699,18 @@ function createManifestEntry( entity: null, } + case `document`: + return { + key: manifest.key, + groupKey: `document`, + groupLabel: `Documents`, + title: manifest.title, + meta: manifest.id, + manifest, + action: { kind: `document`, id: manifest.id }, + entity: null, + } + case `context`: return { key: manifest.key, diff --git a/packages/agents-server-ui/src/components/EntityTimeline.tsx b/packages/agents-server-ui/src/components/EntityTimeline.tsx index 697ff709c5..1d5c33aa81 100644 --- a/packages/agents-server-ui/src/components/EntityTimeline.tsx +++ b/packages/agents-server-ui/src/components/EntityTimeline.tsx @@ -21,9 +21,11 @@ import { Database, ExternalLink, FileJson, + FileText, GitBranch, Radio, Reply, + SplitSquareHorizontal, } from 'lucide-react' import { loadTimelineRowHeights, @@ -815,6 +817,7 @@ function isTimelineFindMatch( function ManifestTimelineRow({ manifest, entityUrl, + tileId, entityStatus, onReply, }: { @@ -828,6 +831,8 @@ function ManifestTimelineRow({ const navigate = useNavigate() const entityTarget = getManifestEntityUrl(manifest) const stateSourceId = getManifestStateSourceId(manifest) + const documentId = manifest.kind === `document` ? manifest.id : null + const splitTargetTileId = tileId ?? workspace?.helpers.activeTileId ?? null const isEntity = entityTarget !== null const title = manifestTitle(manifest) const meta = manifestMeta(manifest) @@ -856,13 +861,62 @@ function ManifestTimelineRow({ }) }, [entityUrl, stateSourceId, workspace]) + const openDocument = useCallback(() => { + if (!entityUrl || !documentId || !workspace) return + workspace.helpers.openEntity(entityUrl, { + viewId: `markdown-doc`, + viewParams: { doc: documentId }, + }) + }, [documentId, entityUrl, workspace]) + + const splitDocumentRight = useCallback(() => { + if (!entityUrl || !documentId || !workspace) return + if (!splitTargetTileId) return + workspace.helpers.openEntity(entityUrl, { + viewId: `markdown-doc`, + viewParams: { doc: documentId }, + target: { tileId: splitTargetTileId, position: `split-right` }, + }) + }, [documentId, entityUrl, splitTargetTileId, workspace]) + const statusBadge = entityStatus ? ( {entityStatus} ) : null - const openAction = stateSourceId ? ( + const openAction = documentId ? ( + <> + + + + + + + + + + + + ) : stateSourceId ? ( - {isEntity || stateSourceId ? ( + {isEntity || stateSourceId || documentId ? ( details ) : ( <> @@ -977,6 +1031,8 @@ function manifestKindLabel(manifest: Manifest): string { return `Effect` case `attachment`: return `Attachment` + case `document`: + return `Markdown document` case `context`: return `Context` case `schedule`: @@ -995,6 +1051,7 @@ function manifestTitle(manifest: Manifest): string { case `shared-state`: case `effect`: case `attachment`: + case `document`: case `context`: case `schedule`: case `goal`: @@ -1014,6 +1071,8 @@ function manifestMeta(manifest: Manifest): string { return manifest.function_ref case `attachment`: return `${manifest.mimeType} · ${manifest.status}` + case `document`: + return manifest.title case `context`: return `${Object.keys(manifest.attrs).length} attrs` case `schedule`: @@ -1064,6 +1123,16 @@ function manifestDetails( value: `${manifest.subject.type}:${manifest.subject.key}`, }, ] + case `document`: + return [ + { label: `Title`, value: manifest.title }, + { label: `MIME`, value: manifest.contentMimeType }, + { label: `Transport`, value: manifest.transportMimeType }, + { label: `Provider`, value: manifest.provider }, + { label: `Y.Text`, value: manifest.yTextName }, + { label: `Doc ID`, value: manifest.docId }, + { label: `Path`, value: manifest.docPath }, + ] case `context`: return [ { label: `Name`, value: manifest.name }, @@ -1099,6 +1168,7 @@ function manifestIcon(manifest: Manifest) { if (getManifestStateSourceId(manifest)) return Database if (getManifestEntityUrl(manifest)) return GitBranch if (manifest.kind === `schedule`) return Radio + if (manifest.kind === `document`) return FileText if (manifest.kind === `attachment`) return FileJson return FileJson } diff --git a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.module.css b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.module.css new file mode 100644 index 0000000000..defc4a595b --- /dev/null +++ b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.module.css @@ -0,0 +1,297 @@ +.root { + --markdown-doc-editor-bg: #fff; + + display: flex; + min-height: 0; + height: 100%; + flex-direction: column; + background: var(--ds-bg); + color: var(--ds-text-1); + font-family: var(--ds-font-body); +} + +.bar { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--ds-space-3); + min-height: 36px; + box-sizing: border-box; + padding: 6px var(--ds-space-3); + border-top: 1px solid var(--ds-divider); + border-bottom: 1px solid var(--ds-divider); + background: var(--ds-surface); +} + +:global(:root[data-theme='dark']) .root { + --markdown-doc-editor-bg: var(--ds-bg); +} + +.title { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: var(--ds-text-1); + font-family: var(--ds-font-heading); + font-size: var(--ds-text-sm); + line-height: var(--ds-text-sm-lh); + font-weight: 600; +} + +.status { + flex: 0 0 auto; + color: var(--ds-text-3); + font-size: var(--ds-text-xs); + line-height: var(--ds-text-xs-lh); +} + +.connectionStatus { + display: inline-flex; + align-items: center; + justify-content: center; + width: var(--ds-icon-sm); + height: var(--ds-icon-sm); + flex: 0 0 auto; + color: var(--ds-text-4); +} + +.connectionStatus[data-status='connected'] { + color: var(--ds-green-11); +} + +.connectionStatus[data-status='connecting'], +.connectionStatus[data-status='loading'] { + color: var(--ds-accent-11); +} + +.connectionStatus[data-status='disconnected'] { + color: var(--ds-text-4); +} + +.connectionStatus[data-status='error'] { + color: var(--ds-red-11); +} + +.editorScrollArea { + min-height: 0; + flex: 1; + overflow: hidden; + background: var(--markdown-doc-editor-bg); +} + +.editorViewport { + background: var(--markdown-doc-editor-bg); +} + +.editor { + min-height: 100%; + background: var(--markdown-doc-editor-bg); +} + +.editor :global(.cm-editor) { + min-height: 100%; + background: var(--markdown-doc-editor-bg); + color: var(--ds-text-1); + font-family: var(--ds-font-mono); + font-size: 13px; + line-height: 1.6; +} + +.editor :global(.cm-editor.cm-focused) { + outline: none; +} + +.editor :global(.cm-scroller) { + font-family: var(--ds-font-mono); + background: var(--markdown-doc-editor-bg); + overflow: visible; +} + +.editor :global(.cm-content) { + min-height: 100%; + padding: 8px 0 36px; + caret-color: var(--ds-accent-11); +} + +.editor :global(.cm-line) { + box-sizing: border-box; + padding: 0 8px; +} + +.editor :global(.cm-cursor) { + border-left-color: var(--ds-accent-11); +} + +.editor :global(.cm-selectionBackground), +.editor :global(.cm-focused .cm-selectionBackground) { + background: var(--ds-accent-a4) !important; +} + +.editor :global(.cm-activeLine) { + background: var(--ds-gray-a2); +} + +.editor :global(.cm-gutters) { + background: var(--ds-bg-subtle); + color: var(--ds-text-4); + border-right: 1px solid var(--ds-divider); + font-family: var(--ds-font-mono); + font-size: var(--ds-text-sm); +} + +.editor :global(.cm-activeLineGutter) { + background: var(--ds-gray-a2); + color: var(--ds-text-2); +} + +.editor :global(.cm-lineNumbers .cm-gutterElement) { + min-width: 24px; + padding: 0 6px; +} + +.editor :global(.cm-foldGutter .cm-gutterElement) { + width: 10px; + padding: 0 2px; + color: var(--ds-text-4); +} + +.editor :global(.cm-matchingBracket), +.editor :global(.cm-nonmatchingBracket) { + background: var(--ds-accent-a3); + color: var(--ds-text-1); +} + +.editor :global(.cm-tooltip), +.editor :global(.cm-tooltip-autocomplete) { + overflow: hidden; + border: 1px solid var(--ds-overlay-border); + border-radius: var(--ds-radius-3); + background: var(--ds-surface-raised); + color: var(--ds-text-1); + box-shadow: var(--ds-overlay-shadow); + font-family: var(--ds-font-body); + font-size: var(--ds-text-sm); +} + +.editor :global(.cm-tooltip-autocomplete ul li[aria-selected]) { + background: var(--ds-bg-hover); + color: var(--ds-text-1); +} + +.editor :global(.cm-panels) { + border-color: var(--ds-divider); + background: var(--ds-surface); + color: var(--ds-text-1); + font-family: var(--ds-font-body); + font-size: var(--ds-text-sm); +} + +.editor :global(.cm-panels-top) { + border-bottom: 1px solid var(--ds-divider); +} + +.editor :global(.cm-panels-bottom) { + border-top: 1px solid var(--ds-divider); +} + +.editor :global(.cm-search) { + display: flex; + align-items: center; + gap: var(--ds-space-2); + padding: 6px var(--ds-space-3); +} + +.editor :global(.cm-search label) { + display: inline-flex; + align-items: center; + gap: 4px; + color: var(--ds-text-2); +} + +.editor :global(.cm-search input) { + min-height: 24px; + box-sizing: border-box; + border: 1px solid var(--ds-border-1); + border-radius: var(--ds-radius-2); + background: var(--ds-input-bg); + color: var(--ds-text-1); + font-family: var(--ds-font-body); + font-size: var(--ds-text-sm); + outline: none; + padding: 2px 8px; +} + +.editor :global(.cm-search input:focus) { + border-color: var(--ds-accent-9); + box-shadow: 0 0 0 1px var(--ds-accent-9); +} + +.editor :global(.cm-search button) { + min-height: 24px; + border: 1px solid transparent; + border-radius: var(--ds-radius-2); + background: transparent; + color: var(--ds-text-2); + cursor: pointer; + font-family: var(--ds-font-body); + font-size: var(--ds-text-sm); + font-weight: 500; + padding: 2px 8px; +} + +.editor :global(.cm-search button:hover) { + background: var(--ds-bg-hover); + color: var(--ds-text-1); +} + +.presence { + display: flex; + align-items: center; + gap: var(--ds-space-2); + min-width: 0; +} + +.presenceDot { + width: 8px; + height: 8px; + flex: 0 0 auto; + border-radius: 999px; + box-shadow: 0 0 0 1px var(--ds-surface); +} + +.empty { + padding: var(--ds-space-5); + color: var(--ds-text-3); + font-family: var(--ds-font-body); + font-size: var(--ds-text-sm); + line-height: var(--ds-text-sm-lh); +} + +:global(.yRemoteSelection) { + opacity: 0.24; +} + +:global(.yRemoteSelectionHead) { + position: absolute; + box-sizing: border-box; + height: 1.2em; + border-left: 2px solid; + opacity: 0.95; +} + +:global(.yRemoteSelectionHead)::after { + position: absolute; + top: -1.05em; + left: -2px; + z-index: 10; + padding: 1px 4px; + border-radius: var(--ds-radius-1); + color: white; + content: attr(data-user-name); + font-family: var(--ds-font-body); + font-size: var(--ds-text-2xs); + line-height: 1.2; + white-space: nowrap; + box-shadow: var(--ds-shadow-1); +} diff --git a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.test.ts b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.test.ts new file mode 100644 index 0000000000..7746958de6 --- /dev/null +++ b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest' +import * as encoding from 'lib0/encoding' +import { + Awareness, + encodeAwarenessUpdate, + type Awareness as AwarenessType, +} from 'y-protocols/awareness' +import * as Y from 'yjs' +import { + applyMarkdownAwarenessFrames, + markdownDocumentConnectionConfig, +} from './MarkdownDocumentView' +import type { ManifestDocumentEntry } from '@electric-ax/agents-runtime/client' + +function frame(update: Uint8Array): Uint8Array { + const encoder = encoding.createEncoder() + encoding.writeVarUint8Array(encoder, update) + return encoding.toUint8Array(encoder) +} + +describe(`markdownDocumentConnectionConfig`, () => { + it(`uses explicit provider doc metadata for editor connections`, () => { + const config = markdownDocumentConnectionConfig( + `http://localhost:4437/app`, + { + key: `document:notes`, + kind: `document`, + id: `notes`, + provider: `y-durable-streams`, + docId: `agents/chat/session/documents/notes`, + docPath: `agents/chat/session/documents/notes`, + streamPath: `/v1/yjs/default/docs/agents/chat/session/documents/notes`, + transportMimeType: `application/vnd.electric-agents.markdown-yjs`, + contentMimeType: `text/markdown`, + yTextName: `markdown`, + title: `Notes`, + createdAt: `2026-01-01T00:00:00.000Z`, + } as ManifestDocumentEntry + ) + + expect(config).toMatchObject({ + providerUrl: `http://localhost:4437/app/v1/yjs/default`, + docId: `agents/chat/session/documents/notes`, + yTextName: `markdown`, + }) + expect(config.docUrl.toString()).toBe( + `http://localhost:4437/app/v1/yjs/default/docs/agents/chat/session/documents/notes` + ) + }) +}) + +describe(`applyMarkdownAwarenessFrames`, () => { + it(`applies lib0-framed awareness updates`, () => { + const sourceDoc = new Y.Doc() + const source = new Awareness(sourceDoc) + source.setLocalState({ + user: { name: `horton`, role: `agent`, status: `editing` }, + cursor: { anchor: 4, head: 4 }, + }) + + const target = new Awareness(new Y.Doc()) as AwarenessType + applyMarkdownAwarenessFrames( + target, + frame(encodeAwarenessUpdate(source, [source.clientID])) + ) + + const remoteState = target.getStates().get(source.clientID) + expect(remoteState).toMatchObject({ + user: { name: `horton`, role: `agent`, status: `editing` }, + cursor: { anchor: 4, head: 4 }, + }) + }) +}) diff --git a/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx new file mode 100644 index 0000000000..75cb31c391 --- /dev/null +++ b/packages/agents-server-ui/src/components/views/MarkdownDocumentView.tsx @@ -0,0 +1,397 @@ +import { useEffect, useMemo, useRef, useState } from 'react' +import { markdown } from '@codemirror/lang-markdown' +import { EditorState } from '@codemirror/state' +import { EditorView, basicSetup } from 'codemirror' +import { keymap } from '@codemirror/view' +import { YjsProvider } from '@durable-streams/y-durable-streams' +import { Plug, TriangleAlert, Unplug } from 'lucide-react' +import { yCollab, yUndoManagerKeymap } from 'y-codemirror.next' +import * as decoding from 'lib0/decoding' +import { + Awareness, + applyAwarenessUpdate, + encodeAwarenessUpdate, + removeAwarenessStates, +} from 'y-protocols/awareness' +import * as Y from 'yjs' +import { useCurrentPrincipal } from '../../hooks/useCurrentPrincipal' +import { getConfiguredServerHeaders, serverFetch } from '../../lib/auth-fetch' +import { principalKeyFromInput } from '../../lib/principals' +import { Icon, ScrollArea } from '../../ui' +import styles from './MarkdownDocumentView.module.css' +import type { EntityViewProps } from '../../lib/workspace/viewRegistry' +import { + MARKDOWN_DOCUMENT_AGENT_PRESENCE_TTL_MS, + type ManifestDocumentEntry, +} from '@electric-ax/agents-runtime/client' +import type { LucideIcon } from 'lucide-react' + +type DocumentResponse = { + document: ManifestDocumentEntry +} + +type DocumentConnectionStatus = + | `loading` + | `connecting` + | `connected` + | `disconnected` + | `error` + +type RemoteUser = { + name: string + status?: string + color?: string + expiresAt?: number +} + +function entityApiUrl(baseUrl: string, entityUrl: string, suffix: string): URL { + const url = new URL(baseUrl) + url.pathname = `${url.pathname.replace(/\/+$/, ``)}/_electric/entities${entityUrl}${suffix}` + return url +} + +function colorFor(value: string): { color: string; light: string } { + const colors = [ + [`#2563eb`, `#2563eb33`], + [`#059669`, `#05966933`], + [`#dc2626`, `#dc262633`], + [`#7c3aed`, `#7c3aed33`], + [`#c2410c`, `#c2410c33`], + [`#0f766e`, `#0f766e33`], + ] as const + let hash = 0 + for (let i = 0; i < value.length; i += 1) { + hash = (hash * 31 + value.charCodeAt(i)) >>> 0 + } + const [color, light] = colors[hash % colors.length]! + return { color, light } +} + +function providerBaseUrl(baseUrl: string, streamPath: string): string { + const docsIndex = streamPath.indexOf(`/docs/`) + const prefix = docsIndex >= 0 ? streamPath.slice(0, docsIndex) : streamPath + const url = new URL(baseUrl) + url.pathname = `${url.pathname.replace(/\/+$/, ``)}${prefix}` + return url.toString().replace(/\/+$/, ``) +} + +function connectionStatusLabel(status: DocumentConnectionStatus): string { + switch (status) { + case `loading`: + return `Loading document` + case `connecting`: + return `Connecting` + case `connected`: + return `Connected` + case `disconnected`: + return `Disconnected` + case `error`: + return `Connection error` + } +} + +function connectionStatusIcon(status: DocumentConnectionStatus): LucideIcon { + switch (status) { + case `error`: + return TriangleAlert + case `disconnected`: + return Unplug + case `loading`: + case `connecting`: + case `connected`: + return Plug + } +} + +export function applyMarkdownAwarenessFrames( + awareness: Awareness, + data: Uint8Array +): void { + if (data.length === 0) return + const decoder = decoding.createDecoder(data) + while (decoding.hasContent(decoder)) { + applyAwarenessUpdate( + awareness, + decoding.readVarUint8Array(decoder), + `server` + ) + } +} + +async function primeMarkdownAwareness( + awareness: Awareness, + docUrl: URL, + signal: AbortSignal +): Promise { + const awarenessUrl = new URL(docUrl) + awarenessUrl.searchParams.set(`awareness`, `default`) + awarenessUrl.searchParams.set(`offset`, `-1`) + const response = await serverFetch(awarenessUrl, { + method: `GET`, + headers: getConfiguredServerHeaders(awarenessUrl), + signal, + }) + if (signal.aborted) return + if (response.status === 404) return + if (!response.ok) return + const bytes = new Uint8Array(await response.arrayBuffer()) + if (signal.aborted) return + + const snapshot = new Awareness(new Y.Doc()) + applyMarkdownAwarenessFrames(snapshot, bytes) + const now = Date.now() + const activeAgents = Array.from(snapshot.getStates()) + .filter(([, state]) => { + const user = ( + state as { + user?: { role?: string; status?: string; expiresAt?: number } + } + ).user + return ( + user?.role === `agent` && + user.status === `editing` && + typeof user.expiresAt === `number` && + user.expiresAt > now + ) + }) + .map(([clientId]) => clientId) + if (activeAgents.length > 0) { + applyAwarenessUpdate( + awareness, + encodeAwarenessUpdate(snapshot, activeAgents), + `server` + ) + } + snapshot.destroy() +} + +export function markdownDocumentConnectionConfig( + baseUrl: string, + documentEntry: ManifestDocumentEntry +): { + providerUrl: string + docUrl: URL + docId: string + yTextName: string +} { + const providerUrl = providerBaseUrl(baseUrl, documentEntry.streamPath) + const docId = documentEntry.docId + return { + providerUrl, + docId, + yTextName: documentEntry.yTextName, + docUrl: new URL(`${providerUrl}/docs/${docId}`), + } +} + +export function MarkdownDocumentView({ + baseUrl, + entityUrl, + viewParams, +}: EntityViewProps): React.ReactElement { + const documentId = viewParams?.doc ?? null + const editorRef = useRef(null) + const editorViewRef = useRef(null) + const remoteStateFirstSeenRef = useRef>(new Map()) + const [documentEntry, setDocumentEntry] = + useState(null) + const [status, setStatus] = useState(`loading`) + const [remoteUsers, setRemoteUsers] = useState>([]) + const { principal } = useCurrentPrincipal() + + useEffect(() => { + let cancelled = false + setDocumentEntry(null) + setStatus(documentId ? `loading` : `error`) + if (!documentId) return + const url = entityApiUrl( + baseUrl, + entityUrl, + `/documents/${encodeURIComponent(documentId)}` + ) + serverFetch(url, { headers: { accept: `application/json` } }) + .then(async (response) => { + if (!response.ok) { + throw new Error(`Document request failed (${response.status})`) + } + return (await response.json()) as DocumentResponse + }) + .then((result) => { + if (!cancelled) setDocumentEntry(result.document) + }) + .catch(() => { + if (!cancelled) setStatus(`error`) + }) + return () => { + cancelled = true + } + }, [baseUrl, entityUrl, documentId]) + + const principalLabel = useMemo( + () => principalKeyFromInput(principal) ?? principal, + [principal] + ) + + useEffect(() => { + if (!editorRef.current || !documentEntry) return + + const ydoc = new Y.Doc() + const awareness = new Awareness(ydoc) + const userColor = colorFor(principalLabel) + awareness.setLocalStateField(`user`, { + name: principalLabel, + color: userColor.color, + colorLight: userColor.light, + }) + + const { providerUrl, docUrl, docId, yTextName } = + markdownDocumentConnectionConfig(baseUrl, documentEntry) + const awarenessPrimeController = new AbortController() + void primeMarkdownAwareness( + awareness, + docUrl, + awarenessPrimeController.signal + ).catch(() => undefined) + const provider = new YjsProvider({ + doc: ydoc, + baseUrl: providerUrl, + docId, + awareness, + headers: getConfiguredServerHeaders(docUrl), + liveMode: `sse`, + }) + const ytext = ydoc.getText(yTextName) + const state = EditorState.create({ + doc: ytext.toString(), + extensions: [ + keymap.of([...yUndoManagerKeymap]), + basicSetup, + markdown(), + EditorView.lineWrapping, + yCollab(ytext, awareness), + ], + }) + const view = new EditorView({ state, parent: editorRef.current }) + editorViewRef.current = view + + const updateRemoteUsers = (): void => { + const users: Array = [] + const staleClients: Array = [] + const seenClients = new Set() + const now = Date.now() + awareness.getStates().forEach((state, clientId) => { + if (clientId === awareness.clientID) return + seenClients.add(clientId) + const user = ( + state as { + user?: { + name?: string + status?: string + color?: string + role?: string + expiresAt?: number + } + } + ).user + const firstSeen = + remoteStateFirstSeenRef.current.get(clientId) ?? Date.now() + remoteStateFirstSeenRef.current.set(clientId, firstSeen) + const isExpired = + typeof user?.expiresAt === `number` + ? user.expiresAt <= now + : user?.role === `agent` && + user.status === `editing` && + now - firstSeen > MARKDOWN_DOCUMENT_AGENT_PRESENCE_TTL_MS + if (isExpired) { + staleClients.push(clientId) + return + } + if (user?.name) { + users.push({ + name: user.name, + status: user.status, + color: user.color, + expiresAt: user.expiresAt, + }) + } + }) + for (const clientId of remoteStateFirstSeenRef.current.keys()) { + if (!seenClients.has(clientId)) { + remoteStateFirstSeenRef.current.delete(clientId) + } + } + if (staleClients.length > 0) { + removeAwarenessStates(awareness, staleClients, `stale-agent-presence`) + } + setRemoteUsers(users) + } + const statusHandler = (next: DocumentConnectionStatus): void => + setStatus(next) + provider.on(`status`, statusHandler) + awareness.on(`change`, updateRemoteUsers) + const stalePresenceInterval = window.setInterval(updateRemoteUsers, 1_000) + provider.connect() + setStatus(`connecting`) + + return () => { + awarenessPrimeController.abort() + window.clearInterval(stalePresenceInterval) + provider.off(`status`, statusHandler) + awareness.off(`change`, updateRemoteUsers) + provider.destroy() + editorViewRef.current?.destroy() + editorViewRef.current = null + ydoc.destroy() + setRemoteUsers([]) + } + }, [baseUrl, documentEntry, principalLabel]) + + if (!documentId) { + return
No document selected.
+ } + + return ( +
+
+
+ {documentEntry?.title ?? `Markdown document`} +
+
+ + + + {remoteUsers.slice(0, 3).map((user) => { + const color = user.color ?? colorFor(user.name).color + return ( + + + + {user.status ? `${user.name} · ${user.status}` : user.name} + + + ) + })} +
+
+ {status === `error` ? ( +
Document could not be opened.
+ ) : ( + +
+ + )} +
+ ) +} diff --git a/packages/agents-server-ui/src/components/workspace/Workspace.tsx b/packages/agents-server-ui/src/components/workspace/Workspace.tsx index 0937b89957..61ac255f76 100644 --- a/packages/agents-server-ui/src/components/workspace/Workspace.tsx +++ b/packages/agents-server-ui/src/components/workspace/Workspace.tsx @@ -15,6 +15,7 @@ import type { LocalRuntimeStatus, ServerConnectionStatus, } from '../../lib/server-connection' +import type { TileViewParams } from '../../lib/workspace/types' import type { ViewId } from '../../lib/workspace/viewRegistry' /** @@ -37,6 +38,7 @@ export function Workspace(): React.ReactElement { const search = useSearch({ strict: false }) as { view?: string source?: string + doc?: string layout?: string } const navigate = useNavigate() @@ -44,6 +46,7 @@ export function Workspace(): React.ReactElement { const entityUrl = splat ? `/${splat}` : null const requestedViewId = (search.view as ViewId | undefined) ?? null const requestedSource = (search.source as string | undefined) ?? null + const requestedDoc = (search.doc as string | undefined) ?? null const layoutParam = (search.layout as string | undefined) ?? null // ---- ?layout= import ------------------------------------------- @@ -73,6 +76,7 @@ export function Workspace(): React.ReactElement { search: { ...(requestedViewId ? { view: requestedViewId } : {}), ...(requestedSource ? { source: requestedSource } : {}), + ...(requestedDoc ? { doc: requestedDoc } : {}), }, replace: true, }) @@ -140,14 +144,21 @@ export function Workspace(): React.ReactElement { const availableViews = entity ? listViews(entity) : [] const defaultViewId = availableViews[0]?.id ?? `chat` const desiredViewId = - requestedViewId && availableViews.some((v) => v.id === requestedViewId) + requestedViewId === `markdown-doc` && requestedDoc ? requestedViewId - : defaultViewId - const desiredViewParams = + : requestedViewId && + availableViews.some((v) => v.id === requestedViewId) + ? requestedViewId + : defaultViewId + const desiredViewParams: TileViewParams | undefined = desiredViewId === `state-explorer` && requestedSource ? { source: requestedSource } - : undefined - const key = `${entityUrl}::${desiredViewId}::${requestedSource ?? ``}` + : desiredViewId === `markdown-doc` && requestedDoc + ? { doc: requestedDoc } + : undefined + const key = `${entityUrl}::${desiredViewId}::${viewParamsKey( + desiredViewParams + )}` if (lastSyncedKey.current === key) return const tiles = listTiles(workspace.root) @@ -157,7 +168,7 @@ export function Workspace(): React.ReactElement { (t) => t.entityUrl === entityUrl && t.viewId === desiredViewId && - (desiredViewParams?.source ?? ``) === (t.viewParams?.source ?? ``) + viewParamsKey(desiredViewParams) === viewParamsKey(t.viewParams) ) if (exactMatch) { helpers.setActiveTile(exactMatch.id) @@ -189,6 +200,7 @@ export function Workspace(): React.ReactElement { entityUrl, requestedViewId, requestedSource, + requestedDoc, entity, workspace.root, helpers, @@ -209,7 +221,7 @@ export function Workspace(): React.ReactElement { const expectedKey = tile.entityUrl === null ? `::${tile.viewId}` - : `${tile.entityUrl}::${tile.viewId}::${tile.viewParams?.source ?? ``}` + : `${tile.entityUrl}::${tile.viewId}::${viewParamsKey(tile.viewParams)}` if (lastSyncedKey.current === expectedKey) return lastSyncedKey.current = expectedKey if (tile.entityUrl === null) { @@ -222,6 +234,7 @@ export function Workspace(): React.ReactElement { search: { ...(tile.viewId === `chat` ? {} : { view: tile.viewId }), ...(tile.viewParams?.source ? { source: tile.viewParams.source } : {}), + ...(tile.viewParams?.doc ? { doc: tile.viewParams.doc } : {}), }, replace: true, }) @@ -306,6 +319,14 @@ export function Workspace(): React.ReactElement { ) } +function viewParamsKey(params: Record | undefined): string { + if (!params) return `` + return Object.entries(params) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, value]) => `${key}=${value}`) + .join(`&`) +} + function getRemoteStatus( connected: boolean, status: ServerConnectionStatus | undefined diff --git a/packages/agents-server-ui/src/lib/workspace/registerViews.ts b/packages/agents-server-ui/src/lib/workspace/registerViews.ts index 20f29596ca..3f94915196 100644 --- a/packages/agents-server-ui/src/lib/workspace/registerViews.ts +++ b/packages/agents-server-ui/src/lib/workspace/registerViews.ts @@ -1,9 +1,16 @@ -import { Database, MessageCircle, MessageSquare, SquarePen } from 'lucide-react' +import { + Database, + FileText, + MessageCircle, + MessageSquare, + SquarePen, +} from 'lucide-react' import { entityTypeSupportsComments } from '../comments-capability' import { registerView } from './viewRegistry' import { NEW_SESSION_VIEW_ID } from './types' import { ChatView, CommentsView } from '../../components/views/ChatView' import { StateExplorerView } from '../../components/views/StateExplorerView' +import { MarkdownDocumentView } from '../../components/views/MarkdownDocumentView' import { NewSessionView } from '../../components/views/NewSessionView' /** @@ -45,6 +52,17 @@ registerView({ Component: StateExplorerView, }) +registerView({ + kind: `entity`, + id: `markdown-doc`, + label: `Markdown Doc`, + shortLabel: `Doc`, + icon: FileText, + description: `Collaborative markdown document editor`, + isAvailable: () => false, + Component: MarkdownDocumentView, +}) + /** * Standalone view: "new session". Doesn't belong to an entity, so it * never appears in the per-entity view-switcher. The workspace mounts diff --git a/packages/agents-server-ui/src/lib/workspace/workspaceReducer.test.ts b/packages/agents-server-ui/src/lib/workspace/workspaceReducer.test.ts index 4cabce3975..c628309daa 100644 --- a/packages/agents-server-ui/src/lib/workspace/workspaceReducer.test.ts +++ b/packages/agents-server-ui/src/lib/workspace/workspaceReducer.test.ts @@ -67,6 +67,33 @@ describe(`workspaceReducer`, () => { expect(right.entityUrl).toBe(`/horton/bar`) }) + it(`opens a markdown document view in a right split`, () => { + const after1 = run(EMPTY_WORKSPACE, { + type: `open-tile`, + tile: { entityUrl: `/horton/foo`, viewId: `chat` }, + }) + const fooId = rootAsTile(after1).id + const ws = workspaceReducer(after1, { + type: `open-tile`, + tile: { + entityUrl: `/horton/foo`, + viewId: `markdown-doc`, + viewParams: { doc: `notes` }, + }, + target: { tileId: fooId, position: `split-right` }, + }) + + const split = rootAsSplit(ws) + expect(split.direction).toBe(`horizontal`) + const right = split.children[1].node as Tile + expect(right).toMatchObject({ + entityUrl: `/horton/foo`, + viewId: `markdown-doc`, + viewParams: { doc: `notes` }, + }) + expect(ws.activeTileId).toBe(right.id) + }) + it(`split-up places the new tile above the existing one`, () => { const after1 = run(EMPTY_WORKSPACE, { type: `open-tile`, diff --git a/packages/agents-server/package.json b/packages/agents-server/package.json index 9fcdbae14a..24ec7f670e 100644 --- a/packages/agents-server/package.json +++ b/packages/agents-server/package.json @@ -59,11 +59,14 @@ "drizzle-orm": "^0.44.0", "fastq": "^1.20.1", "itty-router": "^5.0.23", + "lib0": "^0.2.99", "lmdb": "^3.5.1", "pino": "^10.3.1", "pino-pretty": "^13.0.0", "postgres": "^3.4.0", - "undici": "^7.24.7" + "undici": "^7.24.7", + "y-protocols": "^1.0.6", + "yjs": "^13.6.26" }, "devDependencies": { "@electric-ax/agents": "workspace:*", diff --git a/packages/agents-server/src/entity-manager.ts b/packages/agents-server/src/entity-manager.ts index 200f518661..c9baef7dc9 100644 --- a/packages/agents-server/src/entity-manager.ts +++ b/packages/agents-server/src/entity-manager.ts @@ -7,8 +7,10 @@ import { getCronStreamPath, getSharedStateStreamPath, getNextCronFireAt, + getEntityMarkdownDocumentUrlPath, webhookSourceSubscriptionManifestKey, manifestChildKey, + manifestMarkdownDocumentKey, manifestSharedStateKey, manifestSourceKey, resolveCronScheduleSpec, @@ -52,6 +54,16 @@ import { } from './manifest-side-effects.js' import { DEFAULT_TENANT_ID } from './tenant.js' import { ATTR, withSpan } from './tracing.js' +import { + MARKDOWN_DOCUMENT_CONTENT_MIME, + MARKDOWN_DOCUMENT_PROVIDER, + MARKDOWN_DOCUMENT_TEXT_NAME, + MARKDOWN_DOCUMENT_TRANSPORT_MIME, + assertMarkdownDocumentMatchesEntity, + getMarkdownDocumentDocPath, + getMarkdownDocumentAwarenessStreamPath, + getMarkdownDocumentUpdateStreamPath, +} from './markdown-documents.js' import type { queueAsPromised } from 'fastq' import type { SchedulerClient } from './scheduler.js' import type { WakeEvalResult, WakeRegistry } from './wake-registry.js' @@ -175,6 +187,31 @@ type ManifestAttachmentEntry = { meta?: Record } +export interface CreateMarkdownDocumentRequest { + id?: string + title: string + createdBy?: string + meta?: Record +} + +export type ManifestMarkdownDocumentEntry = { + key: string + kind: `document` + id: string + provider: typeof MARKDOWN_DOCUMENT_PROVIDER + docId: string + docPath: string + streamPath: string + transportMimeType: typeof MARKDOWN_DOCUMENT_TRANSPORT_MIME + contentMimeType: typeof MARKDOWN_DOCUMENT_CONTENT_MIME + yTextName: typeof MARKDOWN_DOCUMENT_TEXT_NAME + title: string + createdAt: string + createdBy?: string + updatedAt?: string + meta?: Record +} + function createInitialQueuePosition(date: Date): string { return `${String(date.getTime()).padStart(16, `0`)}:a0` } @@ -237,6 +274,7 @@ type ForkStateSnapshot = { childStatusesByEntity: Map>> replayWatermarksByEntity: Map>> sharedStateIds: Set + markdownDocumentDocPaths: Set } type ForkResult = { @@ -265,6 +303,16 @@ function manifestAttachmentKey(id: string): string { return `attachment:${id}` } +function validateMarkdownDocumentId(id: string): void { + if (!id || !/^[A-Za-z0-9_-]+$/.test(id)) { + throw new ElectricAgentsError( + ErrCodeInvalidRequest, + `document id must contain only letters, numbers, underscores, or hyphens`, + 400 + ) + } +} + function getEntityAttachmentStreamPath( entityUrl: string, attachmentId: string @@ -1036,6 +1084,7 @@ export class EntityManager { childStatuses: Map> replayWatermarks: Map> sharedStateIds: Set + markdownDocumentDocPaths: Set } | undefined let effectiveForkPointer: EventPointer | undefined = opts.forkPointer @@ -1069,8 +1118,13 @@ export class EntityManager { const filteredEvents = flat.slice(0, target) const rootManifests = this.reduceStateRows(filteredEvents, `manifest`) const sharedStateIds = new Set() + const markdownDocumentDocPaths = new Set() for (const manifest of rootManifests.values()) { this.collectSharedStateIds(manifest, sharedStateIds) + this.collectMarkdownDocumentDocPaths( + manifest, + markdownDocumentDocPaths + ) } preFilteredRoot = { manifests: rootManifests, @@ -1080,6 +1134,7 @@ export class EntityManager { `replay_watermark` ), sharedStateIds, + markdownDocumentDocPaths, } } @@ -1128,6 +1183,9 @@ export class EntityManager { for (const id of preFilteredRoot.sharedStateIds) { snapshot.sharedStateIds.add(id) } + for (const docPath of preFilteredRoot.markdownDocumentDocPaths) { + snapshot.markdownDocumentDocPaths.add(docPath) + } } const suffix = randomUUID().slice(0, 8) @@ -1177,7 +1235,14 @@ export class EntityManager { ) this.addForkLocks( this.forkWriteLockedStreams, - [...snapshot.sharedStateIds].map((id) => getSharedStateStreamPath(id)), + [ + ...[...snapshot.sharedStateIds].map((id) => + getSharedStateStreamPath(id) + ), + ...[...snapshot.markdownDocumentDocPaths].map((docPath) => + getMarkdownDocumentUpdateStreamPath(this.tenantId, docPath) + ), + ], writeStreamLocks ) @@ -1228,6 +1293,34 @@ export class EntityManager { } } + for (const plan of entityPlans) { + const manifests = + snapshot.manifestsByEntity.get(plan.source.url) ?? new Map() + for (const manifest of manifests.values()) { + if ( + manifest.kind !== `document` || + typeof manifest.docPath !== `string` || + typeof manifest.id !== `string` + ) { + continue + } + const forkDocPath = getMarkdownDocumentDocPath( + plan.fork.url, + manifest.id + ) + const sourcePath = getMarkdownDocumentUpdateStreamPath( + this.tenantId, + manifest.docPath + ) + const forkPath = getMarkdownDocumentUpdateStreamPath( + this.tenantId, + forkDocPath + ) + await this.streamClient.fork(forkPath, sourcePath) + createdStreams.push(forkPath) + } + } + for (const plan of entityPlans) { const reconciliation = this.buildForkReconciliation( plan, @@ -1737,6 +1830,7 @@ export class EntityManager { Map> >() const sharedStateIds = new Set() + const markdownDocumentDocPaths = new Set() for (const entity of entitiesToFork) { const events = await this.streamClient.readJson>( @@ -1752,6 +1846,7 @@ export class EntityManager { for (const manifest of manifests.values()) { this.collectSharedStateIds(manifest, sharedStateIds) + this.collectMarkdownDocumentDocPaths(manifest, markdownDocumentDocPaths) } } @@ -1760,6 +1855,7 @@ export class EntityManager { childStatusesByEntity, replayWatermarksByEntity, sharedStateIds, + markdownDocumentDocPaths, } } @@ -1811,6 +1907,16 @@ export class EntityManager { } } + private collectMarkdownDocumentDocPaths( + manifest: Record, + docPaths: Set + ): void { + if (manifest.kind !== `document` || typeof manifest.docPath !== `string`) { + return + } + docPaths.add(manifest.docPath) + } + private async buildForkEntityUrlMap( entitiesToFork: Array, opts: { suffix: string; rootUrl: string; rootInstanceId?: string } @@ -2159,6 +2265,30 @@ export class EntityManager { } } + if ( + next.kind === `document` && + typeof next.docPath === `string` && + typeof next.id === `string` + ) { + for (const [sourceUrl, forkUrl] of entityUrlMap) { + const expectedSourceDocPath = getMarkdownDocumentDocPath( + sourceUrl, + next.id + ) + if (next.docPath !== expectedSourceDocPath) { + continue + } + next.docPath = getMarkdownDocumentDocPath(forkUrl, next.id) + next.docId = next.docPath + next.streamPath = getEntityMarkdownDocumentUrlPath( + this.tenantId, + forkUrl, + next.id + ) + return { key, value: next, changed: true } + } + } + if (next.kind === `schedule` && next.scheduleType === `future_send`) { let changed = false if (typeof next.targetUrl === `string`) { @@ -2821,6 +2951,131 @@ export class EntityManager { return { txid } } + // ========================================================================== + // Markdown Documents + // ========================================================================== + + isMarkdownDocumentUpdateStreamPath(path: string): boolean { + return /^\/yjs\/[^/]+\/docs\/agents\/[^/]+\/[^/]+\/documents\/[A-Za-z0-9_-]+\/\.updates$/.test( + path + ) + } + + async createMarkdownDocument( + entityUrl: string, + req: CreateMarkdownDocumentRequest + ): Promise<{ txid: string; document: ManifestMarkdownDocumentEntry }> { + const entity = await this.registry.getEntity(entityUrl) + if (!entity) { + throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404) + } + if (rejectsNormalWrites(entity.status)) { + throw new ElectricAgentsError( + ErrCodeNotRunning, + `Entity is not accepting writes`, + 409 + ) + } + if (this.isForkWorkLockedEntity(entityUrl)) { + this.assertEntityNotForkWorkLocked(entityUrl) + } + + const id = req.id ?? randomUUID() + validateMarkdownDocumentId(id) + + const docPath = getMarkdownDocumentDocPath(entityUrl, id) + const updateStreamPath = getMarkdownDocumentUpdateStreamPath( + this.tenantId, + docPath + ) + const awarenessStreamPath = getMarkdownDocumentAwarenessStreamPath( + this.tenantId, + docPath, + `default` + ) + const now = new Date().toISOString() + const txid = randomUUID() + const document: ManifestMarkdownDocumentEntry = { + key: manifestMarkdownDocumentKey(id), + kind: `document`, + id, + provider: MARKDOWN_DOCUMENT_PROVIDER, + docId: docPath, + docPath, + streamPath: getEntityMarkdownDocumentUrlPath( + this.tenantId, + entityUrl, + id + ), + transportMimeType: MARKDOWN_DOCUMENT_TRANSPORT_MIME, + contentMimeType: MARKDOWN_DOCUMENT_CONTENT_MIME, + yTextName: MARKDOWN_DOCUMENT_TEXT_NAME, + title: req.title.trim() || `Untitled document`, + createdAt: now, + ...(req.createdBy ? { createdBy: req.createdBy } : {}), + ...(req.meta ? { meta: req.meta } : {}), + } + + let streamCreated = false + let awarenessStreamCreated = false + try { + await this.streamClient.create(updateStreamPath, { + contentType: `application/octet-stream`, + }) + streamCreated = true + await this.streamClient.create(awarenessStreamPath, { + contentType: `application/octet-stream`, + }) + awarenessStreamCreated = true + await this.writeManifestEntry( + entityUrl, + document.key, + `upsert`, + document as unknown as Record, + { txid } + ) + } catch (error) { + if (awarenessStreamCreated) { + await this.streamClient + .delete(awarenessStreamPath) + .catch(() => undefined) + } + if (streamCreated) { + await this.streamClient.delete(updateStreamPath).catch(() => undefined) + } + if (!streamCreated && isStreamCreateConflict(error)) { + throw new ElectricAgentsError( + ErrCodeInvalidRequest, + `Document already exists at id "${id}"`, + 409 + ) + } + throw error + } + + return { txid, document } + } + + async getMarkdownDocument( + entityUrl: string, + id: string + ): Promise { + validateMarkdownDocumentId(id) + const entity = await this.registry.getEntity(entityUrl) + if (!entity) { + throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404) + } + const events = await this.streamClient.readJson>( + entity.streams.main + ) + const manifest = this.reduceStateRows(events, `manifest`).get( + manifestMarkdownDocumentKey(id) + ) + if (!manifest || manifest.kind !== `document`) return null + assertMarkdownDocumentMatchesEntity(entity, manifest.docPath as string) + return manifest as unknown as ManifestMarkdownDocumentEntry + } + // ========================================================================== // Tag Updates // ========================================================================== diff --git a/packages/agents-server/src/markdown-documents.ts b/packages/agents-server/src/markdown-documents.ts new file mode 100644 index 0000000000..8a0d63e907 --- /dev/null +++ b/packages/agents-server/src/markdown-documents.ts @@ -0,0 +1,193 @@ +import * as decoding from 'lib0/decoding' +import * as encoding from 'lib0/encoding' +import { applyAwarenessUpdate } from 'y-protocols/awareness' +import * as Y from 'yjs' +import type { ElectricAgentsEntity } from './electric-agents-types.js' +import type { StreamClient } from './stream-client.js' + +export const MARKDOWN_DOCUMENT_TRANSPORT_MIME = + `application/vnd.electric-agents.markdown-yjs` as const +export const MARKDOWN_DOCUMENT_CONTENT_MIME = `text/markdown` as const +export const MARKDOWN_DOCUMENT_TEXT_NAME = `markdown` as const +export const MARKDOWN_DOCUMENT_PROVIDER = `y-durable-streams` as const + +export interface ParsedMarkdownDocumentPath { + entityType: string + instanceId: string + entityUrl: string + documentId: string +} + +export function getMarkdownDocumentDocPath( + entityUrl: string, + documentId: string +): string { + const match = entityUrl.match(/^\/([^/]+)\/([^/]+)$/) + if (!match) { + throw new Error(`Invalid entity URL for markdown document: ${entityUrl}`) + } + return `agents/${match[1]}/${match[2]}/documents/${documentId}` +} + +export function getMarkdownDocumentUrlPath( + service: string, + entityUrl: string, + documentId: string +): string { + return `/v1/yjs/${encodeURIComponent(service)}/docs/${getMarkdownDocumentDocPath( + entityUrl, + documentId + )}` +} + +export function getMarkdownDocumentUpdateStreamPath( + service: string, + docPath: string +): string { + return `/yjs/${service}/docs/${docPath}/.updates` +} + +export function getMarkdownDocumentAwarenessStreamPath( + service: string, + docPath: string, + name: string +): string { + return `/yjs/${service}/docs/${docPath}/.awareness/${name}` +} + +export function getMarkdownDocumentIndexStreamPath( + service: string, + docPath: string +): string { + return `/yjs/${service}/docs/${docPath}/.index` +} + +export function getMarkdownDocumentSnapshotStreamPath( + service: string, + docPath: string, + snapshotKey: string +): string { + return `/yjs/${service}/docs/${docPath}/.snapshots/${snapshotKey}` +} + +export function parseMarkdownDocumentDocPath( + docPath: string +): ParsedMarkdownDocumentPath | null { + const match = docPath.match( + /^agents\/([^/]+)\/([^/]+)\/documents\/([A-Za-z0-9_-]+)$/ + ) + if (!match) return null + return { + entityType: match[1]!, + instanceId: match[2]!, + entityUrl: `/${match[1]}/${match[2]}`, + documentId: match[3]!, + } +} + +export function parseYjsDocumentRoutePath( + path: string +): { service: string; docPath: string } | null { + const match = path.match(/^\/v1\/yjs\/([^/]+)\/docs\/(.+)$/) + if (!match) return null + let docPath: string + try { + docPath = decodeURIComponent(match[2]!) + } catch { + return null + } + if ( + docPath.includes(`..`) || + docPath.split(`/`).some((segment) => segment === `.` || segment === ``) + ) { + return null + } + return { service: match[1]!, docPath } +} + +export function frameYjsUpdate(update: Uint8Array): Uint8Array { + const encoder = encoding.createEncoder() + encoding.writeVarUint8Array(encoder, update) + return encoding.toUint8Array(encoder) +} + +export function applyFramedYjsUpdates(doc: Y.Doc, data: Uint8Array): void { + if (data.length === 0) return + const decoder = decoding.createDecoder(data) + while (decoding.hasContent(decoder)) { + Y.applyUpdate(doc, decoding.readVarUint8Array(decoder), `server`) + } +} + +export function applyFramedAwarenessUpdates( + awareness: Parameters[0], + data: Uint8Array +): void { + if (data.length === 0) return + const decoder = decoding.createDecoder(data) + while (decoding.hasContent(decoder)) { + applyAwarenessUpdate( + awareness, + decoding.readVarUint8Array(decoder), + `server` + ) + } +} + +export async function readMarkdownYDoc( + streamClient: StreamClient, + updateStreamPath: string +): Promise { + const doc = new Y.Doc() + const result = await streamClient.read(updateStreamPath) + for (const message of result.messages) { + applyFramedYjsUpdates(doc, message.data) + } + return doc +} + +export function markdownText(doc: Y.Doc): Y.Text { + return doc.getText(MARKDOWN_DOCUMENT_TEXT_NAME) +} + +export function replaceMarkdownText(doc: Y.Doc, content: string): Uint8Array { + const before = Y.encodeStateVector(doc) + const text = markdownText(doc) + doc.transact(() => { + text.delete(0, text.length) + if (content.length > 0) text.insert(0, content) + }, `server`) + return Y.encodeStateAsUpdate(doc, before) +} + +export function entityUrlFromYjsDocumentRoutePath(path: string): string | null { + const route = parseYjsDocumentRoutePath(path) + if (!route) return null + return parseMarkdownDocumentDocPath(route.docPath)?.entityUrl ?? null +} + +export function parseMarkdownDocumentStreamPath( + path: string +): { service: string; docPath: string; entityUrl: string } | null { + const match = path.match( + /^\/yjs\/([^/]+)\/docs\/(.+)\/\.(updates|index|awareness|snapshots)(?:\/.*)?$/ + ) + if (!match) return null + const parsed = parseMarkdownDocumentDocPath(match[2]!) + if (!parsed) return null + return { + service: match[1]!, + docPath: match[2]!, + entityUrl: parsed.entityUrl, + } +} + +export function assertMarkdownDocumentMatchesEntity( + entity: ElectricAgentsEntity, + docPath: string +): void { + const parsed = parseMarkdownDocumentDocPath(docPath) + if (!parsed || parsed.entityUrl !== entity.url) { + throw new Error(`Markdown document path does not belong to ${entity.url}`) + } +} diff --git a/packages/agents-server/src/routing/durable-streams-router.ts b/packages/agents-server/src/routing/durable-streams-router.ts index db283021c3..d97aa45022 100644 --- a/packages/agents-server/src/routing/durable-streams-router.ts +++ b/packages/agents-server/src/routing/durable-streams-router.ts @@ -13,7 +13,9 @@ import { } from '../electric-agents-http.js' import { subscriptionWebhooks } from '../db/schema.js' import { + ErrCodeInvalidRequest, ErrCodeNotFound, + ErrCodeNotRunning, ErrCodeUnauthorized, } from '../electric-agents-types.js' import { @@ -29,6 +31,13 @@ import { webhookSigningMetadata, } from '../webhook-signing.js' import { resolveDurableStreamsRoutingAdapter } from './durable-streams-routing-adapter.js' +import { + getMarkdownDocumentAwarenessStreamPath, + getMarkdownDocumentUpdateStreamPath, + parseMarkdownDocumentDocPath, + parseMarkdownDocumentStreamPath, + parseYjsDocumentRoutePath, +} from '../markdown-documents.js' import type { IRequest, RouterType } from 'itty-router' import type { TenantContext } from './context.js' import type { DurableStreamsRoutingAdapter } from './durable-streams-routing-adapter.js' @@ -100,6 +109,7 @@ for (const action of subscriptionControlActions) { durableStreamsRouter.get(`/__ds/jwks.json`, webhookJwks) durableStreamsRouter.all(`/__ds`, controlPassThrough) durableStreamsRouter.all(`/__ds/*`, controlPassThrough) +durableStreamsRouter.all(`/v1/yjs/:service/docs/:docPath+`, yjsDocumentRoute) durableStreamsRouter.post(`*`, streamAppend) durableStreamsRouter.all(`*`, proxyPassThrough) @@ -617,6 +627,111 @@ async function streamAppend( ) } +async function yjsDocumentRoute( + request: IRequest, + ctx: TenantContext +): Promise { + const url = new URL(request.url) + const route = parseYjsDocumentRoutePath(url.pathname) + if (!route || route.service !== ctx.service) { + return apiError(400, ErrCodeInvalidRequest, `Invalid Yjs document path`) + } + const parsed = parseMarkdownDocumentDocPath(route.docPath) + if (!parsed) { + return apiError( + 400, + ErrCodeInvalidRequest, + `Invalid markdown document path` + ) + } + const entity = await ctx.entityManager.registry.getEntity(parsed.entityUrl) + if (!entity) { + return apiError(404, ErrCodeNotFound, `Entity not found`) + } + + const method = request.method.toUpperCase() + const awarenessName = url.searchParams.get(`awareness`) + const isAwareness = awarenessName !== null + const permission = + method === `GET` || method === `HEAD` || isAwareness ? `read` : `write` + if (!(await canAccessEntity(ctx, entity, permission, request as Request))) { + return apiError( + 401, + ErrCodeUnauthorized, + `Principal is not allowed to ${permission} ${entity.url}` + ) + } + + if ( + !isAwareness && + (method === `PUT` || method === `POST` || method === `DELETE`) && + ctx.entityManager.isForkWorkLockedEntity(entity.url) + ) { + return apiError(409, ErrCodeNotRunning, `Entity subtree is being forked`) + } + + if (!isAwareness && url.searchParams.get(`offset`) === `snapshot`) { + const redirect = new URL(request.url) + redirect.searchParams.set(`offset`, `-1`) + return new Response(null, { + status: 307, + headers: { + location: `${redirect.pathname}${redirect.search}`, + 'cache-control': `private, max-age=5`, + }, + }) + } + + const offset = url.searchParams.get(`offset`) + if (!isAwareness && offset?.endsWith(`_snapshot`)) { + return apiError(404, ErrCodeNotFound, `Snapshot not found`) + } + + const streamPath = isAwareness + ? getMarkdownDocumentAwarenessStreamPath( + ctx.service, + route.docPath, + awarenessName || `default` + ) + : getMarkdownDocumentUpdateStreamPath(ctx.service, route.docPath) + + if (method === `PUT`) { + const upstream = await forwardToDurableStreams( + ctx, + request, + undefined, + `stream`, + rewriteYjsDocumentStreamUrl(request.url, streamPath) + ) + if (!isAwareness && (upstream.ok || upstream.status === 409)) { + await ctx.streamClient + .create( + getMarkdownDocumentAwarenessStreamPath( + ctx.service, + route.docPath, + `default` + ), + { contentType: `application/octet-stream` } + ) + .catch(() => undefined) + } + return responseFromUpstream(upstream) + } + + if (method === `GET` || method === `HEAD` || method === `POST`) { + const upstream = await forwardToDurableStreams( + ctx, + request, + undefined, + `stream`, + rewriteYjsDocumentStreamUrl(request.url, streamPath) + ) + return responseFromUpstream(upstream) + } + + return apiError(400, ErrCodeInvalidRequest, `Unsupported Yjs document method`) +} + async function proxyPassThrough( request: IRequest, ctx: TenantContext @@ -640,6 +755,13 @@ async function proxyPassThrough( } } +function rewriteYjsDocumentStreamUrl(requestUrl: string, path: string): string { + const url = new URL(requestUrl) + url.pathname = path + url.searchParams.delete(`awareness`) + return url.toString() +} + async function authorizeDurableStreamAccess( request: IRequest, ctx: TenantContext @@ -685,6 +807,37 @@ async function authorizeDurableStreamAccess( } const sharedStateId = sharedStateIdFromPath(streamPath) + const markdownDocumentStream = parseMarkdownDocumentStreamPath(streamPath) + if (markdownDocumentStream) { + if (markdownDocumentStream.service !== ctx.service) { + return apiError(404, ErrCodeNotFound, `Document stream not found`) + } + const entity = await ctx.entityManager.registry.getEntity( + markdownDocumentStream.entityUrl + ) + if (!entity) { + return apiError(404, ErrCodeNotFound, `Entity not found`) + } + const permission = method === `GET` || method === `HEAD` ? `read` : `write` + if (await canAccessEntity(ctx, entity, permission, request as Request)) { + if ( + permission === `write` && + ctx.entityManager.isForkWorkLockedEntity(entity.url) + ) { + return apiError( + 409, + ErrCodeNotRunning, + `Entity subtree is being forked` + ) + } + return undefined + } + return apiError( + 401, + ErrCodeUnauthorized, + `Principal is not allowed to ${permission} ${entity.url}` + ) + } if (!sharedStateId) { // Durable Streams also hosts non-Agents utility streams. Entity streams, // attachment streams, and shared-state streams are guarded above; paths that diff --git a/packages/agents-server/src/routing/entities-router.ts b/packages/agents-server/src/routing/entities-router.ts index 50ac3e8911..3fff9abde4 100644 --- a/packages/agents-server/src/routing/entities-router.ts +++ b/packages/agents-server/src/routing/entities-router.ts @@ -287,6 +287,15 @@ const setTagBodySchema = Type.Object({ value: Type.String(), }) +const markdownDocumentCreateBodySchema = Type.Object( + { + id: Type.Optional(Type.String()), + title: Type.String(), + meta: Type.Optional(Type.Record(Type.String(), Type.Unknown())), + }, + { additionalProperties: false } +) + const entitySignalSchema = Type.Union([ Type.Literal(`SIGINT`), Type.Literal(`SIGHUP`), @@ -346,6 +355,9 @@ type SendBody = Static type InboxMessageBody = Static type ForkBody = Static type SetTagBody = Static +type MarkdownDocumentCreateBody = Static< + typeof markdownDocumentCreateBodySchema +> type SignalBody = Static type ScheduleBody = Static type WebhookSourceSubscriptionBody = Static< @@ -447,6 +459,19 @@ entitiesRouter.delete( withEntityPermission(`write`), deleteAttachment ) +entitiesRouter.post( + `/:type/:instanceId/documents`, + withExistingEntity, + withSchema(markdownDocumentCreateBodySchema), + withEntityPermission(`write`), + createMarkdownDocument +) +entitiesRouter.get( + `/:type/:instanceId/documents/:documentId`, + withExistingEntity, + withEntityPermission(`read`), + readMarkdownDocument +) entitiesRouter.patch( `/:type/:instanceId/inbox/:messageKey`, withExistingEntity, @@ -1443,6 +1468,50 @@ async function deleteAttachment( return json(result) } +async function createMarkdownDocument( + request: AgentsRouteRequest, + ctx: TenantContext +): Promise { + const principalMutationError = rejectPrincipalEntityMutation( + request, + `given documents` + ) + if (principalMutationError) return principalMutationError + + const parsed = routeBody(request) + const { entityUrl } = requireExistingEntityRoute(request) + const result = await ctx.entityManager.createMarkdownDocument(entityUrl, { + id: parsed.id, + title: parsed.title, + createdBy: ctx.principal.url, + meta: parsed.meta, + }) + return json(result, { status: 201 }) +} + +async function readMarkdownDocument( + request: AgentsRouteRequest, + ctx: TenantContext +): Promise { + const { entityUrl } = requireExistingEntityRoute(request) + const document = await ctx.entityManager.getMarkdownDocument( + entityUrl, + decodeURIComponent(request.params.documentId) + ) + if (!document) { + throw new ElectricAgentsError(ErrCodeNotFound, `Document not found`, 404) + } + return json( + { document }, + { + headers: { + 'content-type': `application/json; charset=utf-8`, + 'cache-control': `no-store`, + }, + } + ) +} + async function updateInboxMessage( request: AgentsRouteRequest, ctx: TenantContext diff --git a/packages/agents-server/src/stream-client.ts b/packages/agents-server/src/stream-client.ts index 96e92de279..b37561931a 100644 --- a/packages/agents-server/src/stream-client.ts +++ b/packages/agents-server/src/stream-client.ts @@ -310,6 +310,49 @@ export class StreamClient { }) } + async appendBytes( + path: string, + data: Uint8Array, + opts?: { + contentType?: string + producerId?: string + epoch?: number + seq?: number + } + ): Promise { + return await withSpan(`stream.appendBytes`, async (span) => { + span.setAttributes({ + [ATTR.STREAM_PATH]: path, + [ATTR.STREAM_OP]: `appendBytes`, + }) + const headers: Record = { + 'content-type': opts?.contentType ?? `application/octet-stream`, + } + if (opts?.producerId) headers[`Producer-Id`] = opts.producerId + if (opts?.epoch !== undefined) { + headers[`Producer-Epoch`] = String(opts.epoch) + } + if (opts?.seq !== undefined) { + headers[`Producer-Seq`] = String(opts.seq) + } + injectTraceHeaders(headers) + + const response = await fetch(this.streamUrl(path), { + method: `POST`, + headers: await this.requestHeaders(headers), + body: data, + }) + if (!response.ok) { + throw new Error( + `Stream append failed: ${response.status} ${await response.text()}` + ) + } + return { + offset: response.headers.get(`stream-next-offset`) ?? ``, + } + }) + } + async appendIdempotent( path: string, data: Uint8Array | string, diff --git a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts index 9a96c4ef40..70c51ea3db 100644 --- a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts +++ b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it, vi } from 'vitest' import { EntityManager } from '../src/entity-manager' import { SchemaValidator } from '../src/electric-agents/schema-validator' +import { + MARKDOWN_DOCUMENT_CONTENT_MIME, + MARKDOWN_DOCUMENT_PROVIDER, + MARKDOWN_DOCUMENT_TEXT_NAME, + MARKDOWN_DOCUMENT_TRANSPORT_MIME, +} from '../src/markdown-documents' const observedItemSchema = { type: `object`, @@ -94,6 +100,65 @@ function attachmentManifest(value: Record) { } } +function createMarkdownDocumentManager() { + const jsonStreams = new Map>>() + const binaryStreams = new Map>() + const streamClient = { + create: vi.fn(async (path: string) => { + if (binaryStreams.has(path)) { + const error = new Error(`Stream already exists`) as Error & { + status?: number + } + error.status = 409 + throw error + } + binaryStreams.set(path, []) + }), + appendBytes: vi.fn(async (path: string, data: Uint8Array) => { + binaryStreams.get(path)?.push(data) + }), + append: vi.fn(async (path: string, data: Uint8Array) => { + const event = JSON.parse(new TextDecoder().decode(data)) + const stream = jsonStreams.get(path) ?? [] + stream.push(event) + jsonStreams.set(path, stream) + }), + delete: vi.fn(async (path: string) => { + binaryStreams.delete(path) + }), + read: vi.fn(async (path: string) => ({ + messages: (binaryStreams.get(path) ?? []).map((data, index) => ({ + data, + offset: String(index), + })), + })), + readJson: vi.fn(async (path: string) => jsonStreams.get(path) ?? []), + } + + return { + manager: new EntityManager({ + registry: { + getEntity: vi.fn().mockResolvedValue({ + url: `/chat/session-1`, + status: `running`, + streams: { main: `/chat/session-1` }, + }), + getEntityType: vi.fn(), + replaceEntityManifestSource: vi.fn(), + replaceSharedStateLink: vi.fn(), + close: vi.fn(), + } as any, + streamClient: streamClient as any, + validator: new SchemaValidator(), + wakeRegistry: { + setTimeoutCallback: vi.fn(), + setDebounceCallback: vi.fn(), + } as any, + }), + streamClient, + } +} + describe(`ElectricAgentsManager.validateWriteEvent`, () => { it(`validates delete events against old_value instead of value`, async () => { const manager = createManager() @@ -140,6 +205,48 @@ describe(`ElectricAgentsManager.validateWriteEvent`, () => { }) }) +describe(`ElectricAgentsManager markdown documents`, () => { + it(`creates Yjs update and awareness streams and exposes a manifest document entry`, async () => { + const { manager, streamClient } = createMarkdownDocumentManager() + + const created = await manager.createMarkdownDocument(`/chat/session-1`, { + id: `notes`, + title: `Session notes`, + createdBy: `/principal/agent:horton`, + }) + + expect(created.document).toMatchObject({ + key: `document:notes`, + kind: `document`, + id: `notes`, + provider: MARKDOWN_DOCUMENT_PROVIDER, + docId: `agents/chat/session-1/documents/notes`, + docPath: `agents/chat/session-1/documents/notes`, + streamPath: `/v1/yjs/default/docs/agents/chat/session-1/documents/notes`, + transportMimeType: MARKDOWN_DOCUMENT_TRANSPORT_MIME, + contentMimeType: MARKDOWN_DOCUMENT_CONTENT_MIME, + yTextName: MARKDOWN_DOCUMENT_TEXT_NAME, + title: `Session notes`, + createdBy: `/principal/agent:horton`, + }) + expect(streamClient.create).toHaveBeenCalledWith( + `/yjs/default/docs/agents/chat/session-1/documents/notes/.updates`, + { contentType: `application/octet-stream` } + ) + expect(streamClient.create).toHaveBeenCalledWith( + `/yjs/default/docs/agents/chat/session-1/documents/notes/.awareness/default`, + { contentType: `application/octet-stream` } + ) + expect(streamClient.appendBytes).not.toHaveBeenCalled() + await expect( + manager.getMarkdownDocument(`/chat/session-1`, `notes`) + ).resolves.toMatchObject({ + id: `notes`, + title: `Session notes`, + }) + }) +}) + describe(`ElectricAgentsManager attachments`, () => { it(`does not delete an existing stream when duplicate attachment creation conflicts`, async () => { const create = vi.fn().mockRejectedValue({ status: 409 }) diff --git a/packages/agents-server/test/electric-agents-routes.test.ts b/packages/agents-server/test/electric-agents-routes.test.ts index 020de60f8e..ce2f6a342d 100644 --- a/packages/agents-server/test/electric-agents-routes.test.ts +++ b/packages/agents-server/test/electric-agents-routes.test.ts @@ -30,7 +30,8 @@ async function routeResponse( key: `system:dev-local`, url: `/principal/system:dev-local`, }, - headers?: HeadersInit + headers?: HeadersInit, + ctxOverrides: Partial = {} ): Promise { const result = await globalRouter.fetch( createRequest(method, path, body, rawBody, headers), @@ -39,6 +40,7 @@ async function routeResponse( entityManager: manager, isShuttingDown: () => false, principal, + ...ctxOverrides, } as unknown as TenantContext ) expect(result).toBeInstanceOf(Response) @@ -280,6 +282,256 @@ describe(`ElectricAgentsRoutes attachment endpoints`, () => { }) }) +describe(`ElectricAgentsRoutes markdown document endpoints`, () => { + it(`routes document create and metadata read requests to the manager`, async () => { + const document = { + key: `document:notes`, + kind: `document`, + id: `notes`, + provider: `y-durable-streams`, + docId: `agents/chat/test/documents/notes`, + docPath: `agents/chat/test/documents/notes`, + streamPath: `/v1/yjs/test/docs/agents/chat/test/documents/notes`, + transportMimeType: `application/vnd.electric-agents.markdown-yjs`, + contentMimeType: `text/markdown`, + yTextName: `markdown`, + title: `Notes`, + createdAt: `2026-01-01T00:00:00.000Z`, + } + const manager = { + registry: { + getEntity: vi.fn().mockResolvedValue({ url: `/chat/test` }), + getEntityType: vi.fn(), + }, + createMarkdownDocument: vi + .fn() + .mockResolvedValue({ txid: `tx-create`, document }), + getMarkdownDocument: vi.fn().mockResolvedValue(document), + } as any + + const createResponse = await routeResponse( + manager, + `POST`, + `/_electric/entities/chat/test/documents`, + { id: `notes`, title: `Notes` } + ) + expect(createResponse.status).toBe(201) + expect(manager.createMarkdownDocument).toHaveBeenCalledWith(`/chat/test`, { + id: `notes`, + title: `Notes`, + createdBy: `/principal/system:dev-local`, + meta: undefined, + }) + + const readResponse = await routeResponse( + manager, + `GET`, + `/_electric/entities/chat/test/documents/notes` + ) + expect(await responseJson(readResponse)).toEqual({ + document, + }) + expect(manager.getMarkdownDocument).toHaveBeenCalledWith( + `/chat/test`, + `notes` + ) + }) + + it(`does not expose semantic markdown document write or edit endpoints`, async () => { + const manager = { + registry: { + getEntity: vi.fn().mockResolvedValue({ url: `/chat/test` }), + getEntityType: vi.fn(), + }, + } as any + + const putResponse = await routeResponse( + manager, + `PUT`, + `/_electric/entities/chat/test/documents/notes`, + { content: `# Ready` } + ) + expect(putResponse.status).toBe(404) + + const patchResponse = await routeResponse( + manager, + `PATCH`, + `/_electric/entities/chat/test/documents/notes`, + { oldString: `Ready`, newString: `Done`, replaceAll: true } + ) + expect(patchResponse.status).toBe(404) + }) + + it(`guards public Yjs document routes and forwards authorized document streams`, async () => { + const fetchSpy = vi + .spyOn(globalThis, `fetch`) + .mockResolvedValue(new Response(null, { status: 204 })) + const manager = { + registry: { + getEntity: vi.fn().mockResolvedValue({ + url: `/chat/test`, + created_by: `/principal/user:owner`, + }), + hasEntityPermission: vi.fn().mockResolvedValue(false), + pruneExpiredPermissionGrants: vi.fn(), + }, + isForkWorkLockedEntity: vi.fn().mockReturnValue(false), + } as any + + try { + const denied = await routeResponse( + manager, + `GET`, + `/v1/yjs/test/docs/agents/chat/test/documents/notes`, + undefined, + false, + { + kind: `user`, + id: `other`, + key: `user:other`, + url: `/principal/user:other`, + } + ) + expect(denied.status).toBe(401) + expect(fetchSpy).not.toHaveBeenCalled() + + const allowed = await routeResponse( + manager, + `GET`, + `/v1/yjs/test/docs/agents/chat/test/documents/notes?offset=-1`, + undefined, + false, + undefined, + undefined, + { + durableStreamsUrl: `http://durable.local`, + } + ) + expect(allowed.status).toBe(204) + expect(fetchSpy).toHaveBeenCalledOnce() + const [url, init] = fetchSpy.mock.calls[0]! + expect(String(url)).toContain( + `/yjs/test/docs/agents/chat/test/documents/notes/.updates?offset=-1` + ) + expect(init).toMatchObject({ method: `GET` }) + } finally { + fetchSpy.mockRestore() + } + }) + + it(`forwards Yjs awareness document routes to the awareness stream`, async () => { + const fetchSpy = vi + .spyOn(globalThis, `fetch`) + .mockResolvedValue(new Response(null, { status: 204 })) + const manager = { + registry: { + getEntity: vi.fn().mockResolvedValue({ + url: `/chat/test`, + created_by: `/principal/user:owner`, + }), + hasEntityPermission: vi.fn().mockResolvedValue(false), + pruneExpiredPermissionGrants: vi.fn(), + }, + isForkWorkLockedEntity: vi.fn().mockReturnValue(false), + } as any + + try { + const allowed = await routeResponse( + manager, + `GET`, + `/v1/yjs/test/docs/agents/chat/test/documents/notes?awareness=default&offset=-1`, + undefined, + false, + undefined, + undefined, + { + durableStreamsUrl: `http://durable.local`, + } + ) + expect(allowed.status).toBe(204) + expect(fetchSpy).toHaveBeenCalledOnce() + const [url, init] = fetchSpy.mock.calls[0]! + expect(String(url)).toContain( + `/yjs/test/docs/agents/chat/test/documents/notes/.awareness/default?offset=-1` + ) + expect(String(url)).not.toContain(`awareness=`) + expect(init).toMatchObject({ method: `GET` }) + } finally { + fetchSpy.mockRestore() + } + }) + + it(`guards private Yjs document streams and forwards authorized awareness streams`, async () => { + const fetchSpy = vi + .spyOn(globalThis, `fetch`) + .mockResolvedValue(new Response(null, { status: 204 })) + const endRead = vi.fn() + const manager = { + registry: { + getEntity: vi.fn().mockResolvedValue({ + url: `/chat/test`, + created_by: `/principal/user:owner`, + }), + hasEntityPermission: vi.fn().mockResolvedValue(false), + pruneExpiredPermissionGrants: vi.fn(), + }, + isForkWorkLockedEntity: vi.fn().mockReturnValue(false), + } as any + + try { + const denied = await routeResponse( + manager, + `GET`, + `/yjs/test/docs/agents/chat/test/documents/notes/.awareness/default`, + undefined, + false, + { + kind: `user`, + id: `other`, + key: `user:other`, + url: `/principal/user:other`, + }, + undefined, + { + durableStreamsUrl: `http://durable.local`, + entityBridgeManager: { + beginClientRead: vi.fn().mockResolvedValue(endRead), + touchByStreamPath: vi.fn(), + } as any, + } + ) + expect(denied.status).toBe(401) + expect(fetchSpy).not.toHaveBeenCalled() + + const allowed = await routeResponse( + manager, + `GET`, + `/yjs/test/docs/agents/chat/test/documents/notes/.awareness/default`, + undefined, + false, + undefined, + undefined, + { + durableStreamsUrl: `http://durable.local`, + entityBridgeManager: { + beginClientRead: vi.fn().mockResolvedValue(endRead), + touchByStreamPath: vi.fn(), + } as any, + } + ) + expect(allowed.status).toBe(204) + expect(fetchSpy).toHaveBeenCalledOnce() + const [url, init] = fetchSpy.mock.calls[0]! + expect(String(url)).toContain( + `/yjs/test/docs/agents/chat/test/documents/notes/.awareness/default` + ) + expect(init).toMatchObject({ method: `GET` }) + } finally { + fetchSpy.mockRestore() + } + }) +}) + describe(`ElectricAgentsRoutes cron stream ensure endpoint`, () => { it(`rejects cron ensure requests without an expression in the schema layer`, async () => { const manager = { diff --git a/packages/agents-server/test/electric-agents-status.test.ts b/packages/agents-server/test/electric-agents-status.test.ts index 277b83119b..3e3efee642 100644 --- a/packages/agents-server/test/electric-agents-status.test.ts +++ b/packages/agents-server/test/electric-agents-status.test.ts @@ -446,6 +446,25 @@ describe(`ElectricAgentsManager.forkSubtree`, () => { }, }, }, + { + type: `manifest`, + key: `document:notes`, + headers: { operation: `insert` }, + value: { + key: `document:notes`, + kind: `document`, + id: `notes`, + provider: `y-durable-streams`, + docId: `agents/manager/root/documents/notes`, + docPath: `agents/manager/root/documents/notes`, + streamPath: `/v1/yjs/default/docs/agents/manager/root/documents/notes`, + transportMimeType: `application/vnd.electric-agents.markdown-yjs`, + contentMimeType: `text/markdown`, + yTextName: `markdown`, + title: `Notes`, + createdAt: `2026-01-01T00:00:00.000Z`, + }, + }, ] }), exists: vi.fn().mockResolvedValue(false), @@ -501,13 +520,19 @@ describe(`ElectricAgentsManager.forkSubtree`, () => { expect.stringMatching(/^\/_electric\/shared-state\/board-fork-/), `/_electric/shared-state/board` ) + expect(streamClient.fork).toHaveBeenCalledWith( + `/yjs/default/docs/agents/manager/root-copy/documents/notes/.updates`, + `/yjs/default/docs/agents/manager/root/documents/notes/.updates` + ) - const manifestInserts = appendedEvents.filter( + const manifestWrites = appendedEvents.filter( (event) => event.type === `manifest` && - (event.headers as Record).operation === `insert` + [`insert`, `update`].includes( + String((event.headers as Record).operation) + ) ) - expect(manifestInserts).toEqual( + expect(manifestWrites).toEqual( expect.arrayContaining([ expect.objectContaining({ value: expect.objectContaining({ @@ -521,6 +546,15 @@ describe(`ElectricAgentsManager.forkSubtree`, () => { id: expect.stringMatching(/^board-fork-/), }), }), + expect.objectContaining({ + value: expect.objectContaining({ + kind: `document`, + id: `notes`, + docId: `agents/manager/root-copy/documents/notes`, + docPath: `agents/manager/root-copy/documents/notes`, + streamPath: `/v1/yjs/default/docs/agents/manager/root-copy/documents/notes`, + }), + }), ]) ) }) diff --git a/packages/agents/skills/markdown-docs.md b/packages/agents/skills/markdown-docs.md new file mode 100644 index 0000000000..73a4a32f5e --- /dev/null +++ b/packages/agents/skills/markdown-docs.md @@ -0,0 +1,122 @@ +--- +description: Create and edit collaborative markdown documents in the app workspace +whenToUse: User wants a markdown doc, notes, plan, draft, report, or document they can open and edit in the app UI +keywords: + - markdown doc + - collaborative document + - notes + - draft + - report + - plan + - workspace editor + - manifest +user-invocable: true +max: 9000 +--- + +# Markdown Docs + +Use this skill when the user wants a document that appears in the Electric +Agents UI and can be opened, edited, and watched live. + +## Core Rule + +Collaborative markdown docs are not filesystem files. + +- Use `create_markdown_doc`, `set_markdown_doc_cursor`, + `insert_markdown_doc`, `replace_markdown_doc_range`, `read_markdown_doc`, + `write_markdown_doc`, and `edit_markdown_doc` for docs the user should open + in the workspace UI. +- Use filesystem `write`/`edit` only when the user asks for an actual file path + in the workspace or repo, such as `docs/foo.md`, `README.md`, or + `/tmp/report.md`. + +## When To Create A Collaborative Doc + +Use `create_markdown_doc` when the user says things like: + +- "make a markdown doc" +- "create a doc" +- "write some notes" +- "draft a plan" +- "make a report I can edit" +- "add this to the manifest" +- "create a document I can open" +- "put this in a doc" + +If the user says "file", "repo", "workspace", or gives a path, ask one short +clarifying question if the destination is ambiguous. + +## Create Workflow + +1. Choose a concise title. +2. Use `create_markdown_doc`. +3. Include initial markdown content if the user supplied enough detail. +4. After creation, tell the user the document is available from this entity's + manifest or timeline and can be opened in the markdown editor. + +Example tool call: + +```json +{ + "title": "Launch Plan", + "content": "# Launch Plan\n\n## Goals\n\n- ...\n" +} +``` + +Do not also write a `.md` file unless the user explicitly asked for a filesystem +copy. + +## Edit Workflow + +For small edits: + +1. Use `read_markdown_doc` first. +2. Use `edit_markdown_doc` with an exact `old_string`. +3. If the target text appears multiple times, make `old_string` more specific or + set `replace_all` only when replacing every occurrence is clearly intended. + +For replacing a section with new long content that should appear live: + +1. Use `read_markdown_doc` if you need to inspect or disambiguate the target. +2. Use `replace_markdown_doc_range` with a unique `old_string`, or with + `old_string` plus `occurrence` for repeated text. +3. For exact offsets, use `index` plus `length` instead of `old_string`. +4. Put the range selector before `content` in the tool arguments so the range is + deleted once and replacement content can stream into that Yjs-relative + position. + +The markdown tools materialize the collaborative Yjs document from its durable +stream during the wake. `write_markdown_doc`, `edit_markdown_doc`, +`insert_markdown_doc`, and `replace_markdown_doc_range` append binary Yjs +updates to that stream; do not write markdown documents to the local filesystem +unless the user explicitly asks for a filesystem file. + +For broad rewrites: + +1. Use `read_markdown_doc` first unless you just created or wrote the doc in the + same wake. +2. Use `write_markdown_doc` with the full replacement markdown. + +For adding new long content to an existing doc: + +1. Use `read_markdown_doc` if you need to inspect the target location. +2. Use `set_markdown_doc_cursor` with `index`, `before`, or `after` when the + insertion belongs at a specific location. +3. Use `insert_markdown_doc`. +4. Pass `id` and optional `index` before `content` in the tool arguments. If + `index` is omitted, the saved Yjs-relative cursor is used; if no cursor is + set, the content is appended to the current document. + +Both write and edit tool results include diffs. Use those diffs to summarize +what changed. + +## Response Style + +After creating a doc, keep the response short: + +- State the title. +- State that it is available in the manifest/timeline. +- Mention any useful next action, such as "open it to edit collaboratively". + +Do not paste the entire document back into chat unless the user asks. diff --git a/packages/agents/src/agents/horton.ts b/packages/agents/src/agents/horton.ts index 851d004bbe..e2b663c80e 100644 --- a/packages/agents/src/agents/horton.ts +++ b/packages/agents/src/agents/horton.ts @@ -218,6 +218,7 @@ export function buildHortonSystemPrompt( hasDocsSupport?: boolean hasWebhookSourceTools?: boolean hasScheduleTools?: boolean + hasMarkdownDocumentTools?: boolean hasSkills?: boolean docsUrl?: string modelProvider?: string @@ -235,9 +236,15 @@ export function buildHortonSystemPrompt( const scheduleTools = opts.hasScheduleTools ? `\n- upsert_cron_schedule: create or update a recurring cron wake for yourself. Always include payload with the concrete instruction/message you should receive when the cron fires.\n- delete_schedule: delete one of your cron or future-send schedules by stable id\n- list_schedules: list your manifest-backed cron and future-send schedules` : `` + const markdownDocumentTools = opts.hasMarkdownDocumentTools + ? `\n- create_markdown_doc: create a collaborative markdown document that appears in this entity's manifest and opens in the workspace editor\n- set_markdown_doc_cursor: choose a stateful Yjs-relative insertion cursor for a collaborative markdown document\n- insert_markdown_doc: stream or insert markdown into an existing collaborative document so open editors can watch content appear\n- replace_markdown_doc_range: delete a range and stream replacement markdown into that location\n- read_markdown_doc: read a collaborative markdown document\n- write_markdown_doc: replace a collaborative markdown document's full content\n- edit_markdown_doc: targeted string replacement in a collaborative markdown document` + : `` const skillsTools = opts.hasSkills ? `\n- use_skill: load a skill (knowledge, instructions, or a tutorial) into your context to help with the user's request\n- remove_skill: unload a skill from context when you're done with it` : `` + const markdownDocumentGuidance = opts.hasMarkdownDocumentTools + ? `\n# Collaborative Markdown Docs\n- If the user asks you to create a markdown doc, notes, draft, brief, plan, report, or any document they should open/edit in the app UI, use create_markdown_doc. Do not use filesystem write unless they ask for a file path or repo/workspace file.\n- For larger document workflows, load the markdown-docs skill first with use_skill, then use the markdown document tools.\n- Use set_markdown_doc_cursor before insert_markdown_doc when inserting at a specific location; insert_markdown_doc can then stream content at that Yjs-relative cursor.\n- Use replace_markdown_doc_range when replacing existing prose with new content that should visibly stream into the deleted range.\n- When spawning a worker to read or edit a collaborative markdown doc, include the doc id in spawn_worker's markdownDocIds and include the specific markdown document tools that worker needs.\n- After creating a collaborative doc, mention that it is available from this entity's manifest/timeline.` + : `` const docsGuidance = opts.hasDocsSupport ? `\n- For ANY question about Electric Agents or this framework, ALWAYS use search_electric_agents_docs FIRST. Do not use web_search or fetch_url for Electric Agents topics unless the docs search returns no useful results.\n- The search tool returns chunk content directly — you do not need to read the source files.\n- Use repo read/bash tools only for non-doc files or when you need to inspect exact implementation code in the workspace.` : `` @@ -303,12 +310,12 @@ When a user opens with a greeting ("hi", "hello", "hey", etc.) or a broad statem - observe_pg_sync: observe an Electric Postgres sync stream and wake on matching changes (see "Observing Postgres tables") - unobserve_pg_sync: stop being woken by a pg-sync stream you previously observed (see "Observing Postgres tables") - send: send a message to an Electric Agent/entity. To schedule future work for yourself, call send with self: true and afterMs. -${webhookSourceTools}${titleTool}${scheduleTools}${docsTools}${skillsTools} +${webhookSourceTools}${titleTool}${scheduleTools}${markdownDocumentTools}${docsTools}${skillsTools} # Working with files - Prefer edit over write when modifying existing files. - Use absolute paths or paths relative to the current working directory. -${modelGuidance}${docsGuidance}${skillsGuidance}${onboardingGuidance}${docsUrlGuidance} +${markdownDocumentGuidance}${modelGuidance}${docsGuidance}${skillsGuidance}${onboardingGuidance}${docsUrlGuidance} # Observing Postgres tables observe_pg_sync subscribes you to row changes in a Postgres table via an Electric shape stream: @@ -682,6 +689,9 @@ function createAssistantHandler(options: { const hasScheduleTools = tools.some( (tool) => getToolName(tool) === `upsert_cron_schedule` ) + const hasMarkdownDocumentTools = tools.some( + (tool) => getToolName(tool) === `create_markdown_doc` + ) const titlePromise = !ctx.tags.title ? (async () => { @@ -859,6 +869,7 @@ function createAssistantHandler(options: { hasWebhookSourceTools, hasScheduleTools, ...(activeGoalPromptInfo && { activeGoal: activeGoalPromptInfo }), + hasMarkdownDocumentTools, }), ...modelConfig, // mcp.tools() inserts sentinel objects that the runtime's diff --git a/packages/agents/src/agents/worker.ts b/packages/agents/src/agents/worker.ts index 1b25201d66..ca14f90bf3 100644 --- a/packages/agents/src/agents/worker.ts +++ b/packages/agents/src/agents/worker.ts @@ -10,7 +10,11 @@ import { createSendTool, } from '@electric-ax/agents-runtime/tools' import type { Sandbox } from '@electric-ax/agents-runtime/sandbox' -import { WORKER_TOOL_NAMES, createSpawnWorkerTool } from '../tools/spawn-worker' +import { + MARKDOWN_WORKER_TOOL_NAMES, + WORKER_TOOL_NAMES, + createSpawnWorkerTool, +} from '../tools/spawn-worker' import { REASONING_EFFORT_VALUES, resolveBuiltinModelConfig, @@ -48,6 +52,21 @@ function isRecord(value: unknown): value is Record { return value !== null && typeof value === `object` } +function isMarkdownWorkerToolName( + value: WorkerToolName +): value is (typeof MARKDOWN_WORKER_TOOL_NAMES)[number] { + return (MARKDOWN_WORKER_TOOL_NAMES as ReadonlyArray).includes(value) +} + +function electricTool( + ctx: HandlerContext, + name: string +): AgentTool | undefined { + return ctx.electricTools.find((tool) => tool.name === name) as + | AgentTool + | undefined +} + function parseWorkerArgs(value: Readonly>): WorkerArgs { if ( typeof value.systemPrompt !== `string` || @@ -155,6 +174,12 @@ function buildToolsForWorker( case `send`: out.push(createSendTool(ctx.send, { selfEntityUrl: ctx.entityUrl })) break + default: + if (isMarkdownWorkerToolName(name)) { + const tool = electricTool(ctx, name) + if (tool) out.push(tool) + } + break } } return out diff --git a/packages/agents/src/bootstrap.ts b/packages/agents/src/bootstrap.ts index 69e3a362a4..9b110cf32a 100644 --- a/packages/agents/src/bootstrap.ts +++ b/packages/agents/src/bootstrap.ts @@ -11,6 +11,7 @@ import { import { createWebhookSourceTools, createScheduleTools, + createMarkdownDocumentTools, } from '@electric-ax/agents-runtime/tools' import { chooseDefaultSandbox, @@ -148,6 +149,7 @@ export function createBuiltinElectricTools( const builtinTools = [ ...createWebhookSourceTools(context), ...createScheduleTools({ ...context, db: context.db as any }), + ...createMarkdownDocumentTools(context), ] const customTools = custom ? await custom(context) : [] return dedupeToolsByName([...builtinTools, ...customTools]) diff --git a/packages/agents/src/tools/spawn-worker.ts b/packages/agents/src/tools/spawn-worker.ts index 57fddd9d5a..b4117b4853 100644 --- a/packages/agents/src/tools/spawn-worker.ts +++ b/packages/agents/src/tools/spawn-worker.ts @@ -3,7 +3,20 @@ import { nanoid } from 'nanoid' import { serverLog } from '../log' import type { BuiltinAgentModelConfig } from '../model-catalog' import type { AgentTool } from '@mariozechner/pi-agent-core' -import type { HandlerContext } from '@electric-ax/agents-runtime' +import type { + HandlerContext, + ManifestDocumentEntry, +} from '@electric-ax/agents-runtime' + +export const MARKDOWN_WORKER_TOOL_NAMES = [ + `create_markdown_doc`, + `set_markdown_doc_cursor`, + `insert_markdown_doc`, + `replace_markdown_doc_range`, + `read_markdown_doc`, + `write_markdown_doc`, + `edit_markdown_doc`, +] as const export const WORKER_TOOL_NAMES = [ `bash`, @@ -14,10 +27,66 @@ export const WORKER_TOOL_NAMES = [ `fetch_url`, `spawn_worker`, `send`, + ...MARKDOWN_WORKER_TOOL_NAMES, ] as const export type WorkerToolName = (typeof WORKER_TOOL_NAMES)[number] +function isManifestDocumentEntry( + value: unknown +): value is ManifestDocumentEntry { + if (!value || typeof value !== `object`) return false + const entry = value as Partial + return ( + entry.kind === `document` && + typeof entry.id === `string` && + entry.provider === `y-durable-streams` && + typeof entry.docPath === `string` && + typeof entry.streamPath === `string` && + entry.transportMimeType === + `application/vnd.electric-agents.markdown-yjs` && + entry.contentMimeType === `text/markdown` && + entry.yTextName === `markdown` && + typeof entry.title === `string` + ) +} + +function manifestMarkdownDocuments( + ctx: HandlerContext +): Array { + const manifests = ctx.db.collections.manifests?.toArray as + | Array + | undefined + const injectedDocs = Array.isArray(ctx.args?.markdownDocs) + ? ctx.args.markdownDocs.filter(isManifestDocumentEntry) + : [] + return [ + ...(manifests?.filter(isManifestDocumentEntry) ?? []), + ...injectedDocs, + ] +} + +function selectedMarkdownDocuments( + ctx: HandlerContext, + ids: ReadonlyArray | undefined +): { documents: Array; missing: Array } { + if (!ids || ids.length === 0) return { documents: [], missing: [] } + const docsById = new Map( + manifestMarkdownDocuments(ctx).map((document) => [document.id, document]) + ) + const documents: Array = [] + const missing: Array = [] + for (const id of [...new Set(ids)]) { + const document = docsById.get(id) + if (document) { + documents.push(document) + } else { + missing.push(id) + } + } + return { documents, missing } +} + export function createSpawnWorkerTool( ctx: HandlerContext, modelConfig?: BuiltinAgentModelConfig @@ -39,13 +108,20 @@ export function createSpawnWorkerTool( initialMessage: Type.String({ description: `First user message sent to the worker. Be concrete: include file paths, line numbers, and the form of answer you want back. This is what kicks off its run — without it the worker will idle. Describe the concrete task to perform and what form of message you want back.`, }), + markdownDocIds: Type.Optional( + Type.Array(Type.String(), { + description: `Optional collaborative markdown document ids from this entity's manifest to make available to the worker. Include the matching markdown tools in tools when the worker should read or edit them.`, + }) + ), }), execute: async (_toolCallId, params) => { - const { systemPrompt, tools, initialMessage } = params as { - systemPrompt: string - tools: Array - initialMessage: string - } + const { systemPrompt, tools, initialMessage, markdownDocIds } = + params as { + systemPrompt: string + tools: Array + initialMessage: string + markdownDocIds?: Array + } if (!Array.isArray(tools) || tools.length === 0) { return { content: [ @@ -68,6 +144,22 @@ export function createSpawnWorkerTool( details: { spawned: false }, } } + const { documents: markdownDocs, missing: missingMarkdownDocIds } = + selectedMarkdownDocuments(ctx, markdownDocIds) + if (missingMarkdownDocIds.length > 0) { + return { + content: [ + { + type: `text` as const, + text: `Error: markdown document ids not found in this entity's manifest: ${missingMarkdownDocIds.join(`, `)}.`, + }, + ], + details: { + spawned: false, + missingMarkdownDocIds, + }, + } + } const id = nanoid(10) const workerModelArgs = modelConfig @@ -83,7 +175,12 @@ export function createSpawnWorkerTool( const handle = await ctx.spawn( `worker`, id, - { systemPrompt, tools, ...workerModelArgs }, + { + systemPrompt, + tools, + ...(markdownDocs.length > 0 ? { markdownDocs } : {}), + ...workerModelArgs, + }, { initialMessage, wake: { on: `runFinished`, includeResponse: true }, diff --git a/packages/agents/test/horton-system-prompt.test.ts b/packages/agents/test/horton-system-prompt.test.ts index e901db8d55..c2daa12d8c 100644 --- a/packages/agents/test/horton-system-prompt.test.ts +++ b/packages/agents/test/horton-system-prompt.test.ts @@ -39,6 +39,27 @@ describe(`buildHortonSystemPrompt`, () => { expect(prompt).not.toContain(`subscribe_webhook_source`) }) + it(`describes collaborative markdown docs when document tools are available`, () => { + const prompt = buildHortonSystemPrompt(`/tmp/test`, { + hasMarkdownDocumentTools: true, + hasSkills: true, + }) + + expect(prompt).toContain(`create_markdown_doc`) + expect(prompt).toContain(`set_markdown_doc_cursor`) + expect(prompt).toContain(`insert_markdown_doc`) + expect(prompt).toContain(`replace_markdown_doc_range`) + expect(prompt).toContain(`Collaborative Markdown Docs`) + expect(prompt).toContain(`Do not use filesystem write`) + expect(prompt).toContain(`markdown-docs skill`) + }) + + it(`omits collaborative markdown docs when document tools are unavailable`, () => { + const prompt = buildHortonSystemPrompt(`/tmp/test`) + expect(prompt).not.toContain(`create_markdown_doc`) + expect(prompt).not.toContain(`Collaborative Markdown Docs`) + }) + it(`includes docs URL guidance alongside local docs support`, () => { const prompt = buildHortonSystemPrompt(`/tmp/test`, { hasDocsSupport: true, diff --git a/packages/agents/test/horton-tool-composition.test.ts b/packages/agents/test/horton-tool-composition.test.ts index ae255e87a8..d7a3ac1a7c 100644 --- a/packages/agents/test/horton-tool-composition.test.ts +++ b/packages/agents/test/horton-tool-composition.test.ts @@ -88,9 +88,24 @@ async function captureToolset(args: Record = {}) { } function createElectricToolsContext() { + const document = { + key: `document:notes`, + kind: `document`, + id: `notes`, + provider: `y-durable-streams`, + docId: `agents/horton/smoke/documents/notes`, + docPath: `agents/horton/smoke/documents/notes`, + streamPath: `/v1/yjs/default/docs/agents/horton/smoke/documents/notes`, + transportMimeType: `application/vnd.electric-agents.markdown-yjs`, + contentMimeType: `text/markdown`, + yTextName: `markdown`, + title: `Notes`, + createdAt: new Date(0).toISOString(), + } return { entityUrl: `/horton/smoke/main`, entityType: `horton`, + principal: { url: `/principal/agent:horton`, kind: `agent` }, args: {}, db: { collections: { manifests: { toArray: [] } }, @@ -121,6 +136,21 @@ function createElectricToolsContext() { unsubscribeFromWebhookSource: vi.fn(async () => ({ txid: `tx-unsubscribe`, })), + createMarkdownDocument: vi.fn(async () => ({ + txid: `tx-create-doc`, + document, + })), + getMarkdownDocumentConnection: vi.fn(async () => ({ + baseUrl: `http://test.local/v1/yjs/default`, + docId: document.docId, + headers: {}, + })), + readMarkdownDocumentStream: vi.fn(async () => ({ + bytes: new Uint8Array(), + })), + appendMarkdownDocumentUpdate: vi.fn(async () => ({})), + appendMarkdownDocumentAwareness: vi.fn(async () => ({})), + registerCleanup: vi.fn(), } as any } @@ -226,7 +256,7 @@ describe(`horton tool composition`, () => { ) }) - it(`adds webhook source and schedule tools through the built-in electric tool factory`, async () => { + it(`adds webhook source, schedule, and markdown document tools through the built-in electric tool factory`, async () => { const tools = await createBuiltinElectricTools()( createElectricToolsContext() ) @@ -241,6 +271,13 @@ describe(`horton tool composition`, () => { `upsert_cron_schedule`, `delete_schedule`, `list_schedules`, + `create_markdown_doc`, + `set_markdown_doc_cursor`, + `insert_markdown_doc`, + `replace_markdown_doc_range`, + `read_markdown_doc`, + `write_markdown_doc`, + `edit_markdown_doc`, ]) ) expect( @@ -253,9 +290,12 @@ describe(`horton tool composition`, () => { tools.find((tool) => tool.name === `list_webhook_source_subscriptions`) ?.description ).not.toContain(`manifest-backed`) + expect( + tools.find((tool) => tool.name === `create_markdown_doc`)?.description + ).toContain(`not a filesystem file`) }) - it(`includes webhook source and schedule electric tools in Horton and describes them in the prompt`, async () => { + it(`includes electric tools in Horton and describes them in the prompt`, async () => { const electricTools = await createBuiltinElectricTools()( createElectricToolsContext() ) @@ -274,6 +314,15 @@ describe(`horton tool composition`, () => { expect(cfg.systemPrompt).toContain(`upsert_cron_schedule`) expect(cfg.systemPrompt).toContain(`delete_schedule`) expect(cfg.systemPrompt).toContain(`list_schedules`) + expect(names).toContain(`create_markdown_doc`) + expect(names).toContain(`set_markdown_doc_cursor`) + expect(names).toContain(`replace_markdown_doc_range`) + expect(names).toContain(`insert_markdown_doc`) + expect(names).toContain(`edit_markdown_doc`) + expect(cfg.systemPrompt).toContain(`create_markdown_doc`) + expect(cfg.systemPrompt).toContain(`set_markdown_doc_cursor`) + expect(cfg.systemPrompt).toContain(`insert_markdown_doc`) + expect(cfg.systemPrompt).toContain(`Collaborative Markdown Docs`) }) it(`includes the default built-in toolset`, async () => { diff --git a/packages/agents/test/spawn-worker-tool.test.ts b/packages/agents/test/spawn-worker-tool.test.ts index 58bfbfebbe..4e543115d8 100644 --- a/packages/agents/test/spawn-worker-tool.test.ts +++ b/packages/agents/test/spawn-worker-tool.test.ts @@ -1,5 +1,23 @@ import { describe, expect, it, vi } from 'vitest' -import { createSpawnWorkerTool } from '../src/tools/spawn-worker' +import { + WORKER_TOOL_NAMES, + createSpawnWorkerTool, +} from '../src/tools/spawn-worker' + +const manifestDocument = { + key: `document:notes`, + kind: `document`, + id: `notes`, + provider: `y-durable-streams`, + docId: `agents/chat/session/documents/notes`, + docPath: `agents/chat/session/documents/notes`, + streamPath: `/v1/yjs/default/docs/agents/chat/session/documents/notes`, + transportMimeType: `application/vnd.electric-agents.markdown-yjs`, + contentMimeType: `text/markdown`, + yTextName: `markdown`, + title: `Notes`, + createdAt: `2026-06-07T00:00:00.000Z`, +} as const describe(`spawn_worker tool`, () => { it(`spawns a worker entity with runFinished + includeResponse and forwards the initial message`, async () => { @@ -67,6 +85,60 @@ describe(`spawn_worker tool`, () => { }) }) + it(`passes selected collaborative markdown document refs to the spawned worker`, async () => { + const spawn = vi.fn(async (type, id) => ({ + entityUrl: `/${type}/${id}`, + writeToken: `tok`, + txid: 1, + })) + const ctx = { + spawn, + db: { collections: { manifests: { toArray: [manifestDocument] } } }, + } as any + const tool = createSpawnWorkerTool(ctx) + + await tool.execute(`call-doc`, { + systemPrompt: `Edit the shared doc.`, + tools: [`read_markdown_doc`, `insert_markdown_doc`], + initialMessage: `Read notes and append a summary.`, + markdownDocIds: [`notes`], + }) + + const [, , args] = spawn.mock.calls[0]! as Array + expect(args).toMatchObject({ + systemPrompt: `Edit the shared doc.`, + tools: [`read_markdown_doc`, `insert_markdown_doc`], + markdownDocs: [manifestDocument], + }) + }) + + it(`rejects unknown collaborative markdown document refs`, async () => { + const spawn = vi.fn() + const ctx = { + spawn, + db: { collections: { manifests: { toArray: [manifestDocument] } } }, + } as any + const tool = createSpawnWorkerTool(ctx) + + const result = await tool.execute(`call-doc-missing`, { + systemPrompt: `Edit the shared doc.`, + tools: [`read_markdown_doc`], + initialMessage: `Read notes.`, + markdownDocIds: [`missing`], + }) + + expect((result.content[0] as { text: string }).text).toMatch( + /not found in this entity's manifest/ + ) + expect(spawn).not.toHaveBeenCalled() + }) + + it(`allows workers to request collaborative markdown document tools`, () => { + expect(WORKER_TOOL_NAMES).toContain(`read_markdown_doc`) + expect(WORKER_TOOL_NAMES).toContain(`insert_markdown_doc`) + expect(WORKER_TOOL_NAMES).toContain(`replace_markdown_doc_range`) + }) + it(`rejects when tools is empty`, async () => { const spawn = vi.fn() const ctx = { spawn } as any diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a71eea6119..d2e9eb140e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -340,7 +340,7 @@ importers: version: 0.2.17(@electric-sql/pglite@0.2.17)(react@18.3.1) '@electric-sql/pglite-repl': specifier: ^0.2.17 - version: 0.2.17(@babel/runtime@7.29.7)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.5.0)(@codemirror/theme-one-dark@6.1.2)(@electric-sql/pglite@0.2.17)(@lezer/common@1.2.3)(codemirror@6.0.1(@lezer/common@1.2.3)) + version: 0.2.17(@babel/runtime@7.29.7)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@electric-sql/pglite@0.2.17)(@lezer/common@1.5.2)(codemirror@6.0.1(@lezer/common@1.5.2)) '@electric-sql/pglite-sync': specifier: ^0.2.20 version: 0.2.20(@electric-sql/pglite@0.2.17) @@ -1822,6 +1822,9 @@ importers: '@durable-streams/state': specifier: ^0.3.1 version: 0.3.1(@tanstack/db@0.6.7(typescript@5.9.3)) + '@durable-streams/y-durable-streams': + specifier: 0.2.7 + version: 0.2.7(lib0@0.2.99)(y-protocols@1.0.6(yjs@13.6.26))(yjs@13.6.26) '@electric-ax/agents-mcp': specifier: workspace:* version: link:../agents-mcp @@ -1858,6 +1861,9 @@ importers: jsdom: specifier: ^28.1.0 version: 28.1.0(@noble/hashes@2.0.1) + lib0: + specifier: ^0.2.99 + version: 0.2.99 pino: specifier: ^10.3.1 version: 10.3.1 @@ -1876,6 +1882,12 @@ importers: xstate: specifier: ^5.32.0 version: 5.32.0 + y-protocols: + specifier: ^1.0.6 + version: 1.0.6(yjs@13.6.26) + yjs: + specifier: ^13.6.26 + version: 13.6.26 zod: specifier: ^4.3.6 version: 4.3.6 @@ -1964,6 +1976,9 @@ importers: itty-router: specifier: ^5.0.23 version: 5.0.23 + lib0: + specifier: ^0.2.99 + version: 0.2.99 lmdb: specifier: ^3.5.1 version: 3.5.4 @@ -1979,6 +1994,12 @@ importers: undici: specifier: ^7.24.7 version: 7.25.0 + y-protocols: + specifier: ^1.0.6 + version: 1.0.6(yjs@13.6.26) + yjs: + specifier: ^13.6.26 + version: 13.6.26 devDependencies: '@electric-ax/agents': specifier: workspace:* @@ -2047,12 +2068,24 @@ importers: '@base-ui/react': specifier: ^1.4.1 version: 1.4.1(@types/react@19.2.14)(date-fns@4.1.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + '@codemirror/lang-markdown': + specifier: ^6.5.0 + version: 6.5.0 + '@codemirror/state': + specifier: ^6.6.0 + version: 6.6.0 + '@codemirror/view': + specifier: ^6.43.0 + version: 6.43.0 '@durable-streams/client': specifier: ^0.2.6 version: 0.2.6 '@durable-streams/state': specifier: ^0.3.1 version: 0.3.1(@tanstack/db@0.6.7(typescript@5.9.3)) + '@durable-streams/y-durable-streams': + specifier: 0.2.7 + version: 0.2.7(lib0@0.2.99)(y-protocols@1.0.6(yjs@13.6.26))(yjs@13.6.26) '@electric-ax/agents-runtime': specifier: workspace:* version: link:../agents-runtime @@ -2080,12 +2113,18 @@ importers: '@tanstack/react-virtual': specifier: ^3.13.23 version: 3.13.24(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + codemirror: + specifier: ^6.0.1 + version: 6.0.1(@lezer/common@1.5.2) fractional-indexing: specifier: ^3.2.0 version: 3.2.0 katex: specifier: ^0.16.45 version: 0.16.45 + lib0: + specifier: ^0.2.99 + version: 0.2.99 lucide-react: specifier: ^0.561.0 version: 0.561.0(react@19.1.0) @@ -2125,6 +2164,15 @@ importers: streamdown: specifier: ^2.5.0 version: 2.5.0(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + y-codemirror.next: + specifier: 0.3.5 + version: 0.3.5(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(yjs@13.6.26) + y-protocols: + specifier: ^1.0.6 + version: 1.0.6(yjs@13.6.26) + yjs: + specifier: ^13.6.26 + version: 13.6.26 zod: specifier: ^3.25.76 version: 3.25.76 @@ -4475,9 +4523,18 @@ packages: '@codemirror/commands@6.7.1': resolution: {integrity: sha512-llTrboQYw5H4THfhN4U3qCnSZ1SOJ60ohhz+SzU0ADGtwlc533DtklQP0vSFaQuCPDn3BPpOd1GbbnUtwNjsrw==} + '@codemirror/lang-css@6.3.1': + resolution: {integrity: sha512-kr5fwBGiGtmz6l0LSJIbno9QrifNMUusivHbnA1H6Dmqy4HZFte3UAICix1VuKo0lMPKQr2rqB+0BkKi/S3Ejg==} + + '@codemirror/lang-html@6.4.11': + resolution: {integrity: sha512-9NsXp7Nwp891pQchI7gPdTwBuSuT3K65NGTHWHNJ55HjYcHLllr0rbIZNdOzas9ztc1EUVBlHou85FFZS4BNnw==} + '@codemirror/lang-javascript@6.2.2': resolution: {integrity: sha512-VGQfY+FCc285AhWuwjYxQyUQcYurWlxdKYT4bqwr3Twnd5wP5WSeu52t4tvvuWmljT4EmgEgZCqSieokhtY8hg==} + '@codemirror/lang-markdown@6.5.0': + resolution: {integrity: sha512-0K40bZ35jpHya6FriukbgaleaqzBLZfOh7HuzqbMxBXkbYMJDxfF39c23xOgxFezR+3G+tR2/Mup+Xk865OMvw==} + '@codemirror/lang-sql@6.8.0': resolution: {integrity: sha512-aGLmY4OwGqN3TdSx3h6QeA1NrvaYtF7kkoWR/+W7/JzB0gQtJ+VJxewlnE3+VImhA4WVlhmkJr109PefOOhjLg==} @@ -4505,9 +4562,6 @@ packages: '@codemirror/view@6.35.2': resolution: {integrity: sha512-u04R04XFCYCNaHoNRr37WUUAfnxKPwPdqV+370NiO6i85qB1J/qCD/WbbMJsyJfRWhXIJXAe2BG/oTzAggqv4A==} - '@codemirror/view@6.41.1': - resolution: {integrity: sha512-ToDnWKbBnke+ZLrP6vgTTDScGi5H37YYuZGniQaBzxMVdtCxMrslsmtnOvbPZk4RX9bvkQqnWR/WS/35tJA0qg==} - '@codemirror/view@6.43.0': resolution: {integrity: sha512-V7ZCLQO3Jus9hzh2jVCCPW3mO4IBMr43O37PqSUYautJSnnJF41YlgLw21x0fLJTYvJ+Vkm6Gp+qKGH9pltgXA==} @@ -4690,6 +4744,15 @@ packages: '@tanstack/db': optional: true + '@durable-streams/y-durable-streams@0.2.7': + resolution: {integrity: sha512-AxHQ0PkW4S4+vyPpyWCUy0IRFOp5c+SmprpmORairZEGV4xeF9EdMIhMm/awgqFgddkGdIhlswvuRDTXIZdv6g==} + engines: {node: '>=18.0.0'} + hasBin: true + peerDependencies: + lib0: ^0.2.0 + y-protocols: ^1.0.0 + yjs: ^13.0.0 + '@ecies/ciphers@0.2.5': resolution: {integrity: sha512-GalEZH4JgOMHYYcYmVqnFirFsjZHeoGMDt9IxEnM9F7GRUUyUksJ7Ou53L83WHJq3RWKD3AcBpo0iQh0oMpf8A==} engines: {bun: '>=1', deno: '>=2', node: '>=16'} @@ -6737,12 +6800,18 @@ packages: '@lezer/common@1.5.2': resolution: {integrity: sha512-sxQE460fPZyU3sdc8lafxiPwJHBzZRy/udNFynGQky1SePYBdhkBl1kOagA9uT3pxR8K09bOrmTUqA9wb/PjSQ==} + '@lezer/css@1.3.3': + resolution: {integrity: sha512-RzBo8r+/6QJeow7aPHIpGVIH59xTcJXp399820gZoMo9noQDRVpJLheIBUicYwKcsbOYoBRoLZlf2720dG/4Tg==} + '@lezer/highlight@1.2.1': resolution: {integrity: sha512-Z5duk4RN/3zuVO7Jq0pGLJ3qynpxUVsh7IbUbGj88+uV2ApSAn6kWg2au3iJb+0Zi7kKtqffIESgNcRXWZWmSA==} '@lezer/highlight@1.2.3': resolution: {integrity: sha512-qXdH7UqTvGfdVBINrgKhDsVTJTxactNNxLk7+UMwZhU13lMHaOBlJe9Vqp907ya56Y3+ed2tlqzys7jDkTmW0g==} + '@lezer/html@1.3.13': + resolution: {integrity: sha512-oI7n6NJml729m7pjm9lvLvmXbdoMoi2f+1pwSDJkl9d68zGr7a9Btz8NdHTGQZtW2DA25ybeuv/SyDb9D5tseg==} + '@lezer/javascript@1.4.21': resolution: {integrity: sha512-lL+1fcuxWYPURMM/oFZLEDm0XuLN128QPV+VuGtKpeaOGdcl9F2LYC3nh1S9LkPqx9M0mndZFdXCipNAZpzIkQ==} @@ -6752,6 +6821,9 @@ packages: '@lezer/lr@1.4.2': resolution: {integrity: sha512-pu0K1jCIdnQ12aWNaAVU5bzi7Bd1w54J3ECgANPmYLtQKP0HBj2cE/5coBD66MT10xbtIuUr7tg0Shbsvk0mDA==} + '@lezer/markdown@1.6.4': + resolution: {integrity: sha512-N0SxazMj4k65DBfaf1azqtMZd6u7MqluP84/NZnB/io8Td9aleFmAhz9hcbvSfsxT5tdYlJ5qgv5aMJGY4zEtA==} + '@lmdb/lmdb-darwin-arm64@3.5.4': resolution: {integrity: sha512-Kk4Kz3iyu1QiLsLZBS9Af1eSKUC8VR2T+/jyE2iAyuGw2VwK08pp5iTbZnXn6sWu0LogO/RFktMxOjiDA2sS3w==} cpu: [arm64] @@ -22576,7 +22648,7 @@ snapshots: '@babel/helper-annotate-as-pure@7.27.3': dependencies: - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 '@babel/helper-annotate-as-pure@7.29.7': dependencies: @@ -24080,7 +24152,7 @@ snapshots: '@babel/helper-module-imports': 7.28.6 '@babel/helper-plugin-utils': 7.28.6 '@babel/plugin-syntax-jsx': 7.27.1(@babel/core@7.28.0) - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color @@ -24091,7 +24163,7 @@ snapshots: '@babel/helper-module-imports': 7.28.6 '@babel/helper-plugin-utils': 7.28.6 '@babel/plugin-syntax-jsx': 7.27.1(@babel/core@7.29.0) - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color @@ -24765,68 +24837,89 @@ snapshots: '@chevrotain/utils@12.0.0': {} - '@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3)': + '@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.2.3)': dependencies: '@codemirror/language': 6.10.6 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 '@lezer/common': 1.2.3 - '@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.5.2)': + '@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2)': dependencies: '@codemirror/language': 6.10.6 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 '@lezer/common': 1.5.2 - '@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.41.1)(@lezer/common@1.2.3)': + '@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.2.3)': dependencies: - '@codemirror/language': 6.10.6 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.41.1 + '@codemirror/language': 6.12.3 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 '@lezer/common': 1.2.3 - '@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)(@lezer/common@1.5.2)': + '@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2)': dependencies: '@codemirror/language': 6.12.3 '@codemirror/state': 6.6.0 - '@codemirror/view': 6.41.1 + '@codemirror/view': 6.43.0 '@lezer/common': 1.5.2 '@codemirror/commands@6.7.1': dependencies: - '@codemirror/language': 6.10.6 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 - '@lezer/common': 1.2.3 + '@codemirror/language': 6.12.3 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 + '@lezer/common': 1.5.2 + + '@codemirror/lang-css@6.3.1(@codemirror/view@6.43.0)': + dependencies: + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) + '@codemirror/language': 6.12.3 + '@codemirror/state': 6.6.0 + '@lezer/common': 1.5.2 + '@lezer/css': 1.3.3 + transitivePeerDependencies: + - '@codemirror/view' + + '@codemirror/lang-html@6.4.11': + dependencies: + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) + '@codemirror/lang-css': 6.3.1(@codemirror/view@6.43.0) + '@codemirror/lang-javascript': 6.2.2 + '@codemirror/language': 6.12.3 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 + '@lezer/common': 1.5.2 + '@lezer/css': 1.3.3 + '@lezer/html': 1.3.13 '@codemirror/lang-javascript@6.2.2': dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3) + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.2.3) '@codemirror/language': 6.10.6 '@codemirror/lint': 6.8.4 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 '@lezer/common': 1.2.3 '@lezer/javascript': 1.4.21 - '@codemirror/lang-sql@6.8.0(@codemirror/view@6.35.2)': + '@codemirror/lang-markdown@6.5.0': dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3) - '@codemirror/language': 6.10.6 - '@codemirror/state': 6.5.0 - '@lezer/common': 1.2.3 - '@lezer/highlight': 1.2.1 - '@lezer/lr': 1.4.2 - transitivePeerDependencies: - - '@codemirror/view' + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) + '@codemirror/lang-html': 6.4.11 + '@codemirror/language': 6.12.3 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 + '@lezer/common': 1.5.2 + '@lezer/markdown': 1.6.4 - '@codemirror/lang-sql@6.8.0(@codemirror/view@6.41.1)': + '@codemirror/lang-sql@6.8.0(@codemirror/view@6.43.0)': dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.41.1)(@lezer/common@1.2.3) - '@codemirror/language': 6.10.6 - '@codemirror/state': 6.5.0 - '@lezer/common': 1.2.3 + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) + '@codemirror/language': 6.12.3 + '@codemirror/state': 6.6.0 + '@lezer/common': 1.5.2 '@lezer/highlight': 1.2.1 '@lezer/lr': 1.4.2 transitivePeerDependencies: @@ -24834,17 +24927,17 @@ snapshots: '@codemirror/language@6.10.6': dependencies: - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 - '@lezer/common': 1.2.3 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 + '@lezer/common': 1.5.2 '@lezer/highlight': 1.2.1 '@lezer/lr': 1.4.2 - style-mod: 4.1.2 + style-mod: 4.1.3 '@codemirror/language@6.12.3': dependencies: '@codemirror/state': 6.6.0 - '@codemirror/view': 6.41.1 + '@codemirror/view': 6.43.0 '@lezer/common': 1.5.2 '@lezer/highlight': 1.2.3 '@lezer/lr': 1.4.10 @@ -24852,14 +24945,14 @@ snapshots: '@codemirror/lint@6.8.4': dependencies: - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.41.1 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 crelt: 1.0.6 '@codemirror/search@6.5.8': dependencies: - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.41.1 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 crelt: 1.0.6 '@codemirror/state@6.5.0': @@ -24878,16 +24971,9 @@ snapshots: '@lezer/highlight': 1.2.3 '@codemirror/view@6.35.2': - dependencies: - '@codemirror/state': 6.5.0 - style-mod: 4.1.2 - w3c-keyname: 2.2.8 - - '@codemirror/view@6.41.1': dependencies: '@codemirror/state': 6.6.0 - crelt: 1.0.6 - style-mod: 4.1.3 + style-mod: 4.1.2 w3c-keyname: 2.2.8 '@codemirror/view@6.43.0': @@ -25172,6 +25258,13 @@ snapshots: optionalDependencies: '@tanstack/db': 0.6.7(typescript@5.9.3) + '@durable-streams/y-durable-streams@0.2.7(lib0@0.2.99)(y-protocols@1.0.6(yjs@13.6.26))(yjs@13.6.26)': + dependencies: + '@durable-streams/client': 0.2.6 + lib0: 0.2.99 + y-protocols: 1.0.6(yjs@13.6.26) + yjs: 13.6.26 + '@ecies/ciphers@0.2.5(@noble/ciphers@1.3.0)': dependencies: '@noble/ciphers': 1.3.0 @@ -25222,18 +25315,18 @@ snapshots: '@electric-sql/pglite': 0.4.5 react: 19.2.5 - '@electric-sql/pglite-repl@0.2.17(@babel/runtime@7.29.7)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.5.0)(@codemirror/theme-one-dark@6.1.2)(@electric-sql/pglite@0.2.17)(@lezer/common@1.2.3)(codemirror@6.0.1(@lezer/common@1.2.3))': + '@electric-sql/pglite-repl@0.2.17(@babel/runtime@7.29.7)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@electric-sql/pglite@0.2.17)(@lezer/common@1.5.2)(codemirror@6.0.1(@lezer/common@1.5.2))': dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3) + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) '@codemirror/commands': 6.7.1 - '@codemirror/lang-sql': 6.8.0(@codemirror/view@6.35.2) + '@codemirror/lang-sql': 6.8.0(@codemirror/view@6.43.0) '@codemirror/language': 6.10.6 - '@codemirror/view': 6.35.2 + '@codemirror/view': 6.43.0 '@electric-sql/pglite-react': 0.2.17(@electric-sql/pglite@0.2.17)(react@19.2.5) - '@uiw/codemirror-theme-github': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2) - '@uiw/codemirror-theme-xcode': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2) - '@uiw/codemirror-themes': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2) - '@uiw/react-codemirror': 4.23.6(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3))(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.5.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.35.2)(codemirror@6.0.1(@lezer/common@1.2.3))(react-dom@19.2.5(react@19.2.5))(react@19.2.5) + '@uiw/codemirror-theme-github': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) + '@uiw/codemirror-theme-xcode': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) + '@uiw/codemirror-themes': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) + '@uiw/react-codemirror': 4.23.6(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.43.0)(codemirror@6.0.1(@lezer/common@1.5.2))(react-dom@19.2.5(react@19.2.5))(react@19.2.5) psql-describe: 0.1.6 react: 19.2.5 react-dom: 19.2.5(react@19.2.5) @@ -25250,16 +25343,16 @@ snapshots: '@electric-sql/pglite-repl@0.3.5(@babel/runtime@7.29.7)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@electric-sql/pglite@0.4.5)(@lezer/common@1.5.2)(codemirror@6.0.1(@lezer/common@1.5.2))': dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)(@lezer/common@1.5.2) + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) '@codemirror/commands': 6.7.1 - '@codemirror/lang-sql': 6.8.0(@codemirror/view@6.41.1) + '@codemirror/lang-sql': 6.8.0(@codemirror/view@6.43.0) '@codemirror/language': 6.12.3 - '@codemirror/view': 6.41.1 + '@codemirror/view': 6.43.0 '@electric-sql/pglite-react': 0.3.5(@electric-sql/pglite@0.4.5)(react@19.2.5) - '@uiw/codemirror-theme-github': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1) - '@uiw/codemirror-theme-xcode': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1) - '@uiw/codemirror-themes': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1) - '@uiw/react-codemirror': 4.23.5(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)(@lezer/common@1.5.2))(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.41.1)(codemirror@6.0.1(@lezer/common@1.5.2))(react-dom@19.2.5(react@19.2.5))(react@19.2.5) + '@uiw/codemirror-theme-github': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) + '@uiw/codemirror-theme-xcode': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) + '@uiw/codemirror-themes': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) + '@uiw/react-codemirror': 4.23.5(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.43.0)(codemirror@6.0.1(@lezer/common@1.5.2))(react-dom@19.2.5(react@19.2.5))(react@19.2.5) psql-describe: 0.1.6 react: 19.2.5 react-dom: 19.2.5(react@19.2.5) @@ -27362,6 +27455,12 @@ snapshots: '@lezer/common@1.5.2': {} + '@lezer/css@1.3.3': + dependencies: + '@lezer/common': 1.5.2 + '@lezer/highlight': 1.2.3 + '@lezer/lr': 1.4.10 + '@lezer/highlight@1.2.1': dependencies: '@lezer/common': 1.5.2 @@ -27370,9 +27469,15 @@ snapshots: dependencies: '@lezer/common': 1.5.2 + '@lezer/html@1.3.13': + dependencies: + '@lezer/common': 1.5.2 + '@lezer/highlight': 1.2.3 + '@lezer/lr': 1.4.10 + '@lezer/javascript@1.4.21': dependencies: - '@lezer/common': 1.2.3 + '@lezer/common': 1.5.2 '@lezer/highlight': 1.2.1 '@lezer/lr': 1.4.2 @@ -27384,6 +27489,11 @@ snapshots: dependencies: '@lezer/common': 1.5.2 + '@lezer/markdown@1.6.4': + dependencies: + '@lezer/common': 1.5.2 + '@lezer/highlight': 1.2.3 + '@lmdb/lmdb-darwin-arm64@3.5.4': optional: true @@ -31586,9 +31696,9 @@ snapshots: '@babel/core': 7.29.0 '@babel/plugin-syntax-jsx': 7.27.1(@babel/core@7.29.0) '@babel/plugin-syntax-typescript': 7.27.1(@babel/core@7.29.0) - '@babel/template': 7.28.6 + '@babel/template': 7.29.7 '@babel/traverse': 7.29.0 - '@babel/types': 7.29.0 + '@babel/types': 7.29.7 '@tanstack/directive-functions-plugin': 1.139.0(vite@7.1.7(@types/node@22.19.1)(jiti@2.6.1)(lightningcss@1.30.1)(terser@5.48.0)(tsx@4.20.3)(yaml@2.9.0)) babel-dead-code-elimination: 1.0.10 tiny-invariant: 1.3.3 @@ -32951,78 +33061,78 @@ snapshots: '@typescript-eslint/types': 8.46.0 eslint-visitor-keys: 4.2.1 - '@uiw/codemirror-extensions-basic-setup@4.23.5(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)(@lezer/common@1.5.2))(@codemirror/commands@6.7.1)(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)': + '@uiw/codemirror-extensions-basic-setup@4.23.5(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/commands@6.7.1)(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)(@lezer/common@1.5.2) + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) '@codemirror/commands': 6.7.1 '@codemirror/language': 6.12.3 '@codemirror/lint': 6.8.4 '@codemirror/search': 6.5.8 '@codemirror/state': 6.6.0 - '@codemirror/view': 6.41.1 + '@codemirror/view': 6.43.0 - '@uiw/codemirror-extensions-basic-setup@4.23.6(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3))(@codemirror/commands@6.7.1)(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)': + '@uiw/codemirror-extensions-basic-setup@4.23.6(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/commands@6.7.1)(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3) + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) '@codemirror/commands': 6.7.1 '@codemirror/language': 6.10.6 '@codemirror/lint': 6.8.4 '@codemirror/search': 6.5.8 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 - '@uiw/codemirror-theme-github@4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)': + '@uiw/codemirror-theme-github@4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: - '@uiw/codemirror-themes': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1) + '@uiw/codemirror-themes': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) transitivePeerDependencies: - '@codemirror/language' - '@codemirror/state' - '@codemirror/view' - '@uiw/codemirror-theme-github@4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)': + '@uiw/codemirror-theme-github@4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: - '@uiw/codemirror-themes': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2) + '@uiw/codemirror-themes': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) transitivePeerDependencies: - '@codemirror/language' - '@codemirror/state' - '@codemirror/view' - '@uiw/codemirror-theme-xcode@4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)': + '@uiw/codemirror-theme-xcode@4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: - '@uiw/codemirror-themes': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1) + '@uiw/codemirror-themes': 4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) transitivePeerDependencies: - '@codemirror/language' - '@codemirror/state' - '@codemirror/view' - '@uiw/codemirror-theme-xcode@4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)': + '@uiw/codemirror-theme-xcode@4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: - '@uiw/codemirror-themes': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2) + '@uiw/codemirror-themes': 4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) transitivePeerDependencies: - '@codemirror/language' - '@codemirror/state' - '@codemirror/view' - '@uiw/codemirror-themes@4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)': + '@uiw/codemirror-themes@4.23.5(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: '@codemirror/language': 6.12.3 '@codemirror/state': 6.6.0 - '@codemirror/view': 6.41.1 + '@codemirror/view': 6.43.0 - '@uiw/codemirror-themes@4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)': + '@uiw/codemirror-themes@4.23.6(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)': dependencies: '@codemirror/language': 6.10.6 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 - '@uiw/react-codemirror@4.23.5(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)(@lezer/common@1.5.2))(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.41.1)(codemirror@6.0.1(@lezer/common@1.5.2))(react-dom@19.2.5(react@19.2.5))(react@19.2.5)': + '@uiw/react-codemirror@4.23.5(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.43.0)(codemirror@6.0.1(@lezer/common@1.5.2))(react-dom@19.2.5(react@19.2.5))(react@19.2.5)': dependencies: '@babel/runtime': 7.29.7 '@codemirror/commands': 6.7.1 '@codemirror/state': 6.6.0 '@codemirror/theme-one-dark': 6.1.2 - '@codemirror/view': 6.41.1 - '@uiw/codemirror-extensions-basic-setup': 4.23.5(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1)(@lezer/common@1.5.2))(@codemirror/commands@6.7.1)(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/view@6.41.1) + '@codemirror/view': 6.43.0 + '@uiw/codemirror-extensions-basic-setup': 4.23.5(@codemirror/autocomplete@6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/commands@6.7.1)(@codemirror/language@6.12.3)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) codemirror: 6.0.1(@lezer/common@1.5.2) react: 19.2.5 react-dom: 19.2.5(react@19.2.5) @@ -33032,15 +33142,15 @@ snapshots: - '@codemirror/lint' - '@codemirror/search' - '@uiw/react-codemirror@4.23.6(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3))(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.5.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.35.2)(codemirror@6.0.1(@lezer/common@1.2.3))(react-dom@19.2.5(react@19.2.5))(react@19.2.5)': + '@uiw/react-codemirror@4.23.6(@babel/runtime@7.29.7)(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.43.0)(codemirror@6.0.1(@lezer/common@1.5.2))(react-dom@19.2.5(react@19.2.5))(react@19.2.5)': dependencies: '@babel/runtime': 7.29.7 '@codemirror/commands': 6.7.1 - '@codemirror/state': 6.5.0 + '@codemirror/state': 6.6.0 '@codemirror/theme-one-dark': 6.1.2 - '@codemirror/view': 6.35.2 - '@uiw/codemirror-extensions-basic-setup': 4.23.6(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3))(@codemirror/commands@6.7.1)(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2) - codemirror: 6.0.1(@lezer/common@1.2.3) + '@codemirror/view': 6.43.0 + '@uiw/codemirror-extensions-basic-setup': 4.23.6(@codemirror/autocomplete@6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2))(@codemirror/commands@6.7.1)(@codemirror/language@6.10.6)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.8)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0) + codemirror: 6.0.1(@lezer/common@1.5.2) react: 19.2.5 react-dom: 19.2.5(react@19.2.5) transitivePeerDependencies: @@ -33970,7 +34080,7 @@ snapshots: ast-kit@1.4.3: dependencies: - '@babel/parser': 7.29.2 + '@babel/parser': 7.29.7 pathe: 2.0.3 ast-types@0.13.4: @@ -34913,25 +35023,25 @@ snapshots: codemirror@6.0.1(@lezer/common@1.2.3): dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.2.3) + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.2.3) '@codemirror/commands': 6.7.1 - '@codemirror/language': 6.10.6 + '@codemirror/language': 6.12.3 '@codemirror/lint': 6.8.4 '@codemirror/search': 6.5.8 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 transitivePeerDependencies: - '@lezer/common' codemirror@6.0.1(@lezer/common@1.5.2): dependencies: - '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(@lezer/common@1.5.2) + '@codemirror/autocomplete': 6.18.3(@codemirror/language@6.12.3)(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(@lezer/common@1.5.2) '@codemirror/commands': 6.7.1 - '@codemirror/language': 6.10.6 + '@codemirror/language': 6.12.3 '@codemirror/lint': 6.8.4 '@codemirror/search': 6.5.8 - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 transitivePeerDependencies: - '@lezer/common' @@ -46788,6 +46898,13 @@ snapshots: lib0: 0.2.99 yjs: 13.6.26 + y-codemirror.next@0.3.5(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(yjs@13.6.26): + dependencies: + '@codemirror/state': 6.6.0 + '@codemirror/view': 6.43.0 + lib0: 0.2.99 + yjs: 13.6.26 + y-indexeddb@9.0.12(yjs@13.6.26): dependencies: lib0: 0.2.99 From a5cd78d4dd622a4a9a028e001463e95f0380f1bf Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 10 Jun 2026 19:15:34 +0100 Subject: [PATCH 05/39] Avoid bundling Yjs in agents client entry --- packages/agents-runtime/src/client.ts | 2 +- .../agents-runtime/src/markdown-document-constants.ts | 2 ++ .../agents-runtime/src/markdown-document-session.ts | 4 ++-- packages/agents-runtime/src/markdown-yjs.ts | 11 +++++++++-- 4 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 packages/agents-runtime/src/markdown-document-constants.ts diff --git a/packages/agents-runtime/src/client.ts b/packages/agents-runtime/src/client.ts index 2c8b2feb9b..3955ca2715 100644 --- a/packages/agents-runtime/src/client.ts +++ b/packages/agents-runtime/src/client.ts @@ -50,7 +50,7 @@ export { export { isGoalCommandText, parseGoalCommand } from './goal-command' export { formatTokenCount } from './token-budget' export type { GoalCommand } from './goal-command' -export { MARKDOWN_DOCUMENT_AGENT_PRESENCE_TTL_MS } from './markdown-yjs' +export { MARKDOWN_DOCUMENT_AGENT_PRESENCE_TTL_MS } from './markdown-document-constants' export type { EntityStreamDB, diff --git a/packages/agents-runtime/src/markdown-document-constants.ts b/packages/agents-runtime/src/markdown-document-constants.ts new file mode 100644 index 0000000000..e58a7ebe69 --- /dev/null +++ b/packages/agents-runtime/src/markdown-document-constants.ts @@ -0,0 +1,2 @@ +export const MARKDOWN_DOCUMENT_TEXT_NAME = `markdown` as const +export const MARKDOWN_DOCUMENT_AGENT_PRESENCE_TTL_MS = 45_000 diff --git a/packages/agents-runtime/src/markdown-document-session.ts b/packages/agents-runtime/src/markdown-document-session.ts index f1879fb823..3e54467f85 100644 --- a/packages/agents-runtime/src/markdown-document-session.ts +++ b/packages/agents-runtime/src/markdown-document-session.ts @@ -4,8 +4,8 @@ import * as Y from 'yjs' import { MARKDOWN_DOCUMENT_AGENT_PRESENCE_TTL_MS, MARKDOWN_DOCUMENT_TEXT_NAME, - markdownText, -} from './markdown-yjs' +} from './markdown-document-constants' +import { markdownText } from './markdown-yjs' import type { ManifestDocumentEntry, MarkdownDocumentConnection, diff --git a/packages/agents-runtime/src/markdown-yjs.ts b/packages/agents-runtime/src/markdown-yjs.ts index 0309508028..57d040d085 100644 --- a/packages/agents-runtime/src/markdown-yjs.ts +++ b/packages/agents-runtime/src/markdown-yjs.ts @@ -3,8 +3,15 @@ import * as encoding from 'lib0/encoding' import { Awareness, encodeAwarenessUpdate } from 'y-protocols/awareness' import * as Y from 'yjs' -export const MARKDOWN_DOCUMENT_TEXT_NAME = `markdown` as const -export const MARKDOWN_DOCUMENT_AGENT_PRESENCE_TTL_MS = 45_000 +import { + MARKDOWN_DOCUMENT_AGENT_PRESENCE_TTL_MS, + MARKDOWN_DOCUMENT_TEXT_NAME, +} from './markdown-document-constants' + +export { + MARKDOWN_DOCUMENT_AGENT_PRESENCE_TTL_MS, + MARKDOWN_DOCUMENT_TEXT_NAME, +} from './markdown-document-constants' export function frameYjsUpdate(update: Uint8Array): Uint8Array { const encoder = encoding.createEncoder() From c4c7722cc24215be04bce1ee366a18f501e4b923 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 10 Jun 2026 19:29:16 +0100 Subject: [PATCH 06/39] Align yjs example CodeMirror dependencies --- examples/yjs/package.json | 4 ++-- pnpm-lock.yaml | 43 +++++---------------------------------- 2 files changed, 7 insertions(+), 40 deletions(-) diff --git a/examples/yjs/package.json b/examples/yjs/package.json index b85c16f412..282d8bf401 100644 --- a/examples/yjs/package.json +++ b/examples/yjs/package.json @@ -23,8 +23,8 @@ }, "dependencies": { "@codemirror/lang-javascript": "^6.2.2", - "@codemirror/state": "^6.4.1", - "@codemirror/view": "^6.32.0", + "@codemirror/state": "^6.6.0", + "@codemirror/view": "^6.43.0", "@electric-sql/y-electric": "workspace:*", "@hono/node-server": "^1.8.2", "codemirror": "^6.0.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d2e9eb140e..27d9a6984a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1452,11 +1452,11 @@ importers: specifier: ^6.2.2 version: 6.2.2 '@codemirror/state': - specifier: ^6.4.1 - version: 6.5.0 + specifier: ^6.6.0 + version: 6.6.0 '@codemirror/view': - specifier: ^6.32.0 - version: 6.35.2 + specifier: ^6.43.0 + version: 6.43.0 '@electric-sql/y-electric': specifier: workspace:* version: link:../../packages/y-electric @@ -1486,7 +1486,7 @@ importers: version: 3.6.35 y-codemirror.next: specifier: 0.3.5 - version: 0.3.5(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(yjs@13.6.26) + version: 0.3.5(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(yjs@13.6.26) y-indexeddb: specifier: ^9.0.12 version: 9.0.12(yjs@13.6.26) @@ -4550,18 +4550,12 @@ packages: '@codemirror/search@6.5.8': resolution: {integrity: sha512-PoWtZvo7c1XFeZWmmyaOp2G0XVbOnm+fJzvghqGAktBW3cufwJUWvSCcNG0ppXiBEM05mZu6RhMtXPv2hpllig==} - '@codemirror/state@6.5.0': - resolution: {integrity: sha512-MwBHVK60IiIHDcoMet78lxt6iw5gJOGSbNbOIVBHWVXIH4/Nq1+GQgLLGgI1KlnN86WDXsPudVaqYHKBIx7Eyw==} - '@codemirror/state@6.6.0': resolution: {integrity: sha512-4nbvra5R5EtiCzr9BTHiTLc+MLXK2QGiAVYMyi8PkQd3SR+6ixar/Q/01Fa21TBIDOZXgeWV4WppsQolSreAPQ==} '@codemirror/theme-one-dark@6.1.2': resolution: {integrity: sha512-F+sH0X16j/qFLMAfbciKTxVOwkdAS336b7AXTKOZhy8BR3eH/RelsnLgLFINrpST63mmN2OuwUt0W2ndUgYwUA==} - '@codemirror/view@6.35.2': - resolution: {integrity: sha512-u04R04XFCYCNaHoNRr37WUUAfnxKPwPdqV+370NiO6i85qB1J/qCD/WbbMJsyJfRWhXIJXAe2BG/oTzAggqv4A==} - '@codemirror/view@6.43.0': resolution: {integrity: sha512-V7ZCLQO3Jus9hzh2jVCCPW3mO4IBMr43O37PqSUYautJSnnJF41YlgLw21x0fLJTYvJ+Vkm6Gp+qKGH9pltgXA==} @@ -6873,9 +6867,6 @@ packages: '@manypkg/get-packages@1.1.3': resolution: {integrity: sha512-fo+QhuU3qE/2TQMQmbVMqaQ6EWbMhi4ABWP+O4AM1NqPBuy0OrApV5LO6BrrgnhtAHS2NH6RrVk9OL181tTi8A==} - '@marijn/find-cluster-break@1.0.0': - resolution: {integrity: sha512-0YSzy7M9mBiK+h1m33rD8vZOfaO8leG6CY3+Q+1Lig86snkc8OAHQVAdndmnXMWJlVIH6S7fSZVVcjLcq6OH1A==} - '@marijn/find-cluster-break@1.0.2': resolution: {integrity: sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==} @@ -19910,9 +19901,6 @@ packages: structured-headers@0.4.1: resolution: {integrity: sha512-0MP/Cxx5SzeeZ10p/bZI0S6MpgD+yxAhi1BOQ34jgnMXsCq3j1t6tQnZu+KdlL7dvJTLT3g9xN8tl10TqgFMcg==} - style-mod@4.1.2: - resolution: {integrity: sha512-wnD1HyVqpJUI2+eKZ+eo1UwghftP6yuFheBqqe+bWCotBjC2K1YnteJILRMs3SM4V/0dLEW1SC27MWP5y+mwmw==} - style-mod@4.1.3: resolution: {integrity: sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ==} @@ -24955,10 +24943,6 @@ snapshots: '@codemirror/view': 6.43.0 crelt: 1.0.6 - '@codemirror/state@6.5.0': - dependencies: - '@marijn/find-cluster-break': 1.0.0 - '@codemirror/state@6.6.0': dependencies: '@marijn/find-cluster-break': 1.0.2 @@ -24970,12 +24954,6 @@ snapshots: '@codemirror/view': 6.43.0 '@lezer/highlight': 1.2.3 - '@codemirror/view@6.35.2': - dependencies: - '@codemirror/state': 6.6.0 - style-mod: 4.1.2 - w3c-keyname: 2.2.8 - '@codemirror/view@6.43.0': dependencies: '@codemirror/state': 6.6.0 @@ -27544,8 +27522,6 @@ snapshots: globby: 11.1.0 read-yaml-file: 1.1.0 - '@marijn/find-cluster-break@1.0.0': {} - '@marijn/find-cluster-break@1.0.2': {} '@mariozechner/pi-agent-core@0.57.1(@modelcontextprotocol/sdk@1.29.0(zod@4.3.6))(ws@8.20.0)(zod@4.3.6)': @@ -44494,8 +44470,6 @@ snapshots: structured-headers@0.4.1: {} - style-mod@4.1.2: {} - style-mod@4.1.3: {} style-to-js@1.1.21: @@ -46891,13 +46865,6 @@ snapshots: xtend@4.0.2: {} - y-codemirror.next@0.3.5(@codemirror/state@6.5.0)(@codemirror/view@6.35.2)(yjs@13.6.26): - dependencies: - '@codemirror/state': 6.5.0 - '@codemirror/view': 6.35.2 - lib0: 0.2.99 - yjs: 13.6.26 - y-codemirror.next@0.3.5(@codemirror/state@6.6.0)(@codemirror/view@6.43.0)(yjs@13.6.26): dependencies: '@codemirror/state': 6.6.0 From 835b8df1af000b5737b8408b6ac27fbcb0e0f9ed Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 17 Jun 2026 11:18:08 +0100 Subject: [PATCH 07/39] Fix manifest schema after markdown docs rebase --- packages/agents-runtime/src/entity-schema.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/agents-runtime/src/entity-schema.ts b/packages/agents-runtime/src/entity-schema.ts index 9f91cf295a..da064d96dc 100644 --- a/packages/agents-runtime/src/entity-schema.ts +++ b/packages/agents-runtime/src/entity-schema.ts @@ -1097,7 +1097,6 @@ export type Manifest = ManifestUnion & { tokenBudget?: number | null tokensUsed?: number summary?: string - updatedAt?: string } export type ReplayWatermark = SequencedPersistedRow From 366b76369dd71220c6d784ba33dbfe229b6fa6e4 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 11:23:56 +0100 Subject: [PATCH 08/39] feat(agents): add realtime stream foundations --- packages/agents-runtime/src/entity-schema.ts | 229 ++++++++++++++++++- packages/agents-runtime/src/index.ts | 12 + packages/agents-runtime/src/types.ts | 12 + packages/agents-server/src/index.ts | 3 + packages/agents-server/src/stream-client.ts | 35 ++- 5 files changed, 282 insertions(+), 9 deletions(-) diff --git a/packages/agents-runtime/src/entity-schema.ts b/packages/agents-runtime/src/entity-schema.ts index d94b99eac9..46ce3e3b85 100644 --- a/packages/agents-runtime/src/entity-schema.ts +++ b/packages/agents-runtime/src/entity-schema.ts @@ -394,6 +394,85 @@ type ManifestGoalEntryValue = { createdAt: string updatedAt: string } +type RealtimeSessionStatusValue = + | `requested` + | `active` + | `closing` + | `closed` + | `failed` +type RealtimeSessionStreamRefsValue = { + audio_in: string + audio_out: string + control_in: string + control_out: string +} +type ManifestRealtimeSessionEntryValue = { + key?: string + kind: `realtime-session` + id: string + provider: string + model: string + status: RealtimeSessionStatusValue + startedAt: string + endedAt?: string | null + streams: RealtimeSessionStreamRefsValue + retention: `forever` + meta?: Record +} +type RealtimeSessionValue = { + key?: string + session_id: string + provider: string + model: string + status: RealtimeSessionStatusValue + started_at: string + ended_at?: string + streams: RealtimeSessionStreamRefsValue + reason?: string + error?: string + meta?: Record +} +type RealtimeAudioSpanValue = { + key?: string + session_id: string + stream: `input` | `output` + producer_id: string + producer_epoch: number + seq: number + offset: string + next_offset?: string + byte_start?: number + byte_end?: number + byte_length: number + sample_start: number + sample_count: number + sample_rate: number + channels: number + codec: `pcm16` + timing_source: `client` | `runtime` | `provider` + captured_at?: string + received_at?: string + participant_id?: string + turn_id?: string + provider_item_id?: string + response_id?: string + created_at: string +} +type RealtimeTranscriptValue = { + key?: string + session_id: string + direction: `input` | `output` + text: string + status: `partial` | `final` + turn_id?: string + response_id?: string + audio_stream?: `input` | `output` + audio_offset?: string + audio_next_offset?: string + sample_start?: number + sample_end?: number + created_at: string +} type ReplayWatermarkValue = { key?: string source_id: string @@ -771,6 +850,20 @@ function createContextRemovedSchema(): Schema { timestamp: z.string(), }) } + +function createRealtimeSessionStreamRefsSchema(): Schema { + return z.object({ + audio_in: z.string(), + audio_out: z.string(), + control_in: z.string(), + control_out: z.string(), + }) +} + +function createRealtimeSessionStatusSchema() { + return z.enum([`requested`, `active`, `closing`, `closed`, `failed`]) +} + function createManifestSchema(): Schema< | ManifestChildEntryValue | ManifestSourceEntryValue @@ -781,6 +874,7 @@ function createManifestSchema(): Schema< | ManifestCronScheduleEntryValue | ManifestFutureSendScheduleEntryValue | ManifestGoalEntryValue + | ManifestRealtimeSessionEntryValue > { return z.union([ z.object({ @@ -896,6 +990,20 @@ function createManifestSchema(): Schema< createdAt: z.string(), updatedAt: z.string(), }), + z.object({ + key: z.string().optional(), + ...timelineOrderField, + kind: z.literal(`realtime-session`), + id: z.string(), + provider: z.string(), + model: z.string(), + status: createRealtimeSessionStatusSchema(), + startedAt: z.string(), + endedAt: z.string().nullable().optional(), + streams: createRealtimeSessionStreamRefsSchema(), + retention: z.literal(`forever`).default(`forever`), + meta: createJsonObjectSchema().optional(), + }), ]) as unknown as Schema< | ManifestChildEntryValue | ManifestSourceEntryValue @@ -906,9 +1014,76 @@ function createManifestSchema(): Schema< | ManifestCronScheduleEntryValue | ManifestFutureSendScheduleEntryValue | ManifestGoalEntryValue + | ManifestRealtimeSessionEntryValue > } +function createRealtimeSessionSchema(): Schema { + return z.object({ + key: z.string().optional(), + ...timelineOrderField, + session_id: z.string(), + provider: z.string(), + model: z.string(), + status: createRealtimeSessionStatusSchema(), + started_at: z.string(), + ended_at: z.string().optional(), + streams: createRealtimeSessionStreamRefsSchema(), + reason: z.string().optional(), + error: z.string().optional(), + meta: createJsonObjectSchema().optional(), + }) +} + +function createRealtimeAudioSpanSchema(): Schema { + return z.object({ + key: z.string().optional(), + ...timelineOrderField, + session_id: z.string(), + stream: z.enum([`input`, `output`]), + producer_id: z.string(), + producer_epoch: z.number().int().nonnegative(), + seq: z.number().int().nonnegative(), + offset: z.string(), + next_offset: z.string().optional(), + byte_start: z.number().int().nonnegative().optional(), + byte_end: z.number().int().nonnegative().optional(), + byte_length: z.number().int().nonnegative(), + sample_start: z.number().int().nonnegative(), + sample_count: z.number().int().nonnegative(), + sample_rate: z.number().int().positive(), + channels: z.number().int().positive(), + codec: z.literal(`pcm16`), + timing_source: z.enum([`client`, `runtime`, `provider`]), + captured_at: z.string().optional(), + received_at: z.string().optional(), + participant_id: z.string().optional(), + turn_id: z.string().optional(), + provider_item_id: z.string().optional(), + response_id: z.string().optional(), + created_at: z.string(), + }) +} + +function createRealtimeTranscriptSchema(): Schema { + return z.object({ + key: z.string().optional(), + ...timelineOrderField, + session_id: z.string(), + direction: z.enum([`input`, `output`]), + text: z.string(), + status: z.enum([`partial`, `final`]), + turn_id: z.string().optional(), + response_id: z.string().optional(), + audio_stream: z.enum([`input`, `output`]).optional(), + audio_offset: z.string().optional(), + audio_next_offset: z.string().optional(), + sample_start: z.number().int().nonnegative().optional(), + sample_end: z.number().int().nonnegative().optional(), + created_at: z.string(), + }) +} + function createReplayWatermarkSchema(): Schema { return z.object({ key: z.string().optional(), @@ -963,6 +1138,10 @@ export type ManifestFutureSendScheduleEntry = SequencedPersistedRow export type GoalStatus = GoalStatusValue export type ManifestGoalEntry = SequencedPersistedRow +export type RealtimeSessionStatus = RealtimeSessionStatusValue +export type RealtimeSessionStreamRefs = RealtimeSessionStreamRefsValue +export type ManifestRealtimeSessionEntry = + SequencedPersistedRow type ManifestUnion = | ManifestChildEntry | ManifestSourceEntry @@ -973,6 +1152,7 @@ type ManifestUnion = | ManifestCronScheduleEntry | ManifestFutureSendScheduleEntry | ManifestGoalEntry + | ManifestRealtimeSessionEntry export type Manifest = ManifestUnion & { id?: string entity_url?: string @@ -1004,7 +1184,11 @@ export type Manifest = ManifestUnion & { targetUrl?: string producerId?: string messageType?: string - status?: FutureSendScheduleStatus | AttachmentStatusValue | GoalStatusValue + status?: + | FutureSendScheduleStatus + | AttachmentStatusValue + | GoalStatusValue + | RealtimeSessionStatusValue sentAt?: string failedAt?: string lastError?: string @@ -1013,7 +1197,16 @@ export type Manifest = ManifestUnion & { tokensUsed?: number summary?: string updatedAt?: string -} + provider?: string + model?: string + startedAt?: string + endedAt?: string | null + streams?: RealtimeSessionStreamRefs + retention?: `forever` +} +export type RealtimeSession = SequencedPersistedRow +export type RealtimeAudioSpan = SequencedPersistedRow +export type RealtimeTranscript = SequencedPersistedRow export type ReplayWatermark = SequencedPersistedRow // ============================================================================ @@ -1038,6 +1231,9 @@ export const ENTITY_COLLECTIONS = { tags: `tags`, slashCommands: `slashCommands`, manifests: `manifests`, + realtimeSessions: `realtimeSessions`, + realtimeAudioSpans: `realtimeAudioSpans`, + realtimeTranscripts: `realtimeTranscripts`, contextInserted: `contextInserted`, contextRemoved: `contextRemoved`, replayWatermarks: `replayWatermarks`, @@ -1073,6 +1269,12 @@ export const BUILT_IN_EVENT_SCHEMAS = { context_removed: createContextRemovedSchema() as unknown as BuiltInEntitySchema, manifest: createManifestSchema() as unknown as BuiltInEntitySchema, + realtime_session: + createRealtimeSessionSchema() as unknown as BuiltInEntitySchema, + realtime_audio_span: + createRealtimeAudioSpanSchema() as unknown as BuiltInEntitySchema, + realtime_transcript: + createRealtimeTranscriptSchema() as unknown as BuiltInEntitySchema, replay_watermark: createReplayWatermarkSchema() as unknown as BuiltInEntitySchema, } as const @@ -1100,6 +1302,9 @@ type EntityCollectionsDefinition = { tags: CollectionDefinition slashCommands: CollectionDefinition manifests: CollectionDefinition + realtimeSessions: CollectionDefinition + realtimeAudioSpans: CollectionDefinition + realtimeTranscripts: CollectionDefinition contextInserted: CollectionDefinition contextRemoved: CollectionDefinition replayWatermarks: CollectionDefinition @@ -1202,6 +1407,24 @@ export const builtInCollections: EntityCollectionsDefinition = { type: `manifest`, primaryKey: `key`, }, + realtimeSessions: { + schema: + BUILT_IN_EVENT_SCHEMAS.realtime_session as StandardSchemaV1, + type: `realtime_session`, + primaryKey: `key`, + }, + realtimeAudioSpans: { + schema: + BUILT_IN_EVENT_SCHEMAS.realtime_audio_span as StandardSchemaV1, + type: `realtime_audio_span`, + primaryKey: `key`, + }, + realtimeTranscripts: { + schema: + BUILT_IN_EVENT_SCHEMAS.realtime_transcript as StandardSchemaV1, + type: `realtime_transcript`, + primaryKey: `key`, + }, contextInserted: { schema: BUILT_IN_EVENT_SCHEMAS.context_inserted as StandardSchemaV1, @@ -1238,6 +1461,8 @@ const MANAGEMENT_TYPES = new Set([ `entity_created`, `signal`, `manifest`, + `realtime_session`, + `realtime_audio_span`, `replay_watermark`, `ack`, ]) diff --git a/packages/agents-runtime/src/index.ts b/packages/agents-runtime/src/index.ts index 61093c87b4..f65431b09f 100644 --- a/packages/agents-runtime/src/index.ts +++ b/packages/agents-runtime/src/index.ts @@ -9,8 +9,14 @@ export type { ManifestContextEntry, ManifestEntry, ManifestEffectEntry, + ManifestRealtimeSessionEntry, ManifestSourceEntry, ManifestSharedStateEntry, + RealtimeAudioSpan, + RealtimeSession, + RealtimeSessionStatus, + RealtimeSessionStreamRefs, + RealtimeTranscript, PendingSend, EffectConfig, ObservationSource, @@ -117,6 +123,12 @@ export type { AttachmentSubject, AttachmentSubjectType, ManifestContextEntry as ManifestContextEntryRow, + ManifestRealtimeSessionEntry as ManifestRealtimeSessionEntryRow, + RealtimeAudioSpan as RealtimeAudioSpanRow, + RealtimeSession as RealtimeSessionRow, + RealtimeSessionStatus as RealtimeSessionStatusRow, + RealtimeSessionStreamRefs as RealtimeSessionStreamRefsRow, + RealtimeTranscript as RealtimeTranscriptRow, ReplayWatermark, WakeConfigValue, } from './entity-schema' diff --git a/packages/agents-runtime/src/types.ts b/packages/agents-runtime/src/types.ts index 3ef89c41b2..6a7f9a7424 100644 --- a/packages/agents-runtime/src/types.ts +++ b/packages/agents-runtime/src/types.ts @@ -45,8 +45,14 @@ import type { ManifestEffectEntry as EntityManifestEffectEntry, ManifestFutureSendScheduleEntry as EntityManifestFutureSendScheduleEntry, ManifestGoalEntry as EntityManifestGoalEntry, + ManifestRealtimeSessionEntry as EntityManifestRealtimeSessionEntry, ManifestSharedStateEntry as EntityManifestSharedStateEntry, ManifestSourceEntry as EntityManifestSourceEntry, + RealtimeAudioSpan as EntityRealtimeAudioSpan, + RealtimeSession as EntityRealtimeSession, + RealtimeSessionStatus as EntityRealtimeSessionStatus, + RealtimeSessionStreamRefs as EntityRealtimeSessionStreamRefs, + RealtimeTranscript as EntityRealtimeTranscript, Signal as EntitySignalEntry, WakeEntry, } from './entity-schema' @@ -324,8 +330,14 @@ export type ManifestEffectEntry = EntityManifestEffectEntry export type ManifestFutureSendScheduleEntry = EntityManifestFutureSendScheduleEntry export type ManifestGoalEntry = EntityManifestGoalEntry +export type ManifestRealtimeSessionEntry = EntityManifestRealtimeSessionEntry export type ManifestSourceEntry = EntityManifestSourceEntry export type ManifestSharedStateEntry = EntityManifestSharedStateEntry +export type RealtimeSession = EntityRealtimeSession +export type RealtimeSessionStatus = EntityRealtimeSessionStatus +export type RealtimeSessionStreamRefs = EntityRealtimeSessionStreamRefs +export type RealtimeAudioSpan = EntityRealtimeAudioSpan +export type RealtimeTranscript = EntityRealtimeTranscript export type ContextInserted = EntityContextInserted export type ContextRemoved = EntityContextRemoved export type ContextEntryAttrs = EntityContextEntryAttrs diff --git a/packages/agents-server/src/index.ts b/packages/agents-server/src/index.ts index 2dda597414..e5aa9af8ee 100644 --- a/packages/agents-server/src/index.ts +++ b/packages/agents-server/src/index.ts @@ -9,7 +9,10 @@ export type { export { StreamClient } from './stream-client.js' export type { DurableStreamsBearerProvider, + StreamAppendOptions, StreamClientOptions, + StreamIdempotentAppendOptions, + StreamProducerHeaderAppendOptions, SubscriptionClaimResponse, SubscriptionCreateInput, SubscriptionResponse, diff --git a/packages/agents-server/src/stream-client.ts b/packages/agents-server/src/stream-client.ts index 96e92de279..da74d5de8d 100644 --- a/packages/agents-server/src/stream-client.ts +++ b/packages/agents-server/src/stream-client.ts @@ -19,6 +19,26 @@ export interface StreamAppendResult { offset: string } +export interface StreamAppendOptions { + close?: boolean + contentType?: string + batching?: boolean +} + +export interface StreamIdempotentAppendOptions { + producerId: string + epoch?: number + contentType?: string + batching?: boolean +} + +export interface StreamProducerHeaderAppendOptions { + producerId: string + epoch: number + seq: number + contentType?: string +} + export interface StreamMessage { data: Uint8Array offset: string @@ -286,7 +306,7 @@ export class StreamClient { async append( path: string, data: Uint8Array | string, - opts?: { close?: boolean } + opts: StreamAppendOptions = {} ): Promise { return await withSpan(`stream.append`, async (span) => { span.setAttributes({ @@ -296,8 +316,8 @@ export class StreamClient { const handle = new DurableStream({ url: this.streamUrl(path), headers: this.streamHeaders(), - contentType: `application/json`, - batching: false, + contentType: opts.contentType ?? `application/json`, + batching: opts.batching ?? false, }) if (opts?.close) { const result = await handle.close({ body: data }) @@ -313,7 +333,7 @@ export class StreamClient { async appendIdempotent( path: string, data: Uint8Array | string, - opts: { producerId: string; epoch?: number } + opts: StreamIdempotentAppendOptions ): Promise { return await withSpan(`stream.appendIdempotent`, async (span) => { span.setAttributes({ @@ -323,7 +343,8 @@ export class StreamClient { const stream = new DurableStream({ url: this.streamUrl(path), headers: this.streamHeaders(), - contentType: `application/json`, + contentType: opts.contentType ?? `application/json`, + batching: opts.batching, }) const producer = new IdempotentProducer(stream, opts.producerId, { epoch: opts.epoch ?? 0, @@ -341,7 +362,7 @@ export class StreamClient { async appendWithProducerHeaders( path: string, data: Uint8Array | string, - opts: { producerId: string; epoch: number; seq: number } + opts: StreamProducerHeaderAppendOptions ): Promise { return await withSpan(`stream.appendWithProducerHeaders`, async (span) => { span.setAttributes({ @@ -349,7 +370,7 @@ export class StreamClient { [ATTR.STREAM_OP]: `appendWithProducerHeaders`, }) const headers: Record = { - 'content-type': `application/json`, + 'content-type': opts.contentType ?? `application/json`, 'Producer-Id': opts.producerId, 'Producer-Epoch': String(opts.epoch), 'Producer-Seq': String(opts.seq), From f1ef114c1009650f87874d9803580109d62a0511 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 11:36:25 +0100 Subject: [PATCH 09/39] feat(agents-runtime): add realtime handler API --- .../agents-runtime/src/context-factory.ts | 278 ++++++++++++++++++ packages/agents-runtime/src/index.ts | 16 + packages/agents-runtime/src/realtime.ts | 42 +++ packages/agents-runtime/src/types.ts | 148 ++++++++++ .../test/realtime-context.test.ts | 65 ++++ 5 files changed, 549 insertions(+) create mode 100644 packages/agents-runtime/src/realtime.ts create mode 100644 packages/agents-runtime/test/realtime-context.test.ts diff --git a/packages/agents-runtime/src/context-factory.ts b/packages/agents-runtime/src/context-factory.ts index e255096f8c..ee11077fa3 100644 --- a/packages/agents-runtime/src/context-factory.ts +++ b/packages/agents-runtime/src/context-factory.ts @@ -47,8 +47,14 @@ import type { HandlerWake, LLMMessage, ManifestAttachmentEntry, + ManifestRealtimeSessionEntry, ObservationHandle, ObservationSource, + RealtimeConfig, + RealtimeHandle, + RealtimeProviderEvent, + RealtimeProviderSession, + RealtimeRunResult, RunHandle, SendResult, SharedStateHandle, @@ -71,6 +77,41 @@ function agentModelProvider(config: AgentConfig): string { : config.model.provider } +function isRealtimeSessionManifest( + entry: unknown +): entry is ManifestRealtimeSessionEntry { + return ( + typeof entry === `object` && + entry !== null && + (entry as { kind?: unknown }).kind === `realtime-session` && + typeof (entry as { id?: unknown }).id === `string` + ) +} + +function realtimeManifestIsActive( + entry: ManifestRealtimeSessionEntry +): boolean { + return entry.status === `requested` || entry.status === `active` +} + +function getToolName(tool: AgentTool): string | null { + const name = (tool as { name?: unknown }).name + return typeof name === `string` ? name : null +} + +function applyRealtimeToolPolicy( + tools: Array, + policy: RealtimeConfig[`toolPolicy`] +): Array { + if (!policy) return tools + const allowed = new Set([...(policy.direct ?? []), ...(policy.confirm ?? [])]) + if (allowed.size === 0) return [] + return tools.filter((tool) => { + const name = getToolName(tool) + return name != null && allowed.has(name) + }) +} + const MAX_HYDRATED_IMAGE_ATTACHMENTS = 4 const MAX_HYDRATED_IMAGE_ATTACHMENT_BYTES = 10 * 1024 * 1024 @@ -476,6 +517,8 @@ export function createHandlerContext( ): HandlerContextResult { let sleepRequested = false let agentConfig: AgentConfig | null = null + let realtimeConfig: RealtimeConfig | null = null + let activeRealtimeProviderSession: RealtimeProviderSession | null = null let useContextConfig: UseContextConfig | null = null let useContextHash = `` let useContextRegistrations = 0 @@ -542,6 +585,20 @@ export function createHandlerContext( }, } + function realtimeSessions(): Array { + const sessions: Array = [] + for (const entry of config.db.collections.manifests.toArray) { + if (isRealtimeSessionManifest(entry)) { + sessions.push(entry) + } + } + return sessions.sort((a, b) => a.startedAt.localeCompare(b.startedAt)) + } + + function activeRealtimeSession(): ManifestRealtimeSessionEntry | undefined { + return realtimeSessions().filter(realtimeManifestIsActive).at(-1) + } + function structuralHash(nextConfig: UseContextConfig): string { const sources = Object.entries(nextConfig.sources) .sort(([leftName], [rightName]) => leftName.localeCompare(rightName)) @@ -950,6 +1007,219 @@ export function createHandlerContext( }, } + const realtimeHandle: RealtimeHandle = { + async run(): Promise { + if (!realtimeConfig) { + throw new Error( + `[agent-runtime] realtime.run() called without useRealtime().` + ) + } + + if (config.prepareAgentRun) { + await config.prepareAgentRun() + } + + const activeRealtimeConfig = realtimeConfig + const bridge = createOutboundBridge( + await loadOutboundIdSeed(config.db), + config.writeEvent + ) + const startedAt = Date.now() + let textStarted = false + let currentToolCall: + | { toolCallId: string; name: string; args: unknown } + | undefined + + const endText = (): void => { + if (!textStarted) return + bridge.onTextEnd() + textStarted = false + } + + const emitText = (delta: string): void => { + if (delta.length === 0) return + if (!textStarted) { + bridge.onTextStart() + textStarted = true + } + bridge.onTextDelta(delta) + } + + const composedTools = (await composeToolsWithProviders( + activeRealtimeConfig.tools ?? [] + )) as Array + const providerTools = applyRealtimeToolPolicy( + composedTools, + activeRealtimeConfig.toolPolicy + ) + const messages = await hydrateAttachmentBlocks( + timelineToMessages(config.db) + ) + + async function handleProviderEvent( + event: RealtimeProviderEvent + ): Promise { + switch (event.type) { + case `session.started`: + case `session.updated`: + case `input_audio.speech_started`: + case `input_audio.speech_stopped`: + case `input_transcript.delta`: + case `input_transcript.completed`: + case `output_audio.delta`: + case `output_audio.completed`: + case `response.started`: + case `response.cancelled`: + break + + case `session.closed`: + case `response.completed`: + endText() + break + + case `session.error`: + throw new Error( + `[agent-runtime] realtime provider error${event.code ? ` ${event.code}` : ``}: ${event.error}` + ) + + case `output_transcript.delta`: + emitText(event.delta) + break + + case `output_transcript.completed`: + if (!textStarted && event.text) { + emitText(event.text) + } + endText() + break + + case `tool_call.started`: + currentToolCall = { + toolCallId: event.toolCallId, + name: event.name, + args: event.args, + } + if (event.args !== undefined) { + bridge.onToolCallStart(event.toolCallId, event.name, event.args) + } + break + + case `tool_call.arguments_delta`: + break + + case `tool_call.arguments_completed`: + currentToolCall = { + toolCallId: event.toolCallId, + name: event.name, + args: event.args, + } + bridge.onToolCallStart(event.toolCallId, event.name, event.args) + break + + case `tool_call.completed`: { + if (currentToolCall?.toolCallId !== event.toolCallId) { + bridge.onToolCallStart(event.toolCallId, event.name, undefined) + } + bridge.onToolCallEnd( + event.toolCallId, + event.name, + event.result, + event.isError ?? false + ) + break + } + } + } + + try { + bridge.onRunStart() + bridge.onStepStart({ + modelProvider: activeRealtimeConfig.provider.id, + modelId: activeRealtimeConfig.provider.model, + }) + + if (activeRealtimeConfig.testResponses) { + const messageText = getTriggerMessageText( + config.db, + config.wakeEvent, + config.events, + config.wakeOffset, + config.hydratedEventSourceWake + ) + const responses = activeRealtimeConfig.testResponses + if (Array.isArray(responses)) { + const priorRunCount = ( + await queryOnce((q) => + q.from({ runs: config.db.collections.runs }) + ) + ).length + emitText( + responses[priorRunCount % Math.max(responses.length, 1)] ?? `` + ) + } else { + const response = await responses(messageText, bridge) + if (response !== undefined) emitText(response) + } + endText() + } else { + activeRealtimeProviderSession = + await activeRealtimeConfig.provider.connect({ + systemPrompt: activeRealtimeConfig.systemPrompt, + messages, + tools: providerTools, + audio: activeRealtimeConfig.audio, + session: activeRealtimeSession(), + signal: config.runSignal, + }) + + for await (const event of activeRealtimeProviderSession.events) { + if (config.runSignal?.aborted) { + break + } + await handleProviderEvent(event) + } + } + + endText() + bridge.onStepEnd({ + finishReason: config.runSignal?.aborted ? `aborted` : `stop`, + durationMs: Date.now() - startedAt, + }) + bridge.onRunEnd({ + finishReason: config.runSignal?.aborted ? `aborted` : `stop`, + }) + } catch (error) { + endText() + bridge.onStepEnd({ + finishReason: `error`, + durationMs: Date.now() - startedAt, + }) + bridge.onRunEnd({ finishReason: `error` }) + throw error + } finally { + activeRealtimeProviderSession = null + } + + return { + writes: [], + toolCalls: [], + usage: { tokens: 0, duration: Date.now() - startedAt }, + } + }, + async close(reason?: string): Promise { + await activeRealtimeProviderSession?.close?.(reason) + }, + async stop(reason?: string): Promise { + await this.close(reason) + }, + async cancelResponse(): Promise { + await activeRealtimeProviderSession?.cancelResponse?.() + }, + async sendText(text: string): Promise { + await activeRealtimeProviderSession?.sendText?.(text) + }, + } + const ctx: DebugHandlerContext = { firstWake: config.firstWake, wake: toHandlerWake(config.wakeEvent), @@ -970,6 +1240,10 @@ export function createHandlerContext( agentConfig = cfg return agent }, + useRealtime(cfg) { + realtimeConfig = cfg + return realtimeHandle + }, useContext(nextConfig) { assertValidUseContextConfig(nextConfig) const hash = structuralHash(nextConfig) @@ -995,6 +1269,10 @@ export function createHandlerContext( useContextRegistrations: () => useContextRegistrations, }, agent, + realtime: { + activeSession: activeRealtimeSession, + sessions: realtimeSessions, + }, observe: ((source: ObservationSource, opts?: { wake?: Wake }) => { return config.doObserve(source, opts?.wake) as Promise< ObservationHandle & EntityHandle & SharedStateHandle diff --git a/packages/agents-runtime/src/index.ts b/packages/agents-runtime/src/index.ts index f65431b09f..dbd32a975c 100644 --- a/packages/agents-runtime/src/index.ts +++ b/packages/agents-runtime/src/index.ts @@ -13,9 +13,23 @@ export type { ManifestSourceEntry, ManifestSharedStateEntry, RealtimeAudioSpan, + RealtimeAudioConfig, + RealtimeAudioFormat, + RealtimeConfig, + RealtimeContextConfig, + RealtimeHandle, + RealtimeHelpers, + RealtimeProviderConfig, + RealtimeProviderConnectInput, + RealtimeProviderEvent, + RealtimeProviderSession, + RealtimeRunResult, RealtimeSession, + RealtimeSessionPolicy, RealtimeSessionStatus, RealtimeSessionStreamRefs, + RealtimeToolPolicy, + RealtimeToolResult, RealtimeTranscript, PendingSend, EffectConfig, @@ -134,6 +148,8 @@ export type { } from './entity-schema' export { createEntityStreamDB } from './entity-stream-db' +export { createTestRealtimeProvider } from './realtime' +export type { TestRealtimeProviderOptions } from './realtime' export { getEntityAttachmentStreamPath, manifestAttachmentKey, diff --git a/packages/agents-runtime/src/realtime.ts b/packages/agents-runtime/src/realtime.ts new file mode 100644 index 0000000000..5916d4ebd6 --- /dev/null +++ b/packages/agents-runtime/src/realtime.ts @@ -0,0 +1,42 @@ +import type { RealtimeProviderConfig, RealtimeProviderEvent } from './types' + +export interface TestRealtimeProviderOptions { + model?: string + events?: Array + response?: string +} + +export function createTestRealtimeProvider( + opts: TestRealtimeProviderOptions = {} +): RealtimeProviderConfig { + return { + id: `test`, + model: opts.model ?? `test-realtime`, + async connect() { + const events = + opts.events ?? + (opts.response != null + ? [ + { type: `session.started` as const }, + { + type: `output_transcript.completed` as const, + text: opts.response, + }, + { type: `response.completed` as const }, + { type: `session.closed` as const }, + ] + : [ + { type: `session.started` as const }, + { type: `session.closed` as const }, + ]) + + return { + events: (async function* () { + for (const event of events) { + yield event + } + })(), + } + }, + } +} diff --git a/packages/agents-runtime/src/types.ts b/packages/agents-runtime/src/types.ts index 6a7f9a7424..5463ad1cec 100644 --- a/packages/agents-runtime/src/types.ts +++ b/packages/agents-runtime/src/types.ts @@ -988,6 +988,152 @@ export interface AgentConfig { testResponses?: TestResponses } +export type RealtimeAudioCodec = `pcm16` + +export interface RealtimeAudioFormat { + codec: RealtimeAudioCodec + sampleRate: number + channels: number +} + +export interface RealtimeAudioConfig { + inputFormat?: RealtimeAudioFormat + outputFormat?: RealtimeAudioFormat +} + +export interface RealtimeToolPolicy { + direct?: Array + confirm?: Array + delegate?: Array +} + +export interface RealtimeSessionPolicy { + textDuringSession?: `route-to-realtime` + retention?: `forever` +} + +export interface RealtimeContextConfig { + includeTimeline?: boolean +} + +export type RealtimeProviderEvent = + | { type: `session.started`; sessionId?: string } + | { type: `session.updated` } + | { type: `session.closed`; reason?: string } + | { type: `session.error`; error: string; code?: string } + | { type: `input_audio.speech_started`; audioOffset?: string } + | { type: `input_audio.speech_stopped`; audioOffset?: string } + | { type: `input_transcript.delta`; delta: string; turnId?: string } + | { type: `input_transcript.completed`; text: string; turnId?: string } + | { + type: `output_audio.delta` + audio: Uint8Array + responseId?: string + itemId?: string + } + | { type: `output_audio.completed`; responseId?: string; itemId?: string } + | { type: `output_transcript.delta`; delta: string; responseId?: string } + | { + type: `output_transcript.completed` + text?: string + responseId?: string + } + | { type: `response.started`; responseId?: string } + | { type: `response.completed`; responseId?: string } + | { type: `response.cancelled`; responseId?: string } + | { + type: `tool_call.started` + toolCallId: string + name: string + args?: unknown + } + | { + type: `tool_call.arguments_delta` + toolCallId: string + delta: string + } + | { + type: `tool_call.arguments_completed` + toolCallId: string + name: string + args: unknown + } + | { + type: `tool_call.completed` + toolCallId: string + name: string + result: unknown + isError?: boolean + } + +export interface RealtimeProviderConnectInput { + systemPrompt: string + messages: Array + tools: Array + audio?: RealtimeAudioConfig + session?: ManifestRealtimeSessionEntry + signal?: AbortSignal +} + +export interface RealtimeToolResult { + toolCallId: string + name: string + result: unknown + isError?: boolean +} + +export interface RealtimeProviderSession { + events: AsyncIterable + updateSession?: (update: unknown) => Promise + appendInputAudio?: ( + chunk: Uint8Array, + meta?: Record + ) => Promise + commitInputAudio?: () => Promise + sendText?: (text: string) => Promise + sendToolResult?: (result: RealtimeToolResult) => Promise + cancelResponse?: () => Promise + truncateOutputAudio?: (opts: { + itemId: string + audioEndMs: number + }) => Promise + close?: (reason?: string) => Promise +} + +export interface RealtimeProviderConfig { + id: string + model: string + connect: ( + input: RealtimeProviderConnectInput + ) => Promise +} + +export interface RealtimeConfig { + systemPrompt: string + provider: RealtimeProviderConfig + tools?: Array + audio?: RealtimeAudioConfig + toolPolicy?: RealtimeToolPolicy + context?: RealtimeContextConfig + session?: RealtimeSessionPolicy + testResponses?: TestResponses +} + +export type RealtimeRunResult = AgentRunResult + +export interface RealtimeHandle { + run: () => Promise + close: (reason?: string) => Promise + stop: (reason?: string) => Promise + cancelResponse: (opts?: { truncateAudio?: boolean }) => Promise + sendText: (text: string) => Promise +} + +export interface RealtimeHelpers { + activeSession: () => ManifestRealtimeSessionEntry | undefined + sessions: () => Array +} + export type TestResponses = Array | TestResponseFn export type TestResponseFn = ( @@ -1087,6 +1233,7 @@ export interface HandlerContext< */ sandbox: Sandbox useAgent: (config: AgentConfig) => AgentHandle + useRealtime: (config: RealtimeConfig) => RealtimeHandle useContext: (config: UseContextConfig) => void timelineMessages: (opts?: TimelineProjectionOpts) => Array insertContext: (id: string, entry: ContextEntryInput) => void @@ -1102,6 +1249,7 @@ export interface HandlerContext< opts?: { status?: GoalEntry[`status`] } ) => GoalEntry | undefined agent: AgentHandle + realtime: RealtimeHelpers spawn: ( type: string, id: string, diff --git a/packages/agents-runtime/test/realtime-context.test.ts b/packages/agents-runtime/test/realtime-context.test.ts new file mode 100644 index 0000000000..9e5ebfac2d --- /dev/null +++ b/packages/agents-runtime/test/realtime-context.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest' +import { createTestRealtimeProvider } from '../src/realtime' +import { createTestHandlerContext } from './helpers/context-test-helpers' + +describe(`ctx.useRealtime()`, () => { + it(`records provider transcript output through the outbound bridge`, async () => { + const { ctx } = createTestHandlerContext() + + const realtime = ctx.useRealtime({ + systemPrompt: `You are realtime.`, + provider: createTestRealtimeProvider({ response: `hello from voice` }), + tools: [], + }) + + await realtime.run() + + expect(ctx.db.collections.runs.toArray).toMatchObject([ + { key: `run-0`, status: `completed`, finish_reason: `stop` }, + ]) + expect(ctx.db.collections.steps.toArray).toMatchObject([ + { + key: `step-0`, + run_id: `run-0`, + model_provider: `test`, + model_id: `test-realtime`, + status: `completed`, + finish_reason: `stop`, + }, + ]) + expect(ctx.db.collections.textDeltas.toArray).toMatchObject([ + { + text_id: `msg-0`, + run_id: `run-0`, + delta: `hello from voice`, + }, + ]) + }) + + it(`finds active realtime sessions from the manifest`, () => { + const { ctx } = createTestHandlerContext() + + ctx.db.collections.manifests.insert({ + key: `realtime-session:rt-1`, + kind: `realtime-session`, + id: `rt-1`, + provider: `openai`, + model: `gpt-realtime-2`, + status: `active`, + startedAt: `2026-06-09T12:00:00.000Z`, + endedAt: null, + retention: `forever`, + streams: { + audio_in: `/entities/test/realtime/rt-1/audio/in`, + audio_out: `/entities/test/realtime/rt-1/audio/out`, + control_in: `/entities/test/realtime/rt-1/control/in`, + control_out: `/entities/test/realtime/rt-1/control/out`, + }, + }) + + expect(ctx.realtime.activeSession()).toMatchObject({ + id: `rt-1`, + status: `active`, + }) + }) +}) From a282191d346230953dbfc224e8e68043867423fb Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 11:45:06 +0100 Subject: [PATCH 10/39] feat(agents-server): add realtime session route --- packages/agents-server/src/entity-manager.ts | 178 ++++++++++++++++++ packages/agents-server/src/index.ts | 4 + .../src/routing/internal-router.ts | 2 + .../src/routing/realtime-router.ts | 89 +++++++++ ...ic-agents-manager-write-validation.test.ts | 96 ++++++++++ 5 files changed, 369 insertions(+) create mode 100644 packages/agents-server/src/routing/realtime-router.ts diff --git a/packages/agents-server/src/entity-manager.ts b/packages/agents-server/src/entity-manager.ts index 200f518661..2de262d8fb 100644 --- a/packages/agents-server/src/entity-manager.ts +++ b/packages/agents-server/src/entity-manager.ts @@ -115,6 +115,35 @@ type ServerSignalEvent = { txid: string } } +type RealtimeAudioRequest = { + codec?: `pcm16` + sampleRate?: number + channels?: number +} + +export type RealtimeSessionCreateRequest = { + id?: string + provider: string + model: string + inputAudio?: RealtimeAudioRequest + outputAudio?: RealtimeAudioRequest + meta?: Record +} + +export type RealtimeSessionCreateResult = { + sessionId: string + entityUrl: string + provider: string + model: string + status: `requested` + startedAt: string + streams: { + audio_in: string + audio_out: string + control_in: string + control_out: string + } +} type AttachmentSubjectType = `inbox` | `run` | `text` | `tool_call` | `context` type AttachmentRole = `input` | `output` @@ -272,6 +301,19 @@ function getEntityAttachmentStreamPath( return `${entityUrl.replace(/\/+$/, ``)}/attachments/${attachmentId}` } +function getRealtimeSessionBasePath( + entityUrl: string, + sessionId: string +): string { + return `${entityUrl.replace(/\/+$/, ``)}/realtime/${sessionId}` +} + +function realtimeAudioContentType(audio?: RealtimeAudioRequest): string { + const sampleRate = audio?.sampleRate ?? 24_000 + const channels = audio?.channels ?? 1 + return `audio/pcm; rate=${sampleRate}; channels=${channels}` +} + function isStreamCreateConflict(error: unknown): boolean { return ( !!error && @@ -304,6 +346,16 @@ function validateAttachmentId(id: string): void { } } +function validateRealtimeSessionId(id: string): void { + if (!id || id.includes(`/`) || id.startsWith(`.`)) { + throw new ElectricAgentsError( + ErrCodeInvalidRequest, + `realtime session id must not be empty, start with ".", or contain forward slashes`, + 400 + ) + } +} + function validateAttachmentSubject( subject: CreateAttachmentRequest[`subject`] ): void { @@ -2635,6 +2687,132 @@ export class EntityManager { return { txid } } + async createRealtimeSession( + entityUrl: string, + req: RealtimeSessionCreateRequest + ): Promise { + const entity = await this.registry.getEntity(entityUrl) + if (!entity) { + throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404) + } + if (rejectsNormalWrites(entity.status)) { + throw new ElectricAgentsError( + ErrCodeNotRunning, + `Entity is not accepting writes`, + 409 + ) + } + if (this.isForkWorkLockedEntity(entityUrl)) { + this.assertEntityNotForkWorkLocked(entityUrl) + } + + const provider = req.provider.trim() + const model = req.model.trim() + if (!provider || !model) { + throw new ElectricAgentsError( + ErrCodeInvalidRequest, + `provider and model are required`, + 400 + ) + } + + const sessionId = req.id ?? `rt-${randomUUID()}` + validateRealtimeSessionId(sessionId) + + const basePath = getRealtimeSessionBasePath(entityUrl, sessionId) + const streams = { + audio_in: `${basePath}/audio/in`, + audio_out: `${basePath}/audio/out`, + control_in: `${basePath}/control/in`, + control_out: `${basePath}/control/out`, + } + const startedAt = new Date().toISOString() + const manifestKey = `realtime-session:${sessionId}` + const txid = randomUUID() + const createdStreams: Array = [] + + try { + for (const [path, contentType] of [ + [streams.audio_in, realtimeAudioContentType(req.inputAudio)], + [streams.audio_out, realtimeAudioContentType(req.outputAudio)], + [streams.control_in, `application/json`], + [streams.control_out, `application/json`], + ] as const) { + await this.streamClient.create(path, { contentType }) + createdStreams.push(path) + } + + await this.writeManifestEntry( + entityUrl, + manifestKey, + `upsert`, + { + kind: `realtime-session`, + id: sessionId, + provider, + model, + status: `requested`, + startedAt, + endedAt: null, + streams, + retention: `forever`, + ...(req.meta ? { meta: req.meta } : {}), + }, + { txid } + ) + + const sessionEvent = entityStateSchema.realtimeSessions.insert({ + key: manifestKey, + value: { + session_id: sessionId, + provider, + model, + status: `requested`, + started_at: startedAt, + streams, + ...(req.meta ? { meta: req.meta } : {}), + }, + } as never) + await this.streamClient.append( + entity.streams.main, + this.encodeChangeEvent(sessionEvent as Record) + ) + + await this.send(entityUrl, { + from: SERVER_SIGNAL_SENDER, + payload: { + type: `realtime_session.started`, + sessionId, + provider, + model, + streams, + }, + }) + } catch (error) { + await Promise.allSettled( + createdStreams.map((path) => this.streamClient.delete(path)) + ) + if (isStreamCreateConflict(error)) { + throw new ElectricAgentsError( + ErrCodeInvalidRequest, + `Realtime session already exists at id "${sessionId}"`, + 409 + ) + } + throw error + } + + return { + sessionId, + entityUrl, + provider, + model, + status: `requested`, + startedAt, + streams, + } + } + // ========================================================================== // Attachments // ========================================================================== diff --git a/packages/agents-server/src/index.ts b/packages/agents-server/src/index.ts index e5aa9af8ee..a381165abf 100644 --- a/packages/agents-server/src/index.ts +++ b/packages/agents-server/src/index.ts @@ -1,6 +1,10 @@ export { createDb, runMigrations } from './db/index.js' export type { DrizzleDB, PgClient } from './db/index.js' export { AgentsHost } from './host.js' +export type { + RealtimeSessionCreateRequest, + RealtimeSessionCreateResult, +} from './entity-manager.js' export type { AgentsHostOptions, AgentsHostTenantConfig, diff --git a/packages/agents-server/src/routing/internal-router.ts b/packages/agents-server/src/routing/internal-router.ts index 4e187e6510..58171f50a5 100644 --- a/packages/agents-server/src/routing/internal-router.ts +++ b/packages/agents-server/src/routing/internal-router.ts @@ -33,6 +33,7 @@ import { entityTypesRouter } from './entity-types-router.js' import { pgSyncRouter } from './pg-sync-router.js' import { getRequestSpan } from './hooks.js' import { observationsRouter } from './observations-router.js' +import { realtimeRouter } from './realtime-router.js' import { runnersRouter } from './runners-router.js' import { routeBody, validateOptionalJsonBody, withSchema } from './schema.js' import { withLeadingSlash } from './tenant-stream-paths.js' @@ -138,6 +139,7 @@ internalRouter.all(`/entities/*`, entitiesRouter.fetch) internalRouter.all(`/entity-types/*`, entityTypesRouter.fetch) internalRouter.all(`/pg-sync/*`, pgSyncRouter.fetch) internalRouter.all(`/observations/*`, observationsRouter.fetch) +internalRouter.all(`/realtime/*`, realtimeRouter.fetch) internalRouter.get(`/electric/*`, electricProxyRouter.fetch) internalRouter.all(`*`, () => status(404)) diff --git a/packages/agents-server/src/routing/realtime-router.ts b/packages/agents-server/src/routing/realtime-router.ts new file mode 100644 index 0000000000..7d65b99d8c --- /dev/null +++ b/packages/agents-server/src/routing/realtime-router.ts @@ -0,0 +1,89 @@ +/** + * HTTP routes for realtime session management. + */ + +import { Type, type Static } from '@sinclair/typebox' +import { Router, json } from 'itty-router' +import { apiError } from '../electric-agents-http.js' +import { + ErrCodeNotFound, + ErrCodeUnauthorized, +} from '../electric-agents-types.js' +import { canAccessEntity } from '../permissions.js' +import { routeBody, withSchema } from './schema.js' +import type { JsonRouteRequest } from './schema.js' +import type { RouterType } from 'itty-router' +import type { TenantContext } from './context.js' + +interface RealtimeRouteRequest extends JsonRouteRequest {} + +type RealtimeRouteArgs = [TenantContext] +type RealtimeRouteResult = Response | undefined + +export type RealtimeRoutes = RouterType< + RealtimeRouteRequest, + RealtimeRouteArgs, + RealtimeRouteResult +> + +const realtimeAudioRequestSchema = Type.Object( + { + codec: Type.Optional(Type.Literal(`pcm16`)), + sampleRate: Type.Optional(Type.Number()), + channels: Type.Optional(Type.Number()), + }, + { additionalProperties: false } +) + +const realtimeSessionCreateBodySchema = Type.Object( + { + entityUrl: Type.String(), + id: Type.Optional(Type.String()), + provider: Type.String(), + model: Type.String(), + inputAudio: Type.Optional(realtimeAudioRequestSchema), + outputAudio: Type.Optional(realtimeAudioRequestSchema), + meta: Type.Optional(Type.Record(Type.String(), Type.Unknown())), + }, + { additionalProperties: false } +) + +type RealtimeSessionCreateBody = Static + +export const realtimeRouter: RealtimeRoutes = Router< + RealtimeRouteRequest, + RealtimeRouteArgs, + RealtimeRouteResult +>({ + base: `/_electric/realtime`, +}) + +realtimeRouter.post( + `/sessions`, + withSchema(realtimeSessionCreateBodySchema), + createRealtimeSession +) + +async function createRealtimeSession( + request: RealtimeRouteRequest, + ctx: TenantContext +): Promise { + const parsed = routeBody(request) + const entity = await ctx.entityManager.registry.getEntity(parsed.entityUrl) + if (!entity) { + return apiError(404, ErrCodeNotFound, `Entity not found`) + } + if (!(await canAccessEntity(ctx, entity, `write`, request as Request))) { + return apiError( + 401, + ErrCodeUnauthorized, + `Principal is not allowed to write ${entity.url}` + ) + } + + const result = await ctx.entityManager.createRealtimeSession( + parsed.entityUrl, + parsed + ) + return json(result, { status: 201 }) +} diff --git a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts index 9a96c4ef40..1cff4f2f19 100644 --- a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts +++ b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts @@ -94,6 +94,13 @@ function attachmentManifest(value: Record) { } } +function decodeAppend(call: unknown[]): Record { + const body = call[1] + const bytes = + body instanceof Uint8Array ? body : new TextEncoder().encode(String(body)) + return JSON.parse(new TextDecoder().decode(bytes)) as Record +} + describe(`ElectricAgentsManager.validateWriteEvent`, () => { it(`validates delete events against old_value instead of value`, async () => { const manager = createManager() @@ -140,6 +147,95 @@ describe(`ElectricAgentsManager.validateWriteEvent`, () => { }) }) +describe(`ElectricAgentsManager realtime sessions`, () => { + it(`creates durable IO streams and records a replayable session manifest`, async () => { + const create = vi.fn() + const append = vi.fn() + const { manager } = createAttachmentManager({ + streamClient: { create, append }, + }) + + const result = await manager.createRealtimeSession(`/chat/session-1`, { + id: `rt-1`, + provider: `openai`, + model: `gpt-realtime-2`, + inputAudio: { codec: `pcm16`, sampleRate: 16_000, channels: 1 }, + outputAudio: { codec: `pcm16`, sampleRate: 24_000, channels: 1 }, + meta: { source: `test` }, + }) + + expect(result.streams).toEqual({ + audio_in: `/chat/session-1/realtime/rt-1/audio/in`, + audio_out: `/chat/session-1/realtime/rt-1/audio/out`, + control_in: `/chat/session-1/realtime/rt-1/control/in`, + control_out: `/chat/session-1/realtime/rt-1/control/out`, + }) + expect(create).toHaveBeenCalledTimes(4) + expect(create.mock.calls).toEqual([ + [ + `/chat/session-1/realtime/rt-1/audio/in`, + { contentType: `audio/pcm; rate=16000; channels=1` }, + ], + [ + `/chat/session-1/realtime/rt-1/audio/out`, + { contentType: `audio/pcm; rate=24000; channels=1` }, + ], + [ + `/chat/session-1/realtime/rt-1/control/in`, + { contentType: `application/json` }, + ], + [ + `/chat/session-1/realtime/rt-1/control/out`, + { contentType: `application/json` }, + ], + ]) + + expect(append).toHaveBeenCalledTimes(3) + const manifestEvent = decodeAppend(append.mock.calls[0]!) + const sessionEvent = decodeAppend(append.mock.calls[1]!) + const inboxEvent = decodeAppend(append.mock.calls[2]!) + + expect(manifestEvent).toMatchObject({ + type: `manifest`, + key: `realtime-session:rt-1`, + headers: { operation: `upsert` }, + value: { + kind: `realtime-session`, + id: `rt-1`, + provider: `openai`, + model: `gpt-realtime-2`, + status: `requested`, + streams: result.streams, + retention: `forever`, + meta: { source: `test` }, + }, + }) + expect(sessionEvent).toMatchObject({ + type: `realtime_session`, + key: `realtime-session:rt-1`, + value: { + session_id: `rt-1`, + provider: `openai`, + model: `gpt-realtime-2`, + status: `requested`, + streams: result.streams, + }, + }) + expect(inboxEvent).toMatchObject({ + type: `inbox`, + value: { + from: `/_electric/server`, + payload: { + type: `realtime_session.started`, + sessionId: `rt-1`, + streams: result.streams, + }, + }, + }) + expect(inboxEvent.value).not.toHaveProperty(`message_type`) + }) +}) + describe(`ElectricAgentsManager attachments`, () => { it(`does not delete an existing stream when duplicate attachment creation conflicts`, async () => { const create = vi.fn().mockRejectedValue({ status: 409 }) From 6e6996aad0f737b05114ac4f32582ec24ad3e681 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 11:47:43 +0100 Subject: [PATCH 11/39] feat(agents-runtime): add realtime session client --- packages/agents-runtime/src/agents-client.ts | 9 +++ packages/agents-runtime/src/client.ts | 5 ++ packages/agents-runtime/src/index.ts | 3 + .../src/runtime-server-client.ts | 49 ++++++++++++ .../test/electric-agents-client.test.ts | 41 ++++++++++ ...time-server-client-update-metadata.test.ts | 78 +++++++++++++++++++ 6 files changed, 185 insertions(+) diff --git a/packages/agents-runtime/src/agents-client.ts b/packages/agents-runtime/src/agents-client.ts index d8995024ac..7c89927838 100644 --- a/packages/agents-runtime/src/agents-client.ts +++ b/packages/agents-runtime/src/agents-client.ts @@ -4,6 +4,10 @@ import { normalizeObservationSchema } from './observation-schema' import { createRuntimeServerClient } from './runtime-server-client' import { appendPathToUrl } from './url' import type { EntitySignal } from './runtime-server-client' +import type { + RealtimeSessionStartResult, + StartRealtimeSessionOptions, +} from './runtime-server-client' import type { EntitiesObservationSource, EntityObservationSource, @@ -32,6 +36,9 @@ export interface AgentsClient { payload?: unknown }) => Promise<{ txid: number }> kill: (entityUrl: string, reason?: string) => Promise<{ txid: number }> + startRealtimeSession: ( + options: StartRealtimeSessionOptions + ) => Promise } export function createAgentsClient(config: AgentsClientConfig): AgentsClient { @@ -45,6 +52,8 @@ export function createAgentsClient(config: AgentsClientConfig): AgentsClient { signal: `SIGKILL`, reason, }), + startRealtimeSession: (options) => + serverClient.startRealtimeSession(options), async observe(source) { if (source.sourceType === `entity`) { const info = await serverClient.getEntity( diff --git a/packages/agents-runtime/src/client.ts b/packages/agents-runtime/src/client.ts index 55306cc03a..72de96eac8 100644 --- a/packages/agents-runtime/src/client.ts +++ b/packages/agents-runtime/src/client.ts @@ -60,6 +60,11 @@ export type { SlashCommandTrigger, } from './composer-input' export type { AgentsClient, AgentsClientConfig } from './agents-client' +export type { + RealtimeAudioOptions, + RealtimeSessionStartResult, + StartRealtimeSessionOptions, +} from './runtime-server-client' export type { AttachmentRole, AttachmentStatus, diff --git a/packages/agents-runtime/src/index.ts b/packages/agents-runtime/src/index.ts index dbd32a975c..e29997aa94 100644 --- a/packages/agents-runtime/src/index.ts +++ b/packages/agents-runtime/src/index.ts @@ -289,6 +289,9 @@ export type { DispatchPolicy, SpawnEntityOptions, SendEntityMessageOptions, + RealtimeAudioOptions, + RealtimeSessionStartResult, + StartRealtimeSessionOptions, } from './runtime-server-client' export { buildWebhookSourceManifestEntry, diff --git a/packages/agents-runtime/src/runtime-server-client.ts b/packages/agents-runtime/src/runtime-server-client.ts index ec7f713166..f38bb10e15 100644 --- a/packages/agents-runtime/src/runtime-server-client.ts +++ b/packages/agents-runtime/src/runtime-server-client.ts @@ -12,6 +12,7 @@ import type { ManifestAttachmentEntry, } from './types' import type { EntitySignal } from './entity-schema' +import type { RealtimeSessionStreamRefs } from './entity-schema' import type { WebhookSourceContract, WebhookSourceSubscription, @@ -95,6 +96,32 @@ export interface SendEntityMessageOptions { writeToken?: string } +export interface RealtimeAudioOptions { + codec?: `pcm16` + sampleRate?: number + channels?: number +} + +export interface StartRealtimeSessionOptions { + entityUrl: string + id?: string + provider: string + model: string + inputAudio?: RealtimeAudioOptions + outputAudio?: RealtimeAudioOptions + meta?: Record +} + +export interface RealtimeSessionStartResult { + sessionId: string + entityUrl: string + provider: string + model: string + status: `requested` + startedAt: string + streams: RealtimeSessionStreamRefs +} + export interface RegisterWakeOptions { subscriberUrl: string sourceUrl: string @@ -120,6 +147,9 @@ export interface SignalEntityOptions { export interface RuntimeServerClient { sendEntityMessage: (options: SendEntityMessageOptions) => Promise + startRealtimeSession: ( + options: StartRealtimeSessionOptions + ) => Promise createAttachment: (options: { entityUrl: string attachment: AttachmentCreateInput @@ -386,6 +416,24 @@ export function createRuntimeServerClient( } } + const startRealtimeSession = async ( + options: StartRealtimeSessionOptions + ): Promise => { + const response = await request(`/_electric/realtime/sessions`, { + method: `POST`, + headers: { 'content-type': `application/json` }, + body: JSON.stringify(options), + }) + + if (!response.ok) { + throw new Error( + `startRealtimeSession ${options.entityUrl} failed (${response.status}): ${await readErrorText(response)}` + ) + } + + return (await response.json()) as RealtimeSessionStartResult + } + const createAttachment = async ({ entityUrl, attachment, @@ -940,6 +988,7 @@ export function createRuntimeServerClient( return { sendEntityMessage, + startRealtimeSession, createAttachment, readAttachment, spawnEntity, diff --git a/packages/agents-runtime/test/electric-agents-client.test.ts b/packages/agents-runtime/test/electric-agents-client.test.ts index b4587f4fd4..858faefff0 100644 --- a/packages/agents-runtime/test/electric-agents-client.test.ts +++ b/packages/agents-runtime/test/electric-agents-client.test.ts @@ -9,6 +9,7 @@ const { mockState } = vi.hoisted(() => ({ ensureCronStream: vi.fn(), registerPgSyncSource: vi.fn(), signalEntity: vi.fn(), + startRealtimeSession: vi.fn(), ensureStream: vi.fn(), createStreamDB: vi.fn(), preload: vi.fn(), @@ -27,6 +28,7 @@ vi.mock(`../src/runtime-server-client`, () => ({ ensureCronStream: mockState.ensureCronStream, registerPgSyncSource: mockState.registerPgSyncSource, signalEntity: mockState.signalEntity, + startRealtimeSession: mockState.startRealtimeSession, ensureStream: mockState.ensureStream, }), })) @@ -55,6 +57,20 @@ describe(`createAgentsClient`, () => { mockState.ensureStream = vi.fn().mockResolvedValue(`/_webhooks/repo`) mockState.createStreamDB = vi.fn() mockState.signalEntity = vi.fn().mockResolvedValue({ txid: 123 }) + mockState.startRealtimeSession = vi.fn().mockResolvedValue({ + sessionId: `rt-1`, + entityUrl: `/horton/demo`, + provider: `openai`, + model: `gpt-realtime-2`, + status: `requested`, + startedAt: `2026-06-09T10:00:00.000Z`, + streams: { + audio_in: `/horton/demo/realtime/rt-1/audio/in`, + audio_out: `/horton/demo/realtime/rt-1/audio/out`, + control_in: `/horton/demo/realtime/rt-1/control/in`, + control_out: `/horton/demo/realtime/rt-1/control/out`, + }, + }) mockState.observedDb = { preload: vi.fn().mockResolvedValue(undefined), collections: { @@ -191,6 +207,31 @@ describe(`createAgentsClient`, () => { }) }) + it(`exposes realtime session start through the server client`, async () => { + const client = createAgentsClient({ + baseUrl: `http://electric-agents.test`, + }) + + await expect( + client.startRealtimeSession({ + entityUrl: `/horton/demo`, + provider: `openai`, + model: `gpt-realtime-2`, + }) + ).resolves.toMatchObject({ + sessionId: `rt-1`, + streams: { + audio_in: `/horton/demo/realtime/rt-1/audio/in`, + }, + }) + + expect(mockState.startRealtimeSession).toHaveBeenCalledWith({ + entityUrl: `/horton/demo`, + provider: `openai`, + model: `gpt-realtime-2`, + }) + }) + it(`observe(webhook(...)) ensures the exact stream before preloading it`, async () => { const client = createAgentsClient({ baseUrl: `http://electric-agents.test/t/tenant-a/v1`, diff --git a/packages/agents-runtime/test/runtime-server-client-update-metadata.test.ts b/packages/agents-runtime/test/runtime-server-client-update-metadata.test.ts index 43398ef7a4..092a72a782 100644 --- a/packages/agents-runtime/test/runtime-server-client-update-metadata.test.ts +++ b/packages/agents-runtime/test/runtime-server-client-update-metadata.test.ts @@ -266,6 +266,84 @@ describe(`runtime-server-client.deleteTag`, () => { }) }) +describe(`runtime-server-client realtime sessions`, () => { + it(`starts a realtime session through the control-plane route`, async () => { + const calls: Array<{ url: string; init?: RequestInit }> = [] + const responseBody = { + sessionId: `rt-1`, + entityUrl: `/horton/demo`, + provider: `openai`, + model: `gpt-realtime-2`, + status: `requested`, + startedAt: `2026-06-09T10:00:00.000Z`, + streams: { + audio_in: `/horton/demo/realtime/rt-1/audio/in`, + audio_out: `/horton/demo/realtime/rt-1/audio/out`, + control_in: `/horton/demo/realtime/rt-1/control/in`, + control_out: `/horton/demo/realtime/rt-1/control/out`, + }, + } + const fakeFetch = vi.fn(async (url: string, init?: RequestInit) => { + calls.push({ url, init }) + return new Response(JSON.stringify(responseBody), { + status: 201, + headers: { 'content-type': `application/json` }, + }) + }) as unknown as typeof fetch + const client = createRuntimeServerClient({ + baseUrl: `http://test.example/t/tenant-a/v1`, + fetch: fakeFetch, + principalKey: `user:sam`, + }) + + await expect( + client.startRealtimeSession({ + entityUrl: `/horton/demo`, + id: `rt-1`, + provider: `openai`, + model: `gpt-realtime-2`, + inputAudio: { codec: `pcm16`, sampleRate: 16_000, channels: 1 }, + meta: { source: `button` }, + }) + ).resolves.toEqual(responseBody) + + expect(calls).toHaveLength(1) + expect(calls[0]!.url).toBe( + `http://test.example/t/tenant-a/v1/_electric/realtime/sessions` + ) + expect(calls[0]!.init?.method).toBe(`POST`) + const headers = new Headers(calls[0]!.init?.headers) + expect(headers.get(`content-type`)).toBe(`application/json`) + expect(headers.get(`electric-principal`)).toBe(`user:sam`) + expect(JSON.parse(calls[0]!.init!.body as string)).toEqual({ + entityUrl: `/horton/demo`, + id: `rt-1`, + provider: `openai`, + model: `gpt-realtime-2`, + inputAudio: { codec: `pcm16`, sampleRate: 16_000, channels: 1 }, + meta: { source: `button` }, + }) + }) + + it(`surfaces realtime session start failures`, async () => { + const fakeFetch = vi.fn( + async () => new Response(`not allowed`, { status: 401 }) + ) as unknown as typeof fetch + const client = createRuntimeServerClient({ + baseUrl: `http://test.example`, + fetch: fakeFetch, + }) + + await expect( + client.startRealtimeSession({ + entityUrl: `/horton/demo`, + provider: `openai`, + model: `gpt-realtime-2`, + }) + ).rejects.toThrow(/startRealtimeSession.*401.*not allowed/) + }) +}) + describe(`runtime-server-client webhook sources`, () => { it(`lists webhook sources from the runtime server`, async () => { const fakeFetch = vi.fn( From 12d55fe776bd71f8ca95747c464ea2f4f0884706 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 11:58:07 +0100 Subject: [PATCH 12/39] feat(agents-runtime): add openai realtime provider --- packages/agents-runtime/src/index.ts | 2 + .../agents-runtime/src/openai-realtime.ts | 615 ++++++++++++++++++ .../test/openai-realtime.test.ts | 217 ++++++ 3 files changed, 834 insertions(+) create mode 100644 packages/agents-runtime/src/openai-realtime.ts create mode 100644 packages/agents-runtime/test/openai-realtime.test.ts diff --git a/packages/agents-runtime/src/index.ts b/packages/agents-runtime/src/index.ts index e29997aa94..ee76bb2814 100644 --- a/packages/agents-runtime/src/index.ts +++ b/packages/agents-runtime/src/index.ts @@ -150,6 +150,8 @@ export type { export { createEntityStreamDB } from './entity-stream-db' export { createTestRealtimeProvider } from './realtime' export type { TestRealtimeProviderOptions } from './realtime' +export { createOpenAIRealtimeProvider } from './openai-realtime' +export type { OpenAIRealtimeProviderOptions } from './openai-realtime' export { getEntityAttachmentStreamPath, manifestAttachmentKey, diff --git a/packages/agents-runtime/src/openai-realtime.ts b/packages/agents-runtime/src/openai-realtime.ts new file mode 100644 index 0000000000..649fe1a25c --- /dev/null +++ b/packages/agents-runtime/src/openai-realtime.ts @@ -0,0 +1,615 @@ +import type { + AgentTool, + LLMMessage, + RealtimeAudioFormat, + RealtimeProviderConfig, + RealtimeProviderConnectInput, + RealtimeProviderEvent, + RealtimeProviderSession, + RealtimeToolResult, +} from './types' + +type MaybePromise = T | Promise +type OpenAIRealtimeSocket = { + send: (data: string) => void + close?: (code?: number, reason?: string) => void + addEventListener?: ( + event: string, + handler: (...args: Array) => void + ) => void + removeEventListener?: ( + event: string, + handler: (...args: Array) => void + ) => void + on?: (event: string, handler: (...args: Array) => void) => void + off?: (event: string, handler: (...args: Array) => void) => void + readyState?: number +} +type OpenAIRealtimeWebSocketConstructor = new ( + url: string, + init?: unknown +) => OpenAIRealtimeSocket + +export interface OpenAIRealtimeProviderOptions { + apiKey: string | (() => MaybePromise) + model?: string + url?: string + voice?: string + safetyIdentifier?: string + headers?: Record + WebSocket?: OpenAIRealtimeWebSocketConstructor +} + +type OpenAIRealtimeEvent = Record & { type?: string } + +class AsyncEventQueue implements AsyncIterable { + private values: Array = [] + private resolvers: Array<(value: IteratorResult) => void> = [] + private closed = false + private error: unknown + + push(value: T): void { + if (this.closed) return + const resolve = this.resolvers.shift() + if (resolve) { + resolve({ value, done: false }) + return + } + this.values.push(value) + } + + close(): void { + if (this.closed) return + this.closed = true + for (const resolve of this.resolvers.splice(0)) { + resolve({ value: undefined as T, done: true }) + } + } + + fail(error: unknown): void { + this.error = error + this.close() + } + + [Symbol.asyncIterator](): AsyncIterator { + return { + next: () => { + if (this.values.length > 0) { + return Promise.resolve({ value: this.values.shift()!, done: false }) + } + if (this.error) { + return Promise.reject(this.error) + } + if (this.closed) { + return Promise.resolve({ value: undefined as T, done: true }) + } + return new Promise>((resolve) => { + this.resolvers.push(resolve) + }) + }, + } + } +} + +function resolveWebSocket( + opts: OpenAIRealtimeProviderOptions +): OpenAIRealtimeWebSocketConstructor { + const ctor = opts.WebSocket ?? globalThis.WebSocket + if (!ctor) { + throw new Error( + `[agent-runtime] OpenAI realtime requires a WebSocket implementation` + ) + } + return ctor as unknown as OpenAIRealtimeWebSocketConstructor +} + +function onSocket( + ws: OpenAIRealtimeSocket, + event: string, + handler: (...args: Array) => void +): void { + if (ws.addEventListener) { + ws.addEventListener(event, handler) + return + } + ws.on?.(event, handler) +} + +function socketMessageData(args: Array): unknown { + const [first] = args + if (first && typeof first === `object` && `data` in first) { + return (first as { data: unknown }).data + } + return first +} + +function dataToString(data: unknown): string { + if (typeof data === `string`) return data + if (data instanceof ArrayBuffer) return new TextDecoder().decode(data) + if (data instanceof Uint8Array) return new TextDecoder().decode(data) + if ( + data && + typeof data === `object` && + `toString` in data && + typeof data.toString === `function` + ) { + return data.toString() + } + return String(data) +} + +function bytesToBase64(bytes: Uint8Array): string { + const bufferCtor = (globalThis as { Buffer?: typeof Buffer }).Buffer + if (bufferCtor) return bufferCtor.from(bytes).toString(`base64`) + let binary = `` + for (const byte of bytes) binary += String.fromCharCode(byte) + return btoa(binary) +} + +function base64ToBytes(value: string): Uint8Array { + const bufferCtor = (globalThis as { Buffer?: typeof Buffer }).Buffer + if (bufferCtor) return new Uint8Array(bufferCtor.from(value, `base64`)) + const binary = atob(value) + const bytes = new Uint8Array(binary.length) + for (let index = 0; index < binary.length; index += 1) { + bytes[index] = binary.charCodeAt(index) + } + return bytes +} + +function sendJson(ws: OpenAIRealtimeSocket, event: unknown): void { + ws.send(JSON.stringify(event)) +} + +function toolName(tool: AgentTool): string { + return tool.name +} + +function toOpenAITool(tool: AgentTool): Record { + return { + type: `function`, + name: tool.name, + description: tool.description, + parameters: tool.parameters, + } +} + +function messageContentText(content: unknown): string { + if (typeof content === `string`) return content + if (!Array.isArray(content)) return `` + return content + .map((part) => { + if (!part || typeof part !== `object`) return `` + const text = (part as { text?: unknown }).text + return typeof text === `string` ? text : `` + }) + .filter(Boolean) + .join(`\n`) +} + +function messageRole(message: LLMMessage): `user` | `assistant` | null { + const role = (message as { role?: unknown }).role + return role === `assistant` ? `assistant` : role === `user` ? `user` : null +} + +function sendConversationMessage( + ws: OpenAIRealtimeSocket, + message: LLMMessage +): void { + const role = messageRole(message) + if (!role) return + const text = messageContentText((message as { content?: unknown }).content) + if (!text) return + sendJson(ws, { + type: `conversation.item.create`, + item: { + type: `message`, + role, + content: [ + { + type: role === `assistant` ? `output_text` : `input_text`, + text, + }, + ], + }, + }) +} + +function realtimeFormat( + format: RealtimeAudioFormat | undefined +): Record | undefined { + if (!format) return undefined + return { + type: `audio/pcm`, + rate: format.sampleRate, + } +} + +function buildSessionUpdate( + opts: OpenAIRealtimeProviderOptions, + input: RealtimeProviderConnectInput +): Record { + const inputFormat = realtimeFormat(input.audio?.inputFormat) + const outputFormat = realtimeFormat(input.audio?.outputFormat) + return { + type: `session.update`, + session: { + type: `realtime`, + model: opts.model ?? `gpt-realtime-2`, + instructions: input.systemPrompt, + output_modalities: outputFormat ? [`audio`] : [`text`], + tool_choice: input.tools.length > 0 ? `auto` : `none`, + ...(input.tools.length > 0 + ? { tools: input.tools.map((tool) => toOpenAITool(tool)) } + : {}), + ...(inputFormat || outputFormat || opts.voice + ? { + audio: { + ...(inputFormat ? { input: { format: inputFormat } } : {}), + ...(outputFormat || opts.voice + ? { + output: { + ...(outputFormat ? { format: outputFormat } : {}), + ...(opts.voice ? { voice: opts.voice } : {}), + }, + } + : {}), + }, + } + : {}), + }, + } +} + +function parseToolArgs(value: unknown): unknown { + if (typeof value !== `string`) return value ?? {} + try { + return JSON.parse(value) as unknown + } catch { + return value + } +} + +function toolResultOutput(result: RealtimeToolResult): string { + if (typeof result.result === `string`) return result.result + return JSON.stringify(result.result) +} + +function mapOpenAIEvent( + event: OpenAIRealtimeEvent +): Array { + switch (event.type) { + case `session.created`: + return [{ type: `session.started`, sessionId: event.session?.id }] + case `session.updated`: + return [{ type: `session.updated` }] + case `error`: + return [ + { + type: `session.error`, + error: + typeof event.error?.message === `string` + ? event.error.message + : `OpenAI realtime error`, + code: + typeof event.error?.code === `string` + ? event.error.code + : undefined, + }, + ] + case `input_audio_buffer.speech_started`: + return [ + { + type: `input_audio.speech_started`, + audioOffset: + typeof event.audio_start_ms === `number` + ? String(event.audio_start_ms) + : undefined, + }, + ] + case `input_audio_buffer.speech_stopped`: + return [ + { + type: `input_audio.speech_stopped`, + audioOffset: + typeof event.audio_end_ms === `number` + ? String(event.audio_end_ms) + : undefined, + }, + ] + case `conversation.item.input_audio_transcription.delta`: + return [ + { + type: `input_transcript.delta`, + delta: String(event.delta ?? ``), + turnId: typeof event.item_id === `string` ? event.item_id : undefined, + }, + ] + case `conversation.item.input_audio_transcription.completed`: + return [ + { + type: `input_transcript.completed`, + text: String(event.transcript ?? ``), + turnId: typeof event.item_id === `string` ? event.item_id : undefined, + }, + ] + case `response.created`: + return [ + { + type: `response.started`, + responseId: + typeof event.response?.id === `string` + ? event.response.id + : undefined, + }, + ] + case `response.audio.delta`: + return [ + { + type: `output_audio.delta`, + audio: base64ToBytes(String(event.delta ?? ``)), + responseId: + typeof event.response_id === `string` + ? event.response_id + : undefined, + itemId: typeof event.item_id === `string` ? event.item_id : undefined, + }, + ] + case `response.audio.done`: + return [ + { + type: `output_audio.completed`, + responseId: + typeof event.response_id === `string` + ? event.response_id + : undefined, + itemId: typeof event.item_id === `string` ? event.item_id : undefined, + }, + ] + case `response.audio_transcript.delta`: + case `response.output_text.delta`: + return [ + { + type: `output_transcript.delta`, + delta: String(event.delta ?? ``), + responseId: + typeof event.response_id === `string` + ? event.response_id + : undefined, + }, + ] + case `response.audio_transcript.done`: + case `response.output_text.done`: + return [ + { + type: `output_transcript.completed`, + text: + typeof event.transcript === `string` + ? event.transcript + : typeof event.text === `string` + ? event.text + : undefined, + responseId: + typeof event.response_id === `string` + ? event.response_id + : undefined, + }, + ] + case `response.done`: + return [ + { + type: `response.completed`, + responseId: + typeof event.response?.id === `string` + ? event.response.id + : typeof event.response_id === `string` + ? event.response_id + : undefined, + }, + ] + case `response.cancelled`: + return [ + { + type: `response.cancelled`, + responseId: + typeof event.response_id === `string` + ? event.response_id + : undefined, + }, + ] + case `response.output_item.added`: + if (event.item?.type !== `function_call`) return [] + return [ + { + type: `tool_call.started`, + toolCallId: String(event.item.call_id ?? event.item.id ?? ``), + name: String(event.item.name ?? ``), + }, + ] + case `response.function_call_arguments.delta`: + return [ + { + type: `tool_call.arguments_delta`, + toolCallId: String(event.call_id ?? event.item_id ?? ``), + delta: String(event.delta ?? ``), + }, + ] + default: + return [] + } +} + +export function createOpenAIRealtimeProvider( + opts: OpenAIRealtimeProviderOptions +): RealtimeProviderConfig { + const model = opts.model ?? `gpt-realtime-2` + + return { + id: `openai`, + model, + async connect(input): Promise { + const apiKey = + typeof opts.apiKey === `function` ? await opts.apiKey() : opts.apiKey + if (!apiKey) { + throw new Error(`[agent-runtime] OpenAI realtime apiKey is required`) + } + + const WebSocketCtor = resolveWebSocket(opts) + const url = new URL(opts.url ?? `wss://api.openai.com/v1/realtime`) + url.searchParams.set(`model`, model) + const headers: Record = { + Authorization: `Bearer ${apiKey}`, + ...opts.headers, + } + if (opts.safetyIdentifier) { + headers[`OpenAI-Safety-Identifier`] = opts.safetyIdentifier + } + + const ws = new WebSocketCtor(url.toString(), { headers }) + const queue = new AsyncEventQueue() + const toolsByName = new Map( + input.tools.map((tool) => [toolName(tool), tool]) + ) + + const sendToolResult = async ( + result: RealtimeToolResult + ): Promise => { + sendJson(ws, { + type: `conversation.item.create`, + item: { + type: `function_call_output`, + call_id: result.toolCallId, + output: toolResultOutput(result), + }, + }) + sendJson(ws, { type: `response.create` }) + } + + const executeToolCall = async ( + event: OpenAIRealtimeEvent + ): Promise => { + const item = event.item ?? {} + const toolCallId = String( + event.call_id ?? item.call_id ?? item.id ?? event.item_id ?? `` + ) + const name = String(event.name ?? item.name ?? ``) + const args = parseToolArgs(event.arguments ?? item.arguments) + queue.push({ + type: `tool_call.arguments_completed`, + toolCallId, + name, + args, + }) + const tool = toolsByName.get(name) + if (!tool) { + const result: RealtimeToolResult = { + toolCallId, + name, + result: `Tool "${name}" is not available.`, + isError: true, + } + queue.push({ type: `tool_call.completed`, ...result }) + await sendToolResult(result) + return + } + + try { + const prepared = + typeof tool.prepareArguments === `function` + ? tool.prepareArguments(args) + : args + const result = await tool.execute( + toolCallId, + prepared as never, + input.signal + ) + const realtimeResult: RealtimeToolResult = { + toolCallId, + name, + result, + } + queue.push({ type: `tool_call.completed`, ...realtimeResult }) + await sendToolResult(realtimeResult) + } catch (error) { + const realtimeResult: RealtimeToolResult = { + toolCallId, + name, + result: error instanceof Error ? error.message : String(error), + isError: true, + } + queue.push({ type: `tool_call.completed`, ...realtimeResult }) + await sendToolResult(realtimeResult) + } + } + + const opened = new Promise((resolve, reject) => { + onSocket(ws, `open`, () => resolve()) + onSocket(ws, `error`, (event) => { + const error = + event instanceof Error + ? event + : new Error(`[agent-runtime] OpenAI realtime WebSocket error`) + queue.fail(error) + reject(error) + }) + }) + + onSocket(ws, `message`, (...args) => { + try { + const parsed = JSON.parse( + dataToString(socketMessageData(args)) + ) as OpenAIRealtimeEvent + if (parsed.type === `response.function_call_arguments.done`) { + void executeToolCall(parsed).catch((error) => queue.fail(error)) + return + } + for (const event of mapOpenAIEvent(parsed)) queue.push(event) + } catch (error) { + queue.fail(error) + } + }) + onSocket(ws, `close`, () => { + queue.push({ type: `session.closed` }) + queue.close() + }) + + await opened + sendJson(ws, buildSessionUpdate(opts, input)) + for (const message of input.messages) { + sendConversationMessage(ws, message) + } + + return { + events: queue, + appendInputAudio: async (chunk) => { + sendJson(ws, { + type: `input_audio_buffer.append`, + audio: bytesToBase64(chunk), + }) + }, + commitInputAudio: async () => { + sendJson(ws, { type: `input_audio_buffer.commit` }) + }, + sendText: async (text) => { + sendJson(ws, { + type: `conversation.item.create`, + item: { + type: `message`, + role: `user`, + content: [{ type: `input_text`, text }], + }, + }) + sendJson(ws, { type: `response.create` }) + }, + sendToolResult, + cancelResponse: async () => { + sendJson(ws, { type: `response.cancel` }) + }, + close: async (reason) => { + ws.close?.(1000, reason) + queue.close() + }, + } + }, + } +} diff --git a/packages/agents-runtime/test/openai-realtime.test.ts b/packages/agents-runtime/test/openai-realtime.test.ts new file mode 100644 index 0000000000..de34722ea6 --- /dev/null +++ b/packages/agents-runtime/test/openai-realtime.test.ts @@ -0,0 +1,217 @@ +import { Type } from '@sinclair/typebox' +import { describe, expect, it, vi } from 'vitest' +import { createOpenAIRealtimeProvider } from '../src/openai-realtime' +import type { AgentTool, RealtimeProviderEvent } from '../src/types' + +type Listener = (...args: Array) => void + +class FakeWebSocket { + static instances: Array = [] + + readonly sent: Array = [] + readonly listeners = new Map>() + + constructor( + readonly url: string, + readonly init?: unknown + ) { + FakeWebSocket.instances.push(this) + queueMicrotask(() => this.emit(`open`)) + } + + addEventListener(event: string, listener: Listener): void { + const listeners = this.listeners.get(event) ?? [] + listeners.push(listener) + this.listeners.set(event, listeners) + } + + send(data: string): void { + this.sent.push(JSON.parse(data) as unknown) + } + + close(): void { + this.emit(`close`) + } + + emit(event: string, payload?: unknown): void { + for (const listener of this.listeners.get(event) ?? []) { + listener(payload) + } + } + + emitMessage(payload: unknown): void { + this.emit(`message`, { data: JSON.stringify(payload) }) + } +} + +function nextEvent(iterator: AsyncIterator) { + return iterator.next().then((result) => result.value) +} + +describe(`createOpenAIRealtimeProvider`, () => { + it(`connects over WebSocket and configures session state`, async () => { + FakeWebSocket.instances = [] + const tool: AgentTool = { + name: `lookup`, + label: `Lookup`, + description: `Look up a value`, + parameters: Type.Object({ q: Type.String() }), + execute: vi.fn(), + } + const provider = createOpenAIRealtimeProvider({ + apiKey: `sk-test`, + safetyIdentifier: `user-1`, + WebSocket: FakeWebSocket, + }) + + await provider.connect({ + systemPrompt: `You are Horton.`, + messages: [{ role: `user`, content: `Previous context` } as never], + tools: [tool], + audio: { + inputFormat: { codec: `pcm16`, sampleRate: 24_000, channels: 1 }, + outputFormat: { codec: `pcm16`, sampleRate: 24_000, channels: 1 }, + }, + }) + + const socket = FakeWebSocket.instances[0]! + expect(socket.url).toBe( + `wss://api.openai.com/v1/realtime?model=gpt-realtime-2` + ) + expect(socket.init).toEqual({ + headers: { + Authorization: `Bearer sk-test`, + 'OpenAI-Safety-Identifier': `user-1`, + }, + }) + expect(socket.sent[0]).toMatchObject({ + type: `session.update`, + session: { + type: `realtime`, + model: `gpt-realtime-2`, + instructions: `You are Horton.`, + output_modalities: [`audio`], + tool_choice: `auto`, + tools: [ + { + type: `function`, + name: `lookup`, + description: `Look up a value`, + }, + ], + audio: { + input: { format: { type: `audio/pcm`, rate: 24_000 } }, + output: { format: { type: `audio/pcm`, rate: 24_000 } }, + }, + }, + }) + expect(socket.sent[1]).toEqual({ + type: `conversation.item.create`, + item: { + type: `message`, + role: `user`, + content: [{ type: `input_text`, text: `Previous context` }], + }, + }) + }) + + it(`sends audio input chunks as OpenAI input buffer events`, async () => { + FakeWebSocket.instances = [] + const provider = createOpenAIRealtimeProvider({ + apiKey: `sk-test`, + WebSocket: FakeWebSocket, + }) + + const session = await provider.connect({ + systemPrompt: `Talk`, + messages: [], + tools: [], + }) + const socket = FakeWebSocket.instances[0]! + + await session.appendInputAudio?.(new Uint8Array([1, 2, 3])) + await session.commitInputAudio?.() + + expect(socket.sent.at(-2)).toEqual({ + type: `input_audio_buffer.append`, + audio: `AQID`, + }) + expect(socket.sent.at(-1)).toEqual({ type: `input_audio_buffer.commit` }) + }) + + it(`maps OpenAI events and executes function calls`, async () => { + FakeWebSocket.instances = [] + const execute = vi.fn().mockResolvedValue({ + content: [{ type: `text`, text: `done` }], + details: { ok: true }, + }) + const tool: AgentTool = { + name: `lookup`, + label: `Lookup`, + description: `Look up a value`, + parameters: Type.Object({ q: Type.String() }), + execute, + } + const provider = createOpenAIRealtimeProvider({ + apiKey: `sk-test`, + WebSocket: FakeWebSocket, + }) + + const session = await provider.connect({ + systemPrompt: `Talk`, + messages: [], + tools: [tool], + }) + const socket = FakeWebSocket.instances[0]! + const iterator = session.events[Symbol.asyncIterator]() + + socket.emitMessage({ type: `session.created`, session: { id: `sess-1` } }) + await expect(nextEvent(iterator)).resolves.toEqual({ + type: `session.started`, + sessionId: `sess-1`, + }) + + socket.emitMessage({ + type: `response.output_item.added`, + item: { + type: `function_call`, + id: `fc-1`, + call_id: `call-1`, + name: `lookup`, + }, + }) + await expect(nextEvent(iterator)).resolves.toEqual({ + type: `tool_call.started`, + toolCallId: `call-1`, + name: `lookup`, + }) + + socket.emitMessage({ + type: `response.function_call_arguments.done`, + call_id: `call-1`, + name: `lookup`, + arguments: JSON.stringify({ q: `status` }), + }) + + await expect(nextEvent(iterator)).resolves.toEqual({ + type: `tool_call.arguments_completed`, + toolCallId: `call-1`, + name: `lookup`, + args: { q: `status` }, + }) + await expect(nextEvent(iterator)).resolves.toMatchObject({ + type: `tool_call.completed`, + toolCallId: `call-1`, + name: `lookup`, + }) + expect(execute).toHaveBeenCalledWith(`call-1`, { q: `status` }, undefined) + expect(socket.sent.at(-2)).toMatchObject({ + type: `conversation.item.create`, + item: { + type: `function_call_output`, + call_id: `call-1`, + }, + }) + expect(socket.sent.at(-1)).toEqual({ type: `response.create` }) + }) +}) From 614eee0eb6303041a87138a85c664b44084b63ff Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 12:03:49 +0100 Subject: [PATCH 13/39] feat(agents-runtime): bridge realtime durable streams --- .../agents-runtime/src/context-factory.ts | 186 +++++++++++++++++- packages/agents-runtime/src/process-wake.ts | 4 + .../test/helpers/context-test-helpers.ts | 5 + .../test/realtime-context.test.ts | 109 +++++++++- 4 files changed, 302 insertions(+), 2 deletions(-) diff --git a/packages/agents-runtime/src/context-factory.ts b/packages/agents-runtime/src/context-factory.ts index ee11077fa3..cbad2d67e8 100644 --- a/packages/agents-runtime/src/context-factory.ts +++ b/packages/agents-runtime/src/context-factory.ts @@ -1,4 +1,5 @@ import { queryOnce } from '@durable-streams/state/db' +import { DurableStream } from '@durable-streams/client' import { assembleContext } from './context-assembly' import { createContextEntriesApi } from './context-entries' import { entityStateSchema } from './entity-schema' @@ -18,6 +19,7 @@ import { getCronStreamPath } from './cron-utils' import { runtimeLog } from './log' import { sliceChars } from './token-budget' import { createContextTools } from './tools/context-tools' +import { appendPathToUrl } from './url' import { CACHE_TIERS } from './types' import { composeToolsWithProviders } from './tool-providers' import { validateSlashCommandDefinitions } from './composer-input' @@ -112,6 +114,176 @@ function applyRealtimeToolPolicy( }) } +type RealtimeStreamConfig = NonNullable +type RealtimeControlInput = + | { type: `input_text`; text: string } + | { type: `input_audio.commit` } + | { type: `response.cancel` } + | { type: `session.close`; reason?: string } +type RealtimeStreamIo = { + writeProviderEvent: (event: RealtimeProviderEvent) => Promise + close: () => Promise +} + +function isRealtimeControlInput(value: unknown): value is RealtimeControlInput { + if (!value || typeof value !== `object`) return false + const type = (value as { type?: unknown }).type + return ( + type === `input_text` || + type === `input_audio.commit` || + type === `response.cancel` || + type === `session.close` + ) +} + +function realtimeDurableStream( + streams: RealtimeStreamConfig, + path: string, + contentType: string +): DurableStream { + return new DurableStream({ + url: appendPathToUrl(streams.baseUrl, path), + headers: streams.headers, + contentType, + batching: false, + }) +} + +function jsonBytes(value: unknown): Uint8Array { + return new TextEncoder().encode(JSON.stringify(value)) +} + +function realtimeControlOutput(event: RealtimeProviderEvent): unknown { + if (event.type !== `output_audio.delta`) return event + return { + type: event.type, + responseId: event.responseId, + itemId: event.itemId, + byteLength: event.audio.byteLength, + } +} + +function createRealtimeStreamIo( + config: HandlerContextConfig, + session: ManifestRealtimeSessionEntry | undefined, + providerSession: RealtimeProviderSession +): RealtimeStreamIo | undefined { + if (!config.realtimeStreams || !session) return undefined + + const abort = new AbortController() + const abortFromRun = (): void => abort.abort() + if (config.runSignal?.aborted) { + abort.abort() + } else { + config.runSignal?.addEventListener(`abort`, abortFromRun, { once: true }) + } + + const audioIn = realtimeDurableStream( + config.realtimeStreams, + session.streams.audio_in, + `audio/pcm` + ) + const audioOut = realtimeDurableStream( + config.realtimeStreams, + session.streams.audio_out, + `audio/pcm` + ) + const controlIn = realtimeDurableStream( + config.realtimeStreams, + session.streams.control_in, + `application/json` + ) + const controlOut = realtimeDurableStream( + config.realtimeStreams, + session.streams.control_out, + `application/json` + ) + const tasks: Array> = [] + + if (providerSession.appendInputAudio) { + tasks.push( + (async () => { + const response = await audioIn.stream({ + live: true, + signal: abort.signal, + warnOnHttp: false, + }) + try { + for await (const chunk of response.bodyStream()) { + if (abort.signal.aborted) break + await providerSession.appendInputAudio?.(chunk) + } + } finally { + response.cancel() + } + })().catch((error) => { + if (!abort.signal.aborted) { + runtimeLog.warn( + `[agent-runtime] realtime audio/in pump failed:`, + error + ) + } + }) + ) + } + + tasks.push( + (async () => { + const response = await controlIn.stream({ + live: true, + signal: abort.signal, + json: true, + warnOnHttp: false, + }) + try { + for await (const command of response.jsonStream()) { + if (abort.signal.aborted || !isRealtimeControlInput(command)) { + continue + } + switch (command.type) { + case `input_text`: + await providerSession.sendText?.(command.text) + break + case `input_audio.commit`: + await providerSession.commitInputAudio?.() + break + case `response.cancel`: + await providerSession.cancelResponse?.() + break + case `session.close`: + await providerSession.close?.(command.reason) + abort.abort() + break + } + } + } finally { + response.cancel() + } + })().catch((error) => { + if (!abort.signal.aborted) { + runtimeLog.warn( + `[agent-runtime] realtime control/in pump failed:`, + error + ) + } + }) + ) + + return { + async writeProviderEvent(event) { + if (event.type === `output_audio.delta`) { + await audioOut.append(event.audio) + } + await controlOut.append(jsonBytes(realtimeControlOutput(event))) + }, + async close() { + abort.abort() + config.runSignal?.removeEventListener(`abort`, abortFromRun) + await Promise.allSettled(tasks) + }, + } +} + const MAX_HYDRATED_IMAGE_ATTACHMENTS = 4 const MAX_HYDRATED_IMAGE_ATTACHMENT_BYTES = 10 * 1024 * 1024 @@ -143,6 +315,10 @@ export interface HandlerContextConfig { }) => void | Promise ) => void hydratedWebhookSourceWake?: HydratedWebhookSourceWake | null + realtimeStreams?: { + baseUrl: string + headers?: Record + } doObserve: ( source: ObservationSource, wake?: Wake @@ -1055,6 +1231,7 @@ export function createHandlerContext( const messages = await hydrateAttachmentBlocks( timelineToMessages(config.db) ) + let realtimeIo: RealtimeStreamIo | undefined async function handleProviderEvent( event: RealtimeProviderEvent @@ -1144,7 +1321,7 @@ export function createHandlerContext( config.wakeEvent, config.events, config.wakeOffset, - config.hydratedEventSourceWake + config.hydratedWebhookSourceWake ) const responses = activeRealtimeConfig.testResponses if (Array.isArray(responses)) { @@ -1171,11 +1348,17 @@ export function createHandlerContext( session: activeRealtimeSession(), signal: config.runSignal, }) + realtimeIo = createRealtimeStreamIo( + config, + activeRealtimeSession(), + activeRealtimeProviderSession + ) for await (const event of activeRealtimeProviderSession.events) { if (config.runSignal?.aborted) { break } + await realtimeIo?.writeProviderEvent(event) await handleProviderEvent(event) } } @@ -1197,6 +1380,7 @@ export function createHandlerContext( bridge.onRunEnd({ finishReason: `error` }) throw error } finally { + await realtimeIo?.close() activeRealtimeProviderSession = null } diff --git a/packages/agents-runtime/src/process-wake.ts b/packages/agents-runtime/src/process-wake.ts index edb7a850a8..9a745b158d 100644 --- a/packages/agents-runtime/src/process-wake.ts +++ b/packages/agents-runtime/src/process-wake.ts @@ -2149,6 +2149,10 @@ export async function processWake( activeSignalHandler = handler }, hydratedWebhookSourceWake: await hydrateCurrentWebhookSourceWake(), + realtimeStreams: { + baseUrl, + headers: serverHeaders, + }, doObserve, doSpawn, doFork, diff --git a/packages/agents-runtime/test/helpers/context-test-helpers.ts b/packages/agents-runtime/test/helpers/context-test-helpers.ts index beaf867202..c6db56af30 100644 --- a/packages/agents-runtime/test/helpers/context-test-helpers.ts +++ b/packages/agents-runtime/test/helpers/context-test-helpers.ts @@ -304,6 +304,10 @@ export function createTestHandlerContext( wakeEvent?: WakeEvent hydratedWebhookSourceWake?: HydratedWebhookSourceWake | null prepareAgentRun?: () => Promise + realtimeStreams?: { + baseUrl: string + headers?: Record + } } = {} ) { const db = opts.db ?? buildStreamFixture([]) @@ -334,6 +338,7 @@ export function createTestHandlerContext( payload: `hi`, }, hydratedWebhookSourceWake: opts.hydratedWebhookSourceWake, + realtimeStreams: opts.realtimeStreams, prepareAgentRun: opts.prepareAgentRun, doObserve: vi.fn(), doSpawn: vi.fn(), diff --git a/packages/agents-runtime/test/realtime-context.test.ts b/packages/agents-runtime/test/realtime-context.test.ts index 9e5ebfac2d..a16c88ccb1 100644 --- a/packages/agents-runtime/test/realtime-context.test.ts +++ b/packages/agents-runtime/test/realtime-context.test.ts @@ -1,8 +1,37 @@ -import { describe, expect, it } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' import { createTestRealtimeProvider } from '../src/realtime' import { createTestHandlerContext } from './helpers/context-test-helpers' +const durableMock = vi.hoisted(() => { + const appends: Array<{ url: string; data: unknown }> = [] + class DurableStream { + constructor(readonly opts: { url: string }) {} + + async append(data: unknown): Promise { + appends.push({ url: this.opts.url, data }) + } + + async stream() { + return { + bodyStream: async function* () {}, + jsonStream: async function* () {}, + cancel: vi.fn(), + } + } + } + + return { appends, DurableStream } +}) + +vi.mock(`@durable-streams/client`, () => ({ + DurableStream: durableMock.DurableStream, +})) + describe(`ctx.useRealtime()`, () => { + beforeEach(() => { + durableMock.appends.length = 0 + }) + it(`records provider transcript output through the outbound bridge`, async () => { const { ctx } = createTestHandlerContext() @@ -62,4 +91,82 @@ describe(`ctx.useRealtime()`, () => { status: `active`, }) }) + + it(`persists provider audio and control output to realtime durable streams`, async () => { + const { ctx } = createTestHandlerContext({ + realtimeStreams: { + baseUrl: `http://server.test`, + headers: { authorization: `Bearer claim` }, + }, + }) + ctx.db.collections.manifests.insert({ + key: `realtime-session:rt-1`, + kind: `realtime-session`, + id: `rt-1`, + provider: `openai`, + model: `gpt-realtime-2`, + status: `active`, + startedAt: `2026-06-09T12:00:00.000Z`, + endedAt: null, + retention: `forever`, + streams: { + audio_in: `/test/entity/realtime/rt-1/audio/in`, + audio_out: `/test/entity/realtime/rt-1/audio/out`, + control_in: `/test/entity/realtime/rt-1/control/in`, + control_out: `/test/entity/realtime/rt-1/control/out`, + }, + }) + + const realtime = ctx.useRealtime({ + systemPrompt: `You are realtime.`, + provider: createTestRealtimeProvider({ + events: [ + { type: `session.started`, sessionId: `rt-1` }, + { + type: `output_audio.delta`, + audio: new Uint8Array([1, 2, 3]), + responseId: `resp-1`, + itemId: `item-1`, + }, + { type: `output_audio.completed`, responseId: `resp-1` }, + { type: `session.closed` }, + ], + }), + tools: [], + }) + + await realtime.run() + + expect(durableMock.appends).toEqual([ + { + url: `http://server.test/test/entity/realtime/rt-1/control/out`, + data: expect.any(Uint8Array), + }, + { + url: `http://server.test/test/entity/realtime/rt-1/audio/out`, + data: new Uint8Array([1, 2, 3]), + }, + { + url: `http://server.test/test/entity/realtime/rt-1/control/out`, + data: expect.any(Uint8Array), + }, + { + url: `http://server.test/test/entity/realtime/rt-1/control/out`, + data: expect.any(Uint8Array), + }, + { + url: `http://server.test/test/entity/realtime/rt-1/control/out`, + data: expect.any(Uint8Array), + }, + ]) + const decoder = new TextDecoder() + expect( + JSON.parse(decoder.decode(durableMock.appends[2]!.data as Uint8Array)) + ).toEqual({ + type: `output_audio.delta`, + responseId: `resp-1`, + itemId: `item-1`, + byteLength: 3, + }) + }) }) From 749c9922ff048d296f8a7d0662a1165d6c0ed682 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 12:07:05 +0100 Subject: [PATCH 14/39] feat(agents): route horton realtime sessions --- packages/agents/src/agents/horton.ts | 83 +++++++++++++++-- .../test/horton-tool-composition.test.ts | 92 +++++++++++++++++++ 2 files changed, 165 insertions(+), 10 deletions(-) diff --git a/packages/agents/src/agents/horton.ts b/packages/agents/src/agents/horton.ts index 851d004bbe..19ac4c45db 100644 --- a/packages/agents/src/agents/horton.ts +++ b/packages/agents/src/agents/horton.ts @@ -21,6 +21,7 @@ import { GOAL_SLASH_COMMAND, buildSkillSlashCommands, createContextSkillLoader, + createOpenAIRealtimeProvider, completeWithLowCostModel, dispatchGoalCommand, formatTokenCount, @@ -59,6 +60,15 @@ const TITLE_USER_PROMPT = (userMessage: string): string => `User request:\n${userMessage}` const TITLE_GENERATION_TIMEOUT_MS = 8_000 const HORTON_SKILLS_SLASH_COMMAND_OWNER = `horton:skills` +const HORTON_REALTIME_DIRECT_TOOLS = new Set([ + `web_search`, + `fetch_url`, + `spawn_worker`, + `send`, + `search_electric_agents_docs`, + `use_skill`, + `remove_skill`, +]) const TITLE_STOP_WORDS = new Set([ `a`, @@ -379,6 +389,25 @@ function getToolName(tool: unknown): string | null { return typeof name === `string` ? name : null } +function hortonRealtimeDirectTools(tools: Array): Array { + return tools + .map((tool) => getToolName(tool)) + .filter( + (name): name is string => + name !== null && HORTON_REALTIME_DIRECT_TOOLS.has(name) + ) +} + +function hortonRealtimeSystemPrompt(systemPrompt: string): string { + return `${systemPrompt} + +# Realtime mode +You are speaking with the user live. Keep responses concise enough for voice. +Prefer dispatching workers for coding, shell, edit, or other long-running tasks. +Use direct tools only for lightweight orchestration, lookup, context loading, and sending messages. +When a task may change files, run commands, or take more than a short exchange, spawn a worker and tell the user you are handing it off.` +} + export function createHortonTools( sandbox: Sandbox, ctx: HandlerContext, @@ -849,17 +878,51 @@ function createAssistantHandler(options: { } : undefined + const systemPrompt = buildHortonSystemPrompt(sandboxCwd, { + hasDocsSupport: Boolean(docsSupport), + hasSkills, + docsUrl, + modelProvider: modelConfig.provider, + modelId: String(modelConfig.model), + hasWebhookSourceTools, + hasScheduleTools, + ...(activeGoalPromptInfo && { activeGoal: activeGoalPromptInfo }), + }) + const activeRealtimeSession = ctx.realtime?.activeSession?.() + if (activeRealtimeSession) { + if (activeRealtimeSession.provider !== `openai`) { + throw new Error( + `Horton realtime currently supports provider "openai", got "${activeRealtimeSession.provider}"` + ) + } + const apiKey = process.env.OPENAI_API_KEY + if (!apiKey) { + throw new Error( + `OPENAI_API_KEY must be set before starting Horton realtime mode` + ) + } + const realtime = ctx.useRealtime({ + systemPrompt: hortonRealtimeSystemPrompt(systemPrompt), + provider: createOpenAIRealtimeProvider({ + apiKey, + model: activeRealtimeSession.model, + }), + tools: tools as AgentTool[], + audio: { + inputFormat: { codec: `pcm16`, sampleRate: 24_000, channels: 1 }, + outputFormat: { codec: `pcm16`, sampleRate: 24_000, channels: 1 }, + }, + toolPolicy: { + direct: hortonRealtimeDirectTools(tools as AgentTool[]), + }, + }) + await realtime.run() + await titlePromise + return + } + ctx.useAgent({ - systemPrompt: buildHortonSystemPrompt(sandboxCwd, { - hasDocsSupport: Boolean(docsSupport), - hasSkills, - docsUrl, - modelProvider: modelConfig.provider, - modelId: String(modelConfig.model), - hasWebhookSourceTools, - hasScheduleTools, - ...(activeGoalPromptInfo && { activeGoal: activeGoalPromptInfo }), - }), + systemPrompt, ...modelConfig, // mcp.tools() inserts sentinel objects that the runtime's // composeToolsWithProviders resolves at wake time. The static type of diff --git a/packages/agents/test/horton-tool-composition.test.ts b/packages/agents/test/horton-tool-composition.test.ts index ae255e87a8..004b02a9d5 100644 --- a/packages/agents/test/horton-tool-composition.test.ts +++ b/packages/agents/test/horton-tool-composition.test.ts @@ -143,6 +143,98 @@ describe(`horton tool composition`, () => { await expect(extractFirstUserMessage(ctx)).resolves.toBe(`first`) }) + it(`uses realtime mode as an OpenAI orchestrator when a session is active`, async () => { + const registry = createEntityRegistry() + registerHorton(registry, { workingDirectory: `/tmp`, modelCatalog }) + const previousOpenAIKey = process.env.OPENAI_API_KEY + process.env.OPENAI_API_KEY = `sk-test` + const realtimeRun = vi.fn(async () => {}) + const useRealtime = vi.fn(() => ({ run: realtimeRun })) + const useAgent = vi.fn(() => ({ run: vi.fn(async () => {}) })) + const fakeCtx = { + args: {}, + electricTools: [], + events: [], + firstWake: false, + tags: { title: `Existing title` }, + db: { collections: { inbox: { toArray: [] } } }, + sandbox: { + workingDirectory: `/work`, + readFile: vi.fn(async () => { + throw new Error(`ENOENT`) + }), + }, + slashCommands: { replaceOwned: vi.fn() }, + insertContext: vi.fn(), + removeContext: vi.fn(), + getContext: vi.fn(), + useContext: vi.fn(), + getGoal: vi.fn(() => undefined), + updateGoalUsage: vi.fn(), + useAgent, + useRealtime, + realtime: { + activeSession: () => ({ + key: `realtime-session:rt-1`, + kind: `realtime-session`, + id: `rt-1`, + provider: `openai`, + model: `gpt-realtime-2`, + status: `active`, + startedAt: `2026-06-09T12:00:00.000Z`, + retention: `forever`, + streams: { + audio_in: `/horton/demo/realtime/rt-1/audio/in`, + audio_out: `/horton/demo/realtime/rt-1/audio/out`, + control_in: `/horton/demo/realtime/rt-1/control/in`, + control_out: `/horton/demo/realtime/rt-1/control/out`, + }, + }), + }, + } as any + + try { + await registry + .get(`horton`)! + .definition.handler(fakeCtx, { type: `inbox` } as any) + } finally { + if (previousOpenAIKey === undefined) { + delete process.env.OPENAI_API_KEY + } else { + process.env.OPENAI_API_KEY = previousOpenAIKey + } + } + + expect(useAgent).not.toHaveBeenCalled() + expect(useRealtime).toHaveBeenCalledTimes(1) + expect(realtimeRun).toHaveBeenCalledTimes(1) + const realtimeConfig = ( + useRealtime.mock.calls as unknown as Array< + [ + { + provider: { id: string; model: string } + toolPolicy: { direct: Array } + }, + ] + > + )[0]![0] + expect(realtimeConfig.provider).toMatchObject({ + id: `openai`, + model: `gpt-realtime-2`, + }) + expect(realtimeConfig.toolPolicy.direct).toEqual( + expect.arrayContaining([ + `web_search`, + `fetch_url`, + `spawn_worker`, + `send`, + ]) + ) + expect(realtimeConfig.toolPolicy.direct).not.toEqual( + expect.arrayContaining([`bash`, `read`, `write`, `edit`]) + ) + }) + it(`orders title candidates with the _seq fallback convention`, async () => { const ctx = { db: { From ca8781f9d756d05624688b6d2745c62582dc7284 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 12:10:08 +0100 Subject: [PATCH 15/39] feat(agents-ui): add realtime voice toggle --- .../src/components/MessageInput.module.css | 5 + .../src/components/MessageInput.tsx | 96 +++++++- .../src/lib/realtime-audio.ts | 213 ++++++++++++++++++ 3 files changed, 304 insertions(+), 10 deletions(-) create mode 100644 packages/agents-server-ui/src/lib/realtime-audio.ts diff --git a/packages/agents-server-ui/src/components/MessageInput.module.css b/packages/agents-server-ui/src/components/MessageInput.module.css index bf5683dd52..d428c3df21 100644 --- a/packages/agents-server-ui/src/components/MessageInput.module.css +++ b/packages/agents-server-ui/src/components/MessageInput.module.css @@ -65,6 +65,11 @@ color: var(--ds-text-1); } +.inlineIconButton.voiceActive { + background: var(--ds-accent-a3); + color: var(--ds-accent-11); +} + .inlineIconButton:focus-visible { outline: 2px solid var(--ds-accent-a6); outline-offset: -2px; diff --git a/packages/agents-server-ui/src/components/MessageInput.tsx b/packages/agents-server-ui/src/components/MessageInput.tsx index 906e4e890b..af930ebb11 100644 --- a/packages/agents-server-ui/src/components/MessageInput.tsx +++ b/packages/agents-server-ui/src/components/MessageInput.tsx @@ -1,5 +1,5 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' -import { ArrowUp, Square } from 'lucide-react' +import { ArrowUp, Mic, MicOff, Square } from 'lucide-react' import { useLiveQuery } from '@tanstack/react-db' import type { EntityStreamDBWithActions } from '@electric-ax/agents-runtime/client' import { @@ -18,6 +18,10 @@ import { parseGoalCommand, serializeComposerInput, } from '@electric-ax/agents-runtime/client' +import { + startRealtimeAudioSession, + type RealtimeAudioSession, +} from '../lib/realtime-audio' import { ComposerEditor } from './ComposerEditor' import { ComposerShell } from './ComposerShell' import { Icon, Stack, Text, Tooltip } from '../ui' @@ -117,6 +121,9 @@ export function MessageInput({ key: string originalText: string } | null>(null) + const [realtimePending, setRealtimePending] = useState(false) + const [realtimeActive, setRealtimeActive] = useState(false) + const realtimeSessionRef = useRef(null) const composerFocusRef = useRef<{ focus: () => void } | null>(null) const inputDisabled = disabled || writeDisabled const isCommentMode = composerMode === `comment` @@ -227,6 +234,15 @@ export function MessageInput({ attachmentCount === 0 && !disabled const canStop = showStop && !stopPending && !stopDisabled + const canUseRealtime = + !inputDisabled && !editingMessage && !isCommentMode && Boolean(baseUrl) + + useEffect(() => { + return () => { + void realtimeSessionRef.current?.stop() + realtimeSessionRef.current = null + } + }, []) const handleSubmit = useCallback( (composerPayload?: ComposerInputPayload) => { @@ -318,6 +334,37 @@ export function MessageInput({ handleSubmit() }, [canStop, handleSubmit, onStop]) + const handleRealtimeToggle = useCallback(() => { + if (realtimePending) return + setError(null) + if (realtimeSessionRef.current) { + const session = realtimeSessionRef.current + realtimeSessionRef.current = null + setRealtimePending(true) + session + .stop() + .catch((err: Error) => setError(err.message)) + .finally(() => { + setRealtimeActive(false) + setRealtimePending(false) + }) + return + } + if (!canUseRealtime) return + setRealtimePending(true) + startRealtimeAudioSession({ baseUrl, entityUrl }) + .then((session) => { + realtimeSessionRef.current = session + setRealtimeActive(true) + }) + .catch((err: Error) => { + setError(err.message) + }) + .finally(() => { + setRealtimePending(false) + }) + }, [baseUrl, canUseRealtime, entityUrl, realtimePending]) + const startEditing = useCallback( (message: EntityTimelineData[`inbox`][number]) => { if (inputDisabled) return @@ -472,15 +519,44 @@ export function MessageInput({ ) : null } controls={ - imageAttachmentsEnabled && !isCommentMode ? ( - - ) : null + <> + {!isCommentMode ? ( + + + + + + ) : null} + {imageAttachmentsEnabled && !isCommentMode ? ( + + ) : null} + } send={ diff --git a/packages/agents-server-ui/src/lib/realtime-audio.ts b/packages/agents-server-ui/src/lib/realtime-audio.ts new file mode 100644 index 0000000000..c5a605b8da --- /dev/null +++ b/packages/agents-server-ui/src/lib/realtime-audio.ts @@ -0,0 +1,213 @@ +import { DurableStream } from '@durable-streams/client' +import { appendPathToUrl } from '@electric-ax/agents-runtime/client' +import { serverFetch, getConfiguredServerHeaders } from './auth-fetch' + +export type RealtimeAudioSession = { + sessionId: string + stop: () => Promise +} + +type RealtimeSessionCreateResult = { + sessionId: string + streams: { + audio_in: string + audio_out: string + control_in: string + control_out: string + } +} + +const REALTIME_SAMPLE_RATE = 24_000 + +function realtimeUrl(baseUrl: string): string { + return appendPathToUrl(baseUrl, `/_electric/realtime/sessions`) +} + +function streamUrl(baseUrl: string, streamPath: string): string { + return appendPathToUrl(baseUrl, streamPath) +} + +function pcm16Bytes(input: Float32Array): Uint8Array { + const bytes = new Uint8Array(input.length * 2) + const view = new DataView(bytes.buffer) + for (let index = 0; index < input.length; index += 1) { + const sample = Math.max(-1, Math.min(1, input[index] ?? 0)) + view.setInt16( + index * 2, + sample < 0 ? sample * 0x8000 : sample * 0x7fff, + true + ) + } + return bytes +} + +function pcm16Floats(bytes: Uint8Array): Float32Array { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + const output = new Float32Array(Math.floor(bytes.byteLength / 2)) + for (let index = 0; index < output.length; index += 1) { + output[index] = view.getInt16(index * 2, true) / 0x8000 + } + return output +} + +function streamHandle( + baseUrl: string, + path: string, + contentType: string +): DurableStream { + const url = streamUrl(baseUrl, path) + return new DurableStream({ + url, + headers: getConfiguredServerHeaders(url), + contentType, + batching: false, + }) +} + +function createAudioContext(): AudioContext { + return new AudioContext({ sampleRate: REALTIME_SAMPLE_RATE }) +} + +async function createRealtimeSession( + baseUrl: string, + entityUrl: string +): Promise { + const response = await serverFetch(realtimeUrl(baseUrl), { + method: `POST`, + headers: { 'content-type': `application/json` }, + body: JSON.stringify({ + entityUrl, + provider: `openai`, + model: `gpt-realtime-2`, + inputAudio: { + codec: `pcm16`, + sampleRate: REALTIME_SAMPLE_RATE, + channels: 1, + }, + outputAudio: { + codec: `pcm16`, + sampleRate: REALTIME_SAMPLE_RATE, + channels: 1, + }, + meta: { source: `agents-server-ui` }, + }), + }) + if (!response.ok) { + throw new Error( + `Failed to start realtime session (${response.status}): ${await response.text()}` + ) + } + return (await response.json()) as RealtimeSessionCreateResult +} + +export async function startRealtimeAudioSession({ + baseUrl, + entityUrl, +}: { + baseUrl: string + entityUrl: string +}): Promise { + const session = await createRealtimeSession(baseUrl, entityUrl) + const abort = new AbortController() + const micContext = createAudioContext() + const playbackContext = createAudioContext() + const media = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + sampleRate: REALTIME_SAMPLE_RATE, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }) + const audioIn = streamHandle( + baseUrl, + session.streams.audio_in, + `audio/pcm; rate=${REALTIME_SAMPLE_RATE}; channels=1` + ) + const audioOut = streamHandle( + baseUrl, + session.streams.audio_out, + `audio/pcm; rate=${REALTIME_SAMPLE_RATE}; channels=1` + ) + const controlIn = streamHandle( + baseUrl, + session.streams.control_in, + `application/json` + ) + + const source = micContext.createMediaStreamSource(media) + const processor = micContext.createScriptProcessor(1024, 1, 1) + const silentOutput = micContext.createGain() + silentOutput.gain.value = 0 + let appendQueue = Promise.resolve() + processor.onaudioprocess = (event) => { + if (abort.signal.aborted) return + const input = event.inputBuffer.getChannelData(0) + const bytes = pcm16Bytes(input) + appendQueue = appendQueue + .then(() => audioIn.append(bytes)) + .catch((error) => { + console.warn(`[realtime-audio] microphone append failed`, error) + }) + } + source.connect(processor) + processor.connect(silentOutput) + silentOutput.connect(micContext.destination) + + let nextPlaybackTime = playbackContext.currentTime + const playback = (async () => { + const response = await audioOut.stream({ + live: true, + signal: abort.signal, + warnOnHttp: false, + }) + try { + for await (const chunk of response.bodyStream()) { + if (abort.signal.aborted || chunk.byteLength === 0) continue + const samples = pcm16Floats(chunk) + const buffer = playbackContext.createBuffer( + 1, + samples.length, + REALTIME_SAMPLE_RATE + ) + const channel = new Float32Array(samples.length) + channel.set(samples) + buffer.copyToChannel(channel, 0) + const node = playbackContext.createBufferSource() + node.buffer = buffer + node.connect(playbackContext.destination) + const startAt = Math.max(playbackContext.currentTime, nextPlaybackTime) + node.start(startAt) + nextPlaybackTime = startAt + buffer.duration + } + } finally { + response.cancel() + } + })().catch((error) => { + if (!abort.signal.aborted) { + console.warn(`[realtime-audio] playback stream failed`, error) + } + }) + + return { + sessionId: session.sessionId, + async stop() { + abort.abort() + processor.disconnect() + silentOutput.disconnect() + source.disconnect() + for (const track of media.getTracks()) track.stop() + await appendQueue.catch(() => undefined) + await controlIn + .append( + new TextEncoder().encode( + JSON.stringify({ type: `session.close`, reason: `client-stop` }) + ) + ) + .catch(() => undefined) + await playback + await Promise.allSettled([micContext.close(), playbackContext.close()]) + }, + } +} From 0f2eac52b68e8545dabad5161add4a58033e14b8 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 12:10:50 +0100 Subject: [PATCH 16/39] feat(agents-ui): route realtime text input --- .../agents-server-ui/src/components/MessageInput.tsx | 10 ++++++++++ packages/agents-server-ui/src/lib/realtime-audio.ts | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/packages/agents-server-ui/src/components/MessageInput.tsx b/packages/agents-server-ui/src/components/MessageInput.tsx index af930ebb11..78361effdd 100644 --- a/packages/agents-server-ui/src/components/MessageInput.tsx +++ b/packages/agents-server-ui/src/components/MessageInput.tsx @@ -270,6 +270,16 @@ export function MessageInput({ return } const files = imageAttachmentsEnabled ? attachments : [] + if (realtimeSessionRef.current && !editingMessage && files.length === 0) { + const session = realtimeSessionRef.current + setValue(``) + onSend?.() + session.sendText(text).catch((err: Error) => { + setError(err.message) + setValue((current) => (current ? current : text)) + }) + return + } const tx = editingMessage ? updateAction?.({ key: editingMessage.key, diff --git a/packages/agents-server-ui/src/lib/realtime-audio.ts b/packages/agents-server-ui/src/lib/realtime-audio.ts index c5a605b8da..524868dfe4 100644 --- a/packages/agents-server-ui/src/lib/realtime-audio.ts +++ b/packages/agents-server-ui/src/lib/realtime-audio.ts @@ -4,6 +4,7 @@ import { serverFetch, getConfiguredServerHeaders } from './auth-fetch' export type RealtimeAudioSession = { sessionId: string + sendText: (text: string) => Promise stop: () => Promise } @@ -192,6 +193,11 @@ export async function startRealtimeAudioSession({ return { sessionId: session.sessionId, + async sendText(text: string) { + await controlIn.append( + new TextEncoder().encode(JSON.stringify({ type: `input_text`, text })) + ) + }, async stop() { abort.abort() processor.disconnect() From 2cea38766db9f73a710a76dddf9508530c867757 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 13:00:53 +0100 Subject: [PATCH 17/39] fix(agents): harden realtime session lifecycle --- .../agents-runtime/src/context-factory.ts | 81 ++++- .../agents-runtime/src/openai-realtime.ts | 52 ++- .../test/openai-realtime.test.ts | 56 +++- .../test/realtime-context.test.ts | 90 +++++ .../src/lib/realtime-audio.ts | 310 ++++++++++++------ packages/agents-server/src/entity-manager.ts | 7 + ...ic-agents-manager-write-validation.test.ts | 15 + packages/agents/src/agents/horton.ts | 16 +- 8 files changed, 516 insertions(+), 111 deletions(-) diff --git a/packages/agents-runtime/src/context-factory.ts b/packages/agents-runtime/src/context-factory.ts index cbad2d67e8..cbf25fc777 100644 --- a/packages/agents-runtime/src/context-factory.ts +++ b/packages/agents-runtime/src/context-factory.ts @@ -119,6 +119,7 @@ type RealtimeControlInput = | { type: `input_text`; text: string } | { type: `input_audio.commit` } | { type: `response.cancel` } + | { type: `output_audio.truncate`; itemId: string; audioEndMs: number } | { type: `session.close`; reason?: string } type RealtimeStreamIo = { writeProviderEvent: (event: RealtimeProviderEvent) => Promise @@ -128,6 +129,12 @@ type RealtimeStreamIo = { function isRealtimeControlInput(value: unknown): value is RealtimeControlInput { if (!value || typeof value !== `object`) return false const type = (value as { type?: unknown }).type + if (type === `output_audio.truncate`) { + return ( + typeof (value as { itemId?: unknown }).itemId === `string` && + typeof (value as { audioEndMs?: unknown }).audioEndMs === `number` + ) + } return ( type === `input_text` || type === `input_audio.commit` || @@ -250,6 +257,12 @@ function createRealtimeStreamIo( case `response.cancel`: await providerSession.cancelResponse?.() break + case `output_audio.truncate`: + await providerSession.truncateOutputAudio?.({ + itemId: command.itemId, + audioEndMs: command.audioEndMs, + }) + break case `session.close`: await providerSession.close?.(command.reason) abort.abort() @@ -775,6 +788,57 @@ export function createHandlerContext( return realtimeSessions().filter(realtimeManifestIsActive).at(-1) } + async function updateRealtimeSessionStatus( + session: ManifestRealtimeSessionEntry | undefined, + status: `active` | `closed` | `failed`, + opts: { reason?: string; error?: string } = {} + ): Promise { + if (!session) return + + const key = session.key ?? `realtime-session:${session.id}` + const terminal = status === `closed` || status === `failed` + const endedAt = terminal ? new Date().toISOString() : session.endedAt + const meta = { + ...(session.meta ?? {}), + ...(opts.reason ? { reason: opts.reason } : {}), + ...(opts.error ? { error: opts.error } : {}), + } + + const nextSession: ManifestRealtimeSessionEntry = { + key, + kind: `realtime-session`, + id: session.id, + provider: session.provider, + model: session.model, + status, + startedAt: session.startedAt, + endedAt: endedAt ?? null, + streams: session.streams, + retention: `forever`, + ...(Object.keys(meta).length > 0 ? { meta } : {}), + } + + config.wakeSession.registerManifestEntry(nextSession) + config.writeEvent( + entityStateSchema.realtimeSessions.update({ + key, + value: { + session_id: session.id, + provider: session.provider, + model: session.model, + status, + started_at: session.startedAt, + ...(endedAt ? { ended_at: endedAt } : {}), + streams: session.streams, + ...(opts.reason ? { reason: opts.reason } : {}), + ...(opts.error ? { error: opts.error } : {}), + ...(Object.keys(meta).length > 0 ? { meta } : {}), + } as never, + }) as ChangeEvent + ) + await config.wakeSession.commitManifestEntries() + } + function structuralHash(nextConfig: UseContextConfig): string { const sources = Object.entries(nextConfig.sources) .sort(([leftName], [rightName]) => leftName.localeCompare(rightName)) @@ -1232,6 +1296,8 @@ export function createHandlerContext( timelineToMessages(config.db) ) let realtimeIo: RealtimeStreamIo | undefined + const realtimeSession = activeRealtimeSession() + let realtimeSessionTerminalWritten = false async function handleProviderEvent( event: RealtimeProviderEvent @@ -1345,12 +1411,13 @@ export function createHandlerContext( messages, tools: providerTools, audio: activeRealtimeConfig.audio, - session: activeRealtimeSession(), + session: realtimeSession, signal: config.runSignal, }) + await updateRealtimeSessionStatus(realtimeSession, `active`) realtimeIo = createRealtimeStreamIo( config, - activeRealtimeSession(), + realtimeSession, activeRealtimeProviderSession ) @@ -1364,6 +1431,10 @@ export function createHandlerContext( } endText() + await updateRealtimeSessionStatus(realtimeSession, `closed`, { + reason: config.runSignal?.aborted ? `aborted` : `completed`, + }) + realtimeSessionTerminalWritten = true bridge.onStepEnd({ finishReason: config.runSignal?.aborted ? `aborted` : `stop`, durationMs: Date.now() - startedAt, @@ -1373,6 +1444,12 @@ export function createHandlerContext( }) } catch (error) { endText() + if (!realtimeSessionTerminalWritten) { + await updateRealtimeSessionStatus(realtimeSession, `failed`, { + error: error instanceof Error ? error.message : String(error), + }) + realtimeSessionTerminalWritten = true + } bridge.onStepEnd({ finishReason: `error`, durationMs: Date.now() - startedAt, diff --git a/packages/agents-runtime/src/openai-realtime.ts b/packages/agents-runtime/src/openai-realtime.ts index 649fe1a25c..72fdb5ea46 100644 --- a/packages/agents-runtime/src/openai-realtime.ts +++ b/packages/agents-runtime/src/openai-realtime.ts @@ -470,6 +470,26 @@ export function createOpenAIRealtimeProvider( const toolsByName = new Map( input.tools.map((tool) => [toolName(tool), tool]) ) + let socketOpen = false + let socketClosed = false + let rejectOpen: ((error: Error) => void) | undefined + + const closeQueue = (reason?: string): void => { + if (socketClosed) return + socketClosed = true + queue.push({ type: `session.closed`, reason }) + queue.close() + input.signal?.removeEventListener(`abort`, handleAbort) + } + + const handleAbort = (): void => { + const error = new Error( + `[agent-runtime] OpenAI realtime WebSocket aborted` + ) + closeQueue(`aborted`) + ws.close?.(1000, `aborted`) + if (!socketOpen) rejectOpen?.(error) + } const sendToolResult = async ( result: RealtimeToolResult @@ -543,12 +563,22 @@ export function createOpenAIRealtimeProvider( } const opened = new Promise((resolve, reject) => { - onSocket(ws, `open`, () => resolve()) + rejectOpen = reject + onSocket(ws, `open`, () => { + if (socketClosed) return + socketOpen = true + if (input.signal?.aborted) { + handleAbort() + return + } + resolve() + }) onSocket(ws, `error`, (event) => { const error = event instanceof Error ? event : new Error(`[agent-runtime] OpenAI realtime WebSocket error`) + input.signal?.removeEventListener(`abort`, handleAbort) queue.fail(error) reject(error) }) @@ -569,10 +599,15 @@ export function createOpenAIRealtimeProvider( } }) onSocket(ws, `close`, () => { - queue.push({ type: `session.closed` }) - queue.close() + closeQueue() }) + if (input.signal?.aborted) { + handleAbort() + } else { + input.signal?.addEventListener(`abort`, handleAbort, { once: true }) + } + await opened sendJson(ws, buildSessionUpdate(opts, input)) for (const message of input.messages) { @@ -589,6 +624,7 @@ export function createOpenAIRealtimeProvider( }, commitInputAudio: async () => { sendJson(ws, { type: `input_audio_buffer.commit` }) + sendJson(ws, { type: `response.create` }) }, sendText: async (text) => { sendJson(ws, { @@ -605,9 +641,17 @@ export function createOpenAIRealtimeProvider( cancelResponse: async () => { sendJson(ws, { type: `response.cancel` }) }, + truncateOutputAudio: async ({ itemId, audioEndMs }) => { + sendJson(ws, { + type: `conversation.item.truncate`, + item_id: itemId, + content_index: 0, + audio_end_ms: audioEndMs, + }) + }, close: async (reason) => { + closeQueue(reason) ws.close?.(1000, reason) - queue.close() }, } }, diff --git a/packages/agents-runtime/test/openai-realtime.test.ts b/packages/agents-runtime/test/openai-realtime.test.ts index de34722ea6..9fc88444c3 100644 --- a/packages/agents-runtime/test/openai-realtime.test.ts +++ b/packages/agents-runtime/test/openai-realtime.test.ts @@ -132,11 +132,63 @@ describe(`createOpenAIRealtimeProvider`, () => { await session.appendInputAudio?.(new Uint8Array([1, 2, 3])) await session.commitInputAudio?.() - expect(socket.sent.at(-2)).toEqual({ + expect(socket.sent.at(-3)).toEqual({ type: `input_audio_buffer.append`, audio: `AQID`, }) - expect(socket.sent.at(-1)).toEqual({ type: `input_audio_buffer.commit` }) + expect(socket.sent.at(-2)).toEqual({ type: `input_audio_buffer.commit` }) + expect(socket.sent.at(-1)).toEqual({ type: `response.create` }) + }) + + it(`unblocks the event stream when the run signal aborts`, async () => { + FakeWebSocket.instances = [] + const controller = new AbortController() + const provider = createOpenAIRealtimeProvider({ + apiKey: `sk-test`, + WebSocket: FakeWebSocket, + }) + + const session = await provider.connect({ + systemPrompt: `Talk`, + messages: [], + tools: [], + signal: controller.signal, + }) + const iterator = session.events[Symbol.asyncIterator]() + + controller.abort() + + await expect(nextEvent(iterator)).resolves.toEqual({ + type: `session.closed`, + reason: `aborted`, + }) + }) + + it(`can truncate output audio for interrupted playback`, async () => { + FakeWebSocket.instances = [] + const provider = createOpenAIRealtimeProvider({ + apiKey: `sk-test`, + WebSocket: FakeWebSocket, + }) + + const session = await provider.connect({ + systemPrompt: `Talk`, + messages: [], + tools: [], + }) + const socket = FakeWebSocket.instances[0]! + + await session.truncateOutputAudio?.({ + itemId: `item-1`, + audioEndMs: 320, + }) + + expect(socket.sent.at(-1)).toEqual({ + type: `conversation.item.truncate`, + item_id: `item-1`, + content_index: 0, + audio_end_ms: 320, + }) }) it(`maps OpenAI events and executes function calls`, async () => { diff --git a/packages/agents-runtime/test/realtime-context.test.ts b/packages/agents-runtime/test/realtime-context.test.ts index a16c88ccb1..7a4c129ccb 100644 --- a/packages/agents-runtime/test/realtime-context.test.ts +++ b/packages/agents-runtime/test/realtime-context.test.ts @@ -92,6 +92,96 @@ describe(`ctx.useRealtime()`, () => { }) }) + it(`marks realtime sessions closed when the provider stream ends`, async () => { + const { ctx } = createTestHandlerContext() + + ctx.db.collections.manifests.insert({ + key: `realtime-session:rt-1`, + kind: `realtime-session`, + id: `rt-1`, + provider: `openai`, + model: `gpt-realtime-2`, + status: `requested`, + startedAt: `2026-06-09T12:00:00.000Z`, + endedAt: null, + retention: `forever`, + streams: { + audio_in: `/entities/test/realtime/rt-1/audio/in`, + audio_out: `/entities/test/realtime/rt-1/audio/out`, + control_in: `/entities/test/realtime/rt-1/control/in`, + control_out: `/entities/test/realtime/rt-1/control/out`, + }, + }) + + const realtime = ctx.useRealtime({ + systemPrompt: `You are realtime.`, + provider: createTestRealtimeProvider({ response: `done` }), + tools: [], + }) + + await realtime.run() + + expect(ctx.realtime.activeSession()).toBeUndefined() + expect( + ctx.db.collections.manifests.get(`realtime-session:rt-1`) + ).toMatchObject({ + status: `closed`, + endedAt: expect.any(String), + meta: { reason: `completed` }, + }) + expect( + ctx.db.collections.realtimeSessions.get(`realtime-session:rt-1`) + ).toMatchObject({ + status: `closed`, + ended_at: expect.any(String), + reason: `completed`, + }) + }) + + it(`marks realtime sessions failed when provider setup fails`, async () => { + const { ctx } = createTestHandlerContext() + + ctx.db.collections.manifests.insert({ + key: `realtime-session:rt-1`, + kind: `realtime-session`, + id: `rt-1`, + provider: `openai`, + model: `gpt-realtime-2`, + status: `requested`, + startedAt: `2026-06-09T12:00:00.000Z`, + endedAt: null, + retention: `forever`, + streams: { + audio_in: `/entities/test/realtime/rt-1/audio/in`, + audio_out: `/entities/test/realtime/rt-1/audio/out`, + control_in: `/entities/test/realtime/rt-1/control/in`, + control_out: `/entities/test/realtime/rt-1/control/out`, + }, + }) + + const realtime = ctx.useRealtime({ + systemPrompt: `You are realtime.`, + provider: { + id: `openai`, + model: `gpt-realtime-2`, + connect: async () => { + throw new Error(`missing key`) + }, + }, + tools: [], + }) + + await expect(realtime.run()).rejects.toThrow(`missing key`) + expect(ctx.realtime.activeSession()).toBeUndefined() + expect( + ctx.db.collections.manifests.get(`realtime-session:rt-1`) + ).toMatchObject({ + status: `failed`, + endedAt: expect.any(String), + meta: { error: `missing key` }, + }) + }) + it(`persists provider audio and control output to realtime durable streams`, async () => { const { ctx } = createTestHandlerContext({ realtimeStreams: { diff --git a/packages/agents-server-ui/src/lib/realtime-audio.ts b/packages/agents-server-ui/src/lib/realtime-audio.ts index 524868dfe4..8d110e8634 100644 --- a/packages/agents-server-ui/src/lib/realtime-audio.ts +++ b/packages/agents-server-ui/src/lib/realtime-audio.ts @@ -18,6 +18,15 @@ type RealtimeSessionCreateResult = { } } +type RealtimeControlOutput = + | { type: `input_audio.speech_started`; audioOffset?: string } + | { type: `output_audio.delta`; itemId?: string; byteLength?: number } + | { type: `output_audio.completed`; responseId?: string; itemId?: string } + | { type: `response.completed`; responseId?: string } + | { type: `response.cancelled`; responseId?: string } + | { type: `session.closed`; reason?: string } + | { type: string; [key: string]: unknown } + const REALTIME_SAMPLE_RATE = 24_000 function realtimeUrl(baseUrl: string): string { @@ -51,6 +60,10 @@ function pcm16Floats(bytes: Uint8Array): Float32Array { return output } +function jsonBytes(value: unknown): Uint8Array { + return new TextEncoder().encode(JSON.stringify(value)) +} + function streamHandle( baseUrl: string, path: string, @@ -108,112 +121,217 @@ export async function startRealtimeAudioSession({ baseUrl: string entityUrl: string }): Promise { - const session = await createRealtimeSession(baseUrl, entityUrl) const abort = new AbortController() const micContext = createAudioContext() const playbackContext = createAudioContext() - const media = await navigator.mediaDevices.getUserMedia({ - audio: { - channelCount: 1, - sampleRate: REALTIME_SAMPLE_RATE, - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, - }, - }) - const audioIn = streamHandle( - baseUrl, - session.streams.audio_in, - `audio/pcm; rate=${REALTIME_SAMPLE_RATE}; channels=1` - ) - const audioOut = streamHandle( - baseUrl, - session.streams.audio_out, - `audio/pcm; rate=${REALTIME_SAMPLE_RATE}; channels=1` - ) - const controlIn = streamHandle( - baseUrl, - session.streams.control_in, - `application/json` - ) - - const source = micContext.createMediaStreamSource(media) - const processor = micContext.createScriptProcessor(1024, 1, 1) - const silentOutput = micContext.createGain() - silentOutput.gain.value = 0 let appendQueue = Promise.resolve() - processor.onaudioprocess = (event) => { - if (abort.signal.aborted) return - const input = event.inputBuffer.getChannelData(0) - const bytes = pcm16Bytes(input) - appendQueue = appendQueue - .then(() => audioIn.append(bytes)) - .catch((error) => { - console.warn(`[realtime-audio] microphone append failed`, error) - }) + let playback = Promise.resolve() + let control = Promise.resolve() + let media: MediaStream | undefined + let source: MediaStreamAudioSourceNode | undefined + let processor: ScriptProcessorNode | undefined + let silentOutput: GainNode | undefined + let controlIn: DurableStream | undefined + let session: RealtimeSessionCreateResult | undefined + let nextPlaybackTime = playbackContext.currentTime + let currentOutputItemId: string | null = null + let currentOutputStartedAt: number | null = null + const playbackNodes = new Set() + + const appendControl = async (value: unknown): Promise => { + await controlIn?.append(jsonBytes(value)) } - source.connect(processor) - processor.connect(silentOutput) - silentOutput.connect(micContext.destination) - let nextPlaybackTime = playbackContext.currentTime - const playback = (async () => { - const response = await audioOut.stream({ - live: true, - signal: abort.signal, - warnOnHttp: false, - }) - try { - for await (const chunk of response.bodyStream()) { - if (abort.signal.aborted || chunk.byteLength === 0) continue - const samples = pcm16Floats(chunk) - const buffer = playbackContext.createBuffer( - 1, - samples.length, - REALTIME_SAMPLE_RATE - ) - const channel = new Float32Array(samples.length) - channel.set(samples) - buffer.copyToChannel(channel, 0) - const node = playbackContext.createBufferSource() - node.buffer = buffer - node.connect(playbackContext.destination) - const startAt = Math.max(playbackContext.currentTime, nextPlaybackTime) - node.start(startAt) - nextPlaybackTime = startAt + buffer.duration + const stopScheduledPlayback = (): void => { + for (const node of playbackNodes) { + try { + node.stop() + } catch { + // Already stopped. } - } finally { - response.cancel() } - })().catch((error) => { - if (!abort.signal.aborted) { - console.warn(`[realtime-audio] playback stream failed`, error) + playbackNodes.clear() + nextPlaybackTime = playbackContext.currentTime + currentOutputStartedAt = null + } + + const interruptPlayback = (): void => { + const audioEndMs = + currentOutputStartedAt === null + ? 0 + : Math.max( + 0, + Math.floor( + (playbackContext.currentTime - currentOutputStartedAt) * 1000 + ) + ) + const itemId = currentOutputItemId + stopScheduledPlayback() + void appendControl({ type: `response.cancel` }).catch((error) => { + console.warn(`[realtime-audio] response cancel failed`, error) + }) + if (itemId) { + void appendControl({ + type: `output_audio.truncate`, + itemId, + audioEndMs, + }).catch((error) => { + console.warn(`[realtime-audio] output truncate failed`, error) + }) + } + } + + const cleanup = async (sendClose: boolean): Promise => { + abort.abort() + processor?.disconnect() + silentOutput?.disconnect() + source?.disconnect() + for (const track of media?.getTracks() ?? []) track.stop() + stopScheduledPlayback() + await appendQueue.catch(() => undefined) + if (sendClose && controlIn) { + await appendControl({ + type: `session.close`, + reason: `client-stop`, + }).catch(() => undefined) } - }) + await Promise.allSettled([playback, control]) + await Promise.allSettled([micContext.close(), playbackContext.close()]) + } - return { - sessionId: session.sessionId, - async sendText(text: string) { - await controlIn.append( - new TextEncoder().encode(JSON.stringify({ type: `input_text`, text })) - ) - }, - async stop() { - abort.abort() - processor.disconnect() - silentOutput.disconnect() - source.disconnect() - for (const track of media.getTracks()) track.stop() - await appendQueue.catch(() => undefined) - await controlIn - .append( - new TextEncoder().encode( - JSON.stringify({ type: `session.close`, reason: `client-stop` }) + try { + media = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + sampleRate: REALTIME_SAMPLE_RATE, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }) + session = await createRealtimeSession(baseUrl, entityUrl) + const audioIn = streamHandle( + baseUrl, + session.streams.audio_in, + `audio/pcm; rate=${REALTIME_SAMPLE_RATE}; channels=1` + ) + const audioOut = streamHandle( + baseUrl, + session.streams.audio_out, + `audio/pcm; rate=${REALTIME_SAMPLE_RATE}; channels=1` + ) + controlIn = streamHandle( + baseUrl, + session.streams.control_in, + `application/json` + ) + const controlOut = streamHandle( + baseUrl, + session.streams.control_out, + `application/json` + ) + + source = micContext.createMediaStreamSource(media) + processor = micContext.createScriptProcessor(1024, 1, 1) + silentOutput = micContext.createGain() + silentOutput.gain.value = 0 + processor.onaudioprocess = (event) => { + if (abort.signal.aborted) return + const input = event.inputBuffer.getChannelData(0) + const bytes = pcm16Bytes(input) + appendQueue = appendQueue + .then(() => audioIn.append(bytes)) + .catch((error) => { + console.warn(`[realtime-audio] microphone append failed`, error) + }) + } + source.connect(processor) + processor.connect(silentOutput) + silentOutput.connect(micContext.destination) + + playback = (async () => { + const response = await audioOut.stream({ + live: true, + signal: abort.signal, + warnOnHttp: false, + }) + try { + for await (const chunk of response.bodyStream()) { + if (abort.signal.aborted || chunk.byteLength === 0) continue + const samples = pcm16Floats(chunk) + const buffer = playbackContext.createBuffer( + 1, + samples.length, + REALTIME_SAMPLE_RATE + ) + const channel = new Float32Array(samples.length) + channel.set(samples) + buffer.copyToChannel(channel, 0) + const node = playbackContext.createBufferSource() + node.buffer = buffer + node.connect(playbackContext.destination) + node.onended = () => playbackNodes.delete(node) + playbackNodes.add(node) + const startAt = Math.max( + playbackContext.currentTime, + nextPlaybackTime ) - ) - .catch(() => undefined) - await playback - await Promise.allSettled([micContext.close(), playbackContext.close()]) - }, + if (currentOutputStartedAt === null) { + currentOutputStartedAt = startAt + } + node.start(startAt) + nextPlaybackTime = startAt + buffer.duration + } + } finally { + response.cancel() + } + })().catch((error) => { + if (!abort.signal.aborted) { + console.warn(`[realtime-audio] playback stream failed`, error) + } + }) + + control = (async () => { + const response = await controlOut.stream({ + live: true, + signal: abort.signal, + json: true, + warnOnHttp: false, + }) + try { + for await (const event of response.jsonStream()) { + if (abort.signal.aborted || !event || typeof event !== `object`) { + continue + } + if ( + event.type === `output_audio.delta` && + typeof event.itemId === `string` + ) { + currentOutputItemId = event.itemId + } else if (event.type === `input_audio.speech_started`) { + interruptPlayback() + } + } + } finally { + response.cancel() + } + })().catch((error) => { + if (!abort.signal.aborted) { + console.warn(`[realtime-audio] control stream failed`, error) + } + }) + + return { + sessionId: session.sessionId, + async sendText(text: string) { + await appendControl({ type: `input_text`, text }) + }, + async stop() { + await cleanup(true) + }, + } + } catch (error) { + await cleanup(Boolean(session)) + throw error } } diff --git a/packages/agents-server/src/entity-manager.ts b/packages/agents-server/src/entity-manager.ts index 2de262d8fb..7c0cbfa053 100644 --- a/packages/agents-server/src/entity-manager.ts +++ b/packages/agents-server/src/entity-manager.ts @@ -2715,6 +2715,13 @@ export class EntityManager { 400 ) } + if (provider !== `openai`) { + throw new ElectricAgentsError( + ErrCodeInvalidRequest, + `Realtime provider "${provider}" is not supported; expected "openai"`, + 400 + ) + } const sessionId = req.id ?? `rt-${randomUUID()}` validateRealtimeSessionId(sessionId) diff --git a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts index 1cff4f2f19..b7830c593c 100644 --- a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts +++ b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts @@ -234,6 +234,21 @@ describe(`ElectricAgentsManager realtime sessions`, () => { }) expect(inboxEvent.value).not.toHaveProperty(`message_type`) }) + + it(`rejects non-OpenAI realtime providers in V1`, async () => { + const { manager } = createAttachmentManager() + + await expect( + manager.createRealtimeSession(`/chat/session-1`, { + id: `rt-1`, + provider: `other`, + model: `other-realtime`, + }) + ).rejects.toMatchObject({ + status: 400, + message: `Realtime provider "other" is not supported; expected "openai"`, + }) + }) }) describe(`ElectricAgentsManager attachments`, () => { diff --git a/packages/agents/src/agents/horton.ts b/packages/agents/src/agents/horton.ts index 19ac4c45db..de5a183580 100644 --- a/packages/agents/src/agents/horton.ts +++ b/packages/agents/src/agents/horton.ts @@ -895,16 +895,18 @@ function createAssistantHandler(options: { `Horton realtime currently supports provider "openai", got "${activeRealtimeSession.provider}"` ) } - const apiKey = process.env.OPENAI_API_KEY - if (!apiKey) { - throw new Error( - `OPENAI_API_KEY must be set before starting Horton realtime mode` - ) - } const realtime = ctx.useRealtime({ systemPrompt: hortonRealtimeSystemPrompt(systemPrompt), provider: createOpenAIRealtimeProvider({ - apiKey, + apiKey: () => { + const apiKey = process.env.OPENAI_API_KEY + if (!apiKey) { + throw new Error( + `OPENAI_API_KEY must be set before starting Horton realtime mode` + ) + } + return apiKey + }, model: activeRealtimeSession.model, }), tools: tools as AgentTool[], From 70b7269dce8e0edfbed8c031ceada44b0867bb52 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 13:04:17 +0100 Subject: [PATCH 18/39] fix(agents): make realtime voice input activate reliably --- packages/agents-runtime/src/openai-realtime.ts | 16 +++++++++++++++- .../agents-runtime/test/openai-realtime.test.ts | 12 +++++++++++- .../agents-server-ui/src/lib/realtime-audio.ts | 5 +++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/packages/agents-runtime/src/openai-realtime.ts b/packages/agents-runtime/src/openai-realtime.ts index 72fdb5ea46..026a5294d1 100644 --- a/packages/agents-runtime/src/openai-realtime.ts +++ b/packages/agents-runtime/src/openai-realtime.ts @@ -245,7 +245,21 @@ function buildSessionUpdate( ...(inputFormat || outputFormat || opts.voice ? { audio: { - ...(inputFormat ? { input: { format: inputFormat } } : {}), + ...(inputFormat + ? { + input: { + format: inputFormat, + turn_detection: { + type: `server_vad`, + threshold: 0.5, + prefix_padding_ms: 300, + silence_duration_ms: 200, + create_response: true, + interrupt_response: true, + }, + }, + } + : {}), ...(outputFormat || opts.voice ? { output: { diff --git a/packages/agents-runtime/test/openai-realtime.test.ts b/packages/agents-runtime/test/openai-realtime.test.ts index 9fc88444c3..ea981e9c43 100644 --- a/packages/agents-runtime/test/openai-realtime.test.ts +++ b/packages/agents-runtime/test/openai-realtime.test.ts @@ -100,7 +100,17 @@ describe(`createOpenAIRealtimeProvider`, () => { }, ], audio: { - input: { format: { type: `audio/pcm`, rate: 24_000 } }, + input: { + format: { type: `audio/pcm`, rate: 24_000 }, + turn_detection: { + type: `server_vad`, + threshold: 0.5, + prefix_padding_ms: 300, + silence_duration_ms: 200, + create_response: true, + interrupt_response: true, + }, + }, output: { format: { type: `audio/pcm`, rate: 24_000 } }, }, }, diff --git a/packages/agents-server-ui/src/lib/realtime-audio.ts b/packages/agents-server-ui/src/lib/realtime-audio.ts index 8d110e8634..f5c14aefc0 100644 --- a/packages/agents-server-ui/src/lib/realtime-audio.ts +++ b/packages/agents-server-ui/src/lib/realtime-audio.ts @@ -124,6 +124,10 @@ export async function startRealtimeAudioSession({ const abort = new AbortController() const micContext = createAudioContext() const playbackContext = createAudioContext() + const resumeAudioContexts = Promise.allSettled([ + micContext.resume(), + playbackContext.resume(), + ]) let appendQueue = Promise.resolve() let playback = Promise.resolve() let control = Promise.resolve() @@ -209,6 +213,7 @@ export async function startRealtimeAudioSession({ autoGainControl: true, }, }) + await resumeAudioContexts session = await createRealtimeSession(baseUrl, entityUrl) const audioIn = streamHandle( baseUrl, From 1e0bc36dd6ff8d92c7abcd0e0ad7e447103bb492 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 13:15:31 +0100 Subject: [PATCH 19/39] fix(agents): avoid inactive realtime response cancel --- .../agents-runtime/src/context-factory.ts | 7 +++++ .../test/realtime-context.test.ts | 27 +++++++++++++++++++ .../src/lib/realtime-audio.ts | 21 +++++++-------- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/packages/agents-runtime/src/context-factory.ts b/packages/agents-runtime/src/context-factory.ts index cbf25fc777..29d3a58e53 100644 --- a/packages/agents-runtime/src/context-factory.ts +++ b/packages/agents-runtime/src/context-factory.ts @@ -1321,6 +1321,13 @@ export function createHandlerContext( break case `session.error`: + if (event.code === `response_cancel_not_active`) { + runtimeLog.warn( + `[agent-runtime]`, + `realtime provider ignored inactive response cancellation: ${event.error}` + ) + break + } throw new Error( `[agent-runtime] realtime provider error${event.code ? ` ${event.code}` : ``}: ${event.error}` ) diff --git a/packages/agents-runtime/test/realtime-context.test.ts b/packages/agents-runtime/test/realtime-context.test.ts index 7a4c129ccb..a07f37f339 100644 --- a/packages/agents-runtime/test/realtime-context.test.ts +++ b/packages/agents-runtime/test/realtime-context.test.ts @@ -182,6 +182,33 @@ describe(`ctx.useRealtime()`, () => { }) }) + it(`does not fail the run when OpenAI reports inactive response cancellation`, async () => { + const { ctx } = createTestHandlerContext() + + const realtime = ctx.useRealtime({ + systemPrompt: `You are realtime.`, + provider: createTestRealtimeProvider({ + events: [ + { type: `session.started` }, + { + type: `session.error`, + code: `response_cancel_not_active`, + error: `Cancellation failed: no active response found`, + }, + { type: `session.closed` }, + ], + }), + tools: [], + }) + + await expect(realtime.run()).resolves.toMatchObject({ + usage: { tokens: 0 }, + }) + expect(ctx.db.collections.runs.toArray).toMatchObject([ + { status: `completed`, finish_reason: `stop` }, + ]) + }) + it(`persists provider audio and control output to realtime durable streams`, async () => { const { ctx } = createTestHandlerContext({ realtimeStreams: { diff --git a/packages/agents-server-ui/src/lib/realtime-audio.ts b/packages/agents-server-ui/src/lib/realtime-audio.ts index f5c14aefc0..4f795b916b 100644 --- a/packages/agents-server-ui/src/lib/realtime-audio.ts +++ b/packages/agents-server-ui/src/lib/realtime-audio.ts @@ -160,6 +160,9 @@ export async function startRealtimeAudioSession({ } const interruptPlayback = (): void => { + const itemId = currentOutputItemId + if (!itemId) return + const audioEndMs = currentOutputStartedAt === null ? 0 @@ -169,20 +172,14 @@ export async function startRealtimeAudioSession({ (playbackContext.currentTime - currentOutputStartedAt) * 1000 ) ) - const itemId = currentOutputItemId stopScheduledPlayback() - void appendControl({ type: `response.cancel` }).catch((error) => { - console.warn(`[realtime-audio] response cancel failed`, error) + void appendControl({ + type: `output_audio.truncate`, + itemId, + audioEndMs, + }).catch((error) => { + console.warn(`[realtime-audio] output truncate failed`, error) }) - if (itemId) { - void appendControl({ - type: `output_audio.truncate`, - itemId, - audioEndMs, - }).catch((error) => { - console.warn(`[realtime-audio] output truncate failed`, error) - }) - } } const cleanup = async (sendClose: boolean): Promise => { From 3dc5e98a910937a6cb4e04184bf910dfaf73e949 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 13:24:05 +0100 Subject: [PATCH 20/39] fix(agents): use supported OpenAI realtime model --- .../agents-runtime/src/openai-realtime.ts | 90 ++++++++++++++++--- .../test/electric-agents-client.test.ts | 6 +- .../test/openai-realtime.test.ts | 30 ++++++- .../test/realtime-context.test.ts | 10 +-- ...time-server-client-update-metadata.test.ts | 8 +- .../src/lib/realtime-audio.ts | 2 +- ...ic-agents-manager-write-validation.test.ts | 6 +- .../test/horton-tool-composition.test.ts | 4 +- 8 files changed, 123 insertions(+), 33 deletions(-) diff --git a/packages/agents-runtime/src/openai-realtime.ts b/packages/agents-runtime/src/openai-realtime.ts index 026a5294d1..f3b36676ae 100644 --- a/packages/agents-runtime/src/openai-realtime.ts +++ b/packages/agents-runtime/src/openai-realtime.ts @@ -30,6 +30,8 @@ type OpenAIRealtimeWebSocketConstructor = new ( init?: unknown ) => OpenAIRealtimeSocket +const DEFAULT_OPENAI_REALTIME_MODEL = `gpt-realtime` + export interface OpenAIRealtimeProviderOptions { apiKey: string | (() => MaybePromise) model?: string @@ -44,15 +46,18 @@ type OpenAIRealtimeEvent = Record & { type?: string } class AsyncEventQueue implements AsyncIterable { private values: Array = [] - private resolvers: Array<(value: IteratorResult) => void> = [] + private resolvers: Array<{ + resolve: (value: IteratorResult) => void + reject: (error: unknown) => void + }> = [] private closed = false private error: unknown push(value: T): void { if (this.closed) return - const resolve = this.resolvers.shift() - if (resolve) { - resolve({ value, done: false }) + const resolver = this.resolvers.shift() + if (resolver) { + resolver.resolve({ value, done: false }) return } this.values.push(value) @@ -61,14 +66,18 @@ class AsyncEventQueue implements AsyncIterable { close(): void { if (this.closed) return this.closed = true - for (const resolve of this.resolvers.splice(0)) { - resolve({ value: undefined as T, done: true }) + for (const resolver of this.resolvers.splice(0)) { + resolver.resolve({ value: undefined as T, done: true }) } } fail(error: unknown): void { + if (this.closed) return this.error = error - this.close() + this.closed = true + for (const resolver of this.resolvers.splice(0)) { + resolver.reject(error) + } } [Symbol.asyncIterator](): AsyncIterator { @@ -83,8 +92,8 @@ class AsyncEventQueue implements AsyncIterable { if (this.closed) { return Promise.resolve({ value: undefined as T, done: true }) } - return new Promise>((resolve) => { - this.resolvers.push(resolve) + return new Promise>((resolve, reject) => { + this.resolvers.push({ resolve, reject }) }) }, } @@ -123,6 +132,48 @@ function socketMessageData(args: Array): unknown { return first } +function socketCloseDetails(args: Array): { + code?: number + reason?: string + wasClean?: boolean +} { + const [first, second] = args + if (typeof first === `number`) { + return { + code: first, + reason: second === undefined ? undefined : dataToString(second), + } + } + if (!first || typeof first !== `object`) return {} + const event = first as { + code?: unknown + reason?: unknown + wasClean?: unknown + } + return { + code: typeof event.code === `number` ? event.code : undefined, + reason: + typeof event.reason === `string` + ? event.reason + : event.reason === undefined + ? undefined + : dataToString(event.reason), + wasClean: typeof event.wasClean === `boolean` ? event.wasClean : undefined, + } +} + +function socketCloseError(details: { + code?: number + reason?: string + wasClean?: boolean +}): string { + const parts = [`OpenAI realtime WebSocket closed before client stop`] + if (details.code !== undefined) parts.push(`code=${details.code}`) + if (details.reason) parts.push(`reason=${details.reason}`) + if (details.wasClean !== undefined) parts.push(`clean=${details.wasClean}`) + return parts.join(` `) +} + function dataToString(data: unknown): string { if (typeof data === `string`) return data if (data instanceof ArrayBuffer) return new TextDecoder().decode(data) @@ -235,7 +286,7 @@ function buildSessionUpdate( type: `session.update`, session: { type: `realtime`, - model: opts.model ?? `gpt-realtime-2`, + model: opts.model ?? DEFAULT_OPENAI_REALTIME_MODEL, instructions: input.systemPrompt, output_modalities: outputFormat ? [`audio`] : [`text`], tool_choice: input.tools.length > 0 ? `auto` : `none`, @@ -456,7 +507,7 @@ function mapOpenAIEvent( export function createOpenAIRealtimeProvider( opts: OpenAIRealtimeProviderOptions ): RealtimeProviderConfig { - const model = opts.model ?? `gpt-realtime-2` + const model = opts.model ?? DEFAULT_OPENAI_REALTIME_MODEL return { id: `openai`, @@ -486,6 +537,7 @@ export function createOpenAIRealtimeProvider( ) let socketOpen = false let socketClosed = false + let clientCloseRequested = false let rejectOpen: ((error: Error) => void) | undefined const closeQueue = (reason?: string): void => { @@ -500,6 +552,7 @@ export function createOpenAIRealtimeProvider( const error = new Error( `[agent-runtime] OpenAI realtime WebSocket aborted` ) + clientCloseRequested = true closeQueue(`aborted`) ws.close?.(1000, `aborted`) if (!socketOpen) rejectOpen?.(error) @@ -612,8 +665,18 @@ export function createOpenAIRealtimeProvider( queue.fail(error) } }) - onSocket(ws, `close`, () => { - closeQueue() + onSocket(ws, `close`, (...args) => { + const details = socketCloseDetails(args) + if (clientCloseRequested || input.signal?.aborted) { + closeQueue(details.reason || undefined) + return + } + queue.push({ + type: `session.error`, + code: `websocket_closed`, + error: socketCloseError(details), + }) + closeQueue(details.reason || `websocket_closed`) }) if (input.signal?.aborted) { @@ -664,6 +727,7 @@ export function createOpenAIRealtimeProvider( }) }, close: async (reason) => { + clientCloseRequested = true closeQueue(reason) ws.close?.(1000, reason) }, diff --git a/packages/agents-runtime/test/electric-agents-client.test.ts b/packages/agents-runtime/test/electric-agents-client.test.ts index 858faefff0..db3c7388f0 100644 --- a/packages/agents-runtime/test/electric-agents-client.test.ts +++ b/packages/agents-runtime/test/electric-agents-client.test.ts @@ -61,7 +61,7 @@ describe(`createAgentsClient`, () => { sessionId: `rt-1`, entityUrl: `/horton/demo`, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, status: `requested`, startedAt: `2026-06-09T10:00:00.000Z`, streams: { @@ -216,7 +216,7 @@ describe(`createAgentsClient`, () => { client.startRealtimeSession({ entityUrl: `/horton/demo`, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, }) ).resolves.toMatchObject({ sessionId: `rt-1`, @@ -228,7 +228,7 @@ describe(`createAgentsClient`, () => { expect(mockState.startRealtimeSession).toHaveBeenCalledWith({ entityUrl: `/horton/demo`, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, }) }) diff --git a/packages/agents-runtime/test/openai-realtime.test.ts b/packages/agents-runtime/test/openai-realtime.test.ts index ea981e9c43..5918853c54 100644 --- a/packages/agents-runtime/test/openai-realtime.test.ts +++ b/packages/agents-runtime/test/openai-realtime.test.ts @@ -76,7 +76,7 @@ describe(`createOpenAIRealtimeProvider`, () => { const socket = FakeWebSocket.instances[0]! expect(socket.url).toBe( - `wss://api.openai.com/v1/realtime?model=gpt-realtime-2` + `wss://api.openai.com/v1/realtime?model=gpt-realtime` ) expect(socket.init).toEqual({ headers: { @@ -88,7 +88,7 @@ describe(`createOpenAIRealtimeProvider`, () => { type: `session.update`, session: { type: `realtime`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, instructions: `You are Horton.`, output_modalities: [`audio`], tool_choice: `auto`, @@ -174,6 +174,32 @@ describe(`createOpenAIRealtimeProvider`, () => { }) }) + it(`surfaces unexpected WebSocket closes as provider errors`, async () => { + FakeWebSocket.instances = [] + const provider = createOpenAIRealtimeProvider({ + apiKey: `sk-test`, + WebSocket: FakeWebSocket, + }) + + const session = await provider.connect({ + systemPrompt: `Talk`, + messages: [], + tools: [], + }) + const socket = FakeWebSocket.instances[0]! + const iterator = session.events[Symbol.asyncIterator]() + + socket.emit(`close`, { code: 1008, reason: `invalid model` }) + + await expect(nextEvent(iterator)).resolves.toEqual({ + type: `session.error`, + code: `websocket_closed`, + error: + `OpenAI realtime WebSocket closed before client stop ` + + `code=1008 reason=invalid model`, + }) + }) + it(`can truncate output audio for interrupted playback`, async () => { FakeWebSocket.instances = [] const provider = createOpenAIRealtimeProvider({ diff --git a/packages/agents-runtime/test/realtime-context.test.ts b/packages/agents-runtime/test/realtime-context.test.ts index a07f37f339..2c488c2dae 100644 --- a/packages/agents-runtime/test/realtime-context.test.ts +++ b/packages/agents-runtime/test/realtime-context.test.ts @@ -73,7 +73,7 @@ describe(`ctx.useRealtime()`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, status: `active`, startedAt: `2026-06-09T12:00:00.000Z`, endedAt: null, @@ -100,7 +100,7 @@ describe(`ctx.useRealtime()`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, status: `requested`, startedAt: `2026-06-09T12:00:00.000Z`, endedAt: null, @@ -146,7 +146,7 @@ describe(`ctx.useRealtime()`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, status: `requested`, startedAt: `2026-06-09T12:00:00.000Z`, endedAt: null, @@ -163,7 +163,7 @@ describe(`ctx.useRealtime()`, () => { systemPrompt: `You are realtime.`, provider: { id: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, connect: async () => { throw new Error(`missing key`) }, @@ -221,7 +221,7 @@ describe(`ctx.useRealtime()`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, status: `active`, startedAt: `2026-06-09T12:00:00.000Z`, endedAt: null, diff --git a/packages/agents-runtime/test/runtime-server-client-update-metadata.test.ts b/packages/agents-runtime/test/runtime-server-client-update-metadata.test.ts index 092a72a782..8b801519b0 100644 --- a/packages/agents-runtime/test/runtime-server-client-update-metadata.test.ts +++ b/packages/agents-runtime/test/runtime-server-client-update-metadata.test.ts @@ -273,7 +273,7 @@ describe(`runtime-server-client realtime sessions`, () => { sessionId: `rt-1`, entityUrl: `/horton/demo`, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, status: `requested`, startedAt: `2026-06-09T10:00:00.000Z`, streams: { @@ -301,7 +301,7 @@ describe(`runtime-server-client realtime sessions`, () => { entityUrl: `/horton/demo`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, inputAudio: { codec: `pcm16`, sampleRate: 16_000, channels: 1 }, meta: { source: `button` }, }) @@ -319,7 +319,7 @@ describe(`runtime-server-client realtime sessions`, () => { entityUrl: `/horton/demo`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, inputAudio: { codec: `pcm16`, sampleRate: 16_000, channels: 1 }, meta: { source: `button` }, }) @@ -338,7 +338,7 @@ describe(`runtime-server-client realtime sessions`, () => { client.startRealtimeSession({ entityUrl: `/horton/demo`, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, }) ).rejects.toThrow(/startRealtimeSession.*401.*not allowed/) }) diff --git a/packages/agents-server-ui/src/lib/realtime-audio.ts b/packages/agents-server-ui/src/lib/realtime-audio.ts index 4f795b916b..bb87b08a33 100644 --- a/packages/agents-server-ui/src/lib/realtime-audio.ts +++ b/packages/agents-server-ui/src/lib/realtime-audio.ts @@ -92,7 +92,7 @@ async function createRealtimeSession( body: JSON.stringify({ entityUrl, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, inputAudio: { codec: `pcm16`, sampleRate: REALTIME_SAMPLE_RATE, diff --git a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts index b7830c593c..02dc3e2c52 100644 --- a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts +++ b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts @@ -158,7 +158,7 @@ describe(`ElectricAgentsManager realtime sessions`, () => { const result = await manager.createRealtimeSession(`/chat/session-1`, { id: `rt-1`, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, inputAudio: { codec: `pcm16`, sampleRate: 16_000, channels: 1 }, outputAudio: { codec: `pcm16`, sampleRate: 24_000, channels: 1 }, meta: { source: `test` }, @@ -203,7 +203,7 @@ describe(`ElectricAgentsManager realtime sessions`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, status: `requested`, streams: result.streams, retention: `forever`, @@ -216,7 +216,7 @@ describe(`ElectricAgentsManager realtime sessions`, () => { value: { session_id: `rt-1`, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, status: `requested`, streams: result.streams, }, diff --git a/packages/agents/test/horton-tool-composition.test.ts b/packages/agents/test/horton-tool-composition.test.ts index 004b02a9d5..1b8d3c019a 100644 --- a/packages/agents/test/horton-tool-composition.test.ts +++ b/packages/agents/test/horton-tool-composition.test.ts @@ -179,7 +179,7 @@ describe(`horton tool composition`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, status: `active`, startedAt: `2026-06-09T12:00:00.000Z`, retention: `forever`, @@ -220,7 +220,7 @@ describe(`horton tool composition`, () => { )[0]![0] expect(realtimeConfig.provider).toMatchObject({ id: `openai`, - model: `gpt-realtime-2`, + model: `gpt-realtime`, }) expect(realtimeConfig.toolPolicy.direct).toEqual( expect.arrayContaining([ From 7d8ae27c7a09773233d2c26d1438774285e54d70 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 13:46:17 +0100 Subject: [PATCH 21/39] fix(agents): wire realtime audio path --- .../agents-runtime/src/context-factory.ts | 46 +++++++++++++++++ .../agents-runtime/src/openai-realtime.ts | 4 ++ .../agents-runtime/src/timeline-context.ts | 16 ++++++ .../test/openai-realtime.test.ts | 51 +++++++++++++++++++ .../src/components/EntityTimeline.tsx | 20 +++++++- .../src/lib/realtime-audio.ts | 24 +++++++++ packages/agents-server/src/entity-manager.ts | 39 ++++++++++---- ...ic-agents-manager-write-validation.test.ts | 30 +++++++---- 8 files changed, 209 insertions(+), 21 deletions(-) diff --git a/packages/agents-runtime/src/context-factory.ts b/packages/agents-runtime/src/context-factory.ts index 29d3a58e53..01a2b387e3 100644 --- a/packages/agents-runtime/src/context-factory.ts +++ b/packages/agents-runtime/src/context-factory.ts @@ -177,6 +177,7 @@ function createRealtimeStreamIo( ): RealtimeStreamIo | undefined { if (!config.realtimeStreams || !session) return undefined + const logPrefix = `[agent-runtime]` const abort = new AbortController() const abortFromRun = (): void => abort.abort() if (config.runSignal?.aborted) { @@ -206,6 +207,17 @@ function createRealtimeStreamIo( `application/json` ) const tasks: Array> = [] + let audioInChunks = 0 + let audioInBytes = 0 + let controlInCommands = 0 + let audioOutChunks = 0 + let audioOutBytes = 0 + let controlOutEvents = 0 + + runtimeLog.info( + logPrefix, + `realtime stream bridge starting session=${session.id} audioIn=${session.streams.audio_in} audioOut=${session.streams.audio_out}` + ) if (providerSession.appendInputAudio) { tasks.push( @@ -218,6 +230,14 @@ function createRealtimeStreamIo( try { for await (const chunk of response.bodyStream()) { if (abort.signal.aborted) break + audioInChunks += 1 + audioInBytes += chunk.byteLength + if (audioInChunks === 1) { + runtimeLog.info( + logPrefix, + `realtime audio/in first chunk session=${session.id} bytes=${chunk.byteLength}` + ) + } await providerSession.appendInputAudio?.(chunk) } } finally { @@ -247,6 +267,13 @@ function createRealtimeStreamIo( if (abort.signal.aborted || !isRealtimeControlInput(command)) { continue } + controlInCommands += 1 + if (controlInCommands === 1) { + runtimeLog.info( + logPrefix, + `realtime control/in first command session=${session.id} type=${command.type}` + ) + } switch (command.type) { case `input_text`: await providerSession.sendText?.(command.text) @@ -284,7 +311,22 @@ function createRealtimeStreamIo( return { async writeProviderEvent(event) { + controlOutEvents += 1 + if (controlOutEvents === 1) { + runtimeLog.info( + logPrefix, + `realtime provider first event session=${session.id} type=${event.type}` + ) + } if (event.type === `output_audio.delta`) { + audioOutChunks += 1 + audioOutBytes += event.audio.byteLength + if (audioOutChunks === 1) { + runtimeLog.info( + logPrefix, + `realtime audio/out first chunk session=${session.id} bytes=${event.audio.byteLength}` + ) + } await audioOut.append(event.audio) } await controlOut.append(jsonBytes(realtimeControlOutput(event))) @@ -293,6 +335,10 @@ function createRealtimeStreamIo( abort.abort() config.runSignal?.removeEventListener(`abort`, abortFromRun) await Promise.allSettled(tasks) + runtimeLog.info( + logPrefix, + `realtime stream bridge closed session=${session.id} audioInChunks=${audioInChunks} audioInBytes=${audioInBytes} controlInCommands=${controlInCommands} providerEvents=${controlOutEvents} audioOutChunks=${audioOutChunks} audioOutBytes=${audioOutBytes}` + ) }, } } diff --git a/packages/agents-runtime/src/openai-realtime.ts b/packages/agents-runtime/src/openai-realtime.ts index f3b36676ae..7914c95cc3 100644 --- a/packages/agents-runtime/src/openai-realtime.ts +++ b/packages/agents-runtime/src/openai-realtime.ts @@ -409,6 +409,7 @@ function mapOpenAIEvent( }, ] case `response.audio.delta`: + case `response.output_audio.delta`: return [ { type: `output_audio.delta`, @@ -421,6 +422,7 @@ function mapOpenAIEvent( }, ] case `response.audio.done`: + case `response.output_audio.done`: return [ { type: `output_audio.completed`, @@ -432,6 +434,7 @@ function mapOpenAIEvent( }, ] case `response.audio_transcript.delta`: + case `response.output_audio_transcript.delta`: case `response.output_text.delta`: return [ { @@ -444,6 +447,7 @@ function mapOpenAIEvent( }, ] case `response.audio_transcript.done`: + case `response.output_audio_transcript.done`: case `response.output_text.done`: return [ { diff --git a/packages/agents-runtime/src/timeline-context.ts b/packages/agents-runtime/src/timeline-context.ts index 461430da4a..a416e14d32 100644 --- a/packages/agents-runtime/src/timeline-context.ts +++ b/packages/agents-runtime/src/timeline-context.ts @@ -194,6 +194,21 @@ function renderSignalMessage(signal: Signal): LLMMessage { } } +function isRealtimeSessionWake(payload: unknown): boolean { + if (!payload || typeof payload !== `object`) return false + const changes = (payload as { changes?: unknown }).changes + if (!Array.isArray(changes)) return false + return changes.some((change) => { + if (!change || typeof change !== `object`) return false + const payload = (change as { payload?: unknown }).payload + return ( + !!payload && + typeof payload === `object` && + (payload as { type?: unknown }).type === `realtime_session.started` + ) + }) +} + export function defaultProjection( item: TimelineItem ): Array | null { @@ -202,6 +217,7 @@ export function defaultProjection( return [{ role: `user`, content: projectInboxPayload(item) }] case `wake`: + if (isRealtimeSessionWake(item.payload)) return null return [{ role: `user`, content: asString(item.payload) }] case `signal`: diff --git a/packages/agents-runtime/test/openai-realtime.test.ts b/packages/agents-runtime/test/openai-realtime.test.ts index 5918853c54..9de8bf4f07 100644 --- a/packages/agents-runtime/test/openai-realtime.test.ts +++ b/packages/agents-runtime/test/openai-realtime.test.ts @@ -227,6 +227,57 @@ describe(`createOpenAIRealtimeProvider`, () => { }) }) + it(`maps GA output audio and transcript events`, async () => { + FakeWebSocket.instances = [] + const provider = createOpenAIRealtimeProvider({ + apiKey: `sk-test`, + WebSocket: FakeWebSocket, + }) + + const session = await provider.connect({ + systemPrompt: `Talk`, + messages: [], + tools: [], + }) + const socket = FakeWebSocket.instances[0]! + const iterator = session.events[Symbol.asyncIterator]() + + socket.emitMessage({ + type: `response.output_audio.delta`, + response_id: `resp-1`, + item_id: `item-1`, + delta: `AQID`, + }) + await expect(nextEvent(iterator)).resolves.toEqual({ + type: `output_audio.delta`, + responseId: `resp-1`, + itemId: `item-1`, + audio: new Uint8Array([1, 2, 3]), + }) + + socket.emitMessage({ + type: `response.output_audio_transcript.delta`, + response_id: `resp-1`, + delta: `hello`, + }) + await expect(nextEvent(iterator)).resolves.toEqual({ + type: `output_transcript.delta`, + responseId: `resp-1`, + delta: `hello`, + }) + + socket.emitMessage({ + type: `response.output_audio.done`, + response_id: `resp-1`, + item_id: `item-1`, + }) + await expect(nextEvent(iterator)).resolves.toEqual({ + type: `output_audio.completed`, + responseId: `resp-1`, + itemId: `item-1`, + }) + }) + it(`maps OpenAI events and executes function calls`, async () => { FakeWebSocket.instances = [] const execute = vi.fn().mockResolvedValue({ diff --git a/packages/agents-server-ui/src/components/EntityTimeline.tsx b/packages/agents-server-ui/src/components/EntityTimeline.tsx index 697ff709c5..79c28b4423 100644 --- a/packages/agents-server-ui/src/components/EntityTimeline.tsx +++ b/packages/agents-server-ui/src/components/EntityTimeline.tsx @@ -110,6 +110,20 @@ function readInboxPayloadDisplay(payload: unknown): string { return stringifyPayload(payload, 2) } +function isRealtimeSessionWake(row: RenderTimelineRow): boolean { + const changes = row.wake?.payload.changes + if (!Array.isArray(changes)) return false + return changes.some((change) => { + if (!change || typeof change !== `object`) return false + const payload = (change as { payload?: unknown }).payload + return ( + !!payload && + typeof payload === `object` && + (payload as { type?: unknown }).type === `realtime_session.started` + ) + }) +} + function stringifySearchPayload(value: unknown): string { if (value == null) return `` if (typeof value === `string`) return value @@ -1492,7 +1506,11 @@ export function EntityTimeline({ const previousStreamingAgentKeyRef = useRef(null) const textColumnWidth = Math.max(0, contentWidth - CHAT_SURFACE_GUTTER) const displayRows = useMemo( - () => rows.filter((row) => !isAttachmentManifest(row.manifest)), + () => + rows.filter( + (row) => + !isAttachmentManifest(row.manifest) && !isRealtimeSessionWake(row) + ), [rows] ) const attachmentsByInboxKey = useMemo(() => { diff --git a/packages/agents-server-ui/src/lib/realtime-audio.ts b/packages/agents-server-ui/src/lib/realtime-audio.ts index bb87b08a33..ffbdb4552f 100644 --- a/packages/agents-server-ui/src/lib/realtime-audio.ts +++ b/packages/agents-server-ui/src/lib/realtime-audio.ts @@ -140,6 +140,9 @@ export async function startRealtimeAudioSession({ let nextPlaybackTime = playbackContext.currentTime let currentOutputItemId: string | null = null let currentOutputStartedAt: number | null = null + let micChunks = 0 + let playbackChunks = 0 + let controlEvents = 0 const playbackNodes = new Set() const appendControl = async (value: unknown): Promise => { @@ -212,6 +215,9 @@ export async function startRealtimeAudioSession({ }) await resumeAudioContexts session = await createRealtimeSession(baseUrl, entityUrl) + console.info( + `[realtime-audio] session started session=${session.sessionId} audioIn=${session.streams.audio_in} audioOut=${session.streams.audio_out}` + ) const audioIn = streamHandle( baseUrl, session.streams.audio_in, @@ -241,6 +247,12 @@ export async function startRealtimeAudioSession({ if (abort.signal.aborted) return const input = event.inputBuffer.getChannelData(0) const bytes = pcm16Bytes(input) + micChunks += 1 + if (micChunks === 1) { + console.info( + `[realtime-audio] microphone first chunk session=${session?.sessionId} bytes=${bytes.byteLength}` + ) + } appendQueue = appendQueue .then(() => audioIn.append(bytes)) .catch((error) => { @@ -260,6 +272,12 @@ export async function startRealtimeAudioSession({ try { for await (const chunk of response.bodyStream()) { if (abort.signal.aborted || chunk.byteLength === 0) continue + playbackChunks += 1 + if (playbackChunks === 1) { + console.info( + `[realtime-audio] playback first chunk session=${session?.sessionId} bytes=${chunk.byteLength}` + ) + } const samples = pcm16Floats(chunk) const buffer = playbackContext.createBuffer( 1, @@ -305,6 +323,12 @@ export async function startRealtimeAudioSession({ if (abort.signal.aborted || !event || typeof event !== `object`) { continue } + controlEvents += 1 + if (controlEvents === 1) { + console.info( + `[realtime-audio] control first event session=${session?.sessionId} type=${event.type}` + ) + } if ( event.type === `output_audio.delta` && typeof event.itemId === `string` diff --git a/packages/agents-server/src/entity-manager.ts b/packages/agents-server/src/entity-manager.ts index 7c0cbfa053..e77cde0493 100644 --- a/packages/agents-server/src/entity-manager.ts +++ b/packages/agents-server/src/entity-manager.ts @@ -2779,22 +2779,41 @@ export class EntityManager { streams, ...(req.meta ? { meta: req.meta } : {}), }, - } as never) + } as any) await this.streamClient.append( entity.streams.main, this.encodeChangeEvent(sessionEvent as Record) ) - await this.send(entityUrl, { - from: SERVER_SIGNAL_SENDER, - payload: { - type: `realtime_session.started`, - sessionId, - provider, - model, - streams, + const wakeEvent = entityStateSchema.wakes.insert({ + key: `wake-realtime-session-${sessionId}`, + value: { + timestamp: startedAt, + source: entityUrl, + timeout: false, + changes: [ + { + collection: `realtimeSessions`, + kind: `insert`, + key: manifestKey, + from: SERVER_SIGNAL_SENDER, + payload: { + type: `realtime_session.started`, + sessionId, + provider, + model, + streams, + }, + timestamp: startedAt, + message_type: `realtime_session.started`, + }, + ], }, - }) + } as any) + await this.streamClient.append( + entity.streams.main, + this.encodeChangeEvent(wakeEvent as Record) + ) } catch (error) { await Promise.allSettled( createdStreams.map((path) => this.streamClient.delete(path)) diff --git a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts index 02dc3e2c52..e4c8ccb4e2 100644 --- a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts +++ b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts @@ -193,7 +193,7 @@ describe(`ElectricAgentsManager realtime sessions`, () => { expect(append).toHaveBeenCalledTimes(3) const manifestEvent = decodeAppend(append.mock.calls[0]!) const sessionEvent = decodeAppend(append.mock.calls[1]!) - const inboxEvent = decodeAppend(append.mock.calls[2]!) + const wakeEvent = decodeAppend(append.mock.calls[2]!) expect(manifestEvent).toMatchObject({ type: `manifest`, @@ -221,18 +221,28 @@ describe(`ElectricAgentsManager realtime sessions`, () => { streams: result.streams, }, }) - expect(inboxEvent).toMatchObject({ - type: `inbox`, + expect(wakeEvent).toMatchObject({ + type: `wake`, + key: `wake-realtime-session-rt-1`, value: { - from: `/_electric/server`, - payload: { - type: `realtime_session.started`, - sessionId: `rt-1`, - streams: result.streams, - }, + source: `/chat/session-1`, + timeout: false, + changes: [ + { + collection: `realtimeSessions`, + kind: `insert`, + key: `realtime-session:rt-1`, + from: `/_electric/server`, + payload: { + type: `realtime_session.started`, + sessionId: `rt-1`, + streams: result.streams, + }, + message_type: `realtime_session.started`, + }, + ], }, }) - expect(inboxEvent.value).not.toHaveProperty(`message_type`) }) it(`rejects non-OpenAI realtime providers in V1`, async () => { From 9cc3f01e464b85e6024ecd493a58bf9ccffa4c36 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 13:55:35 +0100 Subject: [PATCH 22/39] fix(agents): clamp realtime audio truncation --- .../agents-runtime/src/context-factory.ts | 11 +++ .../test/realtime-context.test.ts | 27 ++++++ .../src/components/MessageInput.module.css | 32 +++++++ .../src/components/MessageInput.tsx | 87 +++++++++++++------ .../src/lib/realtime-audio.ts | 50 ++++++++++- 5 files changed, 177 insertions(+), 30 deletions(-) diff --git a/packages/agents-runtime/src/context-factory.ts b/packages/agents-runtime/src/context-factory.ts index 01a2b387e3..1a2b03dfd4 100644 --- a/packages/agents-runtime/src/context-factory.ts +++ b/packages/agents-runtime/src/context-factory.ts @@ -1374,6 +1374,17 @@ export function createHandlerContext( ) break } + if ( + event.code === `invalid_value` && + event.error.includes(`Audio content`) && + event.error.includes(`already shorter than`) + ) { + runtimeLog.warn( + `[agent-runtime]`, + `realtime provider ignored stale output audio truncate: ${event.error}` + ) + break + } throw new Error( `[agent-runtime] realtime provider error${event.code ? ` ${event.code}` : ``}: ${event.error}` ) diff --git a/packages/agents-runtime/test/realtime-context.test.ts b/packages/agents-runtime/test/realtime-context.test.ts index 2c488c2dae..0c7109d46c 100644 --- a/packages/agents-runtime/test/realtime-context.test.ts +++ b/packages/agents-runtime/test/realtime-context.test.ts @@ -209,6 +209,33 @@ describe(`ctx.useRealtime()`, () => { ]) }) + it(`does not fail the run when OpenAI reports a stale output audio truncate`, async () => { + const { ctx } = createTestHandlerContext() + + const realtime = ctx.useRealtime({ + systemPrompt: `You are realtime.`, + provider: createTestRealtimeProvider({ + events: [ + { type: `session.started` }, + { + type: `session.error`, + code: `invalid_value`, + error: `Audio content of 6350ms is already shorter than 8160ms`, + }, + { type: `session.closed` }, + ], + }), + tools: [], + }) + + await expect(realtime.run()).resolves.toMatchObject({ + usage: { tokens: 0 }, + }) + expect(ctx.db.collections.runs.toArray).toMatchObject([ + { status: `completed`, finish_reason: `stop` }, + ]) + }) + it(`persists provider audio and control output to realtime durable streams`, async () => { const { ctx } = createTestHandlerContext({ realtimeStreams: { diff --git a/packages/agents-server-ui/src/components/MessageInput.module.css b/packages/agents-server-ui/src/components/MessageInput.module.css index d428c3df21..86bff9f36e 100644 --- a/packages/agents-server-ui/src/components/MessageInput.module.css +++ b/packages/agents-server-ui/src/components/MessageInput.module.css @@ -70,6 +70,38 @@ color: var(--ds-accent-11); } +.voiceMeter { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 2px; + width: 0; + height: 20px; + color: var(--ds-accent-11); + opacity: 0; + overflow: hidden; + transition: + opacity 0.12s ease, + width 0.12s ease; +} + +.voiceMeter[data-active='true'] { + width: 18px; + opacity: 1; +} + +.voiceMeterBar { + display: block; + width: 3px; + height: 14px; + border-radius: var(--ds-radius-full); + background: currentColor; + transform-origin: center bottom; + transition: + opacity 0.08s linear, + transform 0.08s linear; +} + .inlineIconButton:focus-visible { outline: 2px solid var(--ds-accent-a6); outline-offset: -2px; diff --git a/packages/agents-server-ui/src/components/MessageInput.tsx b/packages/agents-server-ui/src/components/MessageInput.tsx index 78361effdd..e637af15c9 100644 --- a/packages/agents-server-ui/src/components/MessageInput.tsx +++ b/packages/agents-server-ui/src/components/MessageInput.tsx @@ -123,6 +123,7 @@ export function MessageInput({ } | null>(null) const [realtimePending, setRealtimePending] = useState(false) const [realtimeActive, setRealtimeActive] = useState(false) + const [realtimeInputLevel, setRealtimeInputLevel] = useState(0) const realtimeSessionRef = useRef(null) const composerFocusRef = useRef<{ focus: () => void } | null>(null) const inputDisabled = disabled || writeDisabled @@ -356,19 +357,25 @@ export function MessageInput({ .catch((err: Error) => setError(err.message)) .finally(() => { setRealtimeActive(false) + setRealtimeInputLevel(0) setRealtimePending(false) }) return } if (!canUseRealtime) return setRealtimePending(true) - startRealtimeAudioSession({ baseUrl, entityUrl }) + startRealtimeAudioSession({ + baseUrl, + entityUrl, + onInputLevel: setRealtimeInputLevel, + }) .then((session) => { realtimeSessionRef.current = session setRealtimeActive(true) }) .catch((err: Error) => { setError(err.message) + setRealtimeInputLevel(0) }) .finally(() => { setRealtimePending(false) @@ -451,6 +458,12 @@ export function MessageInput({ ) const isButtonActive = canSubmit || (showStop && !stopDisabled) + const voiceLevel = realtimeActive ? realtimeInputLevel : 0 + const voiceBars = [ + Math.max(0.18, Math.min(1, 0.24 + voiceLevel * 0.76)), + Math.max(0.24, Math.min(1, 0.34 + voiceLevel * 0.9)), + Math.max(0.16, Math.min(1, 0.2 + voiceLevel * 0.82)), + ] const sendTooltip = showStop ? stopDisabled ? `Signal permission required` @@ -531,31 +544,53 @@ export function MessageInput({ controls={ <> {!isCommentMode ? ( - - - - - + <> + + + + + + + ) : null} {imageAttachmentsEnabled && !isCommentMode ? ( void }): Promise { const abort = new AbortController() const micContext = createAudioContext() @@ -140,6 +159,7 @@ export async function startRealtimeAudioSession({ let nextPlaybackTime = playbackContext.currentTime let currentOutputItemId: string | null = null let currentOutputStartedAt: number | null = null + let currentOutputReceivedMs = 0 let micChunks = 0 let playbackChunks = 0 let controlEvents = 0 @@ -162,11 +182,21 @@ export async function startRealtimeAudioSession({ currentOutputStartedAt = null } + const setCurrentOutputItem = (itemId: string): void => { + if (currentOutputItemId === itemId) return + currentOutputItemId = itemId + currentOutputStartedAt = null + currentOutputReceivedMs = 0 + } + const interruptPlayback = (): void => { const itemId = currentOutputItemId - if (!itemId) return + if (!itemId) { + stopScheduledPlayback() + return + } - const audioEndMs = + const playedMs = currentOutputStartedAt === null ? 0 : Math.max( @@ -175,7 +205,14 @@ export async function startRealtimeAudioSession({ (playbackContext.currentTime - currentOutputStartedAt) * 1000 ) ) + const maxGeneratedMs = Math.max( + 0, + Math.floor(currentOutputReceivedMs - TRUNCATE_SAFETY_MS) + ) + const audioEndMs = Math.min(playedMs, maxGeneratedMs) stopScheduledPlayback() + if (audioEndMs <= 0) return + void appendControl({ type: `output_audio.truncate`, itemId, @@ -190,6 +227,7 @@ export async function startRealtimeAudioSession({ processor?.disconnect() silentOutput?.disconnect() source?.disconnect() + onInputLevel?.(0) for (const track of media?.getTracks() ?? []) track.stop() stopScheduledPlayback() await appendQueue.catch(() => undefined) @@ -247,6 +285,7 @@ export async function startRealtimeAudioSession({ if (abort.signal.aborted) return const input = event.inputBuffer.getChannelData(0) const bytes = pcm16Bytes(input) + onInputLevel?.(audioLevel(input)) micChunks += 1 if (micChunks === 1) { console.info( @@ -296,7 +335,7 @@ export async function startRealtimeAudioSession({ playbackContext.currentTime, nextPlaybackTime ) - if (currentOutputStartedAt === null) { + if (currentOutputItemId && currentOutputStartedAt === null) { currentOutputStartedAt = startAt } node.start(startAt) @@ -333,7 +372,10 @@ export async function startRealtimeAudioSession({ event.type === `output_audio.delta` && typeof event.itemId === `string` ) { - currentOutputItemId = event.itemId + setCurrentOutputItem(event.itemId) + if (typeof event.byteLength === `number`) { + currentOutputReceivedMs += pcm16DurationMs(event.byteLength) + } } else if (event.type === `input_audio.speech_started`) { interruptPlayback() } From fc789388c45481d12dd7b986eb7db032c48d17b8 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 14:12:56 +0100 Subject: [PATCH 23/39] feat(agents): persist realtime transcripts --- .../agents-runtime/src/context-factory.ts | 153 +++++++++++++++++- .../agents-runtime/src/entity-timeline.ts | 114 ++++++++++++- .../agents-runtime/src/openai-realtime.ts | 17 ++ .../agents-runtime/src/timeline-context.ts | 31 ++++ packages/agents-runtime/src/types.ts | 16 ++ .../test/openai-realtime.test.ts | 71 ++++++++ .../test/realtime-context.test.ts | 65 ++++++++ .../test/timeline-context.test.ts | 37 +++++ .../src/components/EntityTimeline.tsx | 36 ++++- 9 files changed, 534 insertions(+), 6 deletions(-) diff --git a/packages/agents-runtime/src/context-factory.ts b/packages/agents-runtime/src/context-factory.ts index 1a2b03dfd4..c53be6cb5c 100644 --- a/packages/agents-runtime/src/context-factory.ts +++ b/packages/agents-runtime/src/context-factory.ts @@ -1315,6 +1315,7 @@ export function createHandlerContext( let currentToolCall: | { toolCallId: string; name: string; args: unknown } | undefined + const realtimeSession = activeRealtimeSession() const endText = (): void => { if (!textStarted) return @@ -1331,6 +1332,125 @@ export function createHandlerContext( bridge.onTextDelta(delta) } + const transcriptTextByKey = new Map() + const transcriptCreatedAtByKey = new Map() + const transcriptFallbackIds = new Map<`input` | `output`, string>() + let transcriptFallbackCounter = 0 + let providerSessionId = realtimeSession?.id + + const currentTranscriptSessionId = (): string => + realtimeSession?.id ?? providerSessionId ?? `ephemeral` + + const transcriptKey = ( + direction: `input` | `output`, + id?: string + ): string => { + let stableId = id + if (!stableId) { + stableId = transcriptFallbackIds.get(direction) + if (!stableId) { + stableId = `fallback-${transcriptFallbackCounter}` + transcriptFallbackCounter += 1 + transcriptFallbackIds.set(direction, stableId) + } + } + return `realtime-transcript:${currentTranscriptSessionId()}:${direction}:${stableId}` + } + + const writeRealtimeTranscript = (input: { + direction: `input` | `output` + key: string + text: string + status: `partial` | `final` + turnId?: string + responseId?: string + }): void => { + const collection = config.db.collections.realtimeTranscripts + if (input.text.length === 0 && !collection.has(input.key)) return + + const existing = collection.get(input.key) as + | { created_at?: string } + | undefined + const createdAt = + transcriptCreatedAtByKey.get(input.key) ?? + existing?.created_at ?? + new Date().toISOString() + transcriptCreatedAtByKey.set(input.key, createdAt) + + const value = { + session_id: currentTranscriptSessionId(), + direction: input.direction, + text: input.text, + status: input.status, + audio_stream: input.direction, + ...(input.turnId ? { turn_id: input.turnId } : {}), + ...(input.responseId ? { response_id: input.responseId } : {}), + created_at: createdAt, + } + config.writeEvent( + (collection.has(input.key) + ? entityStateSchema.realtimeTranscripts.update({ + key: input.key, + value: value as never, + }) + : entityStateSchema.realtimeTranscripts.insert({ + key: input.key, + value: value as never, + })) as ChangeEvent + ) + } + + const appendRealtimeTranscript = (input: { + direction: `input` | `output` + delta: string + turnId?: string + responseId?: string + }): void => { + if (input.delta.length === 0) return + const key = transcriptKey( + input.direction, + input.direction === `input` ? input.turnId : input.responseId + ) + const text = `${transcriptTextByKey.get(key) ?? ``}${input.delta}` + transcriptTextByKey.set(key, text) + writeRealtimeTranscript({ + direction: input.direction, + key, + text, + status: `partial`, + turnId: input.turnId, + responseId: input.responseId, + }) + } + + const completeRealtimeTranscript = (input: { + direction: `input` | `output` + text?: string + turnId?: string + responseId?: string + }): void => { + const key = transcriptKey( + input.direction, + input.direction === `input` ? input.turnId : input.responseId + ) + const text = input.text ?? transcriptTextByKey.get(key) ?? `` + transcriptTextByKey.set(key, text) + writeRealtimeTranscript({ + direction: input.direction, + key, + text, + status: `final`, + turnId: input.turnId, + responseId: input.responseId, + }) + if ( + (input.direction === `input` && !input.turnId) || + (input.direction === `output` && !input.responseId) + ) { + transcriptFallbackIds.delete(input.direction) + } + } + const composedTools = (await composeToolsWithProviders( activeRealtimeConfig.tools ?? [] )) as Array @@ -1342,7 +1462,6 @@ export function createHandlerContext( timelineToMessages(config.db) ) let realtimeIo: RealtimeStreamIo | undefined - const realtimeSession = activeRealtimeSession() let realtimeSessionTerminalWritten = false async function handleProviderEvent( @@ -1350,17 +1469,35 @@ export function createHandlerContext( ): Promise { switch (event.type) { case `session.started`: + providerSessionId = + realtimeSession?.id ?? event.sessionId ?? providerSessionId + break + case `session.updated`: case `input_audio.speech_started`: case `input_audio.speech_stopped`: - case `input_transcript.delta`: - case `input_transcript.completed`: case `output_audio.delta`: case `output_audio.completed`: case `response.started`: case `response.cancelled`: break + case `input_transcript.delta`: + appendRealtimeTranscript({ + direction: `input`, + delta: event.delta, + turnId: event.turnId, + }) + break + + case `input_transcript.completed`: + completeRealtimeTranscript({ + direction: `input`, + text: event.text, + turnId: event.turnId, + }) + break + case `session.closed`: case `response.completed`: endText() @@ -1390,10 +1527,20 @@ export function createHandlerContext( ) case `output_transcript.delta`: + appendRealtimeTranscript({ + direction: `output`, + delta: event.delta, + responseId: event.responseId, + }) emitText(event.delta) break case `output_transcript.completed`: + completeRealtimeTranscript({ + direction: `output`, + text: event.text, + responseId: event.responseId, + }) if (!textStarted && event.text) { emitText(event.text) } diff --git a/packages/agents-runtime/src/entity-timeline.ts b/packages/agents-runtime/src/entity-timeline.ts index 171c85338f..dd24b0e383 100644 --- a/packages/agents-runtime/src/entity-timeline.ts +++ b/packages/agents-runtime/src/entity-timeline.ts @@ -23,7 +23,12 @@ import type { } from '@tanstack/db' import type { EntityStreamDB } from './entity-stream-db' import { formatPointerOrderToken, type EventPointer } from './event-pointer' -import type { ChildStatusEntry, MessageReceived, Signal } from './entity-schema' +import type { + ChildStatusEntry, + MessageReceived, + RealtimeTranscript, + Signal, +} from './entity-schema' import type { ManifestEntry, Wake, WakeMessage } from './types' export const TIMELINE_ORDER_FALLBACK = `~` @@ -159,6 +164,13 @@ export type IncludesSignal = Omit & { order: TimelineOrder } +export type IncludesRealtimeTranscript = Omit< + RealtimeTranscript, + `_seq` | `_timeline_order` +> & { + order: TimelineOrder +} + export interface IncludesContextInserted { key: string order: TimelineOrder @@ -195,6 +207,7 @@ export interface EntityTimelineData { inbox: Array wakes: Array signals: Array + realtimeTranscripts?: Array contextInserted: Array contextRemoved: Array entities: Array @@ -216,7 +229,7 @@ export interface EntityTimelineQueryOptions { /** * Additional sources merged into the timeline, keyed by row name. Names * must not collide with the built-in sources (`inbox`, `run`, `wake`, - * `signal`, `manifest`). + * `signal`, `error`, `realtimeTranscript`, `manifest`). */ customSources?: Record } @@ -323,6 +336,7 @@ export type EntityTimelineSignalRow = IncludesSignal export type EntityTimelineErrorRow = EntityTimelineErrorItem & { order: TimelineOrder } +export type EntityTimelineRealtimeTranscriptRow = IncludesRealtimeTranscript export type EntityTimelineQueryRow = | { @@ -332,6 +346,7 @@ export type EntityTimelineQueryRow = wake?: undefined signal?: undefined error?: undefined + realtimeTranscript?: undefined manifest?: undefined } | { @@ -341,6 +356,7 @@ export type EntityTimelineQueryRow = wake?: undefined signal?: undefined error?: undefined + realtimeTranscript?: undefined manifest?: undefined } | { @@ -350,6 +366,7 @@ export type EntityTimelineQueryRow = wake: EntityTimelineWakeRow signal?: undefined error?: undefined + realtimeTranscript?: undefined manifest?: undefined } | { @@ -359,6 +376,7 @@ export type EntityTimelineQueryRow = wake?: undefined signal: EntityTimelineSignalRow error?: undefined + realtimeTranscript?: undefined manifest?: undefined } | { @@ -368,6 +386,7 @@ export type EntityTimelineQueryRow = wake?: undefined signal?: undefined error: EntityTimelineErrorRow + realtimeTranscript?: undefined manifest?: undefined } | { @@ -377,6 +396,17 @@ export type EntityTimelineQueryRow = wake?: undefined signal?: undefined error?: undefined + realtimeTranscript: EntityTimelineRealtimeTranscriptRow + manifest?: undefined + } + | { + $key: string + inbox?: undefined + run?: undefined + wake?: undefined + signal?: undefined + error?: undefined + realtimeTranscript?: undefined manifest: ManifestEntry } @@ -492,6 +522,9 @@ export function normalizeEntityTimelineData( inbox: data.inbox, wakes: data.wakes, signals: data.signals ?? [], + realtimeTranscripts: [...(data.realtimeTranscripts ?? [])].sort( + compareTimelineOrder + ), contextInserted: data.contextInserted, contextRemoved: data.contextRemoved, entities: normalizeTimelineEntities(data.entities), @@ -528,6 +561,9 @@ type WakeRow = OrderedValue< type SignalRow = OrderedValue< EntityStreamDB[`collections`][`signals`][`toArray`][number] > +type RealtimeTranscriptValueRow = + EntityStreamDB[`collections`][`realtimeTranscripts`][`toArray`][number] +type RealtimeTranscriptRow = OrderedValue type ContextInsertedValueRow = EntityStreamDB[`collections`][`contextInserted`][`toArray`][number] type ContextRemovedValueRow = @@ -980,6 +1016,22 @@ function buildSignalMessages(signals: Array): Array { }) } +function buildRealtimeTranscriptMessages( + transcripts: Array +): Array { + return [...transcripts].sort(compareTimelineOrder).map((transcript) => { + const { + _seq: _ignoredSeq, + _timeline_order: _ignoredTimelineOrder, + ...value + } = transcript + return { + ...value, + order: transcript.order, + } + }) +} + function buildContextInsertedMessages( entries: Array ): Array { @@ -1098,6 +1150,14 @@ export function buildEntityTimelineData( const inbox = withOrderToken(db.collections.inbox) const wakes = withOrderToken(db.collections.wakes) const signals = withOrderToken(db.collections.signals) + const realtimeTranscripts = withOrderToken( + getOrderableCollection( + db.collections.realtimeTranscripts as + | typeof db.collections.realtimeTranscripts + | undefined, + `realtimeTranscripts` + ) + ) const contextInserted = withOrderToken( getOrderableCollection( db.collections.contextInserted as @@ -1145,6 +1205,7 @@ export function buildEntityTimelineData( inbox, wakes, signals, + realtimeTranscripts, contextInserted, contextRemoved, manifests.filter(hasOrderToken), @@ -1162,6 +1223,9 @@ export function buildEntityTimelineData( inbox: buildInboxMessages(withOrderFromOrderIndex(inbox, orderIndex)), wakes: buildWakeMessages(withOrderFromOrderIndex(wakes, orderIndex)), signals: buildSignalMessages(withOrderFromOrderIndex(signals, orderIndex)), + realtimeTranscripts: buildRealtimeTranscriptMessages( + withOrderFromOrderIndex(realtimeTranscripts, orderIndex) + ), contextInserted: buildContextInsertedMessages( withOrderAndHistoryOffsetFromOrderIndex(contextInserted, orderIndex) ), @@ -1423,6 +1487,28 @@ function buildEntityTimelineQuery( run_id: error.run_id, })) + const realtimeTranscriptSource = q + .from({ realtimeTranscript: db.collections.realtimeTranscripts }) + .where(({ realtimeTranscript }) => + eq(realtimeTranscript.direction, `input`) + ) + .select(({ realtimeTranscript }) => ({ + key: realtimeTranscript.key, + order: coalesce(realtimeTranscript._timeline_order, `~`), + session_id: realtimeTranscript.session_id, + direction: realtimeTranscript.direction, + text: realtimeTranscript.text, + status: realtimeTranscript.status, + turn_id: realtimeTranscript.turn_id, + response_id: realtimeTranscript.response_id, + audio_stream: realtimeTranscript.audio_stream, + audio_offset: realtimeTranscript.audio_offset, + audio_next_offset: realtimeTranscript.audio_next_offset, + sample_start: realtimeTranscript.sample_start, + sample_end: realtimeTranscript.sample_end, + created_at: realtimeTranscript.created_at, + })) + // Union texts + tool calls into a single ordered stream. The // text-delta join lives at this level (vs. inside the consumer's // `items.select`) so the correlation key is `text.key` — a field @@ -1625,6 +1711,7 @@ function buildEntityTimelineQuery( wake: wakeSource, signal: signalSource, error: errorSource, + realtimeTranscript: realtimeTranscriptSource, manifest: db.collections.manifests, } for (const [name, buildSource] of Object.entries(opts.customSources ?? {})) { @@ -1850,6 +1937,29 @@ export function createEntityIncludesQuery( new_state: signal.new_state, })) ), + realtimeTranscripts: toArray( + q + .from({ realtimeTranscript: db.collections.realtimeTranscripts }) + .orderBy(({ realtimeTranscript }) => + coalesce(realtimeTranscript._seq, -1) + ) + .select(({ realtimeTranscript }) => ({ + key: realtimeTranscript.key, + order: coalesce(realtimeTranscript._seq, -1), + session_id: realtimeTranscript.session_id, + direction: realtimeTranscript.direction, + text: realtimeTranscript.text, + status: realtimeTranscript.status, + turn_id: realtimeTranscript.turn_id, + response_id: realtimeTranscript.response_id, + audio_stream: realtimeTranscript.audio_stream, + audio_offset: realtimeTranscript.audio_offset, + audio_next_offset: realtimeTranscript.audio_next_offset, + sample_start: realtimeTranscript.sample_start, + sample_end: realtimeTranscript.sample_end, + created_at: realtimeTranscript.created_at, + })) + ), entities: toArray( q .from({ entity: entitiesCollection }) diff --git a/packages/agents-runtime/src/openai-realtime.ts b/packages/agents-runtime/src/openai-realtime.ts index 7914c95cc3..c5d827e883 100644 --- a/packages/agents-runtime/src/openai-realtime.ts +++ b/packages/agents-runtime/src/openai-realtime.ts @@ -31,6 +31,7 @@ type OpenAIRealtimeWebSocketConstructor = new ( ) => OpenAIRealtimeSocket const DEFAULT_OPENAI_REALTIME_MODEL = `gpt-realtime` +const DEFAULT_OPENAI_INPUT_TRANSCRIPTION_MODEL = `gpt-4o-mini-transcribe` export interface OpenAIRealtimeProviderOptions { apiKey: string | (() => MaybePromise) @@ -276,12 +277,27 @@ function realtimeFormat( } } +function inputTranscription( + input: RealtimeProviderConnectInput +): Record | undefined { + if (!input.audio?.inputFormat || input.audio.inputTranscription === false) { + return undefined + } + const config = input.audio.inputTranscription ?? {} + return { + model: config.model ?? DEFAULT_OPENAI_INPUT_TRANSCRIPTION_MODEL, + ...(config.language ? { language: config.language } : {}), + ...(config.prompt ? { prompt: config.prompt } : {}), + } +} + function buildSessionUpdate( opts: OpenAIRealtimeProviderOptions, input: RealtimeProviderConnectInput ): Record { const inputFormat = realtimeFormat(input.audio?.inputFormat) const outputFormat = realtimeFormat(input.audio?.outputFormat) + const transcription = inputTranscription(input) return { type: `session.update`, session: { @@ -300,6 +316,7 @@ function buildSessionUpdate( ? { input: { format: inputFormat, + ...(transcription ? { transcription } : {}), turn_detection: { type: `server_vad`, threshold: 0.5, diff --git a/packages/agents-runtime/src/timeline-context.ts b/packages/agents-runtime/src/timeline-context.ts index a416e14d32..c3506bdf29 100644 --- a/packages/agents-runtime/src/timeline-context.ts +++ b/packages/agents-runtime/src/timeline-context.ts @@ -7,6 +7,7 @@ import type { IncludesContextInserted, IncludesContextRemoved, IncludesInboxMessage, + IncludesRealtimeTranscript, IncludesRun, IncludesSignal, IncludesWakeMessage, @@ -69,12 +70,14 @@ export function buildTimelineMessages(input: { inbox: Array wakes?: Array signals?: Array + realtimeTranscripts?: Array }): Array { return materializeTimeline({ runs: input.runs, inbox: input.inbox, wakes: input.wakes ?? [], signals: input.signals ?? [], + realtimeTranscripts: input.realtimeTranscripts ?? [], contextInserted: [], contextRemoved: [], entities: [], @@ -223,6 +226,11 @@ export function defaultProjection( case `signal`: return [renderSignalMessage(item.signal)] + case `realtime_transcript`: + return item.direction === `input` && item.text.length > 0 + ? [{ role: `user`, content: item.text }] + : null + case `run`: { const messages: Array = [] @@ -357,6 +365,11 @@ export function materializeTimeline( | { kind: `inbox`; order: TimelineOrder; item: IncludesInboxMessage } | { kind: `wake`; order: TimelineOrder; item: IncludesWakeMessage } | { kind: `signal`; order: TimelineOrder; item: IncludesSignal } + | { + kind: `realtime_transcript` + order: TimelineOrder + item: IncludesRealtimeTranscript + } | { kind: `run`; order: TimelineOrder; item: IncludesRun } | { kind: `context_inserted` @@ -387,6 +400,13 @@ export function materializeTimeline( order: item.order, item, })), + ...(data.realtimeTranscripts ?? []) + .filter((item) => item.direction === `input` && item.text.length > 0) + .map((item) => ({ + kind: `realtime_transcript` as const, + order: item.order, + item, + })), ...data.runs.map((item) => ({ kind: `run` as const, order: item.order, @@ -445,6 +465,17 @@ export function materializeTimeline( signal: entry.item, } + case `realtime_transcript`: + return { + kind: `realtime_transcript`, + at: orderToOffset(entry.order), + key: entry.item.key, + sessionId: entry.item.session_id, + direction: entry.item.direction, + text: entry.item.text, + status: entry.item.status, + } + case `run`: return materializeRunItem(entry.item) diff --git a/packages/agents-runtime/src/types.ts b/packages/agents-runtime/src/types.ts index 5463ad1cec..cdec0e66c9 100644 --- a/packages/agents-runtime/src/types.ts +++ b/packages/agents-runtime/src/types.ts @@ -407,6 +407,15 @@ export type TimelineItem = } | { kind: `wake`; at: number; payload: unknown } | { kind: `signal`; at: number; signal: EntitySignalEntry } + | { + kind: `realtime_transcript` + at: number + key: string + sessionId: string + direction: `input` | `output` + text: string + status: `partial` | `final` + } | { kind: `run` at: number @@ -996,9 +1005,16 @@ export interface RealtimeAudioFormat { channels: number } +export interface RealtimeInputTranscriptionConfig { + model?: string + language?: string + prompt?: string +} + export interface RealtimeAudioConfig { inputFormat?: RealtimeAudioFormat outputFormat?: RealtimeAudioFormat + inputTranscription?: false | RealtimeInputTranscriptionConfig } export interface RealtimeToolPolicy { diff --git a/packages/agents-runtime/test/openai-realtime.test.ts b/packages/agents-runtime/test/openai-realtime.test.ts index 9de8bf4f07..163a246315 100644 --- a/packages/agents-runtime/test/openai-realtime.test.ts +++ b/packages/agents-runtime/test/openai-realtime.test.ts @@ -102,6 +102,7 @@ describe(`createOpenAIRealtimeProvider`, () => { audio: { input: { format: { type: `audio/pcm`, rate: 24_000 }, + transcription: { model: `gpt-4o-mini-transcribe` }, turn_detection: { type: `server_vad`, threshold: 0.5, @@ -125,6 +126,38 @@ describe(`createOpenAIRealtimeProvider`, () => { }) }) + it(`can disable input audio transcription`, async () => { + FakeWebSocket.instances = [] + const provider = createOpenAIRealtimeProvider({ + apiKey: `sk-test`, + WebSocket: FakeWebSocket, + }) + + await provider.connect({ + systemPrompt: `Talk`, + messages: [], + tools: [], + audio: { + inputFormat: { codec: `pcm16`, sampleRate: 24_000, channels: 1 }, + inputTranscription: false, + }, + }) + + const socket = FakeWebSocket.instances[0]! + expect(socket.sent[0]).toMatchObject({ + session: { + audio: { + input: { + format: { type: `audio/pcm`, rate: 24_000 }, + }, + }, + }, + }) + expect( + (socket.sent[0] as any).session.audio.input.transcription + ).toBeUndefined() + }) + it(`sends audio input chunks as OpenAI input buffer events`, async () => { FakeWebSocket.instances = [] const provider = createOpenAIRealtimeProvider({ @@ -278,6 +311,44 @@ describe(`createOpenAIRealtimeProvider`, () => { }) }) + it(`maps GA input audio transcript events`, async () => { + FakeWebSocket.instances = [] + const provider = createOpenAIRealtimeProvider({ + apiKey: `sk-test`, + WebSocket: FakeWebSocket, + }) + + const session = await provider.connect({ + systemPrompt: `Talk`, + messages: [], + tools: [], + }) + const socket = FakeWebSocket.instances[0]! + const iterator = session.events[Symbol.asyncIterator]() + + socket.emitMessage({ + type: `conversation.item.input_audio_transcription.delta`, + item_id: `item-1`, + delta: `hello`, + }) + await expect(nextEvent(iterator)).resolves.toEqual({ + type: `input_transcript.delta`, + turnId: `item-1`, + delta: `hello`, + }) + + socket.emitMessage({ + type: `conversation.item.input_audio_transcription.completed`, + item_id: `item-1`, + transcript: `hello there`, + }) + await expect(nextEvent(iterator)).resolves.toEqual({ + type: `input_transcript.completed`, + turnId: `item-1`, + text: `hello there`, + }) + }) + it(`maps OpenAI events and executes function calls`, async () => { FakeWebSocket.instances = [] const execute = vi.fn().mockResolvedValue({ diff --git a/packages/agents-runtime/test/realtime-context.test.ts b/packages/agents-runtime/test/realtime-context.test.ts index 0c7109d46c..c80e4c61b3 100644 --- a/packages/agents-runtime/test/realtime-context.test.ts +++ b/packages/agents-runtime/test/realtime-context.test.ts @@ -65,6 +65,71 @@ describe(`ctx.useRealtime()`, () => { ]) }) + it(`persists realtime input and output transcripts`, async () => { + const { ctx } = createTestHandlerContext() + + const realtime = ctx.useRealtime({ + systemPrompt: `You are realtime.`, + provider: createTestRealtimeProvider({ + events: [ + { type: `session.started`, sessionId: `provider-session` }, + { + type: `input_transcript.delta`, + delta: `hel`, + turnId: `input-item-1`, + }, + { + type: `input_transcript.delta`, + delta: `lo`, + turnId: `input-item-1`, + }, + { + type: `input_transcript.completed`, + text: `hello there`, + turnId: `input-item-1`, + }, + { + type: `output_transcript.delta`, + delta: `Hi`, + responseId: `resp-1`, + }, + { + type: `output_transcript.completed`, + text: `Hi there`, + responseId: `resp-1`, + }, + { type: `session.closed` }, + ], + }), + tools: [], + }) + + await realtime.run() + + expect(ctx.db.collections.realtimeTranscripts.toArray).toMatchObject([ + { + key: `realtime-transcript:provider-session:input:input-item-1`, + session_id: `provider-session`, + direction: `input`, + text: `hello there`, + status: `final`, + turn_id: `input-item-1`, + audio_stream: `input`, + created_at: expect.any(String), + }, + { + key: `realtime-transcript:provider-session:output:resp-1`, + session_id: `provider-session`, + direction: `output`, + text: `Hi there`, + status: `final`, + response_id: `resp-1`, + audio_stream: `output`, + created_at: expect.any(String), + }, + ]) + }) + it(`finds active realtime sessions from the manifest`, () => { const { ctx } = createTestHandlerContext() diff --git a/packages/agents-runtime/test/timeline-context.test.ts b/packages/agents-runtime/test/timeline-context.test.ts index 0370ca1b1c..6036c3db0a 100644 --- a/packages/agents-runtime/test/timeline-context.test.ts +++ b/packages/agents-runtime/test/timeline-context.test.ts @@ -6,6 +6,7 @@ import { import type { EntityStreamDB } from '../src/entity-stream-db' import type { IncludesInboxMessage, + IncludesRealtimeTranscript, IncludesRun, IncludesSignal, IncludesWakeMessage, @@ -172,6 +173,40 @@ describe(`timeline context`, () => { expect(result).toEqual([{ role: `user`, content: `updated text` }]) }) + it(`projects realtime input transcripts without duplicating output transcripts`, () => { + const realtimeTranscripts: Array = [ + { + key: `rt-in`, + order: order(1), + session_id: `rt-1`, + direction: `input`, + text: `voice question`, + status: `final`, + audio_stream: `input`, + created_at: `2026-03-28T00:00:00.000Z`, + }, + { + key: `rt-out`, + order: order(2), + session_id: `rt-1`, + direction: `output`, + text: `voice answer`, + status: `final`, + audio_stream: `output`, + created_at: `2026-03-28T00:00:01.000Z`, + }, + ] + + expect( + buildTimelineMessages({ + runs: [], + inbox: [], + wakes: [], + realtimeTranscripts, + }) + ).toEqual([{ role: `user`, content: `voice question` }]) + }) + it(`buildTimelineMessages keeps pending tool calls without emitting tool results`, () => { expect( buildTimelineMessages({ @@ -494,6 +529,7 @@ describe(`timeline context`, () => { __electricRowOffsets: new Map([[`wake-1`, offset(7)]]), }, signals: { toArray: [], __electricRowOffsets: new Map() }, + realtimeTranscripts: { toArray: [], __electricRowOffsets: new Map() }, contextInserted: { toArray: [], __electricRowOffsets: new Map() }, contextRemoved: { toArray: [], __electricRowOffsets: new Map() }, manifests: { toArray: [], __electricRowOffsets: new Map() }, @@ -536,6 +572,7 @@ describe(`timeline context`, () => { inbox: { toArray: [] }, wakes: { toArray: [] }, signals: { toArray: [] }, + realtimeTranscripts: { toArray: [] }, contextInserted: { toArray: [] }, contextRemoved: { toArray: [] }, manifests: { toArray: [] }, diff --git a/packages/agents-server-ui/src/components/EntityTimeline.tsx b/packages/agents-server-ui/src/components/EntityTimeline.tsx index 79c28b4423..2f764fc293 100644 --- a/packages/agents-server-ui/src/components/EntityTimeline.tsx +++ b/packages/agents-server-ui/src/components/EntityTimeline.tsx @@ -257,6 +257,13 @@ function estimateRowHeight( const lines = Math.max(1, Math.ceil(row.comment.body.length / charsPerLine)) return Math.max(58, 42 + lines * lineHeight) + timelineRowGap(row, nextRow) } + if (row.realtimeTranscript) { + const lines = Math.max( + 1, + Math.ceil(row.realtimeTranscript.text.length / charsPerLine) + ) + return Math.max(64, 48 + lines * lineHeight) + timelineRowGap(row) + } if (row.wake || row.signal || row.manifest) { return 76 + timelineRowGap(row, nextRow) } @@ -313,6 +320,7 @@ function timelineRowSearchText( ): string { if (row.comment) return row.comment.body if (row.inbox) return readInboxText(row.inbox.payload) + if (row.realtimeTranscript) return row.realtimeTranscript.text if (row.wake) { return wakeSectionText({ kind: `wake`, @@ -330,6 +338,7 @@ function timelineRowLabel(row: RenderTimelineRow): string { if (row.comment) return `Comment` if (row.inbox?.from_agent) return `Agent message` if (row.inbox) return `User message` + if (row.realtimeTranscript) return `Voice message` if (row.wake) return `Wake` if (row.signal) return `Signal` if (row.error) return `Error` @@ -1305,6 +1314,28 @@ const TimelineRow = memo(function TimelineRow({ ) } + if (row.realtimeTranscript) { + const timestamp = Date.parse(row.realtimeTranscript.created_at) + return ( + + ) + } + if (row.wake) { return ( = 0; index--) { const row = displayRows[index] - if (row?.inbox) { + if (row?.inbox || row?.realtimeTranscript) { return row.$key } } @@ -1583,6 +1614,9 @@ export function EntityTimeline({ if (row.inbox) { const timestamp = Date.parse(row.inbox.timestamp) lastUserTimestamp = Number.isFinite(timestamp) ? timestamp : null + } else if (row.realtimeTranscript) { + const timestamp = Date.parse(row.realtimeTranscript.created_at) + lastUserTimestamp = Number.isFinite(timestamp) ? timestamp : null } else if (row.run) { timestampByRowKey.set(row.$key, lastUserTimestamp) } From 9e523f03ac992ae212628c563e2b0cfbed2855d0 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Tue, 9 Jun 2026 14:23:03 +0100 Subject: [PATCH 24/39] feat(agents-ui): start realtime from spawn screen --- .../src/components/EntityContextDrawer.tsx | 2 + .../src/components/EntityTimeline.tsx | 4 + .../src/components/MessageInput.tsx | 22 +++ .../src/components/NewSessionPage.module.css | 30 ++++ .../src/components/views/ChatView.tsx | 15 ++ .../src/components/views/NewSessionView.tsx | 136 +++++++++++++++--- 6 files changed, 190 insertions(+), 19 deletions(-) diff --git a/packages/agents-server-ui/src/components/EntityContextDrawer.tsx b/packages/agents-server-ui/src/components/EntityContextDrawer.tsx index 292b147c89..c89e41dab1 100644 --- a/packages/agents-server-ui/src/components/EntityContextDrawer.tsx +++ b/packages/agents-server-ui/src/components/EntityContextDrawer.tsx @@ -572,6 +572,7 @@ function manifestKindLabel(manifest: Manifest): string { case `goal`: return `Goal` } + return manifest.kind } function createParentEntry(parent: DrawerEntity): DrawerEntry { @@ -714,6 +715,7 @@ function createManifestEntry( case `goal`: return null } + return null } function describeSourceConfig(config: unknown): string { diff --git a/packages/agents-server-ui/src/components/EntityTimeline.tsx b/packages/agents-server-ui/src/components/EntityTimeline.tsx index 2f764fc293..146f507bdf 100644 --- a/packages/agents-server-ui/src/components/EntityTimeline.tsx +++ b/packages/agents-server-ui/src/components/EntityTimeline.tsx @@ -1007,6 +1007,7 @@ function manifestKindLabel(manifest: Manifest): string { case `goal`: return `Goal` } + return manifest.kind } function manifestTitle(manifest: Manifest): string { @@ -1023,6 +1024,7 @@ function manifestTitle(manifest: Manifest): string { case `goal`: return manifest.id } + return manifest.key } function manifestMeta(manifest: Manifest): string { @@ -1046,6 +1048,7 @@ function manifestMeta(manifest: Manifest): string { case `goal`: return manifest.status ?? `active` } + return `` } function manifestDetails( @@ -1116,6 +1119,7 @@ function manifestDetails( }, ] } + return [] } function manifestIcon(manifest: Manifest) { diff --git a/packages/agents-server-ui/src/components/MessageInput.tsx b/packages/agents-server-ui/src/components/MessageInput.tsx index e637af15c9..88967db778 100644 --- a/packages/agents-server-ui/src/components/MessageInput.tsx +++ b/packages/agents-server-ui/src/components/MessageInput.tsx @@ -74,6 +74,8 @@ export function MessageInput({ drawer, onSend, onStop, + autoStartRealtimeSignal, + onRealtimeAutoStartConsumed, }: { db: EntityStreamDBWithActions | null baseUrl: string @@ -95,6 +97,8 @@ export function MessageInput({ onClearCommentTarget?: () => void onSend?: () => void onStop?: () => void + autoStartRealtimeSignal?: string | null + onRealtimeAutoStartConsumed?: () => void /** * Optional content rendered above the composer, sharing its docked * width and lift into the timeline above. The composer is z-indexed @@ -125,6 +129,7 @@ export function MessageInput({ const [realtimeActive, setRealtimeActive] = useState(false) const [realtimeInputLevel, setRealtimeInputLevel] = useState(0) const realtimeSessionRef = useRef(null) + const handledAutoStartRealtimeRef = useRef(null) const composerFocusRef = useRef<{ focus: () => void } | null>(null) const inputDisabled = disabled || writeDisabled const isCommentMode = composerMode === `comment` @@ -382,6 +387,23 @@ export function MessageInput({ }) }, [baseUrl, canUseRealtime, entityUrl, realtimePending]) + useEffect(() => { + if (!autoStartRealtimeSignal) return + if (handledAutoStartRealtimeRef.current === autoStartRealtimeSignal) return + if (!canUseRealtime || realtimePending) return + handledAutoStartRealtimeRef.current = autoStartRealtimeSignal + onRealtimeAutoStartConsumed?.() + if (!realtimeSessionRef.current) { + handleRealtimeToggle() + } + }, [ + autoStartRealtimeSignal, + canUseRealtime, + handleRealtimeToggle, + onRealtimeAutoStartConsumed, + realtimePending, + ]) + const startEditing = useCallback( (message: EntityTimelineData[`inbox`][number]) => { if (inputDisabled) return diff --git a/packages/agents-server-ui/src/components/NewSessionPage.module.css b/packages/agents-server-ui/src/components/NewSessionPage.module.css index 70106efecc..241c4d76e1 100644 --- a/packages/agents-server-ui/src/components/NewSessionPage.module.css +++ b/packages/agents-server-ui/src/components/NewSessionPage.module.css @@ -470,6 +470,36 @@ display: inline-flex; } +.composerVoice { + all: unset; + display: inline-flex; + align-items: center; + justify-content: center; + width: 24px; + height: 24px; + border-radius: var(--ds-radius-full); + background: var(--ds-gray-a3); + color: var(--ds-text-3); + cursor: pointer; + transition: + background 0.12s ease, + color 0.12s ease, + opacity 0.12s ease; + flex-shrink: 0; +} +.composerVoice:hover:not(:disabled) { + background: var(--ds-gray-a4); + color: var(--ds-text-1); +} +.composerVoice:disabled { + cursor: not-allowed; + opacity: 0.55; +} +.composerVoicePending { + background: var(--ds-accent-a3); + color: var(--ds-accent-11); +} + .composerSend { all: unset; display: inline-flex; diff --git a/packages/agents-server-ui/src/components/views/ChatView.tsx b/packages/agents-server-ui/src/components/views/ChatView.tsx index b6b2509332..ce1f1c2160 100644 --- a/packages/agents-server-ui/src/components/views/ChatView.tsx +++ b/packages/agents-server-ui/src/components/views/ChatView.tsx @@ -362,6 +362,19 @@ function GenericChatBody({ setStopPending(false) }, [entityUrl]) + const autoStartRealtimeSignal = + viewParams?.realtime === `start` && entityUrl + ? `${entityUrl}:realtime:start` + : null + const handleRealtimeAutoStartConsumed = useCallback(() => { + const nextParams = Object.fromEntries( + Object.entries(viewParams ?? {}).filter(([key]) => key !== `realtime`) + ) + helpers.setTileView(tileId, `chat`, { + viewParams: Object.keys(nextParams).length > 0 ? nextParams : undefined, + }) + }, [helpers, tileId, viewParams]) + const stopGeneration = useCallback(() => { if (!canSignal) return if (!entityUrl || !signalEntity || !generationActive || stopPending) return @@ -441,6 +454,8 @@ function GenericChatBody({ )} onSend={() => setSentMessageSignal((value) => value + 1)} onStop={stopGeneration} + autoStartRealtimeSignal={autoStartRealtimeSignal} + onRealtimeAutoStartConsumed={handleRealtimeAutoStartConsumed} /> ) diff --git a/packages/agents-server-ui/src/components/views/NewSessionView.tsx b/packages/agents-server-ui/src/components/views/NewSessionView.tsx index 225cb9351f..4b9cc37a7e 100644 --- a/packages/agents-server-ui/src/components/views/NewSessionView.tsx +++ b/packages/agents-server-ui/src/components/views/NewSessionView.tsx @@ -5,6 +5,7 @@ import { ChevronDown, ChevronRight, Cpu, + Mic, Sparkles, } from 'lucide-react' import { eq, not, useLiveQuery } from '@tanstack/react-db' @@ -67,6 +68,7 @@ import type { SlashCommandRow, } from '@electric-ax/agents-runtime/client' import type { StandaloneViewProps } from '../../lib/workspace/viewRegistry' +import type { TileViewParams } from '../../lib/workspace/types' /** * The "default agent" — when an entity type with this name is registered @@ -74,6 +76,7 @@ import type { StandaloneViewProps } from '../../lib/workspace/viewRegistry' * so the most common flow is one keystroke away. */ const DEFAULT_AGENT_NAME = `horton` +const REALTIME_AUTOSTART_VIEW_PARAMS: TileViewParams = { realtime: `start` } const HERO_TITLES = [ `Let’s ship`, @@ -344,7 +347,8 @@ export function NewSessionView({ initialMessage?: unknown, initialMessageType?: string, initialAttachments?: Array, - sandboxProfile?: string | null + sandboxProfile?: string | null, + viewParams?: TileViewParams ): Promise => { if (!spawnEntity) return false setError(null) @@ -402,6 +406,7 @@ export function NewSessionView({ } helpers.openEntity(entityUrl, { target: { tileId, position: `replace` }, + ...(viewParams ? { viewParams } : {}), }) return true } catch (err) { @@ -450,20 +455,15 @@ export function NewSessionView({ return () => setToolbarTitle(null) }, [handleCancelSelected, selected, setToolbarTitle]) - const handleStartDefault = useCallback( - async ( - input: string | ComposerInputPayload, + const prepareDefaultAgentArgs = useCallback( + ( args: Record, - attachments: Array, sandboxProfile: string | null - ): Promise => { - if (!defaultAgent) return false - // Inject the picker's choice into the spawn args for the composer flow - // only — non-default agents have their own schemas and may not - // understand `workingDirectory`. A remote sandbox runs in the provider - // VM, so a host working directory is meaningless there: skip it for - // remote profiles. The spawned session itself becomes the newest - // synced recent for this runner. + ): Record => { + // Inject the picker's choice into the spawn args for the default-agent + // composer only — non-default agents have their own schemas and may not + // understand `workingDirectory`. Remote sandboxes run in provider VMs, so + // host paths are meaningless there. const profileIsRemote = isSandboxProfileRemote( allSandboxProfiles, sandboxProfile @@ -472,7 +472,20 @@ export function NewSessionView({ // factory — require a (non-remote) profile or the arg is a no-op. const includeWorkingDir = workingDirectory !== null && sandboxProfile !== null && !profileIsRemote - const augmented = includeWorkingDir ? { ...args, workingDirectory } : args + return includeWorkingDir ? { ...args, workingDirectory } : args + }, + [allSandboxProfiles, workingDirectory] + ) + + const handleStartDefault = useCallback( + async ( + input: string | ComposerInputPayload, + args: Record, + attachments: Array, + sandboxProfile: string | null + ): Promise => { + if (!defaultAgent) return false + const augmented = prepareDefaultAgentArgs(args, sandboxProfile) const hasAttachments = attachments.length > 0 const initialMessage = typeof input === `string` @@ -493,7 +506,27 @@ export function NewSessionView({ sandboxProfile ) }, - [defaultAgent, doSpawn, workingDirectory, allSandboxProfiles] + [defaultAgent, doSpawn, prepareDefaultAgentArgs] + ) + + const handleStartDefaultRealtime = useCallback( + async ( + args: Record, + sandboxProfile: string | null + ): Promise => { + if (!defaultAgent) return false + const augmented = prepareDefaultAgentArgs(args, sandboxProfile) + return await doSpawn( + defaultAgent.name, + augmented, + undefined, + undefined, + undefined, + sandboxProfile, + REALTIME_AUTOSTART_VIEW_PARAMS + ) + }, + [defaultAgent, doSpawn, prepareDefaultAgentArgs] ) const defaultComposerReady = @@ -531,6 +564,7 @@ export function NewSessionView({ defaultAgentSandboxProfiles={defaultAgent ? allSandboxProfiles : []} onSelectType={handleSelectType} onStartDefault={handleStartDefault} + onStartDefaultRealtime={handleStartDefaultRealtime} spawnReady={Boolean(spawnEntity)} defaultComposerReady={defaultComposerReady} error={error} @@ -553,6 +587,7 @@ function Picker({ defaultAgentSandboxProfiles, onSelectType, onStartDefault, + onStartDefaultRealtime, spawnReady, defaultComposerReady, error, @@ -573,6 +608,10 @@ function Picker({ attachments: Array, sandboxProfile: string | null ) => Promise + onStartDefaultRealtime: ( + args: Record, + sandboxProfile: string | null + ) => Promise spawnReady: boolean defaultComposerReady: boolean error: string | null @@ -608,6 +647,7 @@ function Picker({ agent={defaultAgent} sandboxProfiles={defaultAgentSandboxProfiles} onSubmit={onStartDefault} + onStartRealtime={onStartDefaultRealtime} disabled={!defaultComposerReady} workingDirectory={workingDirectory} onChangeWorkingDirectory={onChangeWorkingDirectory} @@ -888,6 +928,7 @@ function DefaultAgentComposer({ agent, sandboxProfiles, onSubmit, + onStartRealtime, disabled, workingDirectory, onChangeWorkingDirectory, @@ -904,6 +945,10 @@ function DefaultAgentComposer({ attachments: Array, sandboxProfile: string | null ) => Promise + onStartRealtime: ( + args: Record, + sandboxProfile: string | null + ) => Promise disabled?: boolean workingDirectory: string | null onChangeWorkingDirectory: (path: string | null) => void @@ -923,7 +968,11 @@ function DefaultAgentComposer({ [sandboxProfiles, selectedSandboxProfile] ) const [value, setValue] = useState(``) - const [submitting, setSubmitting] = useState(false) + const [submittingMode, setSubmittingMode] = useState< + `message` | `realtime` | null + >(null) + const submitting = submittingMode !== null + const realtimeSubmitting = submittingMode === `realtime` const composerFocusRef = useRef<{ focus: () => void } | null>(null) const inlineProps = useMemo( () => inlineSchemaProperties(agent.creation_schema), @@ -1004,7 +1053,7 @@ function DefaultAgentComposer({ payload ?? serializeComposerInput(value, slashCommands) const trimmed = nextPayload.source.trim() if ((!trimmed && files.length === 0) || disabled || submitting) return - setSubmitting(true) + setSubmittingMode(`message`) const cleaned: Record = {} for (const [k, v] of Object.entries(args)) { if (v !== undefined && v !== ``) cleaned[k] = v @@ -1023,7 +1072,7 @@ function DefaultAgentComposer({ }) .catch(() => undefined) .finally(() => { - setSubmitting(false) + setSubmittingMode(null) }) }, [ @@ -1040,6 +1089,29 @@ function DefaultAgentComposer({ ] ) + const startRealtime = useCallback(() => { + const files = imageAttachmentsEnabled ? attachments : [] + if (disabled || submitting || files.length > 0) return + setSubmittingMode(`realtime`) + const cleaned: Record = {} + for (const [k, v] of Object.entries(args)) { + if (v !== undefined && v !== ``) cleaned[k] = v + } + void onStartRealtime(cleaned, selectedSandboxProfile) + .catch(() => undefined) + .finally(() => { + setSubmittingMode(null) + }) + }, [ + args, + attachments, + disabled, + imageAttachmentsEnabled, + onStartRealtime, + selectedSandboxProfile, + submitting, + ]) + const attachmentCount = imageAttachmentsEnabled ? attachments.length : 0 const isActive = Boolean( (value.trim() || attachmentCount > 0) && !disabled && !submitting @@ -1048,6 +1120,12 @@ function DefaultAgentComposer({ const sendTooltip = submitting ? `Starting ${agent.name} session` : `Start ${agent.name} session` + const realtimeTooltip = + attachmentCount > 0 + ? `Remove attachments to start voice mode` + : realtimeSubmitting + ? `Starting voice session` + : `Start voice session` return (
{submitting && ( - Starting… + + {realtimeSubmitting ? `Starting voice…` : `Starting…`} + )} + + + + + diff --git a/packages/agents-server-ui/src/components/views/NewSessionView.tsx b/packages/agents-server-ui/src/components/views/NewSessionView.tsx index 4b9cc37a7e..40d8efdeb5 100644 --- a/packages/agents-server-ui/src/components/views/NewSessionView.tsx +++ b/packages/agents-server-ui/src/components/views/NewSessionView.tsx @@ -1,11 +1,11 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { ArrowUp, + AudioLines, Check, ChevronDown, ChevronRight, Cpu, - Mic, Sparkles, } from 'lucide-react' import { eq, not, useLiveQuery } from '@tanstack/react-db' @@ -1217,7 +1217,7 @@ function DefaultAgentComposer({ .filter(Boolean) .join(` `)} > - + diff --git a/packages/agents-server-ui/src/lib/realtime-audio.ts b/packages/agents-server-ui/src/lib/realtime-audio.ts index 4379d7ee5c..c4a47279fc 100644 --- a/packages/agents-server-ui/src/lib/realtime-audio.ts +++ b/packages/agents-server-ui/src/lib/realtime-audio.ts @@ -32,6 +32,28 @@ const MIC_CAPTURE_CHUNK_SAMPLES = 1024 const MIC_WORKLET_PROCESSOR_NAME = `realtime-mic-capture` const BYTES_PER_PCM16_SAMPLE = 2 const TRUNCATE_SAFETY_MS = 80 +const MIC_PRE_ROLL_MS = 360 +const MIC_VAD_TAIL_MS = 700 +const MIC_MAX_QUEUE_MS = 1600 +const MIC_APPEND_BATCH_MS = 60 +const MIC_APPEND_DRAIN_WAIT_MS = 350 +const MIC_MIN_START_LEVEL = 0.012 +const MIC_MIN_CONTINUE_LEVEL = 0.006 +const MIC_PLAYBACK_START_LEVEL = 0.035 +const MIC_START_CONFIRM_CHUNKS = 1 +const MIC_PLAYBACK_START_CONFIRM_CHUNKS = 4 +const MIC_NOISE_MARGIN_START = 0.01 +const MIC_NOISE_MARGIN_CONTINUE = 0.004 +const MIC_NOISE_FLOOR_INITIAL = 0.003 +const MIC_NOISE_FLOOR_MAX = 0.018 +const MIC_NOISE_FLOOR_ALPHA = 0.008 + +const NO_RETRY_BACKOFF = { + initialDelay: 100, + maxDelay: 100, + multiplier: 1, + maxRetries: 0, +} type MicCapture = { node: AudioNode @@ -65,6 +87,24 @@ function pcm16DurationMs(byteLength: number): number { return (byteLength / BYTES_PER_PCM16_SAMPLE / REALTIME_SAMPLE_RATE) * 1000 } +function durationBytes(durationMs: number): number { + return Math.ceil( + (durationMs / 1000) * REALTIME_SAMPLE_RATE * BYTES_PER_PCM16_SAMPLE + ) +} + +function combineChunks(chunks: Array): Uint8Array { + if (chunks.length === 1) return chunks[0]! + const totalLength = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0) + const combined = new Uint8Array(totalLength) + let offset = 0 + for (const chunk of chunks) { + combined.set(chunk, offset) + offset += chunk.byteLength + } + return combined +} + function audioLevel(input: Float32Array): number { if (input.length === 0) return 0 let sumSquares = 0 @@ -85,10 +125,41 @@ function pcm16Floats(bytes: Uint8Array): Float32Array { return output } +function alignedPcm16Chunk( + chunk: Uint8Array, + remainder: Uint8Array | undefined +): { bytes: Uint8Array; remainder: Uint8Array | undefined } { + const bytes = remainder ? combineChunks([remainder, chunk]) : chunk + const alignedLength = + bytes.byteLength - (bytes.byteLength % BYTES_PER_PCM16_SAMPLE) + + return { + bytes: + alignedLength === bytes.byteLength + ? bytes + : bytes.subarray(0, alignedLength), + remainder: + alignedLength === bytes.byteLength + ? undefined + : bytes.slice(alignedLength), + } +} + function jsonBytes(value: unknown): Uint8Array { return new TextEncoder().encode(JSON.stringify(value)) } +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +async function settleWithin( + promise: Promise, + timeoutMs: number +): Promise { + await Promise.race([promise, delay(timeoutMs)]) +} + function micWorkletSource(): string { return ` class RealtimeMicCaptureProcessor extends AudioWorkletProcessor { @@ -148,28 +219,18 @@ registerProcessor('${MIC_WORKLET_PROCESSOR_NAME}', RealtimeMicCaptureProcessor) ` } -function trackPendingAppend( - pending: Set>, - append: Promise, - onError: (error: unknown) => void -): void { - let tracked: Promise - tracked = append.catch(onError).finally(() => { - pending.delete(tracked) - }) - pending.add(tracked) -} - function streamHandle( baseUrl: string, path: string, - contentType: string + contentType: string, + opts: { retryWrites?: boolean } = {} ): DurableStream { const url = streamUrl(baseUrl, path) return new DurableStream({ url, headers: getConfiguredServerHeaders(url), contentType, + ...(opts.retryWrites === false ? { backoffOptions: NO_RETRY_BACKOFF } : {}), batching: true, }) } @@ -323,15 +384,153 @@ export async function startRealtimeAudioSession({ let currentOutputStartedAt: number | null = null let currentOutputReceivedMs = 0 let micChunks = 0 + let micSentChunks = 0 let playbackChunks = 0 let controlEvents = 0 - const playbackNodes = new Set() + let speechTurns = 0 + let voiceCandidateChunks = 0 + let noiseFloor = MIC_NOISE_FLOOR_INITIAL + let speechActive = false + let lastVoiceAt = 0 + let audioQueuedBytes = 0 + let audioInputStopping = false + let audioInputError: Error | undefined + let wakeAudioInputWriter: (() => void) | undefined + let activeResponseId: string | undefined + let responseActive = false + const preSpeechChunks: Array = [] + const audioQueue: Array = [] const pendingAudioAppends = new Set>() + const playbackNodes = new Set() + let audioInputWriter = Promise.resolve() const appendControl = async (value: unknown): Promise => { await controlIn?.append(jsonBytes(value)) } + const wakeAudioWriter = (): void => { + wakeAudioInputWriter?.() + wakeAudioInputWriter = undefined + } + + const playbackIsActive = (): boolean => + playbackNodes.size > 0 || + nextPlaybackTime > playbackContext.currentTime + 0.05 + + const trimPreSpeechChunks = (): void => { + const maxBytes = durationBytes(MIC_PRE_ROLL_MS) + let total = preSpeechChunks.reduce( + (sum, chunk) => sum + chunk.byteLength, + 0 + ) + while (total > maxBytes && preSpeechChunks.length > 0) { + const dropped = preSpeechChunks.shift()! + total -= dropped.byteLength + } + } + + const rememberPreSpeechChunk = (bytes: Uint8Array): void => { + preSpeechChunks.push(bytes) + trimPreSpeechChunks() + } + + const dropStaleAudio = (): void => { + const maxBytes = durationBytes(MIC_MAX_QUEUE_MS) + while (audioQueuedBytes > maxBytes && audioQueue.length > 1) { + const dropped = audioQueue.shift()! + audioQueuedBytes -= dropped.byteLength + } + } + + const enqueueAudioInput = (bytes: Uint8Array): void => { + audioQueue.push(bytes) + audioQueuedBytes += bytes.byteLength + dropStaleAudio() + wakeAudioWriter() + } + + const dequeueAudioBatch = (): Uint8Array | null => { + if (audioQueue.length === 0) return null + const maxBytes = durationBytes(MIC_APPEND_BATCH_MS) + let batchBytes = 0 + const chunks: Array = [] + while (audioQueue.length > 0) { + const next = audioQueue[0]! + if (chunks.length > 0 && batchBytes + next.byteLength > maxBytes) break + chunks.push(audioQueue.shift()!) + batchBytes += next.byteLength + audioQueuedBytes -= next.byteLength + } + return combineChunks(chunks) + } + + const toError = (value: unknown): Error => + value instanceof Error ? value : new Error(String(value)) + + const throwIfAudioInputFailed = (): void => { + if (audioInputError) throw audioInputError + } + + const waitForPendingAudioInput = async ( + timeoutMs = MIC_APPEND_DRAIN_WAIT_MS + ): Promise => { + if (pendingAudioAppends.size > 0) { + await Promise.race([ + Promise.all(Array.from(pendingAudioAppends)), + new Promise((resolve) => setTimeout(resolve, timeoutMs)), + ]) + } + throwIfAudioInputFailed() + } + + const trackAudioAppend = ( + audioIn: DurableStream, + batch: Uint8Array + ): void => { + const append = audioIn + .append(batch) + .then(() => { + micSentChunks += 1 + if (micSentChunks === 1) { + console.info( + `[realtime-audio] microphone first sent chunk session=${session?.sessionId} bytes=${batch.byteLength}` + ) + } + }) + .catch((error) => { + audioInputError ??= toError(error) + console.warn(`[realtime-audio] microphone append failed`, error) + }) + .finally(() => { + pendingAudioAppends.delete(append) + }) + pendingAudioAppends.add(append) + } + + const runAudioInputWriter = async (audioIn: DurableStream): Promise => { + while ( + !audioInputStopping || + audioQueue.length > 0 || + pendingAudioAppends.size > 0 + ) { + throwIfAudioInputFailed() + const batch = dequeueAudioBatch() + if (batch) { + trackAudioAppend(audioIn, batch) + continue + } + + if (audioInputStopping && pendingAudioAppends.size > 0) { + await waitForPendingAudioInput(250) + continue + } + + await new Promise((resolve) => { + wakeAudioInputWriter = resolve + }) + } + } + const stopScheduledPlayback = (): void => { for (const node of playbackNodes) { try { @@ -352,8 +551,18 @@ export async function startRealtimeAudioSession({ currentOutputReceivedMs = 0 } - const interruptPlayback = (): void => { + const interruptPlayback = ({ + cancelResponse = true, + }: { cancelResponse?: boolean } = {}): void => { const itemId = currentOutputItemId + const wasResponseActive = responseActive + responseActive = false + if (cancelResponse && (wasResponseActive || itemId)) { + void appendControl({ type: `response.cancel` }).catch((error) => { + console.warn(`[realtime-audio] response cancel failed`, error) + }) + } + if (!itemId) { stopScheduledPlayback() return @@ -386,22 +595,31 @@ export async function startRealtimeAudioSession({ } const cleanup = async (sendClose: boolean): Promise => { - abort.abort() micCapture?.cleanup() micCapture?.node.disconnect() silentOutput?.disconnect() source?.disconnect() onInputLevel?.(0) for (const track of media?.getTracks() ?? []) track.stop() + audioInputStopping = true + wakeAudioWriter() + await settleWithin(audioInputWriter, 250) + abort.abort() stopScheduledPlayback() - await Promise.allSettled(pendingAudioAppends) + await settleWithin(audioInputWriter, 250) if (sendClose && controlIn) { - await appendControl({ - type: `session.close`, - reason: `client-stop`, - }).catch(() => undefined) + await settleWithin( + appendControl({ + type: `session.close`, + reason: `client-stop`, + }).catch(() => undefined), + 500 + ) } - await Promise.allSettled([playback, control]) + await Promise.allSettled([ + settleWithin(playback, 250), + settleWithin(control, 250), + ]) await Promise.allSettled([micContext.close(), playbackContext.close()]) } @@ -423,7 +641,8 @@ export async function startRealtimeAudioSession({ const audioIn = streamHandle( baseUrl, session.streams.audio_in, - `audio/pcm; rate=${REALTIME_SAMPLE_RATE}; channels=1` + `audio/pcm; rate=${REALTIME_SAMPLE_RATE}; channels=1`, + { retryWrites: false } ) const audioOut = streamHandle( baseUrl, @@ -433,13 +652,19 @@ export async function startRealtimeAudioSession({ controlIn = streamHandle( baseUrl, session.streams.control_in, - `application/json` + `application/json`, + { retryWrites: false } ) const controlOut = streamHandle( baseUrl, session.streams.control_out, `application/json` ) + audioInputWriter = runAudioInputWriter(audioIn).catch((error) => { + if (!abort.signal.aborted) { + console.warn(`[realtime-audio] microphone writer failed`, error) + } + }) const handleInputAudio = (bytes: Uint8Array, level: number): void => { if (abort.signal.aborted) return @@ -450,13 +675,60 @@ export async function startRealtimeAudioSession({ `[realtime-audio] microphone first chunk session=${session?.sessionId} bytes=${bytes.byteLength}` ) } - trackPendingAppend( - pendingAudioAppends, - audioIn.append(bytes), - (error) => { - console.warn(`[realtime-audio] microphone append failed`, error) - } + rememberPreSpeechChunk(bytes) + + const now = performance.now() + const startThreshold = Math.max( + MIC_MIN_START_LEVEL, + noiseFloor + MIC_NOISE_MARGIN_START, + playbackIsActive() ? MIC_PLAYBACK_START_LEVEL : 0 + ) + const continueThreshold = Math.max( + MIC_MIN_CONTINUE_LEVEL, + noiseFloor + MIC_NOISE_MARGIN_CONTINUE ) + const hasVoice = + level >= (speechActive ? continueThreshold : startThreshold) + + if (hasVoice) { + lastVoiceAt = now + if (!speechActive) { + voiceCandidateChunks += 1 + const requiredChunks = playbackIsActive() + ? MIC_PLAYBACK_START_CONFIRM_CHUNKS + : MIC_START_CONFIRM_CHUNKS + if (voiceCandidateChunks < requiredChunks) return + + voiceCandidateChunks = 0 + speechActive = true + speechTurns += 1 + console.info( + `[realtime-audio] microphone voice gate opened session=${session?.sessionId} turn=${speechTurns} level=${level.toFixed(4)} threshold=${startThreshold.toFixed(4)} noiseFloor=${noiseFloor.toFixed(4)}` + ) + for (const chunk of preSpeechChunks.splice(0)) { + enqueueAudioInput(chunk) + } + return + } + enqueueAudioInput(bytes) + return + } + + voiceCandidateChunks = 0 + + if (speechActive) { + if (now - lastVoiceAt < MIC_VAD_TAIL_MS) { + enqueueAudioInput(bytes) + return + } + speechActive = false + } + + if (!speechActive && level < startThreshold) { + noiseFloor = + noiseFloor * (1 - MIC_NOISE_FLOOR_ALPHA) + + Math.min(level, MIC_NOISE_FLOOR_MAX) * MIC_NOISE_FLOOR_ALPHA + } } source = micContext.createMediaStreamSource(media) micCapture = await createMicCapture(micContext, handleInputAudio) @@ -475,6 +747,7 @@ export async function startRealtimeAudioSession({ signal: abort.signal, warnOnHttp: false, }) + let playbackRemainder: Uint8Array | undefined try { for await (const chunk of response.bodyStream()) { if (abort.signal.aborted || chunk.byteLength === 0) continue @@ -484,7 +757,11 @@ export async function startRealtimeAudioSession({ `[realtime-audio] playback first chunk session=${session?.sessionId} bytes=${chunk.byteLength}` ) } - const samples = pcm16Floats(chunk) + const aligned = alignedPcm16Chunk(chunk, playbackRemainder) + playbackRemainder = aligned.remainder + if (aligned.bytes.byteLength === 0) continue + + const samples = pcm16Floats(aligned.bytes) const buffer = playbackContext.createBuffer( 1, samples.length, @@ -533,7 +810,25 @@ export async function startRealtimeAudioSession({ `[realtime-audio] control first event session=${session?.sessionId} type=${event.type}` ) } - if ( + if (event.type === `response.started`) { + activeResponseId = + typeof event.responseId === `string` + ? event.responseId + : undefined + responseActive = true + } else if ( + event.type === `response.completed` || + event.type === `response.cancelled` + ) { + if ( + !activeResponseId || + typeof event.responseId !== `string` || + event.responseId === activeResponseId + ) { + activeResponseId = undefined + responseActive = false + } + } else if ( event.type === `output_audio.delta` && typeof event.itemId === `string` ) { @@ -542,7 +837,7 @@ export async function startRealtimeAudioSession({ currentOutputReceivedMs += pcm16DurationMs(event.byteLength) } } else if (event.type === `input_audio.speech_started`) { - interruptPlayback() + interruptPlayback({ cancelResponse: false }) } } } finally { diff --git a/packages/agents-server/src/routing/hooks.ts b/packages/agents-server/src/routing/hooks.ts index 0a5fc39209..b59e2b5e1e 100644 --- a/packages/agents-server/src/routing/hooks.ts +++ b/packages/agents-server/src/routing/hooks.ts @@ -89,6 +89,15 @@ export function applyCors( `electric-owner-entity`, ELECTRIC_PRINCIPAL_HEADER, `ngrok-skip-browser-warning`, + `producer-id`, + `producer-epoch`, + `producer-seq`, + `producer-expected-seq`, + `producer-received-seq`, + `stream-closed`, + `stream-expires-at`, + `stream-seq`, + `stream-ttl`, ].join(`, `) ) headers.set(`access-control-expose-headers`, `*`) diff --git a/packages/agents-server/test/routing-hooks.test.ts b/packages/agents-server/test/routing-hooks.test.ts index 30e653b910..7eef9d4e63 100644 --- a/packages/agents-server/test/routing-hooks.test.ts +++ b/packages/agents-server/test/routing-hooks.test.ts @@ -43,9 +43,12 @@ describe(`routing/hooks`, () => { expect(wrapped?.headers.get(`access-control-allow-methods`)).toContain( `GET` ) - expect(wrapped?.headers.get(`access-control-allow-headers`)).toContain( - `electric-principal` - ) + const allowedHeaders = wrapped?.headers.get(`access-control-allow-headers`) + expect(allowedHeaders).toContain(`electric-principal`) + expect(allowedHeaders).toContain(`producer-id`) + expect(allowedHeaders).toContain(`producer-epoch`) + expect(allowedHeaders).toContain(`producer-seq`) + expect(allowedHeaders).toContain(`stream-closed`) }) it(`errorMapper converts ElectricAgentsError to API error JSON`, async () => { diff --git a/packages/agents/src/agents/horton.ts b/packages/agents/src/agents/horton.ts index a411553188..6a694cf954 100644 --- a/packages/agents/src/agents/horton.ts +++ b/packages/agents/src/agents/horton.ts @@ -956,6 +956,18 @@ function createAssistantHandler(options: { audio: { inputFormat: { codec: `pcm16`, sampleRate: 24_000, channels: 1 }, outputFormat: { codec: `pcm16`, sampleRate: 24_000, channels: 1 }, + inputTranscription: { + model: `gpt-realtime-whisper`, + delay: `minimal`, + }, + turnDetection: { + type: `server_vad`, + threshold: 0.55, + prefixPaddingMs: 300, + silenceDurationMs: 500, + createResponse: true, + interruptResponse: true, + }, }, toolPolicy: { direct: hortonRealtimeDirectTools(tools as AgentTool[]), diff --git a/packages/agents/test/horton-tool-composition.test.ts b/packages/agents/test/horton-tool-composition.test.ts index 1b8d3c019a..ae232a601f 100644 --- a/packages/agents/test/horton-tool-composition.test.ts +++ b/packages/agents/test/horton-tool-composition.test.ts @@ -213,6 +213,12 @@ describe(`horton tool composition`, () => { [ { provider: { id: string; model: string } + audio: { + inputTranscription?: { + model?: string + delay?: string + } + } toolPolicy: { direct: Array } }, ] @@ -222,6 +228,10 @@ describe(`horton tool composition`, () => { id: `openai`, model: `gpt-realtime`, }) + expect(realtimeConfig.audio.inputTranscription).toEqual({ + model: `gpt-realtime-whisper`, + delay: `minimal`, + }) expect(realtimeConfig.toolPolicy.direct).toEqual( expect.arrayContaining([ `web_search`, From 60c29d4368f57405489c264dab8c7ab28b7e792b Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 10 Jun 2026 09:13:29 +0100 Subject: [PATCH 33/39] fix(agents): use gpt-realtime-2 --- packages/agents-runtime/src/openai-realtime.ts | 2 +- .../test/electric-agents-client.test.ts | 6 +++--- .../test/openai-realtime.test.ts | 4 ++-- .../test/realtime-context.test.ts | 18 +++++++++--------- ...ntime-server-client-update-metadata.test.ts | 8 ++++---- .../agents-server-ui/src/lib/realtime-audio.ts | 2 +- ...ric-agents-manager-write-validation.test.ts | 6 +++--- .../test/horton-tool-composition.test.ts | 4 ++-- 8 files changed, 25 insertions(+), 25 deletions(-) diff --git a/packages/agents-runtime/src/openai-realtime.ts b/packages/agents-runtime/src/openai-realtime.ts index b7111d8342..caa48d6a8f 100644 --- a/packages/agents-runtime/src/openai-realtime.ts +++ b/packages/agents-runtime/src/openai-realtime.ts @@ -31,7 +31,7 @@ type OpenAIRealtimeWebSocketConstructor = new ( init?: unknown ) => OpenAIRealtimeSocket -const DEFAULT_OPENAI_REALTIME_MODEL = `gpt-realtime` +const DEFAULT_OPENAI_REALTIME_MODEL = `gpt-realtime-2` const DEFAULT_OPENAI_INPUT_TRANSCRIPTION_MODEL = `gpt-4o-mini-transcribe` export interface OpenAIRealtimeProviderOptions { diff --git a/packages/agents-runtime/test/electric-agents-client.test.ts b/packages/agents-runtime/test/electric-agents-client.test.ts index db3c7388f0..858faefff0 100644 --- a/packages/agents-runtime/test/electric-agents-client.test.ts +++ b/packages/agents-runtime/test/electric-agents-client.test.ts @@ -61,7 +61,7 @@ describe(`createAgentsClient`, () => { sessionId: `rt-1`, entityUrl: `/horton/demo`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, status: `requested`, startedAt: `2026-06-09T10:00:00.000Z`, streams: { @@ -216,7 +216,7 @@ describe(`createAgentsClient`, () => { client.startRealtimeSession({ entityUrl: `/horton/demo`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, }) ).resolves.toMatchObject({ sessionId: `rt-1`, @@ -228,7 +228,7 @@ describe(`createAgentsClient`, () => { expect(mockState.startRealtimeSession).toHaveBeenCalledWith({ entityUrl: `/horton/demo`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, }) }) diff --git a/packages/agents-runtime/test/openai-realtime.test.ts b/packages/agents-runtime/test/openai-realtime.test.ts index 7beba32642..f6703faaf9 100644 --- a/packages/agents-runtime/test/openai-realtime.test.ts +++ b/packages/agents-runtime/test/openai-realtime.test.ts @@ -76,7 +76,7 @@ describe(`createOpenAIRealtimeProvider`, () => { const socket = FakeWebSocket.instances[0]! expect(socket.url).toBe( - `wss://api.openai.com/v1/realtime?model=gpt-realtime` + `wss://api.openai.com/v1/realtime?model=gpt-realtime-2` ) expect(socket.init).toEqual({ headers: { @@ -88,7 +88,7 @@ describe(`createOpenAIRealtimeProvider`, () => { type: `session.update`, session: { type: `realtime`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, instructions: `You are Horton.`, output_modalities: [`audio`], tool_choice: `auto`, diff --git a/packages/agents-runtime/test/realtime-context.test.ts b/packages/agents-runtime/test/realtime-context.test.ts index 66b2d81bb1..5de390de11 100644 --- a/packages/agents-runtime/test/realtime-context.test.ts +++ b/packages/agents-runtime/test/realtime-context.test.ts @@ -519,7 +519,7 @@ describe(`ctx.useRealtime()`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, status: `active`, startedAt: `2026-06-09T12:00:00.000Z`, endedAt: null, @@ -546,7 +546,7 @@ describe(`ctx.useRealtime()`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, status: `requested`, startedAt: `2026-06-09T12:00:00.000Z`, endedAt: null, @@ -592,7 +592,7 @@ describe(`ctx.useRealtime()`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, status: `requested`, startedAt: `2026-06-09T12:00:00.000Z`, endedAt: null, @@ -609,7 +609,7 @@ describe(`ctx.useRealtime()`, () => { systemPrompt: `You are realtime.`, provider: { id: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, connect: async () => { throw new Error(`missing key`) }, @@ -694,7 +694,7 @@ describe(`ctx.useRealtime()`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, status: `active`, startedAt: `2026-06-09T12:00:00.000Z`, endedAt: null, @@ -772,7 +772,7 @@ describe(`ctx.useRealtime()`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, status: `active`, startedAt: `2026-06-09T12:00:00.000Z`, endedAt: null, @@ -844,7 +844,7 @@ describe(`ctx.useRealtime()`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, status: `active`, startedAt: `2026-06-09T12:00:00.000Z`, endedAt: null, @@ -917,7 +917,7 @@ describe(`ctx.useRealtime()`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, status: `active`, startedAt: `2026-06-09T12:00:00.000Z`, endedAt: null, @@ -986,7 +986,7 @@ describe(`ctx.useRealtime()`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, status: `active`, startedAt: `2026-06-09T12:00:00.000Z`, endedAt: null, diff --git a/packages/agents-runtime/test/runtime-server-client-update-metadata.test.ts b/packages/agents-runtime/test/runtime-server-client-update-metadata.test.ts index 8b801519b0..092a72a782 100644 --- a/packages/agents-runtime/test/runtime-server-client-update-metadata.test.ts +++ b/packages/agents-runtime/test/runtime-server-client-update-metadata.test.ts @@ -273,7 +273,7 @@ describe(`runtime-server-client realtime sessions`, () => { sessionId: `rt-1`, entityUrl: `/horton/demo`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, status: `requested`, startedAt: `2026-06-09T10:00:00.000Z`, streams: { @@ -301,7 +301,7 @@ describe(`runtime-server-client realtime sessions`, () => { entityUrl: `/horton/demo`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, inputAudio: { codec: `pcm16`, sampleRate: 16_000, channels: 1 }, meta: { source: `button` }, }) @@ -319,7 +319,7 @@ describe(`runtime-server-client realtime sessions`, () => { entityUrl: `/horton/demo`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, inputAudio: { codec: `pcm16`, sampleRate: 16_000, channels: 1 }, meta: { source: `button` }, }) @@ -338,7 +338,7 @@ describe(`runtime-server-client realtime sessions`, () => { client.startRealtimeSession({ entityUrl: `/horton/demo`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, }) ).rejects.toThrow(/startRealtimeSession.*401.*not allowed/) }) diff --git a/packages/agents-server-ui/src/lib/realtime-audio.ts b/packages/agents-server-ui/src/lib/realtime-audio.ts index c4a47279fc..155387948a 100644 --- a/packages/agents-server-ui/src/lib/realtime-audio.ts +++ b/packages/agents-server-ui/src/lib/realtime-audio.ts @@ -333,7 +333,7 @@ async function createRealtimeSession( body: JSON.stringify({ entityUrl, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, inputAudio: { codec: `pcm16`, sampleRate: REALTIME_SAMPLE_RATE, diff --git a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts index e4c8ccb4e2..20ce20f809 100644 --- a/packages/agents-server/test/electric-agents-manager-write-validation.test.ts +++ b/packages/agents-server/test/electric-agents-manager-write-validation.test.ts @@ -158,7 +158,7 @@ describe(`ElectricAgentsManager realtime sessions`, () => { const result = await manager.createRealtimeSession(`/chat/session-1`, { id: `rt-1`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, inputAudio: { codec: `pcm16`, sampleRate: 16_000, channels: 1 }, outputAudio: { codec: `pcm16`, sampleRate: 24_000, channels: 1 }, meta: { source: `test` }, @@ -203,7 +203,7 @@ describe(`ElectricAgentsManager realtime sessions`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, status: `requested`, streams: result.streams, retention: `forever`, @@ -216,7 +216,7 @@ describe(`ElectricAgentsManager realtime sessions`, () => { value: { session_id: `rt-1`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, status: `requested`, streams: result.streams, }, diff --git a/packages/agents/test/horton-tool-composition.test.ts b/packages/agents/test/horton-tool-composition.test.ts index ae232a601f..5194a434d5 100644 --- a/packages/agents/test/horton-tool-composition.test.ts +++ b/packages/agents/test/horton-tool-composition.test.ts @@ -179,7 +179,7 @@ describe(`horton tool composition`, () => { kind: `realtime-session`, id: `rt-1`, provider: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, status: `active`, startedAt: `2026-06-09T12:00:00.000Z`, retention: `forever`, @@ -226,7 +226,7 @@ describe(`horton tool composition`, () => { )[0]![0] expect(realtimeConfig.provider).toMatchObject({ id: `openai`, - model: `gpt-realtime`, + model: `gpt-realtime-2`, }) expect(realtimeConfig.audio.inputTranscription).toEqual({ model: `gpt-realtime-whisper`, From bf420652f1fbd1ba0b44662e76080c4355e68939 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 10 Jun 2026 09:41:55 +0100 Subject: [PATCH 34/39] feat(agents): expose realtime settings --- packages/agents-desktop/src/app/controller.ts | 18 ++ .../agents-desktop/src/ipc/preferences.ts | 11 + packages/agents-desktop/src/preload.ts | 6 + .../agents-desktop/src/settings/realtime.ts | 66 ++++++ packages/agents-desktop/src/settings/store.ts | 8 +- packages/agents-desktop/src/shared/types.ts | 22 ++ .../components/settings/SettingsSidebar.tsx | 8 + .../settings/pages/RealtimePage.module.css | 61 +++++ .../settings/pages/RealtimePage.tsx | 215 ++++++++++++++++++ .../src/hooks/useDocumentTitle.ts | 1 + .../src/lib/realtime-audio.ts | 4 +- .../src/lib/server-connection.ts | 58 +++++ packages/agents-server-ui/src/router.tsx | 4 + 13 files changed, 480 insertions(+), 2 deletions(-) create mode 100644 packages/agents-desktop/src/settings/realtime.ts create mode 100644 packages/agents-server-ui/src/components/settings/pages/RealtimePage.module.css create mode 100644 packages/agents-server-ui/src/components/settings/pages/RealtimePage.tsx diff --git a/packages/agents-desktop/src/app/controller.ts b/packages/agents-desktop/src/app/controller.ts index 1560564d4b..0b7f961fa1 100644 --- a/packages/agents-desktop/src/app/controller.ts +++ b/packages/agents-desktop/src/app/controller.ts @@ -11,6 +11,7 @@ import * as DesktopIpc from '../ipc/register' import { ensureRuntimeEntry as ensureRuntimeEntryInStore } from '../runtime/entries' import { createRuntimeController } from '../runtime/controller' import * as SettingsBootstrap from '../settings/bootstrap' +import * as RealtimeSettings from '../settings/realtime' import * as ServerSelection from '../settings/selection' import { saveDesktopSettings } from '../settings/store' import { desktopStateForWindow as desktopStateForWindowImpl } from '../state/desktop-state' @@ -30,6 +31,7 @@ import type { DesktopMenuSection, DesktopMenuState, DesktopState, + RealtimeSettings as RealtimeSettingsConfig, RuntimeEntry, ServerConfig, } from '../shared/types' @@ -328,6 +330,20 @@ export function createDesktopMainController(ctx: DesktopAppContext) { runtime.refreshPowerSaveBlocker() } + const getRealtimeSettingsStatus = () => + RealtimeSettings.realtimeSettingsStatus({ + settings, + apiKeys, + launchEnv: ctx.envApiKeysSnapshot, + }) + + const setRealtimeSettings = async ( + next: RealtimeSettingsConfig + ): Promise => { + settings.realtime = RealtimeSettings.normalizeRealtimeSettings(next) + await saveSettings() + } + const syncLaunchAtLoginSetting = async (): Promise => { await LoginItems.setLaunchAtLogin(settings.launchAtLogin === true) } @@ -438,6 +454,8 @@ export function createDesktopMainController(ctx: DesktopAppContext) { setLaunchAtLogin, getPreventAppSuspension, setPreventAppSuspension, + getRealtimeSettingsStatus, + setRealtimeSettings, } const loadSettings = (): Promise => diff --git a/packages/agents-desktop/src/ipc/preferences.ts b/packages/agents-desktop/src/ipc/preferences.ts index cfd50bab3f..2a934afd98 100644 --- a/packages/agents-desktop/src/ipc/preferences.ts +++ b/packages/agents-desktop/src/ipc/preferences.ts @@ -2,6 +2,8 @@ import { ipcMain } from 'electron' import type { LaunchAtLoginStatus, PreventAppSuspensionPreference, + RealtimeSettings, + RealtimeSettingsStatus, } from '../shared/types' export type PreferencesIpcDeps = { @@ -9,6 +11,8 @@ export type PreferencesIpcDeps = { setLaunchAtLogin: (enabled: boolean) => Promise getPreventAppSuspension: () => PreventAppSuspensionPreference setPreventAppSuspension: (enabled: boolean) => Promise + getRealtimeSettingsStatus: () => RealtimeSettingsStatus + setRealtimeSettings: (settings: RealtimeSettings) => Promise } export function registerPreferencesIpcHandlers(deps: PreferencesIpcDeps): void { @@ -25,4 +29,11 @@ export function registerPreferencesIpcHandlers(deps: PreferencesIpcDeps): void { `desktop:set-prevent-app-suspension`, (_event, enabled: boolean) => deps.setPreventAppSuspension(Boolean(enabled)) ) + ipcMain.handle(`desktop:get-realtime-settings`, () => + deps.getRealtimeSettingsStatus() + ) + ipcMain.handle( + `desktop:set-realtime-settings`, + (_event, settings: RealtimeSettings) => deps.setRealtimeSettings(settings) + ) } diff --git a/packages/agents-desktop/src/preload.ts b/packages/agents-desktop/src/preload.ts index 82c437a935..af780edb47 100644 --- a/packages/agents-desktop/src/preload.ts +++ b/packages/agents-desktop/src/preload.ts @@ -21,6 +21,8 @@ import type { McpServerConfig, OnboardingState, PreventAppSuspensionPreference, + RealtimeSettings, + RealtimeSettingsStatus, ServerConfig, } from './shared/types' import type { CloudAgentServersState } from './cloud/cloud-agent-servers' @@ -190,6 +192,10 @@ const api = { ipcRenderer.invoke(`desktop:get-prevent-app-suspension`), setPreventAppSuspension: (enabled: boolean): Promise => ipcRenderer.invoke(`desktop:set-prevent-app-suspension`, enabled), + getRealtimeSettings: (): Promise => + ipcRenderer.invoke(`desktop:get-realtime-settings`), + setRealtimeSettings: (settings: RealtimeSettings): Promise => + ipcRenderer.invoke(`desktop:set-realtime-settings`, settings), getWorkingDirectory: (): Promise => ipcRenderer.invoke(`desktop:get-working-directory`), chooseWorkingDirectory: (): Promise => diff --git a/packages/agents-desktop/src/settings/realtime.ts b/packages/agents-desktop/src/settings/realtime.ts new file mode 100644 index 0000000000..e9481c6221 --- /dev/null +++ b/packages/agents-desktop/src/settings/realtime.ts @@ -0,0 +1,66 @@ +import type { + ApiKeys, + DesktopSettings, + RealtimeModelChoice, + RealtimeSettings, + RealtimeSettingsStatus, +} from '../shared/types' + +export const DEFAULT_REALTIME_SETTINGS: RealtimeSettings = { + provider: `openai`, + model: `gpt-realtime-2`, +} + +export const OPENAI_REALTIME_MODELS: Array = [ + { + id: `gpt-realtime-2`, + label: `GPT-Realtime-2`, + description: `Strongest realtime reasoning, tool use, and instruction following.`, + recommended: true, + }, + { + id: `gpt-realtime-1.5`, + label: `GPT-Realtime-1.5`, + description: `Fast, reliable speech-to-speech model for audio in, audio out.`, + }, + { + id: `gpt-realtime-mini`, + label: `GPT-Realtime mini`, + description: `Cost-efficient realtime voice model.`, + }, +] + +const OPENAI_REALTIME_MODEL_IDS = new Set( + OPENAI_REALTIME_MODELS.map((model) => model.id) +) + +export function normalizeRealtimeSettings(value: unknown): RealtimeSettings { + if (!value || typeof value !== `object`) return DEFAULT_REALTIME_SETTINGS + const maybe = value as Partial> + const model = + typeof maybe.model === `string` && + OPENAI_REALTIME_MODEL_IDS.has(maybe.model) + ? maybe.model + : DEFAULT_REALTIME_SETTINGS.model + return { + provider: `openai`, + model, + } +} + +export function realtimeSettingsStatus({ + settings, + apiKeys, + launchEnv, +}: { + settings: DesktopSettings + apiKeys: ApiKeys + launchEnv: ApiKeys +}): RealtimeSettingsStatus { + return { + settings: normalizeRealtimeSettings(settings.realtime), + availableModels: OPENAI_REALTIME_MODELS, + hasOpenAIApiKey: Boolean(apiKeys.openai || launchEnv.openai), + codexEnabled: settings.codex?.enabled === true, + } +} diff --git a/packages/agents-desktop/src/settings/store.ts b/packages/agents-desktop/src/settings/store.ts index 49a938ec98..4519a34392 100644 --- a/packages/agents-desktop/src/settings/store.ts +++ b/packages/agents-desktop/src/settings/store.ts @@ -17,11 +17,15 @@ import { saveApiKeysToSecret, } from '../credentials/api-keys' import { normalizeEnabledModelValues } from '../credentials/model-picker' +import { + DEFAULT_REALTIME_SETTINGS, + normalizeRealtimeSettings, +} from './realtime' import { normalizeServer, normalizeServers } from './servers' export { settingsPath } from '../shared/paths' -export const SETTINGS_VERSION = 2 +export const SETTINGS_VERSION = 3 export const DEFAULT_SETTINGS: DesktopSettings = { servers: [], @@ -31,6 +35,7 @@ export const DEFAULT_SETTINGS: DesktopSettings = { launchAtLogin: false, preventAppSuspension: true, codex: { enabled: false, source: null }, + realtime: DEFAULT_REALTIME_SETTINGS, } export function normalizeCodexSettings(value: unknown): CodexSettings { @@ -165,6 +170,7 @@ export async function loadDesktopSettings( preventAppSuspension: parsed.preventAppSuspension !== false, onboardingDismissed: parsed.onboardingDismissed === true, codex: normalizeCodexSettings(parsed.codex), + realtime: normalizeRealtimeSettings(parsed.realtime), enabledModelValues: enabledModelValues.length > 0 ? enabledModelValues : undefined, mcp: normalizeMcp(parsed.mcp), diff --git a/packages/agents-desktop/src/shared/types.ts b/packages/agents-desktop/src/shared/types.ts index efce0dd4b4..1f506ddb2f 100644 --- a/packages/agents-desktop/src/shared/types.ts +++ b/packages/agents-desktop/src/shared/types.ts @@ -122,6 +122,27 @@ export type CodexSettings = { source: CodexAuthSource | null } +export type RealtimeProvider = `openai` + +export type RealtimeSettings = { + provider: RealtimeProvider + model: string +} + +export type RealtimeModelChoice = { + id: string + label: string + description: string + recommended?: boolean +} + +export type RealtimeSettingsStatus = { + settings: RealtimeSettings + availableModels: Array + hasOpenAIApiKey: boolean + codexEnabled: boolean +} + export type DesktopSettings = { servers: Array defaultServerId: string | null @@ -131,6 +152,7 @@ export type DesktopSettings = { preventAppSuspension?: boolean codex?: CodexSettings enabledModelValues?: Array + realtime?: RealtimeSettings onboardingDismissed?: boolean mcp?: { servers: Array } seededDefaultMcpServerNames?: Array diff --git a/packages/agents-server-ui/src/components/settings/SettingsSidebar.tsx b/packages/agents-server-ui/src/components/settings/SettingsSidebar.tsx index adbb6d5c95..30019938fa 100644 --- a/packages/agents-server-ui/src/components/settings/SettingsSidebar.tsx +++ b/packages/agents-server-ui/src/components/settings/SettingsSidebar.tsx @@ -6,6 +6,7 @@ import { KeyRound, Palette, Plug, + RadioTower, Server, Settings as SettingsIcon, Terminal, @@ -21,6 +22,7 @@ export type SettingsCategoryId = | `account` | `servers` | `credentials` + | `realtime` | `command-line` | `appearance` | `local-runtime` @@ -105,6 +107,12 @@ export function SettingsSidebar({ icon: , visible: true, }, + { + id: `realtime`, + label: `Realtime`, + icon: , + visible: true, + }, { id: `command-line`, label: `Command Line`, diff --git a/packages/agents-server-ui/src/components/settings/pages/RealtimePage.module.css b/packages/agents-server-ui/src/components/settings/pages/RealtimePage.module.css new file mode 100644 index 0000000000..f7681ab603 --- /dev/null +++ b/packages/agents-server-ui/src/components/settings/pages/RealtimePage.module.css @@ -0,0 +1,61 @@ +.modelSelect { + min-width: 240px; +} + +.modelList { + display: flex; + flex-direction: column; + gap: 0; +} + +.modelItem { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 16px; + padding: 12px 0; + border-top: 1px solid var(--ds-border-1); +} + +.modelItem:first-child { + padding-top: 0; + border-top: 0; +} + +.modelItem:last-child { + padding-bottom: 0; +} + +.modelText { + min-width: 0; + display: flex; + flex-direction: column; + gap: 4px; +} + +.modelTitle { + display: inline-flex; + align-items: center; + gap: 6px; + min-width: 0; + color: var(--ds-text-1); + font-size: var(--ds-text-sm); +} + +.modelId { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: var(--ds-text-3); + font-size: var(--ds-text-xs); +} + +.modelDescription { + color: var(--ds-text-3); + font-size: var(--ds-text-xs); + line-height: 1.45; +} + +.recommended { + flex-shrink: 0; +} diff --git a/packages/agents-server-ui/src/components/settings/pages/RealtimePage.tsx b/packages/agents-server-ui/src/components/settings/pages/RealtimePage.tsx new file mode 100644 index 0000000000..bbb491c562 --- /dev/null +++ b/packages/agents-server-ui/src/components/settings/pages/RealtimePage.tsx @@ -0,0 +1,215 @@ +import { useEffect, useMemo, useState } from 'react' +import { useNavigate } from '@tanstack/react-router' +import { + loadRealtimeSettingsStatus, + saveRealtimeSettings, + type RealtimeSettingsStatus, +} from '../../../lib/server-connection' +import { Button, Select, Text } from '../../../ui' +import { + SettingsPanel, + SettingsRow, + SettingsScreen, + SettingsSection, + SettingsStatusBadge, +} from '../SettingsScreen' +import styles from './RealtimePage.module.css' + +export function RealtimePage(): React.ReactElement { + const isDesktop = typeof window !== `undefined` && Boolean(window.electronAPI) + const navigate = useNavigate() + const [status, setStatus] = useState(null) + const [saving, setSaving] = useState(false) + const [error, setError] = useState(null) + + useEffect(() => { + let cancelled = false + void loadRealtimeSettingsStatus().then((next) => { + if (cancelled) return + setStatus(next) + }) + return () => { + cancelled = true + } + }, []) + + const modelById = useMemo( + () => new Map(status?.availableModels.map((model) => [model.id, model])), + [status?.availableModels] + ) + const selectedModel = status ? modelById.get(status.settings.model) : null + + const saveModel = async (model: string | null): Promise => { + if (!model || !status) return + const next = { + ...status, + settings: { ...status.settings, model }, + } + setStatus(next) + setSaving(true) + setError(null) + try { + await saveRealtimeSettings(next.settings) + } catch (err) { + setStatus(status) + setError(err instanceof Error ? err.message : String(err)) + } finally { + setSaving(false) + } + } + + return ( + + + {!isDesktop ? ( + + + Realtime settings are managed by the connected desktop or server + runtime. This web build uses the default model when starting a + session from the browser. + + + ) : !status ? ( + + + Loading… + + + ) : ( + <> + + + {status.hasOpenAIApiKey ? `Ready` : `API key required`} + + + + } + /> + OpenAI + } + /> + { + void saveModel(model) + }} + disabled={saving} + > + + model ? (modelById.get(model)?.label ?? model) : `Model` + } + /> + + {status.availableModels.map((model) => ( + + {model.label} + + ))} + + + } + /> + {saving && ( + + + Saving… + + + )} + {error && ( + + + {error} + + + )} + + )} + + + {status && ( + + +
+ {status.availableModels.map((model) => ( +
+
+ + {model.label} + {model.recommended && ( + + Recommended + + )} + + {model.id} + + {model.description} + +
+ {model.id === status.settings.model && ( + + + Selected + + + )} +
+ ))} +
+ + + )} + + ) +} + +function authDescription(status: RealtimeSettingsStatus): string { + if (status.hasOpenAIApiKey) { + return `Realtime sessions connect to the OpenAI Realtime API with your OpenAI API key.` + } + if (status.codexEnabled) { + return `ChatGPT / Codex sign-in is enabled, but realtime voice still needs an OpenAI API key.` + } + return `Add an OpenAI API key in Credentials. ChatGPT / Codex sign-in alone does not grant Realtime API access.` +} diff --git a/packages/agents-server-ui/src/hooks/useDocumentTitle.ts b/packages/agents-server-ui/src/hooks/useDocumentTitle.ts index 5477f47c64..2cd6747319 100644 --- a/packages/agents-server-ui/src/hooks/useDocumentTitle.ts +++ b/packages/agents-server-ui/src/hooks/useDocumentTitle.ts @@ -10,6 +10,7 @@ const APP_NAME = `Electric Agents` const SETTINGS_CATEGORY_LABELS: Record = { general: `General`, appearance: `Appearance`, + realtime: `Realtime`, 'local-runtime': `Local Runtime`, } diff --git a/packages/agents-server-ui/src/lib/realtime-audio.ts b/packages/agents-server-ui/src/lib/realtime-audio.ts index 155387948a..521aff0473 100644 --- a/packages/agents-server-ui/src/lib/realtime-audio.ts +++ b/packages/agents-server-ui/src/lib/realtime-audio.ts @@ -1,6 +1,7 @@ import { DurableStream } from '@durable-streams/client' import { appendPathToUrl } from '@electric-ax/agents-runtime/client' import { serverFetch, getConfiguredServerHeaders } from './auth-fetch' +import { loadRealtimeSettingsStatus } from './server-connection' export type RealtimeAudioSession = { sessionId: string @@ -327,13 +328,14 @@ async function createRealtimeSession( baseUrl: string, entityUrl: string ): Promise { + const realtimeSettings = await loadRealtimeSettingsStatus() const response = await serverFetch(realtimeUrl(baseUrl), { method: `POST`, headers: { 'content-type': `application/json` }, body: JSON.stringify({ entityUrl, provider: `openai`, - model: `gpt-realtime-2`, + model: realtimeSettings.settings.model, inputAudio: { codec: `pcm16`, sampleRate: REALTIME_SAMPLE_RATE, diff --git a/packages/agents-server-ui/src/lib/server-connection.ts b/packages/agents-server-ui/src/lib/server-connection.ts index 865fab9b57..8b3f53af80 100644 --- a/packages/agents-server-ui/src/lib/server-connection.ts +++ b/packages/agents-server-ui/src/lib/server-connection.ts @@ -173,6 +173,49 @@ export interface ApiKeysStatus { modelPicker: ModelPickerStatus } +export type RealtimeSettings = { + provider: `openai` + model: string +} + +export type RealtimeModelChoice = { + id: string + label: string + description: string + recommended?: boolean +} + +export type RealtimeSettingsStatus = { + settings: RealtimeSettings + availableModels: Array + hasOpenAIApiKey: boolean + codexEnabled: boolean +} + +const DEFAULT_REALTIME_SETTINGS_STATUS: RealtimeSettingsStatus = { + settings: { provider: `openai`, model: `gpt-realtime-2` }, + availableModels: [ + { + id: `gpt-realtime-2`, + label: `GPT-Realtime-2`, + description: `Strongest realtime reasoning, tool use, and instruction following.`, + recommended: true, + }, + { + id: `gpt-realtime-1.5`, + label: `GPT-Realtime-1.5`, + description: `Fast, reliable speech-to-speech model for audio in, audio out.`, + }, + { + id: `gpt-realtime-mini`, + label: `GPT-Realtime mini`, + description: `Cost-efficient realtime voice model.`, + }, + ], + hasOpenAIApiKey: false, + codexEnabled: false, +} + /** * Snapshot consumed by the renderer's onboarding wizard. * @@ -376,6 +419,8 @@ declare global { setOnboardingDismissed?: (dismissed: boolean) => Promise getPreventAppSuspension?: () => Promise setPreventAppSuspension?: (enabled: boolean) => Promise + getRealtimeSettings?: () => Promise + setRealtimeSettings?: (settings: RealtimeSettings) => Promise getWorkingDirectory?: () => Promise chooseWorkingDirectory?: () => Promise /** @@ -591,6 +636,19 @@ export async function saveEnabledModels(values: Array): Promise { await window.electronAPI?.saveEnabledModels?.(values) } +export async function loadRealtimeSettingsStatus(): Promise { + return ( + (await window.electronAPI?.getRealtimeSettings?.()) ?? + DEFAULT_REALTIME_SETTINGS_STATUS + ) +} + +export async function saveRealtimeSettings( + settings: RealtimeSettings +): Promise { + await window.electronAPI?.setRealtimeSettings?.(settings) +} + export async function codexSignIn(): Promise { return (await window.electronAPI?.codexSignIn?.()) ?? null } diff --git a/packages/agents-server-ui/src/router.tsx b/packages/agents-server-ui/src/router.tsx index 64956671d5..449bf860cc 100644 --- a/packages/agents-server-ui/src/router.tsx +++ b/packages/agents-server-ui/src/router.tsx @@ -53,6 +53,7 @@ import { GeneralPage } from './components/settings/pages/GeneralPage' import { AccountPage } from './components/settings/pages/AccountPage' import { AppearancePage } from './components/settings/pages/AppearancePage' import { CredentialsPage } from './components/settings/pages/CredentialsPage' +import { RealtimePage } from './components/settings/pages/RealtimePage' import { CommandLinePage } from './components/settings/pages/CommandLinePage' import { ServersPage } from './components/settings/pages/ServersPage' import { McpServersPage } from './components/settings/pages/McpServersPage' @@ -64,6 +65,7 @@ const SETTINGS_CATEGORY_IDS: ReadonlyArray = [ `account`, `servers`, `credentials`, + `realtime`, `command-line`, `appearance`, `local-runtime`, @@ -564,6 +566,8 @@ function SettingsCategoryPage(): React.ReactElement { return case `credentials`: return + case `realtime`: + return case `command-line`: return case `local-runtime`: From e5f2220c0398ea3f52c28fac341a10f8f24c9948 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 10 Jun 2026 10:42:45 +0100 Subject: [PATCH 35/39] fix(agents): gate realtime controls on credentials --- .../src/components/MessageInput.tsx | 57 ++++++++++--- .../src/components/views/NewSessionView.tsx | 15 +++- .../src/hooks/useRealtimeAvailability.ts | 82 +++++++++++++++++++ 3 files changed, 139 insertions(+), 15 deletions(-) create mode 100644 packages/agents-server-ui/src/hooks/useRealtimeAvailability.ts diff --git a/packages/agents-server-ui/src/components/MessageInput.tsx b/packages/agents-server-ui/src/components/MessageInput.tsx index e9245475ee..1394f89737 100644 --- a/packages/agents-server-ui/src/components/MessageInput.tsx +++ b/packages/agents-server-ui/src/components/MessageInput.tsx @@ -22,6 +22,7 @@ import { startRealtimeAudioSession, type RealtimeAudioSession, } from '../lib/realtime-audio' +import { useRealtimeAvailability } from '../hooks/useRealtimeAvailability' import { ComposerEditor } from './ComposerEditor' import { ComposerShell } from './ComposerShell' import { Icon, Stack, Text, Tooltip } from '../ui' @@ -131,6 +132,7 @@ export function MessageInput({ const realtimeSessionRef = useRef(null) const handledAutoStartRealtimeRef = useRef(null) const composerFocusRef = useRef<{ focus: () => void } | null>(null) + const realtimeAvailability = useRealtimeAvailability() const inputDisabled = disabled || writeDisabled const isCommentMode = composerMode === `comment` const attachmentsDisabled = @@ -241,8 +243,12 @@ export function MessageInput({ attachmentCount === 0 && !disabled const canStop = showStop && !stopPending && !stopDisabled - const canUseRealtime = - !inputDisabled && !editingMessage && !isCommentMode && Boolean(baseUrl) + const canStartRealtime = + !inputDisabled && + !editingMessage && + !isCommentMode && + Boolean(baseUrl) && + realtimeAvailability.canStart useEffect(() => { return () => { @@ -368,7 +374,12 @@ export function MessageInput({ }) return } - if (!canUseRealtime) return + if (!canStartRealtime) { + if (realtimeAvailability.unavailableReason) { + setError(realtimeAvailability.unavailableReason) + } + return + } setRealtimePending(true) startRealtimeAudioSession({ baseUrl, @@ -386,12 +397,27 @@ export function MessageInput({ .finally(() => { setRealtimePending(false) }) - }, [baseUrl, canUseRealtime, entityUrl, realtimePending]) + }, [ + baseUrl, + canStartRealtime, + entityUrl, + realtimeAvailability.unavailableReason, + realtimePending, + ]) useEffect(() => { if (!autoStartRealtimeSignal) return if (handledAutoStartRealtimeRef.current === autoStartRealtimeSignal) return - if (!canUseRealtime || realtimePending) return + if (realtimeAvailability.loading || realtimePending) return + if (!realtimeAvailability.canStart) { + handledAutoStartRealtimeRef.current = autoStartRealtimeSignal + onRealtimeAutoStartConsumed?.() + if (realtimeAvailability.unavailableReason) { + setError(realtimeAvailability.unavailableReason) + } + return + } + if (!canStartRealtime) return handledAutoStartRealtimeRef.current = autoStartRealtimeSignal onRealtimeAutoStartConsumed?.() if (!realtimeSessionRef.current) { @@ -399,9 +425,12 @@ export function MessageInput({ } }, [ autoStartRealtimeSignal, - canUseRealtime, + canStartRealtime, handleRealtimeToggle, onRealtimeAutoStartConsumed, + realtimeAvailability.canStart, + realtimeAvailability.loading, + realtimeAvailability.unavailableReason, realtimePending, ]) @@ -496,6 +525,13 @@ export function MessageInput({ : `Send message` const replyPreviewLabel = formatReplyBannerLabel(commentTarget) const replyPreviewText = commentTarget?.snapshot.text + const realtimeTooltip = realtimeActive + ? `Stop voice mode` + : realtimeAvailability.loading + ? `Checking realtime credentials` + : (realtimeAvailability.unavailableReason ?? `Start voice mode`) + const realtimeButtonDisabled = + realtimePending || (!realtimeActive && !canStartRealtime) return ( {drawer?.({ @@ -568,12 +604,7 @@ export function MessageInput({ <> {!isCommentMode ? ( <> - +