From bab1a62b5e91c69c03c5d4503e98ad2760e947f2 Mon Sep 17 00:00:00 2001 From: Sora Morimoto Date: Thu, 9 Jul 2026 03:35:00 +0900 Subject: [PATCH 01/13] feat(aws): add AWS plugin (Bedrock LLM, Transcribe STT, Polly TTS) Add @livekit/agents-plugin-aws, providing: - LLM via Amazon Bedrock Converse (streaming, tool calling, prompt caching) - STT via Amazon Transcribe streaming (word-level timestamps, channel identification, idle-timeout reconnects) - TTS via Amazon Polly (PCM output, since the framework has no mp3 decoder) Also adds an 'aws' provider format to @livekit/agents for converting ChatContext into Bedrock Converse messages. Nova Sonic realtime is out of scope for this change. --- .changeset/aws-plugin.md | 6 + README.md | 16 +- agents/src/llm/provider_format/aws.test.ts | 526 ++++++++++++++++++++ agents/src/llm/provider_format/aws.ts | 167 +++++++ agents/src/llm/provider_format/index.ts | 5 +- examples/package.json | 1 + examples/src/aws.ts | 68 +++ plugins/aws/README.md | 91 ++++ plugins/aws/api-extractor.json | 20 + plugins/aws/package.json | 55 +++ plugins/aws/src/index.ts | 22 + plugins/aws/src/llm.test.ts | 249 ++++++++++ plugins/aws/src/llm.ts | 414 ++++++++++++++++ plugins/aws/src/models.ts | 56 +++ plugins/aws/src/stt.test.ts | 530 +++++++++++++++++++++ plugins/aws/src/stt.ts | 383 +++++++++++++++ plugins/aws/src/tts.test.ts | 103 ++++ plugins/aws/src/tts.ts | 197 ++++++++ plugins/aws/src/utils.ts | 25 + plugins/aws/tsconfig.json | 19 + plugins/aws/tsup.config.ts | 7 + pnpm-lock.yaml | 398 ++++++++++++++++ turbo.json | 7 + 23 files changed, 3361 insertions(+), 4 deletions(-) create mode 100644 .changeset/aws-plugin.md create mode 100644 agents/src/llm/provider_format/aws.test.ts create mode 100644 agents/src/llm/provider_format/aws.ts create mode 100644 examples/src/aws.ts create mode 100644 plugins/aws/README.md create mode 100644 plugins/aws/api-extractor.json create mode 100644 plugins/aws/package.json create mode 100644 plugins/aws/src/index.ts create mode 100644 plugins/aws/src/llm.test.ts create mode 100644 plugins/aws/src/llm.ts create mode 100644 plugins/aws/src/models.ts create mode 100644 plugins/aws/src/stt.test.ts create mode 100644 plugins/aws/src/stt.ts create mode 100644 plugins/aws/src/tts.test.ts create mode 100644 plugins/aws/src/tts.ts create mode 100644 plugins/aws/src/utils.ts create mode 100644 plugins/aws/tsconfig.json create mode 100644 plugins/aws/tsup.config.ts diff --git a/.changeset/aws-plugin.md b/.changeset/aws-plugin.md new file mode 100644 index 000000000..05779a8d4 --- /dev/null +++ b/.changeset/aws-plugin.md @@ -0,0 +1,6 @@ +--- +'@livekit/agents-plugin-aws': minor +'@livekit/agents': minor +--- + +Add the AWS plugin (`@livekit/agents-plugin-aws`), providing LLM (Amazon Bedrock Converse), STT (Amazon Transcribe streaming), and TTS (Amazon Polly). Adds an `'aws'` provider format to `@livekit/agents` for converting `ChatContext` to Bedrock Converse messages. diff --git a/README.md b/README.md index e708dcf6e..826f5c475 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ Currently, only the following plugins are supported: | [@livekit/agents-plugin-phonic](https://www.npmjs.com/package/@livekit/agents-plugin-phonic) | Realtime | | [@livekit/agents-plugin-fishaudio](https://www.npmjs.com/package/@livekit/agents-plugin-fishaudio) | TTS | | [@livekit/agents-plugin-hume](https://www.npmjs.com/package/@livekit/agents-plugin-hume) | TTS | +| [@livekit/agents-plugin-aws](https://www.npmjs.com/package/@livekit/agents-plugin-aws) | LLM, STT, TTS | ## Docs and guides @@ -117,9 +118,9 @@ import { WorkerOptions, cli, defineAgent, + inference, llm, voice, - inference, } from '@livekit/agents'; import * as silero from '@livekit/agents-plugin-silero'; import { fileURLToPath } from 'node:url'; @@ -154,7 +155,10 @@ export default defineAgent({ llm: new inference.LLM({ model: 'openai/gpt-4.1-mini' }), // Text-to-speech (TTS) is your agent's voice, turning the LLM's text into speech that the user can hear // See all available models as well as voice selections at https://docs.livekit.io/agents/models/tts/ - tts: new inference.TTS({ model: 'cartesia/sonic-3', voice: '9626c31c-bec5-4cca-baa8-f8ba9e84c8bc' }), + tts: new inference.TTS({ + model: 'cartesia/sonic-3', + voice: '9626c31c-bec5-4cca-baa8-f8ba9e84c8bc', + }), // VAD and turn detection are used to determine when the user is speaking and when the agent should respond // See more at https://docs.livekit.io/agents/build/turns vad: ctx.proc.userData.vad! as silero.VAD, @@ -253,7 +257,10 @@ export default defineAgent({ vad: ctx.proc.userData.vad! as silero.VAD, stt: new inference.STT({ model: 'deepgram/nova-3', language: 'en' }), llm: new inference.LLM({ model: 'openai/gpt-4.1-mini' }), - tts: new inference.TTS({ model: 'cartesia/sonic-3', voice: '9626c31c-bec5-4cca-baa8-f8ba9e84c8bc' }), + tts: new inference.TTS({ + model: 'cartesia/sonic-3', + voice: '9626c31c-bec5-4cca-baa8-f8ba9e84c8bc', + }), userData: userdata, }); @@ -343,6 +350,7 @@ To contribute to this project: To test any changes or plugins: 1. Build the project: + ```bash pnpm build ``` @@ -371,7 +379,9 @@ Refer to [the license](LICENSES/Apache-2.0.txt) for details. The LiveKit turn detection models are licensed under the [LiveKit Model License](MODEL_LICENSE). +
+ diff --git a/agents/src/llm/provider_format/aws.test.ts b/agents/src/llm/provider_format/aws.test.ts new file mode 100644 index 000000000..59fa3bf8a --- /dev/null +++ b/agents/src/llm/provider_format/aws.test.ts @@ -0,0 +1,526 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { initializeLogger } from '../../log.js'; +import { + AgentHandoffItem, + ChatContext, + FunctionCall, + FunctionCallOutput, + Instructions, +} from '../chat_context.js'; +import { serializeImage } from '../utils.js'; +import { toChatCtx } from './aws.js'; + +vi.mock('../utils.js', () => ({ + serializeImage: vi.fn(), +})); + +describe('AWS Provider Format - toChatCtx', () => { + const serializeImageMock = vi.mocked(serializeImage); + + initializeLogger({ level: 'silent', pretty: false }); + + beforeEach(async () => { + vi.clearAllMocks(); + }); + + it('should convert simple text messages', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'Hello' }); + ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { role: 'user', content: [{ text: 'Hello' }] }, + { role: 'assistant', content: [{ text: 'Hi there!' }] }, + ]); + expect(formatData.systemMessages).toBeNull(); + }); + + it('should extract system messages separately', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'system', content: 'You are a helpful assistant' }); + ctx.addMessage({ role: 'user', content: 'Hello' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([{ role: 'user', content: [{ text: 'Hello' }] }]); + expect(formatData.systemMessages).toEqual(['You are a helpful assistant']); + }); + + it('should extract developer messages as system messages', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'developer', content: 'Be concise' }); + ctx.addMessage({ role: 'user', content: 'Hello' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([{ role: 'user', content: [{ text: 'Hello' }] }]); + expect(formatData.systemMessages).toEqual(['Be concise']); + }); + + it('should drop a system message with no text content instead of reattributing it to user', async () => { + serializeImageMock.mockResolvedValue({ + inferenceDetail: 'auto', + mimeType: 'image/png', + base64Data: 'aW1n', + }); + + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'system', + content: [ + { + id: 'img1', + type: 'image_content', + image: 'data:image/png;base64,aW1n', + inferenceDetail: 'auto', + _cache: {}, + }, + ], + }); + ctx.addMessage({ role: 'user', content: 'Hello' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + // The image-only system message must not surface as a 'user' turn. + expect(result).toEqual([{ role: 'user', content: [{ text: 'Hello' }] }]); + expect(formatData.systemMessages).toBeNull(); + }); + + it('should preserve empty string text content instead of dropping the whole message', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: '' }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([{ role: 'user', content: [{ text: '' }] }]); + }); + + it('should render Instructions as their resolved value', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'system', + content: [ + new Instructions({ audio: 'audio instructions', text: 'text instructions' }).asModality( + 'text', + ), + ], + }); + ctx.addMessage({ role: 'user', content: 'Hello' }); + + const [, formatData] = await toChatCtx(ctx, false); + + expect(formatData.systemMessages).toEqual(['text instructions']); + }); + + it('should concatenate multiple system messages as separate entries', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'system', content: 'You are a helpful assistant' }); + ctx.addMessage({ role: 'system', content: 'Be concise in your responses' }); + ctx.addMessage({ role: 'user', content: 'Hello' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'user', + content: [ + { + text: 'New instructions received. Apply them carefully: Be concise in your responses', + }, + { text: 'Hello' }, + ], + }, + ]); + expect(formatData.systemMessages).toEqual(['You are a helpful assistant']); + }); + + it('should merge consecutive messages with the same effective role', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'First user message' }); + ctx.addMessage({ role: 'user', content: 'Second user message' }); + ctx.addMessage({ role: 'assistant', content: 'First assistant response' }); + ctx.addMessage({ role: 'assistant', content: 'Second assistant response' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'user', + content: [{ text: 'First user message' }, { text: 'Second user message' }], + }, + { + role: 'assistant', + content: [{ text: 'First assistant response' }, { text: 'Second assistant response' }], + }, + ]); + expect(formatData.systemMessages).toBeNull(); + }); + + it('should map function calls to assistant toolUse blocks and outputs to user toolResult blocks', async () => { + const ctx = ChatContext.empty(); + + const msg = ctx.addMessage({ role: 'assistant', content: 'Let me check.' }); + const toolCall = FunctionCall.create({ + id: msg.id + '/tool_1', + callId: 'call_123', + name: 'get_weather', + args: '{"location": "San Francisco"}', + }); + const toolOutput = FunctionCallOutput.create({ + callId: 'call_123', + name: 'get_weather', + output: '{"temperature": 72, "condition": "sunny"}', + isError: false, + }); + + ctx.insert([toolCall, toolOutput]); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'assistant', + content: [ + { text: 'Let me check.' }, + { + toolUse: { + toolUseId: 'call_123', + name: 'get_weather', + input: { location: 'San Francisco' }, + }, + }, + ], + }, + { + role: 'user', + content: [ + { + toolResult: { + toolUseId: 'call_123', + content: [{ text: '{"temperature": 72, "condition": "sunny"}' }], + status: 'success', + }, + }, + ], + }, + ]); + expect(formatData.systemMessages).toBeNull(); + }); + + it('should map a failed tool call output to status "error"', async () => { + const ctx = ChatContext.empty(); + + const toolCall = new FunctionCall({ + id: 'func_error', + callId: 'call_error', + name: 'failing_function', + args: '{}', + }); + const toolOutput = new FunctionCallOutput({ + callId: 'call_error', + name: 'failing_function', + output: 'Function failed to execute', + isError: true, + }); + + ctx.insert([toolCall, toolOutput]); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'assistant', + content: [ + { + toolUse: { + toolUseId: 'call_error', + name: 'failing_function', + input: {}, + }, + }, + ], + }, + { + role: 'user', + content: [ + { + toolResult: { + toolUseId: 'call_error', + content: [{ text: 'Function failed to execute' }], + status: 'error', + }, + }, + ], + }, + ]); + }); + + it('should inject a dummy user message when the conversation does not start with user', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); + + const [result, formatData] = await toChatCtx(ctx, true); + + expect(result).toEqual([ + { role: 'user', content: [{ text: '(empty)' }] }, + { role: 'assistant', content: [{ text: 'Hi there!' }] }, + { role: 'user', content: [{ text: '.' }] }, + ]); + expect(formatData.systemMessages).toBeNull(); + }); + + it('should not inject a leading dummy user message when the conversation already starts with user', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'Hello' }); + ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); + + const [result] = await toChatCtx(ctx, true); + + // Still gets a trailing dummy user turn since it ends on 'assistant'. + expect(result).toEqual([ + { role: 'user', content: [{ text: 'Hello' }] }, + { role: 'assistant', content: [{ text: 'Hi there!' }] }, + { role: 'user', content: [{ text: '.' }] }, + ]); + }); + + it('should inject a trailing dummy user message when the conversation ends on assistant', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'Hello' }); + ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); + + const [result] = await toChatCtx(ctx, true); + + expect(result.at(-1)).toEqual({ role: 'user', content: [{ text: '.' }] }); + }); + + it('should not inject a trailing dummy user message when the conversation already ends on user', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'Hello' }); + ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); + ctx.addMessage({ role: 'user', content: 'Follow-up' }); + + const [result] = await toChatCtx(ctx, true); + + expect(result).toEqual([ + { role: 'user', content: [{ text: 'Hello' }] }, + { role: 'assistant', content: [{ text: 'Hi there!' }] }, + { role: 'user', content: [{ text: 'Follow-up' }] }, + ]); + }); + + it('should not inject a trailing dummy user message when disabled', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'Hello' }); + ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { role: 'user', content: [{ text: 'Hello' }] }, + { role: 'assistant', content: [{ text: 'Hi there!' }] }, + ]); + }); + + it('should not inject a dummy user message when disabled', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'system', content: 'You are helpful' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([]); + expect(formatData.systemMessages).toEqual(['You are helpful']); + }); + + it('should inject a dummy user message for an entirely empty conversation', async () => { + const ctx = ChatContext.empty(); + + const [result] = await toChatCtx(ctx, true); + + expect(result).toEqual([{ role: 'user', content: [{ text: '(empty)' }] }]); + }); + + it('should convert supported inline images to Bedrock image blocks', async () => { + serializeImageMock.mockResolvedValue({ + inferenceDetail: 'auto', + mimeType: 'image/png', + base64Data: Buffer.from('fake-png-bytes').toString('base64'), + }); + + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'user', + content: [ + 'Look at this:', + { + id: 'img1', + type: 'image_content', + image: 'data:image/png;base64,ZmFrZS1wbmctYnl0ZXM=', + inferenceDetail: 'auto', + _cache: {}, + }, + ], + }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'user', + content: [ + { text: 'Look at this:' }, + { + image: { + format: 'png', + source: { bytes: Buffer.from('fake-png-bytes') }, + }, + }, + ], + }, + ]); + }); + + it('should default to jpeg format when mimeType is missing', async () => { + serializeImageMock.mockResolvedValue({ + inferenceDetail: 'auto', + base64Data: Buffer.from('raw-bytes').toString('base64'), + }); + + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'user', + content: [ + { + id: 'img1', + type: 'image_content', + image: 'data:image/jpeg;base64,cmF3LWJ5dGVz', + inferenceDetail: 'auto', + _cache: {}, + }, + ], + }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'user', + content: [ + { + image: { + format: 'jpeg', + source: { bytes: Buffer.from('raw-bytes') }, + }, + }, + ], + }, + ]); + }); + + it('should throw for unsupported image mime types', async () => { + serializeImageMock.mockResolvedValue({ + inferenceDetail: 'auto', + mimeType: 'image/bmp', + base64Data: Buffer.from('bmp-bytes').toString('base64'), + }); + + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'user', + content: [ + { + id: 'img1', + type: 'image_content', + image: 'data:image/bmp;base64,Ym1wLWJ5dGVz', + inferenceDetail: 'auto', + _cache: {}, + }, + ], + }); + + await expect(toChatCtx(ctx, false)).rejects.toThrow(/Unsupported mimeType/); + }); + + it('should throw for externalUrl images', async () => { + serializeImageMock.mockResolvedValue({ + inferenceDetail: 'high', + externalUrl: 'https://example.com/image.jpg', + mimeType: 'image/jpeg', + }); + + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'user', + content: [ + { + id: 'img1', + type: 'image_content', + image: 'https://example.com/image.jpg', + inferenceDetail: 'high', + _cache: {}, + }, + ], + }); + + await expect(toChatCtx(ctx, false)).rejects.toThrow(/externalUrl images are not supported/); + }); + + it('should filter out agent handoff items', async () => { + const ctx = ChatContext.empty(); + + ctx.addMessage({ role: 'user', content: 'Hello' }); + ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_1', newAgentId: 'agent_2' })); + ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { role: 'user', content: [{ text: 'Hello' }] }, + { role: 'assistant', content: [{ text: 'Hi there!' }] }, + ]); + }); + + it('should filter out standalone function calls without outputs', async () => { + const ctx = ChatContext.empty(); + + const funcCall = new FunctionCall({ + id: 'func_standalone', + callId: 'call_999', + name: 'standalone_function', + args: '{}', + }); + + ctx.insert(funcCall); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([]); + }); + + it('should skip empty groups', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'Hello', createdAt: 1000 }); + + const orphanOutput = new FunctionCallOutput({ + callId: 'orphan_call', + output: 'This should be ignored', + isError: false, + createdAt: 2000, + }); + ctx.insert(orphanOutput); + + ctx.addMessage({ role: 'assistant', content: 'Hi!', createdAt: 3000 }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { role: 'user', content: [{ text: 'Hello' }] }, + { role: 'assistant', content: [{ text: 'Hi!' }] }, + ]); + }); +}); diff --git a/agents/src/llm/provider_format/aws.ts b/agents/src/llm/provider_format/aws.ts new file mode 100644 index 000000000..5e8c5bf14 --- /dev/null +++ b/agents/src/llm/provider_format/aws.ts @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { ChatContext, ChatItem, ImageContent } from '../chat_context.js'; +import { isInstructions } from '../chat_context.js'; +import { type SerializedImage, serializeImage } from '../utils.js'; +import { convertMidConversationInstructions, groupToolCalls } from './utils.js'; + +export interface AwsFormatData { + systemMessages: string[] | null; +} + +const AWS_IMAGE_FORMATS: Record = { + 'image/jpeg': 'jpeg', + 'image/png': 'png', + 'image/gif': 'gif', + 'image/webp': 'webp', +}; + +/** + * Convert a LiveKit ChatContext into AWS Bedrock Converse API message entries. + * + * - System messages are extracted separately (Bedrock takes `system` outside `messages`). + * - `function_call` items become assistant `toolUse` content blocks. + * - `function_call_output` items become user `toolResult` content blocks. + * - Consecutive same-role turns are merged (Bedrock requires strictly alternating roles). + * - A dummy user message is inserted at the front if the conversation doesn't start with one. + */ +export async function toChatCtx( + chatCtx: ChatContext, + injectDummyUserMessage: boolean = true, +): Promise<[Record[], AwsFormatData]> { + chatCtx = convertMidConversationInstructions(chatCtx); + + const messages: Record[] = []; + const systemMessages: string[] = []; + let currentRole: string | null = null; + let content: Record[] = []; + + const itemGroups = groupToolCalls(chatCtx); + const flattenedItems: ChatItem[] = []; + for (const group of itemGroups) { + flattenedItems.push(...group.flatten()); + } + + for (const msg of flattenedItems) { + if (msg.type === 'message' && (msg.role === 'system' || msg.role === 'developer')) { + // Always exclude system/developer messages from the regular role mapping below, even + // when they carry no usable text (e.g. image-only content) — they must never be + // reattributed to the user/assistant turn merge. + if (msg.textContent) { + systemMessages.push(msg.textContent); + } + continue; + } + + let role: string; + if (msg.type === 'message') { + role = msg.role === 'assistant' ? 'assistant' : 'user'; + } else if (msg.type === 'function_call') { + role = 'assistant'; + } else if (msg.type === 'function_call_output') { + role = 'user'; + } else { + continue; // Skip unknown message types (e.g. agent_handoff, agent_config_update) + } + + // If the effective role changed, finalize the previous turn + if (role !== currentRole) { + if (currentRole !== null && content.length > 0) { + messages.push({ role: currentRole, content: [...content] }); + } + content = []; + currentRole = role; + } + + if (msg.type === 'message') { + for (const part of msg.content) { + if (typeof part === 'string') { + content.push({ text: part }); + } else if (isInstructions(part)) { + content.push({ text: part.value }); + } else if (part && typeof part === 'object' && part.type === 'image_content') { + content.push(await toImagePart(part)); + } + // audio_content is intentionally skipped — Bedrock Converse has no raw audio block + } + } else if (msg.type === 'function_call') { + content.push({ + toolUse: { + toolUseId: msg.callId, + name: msg.name, + input: JSON.parse(msg.args || '{}'), + }, + }); + } else if (msg.type === 'function_call_output') { + content.push({ + toolResult: { + toolUseId: msg.callId, + content: [{ text: msg.output }], + status: msg.isError ? 'error' : 'success', + }, + }); + } + } + + // Finalize the last turn + if (currentRole !== null && content.length > 0) { + messages.push({ role: currentRole, content }); + } + + // Bedrock requires the message list to start with a "user" turn + if (injectDummyUserMessage && (messages.length === 0 || messages[0]!.role !== 'user')) { + messages.unshift({ role: 'user', content: [{ text: '(empty)' }] }); + } + + // Some Bedrock-hosted models (e.g. Anthropic Claude) reject a request ending on an + // assistant turn, or silently treat it as a prefill continuation instead of generating a + // fresh reply. + if ( + injectDummyUserMessage && + messages.length > 0 && + messages[messages.length - 1]!.role === 'assistant' + ) { + messages.push({ role: 'user', content: [{ text: '.' }] }); + } + + return [ + messages, + { + systemMessages: systemMessages.length > 0 ? systemMessages : null, + }, + ]; +} + +async function toImagePart(image: ImageContent): Promise> { + const cacheKey = 'serialized_image'; + if (!image._cache[cacheKey]) { + image._cache[cacheKey] = await serializeImage(image); + } + const img: SerializedImage = image._cache[cacheKey]; + + if (img.externalUrl) { + throw new Error( + 'externalUrl images are not supported by AWS Bedrock, provide inline image data instead', + ); + } + + return { + image: { + format: imageFormat(img.mimeType), + source: { bytes: Buffer.from(img.base64Data ?? '', 'base64') }, + }, + }; +} + +function imageFormat(mimeType?: string): string { + if (!mimeType) return 'jpeg'; + + const format = AWS_IMAGE_FORMATS[mimeType]; + if (!format) { + throw new Error( + `Unsupported mimeType ${mimeType} for AWS Bedrock images. Must be jpeg, png, webp, or gif`, + ); + } + return format; +} diff --git a/agents/src/llm/provider_format/index.ts b/agents/src/llm/provider_format/index.ts index 56e014b7e..6b72b45af 100644 --- a/agents/src/llm/provider_format/index.ts +++ b/agents/src/llm/provider_format/index.ts @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 import type { ChatContext } from '../chat_context.js'; +import { toChatCtx as toChatCtxAws } from './aws.js'; import { toChatCtx as toChatCtxGoogle } from './google.js'; import { toChatCtx as toChatCtxMistralai } from './mistralai.js'; import { @@ -9,7 +10,7 @@ import { toResponsesChatCtx as toResponsesChatCtxOpenai, } from './openai.js'; -export type ProviderFormat = 'openai' | 'openai.responses' | 'google' | 'mistralai'; +export type ProviderFormat = 'openai' | 'openai.responses' | 'google' | 'mistralai' | 'aws'; export async function toChatCtx( format: ProviderFormat, @@ -25,6 +26,8 @@ export async function toChatCtx( return await toChatCtxGoogle(chatCtx, injectDummyUserMessage); case 'mistralai': return toChatCtxMistralai(chatCtx, injectDummyUserMessage); + case 'aws': + return await toChatCtxAws(chatCtx, injectDummyUserMessage); default: throw new Error(`Unsupported provider format: ${format}`); } diff --git a/examples/package.json b/examples/package.json index 2dfff00dc..6b5a4ec17 100644 --- a/examples/package.json +++ b/examples/package.json @@ -26,6 +26,7 @@ "@livekit/agents": "workspace:*", "@livekit/agents-plugin-anam": "workspace:*", "@livekit/agents-plugin-assemblyai": "workspace:*", + "@livekit/agents-plugin-aws": "workspace:*", "@livekit/agents-plugin-baseten": "workspace:*", "@livekit/agents-plugin-bey": "workspace:*", "@livekit/agents-plugin-cartesia": "workspace:*", diff --git a/examples/src/aws.ts b/examples/src/aws.ts new file mode 100644 index 000000000..17b5a4cf6 --- /dev/null +++ b/examples/src/aws.ts @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { + type JobContext, + ServerOptions, + cli, + defineAgent, + log, + metrics, + voice, +} from '@livekit/agents'; +import * as aws from '@livekit/agents-plugin-aws'; +import { fileURLToPath } from 'node:url'; + +// Demonstrates a fully AWS-backed voice pipeline: Amazon Transcribe (STT), +// Amazon Bedrock Converse (LLM), and Amazon Polly (TTS). +// +// Credentials are resolved via the AWS SDK v3 default credential chain +// (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN, a shared +// profile, IMDS, etc.) — see plugins/aws/README.md for details. The region +// defaults to AWS_REGION / AWS_DEFAULT_REGION / "us-east-1". +export default defineAgent({ + entry: async (ctx: JobContext) => { + const agent = new voice.Agent({ + instructions: + "You are a helpful assistant, you can hear the user's message and respond to it.", + }); + + const logger = log(); + + const session = new voice.AgentSession({ + stt: new aws.STT({ language: 'en-US' }), + llm: new aws.LLM({ + // model defaults to BEDROCK_INFERENCE_PROFILE_ARN if set, otherwise 'amazon.nova-2-lite-v1:0' + }), + // Amazon Polly only supports PCM output at 8000 or 16000 Hz. + tts: new aws.TTS({ voice: 'Ruth', sampleRate: 16000 }), + turnHandling: { + turnDetection: 'stt', + }, + }); + + // Log metrics as they are emitted (session.usage is automatically collected) + session.on(voice.AgentSessionEventTypes.MetricsCollected, (ev) => { + metrics.logMetrics(ev.metrics); + }); + + // Log usage summary when job shuts down + ctx.addShutdownCallback(async () => { + logger.info( + { + usage: session.usage, + }, + 'Session usage summary', + ); + }); + + await session.start({ + agent, + room: ctx.room, + }); + + session.say('Hello, how can I help you today?'); + }, +}); + +cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) })); diff --git a/plugins/aws/README.md b/plugins/aws/README.md new file mode 100644 index 000000000..18d5a07c7 --- /dev/null +++ b/plugins/aws/README.md @@ -0,0 +1,91 @@ + + +# AWS plugin for LiveKit Agents + +The Agents Framework is designed for building realtime, programmable +participants that run on servers. Use it to create conversational, multi-modal +voice agents that can see, hear, and understand. + +This package contains the AWS plugin, providing LLM (Amazon Bedrock Converse), +STT (Amazon Transcribe streaming), and TTS (Amazon Polly) capabilities via the +official AWS SDK v3 clients. + +## Installation + +```bash +npm install @livekit/agents-plugin-aws +``` + +## Credentials and region + +All three components resolve AWS credentials via the AWS SDK v3 default +credential chain (environment variables, shared config/credentials files, +IMDS, container credentials, etc.) unless `credentials` is passed explicitly: + +```ts +const llm = new aws.LLM({ + credentials: { + accessKeyId: '...', + secretAccessKey: '...', + sessionToken: '...', // optional + }, +}); +``` + +The region is resolved in this order: the `region` constructor option, then +`AWS_REGION`, then `AWS_DEFAULT_REGION`, falling back to `us-east-1`. + +## Usage + +### LLM + +Uses the [Bedrock Converse API](https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference.html) +with streaming. + +```ts +import * as aws from '@livekit/agents-plugin-aws'; + +const llm = new aws.LLM({ + // model defaults to BEDROCK_INFERENCE_PROFILE_ARN if set, otherwise 'amazon.nova-2-lite-v1:0' + model: 'anthropic.claude-3-5-sonnet-20241022-v2:0', + region: 'us-east-1', +}); +``` + +### STT + +Streaming-only, via [Amazon Transcribe streaming](https://docs.aws.amazon.com/transcribe/latest/dg/streaming.html). +Single-frame (non-streaming) recognition is not supported. + +```ts +const stt = new aws.STT({ + language: 'en-US', + sampleRate: 24000, +}); +``` + +### TTS + +Uses [Amazon Polly](https://docs.aws.amazon.com/polly/latest/dg/API_SynthesizeSpeech.html). +Audio is requested as raw PCM (16-bit little-endian, mono) since the framework +has no MP3 decoder — Amazon Polly only supports PCM output at **8000 Hz or +16000 Hz**, so `sampleRate` is restricted to those two values (default +`16000`). Streaming synthesis is not supported. + +```ts +const tts = new aws.TTS({ + voice: 'Ruth', + speechEngine: 'generative', + sampleRate: 16000, +}); +``` + +### Environment variables + +- `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` / `AWS_SESSION_TOKEN` — picked up by the default credential chain +- `AWS_REGION` / `AWS_DEFAULT_REGION` — region fallback when `region` isn't passed explicitly +- `BEDROCK_INFERENCE_PROFILE_ARN` — LLM model fallback when `model` isn't passed explicitly diff --git a/plugins/aws/api-extractor.json b/plugins/aws/api-extractor.json new file mode 100644 index 000000000..1f75e0708 --- /dev/null +++ b/plugins/aws/api-extractor.json @@ -0,0 +1,20 @@ +/** + * Config file for API Extractor. For more info, please visit: https://api-extractor.com + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + + /** + * Optionally specifies another JSON config file that this file extends from. This provides a way for + * standard settings to be shared across multiple projects. + * + * If the path starts with "./" or "../", the path is resolved relative to the folder of the file that contains + * the "extends" field. Otherwise, the first path segment is interpreted as an NPM package name, and will be + * resolved using NodeJS require(). + * + * SUPPORTED TOKENS: none + * DEFAULT VALUE: "" + */ + "extends": "../../api-extractor-shared.json", + "mainEntryPointFilePath": "./dist/index.d.ts" +} diff --git a/plugins/aws/package.json b/plugins/aws/package.json new file mode 100644 index 000000000..0ac788bb0 --- /dev/null +++ b/plugins/aws/package.json @@ -0,0 +1,55 @@ +{ + "name": "@livekit/agents-plugin-aws", + "version": "1.0.0", + "description": "AWS plugin for LiveKit Node Agents", + "main": "dist/index.js", + "require": "dist/index.cjs", + "types": "dist/index.d.ts", + "exports": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, + "author": "LiveKit", + "type": "module", + "repository": "git@github.com:livekit/agents-js.git", + "license": "Apache-2.0", + "files": [ + "dist", + "src", + "README.md" + ], + "scripts": { + "build": "tsup --onSuccess \"pnpm build:types\"", + "build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js", + "clean": "rm -rf dist", + "clean:build": "pnpm clean && pnpm build", + "lint": "eslint -f unix \"src/**/*.{ts,js}\"", + "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript", + "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose" + }, + "devDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/agents-plugin-silero": "workspace:*", + "@livekit/agents-plugins-test": "workspace:*", + "@livekit/rtc-node": "catalog:", + "@microsoft/api-extractor": "^7.35.0", + "tsup": "^8.3.5", + "typescript": "^5.0.0", + "zod": "^3.25.76 || ^4.1.8" + }, + "dependencies": { + "@aws-sdk/client-bedrock-runtime": "^3.1081.0", + "@aws-sdk/client-polly": "^3.1081.0", + "@aws-sdk/client-transcribe-streaming": "^3.1081.0" + }, + "peerDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/rtc-node": "catalog:" + } +} diff --git a/plugins/aws/src/index.ts b/plugins/aws/src/index.ts new file mode 100644 index 000000000..c123bc7fe --- /dev/null +++ b/plugins/aws/src/index.ts @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { Plugin } from '@livekit/agents'; + +export { LLM, LLMStream, type LLMOptions, buildToolConfig } from './llm.js'; +export * from './models.js'; +export { SpeechStream, STT, type STTOptions } from './stt.js'; +export { ChunkedStream, TTS, type TTSOptions } from './tts.js'; +export { type AwsCredentials, DEFAULT_REGION, resolveRegion } from './utils.js'; + +class AwsPlugin extends Plugin { + constructor() { + super({ + title: 'aws', + version: __PACKAGE_VERSION__, + package: __PACKAGE_NAME__, + }); + } +} + +Plugin.registerPlugin(new AwsPlugin()); diff --git a/plugins/aws/src/llm.test.ts b/plugins/aws/src/llm.test.ts new file mode 100644 index 000000000..e23c82e9b --- /dev/null +++ b/plugins/aws/src/llm.test.ts @@ -0,0 +1,249 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime'; +import { APIStatusError, llm } from '@livekit/agents'; +import { llm as llmTest } from '@livekit/agents-plugins-test'; +import { afterEach, describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { LLM, buildToolConfig, mapConverseStreamException } from './llm.js'; + +const hasAwsCredentials = Boolean(process.env.AWS_ACCESS_KEY_ID || process.env.AWS_PROFILE); + +function fakeClient( + events: Record[], + requestId = 'req_123', +): BedrockRuntimeClient { + return { + send: async () => ({ + $metadata: { requestId }, + stream: (async function* () { + yield* events; + })(), + }), + } as unknown as BedrockRuntimeClient; +} + +function weatherToolCtx() { + return llm.toToolContext({ + getWeather: llm.tool({ + description: 'Get the weather for a given location.', + parameters: z.object({ location: z.string() }), + execute: async () => 'sunny', + }), + }); +} + +describe('AWS Bedrock LLM - buildToolConfig', () => { + it('returns undefined when there are no tools', () => { + expect(buildToolConfig(llm.ToolContext.empty(), undefined, false)).toBeUndefined(); + }); + + it('returns undefined when toolChoice is "none"', () => { + expect(buildToolConfig(weatherToolCtx(), 'none', false)).toBeUndefined(); + }); + + it('maps toolChoice "auto" to {auto: {}}', () => { + expect(buildToolConfig(weatherToolCtx(), 'auto', false)?.toolChoice).toEqual({ auto: {} }); + }); + + it('maps toolChoice "required" to {any: {}}', () => { + expect(buildToolConfig(weatherToolCtx(), 'required', false)?.toolChoice).toEqual({ any: {} }); + }); + + it('maps a named function toolChoice to {tool: {name}}', () => { + const config = buildToolConfig( + weatherToolCtx(), + { type: 'function', function: { name: 'getWeather' } }, + false, + ); + expect(config?.toolChoice).toEqual({ tool: { name: 'getWeather' } }); + }); + + it('leaves toolChoice unset when not specified', () => { + const config = buildToolConfig(weatherToolCtx(), undefined, false); + expect(config?.toolChoice).toBeUndefined(); + expect(config?.tools).toHaveLength(1); + }); + + it('appends a cachePoint block when cacheTools is enabled', () => { + const config = buildToolConfig(weatherToolCtx(), 'auto', true); + expect(config?.tools).toHaveLength(2); + expect(config?.tools?.[1]).toEqual({ cachePoint: { type: 'default' } }); + }); + + it('builds a valid toolSpec from a real ToolContext', () => { + const config = buildToolConfig(weatherToolCtx(), 'auto', false); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const toolSpec = (config?.tools?.[0] as any).toolSpec; + expect(toolSpec.name).toBe('getWeather'); + expect(toolSpec.description).toBe('Get the weather for a given location.'); + expect(toolSpec.inputSchema.json.type).toBe('object'); + expect(Object.keys(toolSpec.inputSchema.json.properties ?? {})).toEqual(['location']); + }); +}); + +describe('AWS Bedrock LLM - constructor', () => { + const originalEnv = process.env.BEDROCK_INFERENCE_PROFILE_ARN; + + afterEach(() => { + if (originalEnv === undefined) { + delete process.env.BEDROCK_INFERENCE_PROFILE_ARN; + } else { + process.env.BEDROCK_INFERENCE_PROFILE_ARN = originalEnv; + } + }); + + it('defaults the model to amazon.nova-2-lite-v1:0', () => { + delete process.env.BEDROCK_INFERENCE_PROFILE_ARN; + const bedrockLlm = new LLM({ client: fakeClient([]) }); + expect(bedrockLlm.model).toBe('amazon.nova-2-lite-v1:0'); + }); + + it('falls back to BEDROCK_INFERENCE_PROFILE_ARN when model is not given', () => { + process.env.BEDROCK_INFERENCE_PROFILE_ARN = + 'arn:aws:bedrock:us-east-1:123456789012:inference-profile/foo'; + const bedrockLlm = new LLM({ client: fakeClient([]) }); + expect(bedrockLlm.model).toBe('arn:aws:bedrock:us-east-1:123456789012:inference-profile/foo'); + }); + + it('prefers an explicit model over BEDROCK_INFERENCE_PROFILE_ARN', () => { + process.env.BEDROCK_INFERENCE_PROFILE_ARN = + 'arn:aws:bedrock:us-east-1:123456789012:inference-profile/foo'; + const bedrockLlm = new LLM({ model: 'anthropic.claude-3-5-sonnet', client: fakeClient([]) }); + expect(bedrockLlm.model).toBe('anthropic.claude-3-5-sonnet'); + }); + + it('reports the provider label', () => { + const bedrockLlm = new LLM({ client: fakeClient([]) }); + expect(bedrockLlm.provider).toBe('AWS Bedrock'); + expect(bedrockLlm.label()).toBe('aws.LLM'); + }); +}); + +describe('AWS Bedrock LLM - streaming', () => { + it('emits text deltas and usage from the Converse stream', async () => { + const bedrockLlm = new LLM({ + client: fakeClient([ + { contentBlockDelta: { contentBlockIndex: 0, delta: { text: 'Hello' } } }, + { contentBlockDelta: { contentBlockIndex: 0, delta: { text: ' world' } } }, + { metadata: { usage: { inputTokens: 5, outputTokens: 2, totalTokens: 7 } } }, + ]), + }); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'Hi' }); + + const chunks: llm.ChatChunk[] = []; + for await (const chunk of bedrockLlm.chat({ chatCtx })) { + chunks.push(chunk); + } + + expect(chunks.map((c) => c.delta?.content ?? '').join('')).toBe('Hello world'); + expect(chunks.at(-1)?.usage).toEqual({ + completionTokens: 2, + promptTokens: 5, + totalTokens: 7, + promptCachedTokens: 0, + }); + }); + + it('emits a FunctionCall assembled from toolUse content blocks', async () => { + const bedrockLlm = new LLM({ + client: fakeClient([ + { + contentBlockStart: { + contentBlockIndex: 0, + start: { toolUse: { toolUseId: 'call_1', name: 'getWeather' } }, + }, + }, + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { toolUse: { input: '{"location":"Tokyo"}' } }, + }, + }, + { contentBlockStop: { contentBlockIndex: 0 } }, + ]), + }); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'weather?' }); + + const toolCalls: llm.FunctionCall[] = []; + for await (const chunk of bedrockLlm.chat({ chatCtx })) { + if (chunk.delta?.toolCalls) toolCalls.push(...chunk.delta.toolCalls); + } + + expect(toolCalls).toHaveLength(1); + expect(toolCalls[0]?.callId).toBe('call_1'); + expect(toolCalls[0]?.name).toBe('getWeather'); + expect(toolCalls[0]?.args).toBe('{"location":"Tokyo"}'); + }); +}); + +describe('AWS Bedrock LLM - mapConverseStreamException', () => { + it('maps validationException to a non-retryable APIStatusError', () => { + const error = mapConverseStreamException( + { validationException: { message: 'bad input' } }, + 'req_1', + true, + ); + expect(error).toBeInstanceOf(APIStatusError); + expect(error?.retryable).toBe(false); + expect(error?.message).toMatch(/bad input/); + }); + + it('maps throttlingException to a 429 APIStatusError honoring the retryable flag', () => { + const error = mapConverseStreamException( + { throttlingException: { message: 'too many requests' } }, + 'req_1', + true, + ); + expect(error?.statusCode).toBe(429); + expect(error?.retryable).toBe(true); + expect(error?.message).toMatch(/too many requests/); + }); + + it('maps internalServerException to a 500 APIStatusError', () => { + const error = mapConverseStreamException( + { internalServerException: { message: 'oops' } }, + 'req_1', + true, + ); + expect(error?.statusCode).toBe(500); + expect(error?.message).toMatch(/oops/); + }); + + it('maps modelStreamErrorException using its originalStatusCode', () => { + const error = mapConverseStreamException( + { modelStreamErrorException: { message: 'stream broke', originalStatusCode: 503 } }, + 'req_1', + true, + ); + expect(error?.statusCode).toBe(503); + expect(error?.message).toMatch(/stream broke/); + }); + + it('maps serviceUnavailableException to a 503 APIStatusError', () => { + const error = mapConverseStreamException( + { serviceUnavailableException: { message: 'down for maintenance' } }, + 'req_1', + true, + ); + expect(error?.statusCode).toBe(503); + expect(error?.message).toMatch(/down for maintenance/); + }); + + it('returns undefined for a non-exception event', () => { + expect(mapConverseStreamException({}, 'req_1', true)).toBeUndefined(); + }); +}); + +describe('AWS Bedrock LLM (live)', () => { + if (hasAwsCredentials) { + it('passes the shared LLM test harness', async () => { + await llmTest(new LLM({ temperature: 0 }), false); + }); + } else { + it.skip('requires AWS_ACCESS_KEY_ID or AWS_PROFILE', () => {}); + } +}); diff --git a/plugins/aws/src/llm.ts b/plugins/aws/src/llm.ts new file mode 100644 index 000000000..904db0650 --- /dev/null +++ b/plugins/aws/src/llm.ts @@ -0,0 +1,414 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { + ConverseStreamCommandInput, + Message, + SystemContentBlock, + Tool, + ToolConfiguration, +} from '@aws-sdk/client-bedrock-runtime'; +import { BedrockRuntimeClient, ConverseStreamCommand } from '@aws-sdk/client-bedrock-runtime'; +import type { APIConnectOptions } from '@livekit/agents'; +import { + APIConnectionError, + APIStatusError, + DEFAULT_API_CONNECT_OPTIONS, + llm, +} from '@livekit/agents'; +import { type AwsCredentials, resolveRegion } from './utils.js'; + +const DEFAULT_MODEL = 'amazon.nova-2-lite-v1:0'; + +interface AwsFormatData { + systemMessages: string[] | null; +} + +export interface LLMOptions { + model?: string; + region?: string; + credentials?: AwsCredentials; + temperature?: number; + maxOutputTokens?: number; + topP?: number; + toolChoice?: llm.ToolChoice; + additionalRequestFields?: Record; + /** Caches system messages with a Bedrock prompt cache point to reduce token usage. */ + cacheSystem?: boolean; + /** Caches tool definitions with a Bedrock prompt cache point to reduce token usage. */ + cacheTools?: boolean; + client?: BedrockRuntimeClient; +} + +/** + * Builds a Bedrock Converse `ToolConfiguration` from a `ToolContext`, or `undefined` when the + * turn should carry no tools (no tools registered, or `toolChoice: 'none'` was requested). + * Exported for unit testing. + */ +export function buildToolConfig( + toolCtx: llm.ToolContext | undefined, + toolChoice: llm.ToolChoice | undefined, + cacheTools: boolean, +): ToolConfiguration | undefined { + if (!toolCtx || Object.keys(toolCtx.functionTools).length === 0) return undefined; + if (toolChoice === 'none') return undefined; + + // The AWS SDK's `Tool` discriminated union requires a `$unknown` tuple on its catch-all + // member, which defeats direct object-literal assignability checks against `Tool[]` here + // (the same constraint the sibling google.ts/mistralai.ts adapters hit at this exact + // ChatContext-to-SDK-type boundary) — build the array in its natural shape and cast once. + const tools: Record[] = llm.sortedToolEntries(toolCtx).map(([name, tool]) => ({ + toolSpec: { + name, + description: tool.description || '', + inputSchema: { + json: tool.parameters + ? llm.toJsonSchema(tool.parameters, false, false) + : { type: 'object', properties: {} }, + }, + }, + })); + + if (cacheTools) { + tools.push({ cachePoint: { type: 'default' } }); + } + + const toolConfig: ToolConfiguration = { tools: tools as unknown as Tool[] }; + + if (toolChoice === 'required') { + toolConfig.toolChoice = { any: {} }; + } else if (toolChoice === 'auto') { + toolConfig.toolChoice = { auto: {} }; + } else if (typeof toolChoice === 'object' && toolChoice.type === 'function') { + toolConfig.toolChoice = { tool: { name: toolChoice.function.name } }; + } + + return toolConfig; +} + +interface ConverseStreamException { + message?: string; + originalStatusCode?: number; +} + +interface ConverseStreamExceptionEvent { + validationException?: ConverseStreamException; + throttlingException?: ConverseStreamException; + internalServerException?: ConverseStreamException; + modelStreamErrorException?: ConverseStreamException; + serviceUnavailableException?: ConverseStreamException; +} + +const CONVERSE_STREAM_EXCEPTIONS: Array<{ + key: keyof ConverseStreamExceptionEvent; + defaultMessage: string; + statusCode?: number; + retryable?: false; +}> = [ + { key: 'validationException', defaultMessage: 'validation error', retryable: false }, + { key: 'throttlingException', defaultMessage: 'throttled', statusCode: 429 }, + { key: 'internalServerException', defaultMessage: 'internal server error', statusCode: 500 }, + { key: 'modelStreamErrorException', defaultMessage: 'model stream error', statusCode: 500 }, + { key: 'serviceUnavailableException', defaultMessage: 'service unavailable', statusCode: 503 }, +]; + +/** + * Bedrock delivers fatal mid-stream errors as regular `ConverseStreamOutput` union events + * rather than thrown exceptions. Maps one to an `APIStatusError`, or `undefined` if the event + * isn't one of the known exception shapes. Exported for unit testing. + */ +export function mapConverseStreamException( + event: ConverseStreamExceptionEvent, + requestId: string, + retryable: boolean, +): APIStatusError | undefined { + for (const { + key, + defaultMessage, + statusCode, + retryable: retryableOverride, + } of CONVERSE_STREAM_EXCEPTIONS) { + const exception = event[key]; + if (!exception) continue; + + return new APIStatusError({ + message: `aws bedrock llm: ${exception.message ?? defaultMessage}`, + options: { + statusCode: exception.originalStatusCode ?? statusCode, + retryable: retryableOverride ?? retryable, + requestId, + }, + }); + } + return undefined; +} + +/** AWS Bedrock Converse LLM. */ +export class LLM extends llm.LLM { + #opts; + #client: BedrockRuntimeClient; + + label(): string { + return 'aws.LLM'; + } + + get model(): string { + return this.#opts.model; + } + + get provider(): string { + return 'AWS Bedrock'; + } + + /** + * Create a new instance of AWS Bedrock Converse LLM. + * + * @remarks + * Credentials are resolved via the AWS SDK v3 default credential chain (environment + * variables, shared config/credentials files, IMDS, etc.) unless `credentials` is provided + * explicitly. The region is resolved from `region`, then `AWS_REGION`, then + * `AWS_DEFAULT_REGION`, falling back to `us-east-1`. `model` defaults to + * `BEDROCK_INFERENCE_PROFILE_ARN` if set, otherwise `amazon.nova-2-lite-v1:0`. + */ + constructor(opts: LLMOptions = {}) { + super(); + + this.#opts = { + model: opts.model ?? process.env.BEDROCK_INFERENCE_PROFILE_ARN ?? DEFAULT_MODEL, + region: resolveRegion(opts.region), + credentials: opts.credentials, + temperature: opts.temperature, + maxOutputTokens: opts.maxOutputTokens, + topP: opts.topP, + toolChoice: opts.toolChoice, + additionalRequestFields: opts.additionalRequestFields, + cacheSystem: opts.cacheSystem ?? false, + cacheTools: opts.cacheTools ?? false, + }; + + this.#client = + opts.client ?? + new BedrockRuntimeClient({ + region: this.#opts.region, + credentials: opts.credentials, + // The framework's own connOptions retry loop handles retries; disable the SDK's + // internal retries so failures aren't retried twice (matches tts.ts's PollyClient). + maxAttempts: 1, + }); + } + + chat({ + chatCtx, + toolCtx: toolCtxInput, + connOptions = DEFAULT_API_CONNECT_OPTIONS, + toolChoice, + extraKwargs, + }: { + chatCtx: llm.ChatContext; + toolCtx?: llm.ToolContextLike; + connOptions?: APIConnectOptions; + parallelToolCalls?: boolean; + toolChoice?: llm.ToolChoice; + extraKwargs?: Record; + }): LLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); + const resolvedToolChoice = toolChoice ?? this.#opts.toolChoice; + + const toolConfig = buildToolConfig(toolCtx, resolvedToolChoice, this.#opts.cacheTools); + if (!toolConfig) { + chatCtx = chatCtx.copy({ excludeFunctionCall: true }); + } + + const inferenceConfig: Record = {}; + if (this.#opts.maxOutputTokens !== undefined) + inferenceConfig.maxTokens = this.#opts.maxOutputTokens; + if (this.#opts.temperature !== undefined) inferenceConfig.temperature = this.#opts.temperature; + if (this.#opts.topP !== undefined) inferenceConfig.topP = this.#opts.topP; + + const extras: Record = { + inferenceConfig, + ...(toolConfig ? { toolConfig } : {}), + ...(this.#opts.additionalRequestFields + ? { additionalModelRequestFields: this.#opts.additionalRequestFields } + : {}), + ...extraKwargs, + }; + + return new LLMStream(this, { + client: this.#client, + model: this.#opts.model, + chatCtx, + toolCtx, + connOptions, + cacheSystem: this.#opts.cacheSystem, + extraKwargs: extras, + }); + } +} + +export class LLMStream extends llm.LLMStream { + #client: BedrockRuntimeClient; + #model: string; + #cacheSystem: boolean; + #extraKwargs: Record; + + constructor( + llmInst: LLM, + { + client, + model, + chatCtx, + toolCtx, + connOptions, + cacheSystem, + extraKwargs, + }: { + client: BedrockRuntimeClient; + model: string; + chatCtx: llm.ChatContext; + toolCtx?: llm.ToolContext; + connOptions: APIConnectOptions; + cacheSystem: boolean; + extraKwargs: Record; + }, + ) { + super(llmInst, { chatCtx, toolCtx, connOptions }); + this.#client = client; + this.#model = model; + this.#cacheSystem = cacheSystem; + this.#extraKwargs = extraKwargs; + } + + protected async run(): Promise { + let retryable = true; + let requestId = ''; + + try { + const [messages, extraData] = (await this.chatCtx.toProviderFormat('aws')) as [ + Record[], + AwsFormatData, + ]; + + const system: Record[] = []; + if (extraData.systemMessages) { + for (const content of extraData.systemMessages) { + system.push({ text: content }); + } + if (this.#cacheSystem) { + system.push({ cachePoint: { type: 'default' } }); + } + } + + const input: ConverseStreamCommandInput = { + modelId: this.#model, + messages: messages as unknown as Message[], + ...(system.length > 0 ? { system: system as unknown as SystemContentBlock[] } : {}), + ...this.#extraKwargs, + }; + + const response = await this.#client.send(new ConverseStreamCommand(input), { + abortSignal: this.abortController.signal, + }); + requestId = response.$metadata.requestId ?? ''; + + if (!response.stream) { + throw new APIStatusError({ + message: 'aws bedrock llm: no stream in the response', + options: { retryable: false, requestId }, + }); + } + + let toolCallId: string | undefined; + let fncName: string | undefined; + let fncRawArgs: string | undefined; + + for await (const event of response.stream) { + if (event.contentBlockStart?.start?.toolUse) { + const toolUse = event.contentBlockStart.start.toolUse; + toolCallId = toolUse.toolUseId; + fncName = toolUse.name; + fncRawArgs = ''; + } else if (event.contentBlockDelta?.delta) { + const delta = event.contentBlockDelta.delta; + if (delta.toolUse?.input !== undefined) { + fncRawArgs = (fncRawArgs ?? '') + delta.toolUse.input; + } else if (delta.text !== undefined) { + this.queue.put({ + id: requestId, + delta: { role: 'assistant', content: delta.text }, + }); + retryable = false; + } + } else if (event.contentBlockStop) { + if (toolCallId !== undefined) { + this.queue.put({ + id: requestId, + delta: { + role: 'assistant', + toolCalls: [ + llm.FunctionCall.create({ + callId: toolCallId, + name: fncName ?? '', + args: fncRawArgs ?? '', + }), + ], + }, + }); + retryable = false; + toolCallId = undefined; + fncName = undefined; + fncRawArgs = undefined; + } + } else if (event.metadata) { + const usage = event.metadata.usage; + if (usage) { + this.queue.put({ + id: requestId, + usage: { + completionTokens: usage.outputTokens ?? 0, + promptTokens: usage.inputTokens ?? 0, + totalTokens: usage.totalTokens ?? 0, + promptCachedTokens: usage.cacheReadInputTokens ?? 0, + }, + }); + } + } else { + const mappedError = mapConverseStreamException(event, requestId, retryable); + if (mappedError) { + throw mappedError; + } + } + } + } catch (error: unknown) { + if (error instanceof APIStatusError || error instanceof APIConnectionError) { + throw error; + } + + // A user interruption aborts the in-flight Bedrock call; the resulting AbortError has + // no HTTP status and would otherwise be classified as a retryable connection error, + // causing the base LLMStream to fire an unwanted duplicate request on every barge-in. + if (this.abortController.signal.aborted) { + return; + } + + const err = error as { message?: string; $metadata?: { httpStatusCode?: number } }; + const statusCode = err.$metadata?.httpStatusCode; + + if (statusCode !== undefined) { + throw new APIStatusError({ + message: `aws bedrock llm: ${err.message ?? 'unknown error'}`, + options: { + statusCode, + retryable: retryable && (statusCode === 429 || statusCode >= 500), + requestId, + }, + }); + } + + throw new APIConnectionError({ + message: `aws bedrock llm: ${err.message ?? String(error)}`, + options: { retryable }, + }); + } + } +} diff --git a/plugins/aws/src/models.ts b/plugins/aws/src/models.ts new file mode 100644 index 000000000..841263a95 --- /dev/null +++ b/plugins/aws/src/models.ts @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** Amazon Polly speech synthesis engine. */ +export type TTSSpeechEngine = 'standard' | 'neural' | 'long-form' | 'generative'; + +/** + * Language code for the Amazon Polly SynthesizeSpeech request. Only necessary when using a + * bilingual voice (e.g. Aditi, which supports both `en-IN` and `hi-IN`). + */ +export type TTSLanguage = + | 'arb' + | 'cmn-CN' + | 'cy-GB' + | 'da-DK' + | 'de-DE' + | 'en-AU' + | 'en-GB' + | 'en-GB-WLS' + | 'en-IN' + | 'en-US' + | 'es-ES' + | 'es-MX' + | 'es-US' + | 'fr-CA' + | 'fr-FR' + | 'is-IS' + | 'it-IT' + | 'ja-JP' + | 'hi-IN' + | 'ko-KR' + | 'nb-NO' + | 'nl-NL' + | 'pl-PL' + | 'pt-BR' + | 'pt-PT' + | 'ro-RO' + | 'ru-RU' + | 'sv-SE' + | 'tr-TR' + | 'en-NZ' + | 'en-ZA' + | 'ca-ES' + | 'de-AT' + | 'yue-CN' + | 'ar-AE' + | 'fi-FI' + | 'en-IE' + | 'nl-BE' + | 'fr-BE' + | 'cs-CZ' + | 'de-CH'; + +/** Whether the Amazon Polly input text is plain text or SSML. */ +export type TTSTextType = 'text' | 'ssml'; diff --git a/plugins/aws/src/stt.test.ts b/plugins/aws/src/stt.test.ts new file mode 100644 index 000000000..4475573be --- /dev/null +++ b/plugins/aws/src/stt.test.ts @@ -0,0 +1,530 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { TranscribeStreamingClient } from '@aws-sdk/client-transcribe-streaming'; +import { stt } from '@livekit/agents'; +import { VAD } from '@livekit/agents-plugin-silero'; +import { stt as sttTest } from '@livekit/agents-plugins-test'; +import { AudioFrame } from '@livekit/rtc-node'; +import { describe, expect, it } from 'vitest'; +import { STT } from './stt.js'; +import type { STTOptions } from './stt.js'; + +const hasAwsCredentials = Boolean(process.env.AWS_ACCESS_KEY_ID || process.env.AWS_PROFILE); + +const baseOpts: Partial = { sampleRate: 16000, language: 'en-US' }; + +function fakeClient( + sessions: Array<() => AsyncGenerator>>, +): TranscribeStreamingClient { + let call = 0; + return { + send: async () => { + const session = sessions[Math.min(call, sessions.length - 1)]!; + call += 1; + return { TranscriptResultStream: session() }; + }, + } as unknown as TranscribeStreamingClient; +} + +function transcriptEvent(...results: Record[]) { + return { TranscriptEvent: { Transcript: { Results: results } } }; +} + +// Exercises the public STT constructor + STT.stream(), rather than instantiating the +// internal SpeechStream class directly, so these tests also cover STT.stream()'s wiring. +function stream(client: TranscribeStreamingClient) { + return new STT({ ...baseOpts, client }).stream(); +} + +describe('AWS Transcribe STT - constructor', () => { + it('throws when identifyLanguage and identifyMultipleLanguages are both set', () => { + expect(() => new STT({ identifyLanguage: true, identifyMultipleLanguages: true })).toThrow( + /mutually exclusive/, + ); + }); + + it('reports streaming-only capabilities with word-aligned transcripts', () => { + const sttInstance = new STT(); + expect(sttInstance.capabilities).toEqual({ + streaming: true, + interimResults: true, + alignedTranscript: 'word', + }); + }); + + it('reports the provider label', () => { + const sttInstance = new STT(); + expect(sttInstance.provider).toBe('Amazon Transcribe'); + expect(sttInstance.label).toBe('aws.STT'); + }); + + it('defaults model to "unknown" without a languageModelName', () => { + expect(new STT().model).toBe('unknown'); + }); + + it('reports languageModelName as the model when provided', () => { + expect(new STT({ languageModelName: 'my-custom-model' }).model).toBe('my-custom-model'); + }); + + it('rejects single-frame recognition', async () => { + const sttInstance = new STT(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await expect((sttInstance as any)._recognize()).rejects.toThrow( + /does not support single-frame recognition/, + ); + }); +}); + +describe('AWS Transcribe STT - SpeechStream event mapping', () => { + it('maps START_OF_SPEECH / FINAL_TRANSCRIPT / END_OF_SPEECH from a TranscriptEvent', async () => { + const client = fakeClient([ + async function* () { + yield transcriptEvent({ + StartTime: 0, + EndTime: 0.5, + IsPartial: false, + Alternatives: [ + { + Transcript: 'hello.', + Items: [ + { + Content: 'hello', + Type: 'pronunciation', + StartTime: 0, + EndTime: 0.4, + Confidence: 0.9, + }, + { Content: '.', Type: 'punctuation' }, + ], + }, + ], + }); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + expect(events.map((e) => e.type)).toEqual([ + stt.SpeechEventType.START_OF_SPEECH, + stt.SpeechEventType.FINAL_TRANSCRIPT, + stt.SpeechEventType.END_OF_SPEECH, + ]); + expect(events[1]?.alternatives?.[0]?.text).toBe('hello.'); + // The punctuation item is excluded from words and from the confidence average. + expect(events[1]?.alternatives?.[0]?.words).toHaveLength(1); + expect(events[1]?.alternatives?.[0]?.words?.[0]?.text).toBe('hello'); + expect(events[1]?.alternatives?.[0]?.confidence).toBe(0.9); + }); + + it('emits START_OF_SPEECH for every utterance, not just the first in the session', async () => { + const client = fakeClient([ + async function* () { + yield transcriptEvent({ + StartTime: 0, + EndTime: 0.5, + IsPartial: false, + Alternatives: [{ Transcript: 'first', Items: [] }], + }); + // A later utterance in the same session never has StartTime === 0. + yield transcriptEvent({ + StartTime: 3.2, + EndTime: 3.8, + IsPartial: false, + Alternatives: [{ Transcript: 'second', Items: [] }], + }); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + expect(events.filter((e) => e.type === stt.SpeechEventType.START_OF_SPEECH)).toHaveLength(2); + expect(events.filter((e) => e.type === stt.SpeechEventType.END_OF_SPEECH)).toHaveLength(2); + }); + + it('tracks speaking state independently per channel within a single TranscriptEvent', async () => { + const client = fakeClient([ + async function* () { + // Both channels start together... + yield transcriptEvent( + { ChannelId: 'ch_0', StartTime: 0, EndTime: 0.3, IsPartial: true, Alternatives: [] }, + { ChannelId: 'ch_1', StartTime: 0, EndTime: 0.3, IsPartial: true, Alternatives: [] }, + ); + // ...channel 0 finishes while channel 1 is still mid-utterance, in the same event. + // Channel 0 finishing must not spuriously re-trigger START_OF_SPEECH for channel 1. + yield transcriptEvent( + { + ChannelId: 'ch_0', + StartTime: 0, + EndTime: 0.5, + IsPartial: false, + Alternatives: [{ Transcript: 'done', Items: [] }], + }, + { + ChannelId: 'ch_1', + StartTime: 0, + EndTime: 0.6, + IsPartial: true, + Alternatives: [{ Transcript: 'still going', Items: [] }], + }, + ); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + // One START_OF_SPEECH per channel (not one per result), and exactly one END_OF_SPEECH + // for the channel that actually finished. + expect(events.filter((e) => e.type === stt.SpeechEventType.START_OF_SPEECH)).toHaveLength(2); + expect(events.filter((e) => e.type === stt.SpeechEventType.END_OF_SPEECH)).toHaveLength(1); + }); + + it('does not drop a final transcript whose EndTime is exactly 0', async () => { + const client = fakeClient([ + async function* () { + yield transcriptEvent({ + StartTime: 0, + EndTime: 0, + IsPartial: false, + Alternatives: [{ Transcript: 'ok', Items: [] }], + }); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + expect(events.some((e) => e.alternatives?.[0]?.text === 'ok')).toBe(true); + }); + + it('reconnects silently on an idle timeout instead of surfacing an error', async () => { + let firstSessionStarted = false; + const client = fakeClient([ + async function* () { + firstSessionStarted = true; + const err = new Error('Your request timed out waiting for input'); + err.name = 'BadRequestException'; + throw err; + }, + async function* () { + yield transcriptEvent({ + StartTime: 0, + EndTime: 0.2, + IsPartial: false, + Alternatives: [{ Transcript: 'hi', Items: [] }], + }); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + expect(firstSessionStarted).toBe(true); + expect(events.some((e) => e.alternatives?.[0]?.text === 'hi')).toBe(true); + }); + + it('resets speaking state on an idle-timeout reconnect so START_OF_SPEECH re-fires', async () => { + const client = fakeClient([ + async function* () { + // A partial result mid-utterance leaves speaking state true when the idle timeout hits. + yield transcriptEvent({ + StartTime: 1, + EndTime: 1.5, + IsPartial: true, + Alternatives: [{ Transcript: 'partial', Items: [] }], + }); + const err = new Error('Your request timed out waiting for input'); + err.name = 'BadRequestException'; + throw err; + }, + async function* () { + yield transcriptEvent({ + StartTime: 0.3, + EndTime: 0.6, + IsPartial: false, + Alternatives: [{ Transcript: 'resumed', Items: [] }], + }); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + expect(events.filter((e) => e.type === stt.SpeechEventType.START_OF_SPEECH)).toHaveLength(2); + }); + + it('keeps word/segment timestamps monotonic across an idle-timeout reconnect', async () => { + const client = fakeClient([ + async function* () { + yield transcriptEvent({ + StartTime: 20, + EndTime: 20.5, + IsPartial: false, + Alternatives: [{ Transcript: 'first', Items: [] }], + }); + const err = new Error('Your request timed out waiting for input'); + err.name = 'BadRequestException'; + throw err; + }, + async function* () { + // The new connection's raw clock resets to ~0. + yield transcriptEvent({ + StartTime: 0.3, + EndTime: 0.6, + IsPartial: false, + Alternatives: [{ Transcript: 'second', Items: [] }], + }); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + const finals = events.filter((e) => e.type === stt.SpeechEventType.FINAL_TRANSCRIPT); + expect(finals).toHaveLength(2); + const firstEnd = finals[0]?.alternatives?.[0]?.endTime ?? -Infinity; + const secondStart = finals[1]?.alternatives?.[0]?.startTime ?? -Infinity; + expect(secondStart).toBeGreaterThanOrEqual(firstEnd); + }); + + it('classifies a non-idle-timeout failure as a hard failure the base class retries', async () => { + // The base SpeechStream intentionally stays silent on the 'error' event for recoverable + // retries (only terminal failures emit), so success-after-retry is observed via the + // recovered transcript rather than an event. + let attempts = 0; + const client = { + send: async () => { + attempts += 1; + if (attempts === 1) { + throw new Error('access denied'); + } + return { + TranscriptResultStream: (async function* () { + yield transcriptEvent({ + StartTime: 0, + EndTime: 0.2, + IsPartial: false, + Alternatives: [{ Transcript: 'recovered', Items: [] }], + }); + })(), + }; + }, + } as unknown as TranscribeStreamingClient; + + const speechStream = new STT({ ...baseOpts, client }).stream({ + connOptions: { maxRetry: 1, retryIntervalMs: 1, timeoutMs: 1000 }, + }); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + expect(attempts).toBe(2); + expect(events.some((e) => e.alternatives?.[0]?.text === 'recovered')).toBe(true); + }); + + it('resets speaking state across a hard-failure retry so START_OF_SPEECH re-fires', async () => { + let attempts = 0; + const client = { + send: async () => { + attempts += 1; + if (attempts === 1) { + return { + TranscriptResultStream: (async function* () { + // Mid-utterance (partial, speaking state left true) when the hard failure hits. + yield transcriptEvent({ + StartTime: 1, + EndTime: 1.5, + IsPartial: true, + Alternatives: [{ Transcript: 'partial', Items: [] }], + }); + throw new Error('connection reset'); + })(), + }; + } + return { + TranscriptResultStream: (async function* () { + yield transcriptEvent({ + StartTime: 0.3, + EndTime: 0.6, + IsPartial: false, + Alternatives: [{ Transcript: 'resumed', Items: [] }], + }); + })(), + }; + }, + } as unknown as TranscribeStreamingClient; + + const speechStream = new STT({ ...baseOpts, client }).stream({ + connOptions: { maxRetry: 1, retryIntervalMs: 1, timeoutMs: 1000 }, + }); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + expect(attempts).toBe(2); + expect(events.filter((e) => e.type === stt.SpeechEventType.START_OF_SPEECH)).toHaveLength(2); + const finals = events.filter((e) => e.type === stt.SpeechEventType.FINAL_TRANSCRIPT); + expect(finals).toHaveLength(1); + // The retried session's raw StartTime (0.3) must be pushed past the failed session's + // furthest known point (1.5), not reported as an earlier absolute timestamp. + expect(finals[0]?.alternatives?.[0]?.startTime ?? -Infinity).toBeGreaterThanOrEqual(1.5); + }); + + it('does not lose or duplicate audio frames across an idle-timeout reconnect', async () => { + // Regression test: SpeechStream must own a single pump over `this.input` and hand each + // session a token-guarded generator over a persistent channel — never let an abandoned + // session keep consuming frames meant for the reconnected session (see the #channel field + // comment in stt.ts). + const receivedChunks: Uint8Array[] = []; + let call = 0; + + const client = { + send: async (command: { + input: { AudioStream: AsyncIterable<{ AudioEvent?: { AudioChunk?: Uint8Array } }> }; + }) => { + call += 1; + if (call === 1) { + // The idle timeout fires on the response side without ever draining the + // request-side AudioStream, mirroring how AWS reports it in practice. + return { + TranscriptResultStream: (async function* () { + const err = new Error('Your request timed out waiting for input'); + err.name = 'BadRequestException'; + throw err; + })(), + }; + } + + // Second session: actually drain AudioStream so we can prove every frame + // pushed around the reconnect reaches this session exactly once. + for await (const event of command.input.AudioStream) { + const chunk = event.AudioEvent?.AudioChunk; + if (chunk) receivedChunks.push(chunk); + } + + return { + TranscriptResultStream: (async function* () { + yield transcriptEvent({ + StartTime: 0, + EndTime: 0.2, + IsPartial: false, + Alternatives: [{ Transcript: 'ok', Items: [] }], + }); + })(), + }; + }, + } as unknown as TranscribeStreamingClient; + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.pushFrame(new AudioFrame(new Int16Array([1, 2, 3, 4]), 16000, 1, 4)); + speechStream.endInput(); + + await collect; + + expect(call).toBe(2); + // 1 pushed frame (8 bytes) + the terminal empty chunk, each exactly once — no loss, + // no duplication, and nothing delivered to the abandoned first session. + expect(receivedChunks).toHaveLength(2); + expect(receivedChunks.reduce((sum, c) => sum + c.length, 0)).toBe(8); + expect(receivedChunks.some((c) => c.length === 0)).toBe(true); + expect(events.some((e) => e.alternatives?.[0]?.text === 'ok')).toBe(true); + }); +}); + +describe('AWS Transcribe STT (live)', () => { + if (hasAwsCredentials) { + it('passes the shared STT test harness', { timeout: 30_000 }, async () => { + await sttTest(new STT(), await VAD.load(), { streaming: true, nonStreaming: false }); + }); + } else { + it.skip('requires AWS_ACCESS_KEY_ID or AWS_PROFILE', () => {}); + } +}); diff --git a/plugins/aws/src/stt.ts b/plugins/aws/src/stt.ts new file mode 100644 index 000000000..e99f261a5 --- /dev/null +++ b/plugins/aws/src/stt.ts @@ -0,0 +1,383 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { + Result, + StartStreamTranscriptionCommandInput, + AudioStream as TranscribeAudioStream, + TranscriptEvent, +} from '@aws-sdk/client-transcribe-streaming'; +import { + StartStreamTranscriptionCommand, + TranscribeStreamingClient, +} from '@aws-sdk/client-transcribe-streaming'; +import { + type APIConnectOptions, + APIConnectionError, + AsyncIterableQueue, + type AudioBuffer, + createTimedString, + log, + normalizeLanguage, + stt, +} from '@livekit/agents'; +import { type AwsCredentials, resolveRegion, stripUndefined } from './utils.js'; + +export interface STTOptions { + sampleRate: number; + language?: string; + region?: string; + credentials?: AwsCredentials; + vocabularyName?: string; + sessionId?: string; + vocabFilterMethod?: string; + vocabFilterName?: string; + showSpeakerLabel?: boolean; + enableChannelIdentification?: boolean; + numberOfChannels?: number; + enablePartialResultsStabilization?: boolean; + partialResultsStability?: string; + languageModelName?: string; + identifyLanguage?: boolean; + identifyMultipleLanguages?: boolean; + languageOptions?: string; + preferredLanguage?: string; + vocabularyNames?: string; + vocabularyFilterNames?: string; + client?: TranscribeStreamingClient; +} + +const defaultSTTOptions: Pick = { + sampleRate: 24000, + language: 'en-US', +}; + +/** Invalidated the instant its session ends, so a superseded audio generator stops pulling. */ +interface SessionToken { + active: boolean; +} + +/** Amazon Transcribe reports this after ~15s of stream inactivity; it's not a hard failure. */ +function isIdleTimeout(error: unknown): boolean { + return ( + error instanceof Error && + error.name === 'BadRequestException' && + (error.message ?? '').startsWith('Your request timed out') + ); +} + +/** + * Amazon Transcribe streaming STT. + * + * @remarks + * Streaming only — {@link STT.stream} is the sole recognition entrypoint, matching Amazon + * Transcribe's streaming-only API surface. + */ +export class STT extends stt.STT { + #opts: STTOptions; + #client: TranscribeStreamingClient; + label = 'aws.STT'; + + get model(): string { + return this.#opts.languageModelName ?? 'unknown'; + } + + get provider(): string { + return 'Amazon Transcribe'; + } + + /** + * Create a new instance of Amazon Transcribe streaming STT. + * + * @remarks + * Credentials are resolved via the AWS SDK v3 default credential chain (environment + * variables, shared config/credentials files, IMDS, etc.) unless `credentials` is provided + * explicitly. The region is resolved from `region`, then `AWS_REGION`, then + * `AWS_DEFAULT_REGION`, falling back to `us-east-1`. + */ + constructor(opts: Partial = {}) { + super({ + streaming: true, + interimResults: true, + alignedTranscript: 'word', + }); + + if (opts.identifyLanguage && opts.identifyMultipleLanguages) { + throw new Error( + 'identifyLanguage and identifyMultipleLanguages are mutually exclusive; set only one to true', + ); + } + + const identifyLanguage = opts.identifyLanguage ?? false; + const identifyMultipleLanguages = opts.identifyMultipleLanguages ?? false; + + this.#opts = { + ...defaultSTTOptions, + ...opts, + identifyLanguage, + identifyMultipleLanguages, + // Auto language detection is mutually exclusive with a fixed language code. + language: + identifyLanguage || identifyMultipleLanguages + ? undefined + : opts.language ?? defaultSTTOptions.language, + }; + + this.#client = + opts.client ?? + new TranscribeStreamingClient({ + region: resolveRegion(opts.region), + credentials: opts.credentials, + // The framework's own connOptions retry loop handles retries; disable the SDK's + // internal retries so failures aren't retried twice (matches tts.ts's PollyClient). + maxAttempts: 1, + }); + } + + async _recognize(_frame: AudioBuffer, _abortSignal?: AbortSignal): Promise { + throw new Error( + 'Amazon Transcribe does not support single-frame recognition, use stream() instead', + ); + } + + stream(options?: { connOptions?: APIConnectOptions }): SpeechStream { + return new SpeechStream(this, this.#opts, this.#client, options?.connOptions); + } +} + +export class SpeechStream extends stt.SpeechStream { + #opts: STTOptions; + #client: TranscribeStreamingClient; + #logger = log(); + // Keyed by Transcribe's `ChannelId` (the empty string when channel identification is off, + // i.e. single-channel audio) so a channel finishing its utterance can't spuriously flip + // speaking state for another channel that's still mid-utterance in the same TranscriptEvent. + #speakingChannels = new Set(); + // Transcribe's StartTime/EndTime are cumulative from the start of the current streaming + // *connection*, and reset to ~0 whenever a reconnect opens a new one. This offset + // accumulates the last known end time of the previous connection so timestamps stay + // monotonic across reconnects, on top of `this.startTimeOffset` (which compensates for the + // framework creating a new SpeechStream instance, a separate concern). + #connectionTimeOffset = 0; + #lastKnownEndTime = 0; + label = 'aws.SpeechStream'; + + // `this.input` may only ever have a single active consumer (calling `.next()` from more + // than one place races for frames). Bedrock's Node HTTP/2 request pipeline can leave an + // abandoned session's `AudioStream` generator alive as a hidden consumer after a session + // fails (the SDK doesn't reliably tear down the request side when the response side + // errors), so a single long-lived pump owns `this.input` and feeds a single persistent + // `#channel` for the lifetime of this stream — the channel itself is never replaced, so + // no already-buffered frame can ever be orphaned on a session transition. Each session + // instead reads through a token-guarded wrapper generator: the token is invalidated the + // moment its session ends, so a superseded generator drops (at most) the one frame it may + // already be mid-await on instead of racing the new session for every subsequent frame. + #channel = new AsyncIterableQueue(); + #pumpStarted = false; + + constructor( + stt: STT, + opts: STTOptions, + client: TranscribeStreamingClient, + connOptions?: APIConnectOptions, + ) { + super(stt, opts.sampleRate, connOptions); + this.#opts = opts; + this.#client = client; + } + + #startPump(): void { + if (this.#pumpStarted) return; + this.#pumpStarted = true; + + (async () => { + for (;;) { + const result = await this.input.next(); + if (result.done) { + this.#channel.close(); + return; + } + const frame = result.value; + if (frame === SpeechStream.FLUSH_SENTINEL) continue; + this.#channel.put( + new Uint8Array(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength), + ); + } + })(); + } + + protected async run(): Promise { + this.#startPump(); + + // Always attempt at least one session, even if the input already closed before run() + // was scheduled (e.g. a very short utterance) — otherwise the final transcript would + // never be requested. + for (;;) { + const token: SessionToken = { active: true }; + try { + await this.#runSession(this.#audioStreamGenerator(token)); + } catch (error) { + token.active = false; + + if (this.closed) { + return; + } + + // Any session end (idle timeout, or a hard failure the base class will retry via a + // fresh run() call) means the *next* session — whether started here or by a retried + // run() — starts on a brand-new Transcribe connection with a reset clock and no + // memory of in-flight utterances. Reset unconditionally so state from the failed + // session can't leak into the next one: carry the previous connection's furthest + // timestamp forward so reported times keep increasing, and clear speaking state so + // the next session's first result re-emits START_OF_SPEECH. + this.#connectionTimeOffset += this.#lastKnownEndTime; + this.#lastKnownEndTime = 0; + this.#speakingChannels.clear(); + + if (isIdleTimeout(error)) { + // AWS times out after 15s of inactivity, which tends to happen at the end of a + // session once the input is gone. Reconnect unconditionally, matching the Python + // plugin, rather than gating on input state. + this.#logger.info('aws transcribe stt: idle timeout, restarting session'); + continue; + } + throw new APIConnectionError({ + message: `aws transcribe stt: ${error instanceof Error ? error.message : String(error)}`, + }); + } + + // #runSession only returns without throwing once its audio generator drains + // `#channel` to completion, which only happens after the pump closes it (input + // exhausted) — so there's nothing left to reconnect for. + token.active = false; + return; + } + } + + async #runSession(audioStream: AsyncGenerator): Promise { + const input = stripUndefined({ + LanguageCode: this.#opts.language, + MediaSampleRateHertz: this.#opts.sampleRate, + MediaEncoding: 'pcm', + AudioStream: audioStream, + VocabularyName: this.#opts.vocabularyName, + SessionId: this.#opts.sessionId, + VocabularyFilterName: this.#opts.vocabFilterName, + VocabularyFilterMethod: this.#opts.vocabFilterMethod, + ShowSpeakerLabel: this.#opts.showSpeakerLabel, + EnableChannelIdentification: this.#opts.enableChannelIdentification, + NumberOfChannels: this.#opts.numberOfChannels, + EnablePartialResultsStabilization: this.#opts.enablePartialResultsStabilization, + PartialResultsStability: this.#opts.partialResultsStability, + LanguageModelName: this.#opts.languageModelName, + IdentifyLanguage: this.#opts.identifyLanguage, + IdentifyMultipleLanguages: this.#opts.identifyMultipleLanguages, + LanguageOptions: this.#opts.languageOptions, + PreferredLanguage: this.#opts.preferredLanguage, + VocabularyNames: this.#opts.vocabularyNames, + VocabularyFilterNames: this.#opts.vocabularyFilterNames, + }) as unknown as StartStreamTranscriptionCommandInput; + + const response = await this.#client.send(new StartStreamTranscriptionCommand(input), { + abortSignal: this.abortSignal, + }); + + if (!response.TranscriptResultStream) { + throw new Error('aws transcribe stt: no TranscriptResultStream in the response'); + } + + for await (const event of response.TranscriptResultStream) { + if (event.TranscriptEvent) { + this.#processTranscriptEvent(event.TranscriptEvent); + } + } + } + + async *#audioStreamGenerator(token: SessionToken): AsyncGenerator { + for (;;) { + if (!token.active) return; + const result = await this.#channel.next(); + // The token may have been invalidated while awaiting a frame that belonged to a + // session that has since ended (e.g. an idle-timeout reconnect) — drop it rather + // than yielding it into a request the SDK has already abandoned. + if (!token.active) return; + if (result.done) break; + yield { AudioEvent: { AudioChunk: result.value } }; + } + + // AWS Transcribe requires an empty chunk to signal the end of the audio stream. + if (token.active) { + yield { AudioEvent: { AudioChunk: new Uint8Array(0) } }; + } + } + + #processTranscriptEvent(transcriptEvent: TranscriptEvent): void { + const results = transcriptEvent.Transcript?.Results; + if (!results) return; + + for (const result of results) { + // Transcribe's StartTime/EndTime are cumulative offsets from the start of the whole + // streaming connection, not per-utterance — so `StartTime === 0` is only ever true for + // the very first result of a session. START_OF_SPEECH must instead be driven by our own + // per-channel speaking state so it fires once per utterance, not once per connection, + // and so one channel finishing doesn't affect another channel's speaking state. + const channelKey = result.ChannelId ?? ''; + if (!this.#speakingChannels.has(channelKey)) { + this.#speakingChannels.add(channelKey); + this.queue.put({ type: stt.SpeechEventType.START_OF_SPEECH }); + } + + if (result.EndTime !== undefined) { + this.#lastKnownEndTime = Math.max(this.#lastKnownEndTime, result.EndTime); + const alternative = this.#toSpeechData(result); + this.queue.put({ + type: result.IsPartial + ? stt.SpeechEventType.INTERIM_TRANSCRIPT + : stt.SpeechEventType.FINAL_TRANSCRIPT, + alternatives: [alternative], + }); + } + + if (!result.IsPartial) { + this.#speakingChannels.delete(channelKey); + this.queue.put({ type: stt.SpeechEventType.END_OF_SPEECH }); + } + } + } + + #toSpeechData(result: Result): stt.SpeechData { + const alternative = result.Alternatives?.[0]; + // Transcribe tags punctuation items with no meaningful timestamp/confidence; excluding + // them keeps word-level alignment and utterance confidence limited to spoken words. + const items = alternative?.Items?.filter((item) => item.Type !== 'punctuation'); + const confidence = items?.length + ? items.reduce((sum, item) => sum + (item.Confidence ?? 0), 0) / items.length + : 0; + const detectedLanguage = result.LanguageCode ?? this.#opts.language ?? 'en-US'; + + const sourceLanguages = + (this.#opts.identifyLanguage || this.#opts.identifyMultipleLanguages) && result.LanguageCode + ? [normalizeLanguage(result.LanguageCode)] + : undefined; + + const offset = this.startTimeOffset + this.#connectionTimeOffset; + + return { + language: normalizeLanguage(detectedLanguage), + startTime: (result.StartTime ?? 0) + offset, + endTime: (result.EndTime ?? 0) + offset, + text: alternative?.Transcript ?? '', + confidence, + sourceLanguages, + words: items?.map((item) => + createTimedString({ + text: item.Content ?? '', + startTime: (item.StartTime ?? 0) + offset, + endTime: (item.EndTime ?? 0) + offset, + confidence: item.Confidence ?? 0, + startTimeOffset: offset, + }), + ), + }; + } +} diff --git a/plugins/aws/src/tts.test.ts b/plugins/aws/src/tts.test.ts new file mode 100644 index 000000000..783be6982 --- /dev/null +++ b/plugins/aws/src/tts.test.ts @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { PollyClient } from '@aws-sdk/client-polly'; +import { tts as ttsTest } from '@livekit/agents-plugins-test'; +import { describe, expect, it } from 'vitest'; +import { STT } from './stt.js'; +import { TTS } from './tts.js'; + +const hasAwsCredentials = Boolean(process.env.AWS_ACCESS_KEY_ID || process.env.AWS_PROFILE); + +function pcmBytes(sampleCount: number): Uint8Array { + return new Uint8Array(sampleCount * 2); // 16-bit PCM, mono +} + +function fakeClient(audioBytes: Uint8Array, requestId = 'req_123'): PollyClient { + return { + send: async () => ({ + $metadata: { requestId }, + AudioStream: { + transformToByteArray: async () => audioBytes, + }, + }), + } as unknown as PollyClient; +} + +describe('AWS Polly TTS - constructor', () => { + it('defaults to Ruth, generative engine, and 16000 Hz', () => { + const tts = new TTS({ client: fakeClient(pcmBytes(0)) }); + expect(tts.sampleRate).toBe(16000); + expect(tts.numChannels).toBe(1); + expect(tts.model).toBe('generative'); + expect(tts.provider).toBe('Amazon Polly'); + expect(tts.capabilities.streaming).toBe(false); + }); + + it('accepts 8000 Hz', () => { + const tts = new TTS({ sampleRate: 8000, client: fakeClient(pcmBytes(0)) }); + expect(tts.sampleRate).toBe(8000); + }); + + it('throws for unsupported sample rates', () => { + expect(() => new TTS({ sampleRate: 24000 })).toThrow(/8000 or 16000/); + }); + + it('throws when streaming is requested', () => { + const tts = new TTS({ client: fakeClient(pcmBytes(0)) }); + expect(() => tts.stream()).toThrow(/Streaming is not supported/); + }); + + it('merges updateOptions onto the existing options', () => { + const tts = new TTS({ client: fakeClient(pcmBytes(0)) }); + tts.updateOptions({ voice: 'Matthew', speechEngine: 'neural' }); + expect(tts.model).toBe('neural'); + }); +}); + +describe('AWS Polly TTS - synthesis', () => { + it('converts the PCM response into audio frames', async () => { + const tts = new TTS({ sampleRate: 16000, client: fakeClient(pcmBytes(1600)) }); + const stream = tts.synthesize('hello world'); + + const events = []; + for await (const event of stream) { + events.push(event); + } + + expect(events.length).toBeGreaterThan(0); + expect(events.at(-1)?.final).toBe(true); + for (const event of events) { + expect(event.frame.sampleRate).toBe(16000); + expect(event.frame.channels).toBe(1); + // A trailing empty flush() frame must never become the reported final frame. + expect(event.frame.samplesPerChannel).toBeGreaterThan(0); + } + }); + + it('does not emit an empty final frame when the PCM divides evenly into 100ms frames', async () => { + // 1600 samples at 16000 Hz is exactly one 100ms frame, leaving nothing buffered for + // flush() — flush() still returns a 0-sample frame in that case, which must be dropped. + const tts = new TTS({ sampleRate: 16000, client: fakeClient(pcmBytes(1600)) }); + const stream = tts.synthesize('hi'); + + const events = []; + for await (const event of stream) { + events.push(event); + } + + expect(events).toHaveLength(1); + expect(events[0]?.final).toBe(true); + expect(events[0]?.frame.samplesPerChannel).toBe(1600); + }); +}); + +describe('AWS Polly TTS (live)', () => { + if (hasAwsCredentials) { + it('passes the shared TTS test harness', async () => { + await ttsTest(new TTS(), new STT(), { streaming: false }); + }); + } else { + it.skip('requires AWS_ACCESS_KEY_ID or AWS_PROFILE', () => {}); + } +}); diff --git a/plugins/aws/src/tts.ts b/plugins/aws/src/tts.ts new file mode 100644 index 000000000..5aada18af --- /dev/null +++ b/plugins/aws/src/tts.ts @@ -0,0 +1,197 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { SynthesizeSpeechCommandInput } from '@aws-sdk/client-polly'; +import { PollyClient, SynthesizeSpeechCommand } from '@aws-sdk/client-polly'; +import { + type APIConnectOptions, + APIConnectionError, + APITimeoutError, + AudioByteStream, + shortuuid, + tts, +} from '@livekit/agents'; +import type { AudioFrame } from '@livekit/rtc-node'; +import type { TTSLanguage, TTSSpeechEngine, TTSTextType } from './models.js'; +import { type AwsCredentials, resolveRegion, stripUndefined } from './utils.js'; + +/** Amazon Polly only returns PCM (16-bit little-endian, mono) at these sample rates. */ +const SUPPORTED_PCM_SAMPLE_RATES = [8000, 16000]; + +export interface TTSOptions { + voice: string; + speechEngine: TTSSpeechEngine; + textType: TTSTextType; + language?: TTSLanguage | string; + sampleRate: number; + region?: string; + credentials?: AwsCredentials; + client?: PollyClient; +} + +const defaultTTSOptions: Omit = { + voice: 'Ruth', + speechEngine: 'generative', + textType: 'text', +}; + +/** + * Amazon Polly TTS. + * + * @remarks + * Audio is requested as raw PCM (16-bit little-endian, mono) since the framework has no mp3 + * decoder. Amazon Polly only supports PCM output at 8000 Hz or 16000 Hz, so `sampleRate` is + * restricted to those two values. + */ +export class TTS extends tts.TTS { + #opts: TTSOptions; + #client: PollyClient; + label = 'aws.TTS'; + private abortController = new AbortController(); + + get model(): string { + return this.#opts.speechEngine; + } + + get provider(): string { + return 'Amazon Polly'; + } + + /** + * Create a new instance of AWS Polly TTS. + * + * @remarks + * Credentials are resolved via the AWS SDK v3 default credential chain (environment + * variables, shared config/credentials files, IMDS, etc.) unless `credentials` is provided + * explicitly. The region is resolved from `region`, then `AWS_REGION`, then + * `AWS_DEFAULT_REGION`, falling back to `us-east-1`. + */ + constructor(opts: Partial = {}) { + const sampleRate = opts.sampleRate ?? 16000; + if (!SUPPORTED_PCM_SAMPLE_RATES.includes(sampleRate)) { + throw new Error( + `AWS Polly TTS only supports PCM output at ${SUPPORTED_PCM_SAMPLE_RATES.join(' or ')} Hz sample rates, got ${sampleRate}`, + ); + } + + super(sampleRate, 1, { streaming: false }); + + this.#opts = { + ...defaultTTSOptions, + ...opts, + sampleRate, + }; + + this.#client = + opts.client ?? + new PollyClient({ + region: resolveRegion(opts.region), + credentials: opts.credentials, + maxAttempts: 1, + }); + } + + updateOptions(opts: { + voice?: string; + language?: TTSLanguage | string; + speechEngine?: TTSSpeechEngine; + textType?: TTSTextType; + }) { + this.#opts = { ...this.#opts, ...opts }; + } + + synthesize( + text: string, + connOptions?: APIConnectOptions, + abortSignal?: AbortSignal, + ): ChunkedStream { + const signal = abortSignal + ? AbortSignal.any([abortSignal, this.abortController.signal]) + : this.abortController.signal; + return new ChunkedStream(this, text, this.#client, this.#opts, connOptions, signal); + } + + stream(): tts.SynthesizeStream { + throw new Error('Streaming is not supported on AWS Polly TTS'); + } + + async close(): Promise { + this.abortController.abort(); + } +} + +export class ChunkedStream extends tts.ChunkedStream { + label = 'aws.ChunkedStream'; + #client: PollyClient; + #opts: TTSOptions; + + constructor( + tts: TTS, + text: string, + client: PollyClient, + opts: TTSOptions, + connOptions?: APIConnectOptions, + abortSignal?: AbortSignal, + ) { + super(text, tts, connOptions, abortSignal); + this.#client = client; + this.#opts = opts; + } + + protected async run() { + try { + const input = stripUndefined({ + Text: this.inputText, + OutputFormat: 'pcm', + Engine: this.#opts.speechEngine, + VoiceId: this.#opts.voice, + TextType: this.#opts.textType, + SampleRate: String(this.#opts.sampleRate), + LanguageCode: this.#opts.language, + }) as unknown as SynthesizeSpeechCommandInput; + + const response = await this.#client.send(new SynthesizeSpeechCommand(input), { + abortSignal: this.abortSignal, + }); + + if (!response.AudioStream) { + throw new Error('aws polly tts: no AudioStream in the response'); + } + + const requestId = response.$metadata.requestId ?? shortuuid(); + const audioBytes = await response.AudioStream.transformToByteArray(); + const audioByteStream = new AudioByteStream(this.#opts.sampleRate, 1); + + let lastFrame: AudioFrame | undefined; + const sendLastFrame = (segmentId: string, final: boolean) => { + if (lastFrame) { + this.queue.put({ requestId, segmentId, frame: lastFrame, final }); + lastFrame = undefined; + } + }; + + // write() only emits complete 100ms frames; flush() drains the trailing remainder (up + // to 100ms), otherwise the tail of the synthesized audio is silently dropped. flush() + // always returns one frame even with nothing buffered (0 samples), so drop it rather + // than let it become the "final" frame in place of the real last frame from write(). + const remainder = audioByteStream.flush().filter((frame) => frame.samplesPerChannel > 0); + for (const frame of [...audioByteStream.write(audioBytes), ...remainder]) { + sendLastFrame(requestId, false); + lastFrame = frame; + } + sendLastFrame(requestId, true); + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + return; + } + if (error instanceof Error && error.name === 'TimeoutError') { + throw new APITimeoutError({ message: error.message }); + } + throw new APIConnectionError({ + message: error instanceof Error ? error.message : String(error), + }); + } finally { + this.queue.close(); + } + } +} diff --git a/plugins/aws/src/utils.ts b/plugins/aws/src/utils.ts new file mode 100644 index 000000000..8f91c49a5 --- /dev/null +++ b/plugins/aws/src/utils.ts @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +export const DEFAULT_REGION = 'us-east-1'; + +/** Explicit static AWS credentials. When omitted, the AWS SDK v3 default credential chain is used. */ +export interface AwsCredentials { + accessKeyId: string; + secretAccessKey: string; + sessionToken?: string; +} + +/** + * Resolves the AWS region to use, in order of precedence: + * the explicit `region` argument, `AWS_REGION`, `AWS_DEFAULT_REGION`, then {@link DEFAULT_REGION}. + */ +export function resolveRegion(region?: string): string { + return region ?? process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION ?? DEFAULT_REGION; +} + +/** Removes `undefined`-valued keys so they aren't sent to AWS SDK calls. */ +export function stripUndefined>(obj: T): T { + return Object.fromEntries(Object.entries(obj).filter(([, value]) => value !== undefined)) as T; +} diff --git a/plugins/aws/tsconfig.json b/plugins/aws/tsconfig.json new file mode 100644 index 000000000..50be4b771 --- /dev/null +++ b/plugins/aws/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../tsconfig.json", + "include": [ + "./src" + ], + "compilerOptions": { + // match output dir to input dir. e.g. dist/index instead of dist/src/index + "rootDir": "./src", + "declarationDir": "./dist", + "outDir": "./dist" + }, + "typedocOptions": { + "name": "plugins/agents-plugin-aws", + "entryPointStrategy": "resolve", + "entryPoints": [ + "src/index.ts" + ] + } +} diff --git a/plugins/aws/tsup.config.ts b/plugins/aws/tsup.config.ts new file mode 100644 index 000000000..8ca20961f --- /dev/null +++ b/plugins/aws/tsup.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'tsup'; + +import defaults from '../../tsup.config'; + +export default defineConfig({ + ...defaults, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f00f5059b..280cbf934 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -251,6 +251,9 @@ importers: '@livekit/agents-plugin-assemblyai': specifier: workspace:* version: link:../plugins/assemblyai + '@livekit/agents-plugin-aws': + specifier: workspace:* + version: link:../plugins/aws '@livekit/agents-plugin-baseten': specifier: workspace:* version: link:../plugins/baseten @@ -478,6 +481,43 @@ importers: specifier: ^5.0.0 version: 5.9.3 + plugins/aws: + dependencies: + '@aws-sdk/client-bedrock-runtime': + specifier: ^3.1081.0 + version: 3.1081.0 + '@aws-sdk/client-polly': + specifier: ^3.1081.0 + version: 3.1081.0 + '@aws-sdk/client-transcribe-streaming': + specifier: ^3.1081.0 + version: 3.1081.0 + devDependencies: + '@livekit/agents': + specifier: workspace:* + version: link:../../agents + '@livekit/agents-plugin-silero': + specifier: workspace:* + version: link:../silero + '@livekit/agents-plugins-test': + specifier: workspace:* + version: link:../test + '@livekit/rtc-node': + specifier: 'catalog:' + version: 0.13.31 + '@microsoft/api-extractor': + specifier: ^7.35.0 + version: 7.43.7(@types/node@25.6.0) + tsup: + specifier: ^8.3.5 + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@25.6.0))(postcss@8.5.9)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.0.0 + version: 5.9.3 + zod: + specifier: ^3.25.76 || ^4.1.8 + version: 4.3.6 + plugins/baseten: dependencies: dotenv: @@ -1450,6 +1490,94 @@ packages: '@anthropic-ai/sdk@0.33.1': resolution: {integrity: sha512-VrlbxiAdVRGuKP2UQlCnsShDHJKWepzvfRCkZMpU+oaUdKLpOfmylLMRojGrAgebV+kDtPjewCVP0laHXg+vsA==} + '@aws-sdk/client-bedrock-runtime@3.1081.0': + resolution: {integrity: sha512-rRAGXY5qV/NCYbVA0QZHPierv3diOOiE4+1f5vedpbyvg7Phh9m/I2pRFwu0koUtMdRSGPWgoNcvMIyaDhDj7Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-polly@3.1081.0': + resolution: {integrity: sha512-nH7X4X9qQmIT3HNMi2LJDu3pVqY6csPwbTbhZ2DXFqVoY2HjGAt+iihWWYoZCrI0d64k8PxPIrqgwAJW40NlcQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-transcribe-streaming@3.1081.0': + resolution: {integrity: sha512-0ToKz8K86yXNNsQAln9x55Hsl31AH+hXssNvg52/Szr3/1iR3F0kJrX0QdWzOBDAn/lrUtpFXGEPn8cfPIGpXQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/core@3.974.29': + resolution: {integrity: sha512-yqKcltLbtRh1ubzhRSldIs8jFHNZlyMlgoIccCC0aDVbrB99nXaBdmfr89mK7obWX/NVg4rAMpCpZ6dCDiVBtA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-env@3.972.55': + resolution: {integrity: sha512-Ah36tYkqyaVnaHkx7VseoTYrHUmwgBps3V+wnrC1idhIIMGlviH0FtrX9EIPdAlVHvXC7FQZLhmHBRz+pLaiWg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-http@3.972.57': + resolution: {integrity: sha512-/vp6i5YEliJqRm5k/BDmYjAyRAMTdkjW6UciVRk9oh/0OfDCWeb/ih7hqte4lFvKXkIbsqe9AdK9LQK6NGardw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.972.62': + resolution: {integrity: sha512-pQIRiQQs+MUlVnJdWJ7/6KS0WxcLRVfut57OFgwC3cnM1F8mXw3Kh4gAVwj6AtvD6CWx8x6+po4ENRcqe64XrQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-login@3.972.61': + resolution: {integrity: sha512-jtrxWwC7slqxh7DnAWHrwsA3UwCsnlypdYtavGT7EX5p791wxWQys7QzkCZ7JvOMAyylDtPoxyV+ic0zg3rV9g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-node@3.972.64': + resolution: {integrity: sha512-zyKVYDyMR9VQL/kPi03ygN2vtD9uLMuWRLoJ77KxgZZaS1VlJloI+SzleF9Zg4HWUI+AIu+ZRs8zsJFNqbrxsw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-process@3.972.55': + resolution: {integrity: sha512-x0XjjF0l1WGRtK2vEhTZqCguQuAIZLep9l2+eeEmuxQQjjD3BlGQXY5xADR+l3t576UX+dxRkRtTjEu40l81Vw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.972.61': + resolution: {integrity: sha512-d/V0VRsz73i+PHhbult/tx0Y1+de1SNQVsXkcQCmpfeBq7uODy/RTxNsOLpT9ZVHxcRNzbQFuywLKC33fUMIxA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.972.61': + resolution: {integrity: sha512-Bv4n3NOI6hPy+rmr6Bw9R6LnBVRkcp3ncj2E2IKSYJG+0UkysSitWMvbgndNvMxDw7gE1pQ/ErwkNceuKwj7zQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/eventstream-handler-node@3.972.25': + resolution: {integrity: sha512-df7HN1ozwMrB9+59re9PM7tSLxLAcheMWc5u/KyfCPCAWtN/vP7y7RTUZOy48uT1K9MESisVeOPPzF3O1AW01A==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-eventstream@3.972.21': + resolution: {integrity: sha512-HvLgDnxBLaHi9E5K++6Vuk+1+qqn7Pmn8zrlzd+NXH3jBzwujnuzZtAR9WHPkbUGPO92FkoQWj/M1IsdxTlBmQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-sdk-transcribe-streaming@3.972.24': + resolution: {integrity: sha512-rLtH6jAHAgcLYFhCShgFGuxHgFNrbGZdryJB1mc1tmBCC8Hb/Dx1DZopwKPGFV9RvdtlhZCaNQpYW5Y+bSuSCQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-websocket@3.972.37': + resolution: {integrity: sha512-u4J2KwTe6hr0hBrcKF7vPNxoQoPdSwdhE8mEQK/ffaY/XgYQ77NRqsbxeNQDaRthQJ3D3KhOdCjrio1E+/ocng==} + engines: {node: '>= 14.0.0'} + + '@aws-sdk/nested-clients@3.997.29': + resolution: {integrity: sha512-ot6v8J5W8P0w6ryyuIkXP1bHZHTlvwtn83mVCYaBE0GJ6tJX4vPSBx7M98w9O4wmmDruFsDBUMjhEHA+OosUFQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.996.38': + resolution: {integrity: sha512-C379Sk+MiFZCfWZphKlMyLHKxV22OjoGM5KJjj5IJNJcOCWL4IGIpnEGzv1FQiRwhYXfq55SJMfxlqPE08JJ9g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1081.0': + resolution: {integrity: sha512-kduAeI6cL+zqwj3gjPh9LhuX7kBZ83msYxutavaR+UPm5K8J7iThJBvNRAsFNyWTji92CSU8dogUgvi9T0BehA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.973.15': + resolution: {integrity: sha512-IULn8uBV/SMtmOIANsm4WHXIOtVPBWfOWs3WGL0j/sI+KhaYehvOw0ET+9urnn8MBpiijuU/0JOpuwKOE451PQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/xml-builder@3.972.33': + resolution: {integrity: sha512-ezbwz9WpuLctm6o7P2t2naDhVVPI5jFGrVefVybhcKGjU57VIyT46pQVO0RI2RYkUdhdj2Z9uSIlAzGZE9NW9A==} + engines: {node: '>=20.0.0'} + + '@aws/lambda-invoke-store@0.3.0': + resolution: {integrity: sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==} + engines: {node: '>=18.0.0'} + '@babel/code-frame@7.24.2': resolution: {integrity: sha512-y5+tLQyV8pg3fsiln67BVLD1P13Eg4lh5RW9mF0zUuvLrv9uIQ4MCL+CRT+FTsBlBjcIan6PGsLcBN0m3ClUyQ==} engines: {node: '>=6.9.0'} @@ -2716,6 +2844,30 @@ packages: '@rushstack/ts-command-line@4.21.0': resolution: {integrity: sha512-z38FLUCn8M9FQf19gJ9eltdwkvc47PxvJmVZS6aKwbBAa3Pis3r3A+ZcBCVPNb9h/Tbga+i0tHdzoSGUoji9GQ==} + '@smithy/core@3.29.1': + resolution: {integrity: sha512-qoiY4nrk5OCu1+eIR1VB8l5DmON/oKiqrd5zZFAhXJXjJlLWQusKEW/SkBDAtGDcPaz86m9kfcE1lngU0GlM6A==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.4.6': + resolution: {integrity: sha512-B2WQ/PV/H6Jeg3lrIq6bKUfa6Hy01mtK7CGs6lhjzHA6k4aagldH6T6eEjnzKl4HI0cJnAsxfJ19pgb5PV+CVQ==} + engines: {node: '>=18.0.0'} + + '@smithy/fetch-http-handler@5.6.3': + resolution: {integrity: sha512-CwCc/7SMTj45y97MUnDTbTaxvtAsiNNRm81z3abROIuMbMsC2Iy5EKfkkVdsKrz8WExQAAMx1EJapq+9j4fFTQ==} + engines: {node: '>=18.0.0'} + + '@smithy/node-http-handler@4.9.3': + resolution: {integrity: sha512-qZTa4gQFUo8RM02rk6q5UVTDLNrQ1oS20LsepBzqq1QBVc/EHJ03OOUADcqMZiXHArW+Y7+OGY0BpdTwZRq/Yg==} + engines: {node: '>=18.0.0'} + + '@smithy/signature-v4@5.6.2': + resolution: {integrity: sha512-QgHflghMoPxCJ9axiCVh8KZfbC9fuP6vkXXyK//E3cq7nLaSSyyLj0GAoqVWezYeDQmXIZhmlRvLE16jsqDK6g==} + engines: {node: '>=18.0.0'} + + '@smithy/types@4.15.1': + resolution: {integrity: sha512-x3L0XSACF6UYzKpa9biqiRMgvH5+wnFFew9Tm/grFYqgaupPwx/+ojDPpPJM8dZON3S9tjz5U+PQYsCBd1Mw5Q==} + engines: {node: '>=18.0.0'} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -3071,6 +3223,9 @@ packages: resolution: {integrity: sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==} deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. + bowser@2.14.1: + resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} + brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} @@ -5277,6 +5432,214 @@ snapshots: transitivePeerDependencies: - encoding + '@aws-sdk/client-bedrock-runtime@3.1081.0': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/credential-provider-node': 3.972.64 + '@aws-sdk/eventstream-handler-node': 3.972.25 + '@aws-sdk/middleware-eventstream': 3.972.21 + '@aws-sdk/middleware-websocket': 3.972.37 + '@aws-sdk/token-providers': 3.1081.0 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/fetch-http-handler': 5.6.3 + '@smithy/node-http-handler': 4.9.3 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/client-polly@3.1081.0': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/credential-provider-node': 3.972.64 + '@aws-sdk/eventstream-handler-node': 3.972.25 + '@aws-sdk/middleware-eventstream': 3.972.21 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/fetch-http-handler': 5.6.3 + '@smithy/node-http-handler': 4.9.3 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/client-transcribe-streaming@3.1081.0': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/credential-provider-node': 3.972.64 + '@aws-sdk/eventstream-handler-node': 3.972.25 + '@aws-sdk/middleware-eventstream': 3.972.21 + '@aws-sdk/middleware-sdk-transcribe-streaming': 3.972.24 + '@aws-sdk/middleware-websocket': 3.972.37 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/fetch-http-handler': 5.6.3 + '@smithy/node-http-handler': 4.9.3 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/core@3.974.29': + dependencies: + '@aws-sdk/types': 3.973.15 + '@aws-sdk/xml-builder': 3.972.33 + '@aws/lambda-invoke-store': 0.3.0 + '@smithy/core': 3.29.1 + '@smithy/signature-v4': 5.6.2 + '@smithy/types': 4.15.1 + bowser: 2.14.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-env@3.972.55': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-http@3.972.57': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/fetch-http-handler': 5.6.3 + '@smithy/node-http-handler': 4.9.3 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-ini@3.972.62': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/credential-provider-env': 3.972.55 + '@aws-sdk/credential-provider-http': 3.972.57 + '@aws-sdk/credential-provider-login': 3.972.61 + '@aws-sdk/credential-provider-process': 3.972.55 + '@aws-sdk/credential-provider-sso': 3.972.61 + '@aws-sdk/credential-provider-web-identity': 3.972.61 + '@aws-sdk/nested-clients': 3.997.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/credential-provider-imds': 4.4.6 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-login@3.972.61': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/nested-clients': 3.997.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-node@3.972.64': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.55 + '@aws-sdk/credential-provider-http': 3.972.57 + '@aws-sdk/credential-provider-ini': 3.972.62 + '@aws-sdk/credential-provider-process': 3.972.55 + '@aws-sdk/credential-provider-sso': 3.972.61 + '@aws-sdk/credential-provider-web-identity': 3.972.61 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/credential-provider-imds': 4.4.6 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-process@3.972.55': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-sso@3.972.61': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/nested-clients': 3.997.29 + '@aws-sdk/token-providers': 3.1081.0 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-web-identity@3.972.61': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/nested-clients': 3.997.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/eventstream-handler-node@3.972.25': + dependencies: + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/middleware-eventstream@3.972.21': + dependencies: + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/middleware-sdk-transcribe-streaming@3.972.24': + dependencies: + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/middleware-websocket@3.972.37': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/fetch-http-handler': 5.6.3 + '@smithy/signature-v4': 5.6.2 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/nested-clients@3.997.29': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/signature-v4-multi-region': 3.996.38 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/fetch-http-handler': 5.6.3 + '@smithy/node-http-handler': 4.9.3 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/signature-v4-multi-region@3.996.38': + dependencies: + '@aws-sdk/types': 3.973.15 + '@smithy/signature-v4': 5.6.2 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/token-providers@3.1081.0': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/nested-clients': 3.997.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/types@3.973.15': + dependencies: + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/xml-builder@3.972.33': + dependencies: + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws/lambda-invoke-store@0.3.0': {} + '@babel/code-frame@7.24.2': dependencies: '@babel/highlight': 7.24.5 @@ -6524,6 +6887,39 @@ snapshots: transitivePeerDependencies: - '@types/node' + '@smithy/core@3.29.1': + dependencies: + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@smithy/credential-provider-imds@4.4.6': + dependencies: + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@smithy/fetch-http-handler@5.6.3': + dependencies: + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@smithy/node-http-handler@4.9.3': + dependencies: + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@smithy/signature-v4@5.6.2': + dependencies: + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@smithy/types@4.15.1': + dependencies: + tslib: 2.6.2 + '@standard-schema/spec@1.1.0': {} '@trivago/prettier-plugin-sort-imports@4.3.0(prettier@3.2.5)': @@ -6928,6 +7324,8 @@ snapshots: boolean@3.2.0: {} + bowser@2.14.1: {} + brace-expansion@1.1.11: dependencies: balanced-match: 1.0.2 diff --git a/turbo.json b/turbo.json index e3d0866ae..848bddcbe 100644 --- a/turbo.json +++ b/turbo.json @@ -2,11 +2,18 @@ "$schema": "https://turborepo.org/schema.json", "globalEnv": [ "ASSEMBLYAI_API_KEY", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_REGION", + "AWS_DEFAULT_REGION", + "AWS_PROFILE", "AZURE_API_KEY", "AZURE_OPENAI_API_KEY", "AZURE_OPENAI_DEPLOYMENT", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_ENTRA_TOKEN", + "BEDROCK_INFERENCE_PROFILE_ARN", "BEY_API_KEY", "BEY_API_URL", "BEY_AVATAR_ID", From f4c34bc03a44f4987d9a2260585ea86604fc4b8e Mon Sep 17 00:00:00 2001 From: Sora Morimoto Date: Thu, 9 Jul 2026 03:41:22 +0900 Subject: [PATCH 02/13] fix(aws): flush trailing PCM bytes after write in Polly TTS flush() ran before write(audioBytes), always draining an empty buffer and dropping the tail of synthesized audio when Polly's byte length isn't an exact 100ms multiple. --- plugins/aws/src/tts.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/aws/src/tts.ts b/plugins/aws/src/tts.ts index 5aada18af..6e8871b59 100644 --- a/plugins/aws/src/tts.ts +++ b/plugins/aws/src/tts.ts @@ -174,8 +174,9 @@ export class ChunkedStream extends tts.ChunkedStream { // to 100ms), otherwise the tail of the synthesized audio is silently dropped. flush() // always returns one frame even with nothing buffered (0 samples), so drop it rather // than let it become the "final" frame in place of the real last frame from write(). + const frames = audioByteStream.write(audioBytes); const remainder = audioByteStream.flush().filter((frame) => frame.samplesPerChannel > 0); - for (const frame of [...audioByteStream.write(audioBytes), ...remainder]) { + for (const frame of [...frames, ...remainder]) { sendLastFrame(requestId, false); lastFrame = frame; } From da90e8f160874725b750de62e19fdb3bacbc9874 Mon Sep 17 00:00:00 2001 From: Sora Morimoto Date: Thu, 9 Jul 2026 04:08:55 +0900 Subject: [PATCH 03/13] fix(aws): harden Polly/Transcribe error handling and language options Map SDK status errors to APIStatusError so non-retryable 4xx inputs are not retried as connection failures, surface Transcribe event-stream exceptions, and require languageOptions when automatic language identification is enabled. --- plugins/aws/README.md | 12 +++++ plugins/aws/src/stt.test.ts | 95 ++++++++++++++++++++++++++++++++++++- plugins/aws/src/stt.ts | 82 ++++++++++++++++++++++++++++++-- plugins/aws/src/tts.test.ts | 27 +++++++++++ plugins/aws/src/tts.ts | 10 ++-- plugins/aws/src/utils.ts | 47 ++++++++++++++++++ 6 files changed, 261 insertions(+), 12 deletions(-) diff --git a/plugins/aws/README.md b/plugins/aws/README.md index 18d5a07c7..f56672ef0 100644 --- a/plugins/aws/README.md +++ b/plugins/aws/README.md @@ -68,6 +68,18 @@ const stt = new aws.STT({ }); ``` +Automatic language identification requires a companion `languageOptions` list +(comma-separated codes Amazon Transcribe should consider). AWS rejects the +request without it: + +```ts +const stt = new aws.STT({ + identifyLanguage: true, + languageOptions: 'en-US,es-US', + preferredLanguage: 'en-US', // optional +}); +``` + ### TTS Uses [Amazon Polly](https://docs.aws.amazon.com/polly/latest/dg/API_SynthesizeSpeech.html). diff --git a/plugins/aws/src/stt.test.ts b/plugins/aws/src/stt.test.ts index 4475573be..6344a75c2 100644 --- a/plugins/aws/src/stt.test.ts +++ b/plugins/aws/src/stt.test.ts @@ -2,14 +2,24 @@ // // SPDX-License-Identifier: Apache-2.0 import type { TranscribeStreamingClient } from '@aws-sdk/client-transcribe-streaming'; -import { stt } from '@livekit/agents'; +import { APIError, APIStatusError, stt } from '@livekit/agents'; import { VAD } from '@livekit/agents-plugin-silero'; import { stt as sttTest } from '@livekit/agents-plugins-test'; import { AudioFrame } from '@livekit/rtc-node'; -import { describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { STT } from './stt.js'; import type { STTOptions } from './stt.js'; +// Failure-path tests drive SpeechStream.mainTask, which re-throws after +// emitting the STT 'error' event (the rethrow only drives `.finally()` cleanup). +// That produces a by-design floating rejection; swallow expected API errors. +const swallowExpectedRejection = (reason: unknown) => { + if (reason instanceof APIError) return; + throw reason; +}; +beforeAll(() => process.on('unhandledRejection', swallowExpectedRejection)); +afterAll(() => void process.off('unhandledRejection', swallowExpectedRejection)); + const hasAwsCredentials = Boolean(process.env.AWS_ACCESS_KEY_ID || process.env.AWS_PROFILE); const baseOpts: Partial = { sampleRate: 16000, language: 'en-US' }; @@ -44,6 +54,20 @@ describe('AWS Transcribe STT - constructor', () => { ); }); + it('throws when identifyLanguage is set without languageOptions', () => { + expect(() => new STT({ identifyLanguage: true })).toThrow(/languageOptions is required/); + }); + + it('throws when identifyMultipleLanguages is set without languageOptions', () => { + expect(() => new STT({ identifyMultipleLanguages: true })).toThrow( + /languageOptions is required/, + ); + }); + + it('accepts identifyLanguage when languageOptions is provided', () => { + expect(() => new STT({ identifyLanguage: true, languageOptions: 'en-US,es-US' })).not.toThrow(); + }); + it('reports streaming-only capabilities with word-aligned transcripts', () => { const sttInstance = new STT(); expect(sttInstance.capabilities).toEqual({ @@ -353,6 +377,73 @@ describe('AWS Transcribe STT - SpeechStream event mapping', () => { expect(secondStart).toBeGreaterThanOrEqual(firstEnd); }); + it('surfaces a BadRequestException stream event as a non-retryable APIStatusError', async () => { + const client = fakeClient([ + async function* () { + yield { BadRequestException: { Message: 'Invalid sample rate' } }; + }, + ]); + + const sttInstance = new STT({ ...baseOpts, client }); + // Fatal errors surface on the STT instance, not through the event iterator. + const errorEvent = new Promise<{ error: Error; recoverable: boolean }>((resolve) => { + sttInstance.on('error', (event) => resolve(event)); + }); + + const speechStream = sttInstance.stream({ + connOptions: { maxRetry: 0, retryIntervalMs: 1, timeoutMs: 1000 }, + }); + const drain = (async () => { + for await (const _event of speechStream) { + // discard + } + })(); + + speechStream.endInput(); + + const { error, recoverable } = await errorEvent; + expect(error).toBeInstanceOf(APIStatusError); + const statusError = error as APIStatusError; + expect(statusError.statusCode).toBe(400); + expect(statusError.retryable).toBe(false); + expect(statusError.message).toMatch(/Invalid sample rate/); + expect(recoverable).toBe(false); + + await drain.catch(() => {}); + }); + + it('reconnects when idle timeout arrives as a BadRequestException stream event', async () => { + const client = fakeClient([ + async function* () { + yield { + BadRequestException: { Message: 'Your request timed out waiting for input' }, + }; + }, + async function* () { + yield transcriptEvent({ + StartTime: 0, + EndTime: 0.2, + IsPartial: false, + Alternatives: [{ Transcript: 'hi', Items: [] }], + }); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + expect(events.some((e) => e.alternatives?.[0]?.text === 'hi')).toBe(true); + }); + it('classifies a non-idle-timeout failure as a hard failure the base class retries', async () => { // The base SpeechStream intentionally stays silent on the 'error' event for recoverable // retries (only terminal failures emit), so success-after-retry is observed via the diff --git a/plugins/aws/src/stt.ts b/plugins/aws/src/stt.ts index e99f261a5..c80e6ce3e 100644 --- a/plugins/aws/src/stt.ts +++ b/plugins/aws/src/stt.ts @@ -6,6 +6,7 @@ import type { StartStreamTranscriptionCommandInput, AudioStream as TranscribeAudioStream, TranscriptEvent, + TranscriptResultStream, } from '@aws-sdk/client-transcribe-streaming'; import { StartStreamTranscriptionCommand, @@ -13,7 +14,7 @@ import { } from '@aws-sdk/client-transcribe-streaming'; import { type APIConnectOptions, - APIConnectionError, + APIStatusError, AsyncIterableQueue, type AudioBuffer, createTimedString, @@ -21,7 +22,7 @@ import { normalizeLanguage, stt, } from '@livekit/agents'; -import { type AwsCredentials, resolveRegion, stripUndefined } from './utils.js'; +import { type AwsCredentials, resolveRegion, stripUndefined, toAwsApiError } from './utils.js'; export interface STTOptions { sampleRate: number; @@ -40,6 +41,11 @@ export interface STTOptions { languageModelName?: string; identifyLanguage?: boolean; identifyMultipleLanguages?: boolean; + /** + * Comma-separated language codes Amazon Transcribe should consider when automatic + * language identification is enabled. Required when {@link identifyLanguage} or + * {@link identifyMultipleLanguages} is true (AWS rejects the request without it). + */ languageOptions?: string; preferredLanguage?: string; vocabularyNames?: string; @@ -47,6 +53,19 @@ export interface STTOptions { client?: TranscribeStreamingClient; } +/** + * HTTP status codes for exception members of {@link TranscriptResultStream}. These arrive + * as event-stream union members rather than thrown SDK errors, so they have no + * `$metadata.httpStatusCode` to read. + */ +const STREAM_EXCEPTION_STATUS: Record = { + BadRequestException: 400, + LimitExceededException: 429, + InternalFailureException: 500, + ConflictException: 409, + ServiceUnavailableException: 503, +}; + const defaultSTTOptions: Pick = { sampleRate: 24000, language: 'en-US', @@ -111,6 +130,15 @@ export class STT extends stt.STT { const identifyLanguage = opts.identifyLanguage ?? false; const identifyMultipleLanguages = opts.identifyMultipleLanguages ?? false; + // StartStreamTranscription requires LanguageOptions whenever either identify flag is set; + // without it AWS returns BadRequest, so fail fast at construction with a clear message. + if ((identifyLanguage || identifyMultipleLanguages) && !opts.languageOptions) { + throw new Error( + 'languageOptions is required when identifyLanguage or identifyMultipleLanguages is true ' + + '(comma-separated language codes Amazon Transcribe should consider, e.g. "en-US,es-US")', + ); + } + this.#opts = { ...defaultSTTOptions, ...opts, @@ -241,9 +269,9 @@ export class SpeechStream extends stt.SpeechStream { this.#logger.info('aws transcribe stt: idle timeout, restarting session'); continue; } - throw new APIConnectionError({ - message: `aws transcribe stt: ${error instanceof Error ? error.message : String(error)}`, - }); + // Preserve APIStatusError (incl. non-retryable 4xx) so the base SpeechStream does not + // treat client errors as retryable connection failures. + throw toAwsApiError(error, 'aws transcribe stt'); } // #runSession only returns without throwing once its audio generator drains @@ -289,7 +317,51 @@ export class SpeechStream extends stt.SpeechStream { for await (const event of response.TranscriptResultStream) { if (event.TranscriptEvent) { this.#processTranscriptEvent(event.TranscriptEvent); + continue; + } + + // Amazon Transcribe can deliver service failures as TranscriptResultStream union + // members rather than thrown errors. Ignoring them lets the stream end "successfully" + // with no transcript and no retry — surface them as API errors instead. + this.#throwIfStreamException(event); + } + } + + /** + * Maps a non-transcript event-stream member onto a thrown error so the session loop in + * {@link run} can classify idle timeouts / retryable failures / hard client errors. + */ + #throwIfStreamException(event: TranscriptResultStream): void { + const entries: Array< + [name: string, value: { Message?: string; message?: string } | undefined] + > = [ + ['BadRequestException', event.BadRequestException], + ['LimitExceededException', event.LimitExceededException], + ['InternalFailureException', event.InternalFailureException], + ['ConflictException', event.ConflictException], + ['ServiceUnavailableException', event.ServiceUnavailableException], + ]; + + for (const [name, value] of entries) { + if (!value) continue; + + const message = value.Message ?? value.message ?? name; + + // Preserve the idle-timeout shape so {@link isIdleTimeout} still recognises it and + // the session reconnects rather than failing the stream. + if (name === 'BadRequestException' && message.startsWith('Your request timed out')) { + const err = new Error(message); + err.name = 'BadRequestException'; + throw err; } + + throw new APIStatusError({ + message: `aws transcribe stt: ${message}`, + options: { + statusCode: STREAM_EXCEPTION_STATUS[name] ?? 500, + body: value as object, + }, + }); } } diff --git a/plugins/aws/src/tts.test.ts b/plugins/aws/src/tts.test.ts index 783be6982..b0d6680d4 100644 --- a/plugins/aws/src/tts.test.ts +++ b/plugins/aws/src/tts.test.ts @@ -2,10 +2,12 @@ // // SPDX-License-Identifier: Apache-2.0 import type { PollyClient } from '@aws-sdk/client-polly'; +import { APIConnectionError, APIStatusError } from '@livekit/agents'; import { tts as ttsTest } from '@livekit/agents-plugins-test'; import { describe, expect, it } from 'vitest'; import { STT } from './stt.js'; import { TTS } from './tts.js'; +import { toAwsApiError } from './utils.js'; const hasAwsCredentials = Boolean(process.env.AWS_ACCESS_KEY_ID || process.env.AWS_PROFILE); @@ -90,6 +92,31 @@ describe('AWS Polly TTS - synthesis', () => { expect(events[0]?.final).toBe(true); expect(events[0]?.frame.samplesPerChannel).toBe(1600); }); + + // Error classification for Polly failures goes through toAwsApiError (used by + // ChunkedStream.run). Exercise the mapper directly — driving the full stream + // would also surface a by-design floating rejection from ThrowsPromise. + it('maps Polly service errors with an HTTP status to non-retryable APIStatusError', () => { + const err = Object.assign(new Error('Invalid voice id'), { + name: 'InvalidParameterException', + $metadata: { httpStatusCode: 400, requestId: 'req_bad_voice' }, + }); + + const error = toAwsApiError(err, 'aws polly tts'); + expect(error).toBeInstanceOf(APIStatusError); + const statusError = error as APIStatusError; + expect(statusError.statusCode).toBe(400); + expect(statusError.retryable).toBe(false); + expect(statusError.requestId).toBe('req_bad_voice'); + expect(statusError.message).toMatch(/Invalid voice id/); + }); + + it('maps Polly errors without an HTTP status to APIConnectionError', () => { + const error = toAwsApiError(new Error('network reset'), 'aws polly tts'); + expect(error).toBeInstanceOf(APIConnectionError); + expect(error).not.toBeInstanceOf(APIStatusError); + expect(error.message).toMatch(/network reset/); + }); }); describe('AWS Polly TTS (live)', () => { diff --git a/plugins/aws/src/tts.ts b/plugins/aws/src/tts.ts index 6e8871b59..27320738b 100644 --- a/plugins/aws/src/tts.ts +++ b/plugins/aws/src/tts.ts @@ -5,7 +5,6 @@ import type { SynthesizeSpeechCommandInput } from '@aws-sdk/client-polly'; import { PollyClient, SynthesizeSpeechCommand } from '@aws-sdk/client-polly'; import { type APIConnectOptions, - APIConnectionError, APITimeoutError, AudioByteStream, shortuuid, @@ -13,7 +12,7 @@ import { } from '@livekit/agents'; import type { AudioFrame } from '@livekit/rtc-node'; import type { TTSLanguage, TTSSpeechEngine, TTSTextType } from './models.js'; -import { type AwsCredentials, resolveRegion, stripUndefined } from './utils.js'; +import { type AwsCredentials, resolveRegion, stripUndefined, toAwsApiError } from './utils.js'; /** Amazon Polly only returns PCM (16-bit little-endian, mono) at these sample rates. */ const SUPPORTED_PCM_SAMPLE_RATES = [8000, 16000]; @@ -188,9 +187,10 @@ export class ChunkedStream extends tts.ChunkedStream { if (error instanceof Error && error.name === 'TimeoutError') { throw new APITimeoutError({ message: error.message }); } - throw new APIConnectionError({ - message: error instanceof Error ? error.message : String(error), - }); + // Prefer APIStatusError when the SDK surfaces an HTTP status (e.g. invalid VoiceId / + // Engine, malformed SSML) so non-retryable 4xx inputs are not retried as connection + // failures by the base ChunkedStream. + throw toAwsApiError(error, 'aws polly tts'); } finally { this.queue.close(); } diff --git a/plugins/aws/src/utils.ts b/plugins/aws/src/utils.ts index 8f91c49a5..17ec6c44a 100644 --- a/plugins/aws/src/utils.ts +++ b/plugins/aws/src/utils.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +import { APIConnectionError, APIStatusError } from '@livekit/agents'; export const DEFAULT_REGION = 'us-east-1'; @@ -23,3 +24,49 @@ export function resolveRegion(region?: string): string { export function stripUndefined>(obj: T): T { return Object.fromEntries(Object.entries(obj).filter(([, value]) => value !== undefined)) as T; } + +/** Shape of AWS SDK v3 service exceptions we care about when classifying failures. */ +interface AwsSdkErrorLike { + message?: string; + Message?: string; + name?: string; + $metadata?: { + httpStatusCode?: number; + requestId?: string; + }; +} + +/** + * Maps an AWS SDK exception (or already-classified framework error) into an + * {@link APIStatusError} when an HTTP status is present, otherwise an + * {@link APIConnectionError}. Passes through existing API errors unchanged. + */ +export function toAwsApiError( + error: unknown, + prefix: string, + options?: { retryable?: boolean; requestId?: string | null }, +): APIConnectionError | APIStatusError { + if (error instanceof APIStatusError || error instanceof APIConnectionError) { + return error; + } + + const err = error as AwsSdkErrorLike; + const message = err.message ?? err.Message ?? String(error); + const statusCode = err.$metadata?.httpStatusCode; + + if (statusCode !== undefined) { + return new APIStatusError({ + message: `${prefix}: ${message}`, + options: { + statusCode, + requestId: options?.requestId ?? err.$metadata?.requestId ?? null, + retryable: options?.retryable, + }, + }); + } + + return new APIConnectionError({ + message: `${prefix}: ${message}`, + options: { retryable: options?.retryable }, + }); +} From 6fb7f6b41a62aaf29053c239adb7e60d4d99160c Mon Sep 17 00:00:00 2001 From: Sora Morimoto Date: Thu, 9 Jul 2026 04:48:39 +0900 Subject: [PATCH 04/13] fix(aws): address PR review findings for Bedrock LLM and Transcribe STT Map validationException to 400, keep modelStreamError 424 retryable, requeue audio frames across STT reconnects, validate channel options, and surface speaker labels from Transcribe. --- plugins/aws/src/llm.test.ts | 19 +++++++- plugins/aws/src/llm.ts | 30 +++++++++++- plugins/aws/src/stt.test.ts | 68 ++++++++++++++++++++++++++ plugins/aws/src/stt.ts | 97 +++++++++++++++++++++++++++++++------ 4 files changed, 195 insertions(+), 19 deletions(-) diff --git a/plugins/aws/src/llm.test.ts b/plugins/aws/src/llm.test.ts index e23c82e9b..0fca34066 100644 --- a/plugins/aws/src/llm.test.ts +++ b/plugins/aws/src/llm.test.ts @@ -181,13 +181,14 @@ describe('AWS Bedrock LLM - streaming', () => { }); describe('AWS Bedrock LLM - mapConverseStreamException', () => { - it('maps validationException to a non-retryable APIStatusError', () => { + it('maps validationException to a non-retryable 400 APIStatusError', () => { const error = mapConverseStreamException( { validationException: { message: 'bad input' } }, 'req_1', true, ); expect(error).toBeInstanceOf(APIStatusError); + expect(error?.statusCode).toBe(400); expect(error?.retryable).toBe(false); expect(error?.message).toMatch(/bad input/); }); @@ -213,13 +214,27 @@ describe('AWS Bedrock LLM - mapConverseStreamException', () => { expect(error?.message).toMatch(/oops/); }); - it('maps modelStreamErrorException using its originalStatusCode', () => { + it('maps modelStreamErrorException using a 5xx originalStatusCode', () => { const error = mapConverseStreamException( { modelStreamErrorException: { message: 'stream broke', originalStatusCode: 503 } }, 'req_1', true, ); expect(error?.statusCode).toBe(503); + expect(error?.retryable).toBe(true); + expect(error?.message).toMatch(/stream broke/); + }); + + it('maps modelStreamErrorException with originalStatusCode 424 to a retryable 500', () => { + // Bedrock documents model stream errors with 424; APIStatusError would otherwise force + // non-retryable for that 4xx. Keep the configured 500 so the base LLM retry loop can recover. + const error = mapConverseStreamException( + { modelStreamErrorException: { message: 'stream broke', originalStatusCode: 424 } }, + 'req_1', + true, + ); + expect(error?.statusCode).toBe(500); + expect(error?.retryable).toBe(true); expect(error?.message).toMatch(/stream broke/); }); diff --git a/plugins/aws/src/llm.ts b/plugins/aws/src/llm.ts index 904db0650..1e05a85b3 100644 --- a/plugins/aws/src/llm.ts +++ b/plugins/aws/src/llm.ts @@ -105,13 +105,39 @@ const CONVERSE_STREAM_EXCEPTIONS: Array<{ statusCode?: number; retryable?: false; }> = [ - { key: 'validationException', defaultMessage: 'validation error', retryable: false }, + { + key: 'validationException', + defaultMessage: 'validation error', + statusCode: 400, + retryable: false, + }, { key: 'throttlingException', defaultMessage: 'throttled', statusCode: 429 }, { key: 'internalServerException', defaultMessage: 'internal server error', statusCode: 500 }, { key: 'modelStreamErrorException', defaultMessage: 'model stream error', statusCode: 500 }, { key: 'serviceUnavailableException', defaultMessage: 'service unavailable', statusCode: 503 }, ]; +/** + * Resolves the HTTP status to report for a Converse stream exception. + * + * `modelStreamErrorException` often carries `originalStatusCode: 424` (Failed Dependency). + * `APIStatusError` treats most 4xx as non-retryable, but AWS documents this event as a + * transient stream failure — keep 5xx originals (and the configured default) so the LLM + * retry loop can recover when no output has been emitted yet. + */ +function resolveConverseStreamStatusCode( + key: keyof ConverseStreamExceptionEvent, + exception: ConverseStreamException, + statusCode: number | undefined, +): number | undefined { + const original = exception.originalStatusCode; + if (key === 'modelStreamErrorException') { + if (original !== undefined && original >= 500) return original; + return statusCode; + } + return original ?? statusCode; +} + /** * Bedrock delivers fatal mid-stream errors as regular `ConverseStreamOutput` union events * rather than thrown exceptions. Maps one to an `APIStatusError`, or `undefined` if the event @@ -134,7 +160,7 @@ export function mapConverseStreamException( return new APIStatusError({ message: `aws bedrock llm: ${exception.message ?? defaultMessage}`, options: { - statusCode: exception.originalStatusCode ?? statusCode, + statusCode: resolveConverseStreamStatusCode(key, exception, statusCode), retryable: retryableOverride ?? retryable, requestId, }, diff --git a/plugins/aws/src/stt.test.ts b/plugins/aws/src/stt.test.ts index 6344a75c2..c8dadff6f 100644 --- a/plugins/aws/src/stt.test.ts +++ b/plugins/aws/src/stt.test.ts @@ -68,6 +68,22 @@ describe('AWS Transcribe STT - constructor', () => { expect(() => new STT({ identifyLanguage: true, languageOptions: 'en-US,es-US' })).not.toThrow(); }); + it('defaults numberOfChannels to 2 when enableChannelIdentification is set', () => { + expect(() => new STT({ enableChannelIdentification: true })).not.toThrow(); + }); + + it('throws when numberOfChannels is set without enableChannelIdentification', () => { + expect(() => new STT({ numberOfChannels: 2 })).toThrow( + /numberOfChannels requires enableChannelIdentification/, + ); + }); + + it('throws when enableChannelIdentification is set with a numberOfChannels other than 2', () => { + expect(() => new STT({ enableChannelIdentification: true, numberOfChannels: 1 })).toThrow( + /numberOfChannels must be 2/, + ); + }); + it('reports streaming-only capabilities with word-aligned transcripts', () => { const sttInstance = new STT(); expect(sttInstance.capabilities).toEqual({ @@ -151,6 +167,58 @@ describe('AWS Transcribe STT - SpeechStream event mapping', () => { expect(events[1]?.alternatives?.[0]?.confidence).toBe(0.9); }); + it('surfaces Transcribe speaker labels on words and the segment', async () => { + const client = fakeClient([ + async function* () { + yield transcriptEvent({ + StartTime: 0, + EndTime: 0.8, + IsPartial: false, + Alternatives: [ + { + Transcript: 'hello world', + Items: [ + { + Content: 'hello', + Type: 'pronunciation', + StartTime: 0, + EndTime: 0.3, + Confidence: 0.95, + Speaker: 'spk_0', + }, + { + Content: 'world', + Type: 'pronunciation', + StartTime: 0.4, + EndTime: 0.8, + Confidence: 0.9, + Speaker: 'spk_0', + }, + ], + }, + ], + }); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + const alt = events.find((e) => e.type === stt.SpeechEventType.FINAL_TRANSCRIPT) + ?.alternatives?.[0]; + expect(alt?.speakerId).toBe('spk_0'); + expect(alt?.words?.map((w) => w.speakerId)).toEqual(['spk_0', 'spk_0']); + }); + it('emits START_OF_SPEECH for every utterance, not just the first in the session', async () => { const client = fakeClient([ async function* () { diff --git a/plugins/aws/src/stt.ts b/plugins/aws/src/stt.ts index c80e6ce3e..08a087a6f 100644 --- a/plugins/aws/src/stt.ts +++ b/plugins/aws/src/stt.ts @@ -129,6 +129,7 @@ export class STT extends stt.STT { const identifyLanguage = opts.identifyLanguage ?? false; const identifyMultipleLanguages = opts.identifyMultipleLanguages ?? false; + const enableChannelIdentification = opts.enableChannelIdentification ?? false; // StartStreamTranscription requires LanguageOptions whenever either identify flag is set; // without it AWS returns BadRequest, so fail fast at construction with a clear message. @@ -139,11 +140,31 @@ export class STT extends stt.STT { ); } + // EnableChannelIdentification and NumberOfChannels must be supplied together; streaming + // Transcribe supports two channels. Default numberOfChannels to 2 when identification is + // enabled without an explicit value, and reject the inverse (channels without identification). + let numberOfChannels = opts.numberOfChannels; + if (enableChannelIdentification && numberOfChannels === undefined) { + numberOfChannels = 2; + } else if (!enableChannelIdentification && numberOfChannels !== undefined) { + throw new Error( + 'numberOfChannels requires enableChannelIdentification to be true ' + + '(Amazon Transcribe rejects NumberOfChannels without EnableChannelIdentification)', + ); + } else if (enableChannelIdentification && numberOfChannels !== 2) { + throw new Error( + 'numberOfChannels must be 2 when enableChannelIdentification is true ' + + '(Amazon Transcribe streaming supports two channels)', + ); + } + this.#opts = { ...defaultSTTOptions, ...opts, identifyLanguage, identifyMultipleLanguages, + enableChannelIdentification, + numberOfChannels, // Auto language detection is mutually exclusive with a fixed language code. language: identifyLanguage || identifyMultipleLanguages @@ -198,9 +219,12 @@ export class SpeechStream extends stt.SpeechStream { // `#channel` for the lifetime of this stream — the channel itself is never replaced, so // no already-buffered frame can ever be orphaned on a session transition. Each session // instead reads through a token-guarded wrapper generator: the token is invalidated the - // moment its session ends, so a superseded generator drops (at most) the one frame it may - // already be mid-await on instead of racing the new session for every subsequent frame. + // moment its session ends. Takes are serialised via `#frameTakeChain` so a superseded + // generator that is mid-await requeues its frame for the active session rather than + // dropping the start of the next utterance. #channel = new AsyncIterableQueue(); + #requeuedFrames: Uint8Array[] = []; + #frameTakeChain: Promise = Promise.resolve(); #pumpStarted = false; constructor( @@ -365,13 +389,49 @@ export class SpeechStream extends stt.SpeechStream { } } + /** + * Exclusively take the next audio frame for `token`. Serialises readers so an abandoned + * session generator cannot race the active one on `#channel`, and requeues a frame that + * was dequeued after the token was invalidated so reconnects do not clip the next utterance. + */ + async #takeFrame(token: SessionToken): Promise> { + let outcome: IteratorResult = { value: undefined, done: true }; + + const previous = this.#frameTakeChain; + let release!: () => void; + this.#frameTakeChain = new Promise((resolve) => { + release = resolve; + }); + + await previous; + try { + if (!token.active) { + return outcome; + } + + if (this.#requeuedFrames.length > 0) { + return { value: this.#requeuedFrames.shift()!, done: false }; + } + + const result = await this.#channel.next(); + // Invalidated while awaiting — stash the frame for the active session instead of dropping it. + if (!token.active) { + if (!result.done) { + this.#requeuedFrames.push(result.value); + } + return outcome; + } + + return result; + } finally { + release(); + } + } + async *#audioStreamGenerator(token: SessionToken): AsyncGenerator { for (;;) { if (!token.active) return; - const result = await this.#channel.next(); - // The token may have been invalidated while awaiting a frame that belonged to a - // session that has since ended (e.g. an idle-timeout reconnect) — drop it rather - // than yielding it into a request the SDK has already abandoned. + const result = await this.#takeFrame(token); if (!token.active) return; if (result.done) break; yield { AudioEvent: { AudioChunk: result.value } }; @@ -434,6 +494,20 @@ export class SpeechStream extends stt.SpeechStream { const offset = this.startTimeOffset + this.#connectionTimeOffset; + // When showSpeakerLabel is enabled, Transcribe attaches a Speaker label to each item. + // Surface it on every word and derive a segment-level speakerId from the first labelled word. + const words = items?.map((item) => + createTimedString({ + text: item.Content ?? '', + startTime: (item.StartTime ?? 0) + offset, + endTime: (item.EndTime ?? 0) + offset, + confidence: item.Confidence ?? 0, + startTimeOffset: offset, + speakerId: item.Speaker ?? null, + }), + ); + const speakerId = words?.find((word) => word.speakerId)?.speakerId ?? null; + return { language: normalizeLanguage(detectedLanguage), startTime: (result.StartTime ?? 0) + offset, @@ -441,15 +515,8 @@ export class SpeechStream extends stt.SpeechStream { text: alternative?.Transcript ?? '', confidence, sourceLanguages, - words: items?.map((item) => - createTimedString({ - text: item.Content ?? '', - startTime: (item.StartTime ?? 0) + offset, - endTime: (item.EndTime ?? 0) + offset, - confidence: item.Confidence ?? 0, - startTimeOffset: offset, - }), - ), + speakerId, + words, }; } } From 7c5592f6621cf0aca732a1b1fd4917e6600ffc8a Mon Sep 17 00:00:00 2001 From: Sora Morimoto Date: Fri, 10 Jul 2026 13:14:29 +0900 Subject: [PATCH 05/13] fix(aws): address review feedback --- agents/src/llm/provider_format/aws.test.ts | 61 +++++++++++++++++++++- agents/src/llm/provider_format/aws.ts | 8 +-- plugins/aws/src/llm.test.ts | 17 +++++- plugins/aws/src/llm.ts | 7 ++- plugins/aws/src/stt.test.ts | 60 ++++++++++++++++++++- plugins/aws/src/stt.ts | 28 ++++++++-- plugins/aws/src/tts.test.ts | 33 ++++++++++++ plugins/aws/src/tts.ts | 2 - plugins/aws/src/utils.ts | 23 ++++++-- 9 files changed, 222 insertions(+), 17 deletions(-) diff --git a/agents/src/llm/provider_format/aws.test.ts b/agents/src/llm/provider_format/aws.test.ts index 59fa3bf8a..38fef36d3 100644 --- a/agents/src/llm/provider_format/aws.test.ts +++ b/agents/src/llm/provider_format/aws.test.ts @@ -91,13 +91,70 @@ describe('AWS Provider Format - toChatCtx', () => { expect(formatData.systemMessages).toBeNull(); }); - it('should preserve empty string text content instead of dropping the whole message', async () => { + it('should drop empty and whitespace-only text blocks', async () => { const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'system', content: ' ' }); ctx.addMessage({ role: 'user', content: '' }); + ctx.addMessage({ + role: 'user', + content: [ + ' ', + new Instructions({ audio: 'audio instructions', text: ' ' }).asModality('text'), + ], + }); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([]); + expect(formatData.systemMessages).toBeNull(); + }); + + it('should replace blank tool output with non-empty placeholder text', async () => { + const ctx = ChatContext.empty(); + ctx.insert( + new FunctionCall({ + id: 'func_blank_output', + callId: 'call_blank_output', + name: 'blank_output', + args: '{}', + }), + ); + ctx.insert( + new FunctionCallOutput({ + callId: 'call_blank_output', + output: ' ', + isError: false, + }), + ); const [result] = await toChatCtx(ctx, false); - expect(result).toEqual([{ role: 'user', content: [{ text: '' }] }]); + expect(result).toEqual([ + { + role: 'assistant', + content: [ + { + toolUse: { + toolUseId: 'call_blank_output', + name: 'blank_output', + input: {}, + }, + }, + ], + }, + { + role: 'user', + content: [ + { + toolResult: { + toolUseId: 'call_blank_output', + content: [{ text: '(empty)' }], + status: 'success', + }, + }, + ], + }, + ]); }); it('should render Instructions as their resolved value', async () => { diff --git a/agents/src/llm/provider_format/aws.ts b/agents/src/llm/provider_format/aws.ts index 5e8c5bf14..e22c70406 100644 --- a/agents/src/llm/provider_format/aws.ts +++ b/agents/src/llm/provider_format/aws.ts @@ -48,7 +48,7 @@ export async function toChatCtx( // Always exclude system/developer messages from the regular role mapping below, even // when they carry no usable text (e.g. image-only content) — they must never be // reattributed to the user/assistant turn merge. - if (msg.textContent) { + if (msg.textContent?.trim()) { systemMessages.push(msg.textContent); } continue; @@ -77,9 +77,9 @@ export async function toChatCtx( if (msg.type === 'message') { for (const part of msg.content) { if (typeof part === 'string') { - content.push({ text: part }); + if (part.trim()) content.push({ text: part }); } else if (isInstructions(part)) { - content.push({ text: part.value }); + if (part.value.trim()) content.push({ text: part.value }); } else if (part && typeof part === 'object' && part.type === 'image_content') { content.push(await toImagePart(part)); } @@ -97,7 +97,7 @@ export async function toChatCtx( content.push({ toolResult: { toolUseId: msg.callId, - content: [{ text: msg.output }], + content: [{ text: msg.output.trim() ? msg.output : '(empty)' }], status: msg.isError ? 'error' : 'success', }, }); diff --git a/plugins/aws/src/llm.test.ts b/plugins/aws/src/llm.test.ts index 0fca34066..2c8bbd929 100644 --- a/plugins/aws/src/llm.test.ts +++ b/plugins/aws/src/llm.test.ts @@ -6,7 +6,12 @@ import { APIStatusError, llm } from '@livekit/agents'; import { llm as llmTest } from '@livekit/agents-plugins-test'; import { afterEach, describe, expect, it } from 'vitest'; import { z } from 'zod'; -import { LLM, buildToolConfig, mapConverseStreamException } from './llm.js'; +import { + LLM, + buildToolConfig, + isRetryableBedrockStatus, + mapConverseStreamException, +} from './llm.js'; const hasAwsCredentials = Boolean(process.env.AWS_ACCESS_KEY_ID || process.env.AWS_PROFILE); @@ -253,6 +258,16 @@ describe('AWS Bedrock LLM - mapConverseStreamException', () => { }); }); +describe('AWS Bedrock LLM - retry classification', () => { + it('retries HTTP 408 model timeouts before any output is emitted', () => { + expect(isRetryableBedrockStatus(408, true)).toBe(true); + }); + + it('does not retry a timeout after output has made the attempt non-retryable', () => { + expect(isRetryableBedrockStatus(408, false)).toBe(false); + }); +}); + describe('AWS Bedrock LLM (live)', () => { if (hasAwsCredentials) { it('passes the shared LLM test harness', async () => { diff --git a/plugins/aws/src/llm.ts b/plugins/aws/src/llm.ts index 1e05a85b3..61a9da3cf 100644 --- a/plugins/aws/src/llm.ts +++ b/plugins/aws/src/llm.ts @@ -99,6 +99,11 @@ interface ConverseStreamExceptionEvent { serviceUnavailableException?: ConverseStreamException; } +/** Whether a Bedrock HTTP failure can be retried before any output has been emitted. */ +export function isRetryableBedrockStatus(statusCode: number, retryable: boolean): boolean { + return retryable && (statusCode === 408 || statusCode === 429 || statusCode >= 500); +} + const CONVERSE_STREAM_EXCEPTIONS: Array<{ key: keyof ConverseStreamExceptionEvent; defaultMessage: string; @@ -425,7 +430,7 @@ export class LLMStream extends llm.LLMStream { message: `aws bedrock llm: ${err.message ?? 'unknown error'}`, options: { statusCode, - retryable: retryable && (statusCode === 429 || statusCode >= 500), + retryable: isRetryableBedrockStatus(statusCode, retryable), requestId, }, }); diff --git a/plugins/aws/src/stt.test.ts b/plugins/aws/src/stt.test.ts index c8dadff6f..1684a84fb 100644 --- a/plugins/aws/src/stt.test.ts +++ b/plugins/aws/src/stt.test.ts @@ -68,6 +68,23 @@ describe('AWS Transcribe STT - constructor', () => { expect(() => new STT({ identifyLanguage: true, languageOptions: 'en-US,es-US' })).not.toThrow(); }); + it('throws when speaker labels and channel identification are both enabled', () => { + expect(() => new STT({ showSpeakerLabel: true, enableChannelIdentification: true })).toThrow( + /mutually exclusive/, + ); + }); + + it('throws when automatic language identification uses a custom language model', () => { + expect( + () => + new STT({ + identifyLanguage: true, + languageOptions: 'en-US,es-US', + languageModelName: 'my-custom-model', + }), + ).toThrow(/languageModelName cannot be used/); + }); + it('defaults numberOfChannels to 2 when enableChannelIdentification is set', () => { expect(() => new STT({ enableChannelIdentification: true })).not.toThrow(); }); @@ -216,7 +233,10 @@ describe('AWS Transcribe STT - SpeechStream event mapping', () => { const alt = events.find((e) => e.type === stt.SpeechEventType.FINAL_TRANSCRIPT) ?.alternatives?.[0]; expect(alt?.speakerId).toBe('spk_0'); - expect(alt?.words?.map((w) => w.speakerId)).toEqual(['spk_0', 'spk_0']); + expect(alt?.words?.map((w: { speakerId?: string | null }) => w.speakerId)).toEqual([ + 'spk_0', + 'spk_0', + ]); }); it('emits START_OF_SPEECH for every utterance, not just the first in the session', async () => { @@ -554,6 +574,44 @@ describe('AWS Transcribe STT - SpeechStream event mapping', () => { expect(events.some((e) => e.alternatives?.[0]?.text === 'recovered')).toBe(true); }); + it('does not retry a failure after audio has already been sent', async () => { + let attempts = 0; + const client = { + send: async (command: { + input: { AudioStream: AsyncIterable<{ AudioEvent?: { AudioChunk?: Uint8Array } }> }; + }) => { + attempts += 1; + return { + TranscriptResultStream: (async function* () { + for await (const event of command.input.AudioStream) { + if ((event.AudioEvent?.AudioChunk?.byteLength ?? 0) > 0) break; + } + throw new Error('connection reset after audio'); + })(), + }; + }, + } as unknown as TranscribeStreamingClient; + + const sttInstance = new STT({ ...baseOpts, client }); + const errors: APIError[] = []; + sttInstance.on('error', ({ error }) => { + if (error instanceof APIError) errors.push(error); + }); + const speechStream = sttInstance.stream({ + connOptions: { maxRetry: 1, retryIntervalMs: 1, timeoutMs: 1000 }, + }); + + speechStream.pushFrame(new AudioFrame(new Int16Array([1, 2, 3, 4]), 16000, 1, 4)); + speechStream.endInput(); + for await (const _event of speechStream) { + // Drain until the terminal error closes the output queue. + } + + expect(attempts).toBe(1); + expect(errors).toHaveLength(1); + expect(errors[0]?.retryable).toBe(false); + }); + it('resets speaking state across a hard-failure retry so START_OF_SPEECH re-fires', async () => { let attempts = 0; const client = { diff --git a/plugins/aws/src/stt.ts b/plugins/aws/src/stt.ts index 08a087a6f..918bccacf 100644 --- a/plugins/aws/src/stt.ts +++ b/plugins/aws/src/stt.ts @@ -74,6 +74,7 @@ const defaultSTTOptions: Pick = { /** Invalidated the instant its session ends, so a superseded audio generator stops pulling. */ interface SessionToken { active: boolean; + audioSent: boolean; } /** Amazon Transcribe reports this after ~15s of stream inactivity; it's not a hard failure. */ @@ -131,6 +132,12 @@ export class STT extends stt.STT { const identifyMultipleLanguages = opts.identifyMultipleLanguages ?? false; const enableChannelIdentification = opts.enableChannelIdentification ?? false; + if (opts.showSpeakerLabel && enableChannelIdentification) { + throw new Error( + 'showSpeakerLabel and enableChannelIdentification are mutually exclusive; set only one to true', + ); + } + // StartStreamTranscription requires LanguageOptions whenever either identify flag is set; // without it AWS returns BadRequest, so fail fast at construction with a clear message. if ((identifyLanguage || identifyMultipleLanguages) && !opts.languageOptions) { @@ -140,6 +147,13 @@ export class STT extends stt.STT { ); } + if ((identifyLanguage || identifyMultipleLanguages) && opts.languageModelName) { + throw new Error( + 'languageModelName cannot be used with identifyLanguage or identifyMultipleLanguages ' + + '(Amazon Transcribe streaming language identification does not support custom language models)', + ); + } + // EnableChannelIdentification and NumberOfChannels must be supplied together; streaming // Transcribe supports two channels. Default numberOfChannels to 2 when identification is // enabled without an explicit value, and reject the inverse (channels without identification). @@ -265,7 +279,7 @@ export class SpeechStream extends stt.SpeechStream { // was scheduled (e.g. a very short utterance) — otherwise the final transcript would // never be requested. for (;;) { - const token: SessionToken = { active: true }; + const token: SessionToken = { active: true, audioSent: false }; try { await this.#runSession(this.#audioStreamGenerator(token)); } catch (error) { @@ -295,7 +309,12 @@ export class SpeechStream extends stt.SpeechStream { } // Preserve APIStatusError (incl. non-retryable 4xx) so the base SpeechStream does not // treat client errors as retryable connection failures. - throw toAwsApiError(error, 'aws transcribe stt'); + // Once AWS has consumed audio, retrying run() cannot replay those frames from the + // persistent channel. Treat the failure as terminal instead of letting a retry open a + // new session that can complete successfully without ever transcribing the lost audio. + throw toAwsApiError(error, 'aws transcribe stt', { + retryable: !token.audioSent, + }); } // #runSession only returns without throwing once its audio generator drains @@ -395,7 +414,7 @@ export class SpeechStream extends stt.SpeechStream { * was dequeued after the token was invalidated so reconnects do not clip the next utterance. */ async #takeFrame(token: SessionToken): Promise> { - let outcome: IteratorResult = { value: undefined, done: true }; + const outcome: IteratorResult = { value: undefined, done: true }; const previous = this.#frameTakeChain; let release!: () => void; @@ -434,6 +453,9 @@ export class SpeechStream extends stt.SpeechStream { const result = await this.#takeFrame(token); if (!token.active) return; if (result.done) break; + if (result.value.byteLength > 0) { + token.audioSent = true; + } yield { AudioEvent: { AudioChunk: result.value } }; } diff --git a/plugins/aws/src/tts.test.ts b/plugins/aws/src/tts.test.ts index b0d6680d4..a365e1bff 100644 --- a/plugins/aws/src/tts.test.ts +++ b/plugins/aws/src/tts.test.ts @@ -93,6 +93,39 @@ describe('AWS Polly TTS - synthesis', () => { expect(events[0]?.frame.samplesPerChannel).toBe(1600); }); + it('keeps the output queue open across a retryable Polly failure', async () => { + let attempts = 0; + const client = { + send: async () => { + attempts += 1; + if (attempts === 1) { + throw Object.assign(new Error('temporary outage'), { + $metadata: { httpStatusCode: 503 }, + }); + } + return { + $metadata: { requestId: 'req_retry' }, + AudioStream: { + transformToByteArray: async () => pcmBytes(1600), + }, + }; + }, + } as unknown as PollyClient; + const tts = new TTS({ sampleRate: 16000, client }); + const stream = tts.synthesize('retry me', { + maxRetry: 1, + retryIntervalMs: 1, + timeoutMs: 1000, + }); + + const events = []; + for await (const event of stream) events.push(event); + + expect(attempts).toBe(2); + expect(events).toHaveLength(1); + expect(events[0]?.final).toBe(true); + }); + // Error classification for Polly failures goes through toAwsApiError (used by // ChunkedStream.run). Exercise the mapper directly — driving the full stream // would also surface a by-design floating rejection from ThrowsPromise. diff --git a/plugins/aws/src/tts.ts b/plugins/aws/src/tts.ts index 27320738b..00aafda5a 100644 --- a/plugins/aws/src/tts.ts +++ b/plugins/aws/src/tts.ts @@ -191,8 +191,6 @@ export class ChunkedStream extends tts.ChunkedStream { // Engine, malformed SSML) so non-retryable 4xx inputs are not retried as connection // failures by the base ChunkedStream. throw toAwsApiError(error, 'aws polly tts'); - } finally { - this.queue.close(); } } } diff --git a/plugins/aws/src/utils.ts b/plugins/aws/src/utils.ts index 17ec6c44a..057c72e64 100644 --- a/plugins/aws/src/utils.ts +++ b/plugins/aws/src/utils.ts @@ -39,15 +39,32 @@ interface AwsSdkErrorLike { /** * Maps an AWS SDK exception (or already-classified framework error) into an * {@link APIStatusError} when an HTTP status is present, otherwise an - * {@link APIConnectionError}. Passes through existing API errors unchanged. + * {@link APIConnectionError}. Passes through existing API errors unchanged unless an explicit + * retryability override is provided. */ export function toAwsApiError( error: unknown, prefix: string, options?: { retryable?: boolean; requestId?: string | null }, ): APIConnectionError | APIStatusError { - if (error instanceof APIStatusError || error instanceof APIConnectionError) { - return error; + if (error instanceof APIStatusError) { + if (options?.retryable === undefined) return error; + return new APIStatusError({ + message: error.message, + options: { + statusCode: error.statusCode, + requestId: options.requestId ?? error.requestId, + body: error.body, + retryable: options.retryable, + }, + }); + } + if (error instanceof APIConnectionError) { + if (options?.retryable === undefined) return error; + return new APIConnectionError({ + message: error.message, + options: { retryable: options.retryable }, + }); } const err = error as AwsSdkErrorLike; From 48d2104b5dc9a98565a54b0a73f551b4b159ce40 Mon Sep 17 00:00:00 2001 From: Sora Morimoto Date: Fri, 10 Jul 2026 13:45:00 +0900 Subject: [PATCH 06/13] refactor(aws): improve reliability and optimise package output Honour request timeouts, preserve Transcribe speaker and language metadata, and close owned AWS clients. Trim redundant tests, exclude test artefacts from the published package, and add API and CI coverage. --- .github/workflows/test.yml | 5 + .gitignore | 1 + README.md | 15 +- agents/src/llm/provider_format/aws.test.ts | 10 - plugins/aws/etc/agents-plugin-aws.api.md | 230 +++++++++++++++++++++ plugins/aws/package.json | 1 + plugins/aws/src/index.ts | 4 +- plugins/aws/src/llm.test.ts | 44 +++- plugins/aws/src/llm.ts | 39 +++- plugins/aws/src/models.ts | 62 ++---- plugins/aws/src/stt.test.ts | 108 ++++------ plugins/aws/src/stt.ts | 117 +++++++---- plugins/aws/src/tts.test.ts | 7 +- plugins/aws/src/tts.ts | 34 ++- plugins/aws/src/utils.test.ts | 19 ++ plugins/aws/src/utils.ts | 50 ++++- plugins/aws/tsconfig.json | 9 +- plugins/aws/tsup.config.ts | 2 +- 18 files changed, 547 insertions(+), 210 deletions(-) create mode 100644 plugins/aws/etc/agents-plugin-aws.api.md create mode 100644 plugins/aws/src/utils.test.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d205e932d..1ff8548da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,11 +49,16 @@ jobs: - 'plugins/test/**' plugins: - 'plugins/**' + aws_plugin: + - 'plugins/aws/**' examples: - 'examples/**' - name: Test agents if: steps.filter.outputs.agents-or-tests == 'true' || github.event_name == 'push' run: pnpm test agents + - name: Test AWS plugin + if: steps.filter.outputs.aws_plugin == 'true' || github.event_name == 'push' + run: pnpm vitest run plugins/aws/src # - name: Test examples # if: (steps.filter.outputs.examples == 'true' || github.event_name == 'push') # env: diff --git a/.gitignore b/.gitignore index a10050c2c..d64cb5a68 100644 --- a/.gitignore +++ b/.gitignore @@ -197,6 +197,7 @@ examples/src/test_*.ts !.changeset/*.md !**/README.md !**/CHANGELOG.md +!**/etc/*.api.md !CONTRIBUTING.md !CLAUDE.md !.CODE_OF_CONDUCT.md diff --git a/README.md b/README.md index 826f5c475..3c2d1da19 100644 --- a/README.md +++ b/README.md @@ -118,9 +118,9 @@ import { WorkerOptions, cli, defineAgent, - inference, llm, voice, + inference, } from '@livekit/agents'; import * as silero from '@livekit/agents-plugin-silero'; import { fileURLToPath } from 'node:url'; @@ -155,10 +155,7 @@ export default defineAgent({ llm: new inference.LLM({ model: 'openai/gpt-4.1-mini' }), // Text-to-speech (TTS) is your agent's voice, turning the LLM's text into speech that the user can hear // See all available models as well as voice selections at https://docs.livekit.io/agents/models/tts/ - tts: new inference.TTS({ - model: 'cartesia/sonic-3', - voice: '9626c31c-bec5-4cca-baa8-f8ba9e84c8bc', - }), + tts: new inference.TTS({ model: 'cartesia/sonic-3', voice: '9626c31c-bec5-4cca-baa8-f8ba9e84c8bc' }), // VAD and turn detection are used to determine when the user is speaking and when the agent should respond // See more at https://docs.livekit.io/agents/build/turns vad: ctx.proc.userData.vad! as silero.VAD, @@ -257,10 +254,7 @@ export default defineAgent({ vad: ctx.proc.userData.vad! as silero.VAD, stt: new inference.STT({ model: 'deepgram/nova-3', language: 'en' }), llm: new inference.LLM({ model: 'openai/gpt-4.1-mini' }), - tts: new inference.TTS({ - model: 'cartesia/sonic-3', - voice: '9626c31c-bec5-4cca-baa8-f8ba9e84c8bc', - }), + tts: new inference.TTS({ model: 'cartesia/sonic-3', voice: '9626c31c-bec5-4cca-baa8-f8ba9e84c8bc' }), userData: userdata, }); @@ -350,7 +344,6 @@ To contribute to this project: To test any changes or plugins: 1. Build the project: - ```bash pnpm build ``` @@ -379,9 +372,7 @@ Refer to [the license](LICENSES/Apache-2.0.txt) for details. The LiveKit turn detection models are licensed under the [LiveKit Model License](MODEL_LICENSE). -
LiveKit Ecosystem
Agents SDKsPython · Node.js
- diff --git a/agents/src/llm/provider_format/aws.test.ts b/agents/src/llm/provider_format/aws.test.ts index 38fef36d3..694f0e17f 100644 --- a/agents/src/llm/provider_format/aws.test.ts +++ b/agents/src/llm/provider_format/aws.test.ts @@ -346,16 +346,6 @@ describe('AWS Provider Format - toChatCtx', () => { ]); }); - it('should inject a trailing dummy user message when the conversation ends on assistant', async () => { - const ctx = ChatContext.empty(); - ctx.addMessage({ role: 'user', content: 'Hello' }); - ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); - - const [result] = await toChatCtx(ctx, true); - - expect(result.at(-1)).toEqual({ role: 'user', content: [{ text: '.' }] }); - }); - it('should not inject a trailing dummy user message when the conversation already ends on user', async () => { const ctx = ChatContext.empty(); ctx.addMessage({ role: 'user', content: 'Hello' }); diff --git a/plugins/aws/etc/agents-plugin-aws.api.md b/plugins/aws/etc/agents-plugin-aws.api.md new file mode 100644 index 000000000..162fd41ed --- /dev/null +++ b/plugins/aws/etc/agents-plugin-aws.api.md @@ -0,0 +1,230 @@ +## API Report File for "@livekit/agents-plugin-aws" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import { APIConnectOptions } from '@livekit/agents'; +import { AudioBuffer as AudioBuffer_2 } from '@livekit/agents'; +import { BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime'; +import type { Engine } from '@aws-sdk/client-polly'; +import type { LanguageCode } from '@aws-sdk/client-transcribe-streaming'; +import type { LanguageCode as LanguageCode_2 } from '@aws-sdk/client-polly'; +import { llm } from '@livekit/agents'; +import type { PartialResultsStability } from '@aws-sdk/client-transcribe-streaming'; +import { PollyClient } from '@aws-sdk/client-polly'; +import { stt } from '@livekit/agents'; +import type { TextType } from '@aws-sdk/client-polly'; +import { TranscribeStreamingClient } from '@aws-sdk/client-transcribe-streaming'; +import { tts } from '@livekit/agents'; +import type { VocabularyFilterMethod } from '@aws-sdk/client-transcribe-streaming'; + +// @public +export interface AwsCredentials { + // (undocumented) + accessKeyId: string; + // (undocumented) + secretAccessKey: string; + // (undocumented) + sessionToken?: string; +} + +// @public (undocumented) +export class ChunkedStream extends tts.ChunkedStream { + constructor(tts: TTS, text: string, client: PollyClient, opts: TTSOptions, connOptions?: APIConnectOptions, abortSignal?: AbortSignal); + // (undocumented) + label: string; + // (undocumented) + protected run(): Promise; +} + +// @public +export class LLM extends llm.LLM { + constructor(opts?: LLMOptions); + // (undocumented) + aclose(): Promise; + // (undocumented) + chat({ chatCtx, toolCtx: toolCtxInput, connOptions, toolChoice, extraKwargs, }: { + chatCtx: llm.ChatContext; + toolCtx?: llm.ToolContextLike; + connOptions?: APIConnectOptions; + parallelToolCalls?: boolean; + toolChoice?: llm.ToolChoice; + extraKwargs?: Record; + }): LLMStream; + // (undocumented) + label(): string; + // (undocumented) + get model(): string; + // (undocumented) + get provider(): string; +} + +// @public (undocumented) +export interface LLMOptions { + // (undocumented) + additionalRequestFields?: Record; + cacheSystem?: boolean; + cacheTools?: boolean; + // (undocumented) + client?: BedrockRuntimeClient; + // (undocumented) + credentials?: AwsCredentials; + // (undocumented) + maxOutputTokens?: number; + // (undocumented) + model?: string; + // (undocumented) + region?: string; + // (undocumented) + temperature?: number; + // (undocumented) + toolChoice?: llm.ToolChoice; + // (undocumented) + topP?: number; +} + +// @public (undocumented) +export class LLMStream extends llm.LLMStream { + constructor(llmInst: LLM, { client, model, chatCtx, toolCtx, connOptions, cacheSystem, extraKwargs, }: { + client: BedrockRuntimeClient; + model: string; + chatCtx: llm.ChatContext; + toolCtx?: llm.ToolContext; + connOptions: APIConnectOptions; + cacheSystem: boolean; + extraKwargs: Record; + }); + // (undocumented) + protected run(): Promise; +} + +// @public (undocumented) +export class SpeechStream extends stt.SpeechStream { + constructor(stt: STT, opts: STTOptions, client: TranscribeStreamingClient, connOptions?: APIConnectOptions); + // (undocumented) + label: string; + // (undocumented) + protected run(): Promise; +} + +// @public +export class STT extends stt.STT { + constructor(opts?: Partial); + // (undocumented) + close(): Promise; + // (undocumented) + label: string; + // (undocumented) + get model(): string; + // (undocumented) + get provider(): string; + // (undocumented) + _recognize(_frame: AudioBuffer_2, _abortSignal?: AbortSignal): Promise; + // (undocumented) + stream(options?: { + connOptions?: APIConnectOptions; + }): SpeechStream; +} + +// @public (undocumented) +export interface STTOptions { + // (undocumented) + client?: TranscribeStreamingClient; + // (undocumented) + credentials?: AwsCredentials; + // (undocumented) + enableChannelIdentification?: boolean; + // (undocumented) + enablePartialResultsStabilization?: boolean; + // (undocumented) + identifyLanguage?: boolean; + // (undocumented) + identifyMultipleLanguages?: boolean; + // (undocumented) + language?: LanguageCode; + // (undocumented) + languageModelName?: string; + languageOptions?: string; + // (undocumented) + numberOfChannels?: number; + // (undocumented) + partialResultsStability?: PartialResultsStability; + // (undocumented) + preferredLanguage?: LanguageCode; + // (undocumented) + region?: string; + // (undocumented) + sampleRate: number; + // (undocumented) + sessionId?: string; + // (undocumented) + showSpeakerLabel?: boolean; + // (undocumented) + vocabFilterMethod?: VocabularyFilterMethod; + // (undocumented) + vocabFilterName?: string; + // (undocumented) + vocabularyFilterNames?: string; + // (undocumented) + vocabularyName?: string; + // (undocumented) + vocabularyNames?: string; +} + +// @public +export class TTS extends tts.TTS { + constructor(opts?: Partial); + // (undocumented) + close(): Promise; + // (undocumented) + label: string; + // (undocumented) + get model(): string; + // (undocumented) + get provider(): string; + // (undocumented) + stream(): tts.SynthesizeStream; + // (undocumented) + synthesize(text: string, connOptions?: APIConnectOptions, abortSignal?: AbortSignal): ChunkedStream; + // (undocumented) + updateOptions(opts: { + voice?: string; + language?: TTSLanguage; + speechEngine?: TTSSpeechEngine; + textType?: TTSTextType; + }): void; +} + +// @public +export type TTSLanguage = LanguageCode_2; + +// @public (undocumented) +export interface TTSOptions { + // (undocumented) + client?: PollyClient; + // (undocumented) + credentials?: AwsCredentials; + // (undocumented) + language?: TTSLanguage; + // (undocumented) + region?: string; + // (undocumented) + sampleRate: number; + // (undocumented) + speechEngine: TTSSpeechEngine; + // (undocumented) + textType: TTSTextType; + // (undocumented) + voice: string; +} + +// @public +export type TTSSpeechEngine = Engine; + +// @public +export type TTSTextType = TextType; + +// (No @packageDocumentation comment for this package) + +``` diff --git a/plugins/aws/package.json b/plugins/aws/package.json index 0ac788bb0..7fec270b4 100644 --- a/plugins/aws/package.json +++ b/plugins/aws/package.json @@ -22,6 +22,7 @@ "files": [ "dist", "src", + "!src/**/*.test.ts", "README.md" ], "scripts": { diff --git a/plugins/aws/src/index.ts b/plugins/aws/src/index.ts index c123bc7fe..b8314017f 100644 --- a/plugins/aws/src/index.ts +++ b/plugins/aws/src/index.ts @@ -3,11 +3,11 @@ // SPDX-License-Identifier: Apache-2.0 import { Plugin } from '@livekit/agents'; -export { LLM, LLMStream, type LLMOptions, buildToolConfig } from './llm.js'; +export { LLM, LLMStream, type LLMOptions } from './llm.js'; export * from './models.js'; export { SpeechStream, STT, type STTOptions } from './stt.js'; export { ChunkedStream, TTS, type TTSOptions } from './tts.js'; -export { type AwsCredentials, DEFAULT_REGION, resolveRegion } from './utils.js'; +export type { AwsCredentials } from './utils.js'; class AwsPlugin extends Plugin { constructor() { diff --git a/plugins/aws/src/llm.test.ts b/plugins/aws/src/llm.test.ts index 2c8bbd929..e9ccefab1 100644 --- a/plugins/aws/src/llm.test.ts +++ b/plugins/aws/src/llm.test.ts @@ -6,12 +6,7 @@ import { APIStatusError, llm } from '@livekit/agents'; import { llm as llmTest } from '@livekit/agents-plugins-test'; import { afterEach, describe, expect, it } from 'vitest'; import { z } from 'zod'; -import { - LLM, - buildToolConfig, - isRetryableBedrockStatus, - mapConverseStreamException, -} from './llm.js'; +import { LLM, buildToolConfig, mapConverseStreamException } from './llm.js'; const hasAwsCredentials = Boolean(process.env.AWS_ACCESS_KEY_ID || process.env.AWS_PROFILE); @@ -259,12 +254,39 @@ describe('AWS Bedrock LLM - mapConverseStreamException', () => { }); describe('AWS Bedrock LLM - retry classification', () => { - it('retries HTTP 408 model timeouts before any output is emitted', () => { - expect(isRetryableBedrockStatus(408, true)).toBe(true); - }); + it('retries an HTTP 408 before output is emitted', async () => { + let attempts = 0; + const client = { + send: async () => { + attempts += 1; + if (attempts === 1) { + throw Object.assign(new Error('model timed out'), { + $metadata: { httpStatusCode: 408, requestId: 'req_timeout' }, + }); + } + return { + $metadata: { requestId: 'req_recovered' }, + stream: (async function* () { + yield { contentBlockDelta: { delta: { text: 'recovered' } } }; + })(), + }; + }, + } as unknown as BedrockRuntimeClient; + const bedrockLlm = new LLM({ client }); + bedrockLlm.on('error', () => {}); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'Hi' }); + + const chunks = []; + for await (const chunk of bedrockLlm.chat({ + chatCtx, + connOptions: { maxRetry: 1, retryIntervalMs: 1, timeoutMs: 1000 }, + })) { + chunks.push(chunk); + } - it('does not retry a timeout after output has made the attempt non-retryable', () => { - expect(isRetryableBedrockStatus(408, false)).toBe(false); + expect(attempts).toBe(2); + expect(chunks.map((chunk) => chunk.delta?.content ?? '').join('')).toBe('recovered'); }); }); diff --git a/plugins/aws/src/llm.ts b/plugins/aws/src/llm.ts index 61a9da3cf..de85a1082 100644 --- a/plugins/aws/src/llm.ts +++ b/plugins/aws/src/llm.ts @@ -13,10 +13,11 @@ import type { APIConnectOptions } from '@livekit/agents'; import { APIConnectionError, APIStatusError, + APITimeoutError, DEFAULT_API_CONNECT_OPTIONS, llm, } from '@livekit/agents'; -import { type AwsCredentials, resolveRegion } from './utils.js'; +import { type AwsCredentials, createRequestSignal, resolveRegion } from './utils.js'; const DEFAULT_MODEL = 'amazon.nova-2-lite-v1:0'; @@ -24,6 +25,7 @@ interface AwsFormatData { systemMessages: string[] | null; } +/** @public */ export interface LLMOptions { model?: string; region?: string; @@ -43,7 +45,7 @@ export interface LLMOptions { /** * Builds a Bedrock Converse `ToolConfiguration` from a `ToolContext`, or `undefined` when the * turn should carry no tools (no tools registered, or `toolChoice: 'none'` was requested). - * Exported for unit testing. + * Kept module-local to the provider implementation; tests import the source module directly. */ export function buildToolConfig( toolCtx: llm.ToolContext | undefined, @@ -100,7 +102,7 @@ interface ConverseStreamExceptionEvent { } /** Whether a Bedrock HTTP failure can be retried before any output has been emitted. */ -export function isRetryableBedrockStatus(statusCode: number, retryable: boolean): boolean { +function isRetryableBedrockStatus(statusCode: number, retryable: boolean): boolean { return retryable && (statusCode === 408 || statusCode === 429 || statusCode >= 500); } @@ -174,10 +176,14 @@ export function mapConverseStreamException( return undefined; } -/** AWS Bedrock Converse LLM. */ +/** + * AWS Bedrock Converse LLM. + * @public + */ export class LLM extends llm.LLM { #opts; #client: BedrockRuntimeClient; + #ownsClient: boolean; label(): string { return 'aws.LLM'; @@ -217,6 +223,7 @@ export class LLM extends llm.LLM { cacheTools: opts.cacheTools ?? false, }; + this.#ownsClient = opts.client === undefined; this.#client = opts.client ?? new BedrockRuntimeClient({ @@ -228,6 +235,10 @@ export class LLM extends llm.LLM { }); } + async aclose(): Promise { + if (this.#ownsClient) this.#client.destroy(); + } + chat({ chatCtx, toolCtx: toolCtxInput, @@ -277,6 +288,7 @@ export class LLM extends llm.LLM { } } +/** @public */ export class LLMStream extends llm.LLMStream { #client: BedrockRuntimeClient; #model: string; @@ -313,6 +325,7 @@ export class LLMStream extends llm.LLMStream { protected async run(): Promise { let retryable = true; let requestId = ''; + const request = createRequestSignal(this.abortController.signal, this.connOptions.timeoutMs); try { const [messages, extraData] = (await this.chatCtx.toProviderFormat('aws')) as [ @@ -338,7 +351,7 @@ export class LLMStream extends llm.LLMStream { }; const response = await this.#client.send(new ConverseStreamCommand(input), { - abortSignal: this.abortController.signal, + abortSignal: request.signal, }); requestId = response.$metadata.requestId ?? ''; @@ -411,6 +424,13 @@ export class LLMStream extends llm.LLMStream { } } } catch (error: unknown) { + if (request.didTimeout()) { + throw new APITimeoutError({ + message: `aws bedrock llm: request timed out after ${this.connOptions.timeoutMs}ms`, + options: { retryable }, + }); + } + if (error instanceof APIStatusError || error instanceof APIConnectionError) { throw error; } @@ -422,7 +442,10 @@ export class LLMStream extends llm.LLMStream { return; } - const err = error as { message?: string; $metadata?: { httpStatusCode?: number } }; + const err = error as { + message?: string; + $metadata?: { httpStatusCode?: number; requestId?: string }; + }; const statusCode = err.$metadata?.httpStatusCode; if (statusCode !== undefined) { @@ -431,7 +454,7 @@ export class LLMStream extends llm.LLMStream { options: { statusCode, retryable: isRetryableBedrockStatus(statusCode, retryable), - requestId, + requestId: err.$metadata?.requestId ?? requestId, }, }); } @@ -440,6 +463,8 @@ export class LLMStream extends llm.LLMStream { message: `aws bedrock llm: ${err.message ?? String(error)}`, options: { retryable }, }); + } finally { + request.dispose(); } } } diff --git a/plugins/aws/src/models.ts b/plugins/aws/src/models.ts index 841263a95..a4f1d0cc4 100644 --- a/plugins/aws/src/models.ts +++ b/plugins/aws/src/models.ts @@ -1,56 +1,22 @@ // SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +import type { Engine, LanguageCode, TextType } from '@aws-sdk/client-polly'; -/** Amazon Polly speech synthesis engine. */ -export type TTSSpeechEngine = 'standard' | 'neural' | 'long-form' | 'generative'; +/** + * Amazon Polly speech synthesis engine. + * @public + */ +export type TTSSpeechEngine = Engine; /** - * Language code for the Amazon Polly SynthesizeSpeech request. Only necessary when using a - * bilingual voice (e.g. Aditi, which supports both `en-IN` and `hi-IN`). + * Language code accepted by Amazon Polly's SynthesizeSpeech request. + * @public */ -export type TTSLanguage = - | 'arb' - | 'cmn-CN' - | 'cy-GB' - | 'da-DK' - | 'de-DE' - | 'en-AU' - | 'en-GB' - | 'en-GB-WLS' - | 'en-IN' - | 'en-US' - | 'es-ES' - | 'es-MX' - | 'es-US' - | 'fr-CA' - | 'fr-FR' - | 'is-IS' - | 'it-IT' - | 'ja-JP' - | 'hi-IN' - | 'ko-KR' - | 'nb-NO' - | 'nl-NL' - | 'pl-PL' - | 'pt-BR' - | 'pt-PT' - | 'ro-RO' - | 'ru-RU' - | 'sv-SE' - | 'tr-TR' - | 'en-NZ' - | 'en-ZA' - | 'ca-ES' - | 'de-AT' - | 'yue-CN' - | 'ar-AE' - | 'fi-FI' - | 'en-IE' - | 'nl-BE' - | 'fr-BE' - | 'cs-CZ' - | 'de-CH'; +export type TTSLanguage = LanguageCode; -/** Whether the Amazon Polly input text is plain text or SSML. */ -export type TTSTextType = 'text' | 'ssml'; +/** + * Whether the Amazon Polly input text is plain text or SSML. + * @public + */ +export type TTSTextType = TextType; diff --git a/plugins/aws/src/stt.test.ts b/plugins/aws/src/stt.test.ts index 1684a84fb..223722a73 100644 --- a/plugins/aws/src/stt.test.ts +++ b/plugins/aws/src/stt.test.ts @@ -30,9 +30,13 @@ function fakeClient( let call = 0; return { send: async () => { + const sessionNumber = call + 1; const session = sessions[Math.min(call, sessions.length - 1)]!; call += 1; - return { TranscriptResultStream: session() }; + return { + $metadata: { requestId: `req_${sessionNumber}` }, + TranscriptResultStream: session(), + }; }, } as unknown as TranscribeStreamingClient; } @@ -107,9 +111,16 @@ describe('AWS Transcribe STT - constructor', () => { streaming: true, interimResults: true, alignedTranscript: 'word', + diarization: false, }); }); + it.each([{ showSpeakerLabel: true }, { enableChannelIdentification: true }] satisfies Array< + Partial + >)('advertises diarization for %o', (opts) => { + expect(new STT(opts).capabilities.diarization).toBe(true); + }); + it('reports the provider label', () => { const sttInstance = new STT(); expect(sttInstance.provider).toBe('Amazon Transcribe'); @@ -182,6 +193,7 @@ describe('AWS Transcribe STT - SpeechStream event mapping', () => { expect(events[1]?.alternatives?.[0]?.words).toHaveLength(1); expect(events[1]?.alternatives?.[0]?.words?.[0]?.text).toBe('hello'); expect(events[1]?.alternatives?.[0]?.confidence).toBe(0.9); + expect(events[1]?.requestId).toBe('req_1'); }); it('surfaces Transcribe speaker labels on words and the segment', async () => { @@ -319,50 +331,51 @@ describe('AWS Transcribe STT - SpeechStream event mapping', () => { // for the channel that actually finished. expect(events.filter((e) => e.type === stt.SpeechEventType.START_OF_SPEECH)).toHaveLength(2); expect(events.filter((e) => e.type === stt.SpeechEventType.END_OF_SPEECH)).toHaveLength(1); + const channelZeroFinal = events.find((event) => event.alternatives?.[0]?.text === 'done'); + expect(channelZeroFinal?.alternatives?.[0]?.speakerId).toBe('ch_0'); }); - it('does not drop a final transcript whose EndTime is exactly 0', async () => { + it('surfaces every language reported by multi-language identification', async () => { const client = fakeClient([ async function* () { yield transcriptEvent({ + LanguageCode: 'en-US', + LanguageIdentification: [ + { LanguageCode: 'en-US', Score: 0.8 }, + { LanguageCode: 'es-US', Score: 0.2 }, + ], StartTime: 0, - EndTime: 0, + EndTime: 0.5, IsPartial: false, - Alternatives: [{ Transcript: 'ok', Items: [] }], + Alternatives: [{ Transcript: 'hello hola', Items: [] }], }); }, ]); - const speechStream = stream(client); - - const events: Awaited>['value'][] = []; - const collect = (async () => { - for await (const event of speechStream) { - events.push(event); - } - })(); - + const speechStream = new STT({ + ...baseOpts, + language: undefined, + identifyMultipleLanguages: true, + languageOptions: 'en-US,es-US', + client, + }).stream(); speechStream.endInput(); - await collect; - expect(events.some((e) => e.alternatives?.[0]?.text === 'ok')).toBe(true); + const events = []; + for await (const event of speechStream) events.push(event); + + const final = events.find((event) => event.type === stt.SpeechEventType.FINAL_TRANSCRIPT); + expect(final?.alternatives?.[0]?.sourceLanguages).toEqual(['en-US', 'es-US']); }); - it('reconnects silently on an idle timeout instead of surfacing an error', async () => { - let firstSessionStarted = false; + it('does not drop a final transcript whose EndTime is exactly 0', async () => { const client = fakeClient([ - async function* () { - firstSessionStarted = true; - const err = new Error('Your request timed out waiting for input'); - err.name = 'BadRequestException'; - throw err; - }, async function* () { yield transcriptEvent({ StartTime: 0, - EndTime: 0.2, + EndTime: 0, IsPartial: false, - Alternatives: [{ Transcript: 'hi', Items: [] }], + Alternatives: [{ Transcript: 'ok', Items: [] }], }); }, ]); @@ -379,8 +392,7 @@ describe('AWS Transcribe STT - SpeechStream event mapping', () => { speechStream.endInput(); await collect; - expect(firstSessionStarted).toBe(true); - expect(events.some((e) => e.alternatives?.[0]?.text === 'hi')).toBe(true); + expect(events.some((e) => e.alternatives?.[0]?.text === 'ok')).toBe(true); }); it('resets speaking state on an idle-timeout reconnect so START_OF_SPEECH re-fires', async () => { @@ -532,48 +544,6 @@ describe('AWS Transcribe STT - SpeechStream event mapping', () => { expect(events.some((e) => e.alternatives?.[0]?.text === 'hi')).toBe(true); }); - it('classifies a non-idle-timeout failure as a hard failure the base class retries', async () => { - // The base SpeechStream intentionally stays silent on the 'error' event for recoverable - // retries (only terminal failures emit), so success-after-retry is observed via the - // recovered transcript rather than an event. - let attempts = 0; - const client = { - send: async () => { - attempts += 1; - if (attempts === 1) { - throw new Error('access denied'); - } - return { - TranscriptResultStream: (async function* () { - yield transcriptEvent({ - StartTime: 0, - EndTime: 0.2, - IsPartial: false, - Alternatives: [{ Transcript: 'recovered', Items: [] }], - }); - })(), - }; - }, - } as unknown as TranscribeStreamingClient; - - const speechStream = new STT({ ...baseOpts, client }).stream({ - connOptions: { maxRetry: 1, retryIntervalMs: 1, timeoutMs: 1000 }, - }); - - const events: Awaited>['value'][] = []; - const collect = (async () => { - for await (const event of speechStream) { - events.push(event); - } - })(); - - speechStream.endInput(); - await collect; - - expect(attempts).toBe(2); - expect(events.some((e) => e.alternatives?.[0]?.text === 'recovered')).toBe(true); - }); - it('does not retry a failure after audio has already been sent', async () => { let attempts = 0; const client = { diff --git a/plugins/aws/src/stt.ts b/plugins/aws/src/stt.ts index 918bccacf..06b1dc431 100644 --- a/plugins/aws/src/stt.ts +++ b/plugins/aws/src/stt.ts @@ -2,11 +2,14 @@ // // SPDX-License-Identifier: Apache-2.0 import type { + LanguageCode, + PartialResultsStability, Result, StartStreamTranscriptionCommandInput, AudioStream as TranscribeAudioStream, TranscriptEvent, TranscriptResultStream, + VocabularyFilterMethod, } from '@aws-sdk/client-transcribe-streaming'; import { StartStreamTranscriptionCommand, @@ -15,39 +18,48 @@ import { import { type APIConnectOptions, APIStatusError, + APITimeoutError, AsyncIterableQueue, type AudioBuffer, + DEFAULT_API_CONNECT_OPTIONS, createTimedString, log, normalizeLanguage, stt, } from '@livekit/agents'; -import { type AwsCredentials, resolveRegion, stripUndefined, toAwsApiError } from './utils.js'; - +import { + type AwsCredentials, + createRequestSignal, + resolveRegion, + stripUndefined, + toAwsApiError, +} from './utils.js'; + +/** @public */ export interface STTOptions { sampleRate: number; - language?: string; + language?: LanguageCode; region?: string; credentials?: AwsCredentials; vocabularyName?: string; sessionId?: string; - vocabFilterMethod?: string; + vocabFilterMethod?: VocabularyFilterMethod; vocabFilterName?: string; showSpeakerLabel?: boolean; enableChannelIdentification?: boolean; numberOfChannels?: number; enablePartialResultsStabilization?: boolean; - partialResultsStability?: string; + partialResultsStability?: PartialResultsStability; languageModelName?: string; identifyLanguage?: boolean; identifyMultipleLanguages?: boolean; /** * Comma-separated language codes Amazon Transcribe should consider when automatic - * language identification is enabled. Required when {@link identifyLanguage} or - * {@link identifyMultipleLanguages} is true (AWS rejects the request without it). + * language identification is enabled. Required when {@link STTOptions.identifyLanguage} or + * {@link STTOptions.identifyMultipleLanguages} is true (AWS rejects the request without it). */ languageOptions?: string; - preferredLanguage?: string; + preferredLanguage?: LanguageCode; vocabularyNames?: string; vocabularyFilterNames?: string; client?: TranscribeStreamingClient; @@ -92,10 +104,12 @@ function isIdleTimeout(error: unknown): boolean { * @remarks * Streaming only — {@link STT.stream} is the sole recognition entrypoint, matching Amazon * Transcribe's streaming-only API surface. + * @public */ export class STT extends stt.STT { #opts: STTOptions; #client: TranscribeStreamingClient; + #ownsClient: boolean; label = 'aws.STT'; get model(): string { @@ -116,10 +130,12 @@ export class STT extends stt.STT { * `AWS_DEFAULT_REGION`, falling back to `us-east-1`. */ constructor(opts: Partial = {}) { + const enableChannelIdentification = opts.enableChannelIdentification ?? false; super({ streaming: true, interimResults: true, alignedTranscript: 'word', + diarization: Boolean(opts.showSpeakerLabel || enableChannelIdentification), }); if (opts.identifyLanguage && opts.identifyMultipleLanguages) { @@ -130,7 +146,6 @@ export class STT extends stt.STT { const identifyLanguage = opts.identifyLanguage ?? false; const identifyMultipleLanguages = opts.identifyMultipleLanguages ?? false; - const enableChannelIdentification = opts.enableChannelIdentification ?? false; if (opts.showSpeakerLabel && enableChannelIdentification) { throw new Error( @@ -186,6 +201,7 @@ export class STT extends stt.STT { : opts.language ?? defaultSTTOptions.language, }; + this.#ownsClient = opts.client === undefined; this.#client = opts.client ?? new TranscribeStreamingClient({ @@ -197,6 +213,10 @@ export class STT extends stt.STT { }); } + async close(): Promise { + if (this.#ownsClient) this.#client.destroy(); + } + async _recognize(_frame: AudioBuffer, _abortSignal?: AbortSignal): Promise { throw new Error( 'Amazon Transcribe does not support single-frame recognition, use stream() instead', @@ -208,9 +228,11 @@ export class STT extends stt.STT { } } +/** @public */ export class SpeechStream extends stt.SpeechStream { #opts: STTOptions; #client: TranscribeStreamingClient; + #timeoutMs: number; #logger = log(); // Keyed by Transcribe's `ChannelId` (the empty string when channel identification is off, // i.e. single-channel audio) so a channel finishing its utterance can't spuriously flip @@ -226,10 +248,10 @@ export class SpeechStream extends stt.SpeechStream { label = 'aws.SpeechStream'; // `this.input` may only ever have a single active consumer (calling `.next()` from more - // than one place races for frames). Bedrock's Node HTTP/2 request pipeline can leave an + // than one place races for frames). Transcribe's Node HTTP/2 request pipeline can leave an // abandoned session's `AudioStream` generator alive as a hidden consumer after a session - // fails (the SDK doesn't reliably tear down the request side when the response side - // errors), so a single long-lived pump owns `this.input` and feeds a single persistent + // fails (the SDK doesn't reliably tear down the request side when the response side errors), + // so a single long-lived pump owns `this.input` and feeds a single persistent // `#channel` for the lifetime of this stream — the channel itself is never replaced, so // no already-buffered frame can ever be orphaned on a session transition. Each session // instead reads through a token-guarded wrapper generator: the token is invalidated the @@ -247,9 +269,11 @@ export class SpeechStream extends stt.SpeechStream { client: TranscribeStreamingClient, connOptions?: APIConnectOptions, ) { - super(stt, opts.sampleRate, connOptions); + const resolvedConnOptions = connOptions ?? DEFAULT_API_CONNECT_OPTIONS; + super(stt, opts.sampleRate, resolvedConnOptions); this.#opts = opts; this.#client = client; + this.#timeoutMs = resolvedConnOptions.timeoutMs; } #startPump(): void { @@ -349,24 +373,39 @@ export class SpeechStream extends stt.SpeechStream { VocabularyFilterNames: this.#opts.vocabularyFilterNames, }) as unknown as StartStreamTranscriptionCommandInput; - const response = await this.#client.send(new StartStreamTranscriptionCommand(input), { - abortSignal: this.abortSignal, - }); - - if (!response.TranscriptResultStream) { - throw new Error('aws transcribe stt: no TranscriptResultStream in the response'); - } + const request = createRequestSignal(this.abortSignal, this.#timeoutMs); + try { + const response = await this.#client.send(new StartStreamTranscriptionCommand(input), { + abortSignal: request.signal, + }); + // `timeoutMs` bounds opening the long-lived stream, not the stream's total lifetime. + request.clearTimeout(); - for await (const event of response.TranscriptResultStream) { - if (event.TranscriptEvent) { - this.#processTranscriptEvent(event.TranscriptEvent); - continue; + if (!response.TranscriptResultStream) { + throw new Error('aws transcribe stt: no TranscriptResultStream in the response'); } - // Amazon Transcribe can deliver service failures as TranscriptResultStream union - // members rather than thrown errors. Ignoring them lets the stream end "successfully" - // with no transcript and no retry — surface them as API errors instead. - this.#throwIfStreamException(event); + const requestId = response.$metadata?.requestId; + for await (const event of response.TranscriptResultStream) { + if (event.TranscriptEvent) { + this.#processTranscriptEvent(event.TranscriptEvent, requestId); + continue; + } + + // Amazon Transcribe can deliver service failures as TranscriptResultStream union + // members rather than thrown errors. Ignoring them lets the stream end "successfully" + // with no transcript and no retry — surface them as API errors instead. + this.#throwIfStreamException(event); + } + } catch (error) { + if (request.didTimeout()) { + throw new APITimeoutError({ + message: `aws transcribe stt: request timed out after ${this.#timeoutMs}ms`, + }); + } + throw error; + } finally { + request.dispose(); } } @@ -465,7 +504,7 @@ export class SpeechStream extends stt.SpeechStream { } } - #processTranscriptEvent(transcriptEvent: TranscriptEvent): void { + #processTranscriptEvent(transcriptEvent: TranscriptEvent, requestId?: string): void { const results = transcriptEvent.Transcript?.Results; if (!results) return; @@ -478,7 +517,7 @@ export class SpeechStream extends stt.SpeechStream { const channelKey = result.ChannelId ?? ''; if (!this.#speakingChannels.has(channelKey)) { this.#speakingChannels.add(channelKey); - this.queue.put({ type: stt.SpeechEventType.START_OF_SPEECH }); + this.queue.put({ type: stt.SpeechEventType.START_OF_SPEECH, requestId }); } if (result.EndTime !== undefined) { @@ -488,13 +527,14 @@ export class SpeechStream extends stt.SpeechStream { type: result.IsPartial ? stt.SpeechEventType.INTERIM_TRANSCRIPT : stt.SpeechEventType.FINAL_TRANSCRIPT, + requestId, alternatives: [alternative], }); } if (!result.IsPartial) { this.#speakingChannels.delete(channelKey); - this.queue.put({ type: stt.SpeechEventType.END_OF_SPEECH }); + this.queue.put({ type: stt.SpeechEventType.END_OF_SPEECH, requestId }); } } } @@ -509,9 +549,16 @@ export class SpeechStream extends stt.SpeechStream { : 0; const detectedLanguage = result.LanguageCode ?? this.#opts.language ?? 'en-US'; + const identifiedLanguages = result.LanguageIdentification?.flatMap(({ LanguageCode }) => + LanguageCode ? [normalizeLanguage(LanguageCode)] : [], + ); const sourceLanguages = - (this.#opts.identifyLanguage || this.#opts.identifyMultipleLanguages) && result.LanguageCode - ? [normalizeLanguage(result.LanguageCode)] + this.#opts.identifyLanguage || this.#opts.identifyMultipleLanguages + ? identifiedLanguages?.length + ? [...new Set(identifiedLanguages)] + : result.LanguageCode + ? [normalizeLanguage(result.LanguageCode)] + : undefined : undefined; const offset = this.startTimeOffset + this.#connectionTimeOffset; @@ -525,10 +572,10 @@ export class SpeechStream extends stt.SpeechStream { endTime: (item.EndTime ?? 0) + offset, confidence: item.Confidence ?? 0, startTimeOffset: offset, - speakerId: item.Speaker ?? null, + speakerId: item.Speaker ?? result.ChannelId ?? null, }), ); - const speakerId = words?.find((word) => word.speakerId)?.speakerId ?? null; + const speakerId = words?.find((word) => word.speakerId)?.speakerId ?? result.ChannelId ?? null; return { language: normalizeLanguage(detectedLanguage), diff --git a/plugins/aws/src/tts.test.ts b/plugins/aws/src/tts.test.ts index a365e1bff..edf423f0f 100644 --- a/plugins/aws/src/tts.test.ts +++ b/plugins/aws/src/tts.test.ts @@ -58,8 +58,8 @@ describe('AWS Polly TTS - constructor', () => { }); describe('AWS Polly TTS - synthesis', () => { - it('converts the PCM response into audio frames', async () => { - const tts = new TTS({ sampleRate: 16000, client: fakeClient(pcmBytes(1600)) }); + it('flushes a PCM response shorter than one 100ms frame', async () => { + const tts = new TTS({ sampleRate: 16000, client: fakeClient(pcmBytes(800)) }); const stream = tts.synthesize('hello world'); const events = []; @@ -67,8 +67,9 @@ describe('AWS Polly TTS - synthesis', () => { events.push(event); } - expect(events.length).toBeGreaterThan(0); + expect(events).toHaveLength(1); expect(events.at(-1)?.final).toBe(true); + expect(events.reduce((sum, event) => sum + event.frame.samplesPerChannel, 0)).toBe(800); for (const event of events) { expect(event.frame.sampleRate).toBe(16000); expect(event.frame.channels).toBe(1); diff --git a/plugins/aws/src/tts.ts b/plugins/aws/src/tts.ts index 00aafda5a..db627ac45 100644 --- a/plugins/aws/src/tts.ts +++ b/plugins/aws/src/tts.ts @@ -7,21 +7,29 @@ import { type APIConnectOptions, APITimeoutError, AudioByteStream, + DEFAULT_API_CONNECT_OPTIONS, shortuuid, tts, } from '@livekit/agents'; import type { AudioFrame } from '@livekit/rtc-node'; import type { TTSLanguage, TTSSpeechEngine, TTSTextType } from './models.js'; -import { type AwsCredentials, resolveRegion, stripUndefined, toAwsApiError } from './utils.js'; +import { + type AwsCredentials, + createRequestSignal, + resolveRegion, + stripUndefined, + toAwsApiError, +} from './utils.js'; /** Amazon Polly only returns PCM (16-bit little-endian, mono) at these sample rates. */ const SUPPORTED_PCM_SAMPLE_RATES = [8000, 16000]; +/** @public */ export interface TTSOptions { voice: string; speechEngine: TTSSpeechEngine; textType: TTSTextType; - language?: TTSLanguage | string; + language?: TTSLanguage; sampleRate: number; region?: string; credentials?: AwsCredentials; @@ -41,10 +49,12 @@ const defaultTTSOptions: Omit = { * Audio is requested as raw PCM (16-bit little-endian, mono) since the framework has no mp3 * decoder. Amazon Polly only supports PCM output at 8000 Hz or 16000 Hz, so `sampleRate` is * restricted to those two values. + * @public */ export class TTS extends tts.TTS { #opts: TTSOptions; #client: PollyClient; + #ownsClient: boolean; label = 'aws.TTS'; private abortController = new AbortController(); @@ -81,6 +91,7 @@ export class TTS extends tts.TTS { sampleRate, }; + this.#ownsClient = opts.client === undefined; this.#client = opts.client ?? new PollyClient({ @@ -92,7 +103,7 @@ export class TTS extends tts.TTS { updateOptions(opts: { voice?: string; - language?: TTSLanguage | string; + language?: TTSLanguage; speechEngine?: TTSSpeechEngine; textType?: TTSTextType; }) { @@ -116,13 +127,16 @@ export class TTS extends tts.TTS { async close(): Promise { this.abortController.abort(); + if (this.#ownsClient) this.#client.destroy(); } } +/** @public */ export class ChunkedStream extends tts.ChunkedStream { label = 'aws.ChunkedStream'; #client: PollyClient; #opts: TTSOptions; + #timeoutMs: number; constructor( tts: TTS, @@ -132,12 +146,15 @@ export class ChunkedStream extends tts.ChunkedStream { connOptions?: APIConnectOptions, abortSignal?: AbortSignal, ) { - super(text, tts, connOptions, abortSignal); + const resolvedConnOptions = connOptions ?? DEFAULT_API_CONNECT_OPTIONS; + super(text, tts, resolvedConnOptions, abortSignal); this.#client = client; this.#opts = opts; + this.#timeoutMs = resolvedConnOptions.timeoutMs; } protected async run() { + const request = createRequestSignal(this.abortSignal, this.#timeoutMs); try { const input = stripUndefined({ Text: this.inputText, @@ -150,7 +167,7 @@ export class ChunkedStream extends tts.ChunkedStream { }) as unknown as SynthesizeSpeechCommandInput; const response = await this.#client.send(new SynthesizeSpeechCommand(input), { - abortSignal: this.abortSignal, + abortSignal: request.signal, }); if (!response.AudioStream) { @@ -181,6 +198,11 @@ export class ChunkedStream extends tts.ChunkedStream { } sendLastFrame(requestId, true); } catch (error) { + if (request.didTimeout()) { + throw new APITimeoutError({ + message: `aws polly tts: request timed out after ${this.#timeoutMs}ms`, + }); + } if (error instanceof Error && error.name === 'AbortError') { return; } @@ -191,6 +213,8 @@ export class ChunkedStream extends tts.ChunkedStream { // Engine, malformed SSML) so non-retryable 4xx inputs are not retried as connection // failures by the base ChunkedStream. throw toAwsApiError(error, 'aws polly tts'); + } finally { + request.dispose(); } } } diff --git a/plugins/aws/src/utils.test.ts b/plugins/aws/src/utils.test.ts new file mode 100644 index 000000000..07043dd66 --- /dev/null +++ b/plugins/aws/src/utils.test.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { describe, expect, it } from 'vitest'; +import { createRequestSignal } from './utils.js'; + +describe('AWS request signals', () => { + it('aborts and records when the request timeout elapses', async () => { + const request = createRequestSignal(new AbortController().signal, 5); + + await new Promise((resolve) => { + request.signal.addEventListener('abort', () => resolve(), { once: true }); + }); + + expect(request.didTimeout()).toBe(true); + expect(request.signal.reason).toMatchObject({ name: 'TimeoutError' }); + request.dispose(); + }); +}); diff --git a/plugins/aws/src/utils.ts b/plugins/aws/src/utils.ts index 057c72e64..23b79a4c1 100644 --- a/plugins/aws/src/utils.ts +++ b/plugins/aws/src/utils.ts @@ -5,7 +5,10 @@ import { APIConnectionError, APIStatusError } from '@livekit/agents'; export const DEFAULT_REGION = 'us-east-1'; -/** Explicit static AWS credentials. When omitted, the AWS SDK v3 default credential chain is used. */ +/** + * Explicit static AWS credentials. When omitted, the AWS SDK v3 default credential chain is used. + * @public + */ export interface AwsCredentials { accessKeyId: string; secretAccessKey: string; @@ -25,6 +28,51 @@ export function stripUndefined>(obj: T): T { return Object.fromEntries(Object.entries(obj).filter(([, value]) => value !== undefined)) as T; } +/** Combines a caller-controlled abort signal with a disposable per-request timeout. */ +export function createRequestSignal( + parent: AbortSignal, + timeoutMs: number, +): { + signal: AbortSignal; + didTimeout: () => boolean; + clearTimeout: () => void; + dispose: () => void; +} { + const controller = new AbortController(); + let timedOut = false; + + const abortFromParent = () => controller.abort(parent.reason); + if (parent.aborted) { + abortFromParent(); + } else { + parent.addEventListener('abort', abortFromParent, { once: true }); + } + + let timer = + timeoutMs > 0 + ? setTimeout(() => { + timedOut = true; + controller.abort(new DOMException('Request timed out', 'TimeoutError')); + }, timeoutMs) + : undefined; + timer?.unref(); + + const clearRequestTimeout = () => { + if (timer) clearTimeout(timer); + timer = undefined; + }; + + return { + signal: controller.signal, + didTimeout: () => timedOut, + clearTimeout: clearRequestTimeout, + dispose: () => { + clearRequestTimeout(); + parent.removeEventListener('abort', abortFromParent); + }, + }; +} + /** Shape of AWS SDK v3 service exceptions we care about when classifying failures. */ interface AwsSdkErrorLike { message?: string; diff --git a/plugins/aws/tsconfig.json b/plugins/aws/tsconfig.json index 50be4b771..9c061c392 100644 --- a/plugins/aws/tsconfig.json +++ b/plugins/aws/tsconfig.json @@ -1,8 +1,7 @@ { "extends": "../../tsconfig.json", - "include": [ - "./src" - ], + "include": ["./src"], + "exclude": ["./src/**/*.test.ts"], "compilerOptions": { // match output dir to input dir. e.g. dist/index instead of dist/src/index "rootDir": "./src", @@ -12,8 +11,6 @@ "typedocOptions": { "name": "plugins/agents-plugin-aws", "entryPointStrategy": "resolve", - "entryPoints": [ - "src/index.ts" - ] + "entryPoints": ["src/index.ts"] } } diff --git a/plugins/aws/tsup.config.ts b/plugins/aws/tsup.config.ts index 8ca20961f..18274714b 100644 --- a/plugins/aws/tsup.config.ts +++ b/plugins/aws/tsup.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from 'tsup'; - import defaults from '../../tsup.config'; export default defineConfig({ ...defaults, + entry: ['src/**/*.ts', '!src/**/*.test.ts'], }); From dd6dd49540729ac82f9cd9c19a3a96b2b7a1eba4 Mon Sep 17 00:00:00 2001 From: Sora Morimoto Date: Fri, 10 Jul 2026 13:57:35 +0900 Subject: [PATCH 07/13] fix(aws): keep established Bedrock streams open Clear the connection timeout once Bedrock has returned a streaming response, while preserving caller-initiated cancellation. Add regression coverage for responses that outlive the connection timeout. --- plugins/aws/src/llm.test.ts | 30 ++++++++++++++++++++++++++++++ plugins/aws/src/llm.ts | 3 +++ 2 files changed, 33 insertions(+) diff --git a/plugins/aws/src/llm.test.ts b/plugins/aws/src/llm.test.ts index e9ccefab1..2af84a3c6 100644 --- a/plugins/aws/src/llm.test.ts +++ b/plugins/aws/src/llm.test.ts @@ -178,6 +178,36 @@ describe('AWS Bedrock LLM - streaming', () => { expect(toolCalls[0]?.name).toBe('getWeather'); expect(toolCalls[0]?.args).toBe('{"location":"Tokyo"}'); }); + + it('does not apply the connection timeout to an established response stream', async () => { + const client = { + send: async (_command: unknown, { abortSignal }: { abortSignal: AbortSignal }) => ({ + $metadata: { requestId: 'req_slow_stream' }, + stream: (async function* () { + await new Promise((resolve) => setTimeout(resolve, 20)); + if (abortSignal.aborted) { + const error = new Error('stream aborted'); + error.name = 'AbortError'; + throw error; + } + yield { contentBlockDelta: { delta: { text: 'finished' } } }; + })(), + }), + } as unknown as BedrockRuntimeClient; + const bedrockLlm = new LLM({ client }); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'Hi' }); + + const chunks = []; + for await (const chunk of bedrockLlm.chat({ + chatCtx, + connOptions: { maxRetry: 0, retryIntervalMs: 1, timeoutMs: 5 }, + })) { + chunks.push(chunk); + } + + expect(chunks.map((chunk) => chunk.delta?.content ?? '').join('')).toBe('finished'); + }); }); describe('AWS Bedrock LLM - mapConverseStreamException', () => { diff --git a/plugins/aws/src/llm.ts b/plugins/aws/src/llm.ts index de85a1082..325a25203 100644 --- a/plugins/aws/src/llm.ts +++ b/plugins/aws/src/llm.ts @@ -354,6 +354,9 @@ export class LLMStream extends llm.LLMStream { abortSignal: request.signal, }); requestId = response.$metadata.requestId ?? ''; + // `timeoutMs` bounds opening the streaming response, not generation time. Keep the + // parent abort signal connected, but do not terminate a healthy long-running response. + request.clearTimeout(); if (!response.stream) { throw new APIStatusError({ From b5cf826457da5e1ce6682e64034876402fe6b7f5 Mon Sep 17 00:00:00 2001 From: Sora Morimoto Date: Fri, 10 Jul 2026 14:16:33 +0900 Subject: [PATCH 08/13] fix(aws): skip assistant images for Bedrock --- agents/src/llm/provider_format/aws.test.ts | 26 ++++++++++++++++++++++ agents/src/llm/provider_format/aws.ts | 8 ++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/agents/src/llm/provider_format/aws.test.ts b/agents/src/llm/provider_format/aws.test.ts index 694f0e17f..1c687d136 100644 --- a/agents/src/llm/provider_format/aws.test.ts +++ b/agents/src/llm/provider_format/aws.test.ts @@ -432,6 +432,32 @@ describe('AWS Provider Format - toChatCtx', () => { ]); }); + it('should drop images from assistant messages because Bedrock only accepts user images', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'Describe the image' }); + ctx.addMessage({ + role: 'assistant', + content: [ + 'Description', + { + id: 'img1', + type: 'image_content', + image: 'data:image/png;base64,ZmFrZS1wbmctYnl0ZXM=', + inferenceDetail: 'auto', + _cache: {}, + }, + ], + }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { role: 'user', content: [{ text: 'Describe the image' }] }, + { role: 'assistant', content: [{ text: 'Description' }] }, + ]); + expect(serializeImageMock).not.toHaveBeenCalled(); + }); + it('should default to jpeg format when mimeType is missing', async () => { serializeImageMock.mockResolvedValue({ inferenceDetail: 'auto', diff --git a/agents/src/llm/provider_format/aws.ts b/agents/src/llm/provider_format/aws.ts index e22c70406..7fc1f692f 100644 --- a/agents/src/llm/provider_format/aws.ts +++ b/agents/src/llm/provider_format/aws.ts @@ -80,7 +80,13 @@ export async function toChatCtx( if (part.trim()) content.push({ text: part }); } else if (isInstructions(part)) { if (part.value.trim()) content.push({ text: part.value }); - } else if (part && typeof part === 'object' && part.type === 'image_content') { + } else if ( + role === 'user' && + part && + typeof part === 'object' && + part.type === 'image_content' + ) { + // Bedrock Converse only accepts image blocks in user messages. content.push(await toImagePart(part)); } // audio_content is intentionally skipped — Bedrock Converse has no raw audio block From 27fdcbd8043a61ed411002b336e804bca7c666aa Mon Sep 17 00:00:00 2001 From: Sora Morimoto Date: Fri, 10 Jul 2026 14:50:03 +0900 Subject: [PATCH 09/13] fix(aws): address Bedrock and Transcribe review feedback --- agents/src/llm/provider_format/aws.test.ts | 91 ++++++++++ agents/src/llm/provider_format/aws.ts | 87 +++++++-- plugins/aws/src/llm.test.ts | 194 ++++++++++++++++++++- plugins/aws/src/llm.ts | 131 ++++++++++---- plugins/aws/src/stt.test.ts | 71 ++++++++ plugins/aws/src/stt.ts | 34 ++++ 6 files changed, 560 insertions(+), 48 deletions(-) diff --git a/agents/src/llm/provider_format/aws.test.ts b/agents/src/llm/provider_format/aws.test.ts index 1c687d136..302a4e3c8 100644 --- a/agents/src/llm/provider_format/aws.test.ts +++ b/agents/src/llm/provider_format/aws.test.ts @@ -218,6 +218,17 @@ describe('AWS Provider Format - toChatCtx', () => { expect(formatData.systemMessages).toBeNull(); }); + it('should ignore empty opposite-role messages without splitting the active turn', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'First' }); + ctx.addMessage({ role: 'assistant', content: ' ' }); + ctx.addMessage({ role: 'user', content: 'Second' }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([{ role: 'user', content: [{ text: 'First' }, { text: 'Second' }] }]); + }); + it('should map function calls to assistant toolUse blocks and outputs to user toolResult blocks', async () => { const ctx = ChatContext.empty(); @@ -317,6 +328,86 @@ describe('AWS Provider Format - toChatCtx', () => { ]); }); + it('should keep tool results separate from a following user message', async () => { + const ctx = ChatContext.empty(); + ctx.insert([ + new FunctionCall({ + id: 'func_separate', + callId: 'call_separate', + name: 'lookup', + args: '{}', + }), + new FunctionCallOutput({ + callId: 'call_separate', + name: 'lookup', + output: 'result', + isError: false, + }), + ]); + ctx.addMessage({ role: 'user', content: 'A separate follow-up' }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'assistant', + content: [ + { + toolUse: { toolUseId: 'call_separate', name: 'lookup', input: {} }, + }, + ], + }, + { + role: 'user', + content: [ + { + toolResult: { + toolUseId: 'call_separate', + content: [{ text: 'result' }], + status: 'success', + }, + }, + ], + }, + { role: 'user', content: [{ text: 'A separate follow-up' }] }, + ]); + }); + + it('should restore Bedrock reasoning content from provider-specific message data', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'assistant', + content: 'Answer', + extra: { + aws: { + reasoningContent: [ + { reasoningText: { text: 'Reasoning', signature: 'signature-1' } }, + { redactedContent: Buffer.from('redacted').toString('base64') }, + ], + }, + }, + }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'assistant', + content: [ + { + reasoningContent: { + reasoningText: { text: 'Reasoning', signature: 'signature-1' }, + }, + }, + { + reasoningContent: { redactedContent: Buffer.from('redacted') }, + }, + { text: 'Answer' }, + ], + }, + ]); + }); + it('should inject a dummy user message when the conversation does not start with user', async () => { const ctx = ChatContext.empty(); ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); diff --git a/agents/src/llm/provider_format/aws.ts b/agents/src/llm/provider_format/aws.ts index 7fc1f692f..717dcbcc1 100644 --- a/agents/src/llm/provider_format/aws.ts +++ b/agents/src/llm/provider_format/aws.ts @@ -10,6 +10,8 @@ export interface AwsFormatData { systemMessages: string[] | null; } +type TurnContentKind = 'conversation' | 'tool_result'; + const AWS_IMAGE_FORMATS: Record = { 'image/jpeg': 'jpeg', 'image/png': 'png', @@ -35,8 +37,16 @@ export async function toChatCtx( const messages: Record[] = []; const systemMessages: string[] = []; let currentRole: string | null = null; + let currentContentKind: TurnContentKind | null = null; let content: Record[] = []; + const flushTurn = () => { + if (currentRole !== null && content.length > 0) { + messages.push({ role: currentRole, content: [...content] }); + } + content = []; + }; + const itemGroups = groupToolCalls(chatCtx); const flattenedItems: ChatItem[] = []; for (const group of itemGroups) { @@ -65,21 +75,17 @@ export async function toChatCtx( continue; // Skip unknown message types (e.g. agent_handoff, agent_config_update) } - // If the effective role changed, finalize the previous turn - if (role !== currentRole) { - if (currentRole !== null && content.length > 0) { - messages.push({ role: currentRole, content: [...content] }); - } - content = []; - currentRole = role; - } + const nextContent: Record[] = []; + const contentKind: TurnContentKind = + msg.type === 'function_call_output' ? 'tool_result' : 'conversation'; if (msg.type === 'message') { + nextContent.push(...reasoningContentFromExtra(msg.extra)); for (const part of msg.content) { if (typeof part === 'string') { - if (part.trim()) content.push({ text: part }); + if (part.trim()) nextContent.push({ text: part }); } else if (isInstructions(part)) { - if (part.value.trim()) content.push({ text: part.value }); + if (part.value.trim()) nextContent.push({ text: part.value }); } else if ( role === 'user' && part && @@ -87,12 +93,13 @@ export async function toChatCtx( part.type === 'image_content' ) { // Bedrock Converse only accepts image blocks in user messages. - content.push(await toImagePart(part)); + nextContent.push(await toImagePart(part)); } // audio_content is intentionally skipped — Bedrock Converse has no raw audio block } } else if (msg.type === 'function_call') { - content.push({ + nextContent.push(...reasoningContentFromExtra(msg.extra)); + nextContent.push({ toolUse: { toolUseId: msg.callId, name: msg.name, @@ -100,7 +107,7 @@ export async function toChatCtx( }, }); } else if (msg.type === 'function_call_output') { - content.push({ + nextContent.push({ toolResult: { toolUseId: msg.callId, content: [{ text: msg.output.trim() ? msg.output : '(empty)' }], @@ -108,12 +115,23 @@ export async function toChatCtx( }, }); } + + // Do not let a skipped/empty item switch the active role. Doing so can split two real + // same-role messages into adjacent turns that Bedrock-hosted models reject. + if (nextContent.length === 0) continue; + + // Tool results must remain isolated from ordinary user conversation content. Bedrock + // models expect a tool-result turn to contain toolResult blocks only. + if (role !== currentRole || contentKind !== currentContentKind) { + flushTurn(); + currentRole = role; + currentContentKind = contentKind; + } + content.push(...nextContent); } // Finalize the last turn - if (currentRole !== null && content.length > 0) { - messages.push({ role: currentRole, content }); - } + flushTurn(); // Bedrock requires the message list to start with a "user" turn if (injectDummyUserMessage && (messages.length === 0 || messages[0]!.role !== 'user')) { @@ -139,6 +157,43 @@ export async function toChatCtx( ]; } +function reasoningContentFromExtra(extra: Record): Record[] { + const aws = extra.aws; + if (!aws || typeof aws !== 'object' || Array.isArray(aws)) return []; + + const reasoningContent = (aws as Record).reasoningContent; + if (!Array.isArray(reasoningContent)) return []; + + const blocks: Record[] = []; + for (const value of reasoningContent) { + if (!value || typeof value !== 'object' || Array.isArray(value)) continue; + const block = value as Record; + const reasoningText = block.reasoningText; + if (reasoningText && typeof reasoningText === 'object' && !Array.isArray(reasoningText)) { + const textBlock = reasoningText as Record; + if (typeof textBlock.text !== 'string') continue; + blocks.push({ + reasoningContent: { + reasoningText: { + text: textBlock.text, + ...(typeof textBlock.signature === 'string' ? { signature: textBlock.signature } : {}), + }, + }, + }); + continue; + } + + if (typeof block.redactedContent === 'string') { + blocks.push({ + reasoningContent: { + redactedContent: Buffer.from(block.redactedContent, 'base64'), + }, + }); + } + } + return blocks; +} + async function toImagePart(image: ImageContent): Promise> { const cacheKey = 'serialized_image'; if (!image._cache[cacheKey]) { diff --git a/plugins/aws/src/llm.test.ts b/plugins/aws/src/llm.test.ts index 2af84a3c6..24523dd71 100644 --- a/plugins/aws/src/llm.test.ts +++ b/plugins/aws/src/llm.test.ts @@ -4,12 +4,23 @@ import type { BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime'; import { APIStatusError, llm } from '@livekit/agents'; import { llm as llmTest } from '@livekit/agents-plugins-test'; -import { afterEach, describe, expect, it } from 'vitest'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; import { z } from 'zod'; import { LLM, buildToolConfig, mapConverseStreamException } from './llm.js'; const hasAwsCredentials = Boolean(process.env.AWS_ACCESS_KEY_ID || process.env.AWS_PROFILE); +// LLMStream reports terminal errors through the LLM `error` event and its background task also +// rejects during cleanup. Swallow the one deterministic failure exercised below. +const swallowExpectedFormatRejection = (reason: unknown) => { + if (reason instanceof Error && reason.message.includes('externalUrl images are not supported')) { + return; + } + throw reason; +}; +beforeAll(() => process.on('unhandledRejection', swallowExpectedFormatRejection)); +afterAll(() => void process.off('unhandledRejection', swallowExpectedFormatRejection)); + function fakeClient( events: Record[], requestId = 'req_123', @@ -81,6 +92,20 @@ describe('AWS Bedrock LLM - buildToolConfig', () => { expect(toolSpec.inputSchema.json.type).toBe('object'); expect(Object.keys(toolSpec.inputSchema.json.properties ?? {})).toEqual(['location']); }); + + it('omits a blank tool description', () => { + const toolCtx = llm.toToolContext({ + noDescription: llm.tool({ + description: ' ', + parameters: z.object({}), + execute: async () => 'ok', + }), + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const toolSpec = (buildToolConfig(toolCtx, 'auto', false)?.tools?.[0] as any).toolSpec; + + expect(toolSpec).not.toHaveProperty('description'); + }); }); describe('AWS Bedrock LLM - constructor', () => { @@ -179,6 +204,173 @@ describe('AWS Bedrock LLM - streaming', () => { expect(toolCalls[0]?.args).toBe('{"location":"Tokyo"}'); }); + it('preserves streamed reasoning text and its signature in response extra data', async () => { + const bedrockLlm = new LLM({ + client: fakeClient([ + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { reasoningContent: { text: 'Think ' } }, + }, + }, + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { reasoningContent: { text: 'carefully' } }, + }, + }, + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { reasoningContent: { signature: 'signature-1' } }, + }, + }, + { contentBlockStop: { contentBlockIndex: 0 } }, + { contentBlockDelta: { contentBlockIndex: 1, delta: { text: 'Answer' } } }, + ]), + }); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'Question' }); + + const response = await bedrockLlm.chat({ chatCtx }).collect(); + + expect(response.text).toBe('Answer'); + expect(response.extra).toEqual({ + aws: { + reasoningContent: [ + { reasoningText: { text: 'Think carefully', signature: 'signature-1' } }, + ], + }, + }); + }); + + it('attaches reasoning data to a tool-only response', async () => { + const bedrockLlm = new LLM({ + client: fakeClient([ + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { reasoningContent: { text: 'Need a tool' } }, + }, + }, + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { reasoningContent: { signature: 'tool-signature' } }, + }, + }, + { contentBlockStop: { contentBlockIndex: 0 } }, + { + contentBlockStart: { + contentBlockIndex: 1, + start: { toolUse: { toolUseId: 'call_reasoning', name: 'getWeather' } }, + }, + }, + { + contentBlockDelta: { + contentBlockIndex: 1, + delta: { toolUse: { input: '{}' } }, + }, + }, + { contentBlockStop: { contentBlockIndex: 1 } }, + ]), + }); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'Question' }); + + const response = await bedrockLlm.chat({ chatCtx }).collect(); + + expect(response.toolCalls[0]?.extra).toEqual({ + aws: { + reasoningContent: [{ reasoningText: { text: 'Need a tool', signature: 'tool-signature' } }], + }, + }); + }); + + it('omits prompt-managed request fields when model is a Prompt ARN', async () => { + let commandInput: Record | undefined; + const client = { + send: async (command: { input: Record }) => { + commandInput = command.input; + return { + $metadata: { requestId: 'req_prompt' }, + stream: (async function* () { + yield { contentBlockDelta: { delta: { text: 'ok' } } }; + })(), + }; + }, + } as unknown as BedrockRuntimeClient; + const bedrockLlm = new LLM({ + model: 'arn:aws:bedrock:us-east-1:123456789012:prompt/ABCDEFGHIJ:1', + temperature: 0.2, + additionalRequestFields: { reasoning_config: { type: 'enabled' } }, + cacheSystem: true, + client, + }); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'system', content: 'Managed by the prompt resource' }); + chatCtx.addMessage({ role: 'user', content: 'Hi' }); + + await bedrockLlm + .chat({ + chatCtx, + toolCtx: weatherToolCtx(), + extraKwargs: { + promptVariables: { topic: { text: 'weather' } }, + system: [{ text: 'must also be omitted' }], + }, + }) + .collect(); + + expect(commandInput).toMatchObject({ + modelId: 'arn:aws:bedrock:us-east-1:123456789012:prompt/ABCDEFGHIJ:1', + promptVariables: { topic: { text: 'weather' } }, + }); + expect(commandInput).not.toHaveProperty('additionalModelRequestFields'); + expect(commandInput).not.toHaveProperty('inferenceConfig'); + expect(commandInput).not.toHaveProperty('system'); + expect(commandInput).not.toHaveProperty('toolConfig'); + }); + + it('does not retry deterministic provider-format errors', async () => { + let attempts = 0; + const client = { + send: async () => { + attempts += 1; + return { $metadata: {}, stream: (async function* () {})() }; + }, + } as unknown as BedrockRuntimeClient; + const bedrockLlm = new LLM({ client }); + const emittedError = new Promise((resolve) => { + bedrockLlm.once('error', ({ error }) => resolve(error)); + }); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ + role: 'user', + content: [ + { + id: 'external-image', + type: 'image_content', + image: 'https://example.com/image.jpg', + inferenceDetail: 'auto', + _cache: {}, + }, + ], + }); + + await bedrockLlm + .chat({ + chatCtx, + connOptions: { maxRetry: 2, retryIntervalMs: 1, timeoutMs: 1000 }, + }) + .collect(); + + await expect(emittedError).resolves.toMatchObject({ + message: expect.stringMatching(/externalUrl images are not supported/), + }); + expect(attempts).toBe(0); + }); + it('does not apply the connection timeout to an established response stream', async () => { const client = { send: async (_command: unknown, { abortSignal }: { abortSignal: AbortSignal }) => ({ diff --git a/plugins/aws/src/llm.ts b/plugins/aws/src/llm.ts index 325a25203..ffacedf0e 100644 --- a/plugins/aws/src/llm.ts +++ b/plugins/aws/src/llm.ts @@ -25,6 +25,12 @@ interface AwsFormatData { systemMessages: string[] | null; } +interface AwsReasoningContentData { + reasoningText?: { text: string; signature?: string }; + /** Base64-encoded so ChatMessage.extra remains JSON serializable. */ + redactedContent?: string; +} + /** @public */ export interface LLMOptions { model?: string; @@ -59,17 +65,20 @@ export function buildToolConfig( // member, which defeats direct object-literal assignability checks against `Tool[]` here // (the same constraint the sibling google.ts/mistralai.ts adapters hit at this exact // ChatContext-to-SDK-type boundary) — build the array in its natural shape and cast once. - const tools: Record[] = llm.sortedToolEntries(toolCtx).map(([name, tool]) => ({ - toolSpec: { - name, - description: tool.description || '', - inputSchema: { - json: tool.parameters - ? llm.toJsonSchema(tool.parameters, false, false) - : { type: 'object', properties: {} }, + const tools: Record[] = llm.sortedToolEntries(toolCtx).map(([name, tool]) => { + const description = tool.description.trim(); + return { + toolSpec: { + name, + ...(description ? { description: tool.description } : {}), + inputSchema: { + json: tool.parameters + ? llm.toJsonSchema(tool.parameters, false, false) + : { type: 'object', properties: {} }, + }, }, - }, - })); + }; + }); if (cacheTools) { tools.push({ cachePoint: { type: 'default' } }); @@ -325,31 +334,47 @@ export class LLMStream extends llm.LLMStream { protected async run(): Promise { let retryable = true; let requestId = ''; - const request = createRequestSignal(this.abortController.signal, this.connOptions.timeoutMs); - try { - const [messages, extraData] = (await this.chatCtx.toProviderFormat('aws')) as [ - Record[], - AwsFormatData, - ]; - - const system: Record[] = []; - if (extraData.systemMessages) { - for (const content of extraData.systemMessages) { - system.push({ text: content }); - } - if (this.#cacheSystem) { - system.push({ cachePoint: { type: 'default' } }); - } + // Keep deterministic local formatting/validation failures outside the API retry wrapper. + // The base LLMStream will surface unknown errors immediately instead of retrying the same + // malformed context as a connection failure. + const [messages, extraData] = (await this.chatCtx.toProviderFormat('aws')) as [ + Record[], + AwsFormatData, + ]; + + const system: Record[] = []; + if (extraData.systemMessages) { + for (const content of extraData.systemMessages) { + system.push({ text: content }); + } + if (this.#cacheSystem) { + system.push({ cachePoint: { type: 'default' } }); } + } - const input: ConverseStreamCommandInput = { - modelId: this.#model, - messages: messages as unknown as Message[], - ...(system.length > 0 ? { system: system as unknown as SystemContentBlock[] } : {}), - ...this.#extraKwargs, - }; + const extraKwargs = { ...this.#extraKwargs }; + const isPromptArn = /^arn:[^:]+:bedrock:[^:]+:[^:]+:prompt\//.test(this.#model); + if (isPromptArn) { + // Prompt management owns these fields and rejects a Converse request that supplies them. + delete extraKwargs.additionalModelRequestFields; + delete extraKwargs.inferenceConfig; + delete extraKwargs.system; + delete extraKwargs.toolConfig; + } + + const input: ConverseStreamCommandInput = { + modelId: this.#model, + messages: messages as unknown as Message[], + ...(!isPromptArn && system.length > 0 + ? { system: system as unknown as SystemContentBlock[] } + : {}), + ...extraKwargs, + }; + + const request = createRequestSignal(this.abortController.signal, this.connOptions.timeoutMs); + try { const response = await this.#client.send(new ConverseStreamCommand(input), { abortSignal: request.signal, }); @@ -368,6 +393,11 @@ export class LLMStream extends llm.LLMStream { let toolCallId: string | undefined; let fncName: string | undefined; let fncRawArgs: string | undefined; + let hasTextOutput = false; + let reasoningText = ''; + let reasoningSignature: string | undefined; + let reasoningRedactedContent: Uint8Array[] = []; + const reasoningContent: AwsReasoningContentData[] = []; for await (const event of response.stream) { if (event.contentBlockStart?.start?.toolUse) { @@ -384,9 +414,45 @@ export class LLMStream extends llm.LLMStream { id: requestId, delta: { role: 'assistant', content: delta.text }, }); + hasTextOutput = true; + retryable = false; + } else if (delta.reasoningContent) { + const reasoningDelta = delta.reasoningContent; + if (reasoningDelta.text !== undefined) reasoningText += reasoningDelta.text; + if (reasoningDelta.signature !== undefined) { + reasoningSignature = reasoningDelta.signature; + } + if (reasoningDelta.redactedContent !== undefined) { + reasoningRedactedContent.push(reasoningDelta.redactedContent); + } retryable = false; } } else if (event.contentBlockStop) { + if (reasoningText || reasoningSignature || reasoningRedactedContent.length > 0) { + if (reasoningRedactedContent.length > 0) { + reasoningContent.push({ + redactedContent: Buffer.concat(reasoningRedactedContent).toString('base64'), + }); + } else { + reasoningContent.push({ + reasoningText: { + text: reasoningText, + ...(reasoningSignature ? { signature: reasoningSignature } : {}), + }, + }); + } + this.queue.put({ + id: requestId, + delta: { + role: 'assistant', + extra: { aws: { reasoningContent: [...reasoningContent] } }, + }, + }); + reasoningText = ''; + reasoningSignature = undefined; + reasoningRedactedContent = []; + } + if (toolCallId !== undefined) { this.queue.put({ id: requestId, @@ -397,6 +463,9 @@ export class LLMStream extends llm.LLMStream { callId: toolCallId, name: fncName ?? '', args: fncRawArgs ?? '', + ...(!hasTextOutput && reasoningContent.length > 0 + ? { extra: { aws: { reasoningContent: [...reasoningContent] } } } + : {}), }), ], }, diff --git a/plugins/aws/src/stt.test.ts b/plugins/aws/src/stt.test.ts index 223722a73..8e4d268d3 100644 --- a/plugins/aws/src/stt.test.ts +++ b/plugins/aws/src/stt.test.ts @@ -72,6 +72,77 @@ describe('AWS Transcribe STT - constructor', () => { expect(() => new STT({ identifyLanguage: true, languageOptions: 'en-US,es-US' })).not.toThrow(); }); + it('throws when languageOptions is provided without automatic language identification', () => { + expect(() => new STT({ languageOptions: 'en-US,es-US' })).toThrow( + /languageOptions requires identifyLanguage or identifyMultipleLanguages/, + ); + }); + + it('requires identifyLanguage for preferredLanguage', () => { + expect(() => new STT({ preferredLanguage: 'en-US' })).toThrow( + /preferredLanguage requires identifyLanguage/, + ); + expect( + () => + new STT({ + identifyMultipleLanguages: true, + languageOptions: 'en-US,es-US', + preferredLanguage: 'en-US', + }), + ).toThrow(/preferredLanguage requires identifyLanguage/); + }); + + it('accepts preferredLanguage with single-language identification', () => { + expect( + () => + new STT({ + identifyLanguage: true, + languageOptions: 'en-US,es-US', + preferredLanguage: 'en-US', + }), + ).not.toThrow(); + }); + + it('rejects plural vocabulary options for a fixed-language request', () => { + expect(() => new STT({ vocabularyNames: 'english,spanish' })).toThrow( + /vocabularyNames and vocabularyFilterNames require/, + ); + expect(() => new STT({ vocabularyFilterNames: 'english,spanish' })).toThrow( + /vocabularyNames and vocabularyFilterNames require/, + ); + }); + + it('accepts plural vocabulary options with automatic language identification', () => { + expect( + () => + new STT({ + identifyMultipleLanguages: true, + languageOptions: 'en-US,es-US', + vocabularyNames: 'english,spanish', + vocabularyFilterNames: 'english,spanish', + }), + ).not.toThrow(); + }); + + it('rejects singular vocabulary options with automatic language identification', () => { + expect( + () => + new STT({ + identifyLanguage: true, + languageOptions: 'en-US,es-US', + vocabularyName: 'english', + }), + ).toThrow(/vocabularyName and vocabFilterName cannot be used/); + expect( + () => + new STT({ + identifyLanguage: true, + languageOptions: 'en-US,es-US', + vocabFilterName: 'english', + }), + ).toThrow(/vocabularyName and vocabFilterName cannot be used/); + }); + it('throws when speaker labels and channel identification are both enabled', () => { expect(() => new STT({ showSpeakerLabel: true, enableChannelIdentification: true })).toThrow( /mutually exclusive/, diff --git a/plugins/aws/src/stt.ts b/plugins/aws/src/stt.ts index 06b1dc431..7498e7543 100644 --- a/plugins/aws/src/stt.ts +++ b/plugins/aws/src/stt.ts @@ -162,6 +162,40 @@ export class STT extends stt.STT { ); } + if (!identifyLanguage && !identifyMultipleLanguages && opts.languageOptions !== undefined) { + throw new Error( + 'languageOptions requires identifyLanguage or identifyMultipleLanguages to be true', + ); + } + + if (opts.preferredLanguage !== undefined && !identifyLanguage) { + throw new Error( + 'preferredLanguage requires identifyLanguage to be true and cannot be used with ' + + 'fixed-language or identifyMultipleLanguages requests', + ); + } + + if ( + !identifyLanguage && + !identifyMultipleLanguages && + (opts.vocabularyNames !== undefined || opts.vocabularyFilterNames !== undefined) + ) { + throw new Error( + 'vocabularyNames and vocabularyFilterNames require identifyLanguage or ' + + 'identifyMultipleLanguages; use vocabularyName or vocabFilterName for a fixed language', + ); + } + + if ( + (identifyLanguage || identifyMultipleLanguages) && + (opts.vocabularyName !== undefined || opts.vocabFilterName !== undefined) + ) { + throw new Error( + 'vocabularyName and vocabFilterName cannot be used with automatic language ' + + 'identification; use vocabularyNames or vocabularyFilterNames instead', + ); + } + if ((identifyLanguage || identifyMultipleLanguages) && opts.languageModelName) { throw new Error( 'languageModelName cannot be used with identifyLanguage or identifyMultipleLanguages ' + From 232228083a4c3fe77bd1533f0ff8bfd9e4175140 Mon Sep 17 00:00:00 2001 From: Sora Morimoto Date: Fri, 10 Jul 2026 14:58:36 +0900 Subject: [PATCH 10/13] fix(aws): trim Bedrock tool descriptions --- plugins/aws/src/llm.test.ts | 14 ++++++++++++++ plugins/aws/src/llm.ts | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/plugins/aws/src/llm.test.ts b/plugins/aws/src/llm.test.ts index 24523dd71..da2ec8646 100644 --- a/plugins/aws/src/llm.test.ts +++ b/plugins/aws/src/llm.test.ts @@ -106,6 +106,20 @@ describe('AWS Bedrock LLM - buildToolConfig', () => { expect(toolSpec).not.toHaveProperty('description'); }); + + it('trims a non-empty tool description', () => { + const toolCtx = llm.toToolContext({ + paddedDescription: llm.tool({ + description: ' Useful description. ', + parameters: z.object({}), + execute: async () => 'ok', + }), + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const toolSpec = (buildToolConfig(toolCtx, 'auto', false)?.tools?.[0] as any).toolSpec; + + expect(toolSpec.description).toBe('Useful description.'); + }); }); describe('AWS Bedrock LLM - constructor', () => { diff --git a/plugins/aws/src/llm.ts b/plugins/aws/src/llm.ts index ffacedf0e..a62faa2f6 100644 --- a/plugins/aws/src/llm.ts +++ b/plugins/aws/src/llm.ts @@ -70,7 +70,7 @@ export function buildToolConfig( return { toolSpec: { name, - ...(description ? { description: tool.description } : {}), + ...(description ? { description } : {}), inputSchema: { json: tool.parameters ? llm.toJsonSchema(tool.parameters, false, false) From 7eeaa8c908a7bdd3143155dec471b394d1912b07 Mon Sep 17 00:00:00 2001 From: Sora Morimoto Date: Fri, 10 Jul 2026 15:12:42 +0900 Subject: [PATCH 11/13] refactor(aws): centralise Bedrock error mapping and tidy stream state Reuse the shared toAwsApiError classifier in the Bedrock LLM stream instead of duplicating its logic, cache decoded image bytes so vision turns aren't re-decoded on every request, and simplify a couple of pieces of stream-tracking state in the Transcribe and Polly plugins that never actually held more than one item. --- agents/src/llm/provider_format/aws.ts | 7 ++++++- plugins/aws/src/llm.ts | 25 ++++++------------------ plugins/aws/src/stt.ts | 28 +++++++++++++++------------ plugins/aws/src/tts.ts | 7 +++++-- 4 files changed, 33 insertions(+), 34 deletions(-) diff --git a/agents/src/llm/provider_format/aws.ts b/agents/src/llm/provider_format/aws.ts index 717dcbcc1..56f96f800 100644 --- a/agents/src/llm/provider_format/aws.ts +++ b/agents/src/llm/provider_format/aws.ts @@ -207,10 +207,15 @@ async function toImagePart(image: ImageContent): Promise ); } + const bytesCacheKey = 'aws_image_bytes'; + if (!image._cache[bytesCacheKey]) { + image._cache[bytesCacheKey] = Buffer.from(img.base64Data ?? '', 'base64'); + } + return { image: { format: imageFormat(img.mimeType), - source: { bytes: Buffer.from(img.base64Data ?? '', 'base64') }, + source: { bytes: image._cache[bytesCacheKey] }, }, }; } diff --git a/plugins/aws/src/llm.ts b/plugins/aws/src/llm.ts index a62faa2f6..c84ddcee9 100644 --- a/plugins/aws/src/llm.ts +++ b/plugins/aws/src/llm.ts @@ -17,7 +17,7 @@ import { DEFAULT_API_CONNECT_OPTIONS, llm, } from '@livekit/agents'; -import { type AwsCredentials, createRequestSignal, resolveRegion } from './utils.js'; +import { type AwsCredentials, createRequestSignal, resolveRegion, toAwsApiError } from './utils.js'; const DEFAULT_MODEL = 'amazon.nova-2-lite-v1:0'; @@ -514,26 +514,13 @@ export class LLMStream extends llm.LLMStream { return; } - const err = error as { - message?: string; - $metadata?: { httpStatusCode?: number; requestId?: string }; - }; + const err = error as { $metadata?: { httpStatusCode?: number; requestId?: string } }; const statusCode = err.$metadata?.httpStatusCode; - if (statusCode !== undefined) { - throw new APIStatusError({ - message: `aws bedrock llm: ${err.message ?? 'unknown error'}`, - options: { - statusCode, - retryable: isRetryableBedrockStatus(statusCode, retryable), - requestId: err.$metadata?.requestId ?? requestId, - }, - }); - } - - throw new APIConnectionError({ - message: `aws bedrock llm: ${err.message ?? String(error)}`, - options: { retryable }, + throw toAwsApiError(error, 'aws bedrock llm', { + retryable: + statusCode !== undefined ? isRetryableBedrockStatus(statusCode, retryable) : retryable, + requestId: err.$metadata?.requestId ?? requestId, }); } finally { request.dispose(); diff --git a/plugins/aws/src/stt.ts b/plugins/aws/src/stt.ts index 7498e7543..248e2e237 100644 --- a/plugins/aws/src/stt.ts +++ b/plugins/aws/src/stt.ts @@ -293,7 +293,9 @@ export class SpeechStream extends stt.SpeechStream { // generator that is mid-await requeues its frame for the active session rather than // dropping the start of the next utterance. #channel = new AsyncIterableQueue(); - #requeuedFrames: Uint8Array[] = []; + // At most one frame is ever in flight for a superseded session, since takes are serialised + // via `#frameTakeChain` and a requeued frame is drained by the very next take. + #requeuedFrame: Uint8Array | undefined; #frameTakeChain: Promise = Promise.resolve(); #pumpStarted = false; @@ -501,15 +503,17 @@ export class SpeechStream extends stt.SpeechStream { return outcome; } - if (this.#requeuedFrames.length > 0) { - return { value: this.#requeuedFrames.shift()!, done: false }; + if (this.#requeuedFrame !== undefined) { + const value = this.#requeuedFrame; + this.#requeuedFrame = undefined; + return { value, done: false }; } const result = await this.#channel.next(); // Invalidated while awaiting — stash the frame for the active session instead of dropping it. if (!token.active) { if (!result.done) { - this.#requeuedFrames.push(result.value); + this.#requeuedFrame = result.value; } return outcome; } @@ -586,14 +590,14 @@ export class SpeechStream extends stt.SpeechStream { const identifiedLanguages = result.LanguageIdentification?.flatMap(({ LanguageCode }) => LanguageCode ? [normalizeLanguage(LanguageCode)] : [], ); - const sourceLanguages = - this.#opts.identifyLanguage || this.#opts.identifyMultipleLanguages - ? identifiedLanguages?.length - ? [...new Set(identifiedLanguages)] - : result.LanguageCode - ? [normalizeLanguage(result.LanguageCode)] - : undefined - : undefined; + let sourceLanguages: ReturnType[] | undefined; + if (this.#opts.identifyLanguage || this.#opts.identifyMultipleLanguages) { + if (identifiedLanguages?.length) { + sourceLanguages = [...new Set(identifiedLanguages)]; + } else if (result.LanguageCode) { + sourceLanguages = [normalizeLanguage(result.LanguageCode)]; + } + } const offset = this.startTimeOffset + this.#connectionTimeOffset; diff --git a/plugins/aws/src/tts.ts b/plugins/aws/src/tts.ts index db627ac45..9238e7713 100644 --- a/plugins/aws/src/tts.ts +++ b/plugins/aws/src/tts.ts @@ -190,9 +190,12 @@ export class ChunkedStream extends tts.ChunkedStream { // to 100ms), otherwise the tail of the synthesized audio is silently dropped. flush() // always returns one frame even with nothing buffered (0 samples), so drop it rather // than let it become the "final" frame in place of the real last frame from write(). - const frames = audioByteStream.write(audioBytes); + for (const frame of audioByteStream.write(audioBytes)) { + sendLastFrame(requestId, false); + lastFrame = frame; + } const remainder = audioByteStream.flush().filter((frame) => frame.samplesPerChannel > 0); - for (const frame of [...frames, ...remainder]) { + for (const frame of remainder) { sendLastFrame(requestId, false); lastFrame = frame; } From 6622b653a536bb7c9c9160fd621d2ef8f0d3b1a2 Mon Sep 17 00:00:00 2001 From: Sora Morimoto Date: Fri, 10 Jul 2026 15:49:16 +0900 Subject: [PATCH 12/13] fix(aws): handle model-specific Bedrock constraints --- agents/src/llm/provider_format/aws.test.ts | 4 +- agents/src/llm/provider_format/aws.ts | 12 +++-- plugins/aws/src/llm.test.ts | 59 +++++++++++++++++++++- plugins/aws/src/llm.ts | 30 ++++++++++- plugins/aws/src/stt.test.ts | 37 ++++++++++++++ plugins/aws/src/stt.ts | 11 ++++ 6 files changed, 146 insertions(+), 7 deletions(-) diff --git a/agents/src/llm/provider_format/aws.test.ts b/agents/src/llm/provider_format/aws.test.ts index 302a4e3c8..562044311 100644 --- a/agents/src/llm/provider_format/aws.test.ts +++ b/agents/src/llm/provider_format/aws.test.ts @@ -328,7 +328,7 @@ describe('AWS Provider Format - toChatCtx', () => { ]); }); - it('should keep tool results separate from a following user message', async () => { + it('should keep a following user message in the tool-result turn', async () => { const ctx = ChatContext.empty(); ctx.insert([ new FunctionCall({ @@ -367,9 +367,9 @@ describe('AWS Provider Format - toChatCtx', () => { status: 'success', }, }, + { text: 'A separate follow-up' }, ], }, - { role: 'user', content: [{ text: 'A separate follow-up' }] }, ]); }); diff --git a/agents/src/llm/provider_format/aws.ts b/agents/src/llm/provider_format/aws.ts index 56f96f800..c5e282bac 100644 --- a/agents/src/llm/provider_format/aws.ts +++ b/agents/src/llm/provider_format/aws.ts @@ -120,9 +120,15 @@ export async function toChatCtx( // same-role messages into adjacent turns that Bedrock-hosted models reject. if (nextContent.length === 0) continue; - // Tool results must remain isolated from ordinary user conversation content. Bedrock - // models expect a tool-result turn to contain toolResult blocks only. - if (role !== currentRole || contentKind !== currentContentKind) { + // Do not append a tool result to an earlier conversation-only user turn. In the reverse + // direction, keep a following user message in the tool-result turn so roles still alternate. + // Bedrock ContentBlock arrays allow both block types, whilst several hosted models reject + // consecutive user turns. + const startsToolResultAfterConversation = + role === currentRole && + contentKind === 'tool_result' && + currentContentKind === 'conversation'; + if (role !== currentRole || startsToolResultAfterConversation) { flushTurn(); currentRole = role; currentContentKind = contentKind; diff --git a/plugins/aws/src/llm.test.ts b/plugins/aws/src/llm.test.ts index da2ec8646..6351c95af 100644 --- a/plugins/aws/src/llm.test.ts +++ b/plugins/aws/src/llm.test.ts @@ -258,7 +258,7 @@ describe('AWS Bedrock LLM - streaming', () => { }); }); - it('attaches reasoning data to a tool-only response', async () => { + it('attaches reasoning data only to the first call in a tool-only response', async () => { const bedrockLlm = new LLM({ client: fakeClient([ { @@ -287,6 +287,19 @@ describe('AWS Bedrock LLM - streaming', () => { }, }, { contentBlockStop: { contentBlockIndex: 1 } }, + { + contentBlockStart: { + contentBlockIndex: 2, + start: { toolUse: { toolUseId: 'call_second', name: 'getWeather' } }, + }, + }, + { + contentBlockDelta: { + contentBlockIndex: 2, + delta: { toolUse: { input: '{"location":"London"}' } }, + }, + }, + { contentBlockStop: { contentBlockIndex: 2 } }, ]), }); const chatCtx = new llm.ChatContext(); @@ -299,6 +312,50 @@ describe('AWS Bedrock LLM - streaming', () => { reasoningContent: [{ reasoningText: { text: 'Need a tool', signature: 'tool-signature' } }], }, }); + expect(response.toolCalls[1]?.extra).toEqual({}); + }); + + it.each([ + ['amazon.nova-2-lite-v1:0', 'success'], + ['anthropic.claude-3-5-sonnet-20240620-v1:0', 'success'], + ['anthropic.claude-sonnet-4-20250514-v1:0', 'success'], + ['meta.llama3-3-70b-instruct-v1:0', undefined], + ])('serialises tool-result status for model %s as %s', async (model, expectedStatus) => { + let commandInput: Record | undefined; + const client = { + send: async (command: { input: Record }) => { + commandInput = command.input; + return { + $metadata: { requestId: 'req_tool_result_status' }, + stream: (async function* () { + yield { contentBlockDelta: { delta: { text: 'ok' } } }; + })(), + }; + }, + } as unknown as BedrockRuntimeClient; + const chatCtx = new llm.ChatContext(); + chatCtx.insert([ + llm.FunctionCall.create({ + callId: 'call_status', + name: 'getWeather', + args: '{"location":"London"}', + }), + llm.FunctionCallOutput.create({ + callId: 'call_status', + name: 'getWeather', + output: 'rainy', + isError: false, + }), + ]); + + await new LLM({ model, client }).chat({ chatCtx, toolCtx: weatherToolCtx() }).collect(); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const messages = commandInput?.messages as any[]; + const toolResult = messages + .flatMap((message) => message.content) + .find((block) => block.toolResult)?.toolResult; + expect(toolResult.status).toBe(expectedStatus); }); it('omits prompt-managed request fields when model is a Prompt ARN', async () => { diff --git a/plugins/aws/src/llm.ts b/plugins/aws/src/llm.ts index c84ddcee9..61fc79848 100644 --- a/plugins/aws/src/llm.ts +++ b/plugins/aws/src/llm.ts @@ -115,6 +115,29 @@ function isRetryableBedrockStatus(statusCode: number, retryable: boolean): boole return retryable && (statusCode === 408 || statusCode === 429 || statusCode >= 500); } +function supportsToolResultStatus(model: string): boolean { + const normalised = model.toLowerCase(); + if (normalised.includes('amazon.nova')) return true; + return /anthropic\.claude-(?:3(?:[-.:]|$)|[^/:]*-4(?:[-.:]|$))/.test(normalised); +} + +function omitUnsupportedToolResultStatuses( + messages: Record[], + model: string, +): void { + if (supportsToolResultStatus(model)) return; + + for (const message of messages) { + if (!Array.isArray(message.content)) continue; + for (const block of message.content) { + if (!block || typeof block !== 'object' || Array.isArray(block)) continue; + const toolResult = (block as Record).toolResult; + if (!toolResult || typeof toolResult !== 'object' || Array.isArray(toolResult)) continue; + delete (toolResult as Record).status; + } + } +} + const CONVERSE_STREAM_EXCEPTIONS: Array<{ key: keyof ConverseStreamExceptionEvent; defaultMessage: string; @@ -342,6 +365,7 @@ export class LLMStream extends llm.LLMStream { Record[], AwsFormatData, ]; + omitUnsupportedToolResultStatuses(messages, this.#model); const system: Record[] = []; if (extraData.systemMessages) { @@ -398,6 +422,7 @@ export class LLMStream extends llm.LLMStream { let reasoningSignature: string | undefined; let reasoningRedactedContent: Uint8Array[] = []; const reasoningContent: AwsReasoningContentData[] = []; + let reasoningAttachedToToolCall = false; for await (const event of response.stream) { if (event.contentBlockStart?.start?.toolUse) { @@ -454,6 +479,8 @@ export class LLMStream extends llm.LLMStream { } if (toolCallId !== undefined) { + const attachReasoning: boolean = + !hasTextOutput && reasoningContent.length > 0 && !reasoningAttachedToToolCall; this.queue.put({ id: requestId, delta: { @@ -463,7 +490,7 @@ export class LLMStream extends llm.LLMStream { callId: toolCallId, name: fncName ?? '', args: fncRawArgs ?? '', - ...(!hasTextOutput && reasoningContent.length > 0 + ...(attachReasoning ? { extra: { aws: { reasoningContent: [...reasoningContent] } } } : {}), }), @@ -471,6 +498,7 @@ export class LLMStream extends llm.LLMStream { }, }); retryable = false; + reasoningAttachedToToolCall ||= attachReasoning; toolCallId = undefined; fncName = undefined; fncRawArgs = undefined; diff --git a/plugins/aws/src/stt.test.ts b/plugins/aws/src/stt.test.ts index 8e4d268d3..752bebb63 100644 --- a/plugins/aws/src/stt.test.ts +++ b/plugins/aws/src/stt.test.ts @@ -149,6 +149,43 @@ describe('AWS Transcribe STT - constructor', () => { ); }); + it('rejects partial result stability when stabilisation is explicitly disabled', () => { + expect( + () => + new STT({ + enablePartialResultsStabilization: false, + partialResultsStability: 'high', + }), + ).toThrow(/partialResultsStability cannot be set/); + }); + + it('enables partial result stabilisation when a stability level is supplied', async () => { + let commandInput: Record | undefined; + const client = { + send: async (command: { input: Record }) => { + commandInput = command.input; + return { + $metadata: { requestId: 'req_stability' }, + TranscriptResultStream: (async function* () {})(), + }; + }, + } as unknown as TranscribeStreamingClient; + const speechStream = new STT({ + sampleRate: 16000, + partialResultsStability: 'high', + client, + }).stream(); + speechStream.endInput(); + for await (const _event of speechStream) { + // Drain the stream so the command is sent and the session completes. + } + + expect(commandInput).toMatchObject({ + EnablePartialResultsStabilization: true, + PartialResultsStability: 'high', + }); + }); + it('throws when automatic language identification uses a custom language model', () => { expect( () => diff --git a/plugins/aws/src/stt.ts b/plugins/aws/src/stt.ts index 248e2e237..fdf4584c7 100644 --- a/plugins/aws/src/stt.ts +++ b/plugins/aws/src/stt.ts @@ -146,6 +146,16 @@ export class STT extends stt.STT { const identifyLanguage = opts.identifyLanguage ?? false; const identifyMultipleLanguages = opts.identifyMultipleLanguages ?? false; + if ( + opts.enablePartialResultsStabilization === false && + opts.partialResultsStability !== undefined + ) { + throw new Error( + 'partialResultsStability cannot be set when enablePartialResultsStabilization is false', + ); + } + const enablePartialResultsStabilization = + opts.enablePartialResultsStabilization ?? opts.partialResultsStability !== undefined; if (opts.showSpeakerLabel && enableChannelIdentification) { throw new Error( @@ -227,6 +237,7 @@ export class STT extends stt.STT { identifyLanguage, identifyMultipleLanguages, enableChannelIdentification, + enablePartialResultsStabilization, numberOfChannels, // Auto language detection is mutually exclusive with a fixed language code. language: From d98d3e13c63819f7ad9e9a323574a0bc5272de6c Mon Sep 17 00:00:00 2001 From: Sora Morimoto Date: Fri, 10 Jul 2026 16:34:45 +0900 Subject: [PATCH 13/13] fix(aws): bridge Bedrock tool-result turns --- agents/src/llm/provider_format/aws.test.ts | 5 +++-- agents/src/llm/provider_format/aws.ts | 21 ++++++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/agents/src/llm/provider_format/aws.test.ts b/agents/src/llm/provider_format/aws.test.ts index 562044311..1d625c7ac 100644 --- a/agents/src/llm/provider_format/aws.test.ts +++ b/agents/src/llm/provider_format/aws.test.ts @@ -328,7 +328,7 @@ describe('AWS Provider Format - toChatCtx', () => { ]); }); - it('should keep a following user message in the tool-result turn', async () => { + it('should separate a following user message from a tool-result turn with an assistant bridge', async () => { const ctx = ChatContext.empty(); ctx.insert([ new FunctionCall({ @@ -367,9 +367,10 @@ describe('AWS Provider Format - toChatCtx', () => { status: 'success', }, }, - { text: 'A separate follow-up' }, ], }, + { role: 'assistant', content: [{ text: '.' }] }, + { role: 'user', content: [{ text: 'A separate follow-up' }] }, ]); }); diff --git a/agents/src/llm/provider_format/aws.ts b/agents/src/llm/provider_format/aws.ts index c5e282bac..bd863a6ca 100644 --- a/agents/src/llm/provider_format/aws.ts +++ b/agents/src/llm/provider_format/aws.ts @@ -120,15 +120,18 @@ export async function toChatCtx( // same-role messages into adjacent turns that Bedrock-hosted models reject. if (nextContent.length === 0) continue; - // Do not append a tool result to an earlier conversation-only user turn. In the reverse - // direction, keep a following user message in the tool-result turn so roles still alternate. - // Bedrock ContentBlock arrays allow both block types, whilst several hosted models reject - // consecutive user turns. - const startsToolResultAfterConversation = - role === currentRole && - contentKind === 'tool_result' && - currentContentKind === 'conversation'; - if (role !== currentRole || startsToolResultAfterConversation) { + // Bedrock rejects tool-result and conversation blocks in the same turn. Some hosted models + // also reject consecutive same-role turns, so bridge a same-role content-kind boundary with + // the smallest non-empty opposite-role turn rather than either mixing or emitting adjacent + // user turns. + if (role === currentRole && contentKind !== currentContentKind) { + flushTurn(); + messages.push({ + role: role === 'user' ? 'assistant' : 'user', + content: [{ text: '.' }], + }); + currentContentKind = contentKind; + } else if (role !== currentRole) { flushTurn(); currentRole = role; currentContentKind = contentKind;
LiveKit Ecosystem
Agents SDKsPython · Node.js