diff --git a/.changeset/aws-plugin.md b/.changeset/aws-plugin.md new file mode 100644 index 000000000..05779a8d4 --- /dev/null +++ b/.changeset/aws-plugin.md @@ -0,0 +1,6 @@ +--- +'@livekit/agents-plugin-aws': minor +'@livekit/agents': minor +--- + +Add the AWS plugin (`@livekit/agents-plugin-aws`), providing LLM (Amazon Bedrock Converse), STT (Amazon Transcribe streaming), and TTS (Amazon Polly). Adds an `'aws'` provider format to `@livekit/agents` for converting `ChatContext` to Bedrock Converse messages. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d205e932d..1ff8548da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,11 +49,16 @@ jobs: - 'plugins/test/**' plugins: - 'plugins/**' + aws_plugin: + - 'plugins/aws/**' examples: - 'examples/**' - name: Test agents if: steps.filter.outputs.agents-or-tests == 'true' || github.event_name == 'push' run: pnpm test agents + - name: Test AWS plugin + if: steps.filter.outputs.aws_plugin == 'true' || github.event_name == 'push' + run: pnpm vitest run plugins/aws/src # - name: Test examples # if: (steps.filter.outputs.examples == 'true' || github.event_name == 'push') # env: diff --git a/.gitignore b/.gitignore index a10050c2c..d64cb5a68 100644 --- a/.gitignore +++ b/.gitignore @@ -197,6 +197,7 @@ examples/src/test_*.ts !.changeset/*.md !**/README.md !**/CHANGELOG.md +!**/etc/*.api.md !CONTRIBUTING.md !CLAUDE.md !.CODE_OF_CONDUCT.md diff --git a/README.md b/README.md index e708dcf6e..3c2d1da19 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ Currently, only the following plugins are supported: | [@livekit/agents-plugin-phonic](https://www.npmjs.com/package/@livekit/agents-plugin-phonic) | Realtime | | [@livekit/agents-plugin-fishaudio](https://www.npmjs.com/package/@livekit/agents-plugin-fishaudio) | TTS | | [@livekit/agents-plugin-hume](https://www.npmjs.com/package/@livekit/agents-plugin-hume) | TTS | +| [@livekit/agents-plugin-aws](https://www.npmjs.com/package/@livekit/agents-plugin-aws) | LLM, STT, TTS | ## Docs and guides diff --git a/agents/src/llm/provider_format/aws.test.ts b/agents/src/llm/provider_format/aws.test.ts new file mode 100644 index 000000000..1d625c7ac --- /dev/null +++ b/agents/src/llm/provider_format/aws.test.ts @@ -0,0 +1,691 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { initializeLogger } from '../../log.js'; +import { + AgentHandoffItem, + ChatContext, + FunctionCall, + FunctionCallOutput, + Instructions, +} from '../chat_context.js'; +import { serializeImage } from '../utils.js'; +import { toChatCtx } from './aws.js'; + +vi.mock('../utils.js', () => ({ + serializeImage: vi.fn(), +})); + +describe('AWS Provider Format - toChatCtx', () => { + const serializeImageMock = vi.mocked(serializeImage); + + initializeLogger({ level: 'silent', pretty: false }); + + beforeEach(async () => { + vi.clearAllMocks(); + }); + + it('should convert simple text messages', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'Hello' }); + ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { role: 'user', content: [{ text: 'Hello' }] }, + { role: 'assistant', content: [{ text: 'Hi there!' }] }, + ]); + expect(formatData.systemMessages).toBeNull(); + }); + + it('should extract system messages separately', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'system', content: 'You are a helpful assistant' }); + ctx.addMessage({ role: 'user', content: 'Hello' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([{ role: 'user', content: [{ text: 'Hello' }] }]); + expect(formatData.systemMessages).toEqual(['You are a helpful assistant']); + }); + + it('should extract developer messages as system messages', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'developer', content: 'Be concise' }); + ctx.addMessage({ role: 'user', content: 'Hello' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([{ role: 'user', content: [{ text: 'Hello' }] }]); + expect(formatData.systemMessages).toEqual(['Be concise']); + }); + + it('should drop a system message with no text content instead of reattributing it to user', async () => { + serializeImageMock.mockResolvedValue({ + inferenceDetail: 'auto', + mimeType: 'image/png', + base64Data: 'aW1n', + }); + + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'system', + content: [ + { + id: 'img1', + type: 'image_content', + image: 'data:image/png;base64,aW1n', + inferenceDetail: 'auto', + _cache: {}, + }, + ], + }); + ctx.addMessage({ role: 'user', content: 'Hello' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + // The image-only system message must not surface as a 'user' turn. + expect(result).toEqual([{ role: 'user', content: [{ text: 'Hello' }] }]); + expect(formatData.systemMessages).toBeNull(); + }); + + it('should drop empty and whitespace-only text blocks', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'system', content: ' ' }); + ctx.addMessage({ role: 'user', content: '' }); + ctx.addMessage({ + role: 'user', + content: [ + ' ', + new Instructions({ audio: 'audio instructions', text: ' ' }).asModality('text'), + ], + }); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([]); + expect(formatData.systemMessages).toBeNull(); + }); + + it('should replace blank tool output with non-empty placeholder text', async () => { + const ctx = ChatContext.empty(); + ctx.insert( + new FunctionCall({ + id: 'func_blank_output', + callId: 'call_blank_output', + name: 'blank_output', + args: '{}', + }), + ); + ctx.insert( + new FunctionCallOutput({ + callId: 'call_blank_output', + output: ' ', + isError: false, + }), + ); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'assistant', + content: [ + { + toolUse: { + toolUseId: 'call_blank_output', + name: 'blank_output', + input: {}, + }, + }, + ], + }, + { + role: 'user', + content: [ + { + toolResult: { + toolUseId: 'call_blank_output', + content: [{ text: '(empty)' }], + status: 'success', + }, + }, + ], + }, + ]); + }); + + it('should render Instructions as their resolved value', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'system', + content: [ + new Instructions({ audio: 'audio instructions', text: 'text instructions' }).asModality( + 'text', + ), + ], + }); + ctx.addMessage({ role: 'user', content: 'Hello' }); + + const [, formatData] = await toChatCtx(ctx, false); + + expect(formatData.systemMessages).toEqual(['text instructions']); + }); + + it('should concatenate multiple system messages as separate entries', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'system', content: 'You are a helpful assistant' }); + ctx.addMessage({ role: 'system', content: 'Be concise in your responses' }); + ctx.addMessage({ role: 'user', content: 'Hello' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'user', + content: [ + { + text: 'New instructions received. Apply them carefully: Be concise in your responses', + }, + { text: 'Hello' }, + ], + }, + ]); + expect(formatData.systemMessages).toEqual(['You are a helpful assistant']); + }); + + it('should merge consecutive messages with the same effective role', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'First user message' }); + ctx.addMessage({ role: 'user', content: 'Second user message' }); + ctx.addMessage({ role: 'assistant', content: 'First assistant response' }); + ctx.addMessage({ role: 'assistant', content: 'Second assistant response' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'user', + content: [{ text: 'First user message' }, { text: 'Second user message' }], + }, + { + role: 'assistant', + content: [{ text: 'First assistant response' }, { text: 'Second assistant response' }], + }, + ]); + expect(formatData.systemMessages).toBeNull(); + }); + + it('should ignore empty opposite-role messages without splitting the active turn', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'First' }); + ctx.addMessage({ role: 'assistant', content: ' ' }); + ctx.addMessage({ role: 'user', content: 'Second' }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([{ role: 'user', content: [{ text: 'First' }, { text: 'Second' }] }]); + }); + + it('should map function calls to assistant toolUse blocks and outputs to user toolResult blocks', async () => { + const ctx = ChatContext.empty(); + + const msg = ctx.addMessage({ role: 'assistant', content: 'Let me check.' }); + const toolCall = FunctionCall.create({ + id: msg.id + '/tool_1', + callId: 'call_123', + name: 'get_weather', + args: '{"location": "San Francisco"}', + }); + const toolOutput = FunctionCallOutput.create({ + callId: 'call_123', + name: 'get_weather', + output: '{"temperature": 72, "condition": "sunny"}', + isError: false, + }); + + ctx.insert([toolCall, toolOutput]); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'assistant', + content: [ + { text: 'Let me check.' }, + { + toolUse: { + toolUseId: 'call_123', + name: 'get_weather', + input: { location: 'San Francisco' }, + }, + }, + ], + }, + { + role: 'user', + content: [ + { + toolResult: { + toolUseId: 'call_123', + content: [{ text: '{"temperature": 72, "condition": "sunny"}' }], + status: 'success', + }, + }, + ], + }, + ]); + expect(formatData.systemMessages).toBeNull(); + }); + + it('should map a failed tool call output to status "error"', async () => { + const ctx = ChatContext.empty(); + + const toolCall = new FunctionCall({ + id: 'func_error', + callId: 'call_error', + name: 'failing_function', + args: '{}', + }); + const toolOutput = new FunctionCallOutput({ + callId: 'call_error', + name: 'failing_function', + output: 'Function failed to execute', + isError: true, + }); + + ctx.insert([toolCall, toolOutput]); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'assistant', + content: [ + { + toolUse: { + toolUseId: 'call_error', + name: 'failing_function', + input: {}, + }, + }, + ], + }, + { + role: 'user', + content: [ + { + toolResult: { + toolUseId: 'call_error', + content: [{ text: 'Function failed to execute' }], + status: 'error', + }, + }, + ], + }, + ]); + }); + + it('should separate a following user message from a tool-result turn with an assistant bridge', async () => { + const ctx = ChatContext.empty(); + ctx.insert([ + new FunctionCall({ + id: 'func_separate', + callId: 'call_separate', + name: 'lookup', + args: '{}', + }), + new FunctionCallOutput({ + callId: 'call_separate', + name: 'lookup', + output: 'result', + isError: false, + }), + ]); + ctx.addMessage({ role: 'user', content: 'A separate follow-up' }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'assistant', + content: [ + { + toolUse: { toolUseId: 'call_separate', name: 'lookup', input: {} }, + }, + ], + }, + { + role: 'user', + content: [ + { + toolResult: { + toolUseId: 'call_separate', + content: [{ text: 'result' }], + status: 'success', + }, + }, + ], + }, + { role: 'assistant', content: [{ text: '.' }] }, + { role: 'user', content: [{ text: 'A separate follow-up' }] }, + ]); + }); + + it('should restore Bedrock reasoning content from provider-specific message data', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'assistant', + content: 'Answer', + extra: { + aws: { + reasoningContent: [ + { reasoningText: { text: 'Reasoning', signature: 'signature-1' } }, + { redactedContent: Buffer.from('redacted').toString('base64') }, + ], + }, + }, + }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'assistant', + content: [ + { + reasoningContent: { + reasoningText: { text: 'Reasoning', signature: 'signature-1' }, + }, + }, + { + reasoningContent: { redactedContent: Buffer.from('redacted') }, + }, + { text: 'Answer' }, + ], + }, + ]); + }); + + it('should inject a dummy user message when the conversation does not start with user', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); + + const [result, formatData] = await toChatCtx(ctx, true); + + expect(result).toEqual([ + { role: 'user', content: [{ text: '(empty)' }] }, + { role: 'assistant', content: [{ text: 'Hi there!' }] }, + { role: 'user', content: [{ text: '.' }] }, + ]); + expect(formatData.systemMessages).toBeNull(); + }); + + it('should not inject a leading dummy user message when the conversation already starts with user', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'Hello' }); + ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); + + const [result] = await toChatCtx(ctx, true); + + // Still gets a trailing dummy user turn since it ends on 'assistant'. + expect(result).toEqual([ + { role: 'user', content: [{ text: 'Hello' }] }, + { role: 'assistant', content: [{ text: 'Hi there!' }] }, + { role: 'user', content: [{ text: '.' }] }, + ]); + }); + + it('should not inject a trailing dummy user message when the conversation already ends on user', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'Hello' }); + ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); + ctx.addMessage({ role: 'user', content: 'Follow-up' }); + + const [result] = await toChatCtx(ctx, true); + + expect(result).toEqual([ + { role: 'user', content: [{ text: 'Hello' }] }, + { role: 'assistant', content: [{ text: 'Hi there!' }] }, + { role: 'user', content: [{ text: 'Follow-up' }] }, + ]); + }); + + it('should not inject a trailing dummy user message when disabled', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'Hello' }); + ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { role: 'user', content: [{ text: 'Hello' }] }, + { role: 'assistant', content: [{ text: 'Hi there!' }] }, + ]); + }); + + it('should not inject a dummy user message when disabled', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'system', content: 'You are helpful' }); + + const [result, formatData] = await toChatCtx(ctx, false); + + expect(result).toEqual([]); + expect(formatData.systemMessages).toEqual(['You are helpful']); + }); + + it('should inject a dummy user message for an entirely empty conversation', async () => { + const ctx = ChatContext.empty(); + + const [result] = await toChatCtx(ctx, true); + + expect(result).toEqual([{ role: 'user', content: [{ text: '(empty)' }] }]); + }); + + it('should convert supported inline images to Bedrock image blocks', async () => { + serializeImageMock.mockResolvedValue({ + inferenceDetail: 'auto', + mimeType: 'image/png', + base64Data: Buffer.from('fake-png-bytes').toString('base64'), + }); + + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'user', + content: [ + 'Look at this:', + { + id: 'img1', + type: 'image_content', + image: 'data:image/png;base64,ZmFrZS1wbmctYnl0ZXM=', + inferenceDetail: 'auto', + _cache: {}, + }, + ], + }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'user', + content: [ + { text: 'Look at this:' }, + { + image: { + format: 'png', + source: { bytes: Buffer.from('fake-png-bytes') }, + }, + }, + ], + }, + ]); + }); + + it('should drop images from assistant messages because Bedrock only accepts user images', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'Describe the image' }); + ctx.addMessage({ + role: 'assistant', + content: [ + 'Description', + { + id: 'img1', + type: 'image_content', + image: 'data:image/png;base64,ZmFrZS1wbmctYnl0ZXM=', + inferenceDetail: 'auto', + _cache: {}, + }, + ], + }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { role: 'user', content: [{ text: 'Describe the image' }] }, + { role: 'assistant', content: [{ text: 'Description' }] }, + ]); + expect(serializeImageMock).not.toHaveBeenCalled(); + }); + + it('should default to jpeg format when mimeType is missing', async () => { + serializeImageMock.mockResolvedValue({ + inferenceDetail: 'auto', + base64Data: Buffer.from('raw-bytes').toString('base64'), + }); + + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'user', + content: [ + { + id: 'img1', + type: 'image_content', + image: 'data:image/jpeg;base64,cmF3LWJ5dGVz', + inferenceDetail: 'auto', + _cache: {}, + }, + ], + }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { + role: 'user', + content: [ + { + image: { + format: 'jpeg', + source: { bytes: Buffer.from('raw-bytes') }, + }, + }, + ], + }, + ]); + }); + + it('should throw for unsupported image mime types', async () => { + serializeImageMock.mockResolvedValue({ + inferenceDetail: 'auto', + mimeType: 'image/bmp', + base64Data: Buffer.from('bmp-bytes').toString('base64'), + }); + + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'user', + content: [ + { + id: 'img1', + type: 'image_content', + image: 'data:image/bmp;base64,Ym1wLWJ5dGVz', + inferenceDetail: 'auto', + _cache: {}, + }, + ], + }); + + await expect(toChatCtx(ctx, false)).rejects.toThrow(/Unsupported mimeType/); + }); + + it('should throw for externalUrl images', async () => { + serializeImageMock.mockResolvedValue({ + inferenceDetail: 'high', + externalUrl: 'https://example.com/image.jpg', + mimeType: 'image/jpeg', + }); + + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'user', + content: [ + { + id: 'img1', + type: 'image_content', + image: 'https://example.com/image.jpg', + inferenceDetail: 'high', + _cache: {}, + }, + ], + }); + + await expect(toChatCtx(ctx, false)).rejects.toThrow(/externalUrl images are not supported/); + }); + + it('should filter out agent handoff items', async () => { + const ctx = ChatContext.empty(); + + ctx.addMessage({ role: 'user', content: 'Hello' }); + ctx.insert(new AgentHandoffItem({ oldAgentId: 'agent_1', newAgentId: 'agent_2' })); + ctx.addMessage({ role: 'assistant', content: 'Hi there!' }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { role: 'user', content: [{ text: 'Hello' }] }, + { role: 'assistant', content: [{ text: 'Hi there!' }] }, + ]); + }); + + it('should filter out standalone function calls without outputs', async () => { + const ctx = ChatContext.empty(); + + const funcCall = new FunctionCall({ + id: 'func_standalone', + callId: 'call_999', + name: 'standalone_function', + args: '{}', + }); + + ctx.insert(funcCall); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([]); + }); + + it('should skip empty groups', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'user', content: 'Hello', createdAt: 1000 }); + + const orphanOutput = new FunctionCallOutput({ + callId: 'orphan_call', + output: 'This should be ignored', + isError: false, + createdAt: 2000, + }); + ctx.insert(orphanOutput); + + ctx.addMessage({ role: 'assistant', content: 'Hi!', createdAt: 3000 }); + + const [result] = await toChatCtx(ctx, false); + + expect(result).toEqual([ + { role: 'user', content: [{ text: 'Hello' }] }, + { role: 'assistant', content: [{ text: 'Hi!' }] }, + ]); + }); +}); diff --git a/agents/src/llm/provider_format/aws.ts b/agents/src/llm/provider_format/aws.ts new file mode 100644 index 000000000..bd863a6ca --- /dev/null +++ b/agents/src/llm/provider_format/aws.ts @@ -0,0 +1,242 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { ChatContext, ChatItem, ImageContent } from '../chat_context.js'; +import { isInstructions } from '../chat_context.js'; +import { type SerializedImage, serializeImage } from '../utils.js'; +import { convertMidConversationInstructions, groupToolCalls } from './utils.js'; + +export interface AwsFormatData { + systemMessages: string[] | null; +} + +type TurnContentKind = 'conversation' | 'tool_result'; + +const AWS_IMAGE_FORMATS: Record = { + 'image/jpeg': 'jpeg', + 'image/png': 'png', + 'image/gif': 'gif', + 'image/webp': 'webp', +}; + +/** + * Convert a LiveKit ChatContext into AWS Bedrock Converse API message entries. + * + * - System messages are extracted separately (Bedrock takes `system` outside `messages`). + * - `function_call` items become assistant `toolUse` content blocks. + * - `function_call_output` items become user `toolResult` content blocks. + * - Consecutive same-role turns are merged (Bedrock requires strictly alternating roles). + * - A dummy user message is inserted at the front if the conversation doesn't start with one. + */ +export async function toChatCtx( + chatCtx: ChatContext, + injectDummyUserMessage: boolean = true, +): Promise<[Record[], AwsFormatData]> { + chatCtx = convertMidConversationInstructions(chatCtx); + + const messages: Record[] = []; + const systemMessages: string[] = []; + let currentRole: string | null = null; + let currentContentKind: TurnContentKind | null = null; + let content: Record[] = []; + + const flushTurn = () => { + if (currentRole !== null && content.length > 0) { + messages.push({ role: currentRole, content: [...content] }); + } + content = []; + }; + + const itemGroups = groupToolCalls(chatCtx); + const flattenedItems: ChatItem[] = []; + for (const group of itemGroups) { + flattenedItems.push(...group.flatten()); + } + + for (const msg of flattenedItems) { + if (msg.type === 'message' && (msg.role === 'system' || msg.role === 'developer')) { + // Always exclude system/developer messages from the regular role mapping below, even + // when they carry no usable text (e.g. image-only content) — they must never be + // reattributed to the user/assistant turn merge. + if (msg.textContent?.trim()) { + systemMessages.push(msg.textContent); + } + continue; + } + + let role: string; + if (msg.type === 'message') { + role = msg.role === 'assistant' ? 'assistant' : 'user'; + } else if (msg.type === 'function_call') { + role = 'assistant'; + } else if (msg.type === 'function_call_output') { + role = 'user'; + } else { + continue; // Skip unknown message types (e.g. agent_handoff, agent_config_update) + } + + const nextContent: Record[] = []; + const contentKind: TurnContentKind = + msg.type === 'function_call_output' ? 'tool_result' : 'conversation'; + + if (msg.type === 'message') { + nextContent.push(...reasoningContentFromExtra(msg.extra)); + for (const part of msg.content) { + if (typeof part === 'string') { + if (part.trim()) nextContent.push({ text: part }); + } else if (isInstructions(part)) { + if (part.value.trim()) nextContent.push({ text: part.value }); + } else if ( + role === 'user' && + part && + typeof part === 'object' && + part.type === 'image_content' + ) { + // Bedrock Converse only accepts image blocks in user messages. + nextContent.push(await toImagePart(part)); + } + // audio_content is intentionally skipped — Bedrock Converse has no raw audio block + } + } else if (msg.type === 'function_call') { + nextContent.push(...reasoningContentFromExtra(msg.extra)); + nextContent.push({ + toolUse: { + toolUseId: msg.callId, + name: msg.name, + input: JSON.parse(msg.args || '{}'), + }, + }); + } else if (msg.type === 'function_call_output') { + nextContent.push({ + toolResult: { + toolUseId: msg.callId, + content: [{ text: msg.output.trim() ? msg.output : '(empty)' }], + status: msg.isError ? 'error' : 'success', + }, + }); + } + + // Do not let a skipped/empty item switch the active role. Doing so can split two real + // same-role messages into adjacent turns that Bedrock-hosted models reject. + if (nextContent.length === 0) continue; + + // Bedrock rejects tool-result and conversation blocks in the same turn. Some hosted models + // also reject consecutive same-role turns, so bridge a same-role content-kind boundary with + // the smallest non-empty opposite-role turn rather than either mixing or emitting adjacent + // user turns. + if (role === currentRole && contentKind !== currentContentKind) { + flushTurn(); + messages.push({ + role: role === 'user' ? 'assistant' : 'user', + content: [{ text: '.' }], + }); + currentContentKind = contentKind; + } else if (role !== currentRole) { + flushTurn(); + currentRole = role; + currentContentKind = contentKind; + } + content.push(...nextContent); + } + + // Finalize the last turn + flushTurn(); + + // Bedrock requires the message list to start with a "user" turn + if (injectDummyUserMessage && (messages.length === 0 || messages[0]!.role !== 'user')) { + messages.unshift({ role: 'user', content: [{ text: '(empty)' }] }); + } + + // Some Bedrock-hosted models (e.g. Anthropic Claude) reject a request ending on an + // assistant turn, or silently treat it as a prefill continuation instead of generating a + // fresh reply. + if ( + injectDummyUserMessage && + messages.length > 0 && + messages[messages.length - 1]!.role === 'assistant' + ) { + messages.push({ role: 'user', content: [{ text: '.' }] }); + } + + return [ + messages, + { + systemMessages: systemMessages.length > 0 ? systemMessages : null, + }, + ]; +} + +function reasoningContentFromExtra(extra: Record): Record[] { + const aws = extra.aws; + if (!aws || typeof aws !== 'object' || Array.isArray(aws)) return []; + + const reasoningContent = (aws as Record).reasoningContent; + if (!Array.isArray(reasoningContent)) return []; + + const blocks: Record[] = []; + for (const value of reasoningContent) { + if (!value || typeof value !== 'object' || Array.isArray(value)) continue; + const block = value as Record; + const reasoningText = block.reasoningText; + if (reasoningText && typeof reasoningText === 'object' && !Array.isArray(reasoningText)) { + const textBlock = reasoningText as Record; + if (typeof textBlock.text !== 'string') continue; + blocks.push({ + reasoningContent: { + reasoningText: { + text: textBlock.text, + ...(typeof textBlock.signature === 'string' ? { signature: textBlock.signature } : {}), + }, + }, + }); + continue; + } + + if (typeof block.redactedContent === 'string') { + blocks.push({ + reasoningContent: { + redactedContent: Buffer.from(block.redactedContent, 'base64'), + }, + }); + } + } + return blocks; +} + +async function toImagePart(image: ImageContent): Promise> { + const cacheKey = 'serialized_image'; + if (!image._cache[cacheKey]) { + image._cache[cacheKey] = await serializeImage(image); + } + const img: SerializedImage = image._cache[cacheKey]; + + if (img.externalUrl) { + throw new Error( + 'externalUrl images are not supported by AWS Bedrock, provide inline image data instead', + ); + } + + const bytesCacheKey = 'aws_image_bytes'; + if (!image._cache[bytesCacheKey]) { + image._cache[bytesCacheKey] = Buffer.from(img.base64Data ?? '', 'base64'); + } + + return { + image: { + format: imageFormat(img.mimeType), + source: { bytes: image._cache[bytesCacheKey] }, + }, + }; +} + +function imageFormat(mimeType?: string): string { + if (!mimeType) return 'jpeg'; + + const format = AWS_IMAGE_FORMATS[mimeType]; + if (!format) { + throw new Error( + `Unsupported mimeType ${mimeType} for AWS Bedrock images. Must be jpeg, png, webp, or gif`, + ); + } + return format; +} diff --git a/agents/src/llm/provider_format/index.ts b/agents/src/llm/provider_format/index.ts index 56e014b7e..6b72b45af 100644 --- a/agents/src/llm/provider_format/index.ts +++ b/agents/src/llm/provider_format/index.ts @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 import type { ChatContext } from '../chat_context.js'; +import { toChatCtx as toChatCtxAws } from './aws.js'; import { toChatCtx as toChatCtxGoogle } from './google.js'; import { toChatCtx as toChatCtxMistralai } from './mistralai.js'; import { @@ -9,7 +10,7 @@ import { toResponsesChatCtx as toResponsesChatCtxOpenai, } from './openai.js'; -export type ProviderFormat = 'openai' | 'openai.responses' | 'google' | 'mistralai'; +export type ProviderFormat = 'openai' | 'openai.responses' | 'google' | 'mistralai' | 'aws'; export async function toChatCtx( format: ProviderFormat, @@ -25,6 +26,8 @@ export async function toChatCtx( return await toChatCtxGoogle(chatCtx, injectDummyUserMessage); case 'mistralai': return toChatCtxMistralai(chatCtx, injectDummyUserMessage); + case 'aws': + return await toChatCtxAws(chatCtx, injectDummyUserMessage); default: throw new Error(`Unsupported provider format: ${format}`); } diff --git a/examples/package.json b/examples/package.json index 2dfff00dc..6b5a4ec17 100644 --- a/examples/package.json +++ b/examples/package.json @@ -26,6 +26,7 @@ "@livekit/agents": "workspace:*", "@livekit/agents-plugin-anam": "workspace:*", "@livekit/agents-plugin-assemblyai": "workspace:*", + "@livekit/agents-plugin-aws": "workspace:*", "@livekit/agents-plugin-baseten": "workspace:*", "@livekit/agents-plugin-bey": "workspace:*", "@livekit/agents-plugin-cartesia": "workspace:*", diff --git a/examples/src/aws.ts b/examples/src/aws.ts new file mode 100644 index 000000000..17b5a4cf6 --- /dev/null +++ b/examples/src/aws.ts @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { + type JobContext, + ServerOptions, + cli, + defineAgent, + log, + metrics, + voice, +} from '@livekit/agents'; +import * as aws from '@livekit/agents-plugin-aws'; +import { fileURLToPath } from 'node:url'; + +// Demonstrates a fully AWS-backed voice pipeline: Amazon Transcribe (STT), +// Amazon Bedrock Converse (LLM), and Amazon Polly (TTS). +// +// Credentials are resolved via the AWS SDK v3 default credential chain +// (AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN, a shared +// profile, IMDS, etc.) — see plugins/aws/README.md for details. The region +// defaults to AWS_REGION / AWS_DEFAULT_REGION / "us-east-1". +export default defineAgent({ + entry: async (ctx: JobContext) => { + const agent = new voice.Agent({ + instructions: + "You are a helpful assistant, you can hear the user's message and respond to it.", + }); + + const logger = log(); + + const session = new voice.AgentSession({ + stt: new aws.STT({ language: 'en-US' }), + llm: new aws.LLM({ + // model defaults to BEDROCK_INFERENCE_PROFILE_ARN if set, otherwise 'amazon.nova-2-lite-v1:0' + }), + // Amazon Polly only supports PCM output at 8000 or 16000 Hz. + tts: new aws.TTS({ voice: 'Ruth', sampleRate: 16000 }), + turnHandling: { + turnDetection: 'stt', + }, + }); + + // Log metrics as they are emitted (session.usage is automatically collected) + session.on(voice.AgentSessionEventTypes.MetricsCollected, (ev) => { + metrics.logMetrics(ev.metrics); + }); + + // Log usage summary when job shuts down + ctx.addShutdownCallback(async () => { + logger.info( + { + usage: session.usage, + }, + 'Session usage summary', + ); + }); + + await session.start({ + agent, + room: ctx.room, + }); + + session.say('Hello, how can I help you today?'); + }, +}); + +cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) })); diff --git a/plugins/aws/README.md b/plugins/aws/README.md new file mode 100644 index 000000000..f56672ef0 --- /dev/null +++ b/plugins/aws/README.md @@ -0,0 +1,103 @@ + + +# AWS plugin for LiveKit Agents + +The Agents Framework is designed for building realtime, programmable +participants that run on servers. Use it to create conversational, multi-modal +voice agents that can see, hear, and understand. + +This package contains the AWS plugin, providing LLM (Amazon Bedrock Converse), +STT (Amazon Transcribe streaming), and TTS (Amazon Polly) capabilities via the +official AWS SDK v3 clients. + +## Installation + +```bash +npm install @livekit/agents-plugin-aws +``` + +## Credentials and region + +All three components resolve AWS credentials via the AWS SDK v3 default +credential chain (environment variables, shared config/credentials files, +IMDS, container credentials, etc.) unless `credentials` is passed explicitly: + +```ts +const llm = new aws.LLM({ + credentials: { + accessKeyId: '...', + secretAccessKey: '...', + sessionToken: '...', // optional + }, +}); +``` + +The region is resolved in this order: the `region` constructor option, then +`AWS_REGION`, then `AWS_DEFAULT_REGION`, falling back to `us-east-1`. + +## Usage + +### LLM + +Uses the [Bedrock Converse API](https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference.html) +with streaming. + +```ts +import * as aws from '@livekit/agents-plugin-aws'; + +const llm = new aws.LLM({ + // model defaults to BEDROCK_INFERENCE_PROFILE_ARN if set, otherwise 'amazon.nova-2-lite-v1:0' + model: 'anthropic.claude-3-5-sonnet-20241022-v2:0', + region: 'us-east-1', +}); +``` + +### STT + +Streaming-only, via [Amazon Transcribe streaming](https://docs.aws.amazon.com/transcribe/latest/dg/streaming.html). +Single-frame (non-streaming) recognition is not supported. + +```ts +const stt = new aws.STT({ + language: 'en-US', + sampleRate: 24000, +}); +``` + +Automatic language identification requires a companion `languageOptions` list +(comma-separated codes Amazon Transcribe should consider). AWS rejects the +request without it: + +```ts +const stt = new aws.STT({ + identifyLanguage: true, + languageOptions: 'en-US,es-US', + preferredLanguage: 'en-US', // optional +}); +``` + +### TTS + +Uses [Amazon Polly](https://docs.aws.amazon.com/polly/latest/dg/API_SynthesizeSpeech.html). +Audio is requested as raw PCM (16-bit little-endian, mono) since the framework +has no MP3 decoder — Amazon Polly only supports PCM output at **8000 Hz or +16000 Hz**, so `sampleRate` is restricted to those two values (default +`16000`). Streaming synthesis is not supported. + +```ts +const tts = new aws.TTS({ + voice: 'Ruth', + speechEngine: 'generative', + sampleRate: 16000, +}); +``` + +### Environment variables + +- `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` / `AWS_SESSION_TOKEN` — picked up by the default credential chain +- `AWS_REGION` / `AWS_DEFAULT_REGION` — region fallback when `region` isn't passed explicitly +- `BEDROCK_INFERENCE_PROFILE_ARN` — LLM model fallback when `model` isn't passed explicitly diff --git a/plugins/aws/api-extractor.json b/plugins/aws/api-extractor.json new file mode 100644 index 000000000..1f75e0708 --- /dev/null +++ b/plugins/aws/api-extractor.json @@ -0,0 +1,20 @@ +/** + * Config file for API Extractor. For more info, please visit: https://api-extractor.com + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + + /** + * Optionally specifies another JSON config file that this file extends from. This provides a way for + * standard settings to be shared across multiple projects. + * + * If the path starts with "./" or "../", the path is resolved relative to the folder of the file that contains + * the "extends" field. Otherwise, the first path segment is interpreted as an NPM package name, and will be + * resolved using NodeJS require(). + * + * SUPPORTED TOKENS: none + * DEFAULT VALUE: "" + */ + "extends": "../../api-extractor-shared.json", + "mainEntryPointFilePath": "./dist/index.d.ts" +} diff --git a/plugins/aws/etc/agents-plugin-aws.api.md b/plugins/aws/etc/agents-plugin-aws.api.md new file mode 100644 index 000000000..162fd41ed --- /dev/null +++ b/plugins/aws/etc/agents-plugin-aws.api.md @@ -0,0 +1,230 @@ +## API Report File for "@livekit/agents-plugin-aws" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import { APIConnectOptions } from '@livekit/agents'; +import { AudioBuffer as AudioBuffer_2 } from '@livekit/agents'; +import { BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime'; +import type { Engine } from '@aws-sdk/client-polly'; +import type { LanguageCode } from '@aws-sdk/client-transcribe-streaming'; +import type { LanguageCode as LanguageCode_2 } from '@aws-sdk/client-polly'; +import { llm } from '@livekit/agents'; +import type { PartialResultsStability } from '@aws-sdk/client-transcribe-streaming'; +import { PollyClient } from '@aws-sdk/client-polly'; +import { stt } from '@livekit/agents'; +import type { TextType } from '@aws-sdk/client-polly'; +import { TranscribeStreamingClient } from '@aws-sdk/client-transcribe-streaming'; +import { tts } from '@livekit/agents'; +import type { VocabularyFilterMethod } from '@aws-sdk/client-transcribe-streaming'; + +// @public +export interface AwsCredentials { + // (undocumented) + accessKeyId: string; + // (undocumented) + secretAccessKey: string; + // (undocumented) + sessionToken?: string; +} + +// @public (undocumented) +export class ChunkedStream extends tts.ChunkedStream { + constructor(tts: TTS, text: string, client: PollyClient, opts: TTSOptions, connOptions?: APIConnectOptions, abortSignal?: AbortSignal); + // (undocumented) + label: string; + // (undocumented) + protected run(): Promise; +} + +// @public +export class LLM extends llm.LLM { + constructor(opts?: LLMOptions); + // (undocumented) + aclose(): Promise; + // (undocumented) + chat({ chatCtx, toolCtx: toolCtxInput, connOptions, toolChoice, extraKwargs, }: { + chatCtx: llm.ChatContext; + toolCtx?: llm.ToolContextLike; + connOptions?: APIConnectOptions; + parallelToolCalls?: boolean; + toolChoice?: llm.ToolChoice; + extraKwargs?: Record; + }): LLMStream; + // (undocumented) + label(): string; + // (undocumented) + get model(): string; + // (undocumented) + get provider(): string; +} + +// @public (undocumented) +export interface LLMOptions { + // (undocumented) + additionalRequestFields?: Record; + cacheSystem?: boolean; + cacheTools?: boolean; + // (undocumented) + client?: BedrockRuntimeClient; + // (undocumented) + credentials?: AwsCredentials; + // (undocumented) + maxOutputTokens?: number; + // (undocumented) + model?: string; + // (undocumented) + region?: string; + // (undocumented) + temperature?: number; + // (undocumented) + toolChoice?: llm.ToolChoice; + // (undocumented) + topP?: number; +} + +// @public (undocumented) +export class LLMStream extends llm.LLMStream { + constructor(llmInst: LLM, { client, model, chatCtx, toolCtx, connOptions, cacheSystem, extraKwargs, }: { + client: BedrockRuntimeClient; + model: string; + chatCtx: llm.ChatContext; + toolCtx?: llm.ToolContext; + connOptions: APIConnectOptions; + cacheSystem: boolean; + extraKwargs: Record; + }); + // (undocumented) + protected run(): Promise; +} + +// @public (undocumented) +export class SpeechStream extends stt.SpeechStream { + constructor(stt: STT, opts: STTOptions, client: TranscribeStreamingClient, connOptions?: APIConnectOptions); + // (undocumented) + label: string; + // (undocumented) + protected run(): Promise; +} + +// @public +export class STT extends stt.STT { + constructor(opts?: Partial); + // (undocumented) + close(): Promise; + // (undocumented) + label: string; + // (undocumented) + get model(): string; + // (undocumented) + get provider(): string; + // (undocumented) + _recognize(_frame: AudioBuffer_2, _abortSignal?: AbortSignal): Promise; + // (undocumented) + stream(options?: { + connOptions?: APIConnectOptions; + }): SpeechStream; +} + +// @public (undocumented) +export interface STTOptions { + // (undocumented) + client?: TranscribeStreamingClient; + // (undocumented) + credentials?: AwsCredentials; + // (undocumented) + enableChannelIdentification?: boolean; + // (undocumented) + enablePartialResultsStabilization?: boolean; + // (undocumented) + identifyLanguage?: boolean; + // (undocumented) + identifyMultipleLanguages?: boolean; + // (undocumented) + language?: LanguageCode; + // (undocumented) + languageModelName?: string; + languageOptions?: string; + // (undocumented) + numberOfChannels?: number; + // (undocumented) + partialResultsStability?: PartialResultsStability; + // (undocumented) + preferredLanguage?: LanguageCode; + // (undocumented) + region?: string; + // (undocumented) + sampleRate: number; + // (undocumented) + sessionId?: string; + // (undocumented) + showSpeakerLabel?: boolean; + // (undocumented) + vocabFilterMethod?: VocabularyFilterMethod; + // (undocumented) + vocabFilterName?: string; + // (undocumented) + vocabularyFilterNames?: string; + // (undocumented) + vocabularyName?: string; + // (undocumented) + vocabularyNames?: string; +} + +// @public +export class TTS extends tts.TTS { + constructor(opts?: Partial); + // (undocumented) + close(): Promise; + // (undocumented) + label: string; + // (undocumented) + get model(): string; + // (undocumented) + get provider(): string; + // (undocumented) + stream(): tts.SynthesizeStream; + // (undocumented) + synthesize(text: string, connOptions?: APIConnectOptions, abortSignal?: AbortSignal): ChunkedStream; + // (undocumented) + updateOptions(opts: { + voice?: string; + language?: TTSLanguage; + speechEngine?: TTSSpeechEngine; + textType?: TTSTextType; + }): void; +} + +// @public +export type TTSLanguage = LanguageCode_2; + +// @public (undocumented) +export interface TTSOptions { + // (undocumented) + client?: PollyClient; + // (undocumented) + credentials?: AwsCredentials; + // (undocumented) + language?: TTSLanguage; + // (undocumented) + region?: string; + // (undocumented) + sampleRate: number; + // (undocumented) + speechEngine: TTSSpeechEngine; + // (undocumented) + textType: TTSTextType; + // (undocumented) + voice: string; +} + +// @public +export type TTSSpeechEngine = Engine; + +// @public +export type TTSTextType = TextType; + +// (No @packageDocumentation comment for this package) + +``` diff --git a/plugins/aws/package.json b/plugins/aws/package.json new file mode 100644 index 000000000..7fec270b4 --- /dev/null +++ b/plugins/aws/package.json @@ -0,0 +1,56 @@ +{ + "name": "@livekit/agents-plugin-aws", + "version": "1.0.0", + "description": "AWS plugin for LiveKit Node Agents", + "main": "dist/index.js", + "require": "dist/index.cjs", + "types": "dist/index.d.ts", + "exports": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, + "author": "LiveKit", + "type": "module", + "repository": "git@github.com:livekit/agents-js.git", + "license": "Apache-2.0", + "files": [ + "dist", + "src", + "!src/**/*.test.ts", + "README.md" + ], + "scripts": { + "build": "tsup --onSuccess \"pnpm build:types\"", + "build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js", + "clean": "rm -rf dist", + "clean:build": "pnpm clean && pnpm build", + "lint": "eslint -f unix \"src/**/*.{ts,js}\"", + "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript", + "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose" + }, + "devDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/agents-plugin-silero": "workspace:*", + "@livekit/agents-plugins-test": "workspace:*", + "@livekit/rtc-node": "catalog:", + "@microsoft/api-extractor": "^7.35.0", + "tsup": "^8.3.5", + "typescript": "^5.0.0", + "zod": "^3.25.76 || ^4.1.8" + }, + "dependencies": { + "@aws-sdk/client-bedrock-runtime": "^3.1081.0", + "@aws-sdk/client-polly": "^3.1081.0", + "@aws-sdk/client-transcribe-streaming": "^3.1081.0" + }, + "peerDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/rtc-node": "catalog:" + } +} diff --git a/plugins/aws/src/index.ts b/plugins/aws/src/index.ts new file mode 100644 index 000000000..b8314017f --- /dev/null +++ b/plugins/aws/src/index.ts @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { Plugin } from '@livekit/agents'; + +export { LLM, LLMStream, type LLMOptions } from './llm.js'; +export * from './models.js'; +export { SpeechStream, STT, type STTOptions } from './stt.js'; +export { ChunkedStream, TTS, type TTSOptions } from './tts.js'; +export type { AwsCredentials } from './utils.js'; + +class AwsPlugin extends Plugin { + constructor() { + super({ + title: 'aws', + version: __PACKAGE_VERSION__, + package: __PACKAGE_NAME__, + }); + } +} + +Plugin.registerPlugin(new AwsPlugin()); diff --git a/plugins/aws/src/llm.test.ts b/plugins/aws/src/llm.test.ts new file mode 100644 index 000000000..6351c95af --- /dev/null +++ b/plugins/aws/src/llm.test.ts @@ -0,0 +1,594 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime'; +import { APIStatusError, llm } from '@livekit/agents'; +import { llm as llmTest } from '@livekit/agents-plugins-test'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { LLM, buildToolConfig, mapConverseStreamException } from './llm.js'; + +const hasAwsCredentials = Boolean(process.env.AWS_ACCESS_KEY_ID || process.env.AWS_PROFILE); + +// LLMStream reports terminal errors through the LLM `error` event and its background task also +// rejects during cleanup. Swallow the one deterministic failure exercised below. +const swallowExpectedFormatRejection = (reason: unknown) => { + if (reason instanceof Error && reason.message.includes('externalUrl images are not supported')) { + return; + } + throw reason; +}; +beforeAll(() => process.on('unhandledRejection', swallowExpectedFormatRejection)); +afterAll(() => void process.off('unhandledRejection', swallowExpectedFormatRejection)); + +function fakeClient( + events: Record[], + requestId = 'req_123', +): BedrockRuntimeClient { + return { + send: async () => ({ + $metadata: { requestId }, + stream: (async function* () { + yield* events; + })(), + }), + } as unknown as BedrockRuntimeClient; +} + +function weatherToolCtx() { + return llm.toToolContext({ + getWeather: llm.tool({ + description: 'Get the weather for a given location.', + parameters: z.object({ location: z.string() }), + execute: async () => 'sunny', + }), + }); +} + +describe('AWS Bedrock LLM - buildToolConfig', () => { + it('returns undefined when there are no tools', () => { + expect(buildToolConfig(llm.ToolContext.empty(), undefined, false)).toBeUndefined(); + }); + + it('returns undefined when toolChoice is "none"', () => { + expect(buildToolConfig(weatherToolCtx(), 'none', false)).toBeUndefined(); + }); + + it('maps toolChoice "auto" to {auto: {}}', () => { + expect(buildToolConfig(weatherToolCtx(), 'auto', false)?.toolChoice).toEqual({ auto: {} }); + }); + + it('maps toolChoice "required" to {any: {}}', () => { + expect(buildToolConfig(weatherToolCtx(), 'required', false)?.toolChoice).toEqual({ any: {} }); + }); + + it('maps a named function toolChoice to {tool: {name}}', () => { + const config = buildToolConfig( + weatherToolCtx(), + { type: 'function', function: { name: 'getWeather' } }, + false, + ); + expect(config?.toolChoice).toEqual({ tool: { name: 'getWeather' } }); + }); + + it('leaves toolChoice unset when not specified', () => { + const config = buildToolConfig(weatherToolCtx(), undefined, false); + expect(config?.toolChoice).toBeUndefined(); + expect(config?.tools).toHaveLength(1); + }); + + it('appends a cachePoint block when cacheTools is enabled', () => { + const config = buildToolConfig(weatherToolCtx(), 'auto', true); + expect(config?.tools).toHaveLength(2); + expect(config?.tools?.[1]).toEqual({ cachePoint: { type: 'default' } }); + }); + + it('builds a valid toolSpec from a real ToolContext', () => { + const config = buildToolConfig(weatherToolCtx(), 'auto', false); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const toolSpec = (config?.tools?.[0] as any).toolSpec; + expect(toolSpec.name).toBe('getWeather'); + expect(toolSpec.description).toBe('Get the weather for a given location.'); + expect(toolSpec.inputSchema.json.type).toBe('object'); + expect(Object.keys(toolSpec.inputSchema.json.properties ?? {})).toEqual(['location']); + }); + + it('omits a blank tool description', () => { + const toolCtx = llm.toToolContext({ + noDescription: llm.tool({ + description: ' ', + parameters: z.object({}), + execute: async () => 'ok', + }), + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const toolSpec = (buildToolConfig(toolCtx, 'auto', false)?.tools?.[0] as any).toolSpec; + + expect(toolSpec).not.toHaveProperty('description'); + }); + + it('trims a non-empty tool description', () => { + const toolCtx = llm.toToolContext({ + paddedDescription: llm.tool({ + description: ' Useful description. ', + parameters: z.object({}), + execute: async () => 'ok', + }), + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const toolSpec = (buildToolConfig(toolCtx, 'auto', false)?.tools?.[0] as any).toolSpec; + + expect(toolSpec.description).toBe('Useful description.'); + }); +}); + +describe('AWS Bedrock LLM - constructor', () => { + const originalEnv = process.env.BEDROCK_INFERENCE_PROFILE_ARN; + + afterEach(() => { + if (originalEnv === undefined) { + delete process.env.BEDROCK_INFERENCE_PROFILE_ARN; + } else { + process.env.BEDROCK_INFERENCE_PROFILE_ARN = originalEnv; + } + }); + + it('defaults the model to amazon.nova-2-lite-v1:0', () => { + delete process.env.BEDROCK_INFERENCE_PROFILE_ARN; + const bedrockLlm = new LLM({ client: fakeClient([]) }); + expect(bedrockLlm.model).toBe('amazon.nova-2-lite-v1:0'); + }); + + it('falls back to BEDROCK_INFERENCE_PROFILE_ARN when model is not given', () => { + process.env.BEDROCK_INFERENCE_PROFILE_ARN = + 'arn:aws:bedrock:us-east-1:123456789012:inference-profile/foo'; + const bedrockLlm = new LLM({ client: fakeClient([]) }); + expect(bedrockLlm.model).toBe('arn:aws:bedrock:us-east-1:123456789012:inference-profile/foo'); + }); + + it('prefers an explicit model over BEDROCK_INFERENCE_PROFILE_ARN', () => { + process.env.BEDROCK_INFERENCE_PROFILE_ARN = + 'arn:aws:bedrock:us-east-1:123456789012:inference-profile/foo'; + const bedrockLlm = new LLM({ model: 'anthropic.claude-3-5-sonnet', client: fakeClient([]) }); + expect(bedrockLlm.model).toBe('anthropic.claude-3-5-sonnet'); + }); + + it('reports the provider label', () => { + const bedrockLlm = new LLM({ client: fakeClient([]) }); + expect(bedrockLlm.provider).toBe('AWS Bedrock'); + expect(bedrockLlm.label()).toBe('aws.LLM'); + }); +}); + +describe('AWS Bedrock LLM - streaming', () => { + it('emits text deltas and usage from the Converse stream', async () => { + const bedrockLlm = new LLM({ + client: fakeClient([ + { contentBlockDelta: { contentBlockIndex: 0, delta: { text: 'Hello' } } }, + { contentBlockDelta: { contentBlockIndex: 0, delta: { text: ' world' } } }, + { metadata: { usage: { inputTokens: 5, outputTokens: 2, totalTokens: 7 } } }, + ]), + }); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'Hi' }); + + const chunks: llm.ChatChunk[] = []; + for await (const chunk of bedrockLlm.chat({ chatCtx })) { + chunks.push(chunk); + } + + expect(chunks.map((c) => c.delta?.content ?? '').join('')).toBe('Hello world'); + expect(chunks.at(-1)?.usage).toEqual({ + completionTokens: 2, + promptTokens: 5, + totalTokens: 7, + promptCachedTokens: 0, + }); + }); + + it('emits a FunctionCall assembled from toolUse content blocks', async () => { + const bedrockLlm = new LLM({ + client: fakeClient([ + { + contentBlockStart: { + contentBlockIndex: 0, + start: { toolUse: { toolUseId: 'call_1', name: 'getWeather' } }, + }, + }, + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { toolUse: { input: '{"location":"Tokyo"}' } }, + }, + }, + { contentBlockStop: { contentBlockIndex: 0 } }, + ]), + }); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'weather?' }); + + const toolCalls: llm.FunctionCall[] = []; + for await (const chunk of bedrockLlm.chat({ chatCtx })) { + if (chunk.delta?.toolCalls) toolCalls.push(...chunk.delta.toolCalls); + } + + expect(toolCalls).toHaveLength(1); + expect(toolCalls[0]?.callId).toBe('call_1'); + expect(toolCalls[0]?.name).toBe('getWeather'); + expect(toolCalls[0]?.args).toBe('{"location":"Tokyo"}'); + }); + + it('preserves streamed reasoning text and its signature in response extra data', async () => { + const bedrockLlm = new LLM({ + client: fakeClient([ + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { reasoningContent: { text: 'Think ' } }, + }, + }, + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { reasoningContent: { text: 'carefully' } }, + }, + }, + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { reasoningContent: { signature: 'signature-1' } }, + }, + }, + { contentBlockStop: { contentBlockIndex: 0 } }, + { contentBlockDelta: { contentBlockIndex: 1, delta: { text: 'Answer' } } }, + ]), + }); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'Question' }); + + const response = await bedrockLlm.chat({ chatCtx }).collect(); + + expect(response.text).toBe('Answer'); + expect(response.extra).toEqual({ + aws: { + reasoningContent: [ + { reasoningText: { text: 'Think carefully', signature: 'signature-1' } }, + ], + }, + }); + }); + + it('attaches reasoning data only to the first call in a tool-only response', async () => { + const bedrockLlm = new LLM({ + client: fakeClient([ + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { reasoningContent: { text: 'Need a tool' } }, + }, + }, + { + contentBlockDelta: { + contentBlockIndex: 0, + delta: { reasoningContent: { signature: 'tool-signature' } }, + }, + }, + { contentBlockStop: { contentBlockIndex: 0 } }, + { + contentBlockStart: { + contentBlockIndex: 1, + start: { toolUse: { toolUseId: 'call_reasoning', name: 'getWeather' } }, + }, + }, + { + contentBlockDelta: { + contentBlockIndex: 1, + delta: { toolUse: { input: '{}' } }, + }, + }, + { contentBlockStop: { contentBlockIndex: 1 } }, + { + contentBlockStart: { + contentBlockIndex: 2, + start: { toolUse: { toolUseId: 'call_second', name: 'getWeather' } }, + }, + }, + { + contentBlockDelta: { + contentBlockIndex: 2, + delta: { toolUse: { input: '{"location":"London"}' } }, + }, + }, + { contentBlockStop: { contentBlockIndex: 2 } }, + ]), + }); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'Question' }); + + const response = await bedrockLlm.chat({ chatCtx }).collect(); + + expect(response.toolCalls[0]?.extra).toEqual({ + aws: { + reasoningContent: [{ reasoningText: { text: 'Need a tool', signature: 'tool-signature' } }], + }, + }); + expect(response.toolCalls[1]?.extra).toEqual({}); + }); + + it.each([ + ['amazon.nova-2-lite-v1:0', 'success'], + ['anthropic.claude-3-5-sonnet-20240620-v1:0', 'success'], + ['anthropic.claude-sonnet-4-20250514-v1:0', 'success'], + ['meta.llama3-3-70b-instruct-v1:0', undefined], + ])('serialises tool-result status for model %s as %s', async (model, expectedStatus) => { + let commandInput: Record | undefined; + const client = { + send: async (command: { input: Record }) => { + commandInput = command.input; + return { + $metadata: { requestId: 'req_tool_result_status' }, + stream: (async function* () { + yield { contentBlockDelta: { delta: { text: 'ok' } } }; + })(), + }; + }, + } as unknown as BedrockRuntimeClient; + const chatCtx = new llm.ChatContext(); + chatCtx.insert([ + llm.FunctionCall.create({ + callId: 'call_status', + name: 'getWeather', + args: '{"location":"London"}', + }), + llm.FunctionCallOutput.create({ + callId: 'call_status', + name: 'getWeather', + output: 'rainy', + isError: false, + }), + ]); + + await new LLM({ model, client }).chat({ chatCtx, toolCtx: weatherToolCtx() }).collect(); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const messages = commandInput?.messages as any[]; + const toolResult = messages + .flatMap((message) => message.content) + .find((block) => block.toolResult)?.toolResult; + expect(toolResult.status).toBe(expectedStatus); + }); + + it('omits prompt-managed request fields when model is a Prompt ARN', async () => { + let commandInput: Record | undefined; + const client = { + send: async (command: { input: Record }) => { + commandInput = command.input; + return { + $metadata: { requestId: 'req_prompt' }, + stream: (async function* () { + yield { contentBlockDelta: { delta: { text: 'ok' } } }; + })(), + }; + }, + } as unknown as BedrockRuntimeClient; + const bedrockLlm = new LLM({ + model: 'arn:aws:bedrock:us-east-1:123456789012:prompt/ABCDEFGHIJ:1', + temperature: 0.2, + additionalRequestFields: { reasoning_config: { type: 'enabled' } }, + cacheSystem: true, + client, + }); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'system', content: 'Managed by the prompt resource' }); + chatCtx.addMessage({ role: 'user', content: 'Hi' }); + + await bedrockLlm + .chat({ + chatCtx, + toolCtx: weatherToolCtx(), + extraKwargs: { + promptVariables: { topic: { text: 'weather' } }, + system: [{ text: 'must also be omitted' }], + }, + }) + .collect(); + + expect(commandInput).toMatchObject({ + modelId: 'arn:aws:bedrock:us-east-1:123456789012:prompt/ABCDEFGHIJ:1', + promptVariables: { topic: { text: 'weather' } }, + }); + expect(commandInput).not.toHaveProperty('additionalModelRequestFields'); + expect(commandInput).not.toHaveProperty('inferenceConfig'); + expect(commandInput).not.toHaveProperty('system'); + expect(commandInput).not.toHaveProperty('toolConfig'); + }); + + it('does not retry deterministic provider-format errors', async () => { + let attempts = 0; + const client = { + send: async () => { + attempts += 1; + return { $metadata: {}, stream: (async function* () {})() }; + }, + } as unknown as BedrockRuntimeClient; + const bedrockLlm = new LLM({ client }); + const emittedError = new Promise((resolve) => { + bedrockLlm.once('error', ({ error }) => resolve(error)); + }); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ + role: 'user', + content: [ + { + id: 'external-image', + type: 'image_content', + image: 'https://example.com/image.jpg', + inferenceDetail: 'auto', + _cache: {}, + }, + ], + }); + + await bedrockLlm + .chat({ + chatCtx, + connOptions: { maxRetry: 2, retryIntervalMs: 1, timeoutMs: 1000 }, + }) + .collect(); + + await expect(emittedError).resolves.toMatchObject({ + message: expect.stringMatching(/externalUrl images are not supported/), + }); + expect(attempts).toBe(0); + }); + + it('does not apply the connection timeout to an established response stream', async () => { + const client = { + send: async (_command: unknown, { abortSignal }: { abortSignal: AbortSignal }) => ({ + $metadata: { requestId: 'req_slow_stream' }, + stream: (async function* () { + await new Promise((resolve) => setTimeout(resolve, 20)); + if (abortSignal.aborted) { + const error = new Error('stream aborted'); + error.name = 'AbortError'; + throw error; + } + yield { contentBlockDelta: { delta: { text: 'finished' } } }; + })(), + }), + } as unknown as BedrockRuntimeClient; + const bedrockLlm = new LLM({ client }); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'Hi' }); + + const chunks = []; + for await (const chunk of bedrockLlm.chat({ + chatCtx, + connOptions: { maxRetry: 0, retryIntervalMs: 1, timeoutMs: 5 }, + })) { + chunks.push(chunk); + } + + expect(chunks.map((chunk) => chunk.delta?.content ?? '').join('')).toBe('finished'); + }); +}); + +describe('AWS Bedrock LLM - mapConverseStreamException', () => { + it('maps validationException to a non-retryable 400 APIStatusError', () => { + const error = mapConverseStreamException( + { validationException: { message: 'bad input' } }, + 'req_1', + true, + ); + expect(error).toBeInstanceOf(APIStatusError); + expect(error?.statusCode).toBe(400); + expect(error?.retryable).toBe(false); + expect(error?.message).toMatch(/bad input/); + }); + + it('maps throttlingException to a 429 APIStatusError honoring the retryable flag', () => { + const error = mapConverseStreamException( + { throttlingException: { message: 'too many requests' } }, + 'req_1', + true, + ); + expect(error?.statusCode).toBe(429); + expect(error?.retryable).toBe(true); + expect(error?.message).toMatch(/too many requests/); + }); + + it('maps internalServerException to a 500 APIStatusError', () => { + const error = mapConverseStreamException( + { internalServerException: { message: 'oops' } }, + 'req_1', + true, + ); + expect(error?.statusCode).toBe(500); + expect(error?.message).toMatch(/oops/); + }); + + it('maps modelStreamErrorException using a 5xx originalStatusCode', () => { + const error = mapConverseStreamException( + { modelStreamErrorException: { message: 'stream broke', originalStatusCode: 503 } }, + 'req_1', + true, + ); + expect(error?.statusCode).toBe(503); + expect(error?.retryable).toBe(true); + expect(error?.message).toMatch(/stream broke/); + }); + + it('maps modelStreamErrorException with originalStatusCode 424 to a retryable 500', () => { + // Bedrock documents model stream errors with 424; APIStatusError would otherwise force + // non-retryable for that 4xx. Keep the configured 500 so the base LLM retry loop can recover. + const error = mapConverseStreamException( + { modelStreamErrorException: { message: 'stream broke', originalStatusCode: 424 } }, + 'req_1', + true, + ); + expect(error?.statusCode).toBe(500); + expect(error?.retryable).toBe(true); + expect(error?.message).toMatch(/stream broke/); + }); + + it('maps serviceUnavailableException to a 503 APIStatusError', () => { + const error = mapConverseStreamException( + { serviceUnavailableException: { message: 'down for maintenance' } }, + 'req_1', + true, + ); + expect(error?.statusCode).toBe(503); + expect(error?.message).toMatch(/down for maintenance/); + }); + + it('returns undefined for a non-exception event', () => { + expect(mapConverseStreamException({}, 'req_1', true)).toBeUndefined(); + }); +}); + +describe('AWS Bedrock LLM - retry classification', () => { + it('retries an HTTP 408 before output is emitted', async () => { + let attempts = 0; + const client = { + send: async () => { + attempts += 1; + if (attempts === 1) { + throw Object.assign(new Error('model timed out'), { + $metadata: { httpStatusCode: 408, requestId: 'req_timeout' }, + }); + } + return { + $metadata: { requestId: 'req_recovered' }, + stream: (async function* () { + yield { contentBlockDelta: { delta: { text: 'recovered' } } }; + })(), + }; + }, + } as unknown as BedrockRuntimeClient; + const bedrockLlm = new LLM({ client }); + bedrockLlm.on('error', () => {}); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'Hi' }); + + const chunks = []; + for await (const chunk of bedrockLlm.chat({ + chatCtx, + connOptions: { maxRetry: 1, retryIntervalMs: 1, timeoutMs: 1000 }, + })) { + chunks.push(chunk); + } + + expect(attempts).toBe(2); + expect(chunks.map((chunk) => chunk.delta?.content ?? '').join('')).toBe('recovered'); + }); +}); + +describe('AWS Bedrock LLM (live)', () => { + if (hasAwsCredentials) { + it('passes the shared LLM test harness', async () => { + await llmTest(new LLM({ temperature: 0 }), false); + }); + } else { + it.skip('requires AWS_ACCESS_KEY_ID or AWS_PROFILE', () => {}); + } +}); diff --git a/plugins/aws/src/llm.ts b/plugins/aws/src/llm.ts new file mode 100644 index 000000000..61fc79848 --- /dev/null +++ b/plugins/aws/src/llm.ts @@ -0,0 +1,557 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { + ConverseStreamCommandInput, + Message, + SystemContentBlock, + Tool, + ToolConfiguration, +} from '@aws-sdk/client-bedrock-runtime'; +import { BedrockRuntimeClient, ConverseStreamCommand } from '@aws-sdk/client-bedrock-runtime'; +import type { APIConnectOptions } from '@livekit/agents'; +import { + APIConnectionError, + APIStatusError, + APITimeoutError, + DEFAULT_API_CONNECT_OPTIONS, + llm, +} from '@livekit/agents'; +import { type AwsCredentials, createRequestSignal, resolveRegion, toAwsApiError } from './utils.js'; + +const DEFAULT_MODEL = 'amazon.nova-2-lite-v1:0'; + +interface AwsFormatData { + systemMessages: string[] | null; +} + +interface AwsReasoningContentData { + reasoningText?: { text: string; signature?: string }; + /** Base64-encoded so ChatMessage.extra remains JSON serializable. */ + redactedContent?: string; +} + +/** @public */ +export interface LLMOptions { + model?: string; + region?: string; + credentials?: AwsCredentials; + temperature?: number; + maxOutputTokens?: number; + topP?: number; + toolChoice?: llm.ToolChoice; + additionalRequestFields?: Record; + /** Caches system messages with a Bedrock prompt cache point to reduce token usage. */ + cacheSystem?: boolean; + /** Caches tool definitions with a Bedrock prompt cache point to reduce token usage. */ + cacheTools?: boolean; + client?: BedrockRuntimeClient; +} + +/** + * Builds a Bedrock Converse `ToolConfiguration` from a `ToolContext`, or `undefined` when the + * turn should carry no tools (no tools registered, or `toolChoice: 'none'` was requested). + * Kept module-local to the provider implementation; tests import the source module directly. + */ +export function buildToolConfig( + toolCtx: llm.ToolContext | undefined, + toolChoice: llm.ToolChoice | undefined, + cacheTools: boolean, +): ToolConfiguration | undefined { + if (!toolCtx || Object.keys(toolCtx.functionTools).length === 0) return undefined; + if (toolChoice === 'none') return undefined; + + // The AWS SDK's `Tool` discriminated union requires a `$unknown` tuple on its catch-all + // member, which defeats direct object-literal assignability checks against `Tool[]` here + // (the same constraint the sibling google.ts/mistralai.ts adapters hit at this exact + // ChatContext-to-SDK-type boundary) — build the array in its natural shape and cast once. + const tools: Record[] = llm.sortedToolEntries(toolCtx).map(([name, tool]) => { + const description = tool.description.trim(); + return { + toolSpec: { + name, + ...(description ? { description } : {}), + inputSchema: { + json: tool.parameters + ? llm.toJsonSchema(tool.parameters, false, false) + : { type: 'object', properties: {} }, + }, + }, + }; + }); + + if (cacheTools) { + tools.push({ cachePoint: { type: 'default' } }); + } + + const toolConfig: ToolConfiguration = { tools: tools as unknown as Tool[] }; + + if (toolChoice === 'required') { + toolConfig.toolChoice = { any: {} }; + } else if (toolChoice === 'auto') { + toolConfig.toolChoice = { auto: {} }; + } else if (typeof toolChoice === 'object' && toolChoice.type === 'function') { + toolConfig.toolChoice = { tool: { name: toolChoice.function.name } }; + } + + return toolConfig; +} + +interface ConverseStreamException { + message?: string; + originalStatusCode?: number; +} + +interface ConverseStreamExceptionEvent { + validationException?: ConverseStreamException; + throttlingException?: ConverseStreamException; + internalServerException?: ConverseStreamException; + modelStreamErrorException?: ConverseStreamException; + serviceUnavailableException?: ConverseStreamException; +} + +/** Whether a Bedrock HTTP failure can be retried before any output has been emitted. */ +function isRetryableBedrockStatus(statusCode: number, retryable: boolean): boolean { + return retryable && (statusCode === 408 || statusCode === 429 || statusCode >= 500); +} + +function supportsToolResultStatus(model: string): boolean { + const normalised = model.toLowerCase(); + if (normalised.includes('amazon.nova')) return true; + return /anthropic\.claude-(?:3(?:[-.:]|$)|[^/:]*-4(?:[-.:]|$))/.test(normalised); +} + +function omitUnsupportedToolResultStatuses( + messages: Record[], + model: string, +): void { + if (supportsToolResultStatus(model)) return; + + for (const message of messages) { + if (!Array.isArray(message.content)) continue; + for (const block of message.content) { + if (!block || typeof block !== 'object' || Array.isArray(block)) continue; + const toolResult = (block as Record).toolResult; + if (!toolResult || typeof toolResult !== 'object' || Array.isArray(toolResult)) continue; + delete (toolResult as Record).status; + } + } +} + +const CONVERSE_STREAM_EXCEPTIONS: Array<{ + key: keyof ConverseStreamExceptionEvent; + defaultMessage: string; + statusCode?: number; + retryable?: false; +}> = [ + { + key: 'validationException', + defaultMessage: 'validation error', + statusCode: 400, + retryable: false, + }, + { key: 'throttlingException', defaultMessage: 'throttled', statusCode: 429 }, + { key: 'internalServerException', defaultMessage: 'internal server error', statusCode: 500 }, + { key: 'modelStreamErrorException', defaultMessage: 'model stream error', statusCode: 500 }, + { key: 'serviceUnavailableException', defaultMessage: 'service unavailable', statusCode: 503 }, +]; + +/** + * Resolves the HTTP status to report for a Converse stream exception. + * + * `modelStreamErrorException` often carries `originalStatusCode: 424` (Failed Dependency). + * `APIStatusError` treats most 4xx as non-retryable, but AWS documents this event as a + * transient stream failure — keep 5xx originals (and the configured default) so the LLM + * retry loop can recover when no output has been emitted yet. + */ +function resolveConverseStreamStatusCode( + key: keyof ConverseStreamExceptionEvent, + exception: ConverseStreamException, + statusCode: number | undefined, +): number | undefined { + const original = exception.originalStatusCode; + if (key === 'modelStreamErrorException') { + if (original !== undefined && original >= 500) return original; + return statusCode; + } + return original ?? statusCode; +} + +/** + * Bedrock delivers fatal mid-stream errors as regular `ConverseStreamOutput` union events + * rather than thrown exceptions. Maps one to an `APIStatusError`, or `undefined` if the event + * isn't one of the known exception shapes. Exported for unit testing. + */ +export function mapConverseStreamException( + event: ConverseStreamExceptionEvent, + requestId: string, + retryable: boolean, +): APIStatusError | undefined { + for (const { + key, + defaultMessage, + statusCode, + retryable: retryableOverride, + } of CONVERSE_STREAM_EXCEPTIONS) { + const exception = event[key]; + if (!exception) continue; + + return new APIStatusError({ + message: `aws bedrock llm: ${exception.message ?? defaultMessage}`, + options: { + statusCode: resolveConverseStreamStatusCode(key, exception, statusCode), + retryable: retryableOverride ?? retryable, + requestId, + }, + }); + } + return undefined; +} + +/** + * AWS Bedrock Converse LLM. + * @public + */ +export class LLM extends llm.LLM { + #opts; + #client: BedrockRuntimeClient; + #ownsClient: boolean; + + label(): string { + return 'aws.LLM'; + } + + get model(): string { + return this.#opts.model; + } + + get provider(): string { + return 'AWS Bedrock'; + } + + /** + * Create a new instance of AWS Bedrock Converse LLM. + * + * @remarks + * Credentials are resolved via the AWS SDK v3 default credential chain (environment + * variables, shared config/credentials files, IMDS, etc.) unless `credentials` is provided + * explicitly. The region is resolved from `region`, then `AWS_REGION`, then + * `AWS_DEFAULT_REGION`, falling back to `us-east-1`. `model` defaults to + * `BEDROCK_INFERENCE_PROFILE_ARN` if set, otherwise `amazon.nova-2-lite-v1:0`. + */ + constructor(opts: LLMOptions = {}) { + super(); + + this.#opts = { + model: opts.model ?? process.env.BEDROCK_INFERENCE_PROFILE_ARN ?? DEFAULT_MODEL, + region: resolveRegion(opts.region), + credentials: opts.credentials, + temperature: opts.temperature, + maxOutputTokens: opts.maxOutputTokens, + topP: opts.topP, + toolChoice: opts.toolChoice, + additionalRequestFields: opts.additionalRequestFields, + cacheSystem: opts.cacheSystem ?? false, + cacheTools: opts.cacheTools ?? false, + }; + + this.#ownsClient = opts.client === undefined; + this.#client = + opts.client ?? + new BedrockRuntimeClient({ + region: this.#opts.region, + credentials: opts.credentials, + // The framework's own connOptions retry loop handles retries; disable the SDK's + // internal retries so failures aren't retried twice (matches tts.ts's PollyClient). + maxAttempts: 1, + }); + } + + async aclose(): Promise { + if (this.#ownsClient) this.#client.destroy(); + } + + chat({ + chatCtx, + toolCtx: toolCtxInput, + connOptions = DEFAULT_API_CONNECT_OPTIONS, + toolChoice, + extraKwargs, + }: { + chatCtx: llm.ChatContext; + toolCtx?: llm.ToolContextLike; + connOptions?: APIConnectOptions; + parallelToolCalls?: boolean; + toolChoice?: llm.ToolChoice; + extraKwargs?: Record; + }): LLMStream { + const toolCtx = llm.toToolContext(toolCtxInput); + const resolvedToolChoice = toolChoice ?? this.#opts.toolChoice; + + const toolConfig = buildToolConfig(toolCtx, resolvedToolChoice, this.#opts.cacheTools); + if (!toolConfig) { + chatCtx = chatCtx.copy({ excludeFunctionCall: true }); + } + + const inferenceConfig: Record = {}; + if (this.#opts.maxOutputTokens !== undefined) + inferenceConfig.maxTokens = this.#opts.maxOutputTokens; + if (this.#opts.temperature !== undefined) inferenceConfig.temperature = this.#opts.temperature; + if (this.#opts.topP !== undefined) inferenceConfig.topP = this.#opts.topP; + + const extras: Record = { + inferenceConfig, + ...(toolConfig ? { toolConfig } : {}), + ...(this.#opts.additionalRequestFields + ? { additionalModelRequestFields: this.#opts.additionalRequestFields } + : {}), + ...extraKwargs, + }; + + return new LLMStream(this, { + client: this.#client, + model: this.#opts.model, + chatCtx, + toolCtx, + connOptions, + cacheSystem: this.#opts.cacheSystem, + extraKwargs: extras, + }); + } +} + +/** @public */ +export class LLMStream extends llm.LLMStream { + #client: BedrockRuntimeClient; + #model: string; + #cacheSystem: boolean; + #extraKwargs: Record; + + constructor( + llmInst: LLM, + { + client, + model, + chatCtx, + toolCtx, + connOptions, + cacheSystem, + extraKwargs, + }: { + client: BedrockRuntimeClient; + model: string; + chatCtx: llm.ChatContext; + toolCtx?: llm.ToolContext; + connOptions: APIConnectOptions; + cacheSystem: boolean; + extraKwargs: Record; + }, + ) { + super(llmInst, { chatCtx, toolCtx, connOptions }); + this.#client = client; + this.#model = model; + this.#cacheSystem = cacheSystem; + this.#extraKwargs = extraKwargs; + } + + protected async run(): Promise { + let retryable = true; + let requestId = ''; + + // Keep deterministic local formatting/validation failures outside the API retry wrapper. + // The base LLMStream will surface unknown errors immediately instead of retrying the same + // malformed context as a connection failure. + const [messages, extraData] = (await this.chatCtx.toProviderFormat('aws')) as [ + Record[], + AwsFormatData, + ]; + omitUnsupportedToolResultStatuses(messages, this.#model); + + const system: Record[] = []; + if (extraData.systemMessages) { + for (const content of extraData.systemMessages) { + system.push({ text: content }); + } + if (this.#cacheSystem) { + system.push({ cachePoint: { type: 'default' } }); + } + } + + const extraKwargs = { ...this.#extraKwargs }; + const isPromptArn = /^arn:[^:]+:bedrock:[^:]+:[^:]+:prompt\//.test(this.#model); + if (isPromptArn) { + // Prompt management owns these fields and rejects a Converse request that supplies them. + delete extraKwargs.additionalModelRequestFields; + delete extraKwargs.inferenceConfig; + delete extraKwargs.system; + delete extraKwargs.toolConfig; + } + + const input: ConverseStreamCommandInput = { + modelId: this.#model, + messages: messages as unknown as Message[], + ...(!isPromptArn && system.length > 0 + ? { system: system as unknown as SystemContentBlock[] } + : {}), + ...extraKwargs, + }; + + const request = createRequestSignal(this.abortController.signal, this.connOptions.timeoutMs); + + try { + const response = await this.#client.send(new ConverseStreamCommand(input), { + abortSignal: request.signal, + }); + requestId = response.$metadata.requestId ?? ''; + // `timeoutMs` bounds opening the streaming response, not generation time. Keep the + // parent abort signal connected, but do not terminate a healthy long-running response. + request.clearTimeout(); + + if (!response.stream) { + throw new APIStatusError({ + message: 'aws bedrock llm: no stream in the response', + options: { retryable: false, requestId }, + }); + } + + let toolCallId: string | undefined; + let fncName: string | undefined; + let fncRawArgs: string | undefined; + let hasTextOutput = false; + let reasoningText = ''; + let reasoningSignature: string | undefined; + let reasoningRedactedContent: Uint8Array[] = []; + const reasoningContent: AwsReasoningContentData[] = []; + let reasoningAttachedToToolCall = false; + + for await (const event of response.stream) { + if (event.contentBlockStart?.start?.toolUse) { + const toolUse = event.contentBlockStart.start.toolUse; + toolCallId = toolUse.toolUseId; + fncName = toolUse.name; + fncRawArgs = ''; + } else if (event.contentBlockDelta?.delta) { + const delta = event.contentBlockDelta.delta; + if (delta.toolUse?.input !== undefined) { + fncRawArgs = (fncRawArgs ?? '') + delta.toolUse.input; + } else if (delta.text !== undefined) { + this.queue.put({ + id: requestId, + delta: { role: 'assistant', content: delta.text }, + }); + hasTextOutput = true; + retryable = false; + } else if (delta.reasoningContent) { + const reasoningDelta = delta.reasoningContent; + if (reasoningDelta.text !== undefined) reasoningText += reasoningDelta.text; + if (reasoningDelta.signature !== undefined) { + reasoningSignature = reasoningDelta.signature; + } + if (reasoningDelta.redactedContent !== undefined) { + reasoningRedactedContent.push(reasoningDelta.redactedContent); + } + retryable = false; + } + } else if (event.contentBlockStop) { + if (reasoningText || reasoningSignature || reasoningRedactedContent.length > 0) { + if (reasoningRedactedContent.length > 0) { + reasoningContent.push({ + redactedContent: Buffer.concat(reasoningRedactedContent).toString('base64'), + }); + } else { + reasoningContent.push({ + reasoningText: { + text: reasoningText, + ...(reasoningSignature ? { signature: reasoningSignature } : {}), + }, + }); + } + this.queue.put({ + id: requestId, + delta: { + role: 'assistant', + extra: { aws: { reasoningContent: [...reasoningContent] } }, + }, + }); + reasoningText = ''; + reasoningSignature = undefined; + reasoningRedactedContent = []; + } + + if (toolCallId !== undefined) { + const attachReasoning: boolean = + !hasTextOutput && reasoningContent.length > 0 && !reasoningAttachedToToolCall; + this.queue.put({ + id: requestId, + delta: { + role: 'assistant', + toolCalls: [ + llm.FunctionCall.create({ + callId: toolCallId, + name: fncName ?? '', + args: fncRawArgs ?? '', + ...(attachReasoning + ? { extra: { aws: { reasoningContent: [...reasoningContent] } } } + : {}), + }), + ], + }, + }); + retryable = false; + reasoningAttachedToToolCall ||= attachReasoning; + toolCallId = undefined; + fncName = undefined; + fncRawArgs = undefined; + } + } else if (event.metadata) { + const usage = event.metadata.usage; + if (usage) { + this.queue.put({ + id: requestId, + usage: { + completionTokens: usage.outputTokens ?? 0, + promptTokens: usage.inputTokens ?? 0, + totalTokens: usage.totalTokens ?? 0, + promptCachedTokens: usage.cacheReadInputTokens ?? 0, + }, + }); + } + } else { + const mappedError = mapConverseStreamException(event, requestId, retryable); + if (mappedError) { + throw mappedError; + } + } + } + } catch (error: unknown) { + if (request.didTimeout()) { + throw new APITimeoutError({ + message: `aws bedrock llm: request timed out after ${this.connOptions.timeoutMs}ms`, + options: { retryable }, + }); + } + + if (error instanceof APIStatusError || error instanceof APIConnectionError) { + throw error; + } + + // A user interruption aborts the in-flight Bedrock call; the resulting AbortError has + // no HTTP status and would otherwise be classified as a retryable connection error, + // causing the base LLMStream to fire an unwanted duplicate request on every barge-in. + if (this.abortController.signal.aborted) { + return; + } + + const err = error as { $metadata?: { httpStatusCode?: number; requestId?: string } }; + const statusCode = err.$metadata?.httpStatusCode; + + throw toAwsApiError(error, 'aws bedrock llm', { + retryable: + statusCode !== undefined ? isRetryableBedrockStatus(statusCode, retryable) : retryable, + requestId: err.$metadata?.requestId ?? requestId, + }); + } finally { + request.dispose(); + } + } +} diff --git a/plugins/aws/src/models.ts b/plugins/aws/src/models.ts new file mode 100644 index 000000000..a4f1d0cc4 --- /dev/null +++ b/plugins/aws/src/models.ts @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { Engine, LanguageCode, TextType } from '@aws-sdk/client-polly'; + +/** + * Amazon Polly speech synthesis engine. + * @public + */ +export type TTSSpeechEngine = Engine; + +/** + * Language code accepted by Amazon Polly's SynthesizeSpeech request. + * @public + */ +export type TTSLanguage = LanguageCode; + +/** + * Whether the Amazon Polly input text is plain text or SSML. + * @public + */ +export type TTSTextType = TextType; diff --git a/plugins/aws/src/stt.test.ts b/plugins/aws/src/stt.test.ts new file mode 100644 index 000000000..752bebb63 --- /dev/null +++ b/plugins/aws/src/stt.test.ts @@ -0,0 +1,825 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { TranscribeStreamingClient } from '@aws-sdk/client-transcribe-streaming'; +import { APIError, APIStatusError, stt } from '@livekit/agents'; +import { VAD } from '@livekit/agents-plugin-silero'; +import { stt as sttTest } from '@livekit/agents-plugins-test'; +import { AudioFrame } from '@livekit/rtc-node'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { STT } from './stt.js'; +import type { STTOptions } from './stt.js'; + +// Failure-path tests drive SpeechStream.mainTask, which re-throws after +// emitting the STT 'error' event (the rethrow only drives `.finally()` cleanup). +// That produces a by-design floating rejection; swallow expected API errors. +const swallowExpectedRejection = (reason: unknown) => { + if (reason instanceof APIError) return; + throw reason; +}; +beforeAll(() => process.on('unhandledRejection', swallowExpectedRejection)); +afterAll(() => void process.off('unhandledRejection', swallowExpectedRejection)); + +const hasAwsCredentials = Boolean(process.env.AWS_ACCESS_KEY_ID || process.env.AWS_PROFILE); + +const baseOpts: Partial = { sampleRate: 16000, language: 'en-US' }; + +function fakeClient( + sessions: Array<() => AsyncGenerator>>, +): TranscribeStreamingClient { + let call = 0; + return { + send: async () => { + const sessionNumber = call + 1; + const session = sessions[Math.min(call, sessions.length - 1)]!; + call += 1; + return { + $metadata: { requestId: `req_${sessionNumber}` }, + TranscriptResultStream: session(), + }; + }, + } as unknown as TranscribeStreamingClient; +} + +function transcriptEvent(...results: Record[]) { + return { TranscriptEvent: { Transcript: { Results: results } } }; +} + +// Exercises the public STT constructor + STT.stream(), rather than instantiating the +// internal SpeechStream class directly, so these tests also cover STT.stream()'s wiring. +function stream(client: TranscribeStreamingClient) { + return new STT({ ...baseOpts, client }).stream(); +} + +describe('AWS Transcribe STT - constructor', () => { + it('throws when identifyLanguage and identifyMultipleLanguages are both set', () => { + expect(() => new STT({ identifyLanguage: true, identifyMultipleLanguages: true })).toThrow( + /mutually exclusive/, + ); + }); + + it('throws when identifyLanguage is set without languageOptions', () => { + expect(() => new STT({ identifyLanguage: true })).toThrow(/languageOptions is required/); + }); + + it('throws when identifyMultipleLanguages is set without languageOptions', () => { + expect(() => new STT({ identifyMultipleLanguages: true })).toThrow( + /languageOptions is required/, + ); + }); + + it('accepts identifyLanguage when languageOptions is provided', () => { + expect(() => new STT({ identifyLanguage: true, languageOptions: 'en-US,es-US' })).not.toThrow(); + }); + + it('throws when languageOptions is provided without automatic language identification', () => { + expect(() => new STT({ languageOptions: 'en-US,es-US' })).toThrow( + /languageOptions requires identifyLanguage or identifyMultipleLanguages/, + ); + }); + + it('requires identifyLanguage for preferredLanguage', () => { + expect(() => new STT({ preferredLanguage: 'en-US' })).toThrow( + /preferredLanguage requires identifyLanguage/, + ); + expect( + () => + new STT({ + identifyMultipleLanguages: true, + languageOptions: 'en-US,es-US', + preferredLanguage: 'en-US', + }), + ).toThrow(/preferredLanguage requires identifyLanguage/); + }); + + it('accepts preferredLanguage with single-language identification', () => { + expect( + () => + new STT({ + identifyLanguage: true, + languageOptions: 'en-US,es-US', + preferredLanguage: 'en-US', + }), + ).not.toThrow(); + }); + + it('rejects plural vocabulary options for a fixed-language request', () => { + expect(() => new STT({ vocabularyNames: 'english,spanish' })).toThrow( + /vocabularyNames and vocabularyFilterNames require/, + ); + expect(() => new STT({ vocabularyFilterNames: 'english,spanish' })).toThrow( + /vocabularyNames and vocabularyFilterNames require/, + ); + }); + + it('accepts plural vocabulary options with automatic language identification', () => { + expect( + () => + new STT({ + identifyMultipleLanguages: true, + languageOptions: 'en-US,es-US', + vocabularyNames: 'english,spanish', + vocabularyFilterNames: 'english,spanish', + }), + ).not.toThrow(); + }); + + it('rejects singular vocabulary options with automatic language identification', () => { + expect( + () => + new STT({ + identifyLanguage: true, + languageOptions: 'en-US,es-US', + vocabularyName: 'english', + }), + ).toThrow(/vocabularyName and vocabFilterName cannot be used/); + expect( + () => + new STT({ + identifyLanguage: true, + languageOptions: 'en-US,es-US', + vocabFilterName: 'english', + }), + ).toThrow(/vocabularyName and vocabFilterName cannot be used/); + }); + + it('throws when speaker labels and channel identification are both enabled', () => { + expect(() => new STT({ showSpeakerLabel: true, enableChannelIdentification: true })).toThrow( + /mutually exclusive/, + ); + }); + + it('rejects partial result stability when stabilisation is explicitly disabled', () => { + expect( + () => + new STT({ + enablePartialResultsStabilization: false, + partialResultsStability: 'high', + }), + ).toThrow(/partialResultsStability cannot be set/); + }); + + it('enables partial result stabilisation when a stability level is supplied', async () => { + let commandInput: Record | undefined; + const client = { + send: async (command: { input: Record }) => { + commandInput = command.input; + return { + $metadata: { requestId: 'req_stability' }, + TranscriptResultStream: (async function* () {})(), + }; + }, + } as unknown as TranscribeStreamingClient; + const speechStream = new STT({ + sampleRate: 16000, + partialResultsStability: 'high', + client, + }).stream(); + speechStream.endInput(); + for await (const _event of speechStream) { + // Drain the stream so the command is sent and the session completes. + } + + expect(commandInput).toMatchObject({ + EnablePartialResultsStabilization: true, + PartialResultsStability: 'high', + }); + }); + + it('throws when automatic language identification uses a custom language model', () => { + expect( + () => + new STT({ + identifyLanguage: true, + languageOptions: 'en-US,es-US', + languageModelName: 'my-custom-model', + }), + ).toThrow(/languageModelName cannot be used/); + }); + + it('defaults numberOfChannels to 2 when enableChannelIdentification is set', () => { + expect(() => new STT({ enableChannelIdentification: true })).not.toThrow(); + }); + + it('throws when numberOfChannels is set without enableChannelIdentification', () => { + expect(() => new STT({ numberOfChannels: 2 })).toThrow( + /numberOfChannels requires enableChannelIdentification/, + ); + }); + + it('throws when enableChannelIdentification is set with a numberOfChannels other than 2', () => { + expect(() => new STT({ enableChannelIdentification: true, numberOfChannels: 1 })).toThrow( + /numberOfChannels must be 2/, + ); + }); + + it('reports streaming-only capabilities with word-aligned transcripts', () => { + const sttInstance = new STT(); + expect(sttInstance.capabilities).toEqual({ + streaming: true, + interimResults: true, + alignedTranscript: 'word', + diarization: false, + }); + }); + + it.each([{ showSpeakerLabel: true }, { enableChannelIdentification: true }] satisfies Array< + Partial + >)('advertises diarization for %o', (opts) => { + expect(new STT(opts).capabilities.diarization).toBe(true); + }); + + it('reports the provider label', () => { + const sttInstance = new STT(); + expect(sttInstance.provider).toBe('Amazon Transcribe'); + expect(sttInstance.label).toBe('aws.STT'); + }); + + it('defaults model to "unknown" without a languageModelName', () => { + expect(new STT().model).toBe('unknown'); + }); + + it('reports languageModelName as the model when provided', () => { + expect(new STT({ languageModelName: 'my-custom-model' }).model).toBe('my-custom-model'); + }); + + it('rejects single-frame recognition', async () => { + const sttInstance = new STT(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await expect((sttInstance as any)._recognize()).rejects.toThrow( + /does not support single-frame recognition/, + ); + }); +}); + +describe('AWS Transcribe STT - SpeechStream event mapping', () => { + it('maps START_OF_SPEECH / FINAL_TRANSCRIPT / END_OF_SPEECH from a TranscriptEvent', async () => { + const client = fakeClient([ + async function* () { + yield transcriptEvent({ + StartTime: 0, + EndTime: 0.5, + IsPartial: false, + Alternatives: [ + { + Transcript: 'hello.', + Items: [ + { + Content: 'hello', + Type: 'pronunciation', + StartTime: 0, + EndTime: 0.4, + Confidence: 0.9, + }, + { Content: '.', Type: 'punctuation' }, + ], + }, + ], + }); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + expect(events.map((e) => e.type)).toEqual([ + stt.SpeechEventType.START_OF_SPEECH, + stt.SpeechEventType.FINAL_TRANSCRIPT, + stt.SpeechEventType.END_OF_SPEECH, + ]); + expect(events[1]?.alternatives?.[0]?.text).toBe('hello.'); + // The punctuation item is excluded from words and from the confidence average. + expect(events[1]?.alternatives?.[0]?.words).toHaveLength(1); + expect(events[1]?.alternatives?.[0]?.words?.[0]?.text).toBe('hello'); + expect(events[1]?.alternatives?.[0]?.confidence).toBe(0.9); + expect(events[1]?.requestId).toBe('req_1'); + }); + + it('surfaces Transcribe speaker labels on words and the segment', async () => { + const client = fakeClient([ + async function* () { + yield transcriptEvent({ + StartTime: 0, + EndTime: 0.8, + IsPartial: false, + Alternatives: [ + { + Transcript: 'hello world', + Items: [ + { + Content: 'hello', + Type: 'pronunciation', + StartTime: 0, + EndTime: 0.3, + Confidence: 0.95, + Speaker: 'spk_0', + }, + { + Content: 'world', + Type: 'pronunciation', + StartTime: 0.4, + EndTime: 0.8, + Confidence: 0.9, + Speaker: 'spk_0', + }, + ], + }, + ], + }); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + const alt = events.find((e) => e.type === stt.SpeechEventType.FINAL_TRANSCRIPT) + ?.alternatives?.[0]; + expect(alt?.speakerId).toBe('spk_0'); + expect(alt?.words?.map((w: { speakerId?: string | null }) => w.speakerId)).toEqual([ + 'spk_0', + 'spk_0', + ]); + }); + + it('emits START_OF_SPEECH for every utterance, not just the first in the session', async () => { + const client = fakeClient([ + async function* () { + yield transcriptEvent({ + StartTime: 0, + EndTime: 0.5, + IsPartial: false, + Alternatives: [{ Transcript: 'first', Items: [] }], + }); + // A later utterance in the same session never has StartTime === 0. + yield transcriptEvent({ + StartTime: 3.2, + EndTime: 3.8, + IsPartial: false, + Alternatives: [{ Transcript: 'second', Items: [] }], + }); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + expect(events.filter((e) => e.type === stt.SpeechEventType.START_OF_SPEECH)).toHaveLength(2); + expect(events.filter((e) => e.type === stt.SpeechEventType.END_OF_SPEECH)).toHaveLength(2); + }); + + it('tracks speaking state independently per channel within a single TranscriptEvent', async () => { + const client = fakeClient([ + async function* () { + // Both channels start together... + yield transcriptEvent( + { ChannelId: 'ch_0', StartTime: 0, EndTime: 0.3, IsPartial: true, Alternatives: [] }, + { ChannelId: 'ch_1', StartTime: 0, EndTime: 0.3, IsPartial: true, Alternatives: [] }, + ); + // ...channel 0 finishes while channel 1 is still mid-utterance, in the same event. + // Channel 0 finishing must not spuriously re-trigger START_OF_SPEECH for channel 1. + yield transcriptEvent( + { + ChannelId: 'ch_0', + StartTime: 0, + EndTime: 0.5, + IsPartial: false, + Alternatives: [{ Transcript: 'done', Items: [] }], + }, + { + ChannelId: 'ch_1', + StartTime: 0, + EndTime: 0.6, + IsPartial: true, + Alternatives: [{ Transcript: 'still going', Items: [] }], + }, + ); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + // One START_OF_SPEECH per channel (not one per result), and exactly one END_OF_SPEECH + // for the channel that actually finished. + expect(events.filter((e) => e.type === stt.SpeechEventType.START_OF_SPEECH)).toHaveLength(2); + expect(events.filter((e) => e.type === stt.SpeechEventType.END_OF_SPEECH)).toHaveLength(1); + const channelZeroFinal = events.find((event) => event.alternatives?.[0]?.text === 'done'); + expect(channelZeroFinal?.alternatives?.[0]?.speakerId).toBe('ch_0'); + }); + + it('surfaces every language reported by multi-language identification', async () => { + const client = fakeClient([ + async function* () { + yield transcriptEvent({ + LanguageCode: 'en-US', + LanguageIdentification: [ + { LanguageCode: 'en-US', Score: 0.8 }, + { LanguageCode: 'es-US', Score: 0.2 }, + ], + StartTime: 0, + EndTime: 0.5, + IsPartial: false, + Alternatives: [{ Transcript: 'hello hola', Items: [] }], + }); + }, + ]); + + const speechStream = new STT({ + ...baseOpts, + language: undefined, + identifyMultipleLanguages: true, + languageOptions: 'en-US,es-US', + client, + }).stream(); + speechStream.endInput(); + + const events = []; + for await (const event of speechStream) events.push(event); + + const final = events.find((event) => event.type === stt.SpeechEventType.FINAL_TRANSCRIPT); + expect(final?.alternatives?.[0]?.sourceLanguages).toEqual(['en-US', 'es-US']); + }); + + it('does not drop a final transcript whose EndTime is exactly 0', async () => { + const client = fakeClient([ + async function* () { + yield transcriptEvent({ + StartTime: 0, + EndTime: 0, + IsPartial: false, + Alternatives: [{ Transcript: 'ok', Items: [] }], + }); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + expect(events.some((e) => e.alternatives?.[0]?.text === 'ok')).toBe(true); + }); + + it('resets speaking state on an idle-timeout reconnect so START_OF_SPEECH re-fires', async () => { + const client = fakeClient([ + async function* () { + // A partial result mid-utterance leaves speaking state true when the idle timeout hits. + yield transcriptEvent({ + StartTime: 1, + EndTime: 1.5, + IsPartial: true, + Alternatives: [{ Transcript: 'partial', Items: [] }], + }); + const err = new Error('Your request timed out waiting for input'); + err.name = 'BadRequestException'; + throw err; + }, + async function* () { + yield transcriptEvent({ + StartTime: 0.3, + EndTime: 0.6, + IsPartial: false, + Alternatives: [{ Transcript: 'resumed', Items: [] }], + }); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + expect(events.filter((e) => e.type === stt.SpeechEventType.START_OF_SPEECH)).toHaveLength(2); + }); + + it('keeps word/segment timestamps monotonic across an idle-timeout reconnect', async () => { + const client = fakeClient([ + async function* () { + yield transcriptEvent({ + StartTime: 20, + EndTime: 20.5, + IsPartial: false, + Alternatives: [{ Transcript: 'first', Items: [] }], + }); + const err = new Error('Your request timed out waiting for input'); + err.name = 'BadRequestException'; + throw err; + }, + async function* () { + // The new connection's raw clock resets to ~0. + yield transcriptEvent({ + StartTime: 0.3, + EndTime: 0.6, + IsPartial: false, + Alternatives: [{ Transcript: 'second', Items: [] }], + }); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + const finals = events.filter((e) => e.type === stt.SpeechEventType.FINAL_TRANSCRIPT); + expect(finals).toHaveLength(2); + const firstEnd = finals[0]?.alternatives?.[0]?.endTime ?? -Infinity; + const secondStart = finals[1]?.alternatives?.[0]?.startTime ?? -Infinity; + expect(secondStart).toBeGreaterThanOrEqual(firstEnd); + }); + + it('surfaces a BadRequestException stream event as a non-retryable APIStatusError', async () => { + const client = fakeClient([ + async function* () { + yield { BadRequestException: { Message: 'Invalid sample rate' } }; + }, + ]); + + const sttInstance = new STT({ ...baseOpts, client }); + // Fatal errors surface on the STT instance, not through the event iterator. + const errorEvent = new Promise<{ error: Error; recoverable: boolean }>((resolve) => { + sttInstance.on('error', (event) => resolve(event)); + }); + + const speechStream = sttInstance.stream({ + connOptions: { maxRetry: 0, retryIntervalMs: 1, timeoutMs: 1000 }, + }); + const drain = (async () => { + for await (const _event of speechStream) { + // discard + } + })(); + + speechStream.endInput(); + + const { error, recoverable } = await errorEvent; + expect(error).toBeInstanceOf(APIStatusError); + const statusError = error as APIStatusError; + expect(statusError.statusCode).toBe(400); + expect(statusError.retryable).toBe(false); + expect(statusError.message).toMatch(/Invalid sample rate/); + expect(recoverable).toBe(false); + + await drain.catch(() => {}); + }); + + it('reconnects when idle timeout arrives as a BadRequestException stream event', async () => { + const client = fakeClient([ + async function* () { + yield { + BadRequestException: { Message: 'Your request timed out waiting for input' }, + }; + }, + async function* () { + yield transcriptEvent({ + StartTime: 0, + EndTime: 0.2, + IsPartial: false, + Alternatives: [{ Transcript: 'hi', Items: [] }], + }); + }, + ]); + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + expect(events.some((e) => e.alternatives?.[0]?.text === 'hi')).toBe(true); + }); + + it('does not retry a failure after audio has already been sent', async () => { + let attempts = 0; + const client = { + send: async (command: { + input: { AudioStream: AsyncIterable<{ AudioEvent?: { AudioChunk?: Uint8Array } }> }; + }) => { + attempts += 1; + return { + TranscriptResultStream: (async function* () { + for await (const event of command.input.AudioStream) { + if ((event.AudioEvent?.AudioChunk?.byteLength ?? 0) > 0) break; + } + throw new Error('connection reset after audio'); + })(), + }; + }, + } as unknown as TranscribeStreamingClient; + + const sttInstance = new STT({ ...baseOpts, client }); + const errors: APIError[] = []; + sttInstance.on('error', ({ error }) => { + if (error instanceof APIError) errors.push(error); + }); + const speechStream = sttInstance.stream({ + connOptions: { maxRetry: 1, retryIntervalMs: 1, timeoutMs: 1000 }, + }); + + speechStream.pushFrame(new AudioFrame(new Int16Array([1, 2, 3, 4]), 16000, 1, 4)); + speechStream.endInput(); + for await (const _event of speechStream) { + // Drain until the terminal error closes the output queue. + } + + expect(attempts).toBe(1); + expect(errors).toHaveLength(1); + expect(errors[0]?.retryable).toBe(false); + }); + + it('resets speaking state across a hard-failure retry so START_OF_SPEECH re-fires', async () => { + let attempts = 0; + const client = { + send: async () => { + attempts += 1; + if (attempts === 1) { + return { + TranscriptResultStream: (async function* () { + // Mid-utterance (partial, speaking state left true) when the hard failure hits. + yield transcriptEvent({ + StartTime: 1, + EndTime: 1.5, + IsPartial: true, + Alternatives: [{ Transcript: 'partial', Items: [] }], + }); + throw new Error('connection reset'); + })(), + }; + } + return { + TranscriptResultStream: (async function* () { + yield transcriptEvent({ + StartTime: 0.3, + EndTime: 0.6, + IsPartial: false, + Alternatives: [{ Transcript: 'resumed', Items: [] }], + }); + })(), + }; + }, + } as unknown as TranscribeStreamingClient; + + const speechStream = new STT({ ...baseOpts, client }).stream({ + connOptions: { maxRetry: 1, retryIntervalMs: 1, timeoutMs: 1000 }, + }); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.endInput(); + await collect; + + expect(attempts).toBe(2); + expect(events.filter((e) => e.type === stt.SpeechEventType.START_OF_SPEECH)).toHaveLength(2); + const finals = events.filter((e) => e.type === stt.SpeechEventType.FINAL_TRANSCRIPT); + expect(finals).toHaveLength(1); + // The retried session's raw StartTime (0.3) must be pushed past the failed session's + // furthest known point (1.5), not reported as an earlier absolute timestamp. + expect(finals[0]?.alternatives?.[0]?.startTime ?? -Infinity).toBeGreaterThanOrEqual(1.5); + }); + + it('does not lose or duplicate audio frames across an idle-timeout reconnect', async () => { + // Regression test: SpeechStream must own a single pump over `this.input` and hand each + // session a token-guarded generator over a persistent channel — never let an abandoned + // session keep consuming frames meant for the reconnected session (see the #channel field + // comment in stt.ts). + const receivedChunks: Uint8Array[] = []; + let call = 0; + + const client = { + send: async (command: { + input: { AudioStream: AsyncIterable<{ AudioEvent?: { AudioChunk?: Uint8Array } }> }; + }) => { + call += 1; + if (call === 1) { + // The idle timeout fires on the response side without ever draining the + // request-side AudioStream, mirroring how AWS reports it in practice. + return { + TranscriptResultStream: (async function* () { + const err = new Error('Your request timed out waiting for input'); + err.name = 'BadRequestException'; + throw err; + })(), + }; + } + + // Second session: actually drain AudioStream so we can prove every frame + // pushed around the reconnect reaches this session exactly once. + for await (const event of command.input.AudioStream) { + const chunk = event.AudioEvent?.AudioChunk; + if (chunk) receivedChunks.push(chunk); + } + + return { + TranscriptResultStream: (async function* () { + yield transcriptEvent({ + StartTime: 0, + EndTime: 0.2, + IsPartial: false, + Alternatives: [{ Transcript: 'ok', Items: [] }], + }); + })(), + }; + }, + } as unknown as TranscribeStreamingClient; + + const speechStream = stream(client); + + const events: Awaited>['value'][] = []; + const collect = (async () => { + for await (const event of speechStream) { + events.push(event); + } + })(); + + speechStream.pushFrame(new AudioFrame(new Int16Array([1, 2, 3, 4]), 16000, 1, 4)); + speechStream.endInput(); + + await collect; + + expect(call).toBe(2); + // 1 pushed frame (8 bytes) + the terminal empty chunk, each exactly once — no loss, + // no duplication, and nothing delivered to the abandoned first session. + expect(receivedChunks).toHaveLength(2); + expect(receivedChunks.reduce((sum, c) => sum + c.length, 0)).toBe(8); + expect(receivedChunks.some((c) => c.length === 0)).toBe(true); + expect(events.some((e) => e.alternatives?.[0]?.text === 'ok')).toBe(true); + }); +}); + +describe('AWS Transcribe STT (live)', () => { + if (hasAwsCredentials) { + it('passes the shared STT test harness', { timeout: 30_000 }, async () => { + await sttTest(new STT(), await VAD.load(), { streaming: true, nonStreaming: false }); + }); + } else { + it.skip('requires AWS_ACCESS_KEY_ID or AWS_PROFILE', () => {}); + } +}); diff --git a/plugins/aws/src/stt.ts b/plugins/aws/src/stt.ts new file mode 100644 index 000000000..fdf4584c7 --- /dev/null +++ b/plugins/aws/src/stt.ts @@ -0,0 +1,640 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { + LanguageCode, + PartialResultsStability, + Result, + StartStreamTranscriptionCommandInput, + AudioStream as TranscribeAudioStream, + TranscriptEvent, + TranscriptResultStream, + VocabularyFilterMethod, +} from '@aws-sdk/client-transcribe-streaming'; +import { + StartStreamTranscriptionCommand, + TranscribeStreamingClient, +} from '@aws-sdk/client-transcribe-streaming'; +import { + type APIConnectOptions, + APIStatusError, + APITimeoutError, + AsyncIterableQueue, + type AudioBuffer, + DEFAULT_API_CONNECT_OPTIONS, + createTimedString, + log, + normalizeLanguage, + stt, +} from '@livekit/agents'; +import { + type AwsCredentials, + createRequestSignal, + resolveRegion, + stripUndefined, + toAwsApiError, +} from './utils.js'; + +/** @public */ +export interface STTOptions { + sampleRate: number; + language?: LanguageCode; + region?: string; + credentials?: AwsCredentials; + vocabularyName?: string; + sessionId?: string; + vocabFilterMethod?: VocabularyFilterMethod; + vocabFilterName?: string; + showSpeakerLabel?: boolean; + enableChannelIdentification?: boolean; + numberOfChannels?: number; + enablePartialResultsStabilization?: boolean; + partialResultsStability?: PartialResultsStability; + languageModelName?: string; + identifyLanguage?: boolean; + identifyMultipleLanguages?: boolean; + /** + * Comma-separated language codes Amazon Transcribe should consider when automatic + * language identification is enabled. Required when {@link STTOptions.identifyLanguage} or + * {@link STTOptions.identifyMultipleLanguages} is true (AWS rejects the request without it). + */ + languageOptions?: string; + preferredLanguage?: LanguageCode; + vocabularyNames?: string; + vocabularyFilterNames?: string; + client?: TranscribeStreamingClient; +} + +/** + * HTTP status codes for exception members of {@link TranscriptResultStream}. These arrive + * as event-stream union members rather than thrown SDK errors, so they have no + * `$metadata.httpStatusCode` to read. + */ +const STREAM_EXCEPTION_STATUS: Record = { + BadRequestException: 400, + LimitExceededException: 429, + InternalFailureException: 500, + ConflictException: 409, + ServiceUnavailableException: 503, +}; + +const defaultSTTOptions: Pick = { + sampleRate: 24000, + language: 'en-US', +}; + +/** Invalidated the instant its session ends, so a superseded audio generator stops pulling. */ +interface SessionToken { + active: boolean; + audioSent: boolean; +} + +/** Amazon Transcribe reports this after ~15s of stream inactivity; it's not a hard failure. */ +function isIdleTimeout(error: unknown): boolean { + return ( + error instanceof Error && + error.name === 'BadRequestException' && + (error.message ?? '').startsWith('Your request timed out') + ); +} + +/** + * Amazon Transcribe streaming STT. + * + * @remarks + * Streaming only — {@link STT.stream} is the sole recognition entrypoint, matching Amazon + * Transcribe's streaming-only API surface. + * @public + */ +export class STT extends stt.STT { + #opts: STTOptions; + #client: TranscribeStreamingClient; + #ownsClient: boolean; + label = 'aws.STT'; + + get model(): string { + return this.#opts.languageModelName ?? 'unknown'; + } + + get provider(): string { + return 'Amazon Transcribe'; + } + + /** + * Create a new instance of Amazon Transcribe streaming STT. + * + * @remarks + * Credentials are resolved via the AWS SDK v3 default credential chain (environment + * variables, shared config/credentials files, IMDS, etc.) unless `credentials` is provided + * explicitly. The region is resolved from `region`, then `AWS_REGION`, then + * `AWS_DEFAULT_REGION`, falling back to `us-east-1`. + */ + constructor(opts: Partial = {}) { + const enableChannelIdentification = opts.enableChannelIdentification ?? false; + super({ + streaming: true, + interimResults: true, + alignedTranscript: 'word', + diarization: Boolean(opts.showSpeakerLabel || enableChannelIdentification), + }); + + if (opts.identifyLanguage && opts.identifyMultipleLanguages) { + throw new Error( + 'identifyLanguage and identifyMultipleLanguages are mutually exclusive; set only one to true', + ); + } + + const identifyLanguage = opts.identifyLanguage ?? false; + const identifyMultipleLanguages = opts.identifyMultipleLanguages ?? false; + if ( + opts.enablePartialResultsStabilization === false && + opts.partialResultsStability !== undefined + ) { + throw new Error( + 'partialResultsStability cannot be set when enablePartialResultsStabilization is false', + ); + } + const enablePartialResultsStabilization = + opts.enablePartialResultsStabilization ?? opts.partialResultsStability !== undefined; + + if (opts.showSpeakerLabel && enableChannelIdentification) { + throw new Error( + 'showSpeakerLabel and enableChannelIdentification are mutually exclusive; set only one to true', + ); + } + + // StartStreamTranscription requires LanguageOptions whenever either identify flag is set; + // without it AWS returns BadRequest, so fail fast at construction with a clear message. + if ((identifyLanguage || identifyMultipleLanguages) && !opts.languageOptions) { + throw new Error( + 'languageOptions is required when identifyLanguage or identifyMultipleLanguages is true ' + + '(comma-separated language codes Amazon Transcribe should consider, e.g. "en-US,es-US")', + ); + } + + if (!identifyLanguage && !identifyMultipleLanguages && opts.languageOptions !== undefined) { + throw new Error( + 'languageOptions requires identifyLanguage or identifyMultipleLanguages to be true', + ); + } + + if (opts.preferredLanguage !== undefined && !identifyLanguage) { + throw new Error( + 'preferredLanguage requires identifyLanguage to be true and cannot be used with ' + + 'fixed-language or identifyMultipleLanguages requests', + ); + } + + if ( + !identifyLanguage && + !identifyMultipleLanguages && + (opts.vocabularyNames !== undefined || opts.vocabularyFilterNames !== undefined) + ) { + throw new Error( + 'vocabularyNames and vocabularyFilterNames require identifyLanguage or ' + + 'identifyMultipleLanguages; use vocabularyName or vocabFilterName for a fixed language', + ); + } + + if ( + (identifyLanguage || identifyMultipleLanguages) && + (opts.vocabularyName !== undefined || opts.vocabFilterName !== undefined) + ) { + throw new Error( + 'vocabularyName and vocabFilterName cannot be used with automatic language ' + + 'identification; use vocabularyNames or vocabularyFilterNames instead', + ); + } + + if ((identifyLanguage || identifyMultipleLanguages) && opts.languageModelName) { + throw new Error( + 'languageModelName cannot be used with identifyLanguage or identifyMultipleLanguages ' + + '(Amazon Transcribe streaming language identification does not support custom language models)', + ); + } + + // EnableChannelIdentification and NumberOfChannels must be supplied together; streaming + // Transcribe supports two channels. Default numberOfChannels to 2 when identification is + // enabled without an explicit value, and reject the inverse (channels without identification). + let numberOfChannels = opts.numberOfChannels; + if (enableChannelIdentification && numberOfChannels === undefined) { + numberOfChannels = 2; + } else if (!enableChannelIdentification && numberOfChannels !== undefined) { + throw new Error( + 'numberOfChannels requires enableChannelIdentification to be true ' + + '(Amazon Transcribe rejects NumberOfChannels without EnableChannelIdentification)', + ); + } else if (enableChannelIdentification && numberOfChannels !== 2) { + throw new Error( + 'numberOfChannels must be 2 when enableChannelIdentification is true ' + + '(Amazon Transcribe streaming supports two channels)', + ); + } + + this.#opts = { + ...defaultSTTOptions, + ...opts, + identifyLanguage, + identifyMultipleLanguages, + enableChannelIdentification, + enablePartialResultsStabilization, + numberOfChannels, + // Auto language detection is mutually exclusive with a fixed language code. + language: + identifyLanguage || identifyMultipleLanguages + ? undefined + : opts.language ?? defaultSTTOptions.language, + }; + + this.#ownsClient = opts.client === undefined; + this.#client = + opts.client ?? + new TranscribeStreamingClient({ + region: resolveRegion(opts.region), + credentials: opts.credentials, + // The framework's own connOptions retry loop handles retries; disable the SDK's + // internal retries so failures aren't retried twice (matches tts.ts's PollyClient). + maxAttempts: 1, + }); + } + + async close(): Promise { + if (this.#ownsClient) this.#client.destroy(); + } + + async _recognize(_frame: AudioBuffer, _abortSignal?: AbortSignal): Promise { + throw new Error( + 'Amazon Transcribe does not support single-frame recognition, use stream() instead', + ); + } + + stream(options?: { connOptions?: APIConnectOptions }): SpeechStream { + return new SpeechStream(this, this.#opts, this.#client, options?.connOptions); + } +} + +/** @public */ +export class SpeechStream extends stt.SpeechStream { + #opts: STTOptions; + #client: TranscribeStreamingClient; + #timeoutMs: number; + #logger = log(); + // Keyed by Transcribe's `ChannelId` (the empty string when channel identification is off, + // i.e. single-channel audio) so a channel finishing its utterance can't spuriously flip + // speaking state for another channel that's still mid-utterance in the same TranscriptEvent. + #speakingChannels = new Set(); + // Transcribe's StartTime/EndTime are cumulative from the start of the current streaming + // *connection*, and reset to ~0 whenever a reconnect opens a new one. This offset + // accumulates the last known end time of the previous connection so timestamps stay + // monotonic across reconnects, on top of `this.startTimeOffset` (which compensates for the + // framework creating a new SpeechStream instance, a separate concern). + #connectionTimeOffset = 0; + #lastKnownEndTime = 0; + label = 'aws.SpeechStream'; + + // `this.input` may only ever have a single active consumer (calling `.next()` from more + // than one place races for frames). Transcribe's Node HTTP/2 request pipeline can leave an + // abandoned session's `AudioStream` generator alive as a hidden consumer after a session + // fails (the SDK doesn't reliably tear down the request side when the response side errors), + // so a single long-lived pump owns `this.input` and feeds a single persistent + // `#channel` for the lifetime of this stream — the channel itself is never replaced, so + // no already-buffered frame can ever be orphaned on a session transition. Each session + // instead reads through a token-guarded wrapper generator: the token is invalidated the + // moment its session ends. Takes are serialised via `#frameTakeChain` so a superseded + // generator that is mid-await requeues its frame for the active session rather than + // dropping the start of the next utterance. + #channel = new AsyncIterableQueue(); + // At most one frame is ever in flight for a superseded session, since takes are serialised + // via `#frameTakeChain` and a requeued frame is drained by the very next take. + #requeuedFrame: Uint8Array | undefined; + #frameTakeChain: Promise = Promise.resolve(); + #pumpStarted = false; + + constructor( + stt: STT, + opts: STTOptions, + client: TranscribeStreamingClient, + connOptions?: APIConnectOptions, + ) { + const resolvedConnOptions = connOptions ?? DEFAULT_API_CONNECT_OPTIONS; + super(stt, opts.sampleRate, resolvedConnOptions); + this.#opts = opts; + this.#client = client; + this.#timeoutMs = resolvedConnOptions.timeoutMs; + } + + #startPump(): void { + if (this.#pumpStarted) return; + this.#pumpStarted = true; + + (async () => { + for (;;) { + const result = await this.input.next(); + if (result.done) { + this.#channel.close(); + return; + } + const frame = result.value; + if (frame === SpeechStream.FLUSH_SENTINEL) continue; + this.#channel.put( + new Uint8Array(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength), + ); + } + })(); + } + + protected async run(): Promise { + this.#startPump(); + + // Always attempt at least one session, even if the input already closed before run() + // was scheduled (e.g. a very short utterance) — otherwise the final transcript would + // never be requested. + for (;;) { + const token: SessionToken = { active: true, audioSent: false }; + try { + await this.#runSession(this.#audioStreamGenerator(token)); + } catch (error) { + token.active = false; + + if (this.closed) { + return; + } + + // Any session end (idle timeout, or a hard failure the base class will retry via a + // fresh run() call) means the *next* session — whether started here or by a retried + // run() — starts on a brand-new Transcribe connection with a reset clock and no + // memory of in-flight utterances. Reset unconditionally so state from the failed + // session can't leak into the next one: carry the previous connection's furthest + // timestamp forward so reported times keep increasing, and clear speaking state so + // the next session's first result re-emits START_OF_SPEECH. + this.#connectionTimeOffset += this.#lastKnownEndTime; + this.#lastKnownEndTime = 0; + this.#speakingChannels.clear(); + + if (isIdleTimeout(error)) { + // AWS times out after 15s of inactivity, which tends to happen at the end of a + // session once the input is gone. Reconnect unconditionally, matching the Python + // plugin, rather than gating on input state. + this.#logger.info('aws transcribe stt: idle timeout, restarting session'); + continue; + } + // Preserve APIStatusError (incl. non-retryable 4xx) so the base SpeechStream does not + // treat client errors as retryable connection failures. + // Once AWS has consumed audio, retrying run() cannot replay those frames from the + // persistent channel. Treat the failure as terminal instead of letting a retry open a + // new session that can complete successfully without ever transcribing the lost audio. + throw toAwsApiError(error, 'aws transcribe stt', { + retryable: !token.audioSent, + }); + } + + // #runSession only returns without throwing once its audio generator drains + // `#channel` to completion, which only happens after the pump closes it (input + // exhausted) — so there's nothing left to reconnect for. + token.active = false; + return; + } + } + + async #runSession(audioStream: AsyncGenerator): Promise { + const input = stripUndefined({ + LanguageCode: this.#opts.language, + MediaSampleRateHertz: this.#opts.sampleRate, + MediaEncoding: 'pcm', + AudioStream: audioStream, + VocabularyName: this.#opts.vocabularyName, + SessionId: this.#opts.sessionId, + VocabularyFilterName: this.#opts.vocabFilterName, + VocabularyFilterMethod: this.#opts.vocabFilterMethod, + ShowSpeakerLabel: this.#opts.showSpeakerLabel, + EnableChannelIdentification: this.#opts.enableChannelIdentification, + NumberOfChannels: this.#opts.numberOfChannels, + EnablePartialResultsStabilization: this.#opts.enablePartialResultsStabilization, + PartialResultsStability: this.#opts.partialResultsStability, + LanguageModelName: this.#opts.languageModelName, + IdentifyLanguage: this.#opts.identifyLanguage, + IdentifyMultipleLanguages: this.#opts.identifyMultipleLanguages, + LanguageOptions: this.#opts.languageOptions, + PreferredLanguage: this.#opts.preferredLanguage, + VocabularyNames: this.#opts.vocabularyNames, + VocabularyFilterNames: this.#opts.vocabularyFilterNames, + }) as unknown as StartStreamTranscriptionCommandInput; + + const request = createRequestSignal(this.abortSignal, this.#timeoutMs); + try { + const response = await this.#client.send(new StartStreamTranscriptionCommand(input), { + abortSignal: request.signal, + }); + // `timeoutMs` bounds opening the long-lived stream, not the stream's total lifetime. + request.clearTimeout(); + + if (!response.TranscriptResultStream) { + throw new Error('aws transcribe stt: no TranscriptResultStream in the response'); + } + + const requestId = response.$metadata?.requestId; + for await (const event of response.TranscriptResultStream) { + if (event.TranscriptEvent) { + this.#processTranscriptEvent(event.TranscriptEvent, requestId); + continue; + } + + // Amazon Transcribe can deliver service failures as TranscriptResultStream union + // members rather than thrown errors. Ignoring them lets the stream end "successfully" + // with no transcript and no retry — surface them as API errors instead. + this.#throwIfStreamException(event); + } + } catch (error) { + if (request.didTimeout()) { + throw new APITimeoutError({ + message: `aws transcribe stt: request timed out after ${this.#timeoutMs}ms`, + }); + } + throw error; + } finally { + request.dispose(); + } + } + + /** + * Maps a non-transcript event-stream member onto a thrown error so the session loop in + * {@link run} can classify idle timeouts / retryable failures / hard client errors. + */ + #throwIfStreamException(event: TranscriptResultStream): void { + const entries: Array< + [name: string, value: { Message?: string; message?: string } | undefined] + > = [ + ['BadRequestException', event.BadRequestException], + ['LimitExceededException', event.LimitExceededException], + ['InternalFailureException', event.InternalFailureException], + ['ConflictException', event.ConflictException], + ['ServiceUnavailableException', event.ServiceUnavailableException], + ]; + + for (const [name, value] of entries) { + if (!value) continue; + + const message = value.Message ?? value.message ?? name; + + // Preserve the idle-timeout shape so {@link isIdleTimeout} still recognises it and + // the session reconnects rather than failing the stream. + if (name === 'BadRequestException' && message.startsWith('Your request timed out')) { + const err = new Error(message); + err.name = 'BadRequestException'; + throw err; + } + + throw new APIStatusError({ + message: `aws transcribe stt: ${message}`, + options: { + statusCode: STREAM_EXCEPTION_STATUS[name] ?? 500, + body: value as object, + }, + }); + } + } + + /** + * Exclusively take the next audio frame for `token`. Serialises readers so an abandoned + * session generator cannot race the active one on `#channel`, and requeues a frame that + * was dequeued after the token was invalidated so reconnects do not clip the next utterance. + */ + async #takeFrame(token: SessionToken): Promise> { + const outcome: IteratorResult = { value: undefined, done: true }; + + const previous = this.#frameTakeChain; + let release!: () => void; + this.#frameTakeChain = new Promise((resolve) => { + release = resolve; + }); + + await previous; + try { + if (!token.active) { + return outcome; + } + + if (this.#requeuedFrame !== undefined) { + const value = this.#requeuedFrame; + this.#requeuedFrame = undefined; + return { value, done: false }; + } + + const result = await this.#channel.next(); + // Invalidated while awaiting — stash the frame for the active session instead of dropping it. + if (!token.active) { + if (!result.done) { + this.#requeuedFrame = result.value; + } + return outcome; + } + + return result; + } finally { + release(); + } + } + + async *#audioStreamGenerator(token: SessionToken): AsyncGenerator { + for (;;) { + if (!token.active) return; + const result = await this.#takeFrame(token); + if (!token.active) return; + if (result.done) break; + if (result.value.byteLength > 0) { + token.audioSent = true; + } + yield { AudioEvent: { AudioChunk: result.value } }; + } + + // AWS Transcribe requires an empty chunk to signal the end of the audio stream. + if (token.active) { + yield { AudioEvent: { AudioChunk: new Uint8Array(0) } }; + } + } + + #processTranscriptEvent(transcriptEvent: TranscriptEvent, requestId?: string): void { + const results = transcriptEvent.Transcript?.Results; + if (!results) return; + + for (const result of results) { + // Transcribe's StartTime/EndTime are cumulative offsets from the start of the whole + // streaming connection, not per-utterance — so `StartTime === 0` is only ever true for + // the very first result of a session. START_OF_SPEECH must instead be driven by our own + // per-channel speaking state so it fires once per utterance, not once per connection, + // and so one channel finishing doesn't affect another channel's speaking state. + const channelKey = result.ChannelId ?? ''; + if (!this.#speakingChannels.has(channelKey)) { + this.#speakingChannels.add(channelKey); + this.queue.put({ type: stt.SpeechEventType.START_OF_SPEECH, requestId }); + } + + if (result.EndTime !== undefined) { + this.#lastKnownEndTime = Math.max(this.#lastKnownEndTime, result.EndTime); + const alternative = this.#toSpeechData(result); + this.queue.put({ + type: result.IsPartial + ? stt.SpeechEventType.INTERIM_TRANSCRIPT + : stt.SpeechEventType.FINAL_TRANSCRIPT, + requestId, + alternatives: [alternative], + }); + } + + if (!result.IsPartial) { + this.#speakingChannels.delete(channelKey); + this.queue.put({ type: stt.SpeechEventType.END_OF_SPEECH, requestId }); + } + } + } + + #toSpeechData(result: Result): stt.SpeechData { + const alternative = result.Alternatives?.[0]; + // Transcribe tags punctuation items with no meaningful timestamp/confidence; excluding + // them keeps word-level alignment and utterance confidence limited to spoken words. + const items = alternative?.Items?.filter((item) => item.Type !== 'punctuation'); + const confidence = items?.length + ? items.reduce((sum, item) => sum + (item.Confidence ?? 0), 0) / items.length + : 0; + const detectedLanguage = result.LanguageCode ?? this.#opts.language ?? 'en-US'; + + const identifiedLanguages = result.LanguageIdentification?.flatMap(({ LanguageCode }) => + LanguageCode ? [normalizeLanguage(LanguageCode)] : [], + ); + let sourceLanguages: ReturnType[] | undefined; + if (this.#opts.identifyLanguage || this.#opts.identifyMultipleLanguages) { + if (identifiedLanguages?.length) { + sourceLanguages = [...new Set(identifiedLanguages)]; + } else if (result.LanguageCode) { + sourceLanguages = [normalizeLanguage(result.LanguageCode)]; + } + } + + const offset = this.startTimeOffset + this.#connectionTimeOffset; + + // When showSpeakerLabel is enabled, Transcribe attaches a Speaker label to each item. + // Surface it on every word and derive a segment-level speakerId from the first labelled word. + const words = items?.map((item) => + createTimedString({ + text: item.Content ?? '', + startTime: (item.StartTime ?? 0) + offset, + endTime: (item.EndTime ?? 0) + offset, + confidence: item.Confidence ?? 0, + startTimeOffset: offset, + speakerId: item.Speaker ?? result.ChannelId ?? null, + }), + ); + const speakerId = words?.find((word) => word.speakerId)?.speakerId ?? result.ChannelId ?? null; + + return { + language: normalizeLanguage(detectedLanguage), + startTime: (result.StartTime ?? 0) + offset, + endTime: (result.EndTime ?? 0) + offset, + text: alternative?.Transcript ?? '', + confidence, + sourceLanguages, + speakerId, + words, + }; + } +} diff --git a/plugins/aws/src/tts.test.ts b/plugins/aws/src/tts.test.ts new file mode 100644 index 000000000..edf423f0f --- /dev/null +++ b/plugins/aws/src/tts.test.ts @@ -0,0 +1,164 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { PollyClient } from '@aws-sdk/client-polly'; +import { APIConnectionError, APIStatusError } from '@livekit/agents'; +import { tts as ttsTest } from '@livekit/agents-plugins-test'; +import { describe, expect, it } from 'vitest'; +import { STT } from './stt.js'; +import { TTS } from './tts.js'; +import { toAwsApiError } from './utils.js'; + +const hasAwsCredentials = Boolean(process.env.AWS_ACCESS_KEY_ID || process.env.AWS_PROFILE); + +function pcmBytes(sampleCount: number): Uint8Array { + return new Uint8Array(sampleCount * 2); // 16-bit PCM, mono +} + +function fakeClient(audioBytes: Uint8Array, requestId = 'req_123'): PollyClient { + return { + send: async () => ({ + $metadata: { requestId }, + AudioStream: { + transformToByteArray: async () => audioBytes, + }, + }), + } as unknown as PollyClient; +} + +describe('AWS Polly TTS - constructor', () => { + it('defaults to Ruth, generative engine, and 16000 Hz', () => { + const tts = new TTS({ client: fakeClient(pcmBytes(0)) }); + expect(tts.sampleRate).toBe(16000); + expect(tts.numChannels).toBe(1); + expect(tts.model).toBe('generative'); + expect(tts.provider).toBe('Amazon Polly'); + expect(tts.capabilities.streaming).toBe(false); + }); + + it('accepts 8000 Hz', () => { + const tts = new TTS({ sampleRate: 8000, client: fakeClient(pcmBytes(0)) }); + expect(tts.sampleRate).toBe(8000); + }); + + it('throws for unsupported sample rates', () => { + expect(() => new TTS({ sampleRate: 24000 })).toThrow(/8000 or 16000/); + }); + + it('throws when streaming is requested', () => { + const tts = new TTS({ client: fakeClient(pcmBytes(0)) }); + expect(() => tts.stream()).toThrow(/Streaming is not supported/); + }); + + it('merges updateOptions onto the existing options', () => { + const tts = new TTS({ client: fakeClient(pcmBytes(0)) }); + tts.updateOptions({ voice: 'Matthew', speechEngine: 'neural' }); + expect(tts.model).toBe('neural'); + }); +}); + +describe('AWS Polly TTS - synthesis', () => { + it('flushes a PCM response shorter than one 100ms frame', async () => { + const tts = new TTS({ sampleRate: 16000, client: fakeClient(pcmBytes(800)) }); + const stream = tts.synthesize('hello world'); + + const events = []; + for await (const event of stream) { + events.push(event); + } + + expect(events).toHaveLength(1); + expect(events.at(-1)?.final).toBe(true); + expect(events.reduce((sum, event) => sum + event.frame.samplesPerChannel, 0)).toBe(800); + for (const event of events) { + expect(event.frame.sampleRate).toBe(16000); + expect(event.frame.channels).toBe(1); + // A trailing empty flush() frame must never become the reported final frame. + expect(event.frame.samplesPerChannel).toBeGreaterThan(0); + } + }); + + it('does not emit an empty final frame when the PCM divides evenly into 100ms frames', async () => { + // 1600 samples at 16000 Hz is exactly one 100ms frame, leaving nothing buffered for + // flush() — flush() still returns a 0-sample frame in that case, which must be dropped. + const tts = new TTS({ sampleRate: 16000, client: fakeClient(pcmBytes(1600)) }); + const stream = tts.synthesize('hi'); + + const events = []; + for await (const event of stream) { + events.push(event); + } + + expect(events).toHaveLength(1); + expect(events[0]?.final).toBe(true); + expect(events[0]?.frame.samplesPerChannel).toBe(1600); + }); + + it('keeps the output queue open across a retryable Polly failure', async () => { + let attempts = 0; + const client = { + send: async () => { + attempts += 1; + if (attempts === 1) { + throw Object.assign(new Error('temporary outage'), { + $metadata: { httpStatusCode: 503 }, + }); + } + return { + $metadata: { requestId: 'req_retry' }, + AudioStream: { + transformToByteArray: async () => pcmBytes(1600), + }, + }; + }, + } as unknown as PollyClient; + const tts = new TTS({ sampleRate: 16000, client }); + const stream = tts.synthesize('retry me', { + maxRetry: 1, + retryIntervalMs: 1, + timeoutMs: 1000, + }); + + const events = []; + for await (const event of stream) events.push(event); + + expect(attempts).toBe(2); + expect(events).toHaveLength(1); + expect(events[0]?.final).toBe(true); + }); + + // Error classification for Polly failures goes through toAwsApiError (used by + // ChunkedStream.run). Exercise the mapper directly — driving the full stream + // would also surface a by-design floating rejection from ThrowsPromise. + it('maps Polly service errors with an HTTP status to non-retryable APIStatusError', () => { + const err = Object.assign(new Error('Invalid voice id'), { + name: 'InvalidParameterException', + $metadata: { httpStatusCode: 400, requestId: 'req_bad_voice' }, + }); + + const error = toAwsApiError(err, 'aws polly tts'); + expect(error).toBeInstanceOf(APIStatusError); + const statusError = error as APIStatusError; + expect(statusError.statusCode).toBe(400); + expect(statusError.retryable).toBe(false); + expect(statusError.requestId).toBe('req_bad_voice'); + expect(statusError.message).toMatch(/Invalid voice id/); + }); + + it('maps Polly errors without an HTTP status to APIConnectionError', () => { + const error = toAwsApiError(new Error('network reset'), 'aws polly tts'); + expect(error).toBeInstanceOf(APIConnectionError); + expect(error).not.toBeInstanceOf(APIStatusError); + expect(error.message).toMatch(/network reset/); + }); +}); + +describe('AWS Polly TTS (live)', () => { + if (hasAwsCredentials) { + it('passes the shared TTS test harness', async () => { + await ttsTest(new TTS(), new STT(), { streaming: false }); + }); + } else { + it.skip('requires AWS_ACCESS_KEY_ID or AWS_PROFILE', () => {}); + } +}); diff --git a/plugins/aws/src/tts.ts b/plugins/aws/src/tts.ts new file mode 100644 index 000000000..9238e7713 --- /dev/null +++ b/plugins/aws/src/tts.ts @@ -0,0 +1,223 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { SynthesizeSpeechCommandInput } from '@aws-sdk/client-polly'; +import { PollyClient, SynthesizeSpeechCommand } from '@aws-sdk/client-polly'; +import { + type APIConnectOptions, + APITimeoutError, + AudioByteStream, + DEFAULT_API_CONNECT_OPTIONS, + shortuuid, + tts, +} from '@livekit/agents'; +import type { AudioFrame } from '@livekit/rtc-node'; +import type { TTSLanguage, TTSSpeechEngine, TTSTextType } from './models.js'; +import { + type AwsCredentials, + createRequestSignal, + resolveRegion, + stripUndefined, + toAwsApiError, +} from './utils.js'; + +/** Amazon Polly only returns PCM (16-bit little-endian, mono) at these sample rates. */ +const SUPPORTED_PCM_SAMPLE_RATES = [8000, 16000]; + +/** @public */ +export interface TTSOptions { + voice: string; + speechEngine: TTSSpeechEngine; + textType: TTSTextType; + language?: TTSLanguage; + sampleRate: number; + region?: string; + credentials?: AwsCredentials; + client?: PollyClient; +} + +const defaultTTSOptions: Omit = { + voice: 'Ruth', + speechEngine: 'generative', + textType: 'text', +}; + +/** + * Amazon Polly TTS. + * + * @remarks + * Audio is requested as raw PCM (16-bit little-endian, mono) since the framework has no mp3 + * decoder. Amazon Polly only supports PCM output at 8000 Hz or 16000 Hz, so `sampleRate` is + * restricted to those two values. + * @public + */ +export class TTS extends tts.TTS { + #opts: TTSOptions; + #client: PollyClient; + #ownsClient: boolean; + label = 'aws.TTS'; + private abortController = new AbortController(); + + get model(): string { + return this.#opts.speechEngine; + } + + get provider(): string { + return 'Amazon Polly'; + } + + /** + * Create a new instance of AWS Polly TTS. + * + * @remarks + * Credentials are resolved via the AWS SDK v3 default credential chain (environment + * variables, shared config/credentials files, IMDS, etc.) unless `credentials` is provided + * explicitly. The region is resolved from `region`, then `AWS_REGION`, then + * `AWS_DEFAULT_REGION`, falling back to `us-east-1`. + */ + constructor(opts: Partial = {}) { + const sampleRate = opts.sampleRate ?? 16000; + if (!SUPPORTED_PCM_SAMPLE_RATES.includes(sampleRate)) { + throw new Error( + `AWS Polly TTS only supports PCM output at ${SUPPORTED_PCM_SAMPLE_RATES.join(' or ')} Hz sample rates, got ${sampleRate}`, + ); + } + + super(sampleRate, 1, { streaming: false }); + + this.#opts = { + ...defaultTTSOptions, + ...opts, + sampleRate, + }; + + this.#ownsClient = opts.client === undefined; + this.#client = + opts.client ?? + new PollyClient({ + region: resolveRegion(opts.region), + credentials: opts.credentials, + maxAttempts: 1, + }); + } + + updateOptions(opts: { + voice?: string; + language?: TTSLanguage; + speechEngine?: TTSSpeechEngine; + textType?: TTSTextType; + }) { + this.#opts = { ...this.#opts, ...opts }; + } + + synthesize( + text: string, + connOptions?: APIConnectOptions, + abortSignal?: AbortSignal, + ): ChunkedStream { + const signal = abortSignal + ? AbortSignal.any([abortSignal, this.abortController.signal]) + : this.abortController.signal; + return new ChunkedStream(this, text, this.#client, this.#opts, connOptions, signal); + } + + stream(): tts.SynthesizeStream { + throw new Error('Streaming is not supported on AWS Polly TTS'); + } + + async close(): Promise { + this.abortController.abort(); + if (this.#ownsClient) this.#client.destroy(); + } +} + +/** @public */ +export class ChunkedStream extends tts.ChunkedStream { + label = 'aws.ChunkedStream'; + #client: PollyClient; + #opts: TTSOptions; + #timeoutMs: number; + + constructor( + tts: TTS, + text: string, + client: PollyClient, + opts: TTSOptions, + connOptions?: APIConnectOptions, + abortSignal?: AbortSignal, + ) { + const resolvedConnOptions = connOptions ?? DEFAULT_API_CONNECT_OPTIONS; + super(text, tts, resolvedConnOptions, abortSignal); + this.#client = client; + this.#opts = opts; + this.#timeoutMs = resolvedConnOptions.timeoutMs; + } + + protected async run() { + const request = createRequestSignal(this.abortSignal, this.#timeoutMs); + try { + const input = stripUndefined({ + Text: this.inputText, + OutputFormat: 'pcm', + Engine: this.#opts.speechEngine, + VoiceId: this.#opts.voice, + TextType: this.#opts.textType, + SampleRate: String(this.#opts.sampleRate), + LanguageCode: this.#opts.language, + }) as unknown as SynthesizeSpeechCommandInput; + + const response = await this.#client.send(new SynthesizeSpeechCommand(input), { + abortSignal: request.signal, + }); + + if (!response.AudioStream) { + throw new Error('aws polly tts: no AudioStream in the response'); + } + + const requestId = response.$metadata.requestId ?? shortuuid(); + const audioBytes = await response.AudioStream.transformToByteArray(); + const audioByteStream = new AudioByteStream(this.#opts.sampleRate, 1); + + let lastFrame: AudioFrame | undefined; + const sendLastFrame = (segmentId: string, final: boolean) => { + if (lastFrame) { + this.queue.put({ requestId, segmentId, frame: lastFrame, final }); + lastFrame = undefined; + } + }; + + // write() only emits complete 100ms frames; flush() drains the trailing remainder (up + // to 100ms), otherwise the tail of the synthesized audio is silently dropped. flush() + // always returns one frame even with nothing buffered (0 samples), so drop it rather + // than let it become the "final" frame in place of the real last frame from write(). + for (const frame of audioByteStream.write(audioBytes)) { + sendLastFrame(requestId, false); + lastFrame = frame; + } + const remainder = audioByteStream.flush().filter((frame) => frame.samplesPerChannel > 0); + for (const frame of remainder) { + sendLastFrame(requestId, false); + lastFrame = frame; + } + sendLastFrame(requestId, true); + } catch (error) { + if (request.didTimeout()) { + throw new APITimeoutError({ + message: `aws polly tts: request timed out after ${this.#timeoutMs}ms`, + }); + } + if (error instanceof Error && error.name === 'AbortError') { + return; + } + if (error instanceof Error && error.name === 'TimeoutError') { + throw new APITimeoutError({ message: error.message }); + } + // Prefer APIStatusError when the SDK surfaces an HTTP status (e.g. invalid VoiceId / + // Engine, malformed SSML) so non-retryable 4xx inputs are not retried as connection + // failures by the base ChunkedStream. + throw toAwsApiError(error, 'aws polly tts'); + } finally { + request.dispose(); + } + } +} diff --git a/plugins/aws/src/utils.test.ts b/plugins/aws/src/utils.test.ts new file mode 100644 index 000000000..07043dd66 --- /dev/null +++ b/plugins/aws/src/utils.test.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { describe, expect, it } from 'vitest'; +import { createRequestSignal } from './utils.js'; + +describe('AWS request signals', () => { + it('aborts and records when the request timeout elapses', async () => { + const request = createRequestSignal(new AbortController().signal, 5); + + await new Promise((resolve) => { + request.signal.addEventListener('abort', () => resolve(), { once: true }); + }); + + expect(request.didTimeout()).toBe(true); + expect(request.signal.reason).toMatchObject({ name: 'TimeoutError' }); + request.dispose(); + }); +}); diff --git a/plugins/aws/src/utils.ts b/plugins/aws/src/utils.ts new file mode 100644 index 000000000..23b79a4c1 --- /dev/null +++ b/plugins/aws/src/utils.ts @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { APIConnectionError, APIStatusError } from '@livekit/agents'; + +export const DEFAULT_REGION = 'us-east-1'; + +/** + * Explicit static AWS credentials. When omitted, the AWS SDK v3 default credential chain is used. + * @public + */ +export interface AwsCredentials { + accessKeyId: string; + secretAccessKey: string; + sessionToken?: string; +} + +/** + * Resolves the AWS region to use, in order of precedence: + * the explicit `region` argument, `AWS_REGION`, `AWS_DEFAULT_REGION`, then {@link DEFAULT_REGION}. + */ +export function resolveRegion(region?: string): string { + return region ?? process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION ?? DEFAULT_REGION; +} + +/** Removes `undefined`-valued keys so they aren't sent to AWS SDK calls. */ +export function stripUndefined>(obj: T): T { + return Object.fromEntries(Object.entries(obj).filter(([, value]) => value !== undefined)) as T; +} + +/** Combines a caller-controlled abort signal with a disposable per-request timeout. */ +export function createRequestSignal( + parent: AbortSignal, + timeoutMs: number, +): { + signal: AbortSignal; + didTimeout: () => boolean; + clearTimeout: () => void; + dispose: () => void; +} { + const controller = new AbortController(); + let timedOut = false; + + const abortFromParent = () => controller.abort(parent.reason); + if (parent.aborted) { + abortFromParent(); + } else { + parent.addEventListener('abort', abortFromParent, { once: true }); + } + + let timer = + timeoutMs > 0 + ? setTimeout(() => { + timedOut = true; + controller.abort(new DOMException('Request timed out', 'TimeoutError')); + }, timeoutMs) + : undefined; + timer?.unref(); + + const clearRequestTimeout = () => { + if (timer) clearTimeout(timer); + timer = undefined; + }; + + return { + signal: controller.signal, + didTimeout: () => timedOut, + clearTimeout: clearRequestTimeout, + dispose: () => { + clearRequestTimeout(); + parent.removeEventListener('abort', abortFromParent); + }, + }; +} + +/** Shape of AWS SDK v3 service exceptions we care about when classifying failures. */ +interface AwsSdkErrorLike { + message?: string; + Message?: string; + name?: string; + $metadata?: { + httpStatusCode?: number; + requestId?: string; + }; +} + +/** + * Maps an AWS SDK exception (or already-classified framework error) into an + * {@link APIStatusError} when an HTTP status is present, otherwise an + * {@link APIConnectionError}. Passes through existing API errors unchanged unless an explicit + * retryability override is provided. + */ +export function toAwsApiError( + error: unknown, + prefix: string, + options?: { retryable?: boolean; requestId?: string | null }, +): APIConnectionError | APIStatusError { + if (error instanceof APIStatusError) { + if (options?.retryable === undefined) return error; + return new APIStatusError({ + message: error.message, + options: { + statusCode: error.statusCode, + requestId: options.requestId ?? error.requestId, + body: error.body, + retryable: options.retryable, + }, + }); + } + if (error instanceof APIConnectionError) { + if (options?.retryable === undefined) return error; + return new APIConnectionError({ + message: error.message, + options: { retryable: options.retryable }, + }); + } + + const err = error as AwsSdkErrorLike; + const message = err.message ?? err.Message ?? String(error); + const statusCode = err.$metadata?.httpStatusCode; + + if (statusCode !== undefined) { + return new APIStatusError({ + message: `${prefix}: ${message}`, + options: { + statusCode, + requestId: options?.requestId ?? err.$metadata?.requestId ?? null, + retryable: options?.retryable, + }, + }); + } + + return new APIConnectionError({ + message: `${prefix}: ${message}`, + options: { retryable: options?.retryable }, + }); +} diff --git a/plugins/aws/tsconfig.json b/plugins/aws/tsconfig.json new file mode 100644 index 000000000..9c061c392 --- /dev/null +++ b/plugins/aws/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../tsconfig.json", + "include": ["./src"], + "exclude": ["./src/**/*.test.ts"], + "compilerOptions": { + // match output dir to input dir. e.g. dist/index instead of dist/src/index + "rootDir": "./src", + "declarationDir": "./dist", + "outDir": "./dist" + }, + "typedocOptions": { + "name": "plugins/agents-plugin-aws", + "entryPointStrategy": "resolve", + "entryPoints": ["src/index.ts"] + } +} diff --git a/plugins/aws/tsup.config.ts b/plugins/aws/tsup.config.ts new file mode 100644 index 000000000..18274714b --- /dev/null +++ b/plugins/aws/tsup.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'tsup'; +import defaults from '../../tsup.config'; + +export default defineConfig({ + ...defaults, + entry: ['src/**/*.ts', '!src/**/*.test.ts'], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f00f5059b..280cbf934 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -251,6 +251,9 @@ importers: '@livekit/agents-plugin-assemblyai': specifier: workspace:* version: link:../plugins/assemblyai + '@livekit/agents-plugin-aws': + specifier: workspace:* + version: link:../plugins/aws '@livekit/agents-plugin-baseten': specifier: workspace:* version: link:../plugins/baseten @@ -478,6 +481,43 @@ importers: specifier: ^5.0.0 version: 5.9.3 + plugins/aws: + dependencies: + '@aws-sdk/client-bedrock-runtime': + specifier: ^3.1081.0 + version: 3.1081.0 + '@aws-sdk/client-polly': + specifier: ^3.1081.0 + version: 3.1081.0 + '@aws-sdk/client-transcribe-streaming': + specifier: ^3.1081.0 + version: 3.1081.0 + devDependencies: + '@livekit/agents': + specifier: workspace:* + version: link:../../agents + '@livekit/agents-plugin-silero': + specifier: workspace:* + version: link:../silero + '@livekit/agents-plugins-test': + specifier: workspace:* + version: link:../test + '@livekit/rtc-node': + specifier: 'catalog:' + version: 0.13.31 + '@microsoft/api-extractor': + specifier: ^7.35.0 + version: 7.43.7(@types/node@25.6.0) + tsup: + specifier: ^8.3.5 + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@25.6.0))(postcss@8.5.9)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.0.0 + version: 5.9.3 + zod: + specifier: ^3.25.76 || ^4.1.8 + version: 4.3.6 + plugins/baseten: dependencies: dotenv: @@ -1450,6 +1490,94 @@ packages: '@anthropic-ai/sdk@0.33.1': resolution: {integrity: sha512-VrlbxiAdVRGuKP2UQlCnsShDHJKWepzvfRCkZMpU+oaUdKLpOfmylLMRojGrAgebV+kDtPjewCVP0laHXg+vsA==} + '@aws-sdk/client-bedrock-runtime@3.1081.0': + resolution: {integrity: sha512-rRAGXY5qV/NCYbVA0QZHPierv3diOOiE4+1f5vedpbyvg7Phh9m/I2pRFwu0koUtMdRSGPWgoNcvMIyaDhDj7Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-polly@3.1081.0': + resolution: {integrity: sha512-nH7X4X9qQmIT3HNMi2LJDu3pVqY6csPwbTbhZ2DXFqVoY2HjGAt+iihWWYoZCrI0d64k8PxPIrqgwAJW40NlcQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-transcribe-streaming@3.1081.0': + resolution: {integrity: sha512-0ToKz8K86yXNNsQAln9x55Hsl31AH+hXssNvg52/Szr3/1iR3F0kJrX0QdWzOBDAn/lrUtpFXGEPn8cfPIGpXQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/core@3.974.29': + resolution: {integrity: sha512-yqKcltLbtRh1ubzhRSldIs8jFHNZlyMlgoIccCC0aDVbrB99nXaBdmfr89mK7obWX/NVg4rAMpCpZ6dCDiVBtA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-env@3.972.55': + resolution: {integrity: sha512-Ah36tYkqyaVnaHkx7VseoTYrHUmwgBps3V+wnrC1idhIIMGlviH0FtrX9EIPdAlVHvXC7FQZLhmHBRz+pLaiWg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-http@3.972.57': + resolution: {integrity: sha512-/vp6i5YEliJqRm5k/BDmYjAyRAMTdkjW6UciVRk9oh/0OfDCWeb/ih7hqte4lFvKXkIbsqe9AdK9LQK6NGardw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.972.62': + resolution: {integrity: sha512-pQIRiQQs+MUlVnJdWJ7/6KS0WxcLRVfut57OFgwC3cnM1F8mXw3Kh4gAVwj6AtvD6CWx8x6+po4ENRcqe64XrQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-login@3.972.61': + resolution: {integrity: sha512-jtrxWwC7slqxh7DnAWHrwsA3UwCsnlypdYtavGT7EX5p791wxWQys7QzkCZ7JvOMAyylDtPoxyV+ic0zg3rV9g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-node@3.972.64': + resolution: {integrity: sha512-zyKVYDyMR9VQL/kPi03ygN2vtD9uLMuWRLoJ77KxgZZaS1VlJloI+SzleF9Zg4HWUI+AIu+ZRs8zsJFNqbrxsw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-process@3.972.55': + resolution: {integrity: sha512-x0XjjF0l1WGRtK2vEhTZqCguQuAIZLep9l2+eeEmuxQQjjD3BlGQXY5xADR+l3t576UX+dxRkRtTjEu40l81Vw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.972.61': + resolution: {integrity: sha512-d/V0VRsz73i+PHhbult/tx0Y1+de1SNQVsXkcQCmpfeBq7uODy/RTxNsOLpT9ZVHxcRNzbQFuywLKC33fUMIxA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.972.61': + resolution: {integrity: sha512-Bv4n3NOI6hPy+rmr6Bw9R6LnBVRkcp3ncj2E2IKSYJG+0UkysSitWMvbgndNvMxDw7gE1pQ/ErwkNceuKwj7zQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/eventstream-handler-node@3.972.25': + resolution: {integrity: sha512-df7HN1ozwMrB9+59re9PM7tSLxLAcheMWc5u/KyfCPCAWtN/vP7y7RTUZOy48uT1K9MESisVeOPPzF3O1AW01A==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-eventstream@3.972.21': + resolution: {integrity: sha512-HvLgDnxBLaHi9E5K++6Vuk+1+qqn7Pmn8zrlzd+NXH3jBzwujnuzZtAR9WHPkbUGPO92FkoQWj/M1IsdxTlBmQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-sdk-transcribe-streaming@3.972.24': + resolution: {integrity: sha512-rLtH6jAHAgcLYFhCShgFGuxHgFNrbGZdryJB1mc1tmBCC8Hb/Dx1DZopwKPGFV9RvdtlhZCaNQpYW5Y+bSuSCQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-websocket@3.972.37': + resolution: {integrity: sha512-u4J2KwTe6hr0hBrcKF7vPNxoQoPdSwdhE8mEQK/ffaY/XgYQ77NRqsbxeNQDaRthQJ3D3KhOdCjrio1E+/ocng==} + engines: {node: '>= 14.0.0'} + + '@aws-sdk/nested-clients@3.997.29': + resolution: {integrity: sha512-ot6v8J5W8P0w6ryyuIkXP1bHZHTlvwtn83mVCYaBE0GJ6tJX4vPSBx7M98w9O4wmmDruFsDBUMjhEHA+OosUFQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.996.38': + resolution: {integrity: sha512-C379Sk+MiFZCfWZphKlMyLHKxV22OjoGM5KJjj5IJNJcOCWL4IGIpnEGzv1FQiRwhYXfq55SJMfxlqPE08JJ9g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1081.0': + resolution: {integrity: sha512-kduAeI6cL+zqwj3gjPh9LhuX7kBZ83msYxutavaR+UPm5K8J7iThJBvNRAsFNyWTji92CSU8dogUgvi9T0BehA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.973.15': + resolution: {integrity: sha512-IULn8uBV/SMtmOIANsm4WHXIOtVPBWfOWs3WGL0j/sI+KhaYehvOw0ET+9urnn8MBpiijuU/0JOpuwKOE451PQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/xml-builder@3.972.33': + resolution: {integrity: sha512-ezbwz9WpuLctm6o7P2t2naDhVVPI5jFGrVefVybhcKGjU57VIyT46pQVO0RI2RYkUdhdj2Z9uSIlAzGZE9NW9A==} + engines: {node: '>=20.0.0'} + + '@aws/lambda-invoke-store@0.3.0': + resolution: {integrity: sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==} + engines: {node: '>=18.0.0'} + '@babel/code-frame@7.24.2': resolution: {integrity: sha512-y5+tLQyV8pg3fsiln67BVLD1P13Eg4lh5RW9mF0zUuvLrv9uIQ4MCL+CRT+FTsBlBjcIan6PGsLcBN0m3ClUyQ==} engines: {node: '>=6.9.0'} @@ -2716,6 +2844,30 @@ packages: '@rushstack/ts-command-line@4.21.0': resolution: {integrity: sha512-z38FLUCn8M9FQf19gJ9eltdwkvc47PxvJmVZS6aKwbBAa3Pis3r3A+ZcBCVPNb9h/Tbga+i0tHdzoSGUoji9GQ==} + '@smithy/core@3.29.1': + resolution: {integrity: sha512-qoiY4nrk5OCu1+eIR1VB8l5DmON/oKiqrd5zZFAhXJXjJlLWQusKEW/SkBDAtGDcPaz86m9kfcE1lngU0GlM6A==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.4.6': + resolution: {integrity: sha512-B2WQ/PV/H6Jeg3lrIq6bKUfa6Hy01mtK7CGs6lhjzHA6k4aagldH6T6eEjnzKl4HI0cJnAsxfJ19pgb5PV+CVQ==} + engines: {node: '>=18.0.0'} + + '@smithy/fetch-http-handler@5.6.3': + resolution: {integrity: sha512-CwCc/7SMTj45y97MUnDTbTaxvtAsiNNRm81z3abROIuMbMsC2Iy5EKfkkVdsKrz8WExQAAMx1EJapq+9j4fFTQ==} + engines: {node: '>=18.0.0'} + + '@smithy/node-http-handler@4.9.3': + resolution: {integrity: sha512-qZTa4gQFUo8RM02rk6q5UVTDLNrQ1oS20LsepBzqq1QBVc/EHJ03OOUADcqMZiXHArW+Y7+OGY0BpdTwZRq/Yg==} + engines: {node: '>=18.0.0'} + + '@smithy/signature-v4@5.6.2': + resolution: {integrity: sha512-QgHflghMoPxCJ9axiCVh8KZfbC9fuP6vkXXyK//E3cq7nLaSSyyLj0GAoqVWezYeDQmXIZhmlRvLE16jsqDK6g==} + engines: {node: '>=18.0.0'} + + '@smithy/types@4.15.1': + resolution: {integrity: sha512-x3L0XSACF6UYzKpa9biqiRMgvH5+wnFFew9Tm/grFYqgaupPwx/+ojDPpPJM8dZON3S9tjz5U+PQYsCBd1Mw5Q==} + engines: {node: '>=18.0.0'} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -3071,6 +3223,9 @@ packages: resolution: {integrity: sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==} deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. + bowser@2.14.1: + resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} + brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} @@ -5277,6 +5432,214 @@ snapshots: transitivePeerDependencies: - encoding + '@aws-sdk/client-bedrock-runtime@3.1081.0': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/credential-provider-node': 3.972.64 + '@aws-sdk/eventstream-handler-node': 3.972.25 + '@aws-sdk/middleware-eventstream': 3.972.21 + '@aws-sdk/middleware-websocket': 3.972.37 + '@aws-sdk/token-providers': 3.1081.0 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/fetch-http-handler': 5.6.3 + '@smithy/node-http-handler': 4.9.3 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/client-polly@3.1081.0': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/credential-provider-node': 3.972.64 + '@aws-sdk/eventstream-handler-node': 3.972.25 + '@aws-sdk/middleware-eventstream': 3.972.21 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/fetch-http-handler': 5.6.3 + '@smithy/node-http-handler': 4.9.3 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/client-transcribe-streaming@3.1081.0': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/credential-provider-node': 3.972.64 + '@aws-sdk/eventstream-handler-node': 3.972.25 + '@aws-sdk/middleware-eventstream': 3.972.21 + '@aws-sdk/middleware-sdk-transcribe-streaming': 3.972.24 + '@aws-sdk/middleware-websocket': 3.972.37 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/fetch-http-handler': 5.6.3 + '@smithy/node-http-handler': 4.9.3 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/core@3.974.29': + dependencies: + '@aws-sdk/types': 3.973.15 + '@aws-sdk/xml-builder': 3.972.33 + '@aws/lambda-invoke-store': 0.3.0 + '@smithy/core': 3.29.1 + '@smithy/signature-v4': 5.6.2 + '@smithy/types': 4.15.1 + bowser: 2.14.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-env@3.972.55': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-http@3.972.57': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/fetch-http-handler': 5.6.3 + '@smithy/node-http-handler': 4.9.3 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-ini@3.972.62': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/credential-provider-env': 3.972.55 + '@aws-sdk/credential-provider-http': 3.972.57 + '@aws-sdk/credential-provider-login': 3.972.61 + '@aws-sdk/credential-provider-process': 3.972.55 + '@aws-sdk/credential-provider-sso': 3.972.61 + '@aws-sdk/credential-provider-web-identity': 3.972.61 + '@aws-sdk/nested-clients': 3.997.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/credential-provider-imds': 4.4.6 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-login@3.972.61': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/nested-clients': 3.997.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-node@3.972.64': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.55 + '@aws-sdk/credential-provider-http': 3.972.57 + '@aws-sdk/credential-provider-ini': 3.972.62 + '@aws-sdk/credential-provider-process': 3.972.55 + '@aws-sdk/credential-provider-sso': 3.972.61 + '@aws-sdk/credential-provider-web-identity': 3.972.61 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/credential-provider-imds': 4.4.6 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-process@3.972.55': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-sso@3.972.61': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/nested-clients': 3.997.29 + '@aws-sdk/token-providers': 3.1081.0 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-web-identity@3.972.61': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/nested-clients': 3.997.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/eventstream-handler-node@3.972.25': + dependencies: + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/middleware-eventstream@3.972.21': + dependencies: + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/middleware-sdk-transcribe-streaming@3.972.24': + dependencies: + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/middleware-websocket@3.972.37': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/fetch-http-handler': 5.6.3 + '@smithy/signature-v4': 5.6.2 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/nested-clients@3.997.29': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/signature-v4-multi-region': 3.996.38 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/fetch-http-handler': 5.6.3 + '@smithy/node-http-handler': 4.9.3 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/signature-v4-multi-region@3.996.38': + dependencies: + '@aws-sdk/types': 3.973.15 + '@smithy/signature-v4': 5.6.2 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/token-providers@3.1081.0': + dependencies: + '@aws-sdk/core': 3.974.29 + '@aws-sdk/nested-clients': 3.997.29 + '@aws-sdk/types': 3.973.15 + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/types@3.973.15': + dependencies: + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws-sdk/xml-builder@3.972.33': + dependencies: + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@aws/lambda-invoke-store@0.3.0': {} + '@babel/code-frame@7.24.2': dependencies: '@babel/highlight': 7.24.5 @@ -6524,6 +6887,39 @@ snapshots: transitivePeerDependencies: - '@types/node' + '@smithy/core@3.29.1': + dependencies: + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@smithy/credential-provider-imds@4.4.6': + dependencies: + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@smithy/fetch-http-handler@5.6.3': + dependencies: + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@smithy/node-http-handler@4.9.3': + dependencies: + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@smithy/signature-v4@5.6.2': + dependencies: + '@smithy/core': 3.29.1 + '@smithy/types': 4.15.1 + tslib: 2.6.2 + + '@smithy/types@4.15.1': + dependencies: + tslib: 2.6.2 + '@standard-schema/spec@1.1.0': {} '@trivago/prettier-plugin-sort-imports@4.3.0(prettier@3.2.5)': @@ -6928,6 +7324,8 @@ snapshots: boolean@3.2.0: {} + bowser@2.14.1: {} + brace-expansion@1.1.11: dependencies: balanced-match: 1.0.2 diff --git a/turbo.json b/turbo.json index e3d0866ae..848bddcbe 100644 --- a/turbo.json +++ b/turbo.json @@ -2,11 +2,18 @@ "$schema": "https://turborepo.org/schema.json", "globalEnv": [ "ASSEMBLYAI_API_KEY", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_REGION", + "AWS_DEFAULT_REGION", + "AWS_PROFILE", "AZURE_API_KEY", "AZURE_OPENAI_API_KEY", "AZURE_OPENAI_DEPLOYMENT", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_ENTRA_TOKEN", + "BEDROCK_INFERENCE_PROFILE_ARN", "BEY_API_KEY", "BEY_API_URL", "BEY_AVATAR_ID",