From b2a64b09eb6c8dc33e0c544bd45840e61fdd2790 Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 22:41:54 +0000 Subject: [PATCH] fix(aws): preserve Bedrock image format --- .changeset/bedrock-image-formats.md | 5 + agents/src/llm/provider_format/aws.test.ts | 49 ++++++++ agents/src/llm/provider_format/aws.ts | 136 +++++++++++++++++++++ agents/src/llm/provider_format/index.ts | 5 +- 4 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 .changeset/bedrock-image-formats.md create mode 100644 agents/src/llm/provider_format/aws.test.ts create mode 100644 agents/src/llm/provider_format/aws.ts diff --git a/.changeset/bedrock-image-formats.md b/.changeset/bedrock-image-formats.md new file mode 100644 index 000000000..8c7f6b7a3 --- /dev/null +++ b/.changeset/bedrock-image-formats.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Preserve image MIME formats when serializing AWS Bedrock chat context images. diff --git a/agents/src/llm/provider_format/aws.test.ts b/agents/src/llm/provider_format/aws.test.ts new file mode 100644 index 000000000..813216944 --- /dev/null +++ b/agents/src/llm/provider_format/aws.test.ts @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { describe, expect, it } from 'vitest'; +import { ChatContext, createImageContent } from '../chat_context.js'; +import { toChatCtx } from './aws.js'; + +const IMAGE_BYTES = Buffer.from('fake image bytes'); + +describe('AWS Provider Format - toChatCtx', () => { + it.each([ + ['image/jpeg', 'jpeg'], + ['image/png', 'png'], + ['image/gif', 'gif'], + ['image/webp', 'webp'], + ])('uses serialized image format for %s', async (mimeType, expectedFormat) => { + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'user', + content: [ + createImageContent({ + image: `data:${mimeType};base64,${IMAGE_BYTES.toString('base64')}`, + }), + ], + }); + + const [messages] = await toChatCtx(ctx); + + const content = messages[0]?.content as Record[]; + const image = content[0]?.image as { format: string; source: { bytes: Buffer } }; + expect(image.format).toBe(expectedFormat); + expect(image.source.bytes).toEqual(IMAGE_BYTES); + }); + + it('rejects external URL images', async () => { + const ctx = ChatContext.empty(); + ctx.addMessage({ + role: 'user', + content: [ + createImageContent({ + image: 'https://example.com/image.png', + mimeType: 'image/png', + }), + ], + }); + + await expect(toChatCtx(ctx)).rejects.toThrow('externalUrl is not supported by AWS Bedrock'); + }); +}); diff --git a/agents/src/llm/provider_format/aws.ts b/agents/src/llm/provider_format/aws.ts new file mode 100644 index 000000000..36a019dd4 --- /dev/null +++ b/agents/src/llm/provider_format/aws.ts @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { ChatContext, ChatItem, ImageContent } from '../chat_context.js'; +import { isInstructions } from '../chat_context.js'; +import { type SerializedImage, serializeImage } from '../utils.js'; +import { convertMidConversationInstructions, groupToolCalls } from './utils.js'; + +const AWS_IMAGE_FORMATS: Record = { + 'image/jpeg': 'jpeg', + 'image/png': 'png', + 'image/gif': 'gif', + 'image/webp': 'webp', +}; + +export interface BedrockFormatData { + systemMessages: string[]; +} + +export async function toChatCtx( + chatCtx: ChatContext, + injectDummyUserMessage: boolean = true, +): Promise<[Record[], BedrockFormatData]> { + chatCtx = convertMidConversationInstructions(chatCtx); + + const messages: Record[] = []; + const systemMessages: string[] = []; + let currentRole: string | null = null; + let currentContent: Record[] = []; + + const flattenedItems: ChatItem[] = []; + for (const group of groupToolCalls(chatCtx)) { + flattenedItems.push(...group.flatten()); + } + + for (const msg of flattenedItems) { + if (msg.type === 'message' && msg.role === 'system' && msg.textContent) { + systemMessages.push(msg.textContent); + continue; + } + + let role: string; + if (msg.type === 'message') { + role = msg.role === 'assistant' ? 'assistant' : 'user'; + } else if (msg.type === 'function_call') { + role = 'assistant'; + } else if (msg.type === 'function_call_output') { + role = 'user'; + } else { + continue; + } + + if (role !== currentRole) { + if (currentRole !== null && currentContent.length > 0) { + messages.push({ role: currentRole, content: currentContent }); + } + currentContent = []; + currentRole = role; + } + + if (msg.type === 'message') { + for (const content of msg.content) { + if (content && typeof content === 'string') { + currentContent.push({ text: content }); + } else if (isInstructions(content)) { + currentContent.push({ text: content.value }); + } else if (content && typeof content === 'object' && content.type === 'image_content') { + currentContent.push(await buildImage(content)); + } + } + } else if (msg.type === 'function_call') { + currentContent.push({ + toolUse: { + toolUseId: msg.callId, + name: msg.name, + input: JSON.parse(msg.args || '{}'), + }, + }); + } else if (msg.type === 'function_call_output') { + currentContent.push({ + toolResult: { + toolUseId: msg.callId, + content: [{ text: msg.output }], + status: 'success', + }, + }); + } + } + + if (currentRole !== null && currentContent.length > 0) { + messages.push({ role: currentRole, content: currentContent }); + } + + if (injectDummyUserMessage && (messages.length === 0 || messages[0]?.role !== 'user')) { + messages.unshift({ role: 'user', content: [{ text: '(empty)' }] }); + } + + return [messages, { systemMessages }]; +} + +async function buildImage(image: ImageContent): Promise> { + const cacheKey = 'serialized_image'; + let img: SerializedImage; + + if (image._cache[cacheKey] === undefined) { + img = await serializeImage(image); + image._cache[cacheKey] = img; + } + img = image._cache[cacheKey]; + + if (img.externalUrl) { + throw new Error('externalUrl is not supported by AWS Bedrock.'); + } + if (img.base64Data === undefined) { + throw new Error('Serialized image has no data bytes'); + } + + return { + image: { + format: imageFormat(img.mimeType), + source: { bytes: Buffer.from(img.base64Data, 'base64') }, + }, + }; +} + +function imageFormat(mimeType: string | undefined): string { + if (!mimeType) { + return 'jpeg'; + } + + const format = AWS_IMAGE_FORMATS[mimeType]; + if (format === undefined) { + throw new Error(`Unsupported mimeType ${mimeType} for AWS Bedrock images`); + } + return format; +} diff --git a/agents/src/llm/provider_format/index.ts b/agents/src/llm/provider_format/index.ts index 56e014b7e..6b72b45af 100644 --- a/agents/src/llm/provider_format/index.ts +++ b/agents/src/llm/provider_format/index.ts @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 import type { ChatContext } from '../chat_context.js'; +import { toChatCtx as toChatCtxAws } from './aws.js'; import { toChatCtx as toChatCtxGoogle } from './google.js'; import { toChatCtx as toChatCtxMistralai } from './mistralai.js'; import { @@ -9,7 +10,7 @@ import { toResponsesChatCtx as toResponsesChatCtxOpenai, } from './openai.js'; -export type ProviderFormat = 'openai' | 'openai.responses' | 'google' | 'mistralai'; +export type ProviderFormat = 'openai' | 'openai.responses' | 'google' | 'mistralai' | 'aws'; export async function toChatCtx( format: ProviderFormat, @@ -25,6 +26,8 @@ export async function toChatCtx( return await toChatCtxGoogle(chatCtx, injectDummyUserMessage); case 'mistralai': return toChatCtxMistralai(chatCtx, injectDummyUserMessage); + case 'aws': + return await toChatCtxAws(chatCtx, injectDummyUserMessage); default: throw new Error(`Unsupported provider format: ${format}`); }