fix(aws): preserve Bedrock image format

.changeset/bedrock-image-formats.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents': patch
+---
+
+Preserve image MIME formats when serializing AWS Bedrock chat context images.

agents/src/llm/provider_format/aws.test.ts

@@ -0,0 +1,49 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { describe, expect, it } from 'vitest';
+import { ChatContext, createImageContent } from '../chat_context.js';
+import { toChatCtx } from './aws.js';
+
+const IMAGE_BYTES = Buffer.from('fake image bytes');
+
+describe('AWS Provider Format - toChatCtx', () => {
+  it.each([
+    ['image/jpeg', 'jpeg'],
+    ['image/png', 'png'],
+    ['image/gif', 'gif'],
+    ['image/webp', 'webp'],
+  ])('uses serialized image format for %s', async (mimeType, expectedFormat) => {
+    const ctx = ChatContext.empty();
+    ctx.addMessage({
+      role: 'user',
+      content: [
+        createImageContent({
+          image: `data:${mimeType};base64,${IMAGE_BYTES.toString('base64')}`,
+        }),
+      ],
+    });
+
+    const [messages] = await toChatCtx(ctx);
+
+    const content = messages[0]?.content as Record<string, unknown>[];
+    const image = content[0]?.image as { format: string; source: { bytes: Buffer } };
+    expect(image.format).toBe(expectedFormat);
+    expect(image.source.bytes).toEqual(IMAGE_BYTES);
+  });
+
+  it('rejects external URL images', async () => {
+    const ctx = ChatContext.empty();
+    ctx.addMessage({
+      role: 'user',
+      content: [
+        createImageContent({
+          image: 'https://example.com/image.png',
+          mimeType: 'image/png',
+        }),
+      ],
+    });
+
+    await expect(toChatCtx(ctx)).rejects.toThrow('externalUrl is not supported by AWS Bedrock');
+  });
+});

agents/src/llm/provider_format/aws.ts

@@ -0,0 +1,136 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { ChatContext, ChatItem, ImageContent } from '../chat_context.js';
+import { isInstructions } from '../chat_context.js';
+import { type SerializedImage, serializeImage } from '../utils.js';
+import { convertMidConversationInstructions, groupToolCalls } from './utils.js';
+
+const AWS_IMAGE_FORMATS: Record<string, string> = {
+  'image/jpeg': 'jpeg',
+  'image/png': 'png',
+  'image/gif': 'gif',
+  'image/webp': 'webp',
+};
+
+export interface BedrockFormatData {
+  systemMessages: string[];
+}
+
+export async function toChatCtx(
+  chatCtx: ChatContext,
+  injectDummyUserMessage: boolean = true,
+): Promise<[Record<string, unknown>[], BedrockFormatData]> {
+  chatCtx = convertMidConversationInstructions(chatCtx);
+
+  const messages: Record<string, unknown>[] = [];
+  const systemMessages: string[] = [];
+  let currentRole: string | null = null;
+  let currentContent: Record<string, unknown>[] = [];
+
+  const flattenedItems: ChatItem[] = [];
+  for (const group of groupToolCalls(chatCtx)) {
+    flattenedItems.push(...group.flatten());
+  }
+
+  for (const msg of flattenedItems) {
+    if (msg.type === 'message' && msg.role === 'system' && msg.textContent) {
+      systemMessages.push(msg.textContent);
+      continue;
+    }
+
+    let role: string;
+    if (msg.type === 'message') {
+      role = msg.role === 'assistant' ? 'assistant' : 'user';
+    } else if (msg.type === 'function_call') {
+      role = 'assistant';
+    } else if (msg.type === 'function_call_output') {
+      role = 'user';
+    } else {
+      continue;
+    }
+
+    if (role !== currentRole) {
+      if (currentRole !== null && currentContent.length > 0) {
+        messages.push({ role: currentRole, content: currentContent });
+      }
+      currentContent = [];
+      currentRole = role;
+    }
+
+    if (msg.type === 'message') {
+      for (const content of msg.content) {
+        if (content && typeof content === 'string') {
+          currentContent.push({ text: content });
+        } else if (isInstructions(content)) {
+          currentContent.push({ text: content.value });
+        } else if (content && typeof content === 'object' && content.type === 'image_content') {
+          currentContent.push(await buildImage(content));
+        }
+      }
+    } else if (msg.type === 'function_call') {
+      currentContent.push({
+        toolUse: {
+          toolUseId: msg.callId,
+          name: msg.name,
+          input: JSON.parse(msg.args || '{}'),
+        },
+      });
+    } else if (msg.type === 'function_call_output') {
+      currentContent.push({
+        toolResult: {
+          toolUseId: msg.callId,
+          content: [{ text: msg.output }],
+          status: 'success',
+        },
+      });
+    }
+  }
+
+  if (currentRole !== null && currentContent.length > 0) {
+    messages.push({ role: currentRole, content: currentContent });
+  }
+
+  if (injectDummyUserMessage && (messages.length === 0 || messages[0]?.role !== 'user')) {
+    messages.unshift({ role: 'user', content: [{ text: '(empty)' }] });
+  }
+
+  return [messages, { systemMessages }];
+}
+
+async function buildImage(image: ImageContent): Promise<Record<string, unknown>> {
+  const cacheKey = 'serialized_image';
+  let img: SerializedImage;
+
+  if (image._cache[cacheKey] === undefined) {
+    img = await serializeImage(image);
+    image._cache[cacheKey] = img;
+  }
+  img = image._cache[cacheKey];
+
+  if (img.externalUrl) {
+    throw new Error('externalUrl is not supported by AWS Bedrock.');
+  }
+  if (img.base64Data === undefined) {
+    throw new Error('Serialized image has no data bytes');
+  }
+
+  return {
+    image: {
+      format: imageFormat(img.mimeType),
+      source: { bytes: Buffer.from(img.base64Data, 'base64') },
+    },
+  };
+}
+
+function imageFormat(mimeType: string | undefined): string {
+  if (!mimeType) {
+    return 'jpeg';
+  }
+
+  const format = AWS_IMAGE_FORMATS[mimeType];
+  if (format === undefined) {
+    throw new Error(`Unsupported mimeType ${mimeType} for AWS Bedrock images`);
+  }
+  return format;
+}

agents/src/llm/provider_format/index.ts

@@ -2,14 +2,15 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { ChatContext } from '../chat_context.js';
+import { toChatCtx as toChatCtxAws } from './aws.js';
 import { toChatCtx as toChatCtxGoogle } from './google.js';
 import { toChatCtx as toChatCtxMistralai } from './mistralai.js';
 import {
   toChatCtx as toChatCtxOpenai,
   toResponsesChatCtx as toResponsesChatCtxOpenai,
 } from './openai.js';
 
-export type ProviderFormat = 'openai' | 'openai.responses' | 'google' | 'mistralai';
+export type ProviderFormat = 'openai' | 'openai.responses' | 'google' | 'mistralai' | 'aws';
 
 export async function toChatCtx(
   format: ProviderFormat,
@@ -25,6 +26,8 @@ export async function toChatCtx(
       return await toChatCtxGoogle(chatCtx, injectDummyUserMessage);
     case 'mistralai':
       return toChatCtxMistralai(chatCtx, injectDummyUserMessage);
+    case 'aws':
+      return await toChatCtxAws(chatCtx, injectDummyUserMessage);
     default:
       throw new Error(`Unsupported provider format: ${format}`);
   }