From ee7f0da9acc66442b56ee953f5b25e2fd7de635d Mon Sep 17 00:00:00 2001 From: Sportinger Date: Fri, 22 May 2026 15:17:12 +0200 Subject: [PATCH 1/2] fix: sanitize AI chat tool result context Refs #141 --- src/components/panels/AIChatPanel.tsx | 117 ++-------- src/components/panels/aiChatSerialization.ts | 223 +++++++++++++++++++ tests/unit/aiChatSerialization.test.ts | 69 ++++++ 3 files changed, 310 insertions(+), 99 deletions(-) create mode 100644 src/components/panels/aiChatSerialization.ts create mode 100644 tests/unit/aiChatSerialization.test.ts diff --git a/src/components/panels/AIChatPanel.tsx b/src/components/panels/AIChatPanel.tsx index 64ef9501..29c6b02b 100644 --- a/src/components/panels/AIChatPanel.tsx +++ b/src/components/panels/AIChatPanel.tsx @@ -23,6 +23,12 @@ import { LEMONADE_MODEL_PRESETS, type LemonadeModelInfo, } from '../../services/lemonadeProvider'; +import { + formatStoredToolMessageForApi, + formatToolResultForApi, + MAX_TOOL_RESULT_MESSAGE_CHARS, + type ModelToolResult, +} from './aiChatSerialization'; import './AIChatPanel.css'; // Available OpenAI models with credit cost per request @@ -182,6 +188,7 @@ interface Message { role: 'user' | 'assistant' | 'tool'; content: string; timestamp: Date; + modelContent?: string; toolCalls?: ToolCall[]; toolName?: string; isToolResult?: boolean; @@ -211,102 +218,12 @@ interface PendingApproval { } interface ExecutedToolResult { - result: { success: boolean; data?: unknown; error?: string }; + result: ModelToolResult; toolName: string; } type SelectorMenu = 'provider' | 'model' | null; -const MAX_TOOL_RESULT_MESSAGE_CHARS = 12000; -const MAX_TOOL_RESULT_ARRAY_ITEMS = 20; -const MAX_TOOL_RESULT_OBJECT_KEYS = 30; -const MAX_TOOL_RESULT_STRING_CHARS = 1200; - -function truncateText(value: string, maxLength: number): string { - if (value.length <= maxLength) { - return value; - } - - return `${value.slice(0, maxLength)}... [truncated]`; -} - -function summarizeToolResultValue(value: unknown, depth = 0): unknown { - if (typeof value === 'string') { - return truncateText(value, MAX_TOOL_RESULT_STRING_CHARS); - } - - if ( - value === null - || typeof value === 'number' - || typeof value === 'boolean' - || typeof value === 'undefined' - ) { - return value; - } - - if (depth >= 3) { - return '[truncated nested value]'; - } - - if (Array.isArray(value)) { - const items = value - .slice(0, MAX_TOOL_RESULT_ARRAY_ITEMS) - .map((item) => summarizeToolResultValue(item, depth + 1)); - - if (value.length > MAX_TOOL_RESULT_ARRAY_ITEMS) { - items.push(`[${value.length - MAX_TOOL_RESULT_ARRAY_ITEMS} more items truncated]`); - } - - return items; - } - - if (typeof value === 'object') { - const entries = Object.entries(value as Record); - const summary: Record = {}; - - for (const [key, nestedValue] of entries.slice(0, MAX_TOOL_RESULT_OBJECT_KEYS)) { - summary[key] = summarizeToolResultValue(nestedValue, depth + 1); - } - - if (entries.length > MAX_TOOL_RESULT_OBJECT_KEYS) { - summary.__truncatedKeys = entries.length - MAX_TOOL_RESULT_OBJECT_KEYS; - } - - return summary; - } - - return String(value); -} - -function formatToolResultForApi( - result: { success: boolean; data?: unknown; error?: string }, - maxLength = MAX_TOOL_RESULT_MESSAGE_CHARS, -): string { - const serialized = JSON.stringify(result); - - if (serialized.length <= maxLength) { - return serialized; - } - - const summarized = JSON.stringify({ - data: summarizeToolResultValue(result.data), - error: result.error ?? null, - success: result.success, - truncated: true, - }); - - if (summarized.length <= maxLength) { - return summarized; - } - - return JSON.stringify({ - error: result.error ?? null, - preview: truncateText(serialized, Math.max(256, maxLength - 128)), - success: result.success, - truncated: true, - }); -} - function getErrorMessage(error: unknown): string { if (error instanceof Error) { return error.message; @@ -715,7 +632,7 @@ export function AIChatPanel() { apiMessages.push({ role: 'tool', - content: msg.content, + content: formatStoredToolMessageForApi(msg.modelContent ?? msg.content), tool_call_id: msg.id, }); } @@ -983,7 +900,7 @@ export function AIChatPanel() { const policy = getToolPolicy(toolCall.name); const needsConfirmation = shouldRequireConfirmation(policy, aiApprovalMode); - let result: { success: boolean; data?: unknown; error?: string }; + let result: ModelToolResult; if (needsConfirmation) { // Show confirmation UI and wait for user response @@ -1009,10 +926,17 @@ export function AIChatPanel() { } } + const modelToolResultContent = formatToolResultForApi( + result, + aiProvider === 'lemonade' + ? LEMONADE_MAX_TOOL_RESULT_MESSAGE_CHARS + : MAX_TOOL_RESULT_MESSAGE_CHARS, + ); const toolResultMessage: Message = { id: toolCall.id, role: 'tool', content: JSON.stringify(result, null, 2), + modelContent: modelToolResultContent, timestamp: new Date(), toolName: toolCall.name, isToolResult: true, @@ -1024,12 +948,7 @@ export function AIChatPanel() { // Add tool result to API messages apiMessages.push({ role: 'tool', - content: formatToolResultForApi( - result, - aiProvider === 'lemonade' - ? LEMONADE_MAX_TOOL_RESULT_MESSAGE_CHARS - : MAX_TOOL_RESULT_MESSAGE_CHARS, - ), + content: modelToolResultContent, tool_call_id: toolCall.id, }); } diff --git a/src/components/panels/aiChatSerialization.ts b/src/components/panels/aiChatSerialization.ts new file mode 100644 index 00000000..c3d0c155 --- /dev/null +++ b/src/components/panels/aiChatSerialization.ts @@ -0,0 +1,223 @@ +export interface ModelToolResult { + success: boolean; + data?: unknown; + error?: string; +} + +export const MAX_TOOL_RESULT_MESSAGE_CHARS = 12000; + +const MAX_TOOL_RESULT_ARRAY_ITEMS = 20; +const MAX_TOOL_RESULT_OBJECT_KEYS = 30; +const MAX_TOOL_RESULT_STRING_CHARS = 1200; + +const IMAGE_DATA_URL_PATTERN = /^data:(image\/[a-z0-9.+-]+);base64,([a-z0-9+/=\s]+)$/i; +const IMAGE_DATA_URL_GLOBAL_PATTERN = /data:image\/[a-z0-9.+-]+;base64,[a-z0-9+/=\s]+/gi; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function truncateText(value: string, maxLength: number): string { + if (value.length <= maxLength) { + return value; + } + + return `${value.slice(0, maxLength)}... [truncated]`; +} + +function estimateDecodedBytes(base64: string): number { + const compact = base64.replace(/\s/g, ''); + if (compact.length === 0) { + return 0; + } + + const padding = compact.endsWith('==') ? 2 : compact.endsWith('=') ? 1 : 0; + return Math.max(0, Math.floor((compact.length * 3) / 4) - padding); +} + +function formatByteCount(bytes: number): string { + if (bytes < 1024) { + return `${bytes} B`; + } + + if (bytes < 1024 * 1024) { + return `${(bytes / 1024).toFixed(1)} KB`; + } + + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +function describeImageDataUrl(value: string): string | null { + const match = IMAGE_DATA_URL_PATTERN.exec(value); + if (!match) { + return null; + } + + const mediaType = match[1]; + const byteCount = estimateDecodedBytes(match[2]); + return `[image data omitted from text context: ${mediaType}, approx ${formatByteCount(byteCount)}]`; +} + +function redactImageDataUrls(value: string): string { + return value.replace(IMAGE_DATA_URL_GLOBAL_PATTERN, (match) => ( + describeImageDataUrl(match) ?? '[image data omitted from text context]' + )); +} + +function sanitizeToolResultValue(value: unknown, depth = 0): unknown { + if (typeof value === 'string') { + return describeImageDataUrl(value) ?? redactImageDataUrls(value); + } + + if ( + value === null + || typeof value === 'number' + || typeof value === 'boolean' + || typeof value === 'undefined' + ) { + return value; + } + + if (depth >= 10) { + return '[truncated nested value]'; + } + + if (Array.isArray(value)) { + return value.map((item) => sanitizeToolResultValue(item, depth + 1)); + } + + if (isRecord(value)) { + const sanitized: Record = {}; + + for (const [key, nestedValue] of Object.entries(value)) { + sanitized[key] = sanitizeToolResultValue(nestedValue, depth + 1); + } + + return sanitized; + } + + return String(value); +} + +function summarizeToolResultValue(value: unknown, depth = 0): unknown { + if (typeof value === 'string') { + return truncateText(describeImageDataUrl(value) ?? redactImageDataUrls(value), MAX_TOOL_RESULT_STRING_CHARS); + } + + if ( + value === null + || typeof value === 'number' + || typeof value === 'boolean' + || typeof value === 'undefined' + ) { + return value; + } + + if (depth >= 3) { + return '[truncated nested value]'; + } + + if (Array.isArray(value)) { + const items = value + .slice(0, MAX_TOOL_RESULT_ARRAY_ITEMS) + .map((item) => summarizeToolResultValue(item, depth + 1)); + + if (value.length > MAX_TOOL_RESULT_ARRAY_ITEMS) { + items.push(`[${value.length - MAX_TOOL_RESULT_ARRAY_ITEMS} more items truncated]`); + } + + return items; + } + + if (isRecord(value)) { + const entries = Object.entries(value); + const summary: Record = {}; + + for (const [key, nestedValue] of entries.slice(0, MAX_TOOL_RESULT_OBJECT_KEYS)) { + summary[key] = summarizeToolResultValue(nestedValue, depth + 1); + } + + if (entries.length > MAX_TOOL_RESULT_OBJECT_KEYS) { + summary.__truncatedKeys = entries.length - MAX_TOOL_RESULT_OBJECT_KEYS; + } + + return summary; + } + + return String(value); +} + +function sanitizeToolResult(result: ModelToolResult): ModelToolResult { + return { + data: sanitizeToolResultValue(result.data), + error: result.error, + success: result.success, + }; +} + +function isToolResultLike(value: unknown): value is ModelToolResult { + return isRecord(value) && typeof value.success === 'boolean'; +} + +function formatGenericStoredValueForApi(value: unknown, maxLength: number): string { + const sanitized = sanitizeToolResultValue(value); + const serialized = JSON.stringify(sanitized); + + if (serialized.length <= maxLength) { + return serialized; + } + + return JSON.stringify({ + preview: truncateText(redactImageDataUrls(serialized), Math.max(256, maxLength - 128)), + truncated: true, + }); +} + +export function formatToolResultForApi( + result: ModelToolResult, + maxLength = MAX_TOOL_RESULT_MESSAGE_CHARS, +): string { + const sanitized = sanitizeToolResult(result); + const serialized = JSON.stringify(sanitized); + + if (serialized.length <= maxLength) { + return serialized; + } + + const summarized = JSON.stringify({ + data: summarizeToolResultValue(sanitized.data), + error: sanitized.error ?? null, + success: sanitized.success, + truncated: true, + }); + + if (summarized.length <= maxLength) { + return summarized; + } + + return JSON.stringify({ + error: sanitized.error ?? null, + preview: truncateText(redactImageDataUrls(serialized), Math.max(256, maxLength - 128)), + success: sanitized.success, + truncated: true, + }); +} + +export function formatStoredToolMessageForApi( + content: string, + maxLength = MAX_TOOL_RESULT_MESSAGE_CHARS, +): string { + try { + const parsed = JSON.parse(content); + if (isToolResultLike(parsed)) { + return formatToolResultForApi(parsed, maxLength); + } + + return formatGenericStoredValueForApi(parsed, maxLength); + } catch { + const redacted = redactImageDataUrls(content); + return redacted.length <= maxLength + ? redacted + : truncateText(redacted, Math.max(256, maxLength)); + } +} diff --git a/tests/unit/aiChatSerialization.test.ts b/tests/unit/aiChatSerialization.test.ts new file mode 100644 index 00000000..40509e68 --- /dev/null +++ b/tests/unit/aiChatSerialization.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest'; +import { + formatStoredToolMessageForApi, + formatToolResultForApi, +} from '../../src/components/panels/aiChatSerialization'; + +const SAMPLE_DATA_URL = 'data:image/png;base64,QUJDREVGR0g='; + +describe('AI chat tool-result serialization', () => { + it('omits raw image data URLs even when the tool result is below the truncation limit', () => { + const content = formatToolResultForApi({ + success: true, + data: { + capturedAt: 1.25, + dataUrl: SAMPLE_DATA_URL, + height: 360, + width: 640, + }, + }); + + expect(content).not.toContain('data:image/png;base64'); + expect(content).not.toContain('QUJDREVGR0g='); + + const parsed = JSON.parse(content) as { + data: { capturedAt: number; dataUrl: string; height: number; width: number }; + success: boolean; + }; + expect(parsed.success).toBe(true); + expect(parsed.data.width).toBe(640); + expect(parsed.data.height).toBe(360); + expect(parsed.data.capturedAt).toBe(1.25); + expect(parsed.data.dataUrl).toContain('image data omitted from text context'); + }); + + it('sanitizes old full UI tool messages before rebuilding API messages from history', () => { + const storedUiContent = JSON.stringify({ + success: true, + data: { + frameCount: 8, + gridSize: '4x2', + dataUrl: SAMPLE_DATA_URL, + width: 1280, + height: 360, + }, + }, null, 2); + + const apiContent = formatStoredToolMessageForApi(storedUiContent); + + expect(apiContent).not.toContain('data:image/png;base64'); + expect(apiContent).not.toContain('QUJDREVGR0g='); + + const parsed = JSON.parse(apiContent) as { + data: { dataUrl: string; frameCount: number; gridSize: string }; + success: boolean; + }; + expect(parsed.success).toBe(true); + expect(parsed.data.frameCount).toBe(8); + expect(parsed.data.gridSize).toBe('4x2'); + expect(parsed.data.dataUrl).toContain('image data omitted from text context'); + }); + + it('redacts image data URLs from non-JSON stored tool content as a fallback', () => { + const apiContent = formatStoredToolMessageForApi(`preview=${SAMPLE_DATA_URL}`); + + expect(apiContent).not.toContain('data:image/png;base64'); + expect(apiContent).not.toContain('QUJDREVGR0g='); + expect(apiContent).toContain('image data omitted from text context'); + }); +}); From f3cd979079c2296ee9c320c5e22d11af9038e918 Mon Sep 17 00:00:00 2001 From: Sportinger Date: Fri, 22 May 2026 15:22:27 +0200 Subject: [PATCH 2/2] chore: release 1.8.3 --- src/changelog-data.json | 8 ++++++++ src/version.ts | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/changelog-data.json b/src/changelog-data.json index cf457be9..326b3482 100644 --- a/src/changelog-data.json +++ b/src/changelog-data.json @@ -1,4 +1,12 @@ [ + { + "date": "2026-05-22", + "type": "fix", + "title": "AI Preview Tool Context Sanitization", + "description": "AI chat tool history now keeps preview image payloads out of model-visible text context while preserving full preview results for the UI.", + "section": "AI / Reliability", + "commits": ["ee7f0da9"] + }, { "date": "2026-05-20", "type": "new", diff --git a/src/version.ts b/src/version.ts index 982a4882..1f386653 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,6 +1,6 @@ // App version // Format: MAJOR.MINOR.PATCH -export const APP_VERSION = '1.8.2'; +export const APP_VERSION = '1.8.3'; export interface ChangelogNotice { type: 'info' | 'warning' | 'success' | 'danger';