diff --git a/.changeset/fuzzy-crabs-express.md b/.changeset/fuzzy-crabs-express.md new file mode 100644 index 000000000..1783e902d --- /dev/null +++ b/.changeset/fuzzy-crabs-express.md @@ -0,0 +1,7 @@ +--- +'@livekit/agents': minor +'@livekit/agents-plugin-anthropic': patch +'@livekit/agents-plugin-phonic': patch +--- + +Preserve raw expressive markup for provider-facing text while stripping LiveKit expr tags from assistant text content. diff --git a/agents/src/llm/chat_context.test.ts b/agents/src/llm/chat_context.test.ts index 101a9fe46..8de9e825e 100644 --- a/agents/src/llm/chat_context.test.ts +++ b/agents/src/llm/chat_context.test.ts @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from 'vitest'; import { initializeLogger } from '../log.js'; +import { stripExprMarkup } from '../tts/provider_format.js'; import { INSTRUCTIONS_MESSAGE_ID, applyInstructionsModality } from '../voice/generation.js'; import { FakeLLM } from '../voice/testing/fake_llm.js'; import { @@ -26,6 +27,16 @@ initializeLogger({ pretty: false, level: 'error' }); const summaryXml = (summary: string) => ['', summary, ''].join('\n'); +const mixedMarkup = + ' Press [Enter] to see bold, ' + + 'read [the docs](https://docs.livekit.io), then 1 < 2. ' + + 'keep it secret'; + +const mixedMarkupClean = + ' Press [Enter] to see bold, ' + + 'read [the docs](https://docs.livekit.io), then 1 < 2. ' + + 'keep it secret'; + class TrackingFakeLLM extends FakeLLM { chatCalls = 0; @@ -305,6 +316,50 @@ describe('ChatContext.toJSON', () => { }); }); +describe('stripExprMarkup and ChatMessage text content', () => { + it('stripExprMarkup only touches expr tags', () => { + expect(stripExprMarkup(mixedMarkup)).toBe(mixedMarkupClean); + }); + + it('stripExprMarkup is a noop without expr tags', () => { + const text = 'plain text with [brackets] and '; + expect(stripExprMarkup(text)).toBe(text); + }); + + it('strips expr tags from assistant textContent only', () => { + const msg = ChatMessage.create({ role: 'assistant', content: [mixedMarkup] }); + expect(msg.textContent).toBe(mixedMarkupClean); + expect(msg.rawTextContent).toBe(mixedMarkup); + }); + + for (const role of ['user', 'system', 'developer'] as const) { + it(`keeps ${role} textContent raw`, () => { + const msg = ChatMessage.create({ role, content: [mixedMarkup] }); + expect(msg.textContent).toBe(mixedMarkup); + expect(msg.rawTextContent).toBe(mixedMarkup); + }); + } + + it('returns undefined without text', () => { + const msg = ChatMessage.create({ role: 'assistant', content: [] }); + expect(msg.textContent).toBeUndefined(); + expect(msg.rawTextContent).toBeUndefined(); + }); + + it('toJSON stripMarkup strips expr tags from assistant messages only', () => { + const chatCtx = new ChatContext(); + chatCtx.addMessage({ role: 'user', content: [mixedMarkup] }); + chatCtx.addMessage({ role: 'assistant', content: [mixedMarkup] }); + + const strippedItems = chatCtx.toJSON({ stripMarkup: true }).items; + expect(strippedItems[0]).toMatchObject({ content: [mixedMarkup] }); + expect(strippedItems[1]).toMatchObject({ content: [mixedMarkupClean] }); + + const rawItems = chatCtx.toJSON().items; + expect(rawItems[1]).toMatchObject({ content: [mixedMarkup] }); + }); +}); + describe('ChatContext._summarize', () => { it('includes function calls in the summarization source and keeps chronological order', async () => { const ctx = new ChatContext(); diff --git a/agents/src/llm/chat_context.ts b/agents/src/llm/chat_context.ts index 34cf39200..9cae0f53c 100644 --- a/agents/src/llm/chat_context.ts +++ b/agents/src/llm/chat_context.ts @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 import type { AudioFrame, VideoFrame } from '@livekit/rtc-node'; +import { stripExprMarkup } from '../tts/provider_format.js'; import { createImmutableArray, shortuuid } from '../utils.js'; import type { LLM } from './llm.js'; import { type ProviderFormat, toChatCtx } from './provider_format/index.js'; @@ -357,10 +358,20 @@ export class ChatMessage { } /** - * Returns a single string with all text parts of the message joined by new - * lines. If no string content is present, returns `null`. + * Returns text content with LiveKit expressive `` tags removed from assistant messages. + * Use {@link rawTextContent} for the exact model-facing content. */ get textContent(): string | undefined { + const raw = this.rawTextContent; + if (raw === undefined || this.role !== 'assistant') { + return raw; + } + + return stripExprMarkup(raw); + } + + /** Returns the exact text content as generated, joined by new lines. */ + get rawTextContent(): string | undefined { const parts = this.content .filter((c): c is string | Instructions => typeof c === 'string' || isInstructions(c)) .map((c) => (typeof c === 'string' ? c : c.value)); @@ -925,6 +936,7 @@ export class ChatContext { excludeTimestamp?: boolean; excludeFunctionCall?: boolean; excludeConfigUpdate?: boolean; + stripMarkup?: boolean; } = {}, // eslint-disable-next-line @typescript-eslint/no-explicit-any ): JSONObject { @@ -934,6 +946,7 @@ export class ChatContext { excludeTimestamp = true, excludeFunctionCall = false, excludeConfigUpdate = false, + stripMarkup = false, } = options; const items: ChatItem[] = []; @@ -973,6 +986,12 @@ export class ChatContext { return !(typeof c === 'object' && c.type === 'audio_content'); }); } + + if (stripMarkup && processedItem.role === 'assistant') { + processedItem.content = processedItem.content.map((c) => + typeof c === 'string' ? stripExprMarkup(c) : c, + ); + } } items.push(processedItem); @@ -1190,7 +1209,7 @@ export class ChatContext { return toXml(item.role, (item.textContent ?? '').trim()); } - return functionCallItemToMessage(item).textContent ?? ''; + return functionCallItemToMessage(item).rawTextContent ?? ''; }) .join('\n') .trim(); diff --git a/agents/src/llm/provider_format/google.ts b/agents/src/llm/provider_format/google.ts index 2ef717f28..d7d2197d2 100644 --- a/agents/src/llm/provider_format/google.ts +++ b/agents/src/llm/provider_format/google.ts @@ -31,8 +31,8 @@ export async function toChatCtx( for (const msg of flattenedItems) { // Handle system messages separately - if (msg.type === 'message' && msg.role === 'system' && msg.textContent) { - systemMessages.push(msg.textContent); + if (msg.type === 'message' && msg.role === 'system' && msg.rawTextContent) { + systemMessages.push(msg.rawTextContent); continue; } diff --git a/agents/src/llm/provider_format/utils.ts b/agents/src/llm/provider_format/utils.ts index a0fcdbe13..d8a497b5c 100644 --- a/agents/src/llm/provider_format/utils.ts +++ b/agents/src/llm/provider_format/utils.ts @@ -117,12 +117,12 @@ export function convertMidConversationInstructions( item.type === 'message' && (item.role === 'system' || item.role === 'developer') && firstSystemSeen && - item.textContent + item.rawTextContent ) { items.push( ChatMessage.create({ role, - content: template.replace('{instructions}', item.textContent), + content: template.replace('{instructions}', item.rawTextContent), id: item.id, createdAt: item.createdAt, }), diff --git a/agents/src/llm/utils.ts b/agents/src/llm/utils.ts index 0db2bb802..0b138f687 100644 --- a/agents/src/llm/utils.ts +++ b/agents/src/llm/utils.ts @@ -706,6 +706,18 @@ interface DiffOps { toCreate: Array<[string | null, string]>; // (previous_item_id, id), if previous_item_id is null, add to the root } +function isUnchangedChatItem(oldItem: ChatItem | undefined, newItem: ChatItem): boolean { + if (!oldItem || oldItem.type !== newItem.type) { + return false; + } + + if (oldItem.type === 'message' && newItem.type === 'message') { + return oldItem.rawTextContent === newItem.rawTextContent; + } + + return true; +} + /** * Compute the minimal list of create/remove operations to transform oldCtx into newCtx. * @@ -717,6 +729,13 @@ export function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): Di const oldIds = oldCtx.items.map((item: ChatItem) => item.id); const newIds = newCtx.items.map((item: ChatItem) => item.id); const lcsIds = new Set(computeLCS(oldIds, newIds)); + const oldCtxById = new Map(oldCtx.items.map((item) => [item.id, item])); + + for (const newItem of newCtx.items) { + if (lcsIds.has(newItem.id) && !isUnchangedChatItem(oldCtxById.get(newItem.id), newItem)) { + lcsIds.delete(newItem.id); + } + } const toRemove = oldCtx.items.filter((msg) => !lcsIds.has(msg.id)).map((msg) => msg.id); const toCreate: Array<[string | null, string]> = []; diff --git a/agents/src/tts/provider_format.ts b/agents/src/tts/provider_format.ts new file mode 100644 index 000000000..9f84d1f2d --- /dev/null +++ b/agents/src/tts/provider_format.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** Strip only LiveKit expressive `` tags, leaving provider-native markup untouched. */ +export function stripExprMarkup(text: string): string { + let stripped = text; + let previous: string; + + do { + previous = stripped; + stripped = stripped + .replace(/]*\/>/gi, '') + .replace(/]*>([\s\S]*?)<\/expr>/gi, '$1'); + } while (stripped !== previous); + + return stripped; +} diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 9fd906783..0839f4d93 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -2117,7 +2117,7 @@ export class AgentActivity implements RecognitionHooks { this.realtimeReplyTask({ speechHandle: handle, // TODO(brian): support llm.ChatMessage for the realtime model - userInput: userMessage?.textContent, + userInput: userMessage?.rawTextContent, instructions, modelSettings: { // isGiven(toolChoice) = toolChoice !== undefined @@ -2391,7 +2391,7 @@ export class AgentActivity implements RecognitionHooks { // make sure the onUserTurnCompleted didn't change some request parameters // otherwise invalidate the preemptive generation if ( - preemptive.info.newTranscript === userMessage?.textContent && + preemptive.info.newTranscript === userMessage?.rawTextContent && preemptive.chatCtx.isEquivalent(chatCtx) && preemptive.tools.equals(this.tools) && isSameToolChoice(preemptive.toolChoice, this.toolChoice) @@ -2649,7 +2649,7 @@ export class AgentActivity implements RecognitionHooks { span.setAttribute(traceTypes.ATTR_INSTRUCTIONS, renderInstructions(instructions)); } if (newMessage) { - span.setAttribute(traceTypes.ATTR_USER_INPUT, newMessage.textContent || ''); + span.setAttribute(traceTypes.ATTR_USER_INPUT, newMessage.rawTextContent || ''); } const localParticipant = this.agentSession._roomIO?.localParticipant; diff --git a/plugins/anthropic/src/llm.ts b/plugins/anthropic/src/llm.ts index 7f9fe9056..0c85bbf90 100644 --- a/plugins/anthropic/src/llm.ts +++ b/plugins/anthropic/src/llm.ts @@ -113,7 +113,7 @@ export class LLM extends llm.LLM { for (const msg of chatCtx.items) { if (msg.type === 'message') { - const textContent = msg.textContent || ''; + const textContent = msg.rawTextContent || ''; if (msg.role === 'system' || msg.role === 'developer') { system.push({ type: 'text', text: textContent }); } else if (msg.role === 'user' || msg.role === 'assistant') { diff --git a/plugins/phonic/src/realtime/realtime_model.ts b/plugins/phonic/src/realtime/realtime_model.ts index 6ccfb2102..d2a92cfd4 100644 --- a/plugins/phonic/src/realtime/realtime_model.ts +++ b/plugins/phonic/src/realtime/realtime_model.ts @@ -369,19 +369,19 @@ export class RealtimeSession extends llm.RealtimeSession { } } if (item?.type === 'message') { - if ((item.role === 'system' || item.role === 'developer') && item.textContent) { - this.#logger.debug(`Sending add system message: ${item.textContent}`); + if ((item.role === 'system' || item.role === 'developer') && item.rawTextContent) { + this.#logger.debug(`Sending add system message: ${item.rawTextContent}`); this.socket?.sendAddSystemMessage({ type: 'add_system_message', - system_message: item.textContent, + system_message: item.rawTextContent, }); sentAddSystemMessage = true; } // Only treat a user message as text input when it's appended at the tail of the context. - if (item.role === 'user' && itemId === lastItemId && item.textContent) { - this.#logger.debug(`Received user text input: ${item.textContent}`); - this.pendingUserText = item.textContent; + if (item.role === 'user' && itemId === lastItemId && item.rawTextContent) { + this.#logger.debug(`Received user text input: ${item.rawTextContent}`); + this.pendingUserText = item.rawTextContent; bufferedUserText = true; } } @@ -1011,7 +1011,7 @@ export class RealtimeSession extends llm.RealtimeSession { function chatItemToText(item: llm.ChatItem): string | undefined { if (item.type === 'message') { - const text = item.textContent?.trim(); + const text = item.rawTextContent?.trim(); if (!text) return undefined; return `<${item.role}>${text}`; }