fix(core): flip text content fields

.changeset/fuzzy-crabs-express.md

@@ -0,0 +1,7 @@
+---
+'@livekit/agents': minor
+'@livekit/agents-plugin-anthropic': patch
+'@livekit/agents-plugin-phonic': patch
+---
+
+Preserve raw expressive markup for provider-facing text while stripping LiveKit expr tags from assistant text content.

agents/src/llm/chat_context.test.ts

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 import { describe, expect, it } from 'vitest';
 import { initializeLogger } from '../log.js';
+import { stripExprMarkup } from '../tts/provider_format.js';
 import { INSTRUCTIONS_MESSAGE_ID, applyInstructionsModality } from '../voice/generation.js';
 import { FakeLLM } from '../voice/testing/fake_llm.js';
 import {
@@ -26,6 +27,16 @@ initializeLogger({ pretty: false, level: 'error' });
 const summaryXml = (summary: string) =>
   ['<chat_history_summary>', summary, '</chat_history_summary>'].join('\n');
 
+const mixedMarkup =
+  '<expr type="expression" label="happy"/> Press [Enter] to see <b>bold</b>, ' +
+  'read [the docs](https://docs.livekit.io), then 1 < 2. <break time="1s"/> ' +
+  '<expr type="prosody" label="whisper">keep it secret</expr>';
+
+const mixedMarkupClean =
+  ' Press [Enter] to see <b>bold</b>, ' +
+  'read [the docs](https://docs.livekit.io), then 1 < 2. <break time="1s"/> ' +
+  'keep it secret';
+
 class TrackingFakeLLM extends FakeLLM {
   chatCalls = 0;
 
@@ -305,6 +316,50 @@ describe('ChatContext.toJSON', () => {
   });
 });
 
+describe('stripExprMarkup and ChatMessage text content', () => {
+  it('stripExprMarkup only touches expr tags', () => {
+    expect(stripExprMarkup(mixedMarkup)).toBe(mixedMarkupClean);
+  });
+
+  it('stripExprMarkup is a noop without expr tags', () => {
+    const text = 'plain text with [brackets] and <sound value="laugh"/>';
+    expect(stripExprMarkup(text)).toBe(text);
+  });
+
+  it('strips expr tags from assistant textContent only', () => {
+    const msg = ChatMessage.create({ role: 'assistant', content: [mixedMarkup] });
+    expect(msg.textContent).toBe(mixedMarkupClean);
+    expect(msg.rawTextContent).toBe(mixedMarkup);
+  });
+
+  for (const role of ['user', 'system', 'developer'] as const) {
+    it(`keeps ${role} textContent raw`, () => {
+      const msg = ChatMessage.create({ role, content: [mixedMarkup] });
+      expect(msg.textContent).toBe(mixedMarkup);
+      expect(msg.rawTextContent).toBe(mixedMarkup);
+    });
+  }
+
+  it('returns undefined without text', () => {
+    const msg = ChatMessage.create({ role: 'assistant', content: [] });
+    expect(msg.textContent).toBeUndefined();
+    expect(msg.rawTextContent).toBeUndefined();
+  });
+
+  it('toJSON stripMarkup strips expr tags from assistant messages only', () => {
+    const chatCtx = new ChatContext();
+    chatCtx.addMessage({ role: 'user', content: [mixedMarkup] });
+    chatCtx.addMessage({ role: 'assistant', content: [mixedMarkup] });
+
+    const strippedItems = chatCtx.toJSON({ stripMarkup: true }).items;
+    expect(strippedItems[0]).toMatchObject({ content: [mixedMarkup] });
+    expect(strippedItems[1]).toMatchObject({ content: [mixedMarkupClean] });
+
+    const rawItems = chatCtx.toJSON().items;
+    expect(rawItems[1]).toMatchObject({ content: [mixedMarkup] });
+  });
+});
+
 describe('ChatContext._summarize', () => {
   it('includes function calls in the summarization source and keeps chronological order', async () => {
     const ctx = new ChatContext();

agents/src/llm/chat_context.ts

@@ -2,6 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import type { AudioFrame, VideoFrame } from '@livekit/rtc-node';
+import { stripExprMarkup } from '../tts/provider_format.js';
 import { createImmutableArray, shortuuid } from '../utils.js';
 import type { LLM } from './llm.js';
 import { type ProviderFormat, toChatCtx } from './provider_format/index.js';
@@ -357,10 +358,20 @@ export class ChatMessage {
   }
 
   /**
-   * Returns a single string with all text parts of the message joined by new
-   * lines. If no string content is present, returns `null`.
+   * Returns text content with LiveKit expressive `<expr/>` tags removed from assistant messages.
+   * Use {@link rawTextContent} for the exact model-facing content.
    */
   get textContent(): string | undefined {
+    const raw = this.rawTextContent;
+    if (raw === undefined || this.role !== 'assistant') {
+      return raw;
+    }
+
+    return stripExprMarkup(raw);
+  }
+
+  /** Returns the exact text content as generated, joined by new lines. */
+  get rawTextContent(): string | undefined {
     const parts = this.content
       .filter((c): c is string | Instructions => typeof c === 'string' || isInstructions(c))
       .map((c) => (typeof c === 'string' ? c : c.value));
@@ -925,6 +936,7 @@ export class ChatContext {
       excludeTimestamp?: boolean;
       excludeFunctionCall?: boolean;
       excludeConfigUpdate?: boolean;
+      stripMarkup?: boolean;
     } = {},
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
   ): JSONObject {
@@ -934,6 +946,7 @@ export class ChatContext {
       excludeTimestamp = true,
       excludeFunctionCall = false,
       excludeConfigUpdate = false,
+      stripMarkup = false,
     } = options;
 
     const items: ChatItem[] = [];
@@ -973,6 +986,12 @@ export class ChatContext {
             return !(typeof c === 'object' && c.type === 'audio_content');
           });
         }
+
+        if (stripMarkup && processedItem.role === 'assistant') {
+          processedItem.content = processedItem.content.map((c) =>
+            typeof c === 'string' ? stripExprMarkup(c) : c,
+          );
+        }
       }
 
       items.push(processedItem);
@@ -1190,7 +1209,7 @@ export class ChatContext {
           return toXml(item.role, (item.textContent ?? '').trim());
         }
 
-        return functionCallItemToMessage(item).textContent ?? '';
+        return functionCallItemToMessage(item).rawTextContent ?? '';
       })
       .join('\n')
       .trim();

agents/src/llm/provider_format/google.ts

@@ -31,8 +31,8 @@ export async function toChatCtx(
 
   for (const msg of flattenedItems) {
     // Handle system messages separately
-    if (msg.type === 'message' && msg.role === 'system' && msg.textContent) {
-      systemMessages.push(msg.textContent);
+    if (msg.type === 'message' && msg.role === 'system' && msg.rawTextContent) {
+      systemMessages.push(msg.rawTextContent);
       continue;
     }
 

agents/src/llm/provider_format/utils.ts

@@ -117,12 +117,12 @@ export function convertMidConversationInstructions(
       item.type === 'message' &&
       (item.role === 'system' || item.role === 'developer') &&
       firstSystemSeen &&
-      item.textContent
+      item.rawTextContent
     ) {
       items.push(
         ChatMessage.create({
           role,
-          content: template.replace('{instructions}', item.textContent),
+          content: template.replace('{instructions}', item.rawTextContent),
           id: item.id,
           createdAt: item.createdAt,
         }),

agents/src/llm/utils.ts

@@ -706,6 +706,18 @@ interface DiffOps {
   toCreate: Array<[string | null, string]>; // (previous_item_id, id), if previous_item_id is null, add to the root
 }
 
+function isUnchangedChatItem(oldItem: ChatItem | undefined, newItem: ChatItem): boolean {
+  if (!oldItem || oldItem.type !== newItem.type) {
+    return false;
+  }
+
+  if (oldItem.type === 'message' && newItem.type === 'message') {
+    return oldItem.rawTextContent === newItem.rawTextContent;
+  }
+
+  return true;
+}
+
 /**
  * Compute the minimal list of create/remove operations to transform oldCtx into newCtx.
  *
@@ -717,6 +729,13 @@ export function computeChatCtxDiff(oldCtx: ChatContext, newCtx: ChatContext): Di
   const oldIds = oldCtx.items.map((item: ChatItem) => item.id);
   const newIds = newCtx.items.map((item: ChatItem) => item.id);
   const lcsIds = new Set(computeLCS(oldIds, newIds));
+  const oldCtxById = new Map(oldCtx.items.map((item) => [item.id, item]));
+
+  for (const newItem of newCtx.items) {
+    if (lcsIds.has(newItem.id) && !isUnchangedChatItem(oldCtxById.get(newItem.id), newItem)) {
+      lcsIds.delete(newItem.id);
+    }
+  }
 
   const toRemove = oldCtx.items.filter((msg) => !lcsIds.has(msg.id)).map((msg) => msg.id);
   const toCreate: Array<[string | null, string]> = [];

agents/src/tts/provider_format.ts

@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/** Strip only LiveKit expressive `<expr/>` tags, leaving provider-native markup untouched. */
+export function stripExprMarkup(text: string): string {
+  let stripped = text;
+  let previous: string;
+
+  do {
+    previous = stripped;
+    stripped = stripped
+      .replace(/<expr\b[^>]*\/>/gi, '')
+      .replace(/<expr\b[^>]*>([\s\S]*?)<\/expr>/gi, '$1');
+  } while (stripped !== previous);
+
+  return stripped;
+}

agents/src/voice/agent_activity.ts

@@ -2117,7 +2117,7 @@ export class AgentActivity implements RecognitionHooks {
           this.realtimeReplyTask({
             speechHandle: handle,
             // TODO(brian): support llm.ChatMessage for the realtime model
-            userInput: userMessage?.textContent,
+            userInput: userMessage?.rawTextContent,
             instructions,
             modelSettings: {
               // isGiven(toolChoice) = toolChoice !== undefined
@@ -2391,7 +2391,7 @@ export class AgentActivity implements RecognitionHooks {
       // make sure the onUserTurnCompleted didn't change some request parameters
       // otherwise invalidate the preemptive generation
       if (
-        preemptive.info.newTranscript === userMessage?.textContent &&
+        preemptive.info.newTranscript === userMessage?.rawTextContent &&
         preemptive.chatCtx.isEquivalent(chatCtx) &&
         preemptive.tools.equals(this.tools) &&
         isSameToolChoice(preemptive.toolChoice, this.toolChoice)
@@ -2649,7 +2649,7 @@ export class AgentActivity implements RecognitionHooks {
       span.setAttribute(traceTypes.ATTR_INSTRUCTIONS, renderInstructions(instructions));
     }
     if (newMessage) {
-      span.setAttribute(traceTypes.ATTR_USER_INPUT, newMessage.textContent || '');
+      span.setAttribute(traceTypes.ATTR_USER_INPUT, newMessage.rawTextContent || '');
     }
 
     const localParticipant = this.agentSession._roomIO?.localParticipant;

plugins/anthropic/src/llm.ts

@@ -113,7 +113,7 @@ export class LLM extends llm.LLM {
 
     for (const msg of chatCtx.items) {
       if (msg.type === 'message') {
-        const textContent = msg.textContent || '';
+        const textContent = msg.rawTextContent || '';
         if (msg.role === 'system' || msg.role === 'developer') {
           system.push({ type: 'text', text: textContent });
         } else if (msg.role === 'user' || msg.role === 'assistant') {

plugins/phonic/src/realtime/realtime_model.ts

@@ -369,19 +369,19 @@ export class RealtimeSession extends llm.RealtimeSession {
         }
       }
       if (item?.type === 'message') {
-        if ((item.role === 'system' || item.role === 'developer') && item.textContent) {
-          this.#logger.debug(`Sending add system message: ${item.textContent}`);
+        if ((item.role === 'system' || item.role === 'developer') && item.rawTextContent) {
+          this.#logger.debug(`Sending add system message: ${item.rawTextContent}`);
           this.socket?.sendAddSystemMessage({
             type: 'add_system_message',
-            system_message: item.textContent,
+            system_message: item.rawTextContent,
           });
           sentAddSystemMessage = true;
         }
 
         // Only treat a user message as text input when it's appended at the tail of the context.
-        if (item.role === 'user' && itemId === lastItemId && item.textContent) {
-          this.#logger.debug(`Received user text input: ${item.textContent}`);
-          this.pendingUserText = item.textContent;
+        if (item.role === 'user' && itemId === lastItemId && item.rawTextContent) {
+          this.#logger.debug(`Received user text input: ${item.rawTextContent}`);
+          this.pendingUserText = item.rawTextContent;
           bufferedUserText = true;
         }
       }
@@ -1011,7 +1011,7 @@ export class RealtimeSession extends llm.RealtimeSession {
 
 function chatItemToText(item: llm.ChatItem): string | undefined {
   if (item.type === 'message') {
-    const text = item.textContent?.trim();
+    const text = item.rawTextContent?.trim();
     if (!text) return undefined;
     return `<${item.role}>${text}</${item.role}>`;
   }