From cf07bd17b6539945630b9757f31dfc20cd9582fa Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 23:31:57 +0000 Subject: [PATCH] feat(core): add plain text chat content --- .changeset/plain-chat-content.md | 6 ++ agents/src/llm/chat_context.ts | 81 ++++++++++++++++++- plugins/livekit/src/turn_detector/base.ts | 13 ++- .../livekit/src/turn_detector/multilingual.ts | 1 + 4 files changed, 92 insertions(+), 9 deletions(-) create mode 100644 .changeset/plain-chat-content.md diff --git a/.changeset/plain-chat-content.md b/.changeset/plain-chat-content.md new file mode 100644 index 000000000..c7ce71791 --- /dev/null +++ b/.changeset/plain-chat-content.md @@ -0,0 +1,6 @@ +--- +'@livekit/agents': minor +'@livekit/agents-plugin-livekit': minor +--- + +Add plain text content access for chat messages and use it for assistant EOU context. diff --git a/agents/src/llm/chat_context.ts b/agents/src/llm/chat_context.ts index 34cf39200..c088df67f 100644 --- a/agents/src/llm/chat_context.ts +++ b/agents/src/llm/chat_context.ts @@ -229,6 +229,63 @@ export function concatInstructions(...parts: Array): stri export type ChatContent = ImageContent | AudioContent | Instructions | string; +const EXPRESSIVE_MARKUP_TAGS = [ + 'angry', + 'break', + 'build-intensity', + 'calm', + 'confident', + 'curious', + 'decrease-intensity', + 'emotion', + 'emphasis', + 'excited', + 'fast', + 'happy', + 'higher-pitch', + 'laugh-speak', + 'loud', + 'lower-pitch', + 'nervous', + 'playful', + 'sad', + 'sarcastic', + 'singing', + 'sing-song', + 'slow', + 'soft', + 'sound', + 'spell', + 'speed', + 'surprised', + 'sympathetic', + 'volume', + 'whisper', +].join('|'); + +const EXPRESSIVE_WRAPPER_RE = new RegExp( + `<(${EXPRESSIVE_MARKUP_TAGS})\\b[^>]*\\s*>([\\s\\S]*?)<\\/\\1\\s*>`, + 'g', +); +const EXPRESSIVE_OPEN_RE = new RegExp(`<(?:${EXPRESSIVE_MARKUP_TAGS})\\b[^>]*\\s*\\/?\\s*>`, 'g'); +const EXPRESSIVE_CLOSE_RE = new RegExp(`<\\/(?:${EXPRESSIVE_MARKUP_TAGS})\\s*>`, 'g'); + +function stripAllMarkup(text: string): string { + let clean = text.replace(/]*>/g, '').replace(/<\/expr\s*>/g, ''); + let previous: string | undefined; + + while (clean !== previous) { + previous = clean; + clean = clean + .replace(EXPRESSIVE_WRAPPER_RE, '$2') + .replace(EXPRESSIVE_OPEN_RE, '') + .replace(EXPRESSIVE_CLOSE_RE, '') + .replace(/\[[^\]]+\]/g, ''); + } + + return clean; +} + export function createImageContent(params: { image: string | VideoFrame; id?: string; @@ -367,6 +424,16 @@ export class ChatMessage { return parts.length > 0 ? parts.join('\n') : undefined; } + /** + * Returns a single string with all text parts of the message joined by new + * lines, without expressive markup tags. If no string content is present, + * returns `undefined`. + */ + get plainTextContent(): string | undefined { + const raw = this.textContent; + return raw === undefined ? undefined : stripAllMarkup(raw); + } + toJSONContent(): JSONValue[] { return this.content.map((c) => { if (typeof c === 'string') { @@ -925,6 +992,7 @@ export class ChatContext { excludeTimestamp?: boolean; excludeFunctionCall?: boolean; excludeConfigUpdate?: boolean; + stripMarkup?: boolean; } = {}, // eslint-disable-next-line @typescript-eslint/no-explicit-any ): JSONObject { @@ -934,6 +1002,7 @@ export class ChatContext { excludeTimestamp = true, excludeFunctionCall = false, excludeConfigUpdate = false, + stripMarkup = false, } = options; const items: ChatItem[] = []; @@ -973,6 +1042,12 @@ export class ChatContext { return !(typeof c === 'object' && c.type === 'audio_content'); }); } + + if (stripMarkup && processedItem.role === 'assistant') { + processedItem.content = processedItem.content.map((c) => + typeof c === 'string' ? stripAllMarkup(c) : c, + ); + } } items.push(processedItem); @@ -1171,7 +1246,8 @@ export class ChatContext { if (item.role !== 'user' && item.role !== 'assistant') continue; if (item.extra?.is_summary === true) continue; - const text = (item.textContent ?? '').trim(); + const content = item.role === 'assistant' ? item.plainTextContent : item.textContent; + const text = (content ?? '').trim(); if (text) { toSummarize.push(item); } @@ -1187,7 +1263,8 @@ export class ChatContext { const sourceText = toSummarize .map((item) => { if (item.type === 'message') { - return toXml(item.role, (item.textContent ?? '').trim()); + const content = item.role === 'assistant' ? item.plainTextContent : item.textContent; + return toXml(item.role, (content ?? '').trim()); } return functionCallItemToMessage(item).textContent ?? ''; diff --git a/plugins/livekit/src/turn_detector/base.ts b/plugins/livekit/src/turn_detector/base.ts index 3fa1dd139..bab9a97af 100644 --- a/plugins/livekit/src/turn_detector/base.ts +++ b/plugins/livekit/src/turn_detector/base.ts @@ -244,13 +244,12 @@ export abstract class EOUModel { continue; } - for (const content of message.content) { - if (typeof content === 'string') { - messages.push({ - role: message.role === 'assistant' ? 'assistant' : 'user', - content: content, - }); - } + const content = message.role === 'assistant' ? message.plainTextContent : message.textContent; + if (content) { + messages.push({ + role: message.role === 'assistant' ? 'assistant' : 'user', + content, + }); } } diff --git a/plugins/livekit/src/turn_detector/multilingual.ts b/plugins/livekit/src/turn_detector/multilingual.ts index cd0423913..7471e3305 100644 --- a/plugins/livekit/src/turn_detector/multilingual.ts +++ b/plugins/livekit/src/turn_detector/multilingual.ts @@ -90,6 +90,7 @@ export class MultilingualModel extends EOUModel { excludeImage: true, excludeAudio: true, excludeTimestamp: true, + stripMarkup: true, }), jobId: ctx.job.id, workerId: ctx.workerId,