diff --git a/.changeset/expr-marker-dialect.md b/.changeset/expr-marker-dialect.md new file mode 100644 index 000000000..ac5b47084 --- /dev/null +++ b/.changeset/expr-marker-dialect.md @@ -0,0 +1,7 @@ +--- +"@livekit/agents": patch +"@livekit/agents-plugin-cartesia": patch +"@livekit/agents-plugin-inworld": patch +--- + +Add expressive TTS `` marker dialect support. diff --git a/agents/src/inference/tts.ts b/agents/src/inference/tts.ts index f4011f69b..ad9c11385 100644 --- a/agents/src/inference/tts.ts +++ b/agents/src/inference/tts.ts @@ -13,6 +13,7 @@ import { createStreamChannel } from '../stream/stream_channel.js'; import { basic as tokenizeBasic } from '../tokenize/index.js'; import type { ChunkedStream } from '../tts/index.js'; import { SynthesizeStream as BaseSynthesizeStream, TTS as BaseTTS } from '../tts/index.js'; +import { convertMarkup, normalizeMarkup } from '../tts/provider_format.js'; import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js'; import { Event, @@ -597,7 +598,7 @@ export class SynthesizeStream extends BaseSynthesizeSt sendTokenizerStream.flush(); continue; } - sendTokenizerStream.pushText(data); + sendTokenizerStream.pushText(normalizeMarkup(this.opts.model, data)); } // Only call endInput if the stream hasn't been closed by cleanup if (!closing) { @@ -617,11 +618,12 @@ export class SynthesizeStream extends BaseSynthesizeSt if (this.opts.model) generationConfig.model = this.opts.model; if (this.opts.language) generationConfig.language = this.opts.language; + const transcript = convertMarkup(this.opts.model, ev.token); this.markStarted(); await sendClientEvent( { type: 'input_transcript', - transcript: ev.token + ' ', + transcript: transcript + ' ', generation_config: generationConfig, extra: (this.opts.modelOptions as Record) ?? {}, }, diff --git a/agents/src/tts/index.ts b/agents/src/tts/index.ts index 8f879990a..46b8c3c74 100644 --- a/agents/src/tts/index.ts +++ b/agents/src/tts/index.ts @@ -11,3 +11,10 @@ export { } from './tts.js'; export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js'; export { FallbackAdapter, type AvailabilityChangedEvent } from './fallback_adapter.js'; +export { + convertMarkup, + llmInstructions, + normalizeMarkup, + providerKey, + stripAllMarkup, +} from './provider_format.js'; diff --git a/agents/src/tts/provider_format.ts b/agents/src/tts/provider_format.ts new file mode 100644 index 000000000..59fc23d95 --- /dev/null +++ b/agents/src/tts/provider_format.ts @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +const CARTESIA_TAGS = ['emotion', 'speed', 'volume', 'break', 'spell']; +const INWORLD_TAGS = ['expression', 'sound', 'break']; + +const XAI_INLINE = [ + 'breath', + 'inhale', + 'exhale', + 'sigh', + 'laugh', + 'chuckle', + 'giggle', + 'cry', + 'tsk', + 'tongue-click', + 'lip-smack', + 'hum-tune', +]; + +const XAI_WRAPPING = [ + 'emphasis', + 'whisper', + 'soft', + 'loud', + 'build-intensity', + 'decrease-intensity', + 'higher-pitch', + 'lower-pitch', + 'slow', + 'fast', + 'sing-song', + 'singing', + 'laugh-speak', +]; + +const XAI_TAGS = [ + 'happy', + 'sad', + 'angry', + 'excited', + 'calm', + 'surprised', + 'sympathetic', + 'curious', + 'sarcastic', + 'confident', + 'playful', + 'nervous', + ...XAI_WRAPPING, + 'sound', + 'break', +]; + +const EXPR_PREAMBLE = `Expand all numbers, symbols, and abbreviations into spoken form (e.g. $42.50 to forty-two dollars and fifty cents, Dr. to Doctor). + +You control speech delivery with a single XML marker tag: . Every marker has a type attribute. The types below are the ONLY ones this voice supports, and where a type lists a label vocabulary, use only those labels. Reach for the markers often and mix them so the voice never sounds flat, but keep each one motivated by the moment.`; + +const CARTESIA_EXPR_LLM_INSTRUCTIONS = `${EXPR_PREAMBLE} + +1. Emotion - sets the emotional tone. Self-closing; place before EVERY sentence. + + Labels are a fixed vocabulary: neutral, angry, excited, content, sad, scared, happy, enthusiastic, elated, triumphant, amazed, surprised, flirtatious, curious, peaceful, serene, calm, grateful, affectionate, sympathetic, mysterious, frustrated, disgusted, sarcastic, ironic, dejected, melancholic, disappointed, apologetic, hesitant, confused, anxious, panicked, proud, confident, contemplative, determined, joking/comedic. + +2. Pauses - insert silence when appropriate. Self-closing. + - label is a duration in seconds or milliseconds. + +3. Prosody - adjusts pacing and loudness from that point on. Self-closing. + slower faster + quieter louder + Labels are a fixed vocabulary: slow, fast, soft, loud. + +4. Spell - wraps text read character by character. + A7X9 + +This voice has no non-verbal sounds and no free-form delivery descriptions.`; + +const INWORLD_EXPR_LLM_INSTRUCTIONS = `${EXPR_PREAMBLE} + +1. Delivery - controls how a sentence sounds. Self-closing; place before EVERY sentence. + + The label is free-form: describe vocal quality, pitch, volume, pace, and intonation in plain English. + +2. Sounds - a non-verbal sound between sentences. Self-closing. + + Labels are a fixed vocabulary: laugh, sigh, breathe, clear throat, cough, yawn. + +3. Pauses - insert silence when appropriate. Self-closing. + or (max 10s). + +There is no wrapping prosody marker for this voice; put pace, pitch, and volume in the expression label instead.`; + +const XAI_EXPR_LLM_INSTRUCTIONS = `${EXPR_PREAMBLE} + +1. Sounds - a non-verbal vocalization at the exact point where it happens. Self-closing. + + Labels are a fixed vocabulary: ${XAI_INLINE.join(', ')}. + +2. Pauses - insert a beat. Self-closing. + a brief pause a longer, dramatic pause + +3. Prosody - wraps the exact words it affects to shape HOW they're said. + the words it affects + Labels are a fixed vocabulary: ${XAI_WRAPPING.join(', ')}. + Never nest one prosody marker inside another, and always close it with . + +This voice has no free-form delivery descriptions.`; + +const EXPR_LLM_INSTRUCTIONS: Record = { + cartesia: CARTESIA_EXPR_LLM_INSTRUCTIONS, + inworld: INWORLD_EXPR_LLM_INSTRUCTIONS, + xai: XAI_EXPR_LLM_INSTRUCTIONS, +}; + +const PROVIDER_MARKUP: Record = { + cartesia: { xmlTags: CARTESIA_TAGS, brackets: false }, + inworld: { xmlTags: INWORLD_TAGS, brackets: true }, + xai: { xmlTags: XAI_TAGS, brackets: false }, +}; +const ALL_MARKUP_TAGS = [ + ...new Set(Object.values(PROVIDER_MARKUP).flatMap((v) => v.xmlTags)), +].sort(); + +const XAI_SOUND_ALIASES: Record = { breathe: 'breath' }; +const CARTESIA_PROSODY: Record = { + slow: '', + fast: '', + soft: '', + loud: '', +}; + +const EXPR_ATTR_RE = /([\w-]+)\s*=\s*"([^"]*)"/g; +const EXPR_OPEN_RE = /]*?)\/?\s*>/g; +const EXPR_CLOSE_RE = /<\/expr\s*>/g; +const EXPR_SELF_RE = /]*?)\/\s*>/g; +const EXPR_WRAP_RE = /]*type="(?:prosody|spell)")([^>]*?)>(.*?)<\/expr\s*>/gs; +const EXPR_UNCLOSED_RE = /(]*type="(?:expression|break|sound)")[^>]*[^/>\s])\s*>/g; +const EXPRESSION_RE = /|>(?:.*?)<\/expression>)/gs; +const SOUND_RE = /|>(?:.*?)<\/sound>)/gs; +const XAI_BREAK_RE = //g; + +function exprAttrs(attrs: string): Record { + return Object.fromEntries([...attrs.matchAll(EXPR_ATTR_RE)].map((m) => [m[1]!, m[2]!])); +} + +function convertExpressionTags(text: string): string { + return text + .replace(EXPRESSION_RE, (_, value: string) => `[${value}]`) + .replace(SOUND_RE, (_, value: string) => `[${value}]`); +} + +function xaiBreakToBracket(rawValue: string): string { + const raw = rawValue.trim().toLowerCase(); + const seconds = raw.endsWith('ms') + ? Number(raw.slice(0, -2)) / 1000 + : Number(raw.replace(/s$/, '')); + return Number.isFinite(seconds) && seconds >= 1 ? '[long-pause]' : '[pause]'; +} + +function convertExpr(provider: string, text: string): string { + if (!text.includes(' { + const attrs = exprAttrs(rawAttrs); + const markerType = attrs.type ?? ''; + const label = (attrs.label ?? '').trim().toLowerCase(); + if (markerType === 'spell') return provider === 'cartesia' ? `${inner}` : inner; + if (provider === 'xai') { + const native = label.replaceAll(' ', '-'); + return XAI_WRAPPING.includes(native) ? `<${native}>${inner}` : inner; + } + if (provider === 'inworld') return `${inner}`; + if (provider === 'cartesia') return (CARTESIA_PROSODY[label] ?? '') + inner; + return inner; + }); + + text = text.replace(EXPR_SELF_RE, (_match: string, rawAttrs: string) => { + const attrs = exprAttrs(rawAttrs); + const markerType = attrs.type ?? ''; + let label = attrs.label ?? ''; + if (markerType === 'expression') { + if (provider === 'cartesia') return ``; + if (provider === 'inworld') return ``; + return ''; + } + if (markerType === 'sound') { + if (provider === 'cartesia') return ''; + if (provider === 'xai') label = XAI_SOUND_ALIASES[label.toLowerCase()] ?? label; + return ``; + } + if (markerType === 'break') return ``; + if (markerType === 'prosody' && provider === 'cartesia') { + return CARTESIA_PROSODY[label.trim().toLowerCase()] ?? ''; + } + return ''; + }); + + return text.replace(EXPR_OPEN_RE, '').replace(EXPR_CLOSE_RE, ''); +} + +export function providerKey(provider: string | undefined): string | undefined { + return provider?.toLowerCase().split('/')[0]; +} + +export function llmInstructions(provider: string | undefined): string | undefined { + const key = providerKey(provider); + return key ? EXPR_LLM_INSTRUCTIONS[key] : undefined; +} + +export function normalizeMarkup(provider: string | undefined, text: string): string { + const key = providerKey(provider); + if (!key || !(key in PROVIDER_MARKUP)) return text; + return text.replace(EXPR_UNCLOSED_RE, '$1/>'); +} + +export function convertMarkup(provider: string | undefined, text: string): string { + const key = providerKey(provider); + if (!key || !(key in PROVIDER_MARKUP)) return text; + text = convertExpr(key, text); + if (key === 'inworld' || key === 'xai') text = convertExpressionTags(text); + if (key === 'xai') + text = text.replace(XAI_BREAK_RE, (_match, value: string) => xaiBreakToBracket(value)); + return text; +} + +export function stripAllMarkup(text: string): string { + let clean = text.replace(EXPR_OPEN_RE, '').replace(EXPR_CLOSE_RE, ''); + const tagPattern = ALL_MARKUP_TAGS.map((tag) => tag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join( + '|', + ); + clean = clean.replace( + new RegExp(`<(${tagPattern})\\b([^>]*?)\\s*\\/?\\s*>(?:(.*?)<\\/\\1\\s*>)?`, 'gs'), + (_match, _tag, _attrs, inner: string | undefined) => inner ?? '', + ); + clean = clean.replace(new RegExp(`<\\/(?:${tagPattern})\\s*>`, 'g'), ''); + return clean.replace(/\[[^\]]+\]/g, ''); +} diff --git a/agents/src/voice/agent.ts b/agents/src/voice/agent.ts index 05536663b..a1fb68613 100644 --- a/agents/src/voice/agent.ts +++ b/agents/src/voice/agent.ts @@ -34,7 +34,7 @@ import { type FlushSentinel, USERDATA_TIMED_TRANSCRIPT } from '../types.js'; import { Future, Task, toStream } from '../utils.js'; import type { VAD } from '../vad.js'; import { type AgentActivity, agentActivityStorage } from './agent_activity.js'; -import type { AgentSession, TurnDetectionMode } from './agent_session.js'; +import type { AgentSession, ExpressiveOptions, TurnDetectionMode } from './agent_session.js'; import { type AgentCreateOptions, type AgentTaskCreateOptions, @@ -141,6 +141,7 @@ export interface AgentOptions { vad?: VAD; llm?: LLM | RealtimeModel | LLMModels; tts?: TTS | TTSModelString; + expressive?: boolean | ExpressiveOptions; turnHandling?: TurnHandlingOptions; toolHandling?: ToolHandlingOptions; minConsecutiveSpeechDelay?: number; @@ -158,6 +159,7 @@ export class Agent { private _llm?: LLM | RealtimeModel; private _tts?: TTS; private _turnHandling?: Partial; + private _expressive?: boolean | ExpressiveOptions; private _minConsecutiveSpeechDelay?: number; private _useTtsAlignedTranscript?: boolean; @@ -191,6 +193,7 @@ export class Agent { vad, llm, tts, + expressive, allowInterruptions, turnHandling, toolHandling, @@ -250,6 +253,7 @@ export class Agent { this._minConsecutiveSpeechDelay = minConsecutiveSpeechDelay; this._useTtsAlignedTranscript = useTtsAlignedTranscript; + this._expressive = expressive; this._agentActivity = undefined; } @@ -274,6 +278,10 @@ export class Agent { return this._useTtsAlignedTranscript; } + get expressive(): boolean | ExpressiveOptions | undefined { + return this._expressive; + } + get chatCtx(): ReadonlyChatContext { return new ReadonlyChatContext(this._chatCtx.items); } diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 9fd906783..ae28aa675 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -68,6 +68,10 @@ import { MultiInputStream } from '../stream/multi_input_stream.js'; import { STT, type STTError, type SpeechEvent } from '../stt/stt.js'; import { recordRealtimeMetrics, traceTypes, tracer } from '../telemetry/index.js'; import { splitWords } from '../tokenize/basic/word.js'; +import { + stripAllMarkup, + llmInstructions as ttsMarkupInstructions, +} from '../tts/provider_format.js'; import { TTS, type TTSError } from '../tts/tts.js'; import { isFlushSentinel } from '../types.js'; import { @@ -91,7 +95,12 @@ import { _setActivityTaskInfo, speechHandleStorage, } from './agent.js'; -import { type AgentSession, type TurnDetectionMode } from './agent_session.js'; +import { + type AgentSession, + DEFAULT_EXPRESSIVE_OPTIONS, + type ExpressiveOptions, + type TurnDetectionMode, +} from './agent_session.js'; import { AudioRecognition, type EndOfTurnInfo, @@ -796,6 +805,44 @@ export class AgentActivity implements RecognitionHooks { return this.agent.tts || this.agentSession.tts; } + private getTtsMarkupProvider(): string | undefined { + const tts = this.tts; + if (!tts) return undefined; + if (tts.provider.toLowerCase() === 'livekit') return tts.model.split('/')[0]; + return tts.provider; + } + + private expressiveInstructions(modality: 'audio' | 'text'): string | undefined { + const providerInstructions = ttsMarkupInstructions(this.getTtsMarkupProvider()); + if (!providerInstructions) return undefined; + + const expressive = this.agent.expressive ?? this.agentSession.sessionOptions.expressive; + if (!expressive) return undefined; + + const options: Required = { + ...DEFAULT_EXPRESSIVE_OPTIONS, + ...(typeof expressive === 'object' ? expressive : {}), + }; + const template = renderInstructions(options.ttsInstructionsTemplate, modality); + const rendered = template.replace('{tts.markup.llm_instructions}', providerInstructions); + return concatInstructions(rendered, options.ttsInstructionsAppend).toString().trim(); + } + + private injectExpressiveInstructions(chatCtx: ChatContext, modality: 'audio' | 'text'): void { + const instructions = this.expressiveInstructions(modality); + if (!instructions) return; + chatCtx.items.push( + ChatMessage.create({ + role: 'system', + content: [instructions], + }), + ); + } + + private stripExpressiveMarkup(text: string, modality: 'audio' | 'text'): string { + return this.expressiveInstructions(modality) ? stripAllMarkup(text) : text; + } + get tools(): ToolContext { const tools: ToolContextEntry[] = [ ...this.agentSession.toolCtx.tools, @@ -2602,7 +2649,10 @@ export class AgentActivity implements RecognitionHooks { const message = ChatMessage.create({ role: 'assistant', - content: textOut?.text || '', + content: this.stripExpressiveMarkup( + textOut?.text || '', + speechHandle.inputDetails.modality, + ), interrupted: speechHandle.interrupted, metrics: replyAssistantMetrics, }); @@ -2687,6 +2737,7 @@ export class AgentActivity implements RecognitionHooks { // apply the correct variant of the instructions for the turn's input modality applyInstructionsModality(chatCtx, { modality: speechHandle.inputDetails.modality }); + this.injectExpressiveInstructions(chatCtx, speechHandle.inputDetails.modality); const tasks: Array> = []; const [llmTask, llmGenData] = performLLMInference( @@ -3048,7 +3099,10 @@ export class AgentActivity implements RecognitionHooks { replyAbortController.abort(); await cancelAndWait(tasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT); - const forwardedText = segmentOutputs.map(forwardedTextFor).join(''); + const forwardedText = this.stripExpressiveMarkup( + segmentOutputs.map(forwardedTextFor).join(''), + speechHandle.inputDetails.modality, + ); if (forwardedText) { hasSpeechMessage = true; @@ -3091,7 +3145,10 @@ export class AgentActivity implements RecognitionHooks { return; } - const forwardedText = segmentOutputs.map(forwardedTextFor).join(''); + const forwardedText = this.stripExpressiveMarkup( + segmentOutputs.map(forwardedTextFor).join(''), + speechHandle.inputDetails.modality, + ); if (forwardedText) { hasSpeechMessage = true; const message = ChatMessage.create({ diff --git a/agents/src/voice/agent_session.ts b/agents/src/voice/agent_session.ts index c48df175b..e341f175d 100644 --- a/agents/src/voice/agent_session.ts +++ b/agents/src/voice/agent_session.ts @@ -123,6 +123,21 @@ export interface AgentSessionUsage { modelUsage: Array>; } +export interface ExpressiveOptions { + /** Template used to inject provider-specific TTS marker instructions. */ + ttsInstructionsTemplate?: string | Instructions; + /** Extra expressive guidance appended after the rendered template. */ + ttsInstructionsAppend?: string; +} + +export const DEFAULT_EXPRESSIVE_OPTIONS: Required = { + ttsInstructionsTemplate: + 'You can control how you speak using the following formatting tags. ' + + 'Use them when appropriate to make your speech more expressive and natural:\n\n' + + '{tts.markup.llm_instructions}', + ttsInstructionsAppend: '', +}; + /** * Granular control over which recording features are active. * @@ -187,6 +202,7 @@ export interface InternalSessionOptions extends AgentSessionOptions = { * and `filter_emoji`; pass `null` to disable text transforms. */ ttsTextTransforms?: readonly TextTransform[] | null; + + /** Enable expressive TTS marker prompting for supported inference TTS providers. */ + expressive?: boolean | ExpressiveOptions; }; export type AgentSessionUpdateOptions = { diff --git a/agents/src/voice/index.ts b/agents/src/voice/index.ts index 02872fc8a..d2d5d27c8 100644 --- a/agents/src/voice/index.ts +++ b/agents/src/voice/index.ts @@ -17,8 +17,10 @@ export { export * from './amd.js'; export { AgentSession, + DEFAULT_EXPRESSIVE_OPTIONS, type AgentSessionOptions, type AgentSessionUsage, + type ExpressiveOptions, type VoiceOptions, } from './agent_session.js'; export * from './avatar/index.js'; diff --git a/agents/src/voice/turn_config/utils.ts b/agents/src/voice/turn_config/utils.ts index 100cda367..afceca7e8 100644 --- a/agents/src/voice/turn_config/utils.ts +++ b/agents/src/voice/turn_config/utils.ts @@ -28,6 +28,7 @@ const defaultSessionOptions = { turnHandling: {}, useTtsAlignedTranscript: true, ttsTextTransforms: ['filter_markdown', 'filter_emoji'], + expressive: false, } as const satisfies AgentSessionOptions; const defaultLegacyVoiceOptions: VoiceOptions = { diff --git a/plugins/cartesia/src/tts.ts b/plugins/cartesia/src/tts.ts index cc622f36d..aeefcf107 100644 --- a/plugins/cartesia/src/tts.ts +++ b/plugins/cartesia/src/tts.ts @@ -219,7 +219,7 @@ export class ChunkedStream extends tts.ChunkedStream { const requestId = shortuuid(); const bstream = new AudioByteStream(this.#opts.sampleRate, NUM_CHANNELS); const json = toCartesiaOptions(this.#opts); - json.transcript = this.#text; + json.transcript = tts.convertMarkup('cartesia', this.#text); const baseUrl = new URL(this.#opts.baseUrl); const doneFut = new Future(); @@ -326,7 +326,7 @@ export class SynthesizeStream extends tts.SynthesizeStream { const msg = { ...packet, context_id: requestId, - transcript: event.token + ' ', + transcript: tts.convertMarkup('cartesia', event.token) + ' ', continue: true, }; this.markStarted(); @@ -350,7 +350,7 @@ export class SynthesizeStream extends tts.SynthesizeStream { this.#tokenizer.flush(); continue; } - this.#tokenizer.pushText(data); + this.#tokenizer.pushText(tts.normalizeMarkup('cartesia', data)); } this.#tokenizer.endInput(); this.#tokenizer.close(); diff --git a/plugins/inworld/src/tts.ts b/plugins/inworld/src/tts.ts index bbf23e0f5..a74d91ed4 100644 --- a/plugins/inworld/src/tts.ts +++ b/plugins/inworld/src/tts.ts @@ -423,7 +423,7 @@ class ChunkedStream extends tts.ChunkedStream { }; const bodyParams: SynthesizeRequest = { - text: this.inputText, + text: tts.convertMarkup('inworld', this.inputText), voiceId: this.#opts.voice, modelId: this.#opts.model, audioConfig: audioConfig, @@ -726,7 +726,7 @@ class SynthesizeStream extends tts.SynthesizeStream { const sendLoop = async () => { for await (const ev of tokenizerStream) { - await this.#sendText(ws, ev.token); + await this.#sendText(ws, tts.convertMarkup('inworld', ev.token)); } }; const sendPromise = sendLoop(); @@ -738,7 +738,7 @@ class SynthesizeStream extends tts.SynthesizeStream { if (text === tts.SynthesizeStream.FLUSH_SENTINEL) { tokenizerStream.flush(); } else { - tokenizerStream.pushText(text); + tokenizerStream.pushText(tts.normalizeMarkup('inworld', text)); } } tokenizerStream.endInput();