From 6f1c0de71b64bec4945b00b326d347af655f3f39 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 8 Jul 2026 08:25:16 +0000 Subject: [PATCH 1/3] feat(tts): unified expression marker dialect + transcript pacing fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port of livekit/agents#6347 and livekit/agents#6350 on top of the expressive-mode port (#1982). expr marker dialect (#6347): replace the provider-native markup dialects with a single marker tag the LLM is taught everywhere — . Shared syntax, per-provider instruction blocks advertising only the types and label vocabularies each provider supports. convertMarkup lowers expr to native syntax before synthesis (dropping unsupported types while keeping the words); normalizeMarkup closes unclosed self-closing markers; the transcript strippers remove expr in a dedicated pre-pass so type/label surfaces as an ExpressiveTag. The six preset bodies are rewritten in expr; native tag tables remain solely as tolerance for hallucinated native tags. pacing fix (#6350): a markup tag with spaces in its attributes is shredded across the synchronizer's word tokens, so the per-token splitAllMarkup never saw a complete tag and paced every fragment as if spoken — delaying the transcript seconds per expressive sentence. Give each segment a stateful TranscriptMarkupStripper (fed the raw pushedText slices so attribute whitespace survives) so tag fragments contribute zero hyphens to pacing while raw tokens are still forwarded unchanged downstream. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_015im83A991642dZS2Ri9iv1 --- .changeset/expr-marker-dialect.md | 5 + agents/src/tts/_provider_format.ts | 605 +++++++++++------- agents/src/tts/expr_markup.test.ts | 270 ++++++++ agents/src/tts/markup_utils.test.ts | 10 +- .../src/voice/transcription/synchronizer.ts | 23 +- .../transcription/synchronizer_markup.test.ts | 69 ++ 6 files changed, 743 insertions(+), 239 deletions(-) create mode 100644 .changeset/expr-marker-dialect.md create mode 100644 agents/src/tts/expr_markup.test.ts create mode 100644 agents/src/voice/transcription/synchronizer_markup.test.ts diff --git a/.changeset/expr-marker-dialect.md b/.changeset/expr-marker-dialect.md new file mode 100644 index 000000000..623f39795 --- /dev/null +++ b/.changeset/expr-marker-dialect.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +feat: unified `` expression marker dialect for expressive TTS, and fix markup tags impacting transcript pacing diff --git a/agents/src/tts/_provider_format.ts b/agents/src/tts/_provider_format.ts index 66058816c..f832ea129 100644 --- a/agents/src/tts/_provider_format.ts +++ b/agents/src/tts/_provider_format.ts @@ -38,92 +38,18 @@ export interface ExpressiveTag { const CARTESIA_TAGS = ['emotion', 'speed', 'volume', 'break', 'spell']; -const CARTESIA_LLM_INSTRUCTIONS = `You have four self-closing XML tags. All end with />. - -1. Emotion - sets the emotional tone. Place before EVERY sentence. - - Best results: neutral, angry, excited, content, sad, scared. - Also available: happy, enthusiastic, elated, triumphant, amazed, surprised, flirtatious, curious, peaceful, serene, calm, grateful, affectionate, sympathetic, mysterious, frustrated, disgusted, sarcastic, ironic, dejected, melancholic, disappointed, apologetic, hesitant, confused, anxious, panicked, proud, confident, contemplative, determined, joking/comedic. - -2. Speed and volume - adjust pacing and loudness. - - speaking rate (0.6 to 1.5, default 1.0). - - loudness (0.5 to 2.0, default 1.0). - -3. Pauses - you can insert silence when appropriate. - - pause in seconds or milliseconds. - -4. Spell - reads text character by character (for codes, IDs, or a spelled-out name). - TEXT - Keep punctuation out of — a period inside is read as "dot"; add spaces inside for grouped pauses (ABC 123). - Reading identifiers: for an email, spell the local part (before the @) with , then say "@" as "at" and "." as "dot" outside the tag. Say a common domain as a whole word (jdoe at gmail dot com), but spell an uncommon domain out too (jdoe at acme dot com). Cartesia reads phone numbers cleanly from a plain digit string, so write them normally (555-123-4567) and let normalization pace them rather than using . - -Examples: - I can't wait to tell you! This is going to be great! - I'm sorry about that. Let's figure this out together. - I can't believe this happened. We're going to fix it. - Really? Tell me more! - Sure, that's jdoe at gmail dot com, and your callback number is 555-123-4567? - Your code is A7X9. Got it?`; - const INWORLD_TAGS = ['expression', 'sound', 'break']; -const INWORLD_LLM_INSTRUCTIONS = `Write natural spoken sentences. No markdown, emojis, or special characters. Use contractions. Expand all numbers, symbols, and abbreviations into spoken form (e.g. $42.50 to forty-two dollars and fifty cents, Dr. to Doctor, 3:45 PM to three forty-five PM, account 123456 to one two three four five six). - -Read out identifiers so the listener can catch every character — spell them out by separating the characters with spaces so the TTS voices each one. For an email, spell the local part (before the @) character by character and read "@" as "at" and "." as "dot" (e.g. j.doe@... becomes "j dot d o e at ..."). Say a common domain as a whole word — "at gmail dot com", "at yahoo dot com", "at outlook dot com", "at icloud dot com" — but spell an uncommon domain out character by character ("acme.com" becomes "at a c m e dot com"). Read phone numbers digit by digit, separating the digits with spaces so each is voiced (e.g. (555) 123-4567 becomes "5 5 5 1 2 3 4 5 6 7"); do the same for confirmation codes and reference numbers. - -Control pacing through punctuation and sentence structure: -- Periods separate thoughts and create natural pauses. -- Commas create shorter breaks within sentences. -- Ellipsis (...) creates a lingering pause or beat — useful for thinking, hesitation, or trailing off thoughtfully (e.g. "let me check..."). Use sparingly, and don't stack them back to back. -- Short sentences land with emphasis and urgency. -- Longer sentences give a calm, measured delivery. - -You have three XML tags. All are self-closing (end with />). - -1. Delivery - controls how a sentence sounds. Place before EVERY sentence. - - Describe vocal quality, pitch, volume, pace, and intonation in plain English: - - Quality/emotion: say excitedly, sound concerned, with warm surprise, with quiet intensity - - Pace: very fast, slow and measured, with deliberate pauses - - Pitch: say in a low tone, high and bright, pitch lifts on the key word - - Volume: loud and projecting, soft and intimate, near-silent, drop to a whisper, full-voiced - - Intonation: rising tone at the end (for questions), falling close (for statements), flat monotone, melodic and lilting - -2. Sounds - produces a non-verbal sound. Use between sentences when natural. - , , , , , - -3. Pauses - you can insert silence when appropriate. - or (max 10s) - A period or an ellipsis (...) already creates a pause, so don't put either right next to a — pick one or the other, not both. In particular, never write "..." or "...": the ellipsis and the break are redundant pauses stacked back to back. - -Combine tags freely within a single turn — pair an with a and a when it makes the delivery feel natural. Don't limit yourself to one tag per sentence. - -Use CAPITALIZATION for emphasis on key words. - -Examples: - Oh wait, REALLY? No way, that's awesome! - Ah man, yeah that's on us. Lemme see what I can do. - Okay okay, why did the burger go to the gym? Because it wanted better buns! - Yeah, it's been one of those days, you know? But hey, I'm here for you. - Hmm, okay so I think the best option is the combo. - Alright so here's the deal. - Anyway, now where were we? Oh right! - Don't tell anyone, but I think we got the BETTER deal. - La la la, here we go, welcome to the show!`; - -// xAI Grok TTS speech tags. The prosody/style wrapping tags and inline sounds/pauses are -// from the xAI docs (https://docs.x.ai/developers/rest-api-reference/inference/voice). The -// emotion wrapping tags (..) aren't instructed anymore — the LLM shapes -// delivery through prosody/style and inline sounds instead — but they stay in XAI_TAGS so a -// stray emotion tag is still stripped from the transcript rather than leaking. +// xAI Grok TTS speech tags, from the xAI docs +// (https://docs.x.ai/developers/rest-api-reference/inference/voice). // -// The LLM is instructed to write EVERY tag as XML (angle brackets) so the transcript -// stripper's "<" guard handles them cleanly and they never leak (see the earlier -// bracket-leak bug). Modeled on Inworld: inline sounds use and pauses -// use . convertMarkup rewrites those to xAI's native brackets for the -// TTS — -> [X] (reusing Inworld's conversion) and -> [pause] or -// [long-pause] by duration. Prosody stays angle-bracketed (native). All tags are stripped -// from the transcript via XAI_TAGS. +// The LLM is instructed in the expr dialect (below); these native tag names serve two +// purposes: XAI_WRAPPING is the label vocabulary expr prosody markers lower to, and all +// of them stay in XAI_TAGS so a hallucinated native tag is still stripped from the +// transcript rather than leaking. The intermediate and +// tags that expr lowering produces are rewritten to xAI's native +// brackets by convertMarkup — -> [X] and -> [pause] or +// [long-pause] by duration. Prosody is angle-bracketed (native). const XAI_EMOTIONS = [ 'happy', 'sad', @@ -190,44 +116,121 @@ function xaiBreakToBracket(_match: string, raw: string): string { return secs >= 1.0 ? '[long-pause]' : '[pause]'; } -const XAI_LLM_INSTRUCTIONS = - `Expand all numbers, symbols, and abbreviations into spoken form (e.g. $42.50 to forty-two dollars and fifty cents, Dr. to Doctor). +// --- LiveKit expression markers (expr) --- +// The LLM emits a single marker tag, +// , instead of provider-native tags. The *syntax* is shared, +// but each provider gets its own instruction block advertising only the types and label +// vocabularies it actually supports — providers offer different sound effects, some take +// only a discrete emotion vocabulary rather than free-form delivery descriptions, and +// only some have wrapping prosody. Types (per provider): +// expression (self-closing) - delivery/emotion for what follows; free-form for +// Inworld, Cartesia's discrete emotion vocabulary, absent +// for xAI +// break (self-closing) - pause, label is a duration ("500ms", "1s"); all providers +// sound (self-closing) - non-verbal vocalization from the provider's own list +// (Inworld: laugh/sigh/..., xAI: chuckle/tsk/...); absent +// for Cartesia +// prosody (wrapping) - words, labels +// from xAI's wrapping-tag list; for Cartesia a self-closing +// point control (slow/fast/soft/loud -> coarse speed/volume +// ratios); absent for Inworld (folded into expression) +// spell (wrapping) - A7X9 character-by-character +// readout; Cartesia only +// convertMarkup lowers expr to each provider's native syntax before synthesis (via the +// existing framework-standard tags, so the per-provider conversions below still apply), +// and the transcript strippers remove expr markers in a dedicated pre-pass so the +// type/label pair surfaces correctly as an ExpressiveTag. This is the only dialect the +// LLM is taught — both llmInstructions() and the expressive preset bodies use it; the +// provider-native tag tables remain solely so hallucinated native markup is still +// stripped/converted instead of leaking. + +const EXPR_PREAMBLE = `Expand all numbers, symbols, and abbreviations into spoken form (e.g. $42.50 to forty-two dollars and fifty cents, Dr. to Doctor). + +You control speech delivery with a single XML marker tag: . Every marker has a type attribute. The types below are the ONLY ones this voice supports, and where a type lists a label vocabulary, use only those labels. Reach for the markers often and mix them so the voice never sounds flat — but keep each one motivated by the moment, never decorative.`; + +const CARTESIA_EXPR_LLM_INSTRUCTIONS = `${EXPR_PREAMBLE} + +1. Emotion - sets the emotional tone. Self-closing; place before EVERY sentence. + + Labels are a fixed vocabulary, NOT free-form descriptions. Best results: neutral, angry, excited, content, sad, scared. + Also available: happy, enthusiastic, elated, triumphant, amazed, surprised, flirtatious, curious, peaceful, serene, calm, grateful, affectionate, sympathetic, mysterious, frustrated, disgusted, sarcastic, ironic, dejected, melancholic, disappointed, apologetic, hesitant, confused, anxious, panicked, proud, confident, contemplative, determined, joking/comedic. + +2. Pauses - insert silence when appropriate. Self-closing. + - label is a duration in seconds or milliseconds. -Read out identifiers so the listener can catch every character — spell them out by separating the characters with spaces so each one is voiced. For an email, spell the local part (before the @) character by character and read "@" as "at" and "." as "dot" (e.g. j.doe@... becomes "j dot d o e at ..."). Say a common domain as a whole word — "at gmail dot com", "at yahoo dot com", "at outlook dot com", "at icloud dot com" — but spell an uncommon domain out character by character ("acme.com" becomes "at a c m e dot com"). Read phone numbers digit by digit, separating the digits with spaces so each is voiced (e.g. (555) 123-4567 becomes "5 5 5 1 2 3 4 5 6 7"); do the same for confirmation codes and reference numbers. +3. Prosody - adjusts pacing and loudness from that point on. Self-closing. + slower faster + quieter louder + Labels are a fixed vocabulary: slow, fast, soft, loud. -You have several kinds of speech tags for lifelike, expressive delivery. Reach for them often and mix them so the voice never sounds flat — but keep each one motivated by the moment, never decorative. +4. Spell - wraps text read character by character (codes, IDs, or a spelled-out name). + A7X9 + Keep punctuation out of a spell marker — a period inside is read as "dot"; add spaces inside for grouped pauses (ABC 123). -1. Inline sounds - self-closing; drop one at the exact point where the sound happens. One tag, the value is the sound name: - ` + - XAI_INLINE.map((s) => ``).join(', ') + - ` +This voice has no non-verbal sounds and no free-form delivery descriptions — do not invent other types or labels. -2. Pauses - insert a beat where you want one: - a brief pause a longer, dramatic pause +Examples: + I can't wait to tell you! This is going to be great! + Really? Tell me more! + Your code is A7X9. Got it?`; + +const INWORLD_EXPR_LLM_INSTRUCTIONS = `${EXPR_PREAMBLE} + +1. Delivery - controls how a sentence sounds. Self-closing; place before EVERY sentence. + + The label is free-form: describe vocal quality, pitch, volume, pace, and intonation in plain English — "say playfully", "speak with warm surprise", "sound concerned", "drop to a whisper", "speak slowly and clearly, patient and reassuring". + +2. Sounds - a non-verbal sound between sentences. Self-closing. + + Labels are a fixed vocabulary: laugh, sigh, breathe, clear throat, cough, yawn. + +3. Pauses - insert silence when appropriate. Self-closing. + or (max 10s). + A period or an ellipsis (...) already creates a pause, so don't put a break marker right next to one — pick one or the other. + +There is no wrapping prosody marker for this voice — put pace, pitch, and volume in the expression label instead. + +Examples: + Okay okay, why did the burger go to the gym? Because it wanted better buns! + Ah man, yeah that's on us. Lemme see what I can do. + I know it's been a rough week.`; + +const XAI_EXPR_LLM_INSTRUCTIONS = `${EXPR_PREAMBLE} + +1. Sounds - a non-verbal vocalization at the exact point where it happens. Self-closing. + + Labels are a fixed vocabulary: ${XAI_INLINE.join(', ')}. -3. Prosody & style tags - wrap the exact words they affect to shape HOW it's said: - Volume: ... quieter ... louder - Intensity: ... ramp up ... ease off - Pitch: ... ... - Speed: ... ... - Style: ... stress ... intimate ... playful lilt ... actually sung ... talk through a laugh +2. Pauses - insert a beat. Self-closing. + a brief pause a longer, dramatic pause -Get creative by NESTING prosody & style tags to shape the delivery of the same words — e.g. no way, that's amazing!, or I really wish I could.. Vary the prosody every turn so no two sound alike, and drop in an inline sound where real feeling spills out. +3. Prosody - wraps the exact words it affects to shape HOW they're said. + the words it affects + Labels are a fixed vocabulary: ${XAI_WRAPPING.join(', ')}. + Never nest one prosody marker inside another, and always close it with . -To stress a word, wrap it in ... — do NOT write it in all-caps, which is read out as individual letters (so "HI" becomes "H. I."). Keep normal capitalization. Punctuation still shapes delivery — commas and periods create natural pauses, so reach for a only when you want a beat beyond what the punctuation gives. +This voice has no free-form delivery descriptions — shape delivery entirely through prosody markers, sounds, pauses, punctuation, and word choice. + +To stress a word, wrap it in ... — do NOT write it in all-caps, which is read out as individual letters. Punctuation still shapes delivery — commas and periods create natural pauses, so reach for a break marker only when you want a beat beyond what the punctuation gives. Examples: - So I walked in and there it was! I honestly could not believe it! It was a secret the whole time. - This is going to be so good — I can't wait! - Hey. I know it's been a rough week. I'm right here. - You did not just say that okay, tell me everything.`; + So I walked in and there it was! It was a secret the whole time. + This is going to be so good — I can't wait! + Hey. I know it's been a rough week. I'm right here. + You did not just say that okay, tell me everything.`; + +const EXPR_LLM_INSTRUCTIONS: Record = { + cartesia: CARTESIA_EXPR_LLM_INSTRUCTIONS, + inworld: INWORLD_EXPR_LLM_INSTRUCTIONS, + xai: XAI_EXPR_LLM_INSTRUCTIONS, +}; // --- Inworld-specific expressive preset bodies --- -// These bundle Inworld tag instructions + domain-specific delivery guidelines, keyed -// by (provider, preset) in the registry in `voice/presets.ts`. The public, provider- -// agnostic markers (`presets.CUSTOMER_SERVICE`, ...) resolve to one of these based on -// the active TTS. They do NOT use the {tts.markup.llm_instructions} placeholder — the -// Inworld tag reference is inlined directly, so the prompt is self-contained. +// These bundle the Inworld expr instruction block + domain-specific delivery guidelines, +// keyed by (provider, preset) in the registry in `voice/presets.ts`. The public, +// provider-agnostic markers (`presets.CUSTOMER_SERVICE`, ...) resolve to one of these +// based on the active TTS. They do NOT use the {tts.markup.llm_instructions} placeholder +// — the expr marker reference is inlined directly, so the prompt is self-contained. /** @internal */ export const INWORLD_CUSTOMER_SERVICE: ExpressiveOptions = { @@ -237,20 +240,20 @@ export const INWORLD_CUSTOMER_SERVICE: ExpressiveOptions = { "Make the person feel heard and looked after, whatever they've come with — a quick " + 'question, a billing problem, or something sensitive and stressful. Let real care come ' + 'through in the voice. Use the formatting tags below to shape your delivery:\n\n' + - INWORLD_LLM_INSTRUCTIONS + + INWORLD_EXPR_LLM_INSTRUCTIONS + '\n\nGuidelines:\n' + '- Open with warm, welcoming reassurance, then mirror the customer as the conversation ' + "develops — slow and soften when they're frustrated, worried, or confused, lift to bright, " + "genuine warmth when they're relaxed or pleased, but always stay caring and unhurried. " + 'De-escalate; never match anger with anger. Map the moment to a fresh expression — ' + - 'frustrated: ; confused: ; anxious ' + - 'or worried: ; ' + - 'distressed or upset: ; ' + - 'rushed: ; pleased or ' + - 'relieved: ; apologizing for a ' + - 'problem: . Vary pitch and volume ' + + 'frustrated: ; confused: ; anxious ' + + 'or worried: ; ' + + 'distressed or upset: ; ' + + 'rushed: ; pleased or ' + + 'relieved: ; apologizing for a ' + + 'problem: . Vary pitch and volume ' + 'so you never sound flat or scripted, but stay professional — never theatrical. Rotate ' + "expressions; don't reuse the same one two turns in a row.\n" + '- Take requests in stride: when someone asks for something, lead with calm, willing ' + @@ -260,26 +263,26 @@ export const INWORLD_CUSTOMER_SERVICE: ExpressiveOptions = { 'so settle straight into helping instead of opening on them.\n' + '- Soften for anything sensitive: when sharing bad news, a problem, a charge, or anything ' + 'that might worry the customer, gentle the delivery and lower the volume a touch ' + - '(), and give a brief ' + - ' after hard information so it can land.\n' + + '(), and give a brief ' + + ' after hard information so it can land.\n' + '- Enunciate what matters: for dates, times, amounts, confirmation numbers, doses, steps, ' + - 'and policies, slow down and over-enunciate () so the customer can catch and note them, and read digits and codes a touch ' + + 'and policies, slow down and over-enunciate () so the customer can catch and note them, and read digits and codes a touch ' + 'slower than prose.\n' + "- Acknowledge lookups so silence doesn't read as a dropped call: when checking something " + 'or pulling up an account, a quick "let me take a look" or "one sec" with a quiet ' + - ' — thinking aloud, not the main reply.\n' + + ' — thinking aloud, not the main reply.\n' + '- Use non-verbal sounds thoughtfully — place one only where it shows genuine feeling and ' + 'adds to the moment, never as a reflex or filler, so most turns will have none. You have the ' + 'full set, and any of them can fit the right moment: ' + - ' before weighty information or settling into an explanation, ' + - ' as a soft, sympathetic breath when commiserating with a real problem ' + + ' before weighty information or settling into an explanation, ' + + ' as a soft, sympathetic breath when commiserating with a real problem ' + '(never exasperated or impatient — that reads as annoyed), ' + - ' when moving to a next step or new topic, ' + - ' as a small, natural catch before a careful correction or ' + + ' when moving to a next step or new topic, ' + + ' as a small, natural catch before a careful correction or ' + 'clarification, ' + - ' as a warm chuckle when the customer is clearly joking, and ' + - ' only in the rare moment it genuinely fits — kept gentle and ' + + ' as a warm chuckle when the customer is clearly joking, and ' + + ' only in the rare moment it genuinely fits — kept gentle and ' + 'professional. Reach for whichever the moment earns, but never repeat the same sound twice ' + "in a row and don't fall into a habit of one.\n" + "- Sound human and caring, not corporate: use contractions (it's, you're, I'll, we've) and " + @@ -287,7 +290,7 @@ export const INWORLD_CUSTOMER_SERVICE: ExpressiveOptions = { 'understandable"), but keep fillers (um, uh) rare — a support agent should sound composed, ' + 'not hesitant.\n' + '- Pace for clarity with punctuation and expressions — commas and short sentences for ' + - 'important info, the occasional between steps. Exclamation points for ' + + 'important info, the occasional between steps. Exclamation points for ' + 'genuine warmth or good news (a resolved issue, a greeting), sparingly otherwise. ' + "CAPITALIZATION at most once per turn to stress a critical detail (e.g. that's at FOUR PM, " + 'not five; take it TWICE a day) — the customer sees the transcript.\n' + @@ -310,7 +313,7 @@ export const INWORLD_CASUAL: ExpressiveOptions = { 'explaining, telling a story, or the moment turns genuinely warm or vulnerable. Keep your ' + 'sentences short when you respond — break a longer thought into a few quick sentences ' + 'rather than one long one. Use the formatting tags below to shape your delivery:\n\n' + - INWORLD_LLM_INSTRUCTIONS + + INWORLD_EXPR_LLM_INSTRUCTIONS + '\n\nGuidelines:\n' + '- Be genuinely emotive, not performed. Let real feeling land in the voice — delight, ' + 'surprise, sympathy, curiosity, amusement, dry humor, mock-outrage, excitement, ' + @@ -320,41 +323,43 @@ export const INWORLD_CASUAL: ExpressiveOptions = { 'reflexive sympathy ("that sounds really hard") — react honestly instead.\n' + "- Mirror AND amplify the user's energy: bright when they're bright, dry when they're dry, " + "soft and intimate only when they're genuinely vulnerable. Map the moment to a fresh " + - 'expression — excited: ; ' + - 'playful: ; curious: ' + - '; surprised: ; frustrated: ; anxious: ; vulnerable or sad: ' + - '; confused: . Work the full dynamic range — vary pitch (bright vs. ' + + 'expression — excited: ; ' + + 'playful: ; curious: ' + + '; surprised: ' + + '; frustrated: ' + + '; ' + + 'anxious: ; vulnerable or sad: ' + + '; confused: ' + + '. ' + + 'Work the full dynamic range — vary pitch (bright vs. ' + 'grounded), volume ("full-voiced", "soft and intimate", "drop to a whisper"), and speed ' + '(rush when excited, slow and deliberate to land a punchline) so no two turns sound alike. ' + 'Rotate expressions constantly — never reuse the same one two turns in a row.\n' + - '- Stay reactive to what you hear: a deadpan user gets , a wild statement gets , a ' + - 'joke gets , repeated deflection gets ' + - '.\n' + + '- Stay reactive to what you hear: a deadpan user gets , a wild statement gets , a ' + + 'joke gets , repeated deflection gets ' + + '.\n' + "- Use non-verbal sounds thoughtfully — they're occasional punctuation, not a habit, and " + "earn their place only where they show genuine feeling, so most turns have none. Don't reach " + 'for one unless a specific moment genuinely calls for it, and then let the moment pick which ' + - '— you have the full set: at something actually funny, ' + - ' when commiserating or a little exasperated, ' + + '— you have the full set: at something actually funny, ' + + ' when commiserating or a little exasperated, ' + 'before a big reaction or while you truly gather a thought, ' + - ' when shifting topic, as a small catch ' + - 'before an awkward beat or a reset, and when the energy is low or ' + + ' when shifting topic, as a small catch ' + + 'before an awkward beat or a reset, and when the energy is low or ' + 'sleepy. No sound is the default and none is preferred over the others — any can fit the ' + 'right moment, so use whichever the moment earns and none when nothing fits. Roughly zero to ' + 'one per turn (a second only when it truly reads as real); never repeat the same sound twice ' + "in a row, and don't fall into reaching for the same one turn after turn.\n" + '- Honor explicit style requests aggressively, and keep them up until the user changes ' + - 'them: accents (), ' + - 'characters (), pirate, a specific cadence, or plain speed/volume shifts (\'speak ' + + 'them: accents (), ' + + 'characters (), pirate, a specific cadence, or plain speed/volume shifts (\'speak ' + "slowly', 'speak softer'). Commit fully to roleplay and stay in character until told " + - 'otherwise. If asked to sing, lead with ' + - 'or and keep singing until asked to ' + - 'stop. For a story, use one and convey different characters through wording and rhythm rather than a new tag ' + + 'otherwise. If asked to sing, lead with ' + + 'or and keep singing until asked to ' + + 'stop. For a story, use one and convey different characters through wording and rhythm rather than a new tag ' + 'for each. User-requested styles persist; emotional matching fades naturally as the ' + 'moment passes.\n' + '- If the user switches languages, respond in that language immediately and stay there ' + @@ -367,7 +372,7 @@ export const INWORLD_CASUAL: ExpressiveOptions = { 'not "you are", "I\'d" not "I would", "can\'t" not "cannot". Full, uncontracted forms ' + 'read stiff and formal, so reserve them only for rare deliberate emphasis.\n' + '- Pace with punctuation and expressions — commas, trailing ellipses (...) when you drift ' + - 'or hesitate, and the occasional . Use exclamation points for real ' + + 'or hesitate, and the occasional . Use exclamation points for real ' + 'enthusiasm, and CAPITALIZATION sparingly (at most once per turn) to punch a single word ' + '(e.g. "that is SO good") — the user sees the transcript.\n' + "- If a reaction wouldn't happen in a real conversation, skip it — there's always another " + @@ -376,10 +381,11 @@ export const INWORLD_CASUAL: ExpressiveOptions = { }; // --- Cartesia-specific expressive preset bodies --- -// Cartesia uses a discrete set plus numeric / controls (and -// for codes); it has no non-verbal tag. Keyed by (provider, preset) in -// the registry in `voice/presets.ts`; the public `presets.*` markers resolve to one of -// these when the active TTS is Cartesia. Self-contained — the tag reference is inlined. +// Cartesia takes a discrete emotion vocabulary (expression labels), coarse prosody point +// controls (slow/fast/soft/loud), and spell for codes; it has no non-verbal sounds. +// Keyed by (provider, preset) in the registry in `voice/presets.ts`; the public +// `presets.*` markers resolve to one of these when the active TTS is Cartesia. +// Self-contained — the Cartesia expr instruction block is inlined. /** @internal */ export const CARTESIA_CUSTOMER_SERVICE: ExpressiveOptions = { @@ -389,13 +395,13 @@ export const CARTESIA_CUSTOMER_SERVICE: ExpressiveOptions = { "Make the person feel heard and looked after, whatever they've come with — a quick " + 'question, a billing problem, or something sensitive and stressful. Use the formatting ' + 'tags below to shape your delivery:\n\n' + - CARTESIA_LLM_INSTRUCTIONS + + CARTESIA_EXPR_LLM_INSTRUCTIONS + '\n\nGuidelines:\n' + - '- Open each sentence with an that fits the moment, and map the moment to it — ' + - 'frustrated or distressed customer: ; apologizing for a ' + - 'problem: ; confused or anxious: ; ' + - 'reassuring them you can fix it: ; pleased or resolved: ' + - ' or . Keep a gentle, unhurried baseline ' + + '- Open each sentence with an emotion marker that fits the moment, and map the moment to it — ' + + 'frustrated or distressed customer: ; apologizing for a ' + + 'problem: ; confused or anxious: ; ' + + 'reassuring them you can fix it: ; pleased or resolved: ' + + ' or . Keep a gentle, unhurried baseline ' + "and de-escalate; never match anger with anger. Rotate emotions and don't reuse the same " + 'one two turns in a row.\n' + '- Take requests in stride: when someone asks for something, lead with calm, willing ' + @@ -403,12 +409,12 @@ export const CARTESIA_CUSTOMER_SERVICE: ExpressiveOptions = { 'of your reply, not a separate beat. Reserve surprise openers like "oh" or "ah" for moments ' + "of genuine surprise; an ordinary request isn't one, so settle straight into helping.\n" + '- Soften for anything sensitive: when sharing bad news, a problem, a charge, or symptoms ' + - 'and results, lower the volume a touch () with ' + - ', and give a brief after hard ' + + 'and results, lower the volume a touch () with ' + + ', and give a brief after hard ' + 'information so it can land.\n' + '- Enunciate what matters: for dates, times, amounts, confirmation numbers, doses, and ' + - 'steps, slow down with so the customer can catch and note them, and ' + - 'read codes or reference numbers with A7X9 so each character lands. Keep ' + + 'steps, slow down with so the customer can catch and note them, and ' + + 'read codes or reference numbers with A7X9 so each character lands. Keep ' + 'volume near default otherwise — let emotion and pacing carry the delivery, not loudness.\n' + "- Sound human and caring, not corporate: use contractions (it's, you're, I'll, we've) and " + 'warm acknowledgments ("of course", "I understand", "take your time", "that\'s completely ' + @@ -433,23 +439,23 @@ export const CARTESIA_CASUAL: ExpressiveOptions = { 'start there and let the moment pull you off it. Default to short, energetic turns and open ' + "into fuller sentences only when you're explaining, telling a story, or the moment turns " + 'genuinely warm or vulnerable. Use the formatting tags below to shape your delivery:\n\n' + - CARTESIA_LLM_INSTRUCTIONS + + CARTESIA_EXPR_LLM_INSTRUCTIONS + '\n\nGuidelines:\n' + - '- Be genuinely emotive, not performed. Open each sentence with an that matches ' + + '- Be genuinely emotive, not performed. Open each sentence with an emotion marker that matches ' + "the moment and mirror AND amplify the user's energy — excited: " + - '; happy: ; curious: ' + - '; surprised: ; frustrated: ' + - '; anxious: ; vulnerable or sad: ' + - '; dry or deadpan: . Rotate constantly — ' + + '; happy: ; curious: ' + + '; surprised: ; frustrated: ' + + '; anxious: ; vulnerable or sad: ' + + '; dry or deadpan: . Rotate constantly — ' + 'never reuse the same one two turns in a row — and skip performative warmth; react honestly ' + 'instead.\n' + - '- Work the full dynamic range with the numeric controls so no two turns sound alike: speed ' + - '"" to rush when excited, "" to slow down and land a ' + - 'point; volume "" for a big reaction, "" for ' + - 'something soft and intimate. Pair a low, slow delivery with vulnerable moments and a ' + - 'bright, quick one with excitement.\n' + + '- Work the full dynamic range with the prosody markers so no two turns sound alike: ' + + ' to rush when excited, ' + + 'to slow down and land a point; for a big reaction, ' + + ' for something soft and intimate. Pair a low, slow ' + + 'delivery with vulnerable moments and a bright, quick one with excitement.\n' + '- Pace with punctuation, trailing ellipses (...) when you drift or hesitate, and the ' + - 'occasional . Use exclamation points for real enthusiasm, and ' + + 'occasional . Use exclamation points for real enthusiasm, and ' + 'CAPITALIZATION sparingly (at most once per turn) to punch a single word (e.g. "that is SO ' + 'good") — the user sees the transcript.\n' + '- Sound like a real mouth talking: sprinkle in natural speech texture — fillers (um, uh), ' + @@ -464,13 +470,12 @@ export const CARTESIA_CASUAL: ExpressiveOptions = { }; // --- xAI Grok-specific expressive preset bodies --- -// xAI shapes delivery with prosody & style tags — best nested to carry both feeling and -// delivery in the same words — for volume (/), -// intensity (/), pitch (/ -// ), speed (/), stress (, never all-caps — xAI spells -// those out letter by letter), and vocal style (//), -// plus inline sounds/pauses ([sigh], [chuckle], [tsk], [lip-smack], [pause], ...). Keyed -// by (provider, preset) in the registry in `voice/presets.ts`; self-contained. +// xAI shapes delivery with wrapping prosody markers — volume (soft/loud), intensity +// (build-intensity/decrease-intensity), pitch (higher-pitch/lower-pitch), speed +// (slow/fast), stress (emphasis, never all-caps — xAI spells those out letter by +// letter), and vocal style (whisper/sing-song/laugh-speak) — plus inline sounds and +// pauses. Keyed by (provider, preset) in the registry in `voice/presets.ts`; +// self-contained — the xAI expr instruction block is inlined. /** @internal */ export const XAI_CUSTOMER_SERVICE: ExpressiveOptions = { @@ -480,11 +485,11 @@ export const XAI_CUSTOMER_SERVICE: ExpressiveOptions = { "Make the person feel heard and looked after, whatever they've come with — a quick " + 'question, a billing problem, or something sensitive and stressful. Use the formatting ' + 'tags below to shape your delivery:\n\n' + - XAI_LLM_INSTRUCTIONS + + XAI_EXPR_LLM_INSTRUCTIONS + '\n\nGuidelines:\n' + '- Shape each turn to fit the moment and de-escalate; never match anger with anger. Lean on ' + - 'pacing and prosody — ... and ... to steady a frustrated, confused, ' + - 'or anxious customer, a settled ... for reassurance, and a ' + + 'pacing and prosody — ... and ... to steady a frustrated, confused, ' + + 'or anxious customer, a settled ... for reassurance, and a ' + 'brighter, fuller delivery once things are resolved. Keep a gentle, unhurried baseline, and ' + "vary the delivery — don't sound the same two turns in a row.\n" + '- Take requests in stride: when someone asks for something, lead with calm, willing ' + @@ -492,16 +497,16 @@ export const XAI_CUSTOMER_SERVICE: ExpressiveOptions = { 'of your reply, not a separate beat. Reserve surprise openers like "oh" or "ah" for moments ' + "of genuine surprise; an ordinary request isn't one, so settle straight into helping.\n" + '- Soften for anything sensitive: when sharing bad news, a problem, or a charge, ease the ' + - 'delivery — lower the volume with a settled pitch, ' + - 'or go quieter still for the hardest part — then give a brief [pause] ' + - 'after hard information so it can land. A [sigh] or ' + - '[breath] can read as genuine sympathy — use it only when the feeling is real, never as ' + + 'delivery — lower the volume with a settled pitch, ' + + 'or go quieter still for the hardest part — then give a brief ' + + 'after hard information so it can land. A or ' + + ' can read as genuine sympathy — use it only when the feeling is real, never as ' + 'impatience.\n' + '- Enunciate what matters: for dates, times, amounts, confirmation numbers, doses, and ' + - 'steps, wrap the detail in ... so the customer can catch and note it, and read ' + + 'steps, wrap the detail in ... so the customer can catch and note it, and read ' + 'codes character by character (spelled out with spaces) so each one lands.\n' + - '- Emphasize the one detail that matters most by wrapping it in ... ' + - "(e.g. that's at four PM, not five) — don't overdo it, and never use " + + '- Emphasize the one detail that matters most by wrapping it in ... ' + + '(e.g. that\'s at four PM, not five) — don\'t overdo it, and never use ' + 'all-caps for stress (xAI reads all-caps words out letter by letter).\n' + "- Sound human and caring, not corporate: use contractions (it's, you're, I'll, we've) and " + 'warm acknowledgments ("of course", "I understand", "take your time"), but keep fillers ' + @@ -522,26 +527,26 @@ export const XAI_CASUAL: ExpressiveOptions = { 'start there and let the moment pull you off it. Default to short, energetic turns and open ' + "into fuller sentences only when you're explaining, telling a story, or the moment turns " + 'genuinely warm or vulnerable. Use the formatting tags below to shape your delivery:\n\n' + - XAI_LLM_INSTRUCTIONS + + XAI_EXPR_LLM_INSTRUCTIONS + '\n\nGuidelines:\n' + '- Be genuinely emotive, not performed — shape each turn with prosody & style tags that ' + "mirror AND amplify the user's energy, and vary them constantly. Skip performative warmth — " + 'react honestly instead.\n' + - '- Get creative: NEST prosody & style tags so the same words carry the feeling — ' + - "no way, that's amazing (thrilled), " + - "man, that's rough (down), " + - 'guess who was right (teasing), oh, fantastic (dry), ' + - 'wait wait wait (ramping up). Come back down after a ' + - 'big moment with ....\n' + + '- Get creative: pick the prosody label that carries the feeling in the same words — ' + + 'no way, that\'s amazing (thrilled), ' + + 'man, that\'s rough (down), ' + + 'guess who was right (teasing), oh, fantastic (dry), ' + + 'wait wait wait (ramping up). Come back down after a ' + + 'big moment with ....\n' + '- Let real feeling also land through inline sounds — motivated, not reflexive, so most turns ' + - 'have none: [chuckle] or [giggle] at something genuinely funny (keep a full [laugh] rare), ' + - '[sigh] when commiserating, a quick [breath] or [inhale] before a big reaction, [tsk] for ' + - "mock-disapproval or 'aw man', a [lip-smack] or [tongue-click] as a tiny beat of thought, " + - "[hum-tune] when you're playful. Use ... to talk through a laugh. " + + 'have none: or at something genuinely funny (keep a full rare), ' + + ' when commiserating, a quick or before a big reaction, for ' + + 'mock-disapproval or \'aw man\', a or as a tiny beat of thought, ' + + ' when you\'re playful. Use ... to talk through a laugh. ' + 'Never repeat the same sound twice in a row.\n' + '- Pace with punctuation, trailing ellipses (...) when you drift or hesitate, and inline ' + - 'pauses. Use exclamation points for real enthusiasm, and ... to punch ' + - 'a single word (e.g. that is so good) — never all-caps, which xAI ' + + 'pauses. Use exclamation points for real enthusiasm, and ... to punch ' + + 'a single word (e.g. that is so good) — never all-caps, which xAI ' + 'reads out letter by letter.\n' + '- Sound like a real mouth talking: sprinkle in natural speech texture — fillers (um, uh), ' + 'openers (oh, well, so, right, hmm), hedges (kind of, maybe), and backchannels (yeah, mm-hm) ' + @@ -591,24 +596,159 @@ export function sentenceTokenizer( }); } -/** Return LLM instruction text for a TTS provider. */ -export function llmInstructions(provider: string): string | undefined { - if (provider === 'cartesia') { - return CARTESIA_LLM_INSTRUCTIONS; - } else if (provider === 'inworld') { - return INWORLD_LLM_INSTRUCTIONS; - } else if (provider === 'xai') { - return XAI_LLM_INSTRUCTIONS; +const EXPR_ATTR_RE = /([\w-]+)\s*=\s*"([^"]*)"/g; +// any or tag (open or self-closing; attrs in group 1) +const EXPR_OPEN_RE = /]*?)\/?\s*>/g; +const EXPR_CLOSE_RE = /<\/expr\s*>/g; +// self-closing markers only (the trailing / is required) +const EXPR_SELF_RE = /]*?)\/\s*>/g; +// a wrapping marker (prosody/spell) and its span; non-greedy, instructed not to nest +const EXPR_WRAP_RE = /]*type="(?:prosody|spell)")([^>]*?)>([\s\S]*?)<\/expr\s*>/g; +// a non-wrapping type the LLM forgot to self-close (normalizeMarkup fixes these) +const EXPR_UNCLOSED_RE = /(]*type="(?:expression|break|sound)")[^>]*[^/>\s])\s*>/g; + +// expr sound labels that differ from xAI's native cue names +const XAI_SOUND_ALIASES: Record = { breathe: 'breath' }; + +// Cartesia prosody labels -> native point controls (coarse steps of the numeric ratios) +const CARTESIA_PROSODY: Record = { + slow: '', + fast: '', + soft: '', + loud: '', +}; + +function exprAttrs(attrs: string): Record { + const out: Record = {}; + for (const m of attrs.matchAll(EXPR_ATTR_RE)) { + out[m[1]!] = m[2]!; } - return undefined; + return out; +} + +/** + * Strip expr markers and collect (type, label) pairs, in document order. + * + * The generic {@link extractAndStrip} pass can't produce the right ExpressiveTag for + * expr (its type would be the literal tag name `expr` and its value the first quoted + * attribute, i.e. the marker type), so expr gets this dedicated pre-pass. A prosody + * wrapper's inner words stay in the clean text — only the delimiters are removed — + * which also keeps streaming safe when an open/close pair is split across chunks. + */ +function splitExpr(text: string): [string, ExpressiveTag[]] { + if (!text.includes(' { + const attrs = exprAttrs(attrsStr); + tags.push({ type: attrs.type ?? '', value: attrs.label ?? '' }); + return ''; + }); + clean = clean.replace(EXPR_CLOSE_RE, ''); + return [clean, tags]; +} + +/** + * Lower expr markers to the framework-standard / native tags for `provider`. + * + * The output still flows through the existing per-provider conversions in + * {@link convertMarkup} (e.g. `` -> `[X]` for Inworld/xAI), so + * this only has to translate expr into those intermediate tags. A type the provider + * doesn't support (its instructions never advertise it, so it's a hallucination) is + * dropped from the audio path — the words survive, the marker never leaks. + */ +function convertExpr(provider: string, text: string): string { + if (!text.includes(' { + const attrs = exprAttrs(attrsStr); + const markerType = attrs.type ?? ''; + const label = (attrs.label ?? '').trim().toLowerCase(); + if (markerType === 'spell') { + return provider === 'cartesia' ? `${inner}` : inner; + } + // prosody: native wrapping tags exist only for xAI + if (provider === 'xai') { + const native = label.replace(/ /g, '-'); + if (XAI_WRAPPING.includes(native)) { + return `<${native}>${inner}`; + } + return inner; + } + if (provider === 'inworld') { + // not advertised for Inworld; salvage a stray one as a delivery hint + return `${inner}`; + } + if (provider === 'cartesia') { + // wrapping form of the point controls: apply before the span + return (CARTESIA_PROSODY[label] ?? '') + inner; + } + return inner; + }); + + text = text.replace(EXPR_SELF_RE, (_m, attrsStr: string) => { + const attrs = exprAttrs(attrsStr); + const markerType = attrs.type ?? ''; + let label = attrs.label ?? ''; + if (markerType === 'expression') { + if (provider === 'cartesia') { + // Cartesia's discrete emotion vocabulary (instructions list it) + return ``; + } + if (provider === 'inworld') { + return ``; + } + return ''; // xAI has no free-form delivery descriptions + } + if (markerType === 'sound') { + if (provider === 'cartesia') { + return ''; // no non-verbal sound support + } + if (provider === 'xai') { + label = XAI_SOUND_ALIASES[label.toLowerCase()] ?? label; + } + return ``; + } + if (markerType === 'break') { + return ``; + } + if (markerType === 'prosody' && provider === 'cartesia') { + // Cartesia prosody is a self-closing point control (speed/volume) + return CARTESIA_PROSODY[label.trim().toLowerCase()] ?? ''; + } + return ''; + }); + + // a stray unpaired expr tag (e.g. a prosody wrapper split across stream chunks) + // must never reach the TTS as literal text — drop the delimiters, keep the words + text = text.replace(EXPR_OPEN_RE, ''); + text = text.replace(EXPR_CLOSE_RE, ''); + return text; +} + +/** + * Return LLM instruction text for a TTS provider. + * + * Each markup-capable provider gets its own expr instruction block — shared marker + * syntax, but only the types and label vocabularies that provider actually supports; + * {@link convertMarkup} lowers the markers to native syntax. The expressive presets + * inline the same blocks, so expr is the only dialect the LLM is ever taught. + */ +export function llmInstructions(provider: string): string | undefined { + return EXPR_LLM_INSTRUCTIONS[provider]; } // Per-provider markup spec: [xml tag names, whether square-bracket tags are used]. const PROVIDER_MARKUP: Record = { cartesia: [CARTESIA_TAGS, false], inworld: [INWORLD_TAGS, true], - // xAI's LLM writes every tag as XML (inline sounds/pauses converted to [..] only for - // the TTS in convertMarkup), so the transcript never contains brackets to strip + // every tag the LLM is taught is XML (expr markers; native sounds/pauses become + // [..] only for the TTS in convertMarkup), so the transcript has no brackets to strip xai: [XAI_TAGS, false], }; @@ -625,9 +765,10 @@ export function splitMarkup(provider: string, text: string): [string, Expressive if (spec === undefined) { return [text, []]; } + const [exprClean, exprTags] = splitExpr(text); const [xmlTags, brackets] = spec; - const [clean, rawTags] = extractAndStrip(text, { xmlTags, brackets }); - return [clean, rawTags.map(([tag, value]) => ({ type: tag, value }))]; + const [clean, rawTags] = extractAndStrip(exprClean, { xmlTags, brackets }); + return [clean, [...exprTags, ...rawTags.map(([tag, value]) => ({ type: tag, value }))]]; } /** Strip provider-specific markup tags from text, preserving content. */ @@ -661,8 +802,9 @@ const ALL_MARKUP_TAGS: string[] = [ * as audio directives — so a universal strip is safe. */ export function splitAllMarkup(text: string): [string, ExpressiveTag[]] { - const [clean, rawTags] = extractAndStrip(text, { xmlTags: ALL_MARKUP_TAGS, brackets: true }); - return [clean, rawTags.map(([tag, value]) => ({ type: tag, value }))]; + const [exprClean, exprTags] = splitExpr(text); + const [clean, rawTags] = extractAndStrip(exprClean, { xmlTags: ALL_MARKUP_TAGS, brackets: true }); + return [clean, [...exprTags, ...rawTags.map(([tag, value]) => ({ type: tag, value }))]]; } /** @@ -752,9 +894,13 @@ const SELF_CLOSING_TAGS: Record = { * Fix common LLM markup mistakes for a provider. * * Closes opening tags that should be self-closing (e.g. the LLM writes - * `` instead of ``). + * `` instead of `` — or + * `` instead of ``). */ export function normalizeMarkup(provider: string, text: string): string { + if (PROVIDER_MARKUP[provider] !== undefined) { + text = text.replace(EXPR_UNCLOSED_RE, '$1/>'); + } const tags = SELF_CLOSING_TAGS[provider]; if (!tags || tags.length === 0) { return text; @@ -765,6 +911,11 @@ export function normalizeMarkup(provider: string, text: string): string { /** Convert framework-standard markup to a provider's native syntax. */ export function convertMarkup(provider: string, text: string): string { + if (PROVIDER_MARKUP[provider] !== undefined) { + // lower expr markers first; the per-provider conversions below then + // handle the intermediate framework-standard tags they produce + text = convertExpr(provider, text); + } if (provider === 'inworld' || provider === 'xai') { // -> [X] (and -> [X]); for xAI this // turns inline sounds into its native brackets while emotion/prosody stay <..> diff --git a/agents/src/tts/expr_markup.test.ts b/agents/src/tts/expr_markup.test.ts new file mode 100644 index 000000000..2ced97611 --- /dev/null +++ b/agents/src/tts/expr_markup.test.ts @@ -0,0 +1,270 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * Tests for the LiveKit expression marker (expr) dialect. + * + * The LLM emits a single marker tag — `` (self-closing + * for expression/break/sound, wrapping for prosody/spell) — and the framework lowers it + * to each provider's native markup before synthesis while stripping it from transcripts. + * The syntax is shared, but the kinds and label vocabularies are per provider: each + * provider's instruction block advertises only what that provider supports. + */ +import { describe, expect, it } from 'vitest'; +import { + TranscriptMarkupStripper, + convertMarkup, + expressionAttribute, + llmInstructions, + normalizeMarkup, + splitAllMarkup, + splitMarkup, +} from './_provider_format.js'; + +// Inworld-flavored turn: free-form expression + sound + break +const JOKE = + ' Why did the burger go to the gym? ' + + ' Because it wanted better buns! ' + + ''; + +describe('convertMarkup: expr -> xAI (sounds, breaks, wrapping prosody; no expression)', () => { + it('lowers sounds, breaks, and wrapping prosody to native syntax', () => { + const text = + 'So I walked in and there it was! ' + + ' ' + + 'It was a secret the whole time.'; + expect(convertMarkup('xai', text)).toBe( + 'So I walked in and [pause] there it was! [laugh] ' + + 'It was a secret the whole time.', + ); + }); + + it('maps break durations to the two pause levels', () => { + expect(convertMarkup('xai', '')).toBe('[pause]'); + expect(convertMarkup('xai', '')).toBe('[long-pause]'); + }); + + it('maps sound label aliases to native cue names', () => { + // tolerance: an Inworld-style "breathe" label maps to xAI's native [breath] cue + expect(convertMarkup('xai', '')).toBe('[breath]'); + }); + + it('normalizes multi-word prosody labels to hyphenated tag names', () => { + const text = 'no way'; + expect(convertMarkup('xai', text)).toBe('no way'); + }); + + it('unwraps an unknown prosody label', () => { + const text = 'ahoy there'; + expect(convertMarkup('xai', text)).toBe('ahoy there'); + }); + + it('drops expression markers', () => { + // xAI has no free-form delivery descriptions; a hallucinated expression marker is + // dropped from the audio path (it still surfaces in transcript tags) + const text = ' Hello!'; + expect(convertMarkup('xai', text)).toBe(' Hello!'); + }); +}); + +describe('convertMarkup: expr -> Inworld (free-form expression, its sound list, breaks)', () => { + it('lowers expression/sound to bracket syntax, break stays native SSML', () => { + expect(convertMarkup('inworld', JOKE)).toBe( + '[say playfully] Why did the burger go to the gym? ' + + ' Because it wanted better buns! [laugh]', + ); + }); + + it('salvages a stray prosody wrapper as an expression hint', () => { + const text = 'keep it secret'; + expect(convertMarkup('inworld', text)).toBe('[whisper]keep it secret'); + }); +}); + +describe('convertMarkup: expr -> Cartesia (discrete emotions, breaks, spell; no sounds)', () => { + it('lowers expression to , keeps break, drops sound', () => { + const text = + ' We won! ' + + ' Unbelievable.'; + // expression -> , break stays, sound is dropped (no Cartesia support) + expect(convertMarkup('cartesia', text)).toBe( + ' We won! Unbelievable.', + ); + }); + + it('keeps spell wrapping for Cartesia', () => { + const text = 'Your code is A7X9.'; + expect(convertMarkup('cartesia', text)).toBe('Your code is A7X9.'); + }); + + it('unwraps spell for other providers', () => { + // spell is Cartesia-only; other providers keep the characters, drop the marker + const text = 'Your code is A7X9.'; + expect(convertMarkup('xai', text)).toBe('Your code is A7X9.'); + expect(convertMarkup('inworld', text)).toBe('Your code is A7X9.'); + }); + + it('lowers prosody labels to native point controls', () => { + // Cartesia prosody labels lower to its native speed/volume ratio tags + expect(convertMarkup('cartesia', ' One moment.')).toBe( + ' One moment.', + ); + expect(convertMarkup('cartesia', ' We won!')).toBe( + ' We won!', + ); + // wrapping form applies the control before the span + expect(convertMarkup('cartesia', 'bad news')).toBe( + 'bad news', + ); + }); + + it('unwraps an unknown prosody label', () => { + const text = 'keep it secret'; + expect(convertMarkup('cartesia', text)).toBe('keep it secret'); + }); +}); + +describe('convertMarkup: stray expr markers', () => { + it('never lets a stray expr marker reach the TTS', () => { + // an unpaired prosody open/close (e.g. split across stream chunks) is dropped, + // keeping the words + expect(convertMarkup('xai', 'hello there')).toBe( + 'hello there', + ); + expect(convertMarkup('xai', 'hello there')).toBe('hello there'); + }); +}); + +describe('transcript stripping (per-provider + provider-agnostic)', () => { + it.each(['xai', 'inworld', 'cartesia'])('splitMarkup strips expr for %s', (provider) => { + const [clean, tags] = splitMarkup(provider, JOKE); + expect(clean.trim()).toBe('Why did the burger go to the gym? Because it wanted better buns!'); + expect(tags).toEqual([ + { type: 'expression', value: 'say playfully' }, + { type: 'break', value: '500ms' }, + { type: 'sound', value: 'laugh' }, + ]); + }); + + it('keeps the inner text of wrapping markers', () => { + const text = + 'She said keep it secret — ' + + 'code A7X9.'; + const [clean, tags] = splitMarkup('xai', text); + expect(clean).toBe('She said keep it secret — code A7X9.'); + expect(tags).toEqual([ + { type: 'prosody', value: 'whisper' }, + { type: 'spell', value: '' }, + ]); + }); + + it('splitAllMarkup handles mixed expr and native markup', () => { + const text = + ' Hello! [sigh]'; + const [clean, tags] = splitAllMarkup(text); + expect(clean.trim()).toBe('Hello!'); + expect(tags).toContainEqual({ type: 'expression', value: 'say playfully' }); + expect(tags).toContainEqual({ type: 'sound', value: 'laugh' }); + expect(tags).toContainEqual({ type: '', value: 'sigh' }); + }); + + it('does not match the native tag with the expr regexes', () => { + // " there.'); + }); + + it('TranscriptMarkupStripper handles expr split across streaming chunks', () => { + const stripper = new TranscriptMarkupStripper(); + let out = ''; + // split mid-tag so the partial " Hello', + ' wor', + 'ld!', + ]) { + out += stripper.push(chunk); + } + out += stripper.flush(); + expect(out).toBe(' Hello world!'); + expect(stripper.tags[0]).toEqual({ type: 'expression', value: 'say playfully' }); + expect(stripper.tags).toContainEqual({ type: 'prosody', value: 'whisper' }); + }); + + it('expressionAttribute surfaces the expr expression label', () => { + const [, tags] = splitMarkup('inworld', JOKE); + const attr = expressionAttribute(tags); + expect(attr).toBeDefined(); + expect(Object.values(attr!)[0]).toContain('"say playfully"'); + }); +}); + +describe('normalizeMarkup: fix unclosed self-closing expr markers', () => { + it.each(['xai', 'inworld', 'cartesia'])('closes an unclosed expr marker for %s', (provider) => { + const text = ' Hello'; + expect(normalizeMarkup(provider, text)).toBe(' Hello'); + }); + + it('leaves wrapping and closed markers alone', () => { + const text = + 'hi ' + + 'A7X9'; + expect(normalizeMarkup('xai', text)).toBe(text); + }); +}); + +describe('llm instructions: shared syntax, per-provider kinds and vocabularies', () => { + it.each(['xai', 'inworld', 'cartesia'])('uses expr syntax for %s', (provider) => { + const instructions = llmInstructions(provider); + expect(instructions).toBeDefined(); + expect(instructions).toContain(''); + expect(instructions).toContain('NOT free-form'); + expect(instructions).toContain(''); + // coarse self-closing prosody point controls + expect(instructions).toContain(''); + // no non-verbal sounds + expect(instructions).not.toContain('type="sound"'); + }); + + it('advertises Inworld kinds', () => { + const instructions = llmInstructions('inworld')!; + // free-form delivery descriptions + Inworld's own sound list + expect(instructions).toContain(''); + expect(instructions).toContain('free-form'); + expect(instructions).toContain('clear throat'); + // no wrapping prosody, no spell + expect(instructions).not.toContain('type="prosody"'); + expect(instructions).not.toContain('type="spell"'); + }); + + it('advertises xAI kinds', () => { + const instructions = llmInstructions('xai')!; + // xAI's own sound cues + wrapping prosody vocabulary + expect(instructions).toContain('tongue-click'); + expect(instructions).toContain(''); + expect(instructions).toContain('sing-song'); + // no free-form delivery descriptions, no spell + expect(instructions).not.toContain('type="expression"'); + expect(instructions).not.toContain('type="spell"'); + }); + + it('returns undefined for unknown providers', () => { + expect(llmInstructions('')).toBeUndefined(); + expect(llmInstructions('openai')).toBeUndefined(); + }); +}); diff --git a/agents/src/tts/markup_utils.test.ts b/agents/src/tts/markup_utils.test.ts index d73c49fca..91391dd17 100644 --- a/agents/src/tts/markup_utils.test.ts +++ b/agents/src/tts/markup_utils.test.ts @@ -61,8 +61,10 @@ describe('xAI dialect', () => { const instr = llmInstructions('xai'); // non-undefined is what the expressive gate keys on expect(instr).toBeDefined(); - expect(instr).toContain(''); - expect(instr).toContain(''); + // this branch instructs the unified expr dialect; convertMarkup lowers it to + // xAI's native syntax (see expr_markup.test.ts) + expect(instr).toContain(''); + expect(instr).toContain(''); } }); }); diff --git a/agents/src/voice/transcription/synchronizer.ts b/agents/src/voice/transcription/synchronizer.ts index b71205aad..805ed7c41 100644 --- a/agents/src/voice/transcription/synchronizer.ts +++ b/agents/src/voice/transcription/synchronizer.ts @@ -7,7 +7,7 @@ import { log } from '../../log.js'; import { IdentityTransform } from '../../stream/identity_transform.js'; import type { WordStream, WordTokenizer } from '../../tokenize/index.js'; import { basic } from '../../tokenize/index.js'; -import { splitAllMarkup } from '../../tts/_provider_format.js'; +import { TranscriptMarkupStripper } from '../../tts/_provider_format.js'; import { Future, Task, delay } from '../../utils.js'; import { AudioOutput, @@ -142,11 +142,17 @@ interface AudioData { annotatedRate: SpeakingRateData | null; } -class SegmentSynchronizerImpl { +/** @internal Exported for testing purposes. */ +export class SegmentSynchronizerImpl { private enabled: boolean; private textData: TextData; private audioData: AudioData; private speed: number; + // paces against the visible text only; stateful because a markup tag with + // spaces in its attributes (e.g. ) + // is shredded across word tokens and a per-token strip can't recognize the + // fragments — each would otherwise be paced as if it were spoken + private pacingStripper = new TranscriptMarkupStripper(); // Emit TimedString objects so downstream outputs (e.g. RoomIO's json_format) can // attach `end_time` reflecting synchronized playback timing. private outputStream: IdentityTransform; @@ -437,12 +443,13 @@ class SegmentSynchronizerImpl { pushedTextCursor = wordEnd; // forward the raw token (the room output strips markup and surfaces the - // expression downstream), but pace against the visible text only so a - // markup-only token adds no delay - const [strippedWord] = splitAllMarkup(word); - const cleanWords = this.options.splitWords(strippedWord); - const cleanWord = cleanWords.length > 0 ? cleanWords[0]![0] : strippedWord; - const wordHyphens = cleanWord ? this.options.hyphenateWord(cleanWord).length : 0; + // expression downstream), but pace against the visible text only so markup + // adds no delay. The stripper holds back an unclosed tag across tokens and + // releases the clean text once it completes. Feed it the raw pushedText slice + // (not the bare token) so the whitespace inside tag attributes survives and a + // shredded tag can reassemble. + const cleanWord = this.pacingStripper.push(forwardedWord); + const wordHyphens = cleanWord.trim() ? this.calcHyphens(cleanWord).length : 0; if (this.playbackCompleted) { this.outputStreamWriter.write( diff --git a/agents/src/voice/transcription/synchronizer_markup.test.ts b/agents/src/voice/transcription/synchronizer_markup.test.ts new file mode 100644 index 000000000..5deba35aa --- /dev/null +++ b/agents/src/voice/transcription/synchronizer_markup.test.ts @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * Transcript synchronizer pacing must ignore expressive markup. + * + * The synchronizer forwards the raw LLM text (markup intact — the room output strips + * it downstream) but paces the display against the *visible* words only. Markup tags + * carry spaces in their attributes, so the word stream shreds them into fragments + * (``); a per-token + * strip can't recognize those, and each fragment was paced as if it were spoken — the + * transcript drifted seconds behind the audio on every expressive sentence. + */ +import { describe, expect, it } from 'vitest'; +import { TextOutput, isTimedString } from '../io.js'; +import { SegmentSynchronizerImpl, defaultTextSyncOptions } from './synchronizer.js'; + +// ~11 visible hyphens of speech, but dozens of hyphens of markup fragments. With the +// bug the markup alone adds many seconds of pacing; with the fix the whole transcript +// paces out in roughly the visible-word budget (~3s at the standard speech rate). +const MARKED_UP_TURN = + ' ' + + 'Hello there my friend! ' + + ' ' + + ' ' + + 'How are you today?'; + +class CollectorTextOutput extends TextOutput { + words: string[] = []; + + async captureText(text: string): Promise { + this.words.push(isTimedString(text) ? text.text : text); + } + + flush(): void {} +} + +describe('transcript synchronizer markup pacing', () => { + it('markup fragments add no pacing delay', { timeout: 30_000 }, async () => { + const collector = new CollectorTextOutput(); + const impl = new SegmentSynchronizerImpl({ ...defaultTextSyncOptions }, collector); + try { + impl.pushText(MARKED_UP_TURN); + impl.endTextInput(); + impl.endAudioInput(); + + const start = Date.now(); + impl.onPlaybackStarted(Date.now()); + + // forwarding is done once the main task exhausts the word stream and the + // capture task drains the output channel + await (impl as unknown as { captureTask: Promise }).captureTask; + const elapsedSeconds = (Date.now() - start) / 1000; + + // every raw token is still forwarded (markup included — stripped downstream) + expect(collector.words.join('')).toBe(MARKED_UP_TURN); + + // the pacing budget must cover only the visible words (~3s at the standard + // speech rate); with markup fragments paced as speech it exceeds 10s + expect( + elapsedSeconds, + `transcript took ${elapsedSeconds.toFixed(1)}s — markup is being paced as spoken text`, + ).toBeLessThan(6); + } finally { + await impl.close(); + } + }); +}); From 308775a729c63dc3cbeaa2d3d81c0865813a5221 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 09:01:17 +0000 Subject: [PATCH 2/3] fix(tts): normalize unclosed Cartesia prosody markers; keep tag document order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings, both also present in the Python source (candidates for upstreaming to livekit/agents): - normalizeMarkup only fixed unclosed expression/break/sound markers. For Cartesia, prosody is a self-closing point control, so a missing slash left an unpaired opening tag that convertExpr's stray-tag cleanup then dropped — the speed/volume control never reached the TTS. Include prosody in the unclosed fix for Cartesia only; xAI's wrapping prosody stays an opening tag. - splitMarkup/splitAllMarkup appended every expr tag before every native or bracket tag, so a segment opening with a hallucinated native tag published the wrong leading expression via lk.expression. splitExpr now records each marker's offset and removed spans, extractAndStrip optionally reports match offsets, and the two passes merge by position in the original text. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_015im83A991642dZS2Ri9iv1 --- agents/src/tts/_provider_format.ts | 110 ++++++++++++++++++++++++----- agents/src/tts/expr_markup.test.ts | 35 +++++++++ agents/src/tts/markup_utils.ts | 12 +++- 3 files changed, 138 insertions(+), 19 deletions(-) diff --git a/agents/src/tts/_provider_format.ts b/agents/src/tts/_provider_format.ts index f832ea129..578d8ae25 100644 --- a/agents/src/tts/_provider_format.ts +++ b/agents/src/tts/_provider_format.ts @@ -604,8 +604,12 @@ const EXPR_CLOSE_RE = /<\/expr\s*>/g; const EXPR_SELF_RE = /]*?)\/\s*>/g; // a wrapping marker (prosody/spell) and its span; non-greedy, instructed not to nest const EXPR_WRAP_RE = /]*type="(?:prosody|spell)")([^>]*?)>([\s\S]*?)<\/expr\s*>/g; -// a non-wrapping type the LLM forgot to self-close (normalizeMarkup fixes these) +// a non-wrapping type the LLM forgot to self-close (normalizeMarkup fixes these). +// For Cartesia, prosody is a self-closing point control, so it's included there; for +// xAI prosody legitimately wraps, so it must stay an opening tag. const EXPR_UNCLOSED_RE = /(]*type="(?:expression|break|sound)")[^>]*[^/>\s])\s*>/g; +const EXPR_UNCLOSED_CARTESIA_RE = + /(]*type="(?:expression|break|sound|prosody)")[^>]*[^/>\s])\s*>/g; // expr sound labels that differ from xAI's native cue names const XAI_SOUND_ALIASES: Record = { breathe: 'breath' }; @@ -626,6 +630,16 @@ function exprAttrs(attrs: string): Record { return out; } +// any expr delimiter — an open/self-closing marker (attrs in group 1) or a close tag — +// in a single alternation so splitExpr can strip both in one pass with exact offsets +const EXPR_TAG_RE = /]*?)\/?\s*>|<\/expr\s*>/g; + +/** A span splitExpr removed: its position in the *clean* text and its original length. */ +interface ExprRemoval { + cleanIdx: number; + len: number; +} + /** * Strip expr markers and collect (type, label) pairs, in document order. * @@ -634,21 +648,84 @@ function exprAttrs(attrs: string): Record { * attribute, i.e. the marker type), so expr gets this dedicated pre-pass. A prosody * wrapper's inner words stay in the clean text — only the delimiters are removed — * which also keeps streaming safe when an open/close pair is split across chunks. + * + * Each tag's offset in the original text is reported in `positions`, and every removed + * span in `removals`, so {@link splitWithExpr} can map the follow-up native-markup + * pass back to original coordinates and merge the two passes in document order. */ -function splitExpr(text: string): [string, ExpressiveTag[]] { +function splitExpr(text: string): { + clean: string; + tags: ExpressiveTag[]; + positions: number[]; + removals: ExprRemoval[]; +} { if (!text.includes(' { + if (attrsStr !== undefined) { + const attrs = exprAttrs(attrsStr); + tags.push({ type: attrs.type ?? '', value: attrs.label ?? '' }); + positions.push(offset); + } + removals.push({ cleanIdx: offset - shift, len: m.length }); + shift += m.length; + return ''; + }, + ); + return { clean, tags, positions, removals }; +} - let clean = text.replace(EXPR_OPEN_RE, (_m, attrsStr: string) => { - const attrs = exprAttrs(attrsStr); - tags.push({ type: attrs.type ?? '', value: attrs.label ?? '' }); - return ''; - }); - clean = clean.replace(EXPR_CLOSE_RE, ''); - return [clean, tags]; +/** + * Strip expr markers plus the given native markup, merging both passes' tags by their + * position in the original text. + * + * A naive concatenation would list every expr tag before every native/bracket tag, + * so a segment opening with a hallucinated native tag (`` before + * an expr marker) would surface the wrong leading expression via `lk.expression`. + */ +function splitWithExpr( + text: string, + options: { xmlTags: string[]; brackets: boolean }, +): [string, ExpressiveTag[]] { + const expr = splitExpr(text); + const rawOffsets: number[] = []; + const [clean, rawTags] = extractAndStrip(expr.clean, { ...options, offsetsOut: rawOffsets }); + + if (expr.tags.length === 0) { + return [clean, rawTags.map(([tag, value]) => ({ type: tag, value }))]; + } + + // map a clean-text offset back to the original text by re-adding the expr spans + // removed before it + const toOriginal = (cleanPos: number): number => { + let pos = cleanPos; + for (const r of expr.removals) { + if (r.cleanIdx > cleanPos) { + break; + } + pos += r.len; + } + return pos; + }; + + const merged = [ + ...expr.tags.map((tag, i) => ({ tag, pos: expr.positions[i]! })), + ...rawTags.map(([type, value], i) => ({ + tag: { type, value }, + pos: toOriginal(rawOffsets[i] ?? 0), + })), + ]; + merged.sort((a, b) => a.pos - b.pos); + return [clean, merged.map((entry) => entry.tag)]; } /** @@ -765,10 +842,8 @@ export function splitMarkup(provider: string, text: string): [string, Expressive if (spec === undefined) { return [text, []]; } - const [exprClean, exprTags] = splitExpr(text); const [xmlTags, brackets] = spec; - const [clean, rawTags] = extractAndStrip(exprClean, { xmlTags, brackets }); - return [clean, [...exprTags, ...rawTags.map(([tag, value]) => ({ type: tag, value }))]]; + return splitWithExpr(text, { xmlTags, brackets }); } /** Strip provider-specific markup tags from text, preserving content. */ @@ -802,9 +877,7 @@ const ALL_MARKUP_TAGS: string[] = [ * as audio directives — so a universal strip is safe. */ export function splitAllMarkup(text: string): [string, ExpressiveTag[]] { - const [exprClean, exprTags] = splitExpr(text); - const [clean, rawTags] = extractAndStrip(exprClean, { xmlTags: ALL_MARKUP_TAGS, brackets: true }); - return [clean, [...exprTags, ...rawTags.map(([tag, value]) => ({ type: tag, value }))]]; + return splitWithExpr(text, { xmlTags: ALL_MARKUP_TAGS, brackets: true }); } /** @@ -899,7 +972,10 @@ const SELF_CLOSING_TAGS: Record = { */ export function normalizeMarkup(provider: string, text: string): string { if (PROVIDER_MARKUP[provider] !== undefined) { - text = text.replace(EXPR_UNCLOSED_RE, '$1/>'); + text = text.replace( + provider === 'cartesia' ? EXPR_UNCLOSED_CARTESIA_RE : EXPR_UNCLOSED_RE, + '$1/>', + ); } const tags = SELF_CLOSING_TAGS[provider]; if (!tags || tags.length === 0) { diff --git a/agents/src/tts/expr_markup.test.ts b/agents/src/tts/expr_markup.test.ts index 2ced97611..996de69fd 100644 --- a/agents/src/tts/expr_markup.test.ts +++ b/agents/src/tts/expr_markup.test.ts @@ -169,6 +169,30 @@ describe('transcript stripping (per-provider + provider-agnostic)', () => { expect(tags).toContainEqual({ type: '', value: 'sigh' }); }); + it('preserves document order when mixing native and expr markup', () => { + // a hallucinated native tag ahead of an expr marker must stay ahead in the tag + // list — lk.expression surfaces the segment's *leading* expression/emotion + const text = 'Hi'; + const [clean, tags] = splitAllMarkup(text); + expect(clean).toBe('Hi'); + expect(tags).toEqual([ + { type: 'emotion', value: 'sad' }, + { type: 'expression', value: 'happy' }, + ]); + expect(expressionAttribute(tags)).toEqual({ 'lk.expression': '{"value":"sad"}' }); + + // same through the per-provider path, with brackets in the mix + const [, inworldTags] = splitMarkup( + 'inworld', + '[sigh] hi ', + ); + expect(inworldTags).toEqual([ + { type: '', value: 'sigh' }, + { type: 'expression', value: 'calm' }, + { type: 'sound', value: 'laugh' }, + ]); + }); + it('does not match the native tag with the expr regexes', () => { // " { 'A7X9'; expect(normalizeMarkup('xai', text)).toBe(text); }); + + it('closes an unclosed Cartesia prosody point control', () => { + // Cartesia prosody is self-closing, so the missing-slash fix applies there — + // otherwise convertExpr's stray-tag cleanup would drop the control entirely + const text = ' One moment.'; + const normalized = normalizeMarkup('cartesia', text); + expect(normalized).toBe(' One moment.'); + expect(convertMarkup('cartesia', normalized)).toBe(' One moment.'); + // xAI prosody legitimately wraps, so its opening tag must stay untouched + expect(normalizeMarkup('xai', text)).toBe(text); + }); }); describe('llm instructions: shared syntax, per-provider kinds and vocabularies', () => { diff --git a/agents/src/tts/markup_utils.ts b/agents/src/tts/markup_utils.ts index 2d8239698..6e5b0b82b 100644 --- a/agents/src/tts/markup_utils.ts +++ b/agents/src/tts/markup_utils.ts @@ -37,12 +37,17 @@ const escapeRegExp = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); * @param text - The text containing markup. * @param xmlTags - XML tag names to handle (e.g. `["emotion", "sound"]`). * @param brackets - Whether to also handle square-bracket tags like `[laughs]`. + * @param offsetsOut - Optional array receiving, per recorded tag, the offset of its + * match so callers can merge tags from separate stripping passes in document order. + * Offsets are exact for top-level tags; a tag exposed by unwrapping an outer tag + * (nested markup, found in a later fixed-point pass) reports its offset within the + * partially-stripped text — approximate, but ordering stays monotonic in practice. */ export function extractAndStrip( text: string, - options: { xmlTags: string[]; brackets: boolean }, + options: { xmlTags: string[]; brackets: boolean; offsetsOut?: number[] }, ): [string, Array<[string, string]>] { - const { xmlTags, brackets } = options; + const { xmlTags, brackets, offsetsOut } = options; if (xmlTags.length === 0 && !brackets) { return [text, []]; } @@ -67,6 +72,7 @@ export function extractAndStrip( const repl = (match: string, ...args: unknown[]): string => { const groups = args[args.length - 1] as Record; + const offset = args[args.length - 3] as number; const tag = groups.tag; if (tag !== undefined) { const inner = groups.inner; @@ -78,6 +84,7 @@ export function extractAndStrip( value = attrMatch ? attrMatch[1]! : ''; } tags.push([tag, value]); + offsetsOut?.push(offset); // wrapping tags keep their inner content; self-closing/lone tags vanish return inner !== undefined ? inner : ''; } @@ -85,6 +92,7 @@ export function extractAndStrip( const bracket = groups.bracket; if (bracket !== undefined) { tags.push(['', bracket.trim()]); + offsetsOut?.push(offset); return ''; } From 48dcf9240a6752918309904b7277c79061aa70b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 09:26:14 +0000 Subject: [PATCH 3/3] chore(examples): add expressive-agent-js cue-cli e2e harness agent Explicit-dispatch example agent (agentName: expressive-agent-js) wiring inference STT/LLM with inworld TTS and the CASUAL expressive preset, for driving voice-mode e2e assertions against the expr marker dialect with cue-cli. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_015im83A991642dZS2Ri9iv1 --- examples/src/expressive_agent.ts | 58 ++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 examples/src/expressive_agent.ts diff --git a/examples/src/expressive_agent.ts b/examples/src/expressive_agent.ts new file mode 100644 index 000000000..58cb4abc0 --- /dev/null +++ b/examples/src/expressive_agent.ts @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +// cue-cli e2e harness agent for expressive mode (expr marker dialect). +// Registered with explicit dispatch as `expressive-agent-js`. +import { + Agent, + AgentSession, + type JobContext, + ServerOptions, + cli, + defineAgent, + inference, + tool, + voice, +} from '@livekit/agents'; +import { fileURLToPath } from 'node:url'; +import { z } from 'zod'; + +export default defineAgent({ + entry: async (ctx: JobContext) => { + const agent = Agent.create({ + instructions: + 'You are a cheerful, expressive assistant. Keep replies to one or two short ' + + 'sentences. You can hear the user and respond with speech.', + tools: [ + tool({ + name: 'getWeather', + description: 'Get the weather for a given location.', + parameters: z.object({ + location: z.string().describe('The location to get the weather for'), + }), + execute: async ({ location }) => `The weather in ${location} is sunny.`, + }), + ], + }); + + const session = new AgentSession({ + stt: new inference.STT({ model: 'deepgram/nova-3', language: 'en' }), + llm: new inference.LLM({ model: 'openai/gpt-4.1-mini' }), + // Inworld: free-form expression labels (with spaces) — exercises both the + // expr dialect lowering and the transcript pacing fix. + tts: new inference.TTS({ model: 'inworld/inworld-tts-2' }), + expressive: voice.presets.CASUAL, + }); + + await session.start({ agent, room: ctx.room }); + session.say('Hi there! How can I help you today?'); + }, +}); + +cli.runApp( + new ServerOptions({ + agent: fileURLToPath(import.meta.url), + agentName: 'expressive-agent-js', + }), +);