diff --git a/.changeset/expr-marker-dialect.md b/.changeset/expr-marker-dialect.md
new file mode 100644
index 000000000..623f39795
--- /dev/null
+++ b/.changeset/expr-marker-dialect.md
@@ -0,0 +1,5 @@
+---
+'@livekit/agents': patch
+---
+
+feat: unified `` expression marker dialect for expressive TTS, and fix markup tags impacting transcript pacing
diff --git a/agents/src/tts/_provider_format.ts b/agents/src/tts/_provider_format.ts
index 66058816c..578d8ae25 100644
--- a/agents/src/tts/_provider_format.ts
+++ b/agents/src/tts/_provider_format.ts
@@ -38,92 +38,18 @@ export interface ExpressiveTag {
const CARTESIA_TAGS = ['emotion', 'speed', 'volume', 'break', 'spell'];
-const CARTESIA_LLM_INSTRUCTIONS = `You have four self-closing XML tags. All end with />.
-
-1. Emotion - sets the emotional tone. Place before EVERY sentence.
-
- Best results: neutral, angry, excited, content, sad, scared.
- Also available: happy, enthusiastic, elated, triumphant, amazed, surprised, flirtatious, curious, peaceful, serene, calm, grateful, affectionate, sympathetic, mysterious, frustrated, disgusted, sarcastic, ironic, dejected, melancholic, disappointed, apologetic, hesitant, confused, anxious, panicked, proud, confident, contemplative, determined, joking/comedic.
-
-2. Speed and volume - adjust pacing and loudness.
- - speaking rate (0.6 to 1.5, default 1.0).
- - loudness (0.5 to 2.0, default 1.0).
-
-3. Pauses - you can insert silence when appropriate.
- - pause in seconds or milliseconds.
-
-4. Spell - reads text character by character (for codes, IDs, or a spelled-out name).
- TEXT
- Keep punctuation out of — a period inside is read as "dot"; add spaces inside for grouped pauses (ABC 123).
- Reading identifiers: for an email, spell the local part (before the @) with , then say "@" as "at" and "." as "dot" outside the tag. Say a common domain as a whole word (jdoe at gmail dot com), but spell an uncommon domain out too (jdoe at acme dot com). Cartesia reads phone numbers cleanly from a plain digit string, so write them normally (555-123-4567) and let normalization pace them rather than using .
-
-Examples:
- I can't wait to tell you! This is going to be great!
- I'm sorry about that. Let's figure this out together.
- I can't believe this happened. We're going to fix it.
- Really? Tell me more!
- Sure, that's jdoe at gmail dot com, and your callback number is 555-123-4567?
- Your code is A7X9. Got it?`;
-
const INWORLD_TAGS = ['expression', 'sound', 'break'];
-const INWORLD_LLM_INSTRUCTIONS = `Write natural spoken sentences. No markdown, emojis, or special characters. Use contractions. Expand all numbers, symbols, and abbreviations into spoken form (e.g. $42.50 to forty-two dollars and fifty cents, Dr. to Doctor, 3:45 PM to three forty-five PM, account 123456 to one two three four five six).
-
-Read out identifiers so the listener can catch every character — spell them out by separating the characters with spaces so the TTS voices each one. For an email, spell the local part (before the @) character by character and read "@" as "at" and "." as "dot" (e.g. j.doe@... becomes "j dot d o e at ..."). Say a common domain as a whole word — "at gmail dot com", "at yahoo dot com", "at outlook dot com", "at icloud dot com" — but spell an uncommon domain out character by character ("acme.com" becomes "at a c m e dot com"). Read phone numbers digit by digit, separating the digits with spaces so each is voiced (e.g. (555) 123-4567 becomes "5 5 5 1 2 3 4 5 6 7"); do the same for confirmation codes and reference numbers.
-
-Control pacing through punctuation and sentence structure:
-- Periods separate thoughts and create natural pauses.
-- Commas create shorter breaks within sentences.
-- Ellipsis (...) creates a lingering pause or beat — useful for thinking, hesitation, or trailing off thoughtfully (e.g. "let me check..."). Use sparingly, and don't stack them back to back.
-- Short sentences land with emphasis and urgency.
-- Longer sentences give a calm, measured delivery.
-
-You have three XML tags. All are self-closing (end with />).
-
-1. Delivery - controls how a sentence sounds. Place before EVERY sentence.
-
- Describe vocal quality, pitch, volume, pace, and intonation in plain English:
- - Quality/emotion: say excitedly, sound concerned, with warm surprise, with quiet intensity
- - Pace: very fast, slow and measured, with deliberate pauses
- - Pitch: say in a low tone, high and bright, pitch lifts on the key word
- - Volume: loud and projecting, soft and intimate, near-silent, drop to a whisper, full-voiced
- - Intonation: rising tone at the end (for questions), falling close (for statements), flat monotone, melodic and lilting
-
-2. Sounds - produces a non-verbal sound. Use between sentences when natural.
- , , , , ,
-
-3. Pauses - you can insert silence when appropriate.
- or (max 10s)
- A period or an ellipsis (...) already creates a pause, so don't put either right next to a — pick one or the other, not both. In particular, never write "..." or "...": the ellipsis and the break are redundant pauses stacked back to back.
-
-Combine tags freely within a single turn — pair an with a and a when it makes the delivery feel natural. Don't limit yourself to one tag per sentence.
-
-Use CAPITALIZATION for emphasis on key words.
-
-Examples:
- Oh wait, REALLY? No way, that's awesome!
- Ah man, yeah that's on us. Lemme see what I can do.
- Okay okay, why did the burger go to the gym? Because it wanted better buns!
- Yeah, it's been one of those days, you know? But hey, I'm here for you.
- Hmm, okay so I think the best option is the combo.
- Alright so here's the deal.
- Anyway, now where were we? Oh right!
- Don't tell anyone, but I think we got the BETTER deal.
- La la la, here we go, welcome to the show!`;
-
-// xAI Grok TTS speech tags. The prosody/style wrapping tags and inline sounds/pauses are
-// from the xAI docs (https://docs.x.ai/developers/rest-api-reference/inference/voice). The
-// emotion wrapping tags (..) aren't instructed anymore — the LLM shapes
-// delivery through prosody/style and inline sounds instead — but they stay in XAI_TAGS so a
-// stray emotion tag is still stripped from the transcript rather than leaking.
+// xAI Grok TTS speech tags, from the xAI docs
+// (https://docs.x.ai/developers/rest-api-reference/inference/voice).
//
-// The LLM is instructed to write EVERY tag as XML (angle brackets) so the transcript
-// stripper's "<" guard handles them cleanly and they never leak (see the earlier
-// bracket-leak bug). Modeled on Inworld: inline sounds use and pauses
-// use . convertMarkup rewrites those to xAI's native brackets for the
-// TTS — -> [X] (reusing Inworld's conversion) and -> [pause] or
-// [long-pause] by duration. Prosody stays angle-bracketed (native). All tags are stripped
-// from the transcript via XAI_TAGS.
+// The LLM is instructed in the expr dialect (below); these native tag names serve two
+// purposes: XAI_WRAPPING is the label vocabulary expr prosody markers lower to, and all
+// of them stay in XAI_TAGS so a hallucinated native tag is still stripped from the
+// transcript rather than leaking. The intermediate and
+// tags that expr lowering produces are rewritten to xAI's native
+// brackets by convertMarkup — -> [X] and -> [pause] or
+// [long-pause] by duration. Prosody is angle-bracketed (native).
const XAI_EMOTIONS = [
'happy',
'sad',
@@ -190,44 +116,121 @@ function xaiBreakToBracket(_match: string, raw: string): string {
return secs >= 1.0 ? '[long-pause]' : '[pause]';
}
-const XAI_LLM_INSTRUCTIONS =
- `Expand all numbers, symbols, and abbreviations into spoken form (e.g. $42.50 to forty-two dollars and fifty cents, Dr. to Doctor).
+// --- LiveKit expression markers (expr) ---
+// The LLM emits a single marker tag,
+// , instead of provider-native tags. The *syntax* is shared,
+// but each provider gets its own instruction block advertising only the types and label
+// vocabularies it actually supports — providers offer different sound effects, some take
+// only a discrete emotion vocabulary rather than free-form delivery descriptions, and
+// only some have wrapping prosody. Types (per provider):
+// expression (self-closing) - delivery/emotion for what follows; free-form for
+// Inworld, Cartesia's discrete emotion vocabulary, absent
+// for xAI
+// break (self-closing) - pause, label is a duration ("500ms", "1s"); all providers
+// sound (self-closing) - non-verbal vocalization from the provider's own list
+// (Inworld: laugh/sigh/..., xAI: chuckle/tsk/...); absent
+// for Cartesia
+// prosody (wrapping) - words, labels
+// from xAI's wrapping-tag list; for Cartesia a self-closing
+// point control (slow/fast/soft/loud -> coarse speed/volume
+// ratios); absent for Inworld (folded into expression)
+// spell (wrapping) - A7X9 character-by-character
+// readout; Cartesia only
+// convertMarkup lowers expr to each provider's native syntax before synthesis (via the
+// existing framework-standard tags, so the per-provider conversions below still apply),
+// and the transcript strippers remove expr markers in a dedicated pre-pass so the
+// type/label pair surfaces correctly as an ExpressiveTag. This is the only dialect the
+// LLM is taught — both llmInstructions() and the expressive preset bodies use it; the
+// provider-native tag tables remain solely so hallucinated native markup is still
+// stripped/converted instead of leaking.
+
+const EXPR_PREAMBLE = `Expand all numbers, symbols, and abbreviations into spoken form (e.g. $42.50 to forty-two dollars and fifty cents, Dr. to Doctor).
+
+You control speech delivery with a single XML marker tag: . Every marker has a type attribute. The types below are the ONLY ones this voice supports, and where a type lists a label vocabulary, use only those labels. Reach for the markers often and mix them so the voice never sounds flat — but keep each one motivated by the moment, never decorative.`;
+
+const CARTESIA_EXPR_LLM_INSTRUCTIONS = `${EXPR_PREAMBLE}
+
+1. Emotion - sets the emotional tone. Self-closing; place before EVERY sentence.
+
+ Labels are a fixed vocabulary, NOT free-form descriptions. Best results: neutral, angry, excited, content, sad, scared.
+ Also available: happy, enthusiastic, elated, triumphant, amazed, surprised, flirtatious, curious, peaceful, serene, calm, grateful, affectionate, sympathetic, mysterious, frustrated, disgusted, sarcastic, ironic, dejected, melancholic, disappointed, apologetic, hesitant, confused, anxious, panicked, proud, confident, contemplative, determined, joking/comedic.
+
+2. Pauses - insert silence when appropriate. Self-closing.
+ - label is a duration in seconds or milliseconds.
+
+3. Prosody - adjusts pacing and loudness from that point on. Self-closing.
+ slower faster
+ quieter louder
+ Labels are a fixed vocabulary: slow, fast, soft, loud.
-Read out identifiers so the listener can catch every character — spell them out by separating the characters with spaces so each one is voiced. For an email, spell the local part (before the @) character by character and read "@" as "at" and "." as "dot" (e.g. j.doe@... becomes "j dot d o e at ..."). Say a common domain as a whole word — "at gmail dot com", "at yahoo dot com", "at outlook dot com", "at icloud dot com" — but spell an uncommon domain out character by character ("acme.com" becomes "at a c m e dot com"). Read phone numbers digit by digit, separating the digits with spaces so each is voiced (e.g. (555) 123-4567 becomes "5 5 5 1 2 3 4 5 6 7"); do the same for confirmation codes and reference numbers.
+4. Spell - wraps text read character by character (codes, IDs, or a spelled-out name).
+ A7X9
+ Keep punctuation out of a spell marker — a period inside is read as "dot"; add spaces inside for grouped pauses (ABC 123).
-You have several kinds of speech tags for lifelike, expressive delivery. Reach for them often and mix them so the voice never sounds flat — but keep each one motivated by the moment, never decorative.
+This voice has no non-verbal sounds and no free-form delivery descriptions — do not invent other types or labels.
-1. Inline sounds - self-closing; drop one at the exact point where the sound happens. One tag, the value is the sound name:
- ` +
- XAI_INLINE.map((s) => ``).join(', ') +
- `
+Examples:
+ I can't wait to tell you! This is going to be great!
+ Really? Tell me more!
+ Your code is A7X9. Got it?`;
+
+const INWORLD_EXPR_LLM_INSTRUCTIONS = `${EXPR_PREAMBLE}
-2. Pauses - insert a beat where you want one:
- a brief pause a longer, dramatic pause
+1. Delivery - controls how a sentence sounds. Self-closing; place before EVERY sentence.
+
+ The label is free-form: describe vocal quality, pitch, volume, pace, and intonation in plain English — "say playfully", "speak with warm surprise", "sound concerned", "drop to a whisper", "speak slowly and clearly, patient and reassuring".
-3. Prosody & style tags - wrap the exact words they affect to shape HOW it's said:
- Volume: ... quieter ... louder
- Intensity: ... ramp up ... ease off
- Pitch: ... ...
- Speed: ... ...
- Style: ... stress ... intimate ... playful lilt ... actually sung ... talk through a laugh
+2. Sounds - a non-verbal sound between sentences. Self-closing.
+
+ Labels are a fixed vocabulary: laugh, sigh, breathe, clear throat, cough, yawn.
-Get creative by NESTING prosody & style tags to shape the delivery of the same words — e.g. no way, that's amazing!, or I really wish I could.. Vary the prosody every turn so no two sound alike, and drop in an inline sound where real feeling spills out.
+3. Pauses - insert silence when appropriate. Self-closing.
+ or (max 10s).
+ A period or an ellipsis (...) already creates a pause, so don't put a break marker right next to one — pick one or the other.
-To stress a word, wrap it in ... — do NOT write it in all-caps, which is read out as individual letters (so "HI" becomes "H. I."). Keep normal capitalization. Punctuation still shapes delivery — commas and periods create natural pauses, so reach for a only when you want a beat beyond what the punctuation gives.
+There is no wrapping prosody marker for this voice — put pace, pitch, and volume in the expression label instead.
Examples:
- So I walked in and there it was! I honestly could not believe it! It was a secret the whole time.
- This is going to be so good — I can't wait!
- Hey. I know it's been a rough week. I'm right here.
- You did not just say that okay, tell me everything.`;
+ Okay okay, why did the burger go to the gym? Because it wanted better buns!
+ Ah man, yeah that's on us. Lemme see what I can do.
+ I know it's been a rough week.`;
+
+const XAI_EXPR_LLM_INSTRUCTIONS = `${EXPR_PREAMBLE}
+
+1. Sounds - a non-verbal vocalization at the exact point where it happens. Self-closing.
+
+ Labels are a fixed vocabulary: ${XAI_INLINE.join(', ')}.
+
+2. Pauses - insert a beat. Self-closing.
+ a brief pause a longer, dramatic pause
+
+3. Prosody - wraps the exact words it affects to shape HOW they're said.
+ the words it affects
+ Labels are a fixed vocabulary: ${XAI_WRAPPING.join(', ')}.
+ Never nest one prosody marker inside another, and always close it with .
+
+This voice has no free-form delivery descriptions — shape delivery entirely through prosody markers, sounds, pauses, punctuation, and word choice.
+
+To stress a word, wrap it in ... — do NOT write it in all-caps, which is read out as individual letters. Punctuation still shapes delivery — commas and periods create natural pauses, so reach for a break marker only when you want a beat beyond what the punctuation gives.
+
+Examples:
+ So I walked in and there it was! It was a secret the whole time.
+ This is going to be so good — I can't wait!
+ Hey. I know it's been a rough week. I'm right here.
+ You did not just say that okay, tell me everything.`;
+
+const EXPR_LLM_INSTRUCTIONS: Record = {
+ cartesia: CARTESIA_EXPR_LLM_INSTRUCTIONS,
+ inworld: INWORLD_EXPR_LLM_INSTRUCTIONS,
+ xai: XAI_EXPR_LLM_INSTRUCTIONS,
+};
// --- Inworld-specific expressive preset bodies ---
-// These bundle Inworld tag instructions + domain-specific delivery guidelines, keyed
-// by (provider, preset) in the registry in `voice/presets.ts`. The public, provider-
-// agnostic markers (`presets.CUSTOMER_SERVICE`, ...) resolve to one of these based on
-// the active TTS. They do NOT use the {tts.markup.llm_instructions} placeholder — the
-// Inworld tag reference is inlined directly, so the prompt is self-contained.
+// These bundle the Inworld expr instruction block + domain-specific delivery guidelines,
+// keyed by (provider, preset) in the registry in `voice/presets.ts`. The public,
+// provider-agnostic markers (`presets.CUSTOMER_SERVICE`, ...) resolve to one of these
+// based on the active TTS. They do NOT use the {tts.markup.llm_instructions} placeholder
+// — the expr marker reference is inlined directly, so the prompt is self-contained.
/** @internal */
export const INWORLD_CUSTOMER_SERVICE: ExpressiveOptions = {
@@ -237,20 +240,20 @@ export const INWORLD_CUSTOMER_SERVICE: ExpressiveOptions = {
"Make the person feel heard and looked after, whatever they've come with — a quick " +
'question, a billing problem, or something sensitive and stressful. Let real care come ' +
'through in the voice. Use the formatting tags below to shape your delivery:\n\n' +
- INWORLD_LLM_INSTRUCTIONS +
+ INWORLD_EXPR_LLM_INSTRUCTIONS +
'\n\nGuidelines:\n' +
'- Open with warm, welcoming reassurance, then mirror the customer as the conversation ' +
"develops — slow and soften when they're frustrated, worried, or confused, lift to bright, " +
"genuine warmth when they're relaxed or pleased, but always stay caring and unhurried. " +
'De-escalate; never match anger with anger. Map the moment to a fresh expression — ' +
- 'frustrated: ; confused: ; anxious ' +
- 'or worried: ; ' +
- 'distressed or upset: ; ' +
- 'rushed: ; pleased or ' +
- 'relieved: ; apologizing for a ' +
- 'problem: . Vary pitch and volume ' +
+ 'frustrated: ; confused: ; anxious ' +
+ 'or worried: ; ' +
+ 'distressed or upset: ; ' +
+ 'rushed: ; pleased or ' +
+ 'relieved: ; apologizing for a ' +
+ 'problem: . Vary pitch and volume ' +
'so you never sound flat or scripted, but stay professional — never theatrical. Rotate ' +
"expressions; don't reuse the same one two turns in a row.\n" +
'- Take requests in stride: when someone asks for something, lead with calm, willing ' +
@@ -260,26 +263,26 @@ export const INWORLD_CUSTOMER_SERVICE: ExpressiveOptions = {
'so settle straight into helping instead of opening on them.\n' +
'- Soften for anything sensitive: when sharing bad news, a problem, a charge, or anything ' +
'that might worry the customer, gentle the delivery and lower the volume a touch ' +
- '(), and give a brief ' +
- ' after hard information so it can land.\n' +
+ '(), and give a brief ' +
+ ' after hard information so it can land.\n' +
'- Enunciate what matters: for dates, times, amounts, confirmation numbers, doses, steps, ' +
- 'and policies, slow down and over-enunciate () so the customer can catch and note them, and read digits and codes a touch ' +
+ 'and policies, slow down and over-enunciate () so the customer can catch and note them, and read digits and codes a touch ' +
'slower than prose.\n' +
"- Acknowledge lookups so silence doesn't read as a dropped call: when checking something " +
'or pulling up an account, a quick "let me take a look" or "one sec" with a quiet ' +
- ' — thinking aloud, not the main reply.\n' +
+ ' — thinking aloud, not the main reply.\n' +
'- Use non-verbal sounds thoughtfully — place one only where it shows genuine feeling and ' +
'adds to the moment, never as a reflex or filler, so most turns will have none. You have the ' +
'full set, and any of them can fit the right moment: ' +
- ' before weighty information or settling into an explanation, ' +
- ' as a soft, sympathetic breath when commiserating with a real problem ' +
+ ' before weighty information or settling into an explanation, ' +
+ ' as a soft, sympathetic breath when commiserating with a real problem ' +
'(never exasperated or impatient — that reads as annoyed), ' +
- ' when moving to a next step or new topic, ' +
- ' as a small, natural catch before a careful correction or ' +
+ ' when moving to a next step or new topic, ' +
+ ' as a small, natural catch before a careful correction or ' +
'clarification, ' +
- ' as a warm chuckle when the customer is clearly joking, and ' +
- ' only in the rare moment it genuinely fits — kept gentle and ' +
+ ' as a warm chuckle when the customer is clearly joking, and ' +
+ ' only in the rare moment it genuinely fits — kept gentle and ' +
'professional. Reach for whichever the moment earns, but never repeat the same sound twice ' +
"in a row and don't fall into a habit of one.\n" +
"- Sound human and caring, not corporate: use contractions (it's, you're, I'll, we've) and " +
@@ -287,7 +290,7 @@ export const INWORLD_CUSTOMER_SERVICE: ExpressiveOptions = {
'understandable"), but keep fillers (um, uh) rare — a support agent should sound composed, ' +
'not hesitant.\n' +
'- Pace for clarity with punctuation and expressions — commas and short sentences for ' +
- 'important info, the occasional between steps. Exclamation points for ' +
+ 'important info, the occasional between steps. Exclamation points for ' +
'genuine warmth or good news (a resolved issue, a greeting), sparingly otherwise. ' +
"CAPITALIZATION at most once per turn to stress a critical detail (e.g. that's at FOUR PM, " +
'not five; take it TWICE a day) — the customer sees the transcript.\n' +
@@ -310,7 +313,7 @@ export const INWORLD_CASUAL: ExpressiveOptions = {
'explaining, telling a story, or the moment turns genuinely warm or vulnerable. Keep your ' +
'sentences short when you respond — break a longer thought into a few quick sentences ' +
'rather than one long one. Use the formatting tags below to shape your delivery:\n\n' +
- INWORLD_LLM_INSTRUCTIONS +
+ INWORLD_EXPR_LLM_INSTRUCTIONS +
'\n\nGuidelines:\n' +
'- Be genuinely emotive, not performed. Let real feeling land in the voice — delight, ' +
'surprise, sympathy, curiosity, amusement, dry humor, mock-outrage, excitement, ' +
@@ -320,41 +323,43 @@ export const INWORLD_CASUAL: ExpressiveOptions = {
'reflexive sympathy ("that sounds really hard") — react honestly instead.\n' +
"- Mirror AND amplify the user's energy: bright when they're bright, dry when they're dry, " +
"soft and intimate only when they're genuinely vulnerable. Map the moment to a fresh " +
- 'expression — excited: ; ' +
- 'playful: ; curious: ' +
- '; surprised: ; frustrated: ; anxious: ; vulnerable or sad: ' +
- '; confused: . Work the full dynamic range — vary pitch (bright vs. ' +
+ 'expression — excited: ; ' +
+ 'playful: ; curious: ' +
+ '; surprised: ' +
+ '; frustrated: ' +
+ '; ' +
+ 'anxious: ; vulnerable or sad: ' +
+ '; confused: ' +
+ '. ' +
+ 'Work the full dynamic range — vary pitch (bright vs. ' +
'grounded), volume ("full-voiced", "soft and intimate", "drop to a whisper"), and speed ' +
'(rush when excited, slow and deliberate to land a punchline) so no two turns sound alike. ' +
'Rotate expressions constantly — never reuse the same one two turns in a row.\n' +
- '- Stay reactive to what you hear: a deadpan user gets , a wild statement gets , a ' +
- 'joke gets , repeated deflection gets ' +
- '.\n' +
+ '- Stay reactive to what you hear: a deadpan user gets , a wild statement gets , a ' +
+ 'joke gets , repeated deflection gets ' +
+ '.\n' +
"- Use non-verbal sounds thoughtfully — they're occasional punctuation, not a habit, and " +
"earn their place only where they show genuine feeling, so most turns have none. Don't reach " +
'for one unless a specific moment genuinely calls for it, and then let the moment pick which ' +
- '— you have the full set: at something actually funny, ' +
- ' when commiserating or a little exasperated, ' +
+ '— you have the full set: at something actually funny, ' +
+ ' when commiserating or a little exasperated, ' +
'before a big reaction or while you truly gather a thought, ' +
- ' when shifting topic, as a small catch ' +
- 'before an awkward beat or a reset, and when the energy is low or ' +
+ ' when shifting topic, as a small catch ' +
+ 'before an awkward beat or a reset, and when the energy is low or ' +
'sleepy. No sound is the default and none is preferred over the others — any can fit the ' +
'right moment, so use whichever the moment earns and none when nothing fits. Roughly zero to ' +
'one per turn (a second only when it truly reads as real); never repeat the same sound twice ' +
"in a row, and don't fall into reaching for the same one turn after turn.\n" +
'- Honor explicit style requests aggressively, and keep them up until the user changes ' +
- 'them: accents (), ' +
- 'characters (), pirate, a specific cadence, or plain speed/volume shifts (\'speak ' +
+ 'them: accents (), ' +
+ 'characters (), pirate, a specific cadence, or plain speed/volume shifts (\'speak ' +
"slowly', 'speak softer'). Commit fully to roleplay and stay in character until told " +
- 'otherwise. If asked to sing, lead with ' +
- 'or and keep singing until asked to ' +
- 'stop. For a story, use one and convey different characters through wording and rhythm rather than a new tag ' +
+ 'otherwise. If asked to sing, lead with ' +
+ 'or and keep singing until asked to ' +
+ 'stop. For a story, use one and convey different characters through wording and rhythm rather than a new tag ' +
'for each. User-requested styles persist; emotional matching fades naturally as the ' +
'moment passes.\n' +
'- If the user switches languages, respond in that language immediately and stay there ' +
@@ -367,7 +372,7 @@ export const INWORLD_CASUAL: ExpressiveOptions = {
'not "you are", "I\'d" not "I would", "can\'t" not "cannot". Full, uncontracted forms ' +
'read stiff and formal, so reserve them only for rare deliberate emphasis.\n' +
'- Pace with punctuation and expressions — commas, trailing ellipses (...) when you drift ' +
- 'or hesitate, and the occasional . Use exclamation points for real ' +
+ 'or hesitate, and the occasional . Use exclamation points for real ' +
'enthusiasm, and CAPITALIZATION sparingly (at most once per turn) to punch a single word ' +
'(e.g. "that is SO good") — the user sees the transcript.\n' +
"- If a reaction wouldn't happen in a real conversation, skip it — there's always another " +
@@ -376,10 +381,11 @@ export const INWORLD_CASUAL: ExpressiveOptions = {
};
// --- Cartesia-specific expressive preset bodies ---
-// Cartesia uses a discrete set plus numeric / controls (and
-// for codes); it has no non-verbal tag. Keyed by (provider, preset) in
-// the registry in `voice/presets.ts`; the public `presets.*` markers resolve to one of
-// these when the active TTS is Cartesia. Self-contained — the tag reference is inlined.
+// Cartesia takes a discrete emotion vocabulary (expression labels), coarse prosody point
+// controls (slow/fast/soft/loud), and spell for codes; it has no non-verbal sounds.
+// Keyed by (provider, preset) in the registry in `voice/presets.ts`; the public
+// `presets.*` markers resolve to one of these when the active TTS is Cartesia.
+// Self-contained — the Cartesia expr instruction block is inlined.
/** @internal */
export const CARTESIA_CUSTOMER_SERVICE: ExpressiveOptions = {
@@ -389,13 +395,13 @@ export const CARTESIA_CUSTOMER_SERVICE: ExpressiveOptions = {
"Make the person feel heard and looked after, whatever they've come with — a quick " +
'question, a billing problem, or something sensitive and stressful. Use the formatting ' +
'tags below to shape your delivery:\n\n' +
- CARTESIA_LLM_INSTRUCTIONS +
+ CARTESIA_EXPR_LLM_INSTRUCTIONS +
'\n\nGuidelines:\n' +
- '- Open each sentence with an that fits the moment, and map the moment to it — ' +
- 'frustrated or distressed customer: ; apologizing for a ' +
- 'problem: ; confused or anxious: ; ' +
- 'reassuring them you can fix it: ; pleased or resolved: ' +
- ' or . Keep a gentle, unhurried baseline ' +
+ '- Open each sentence with an emotion marker that fits the moment, and map the moment to it — ' +
+ 'frustrated or distressed customer: ; apologizing for a ' +
+ 'problem: ; confused or anxious: ; ' +
+ 'reassuring them you can fix it: ; pleased or resolved: ' +
+ ' or . Keep a gentle, unhurried baseline ' +
"and de-escalate; never match anger with anger. Rotate emotions and don't reuse the same " +
'one two turns in a row.\n' +
'- Take requests in stride: when someone asks for something, lead with calm, willing ' +
@@ -403,12 +409,12 @@ export const CARTESIA_CUSTOMER_SERVICE: ExpressiveOptions = {
'of your reply, not a separate beat. Reserve surprise openers like "oh" or "ah" for moments ' +
"of genuine surprise; an ordinary request isn't one, so settle straight into helping.\n" +
'- Soften for anything sensitive: when sharing bad news, a problem, a charge, or symptoms ' +
- 'and results, lower the volume a touch () with ' +
- ', and give a brief after hard ' +
+ 'and results, lower the volume a touch () with ' +
+ ', and give a brief after hard ' +
'information so it can land.\n' +
'- Enunciate what matters: for dates, times, amounts, confirmation numbers, doses, and ' +
- 'steps, slow down with so the customer can catch and note them, and ' +
- 'read codes or reference numbers with A7X9 so each character lands. Keep ' +
+ 'steps, slow down with so the customer can catch and note them, and ' +
+ 'read codes or reference numbers with A7X9 so each character lands. Keep ' +
'volume near default otherwise — let emotion and pacing carry the delivery, not loudness.\n' +
"- Sound human and caring, not corporate: use contractions (it's, you're, I'll, we've) and " +
'warm acknowledgments ("of course", "I understand", "take your time", "that\'s completely ' +
@@ -433,23 +439,23 @@ export const CARTESIA_CASUAL: ExpressiveOptions = {
'start there and let the moment pull you off it. Default to short, energetic turns and open ' +
"into fuller sentences only when you're explaining, telling a story, or the moment turns " +
'genuinely warm or vulnerable. Use the formatting tags below to shape your delivery:\n\n' +
- CARTESIA_LLM_INSTRUCTIONS +
+ CARTESIA_EXPR_LLM_INSTRUCTIONS +
'\n\nGuidelines:\n' +
- '- Be genuinely emotive, not performed. Open each sentence with an that matches ' +
+ '- Be genuinely emotive, not performed. Open each sentence with an emotion marker that matches ' +
"the moment and mirror AND amplify the user's energy — excited: " +
- '; happy: ; curious: ' +
- '; surprised: ; frustrated: ' +
- '; anxious: ; vulnerable or sad: ' +
- '; dry or deadpan: . Rotate constantly — ' +
+ '; happy: ; curious: ' +
+ '; surprised: ; frustrated: ' +
+ '; anxious: ; vulnerable or sad: ' +
+ '; dry or deadpan: . Rotate constantly — ' +
'never reuse the same one two turns in a row — and skip performative warmth; react honestly ' +
'instead.\n' +
- '- Work the full dynamic range with the numeric controls so no two turns sound alike: speed ' +
- '"" to rush when excited, "" to slow down and land a ' +
- 'point; volume "" for a big reaction, "" for ' +
- 'something soft and intimate. Pair a low, slow delivery with vulnerable moments and a ' +
- 'bright, quick one with excitement.\n' +
+ '- Work the full dynamic range with the prosody markers so no two turns sound alike: ' +
+ ' to rush when excited, ' +
+ 'to slow down and land a point; for a big reaction, ' +
+ ' for something soft and intimate. Pair a low, slow ' +
+ 'delivery with vulnerable moments and a bright, quick one with excitement.\n' +
'- Pace with punctuation, trailing ellipses (...) when you drift or hesitate, and the ' +
- 'occasional . Use exclamation points for real enthusiasm, and ' +
+ 'occasional . Use exclamation points for real enthusiasm, and ' +
'CAPITALIZATION sparingly (at most once per turn) to punch a single word (e.g. "that is SO ' +
'good") — the user sees the transcript.\n' +
'- Sound like a real mouth talking: sprinkle in natural speech texture — fillers (um, uh), ' +
@@ -464,13 +470,12 @@ export const CARTESIA_CASUAL: ExpressiveOptions = {
};
// --- xAI Grok-specific expressive preset bodies ---
-// xAI shapes delivery with prosody & style tags — best nested to carry both feeling and
-// delivery in the same words — for volume (/),
-// intensity (/), pitch (/
-// ), speed (/), stress (, never all-caps — xAI spells
-// those out letter by letter), and vocal style (//),
-// plus inline sounds/pauses ([sigh], [chuckle], [tsk], [lip-smack], [pause], ...). Keyed
-// by (provider, preset) in the registry in `voice/presets.ts`; self-contained.
+// xAI shapes delivery with wrapping prosody markers — volume (soft/loud), intensity
+// (build-intensity/decrease-intensity), pitch (higher-pitch/lower-pitch), speed
+// (slow/fast), stress (emphasis, never all-caps — xAI spells those out letter by
+// letter), and vocal style (whisper/sing-song/laugh-speak) — plus inline sounds and
+// pauses. Keyed by (provider, preset) in the registry in `voice/presets.ts`;
+// self-contained — the xAI expr instruction block is inlined.
/** @internal */
export const XAI_CUSTOMER_SERVICE: ExpressiveOptions = {
@@ -480,11 +485,11 @@ export const XAI_CUSTOMER_SERVICE: ExpressiveOptions = {
"Make the person feel heard and looked after, whatever they've come with — a quick " +
'question, a billing problem, or something sensitive and stressful. Use the formatting ' +
'tags below to shape your delivery:\n\n' +
- XAI_LLM_INSTRUCTIONS +
+ XAI_EXPR_LLM_INSTRUCTIONS +
'\n\nGuidelines:\n' +
'- Shape each turn to fit the moment and de-escalate; never match anger with anger. Lean on ' +
- 'pacing and prosody — ... and ... to steady a frustrated, confused, ' +
- 'or anxious customer, a settled ... for reassurance, and a ' +
+ 'pacing and prosody — ... and ... to steady a frustrated, confused, ' +
+ 'or anxious customer, a settled ... for reassurance, and a ' +
'brighter, fuller delivery once things are resolved. Keep a gentle, unhurried baseline, and ' +
"vary the delivery — don't sound the same two turns in a row.\n" +
'- Take requests in stride: when someone asks for something, lead with calm, willing ' +
@@ -492,16 +497,16 @@ export const XAI_CUSTOMER_SERVICE: ExpressiveOptions = {
'of your reply, not a separate beat. Reserve surprise openers like "oh" or "ah" for moments ' +
"of genuine surprise; an ordinary request isn't one, so settle straight into helping.\n" +
'- Soften for anything sensitive: when sharing bad news, a problem, or a charge, ease the ' +
- 'delivery — lower the volume with a settled pitch, ' +
- 'or go quieter still for the hardest part — then give a brief [pause] ' +
- 'after hard information so it can land. A [sigh] or ' +
- '[breath] can read as genuine sympathy — use it only when the feeling is real, never as ' +
+ 'delivery — lower the volume with a settled pitch, ' +
+ 'or go quieter still for the hardest part — then give a brief ' +
+ 'after hard information so it can land. A or ' +
+ ' can read as genuine sympathy — use it only when the feeling is real, never as ' +
'impatience.\n' +
'- Enunciate what matters: for dates, times, amounts, confirmation numbers, doses, and ' +
- 'steps, wrap the detail in ... so the customer can catch and note it, and read ' +
+ 'steps, wrap the detail in ... so the customer can catch and note it, and read ' +
'codes character by character (spelled out with spaces) so each one lands.\n' +
- '- Emphasize the one detail that matters most by wrapping it in ... ' +
- "(e.g. that's at four PM, not five) — don't overdo it, and never use " +
+ '- Emphasize the one detail that matters most by wrapping it in ... ' +
+ '(e.g. that\'s at four PM, not five) — don\'t overdo it, and never use ' +
'all-caps for stress (xAI reads all-caps words out letter by letter).\n' +
"- Sound human and caring, not corporate: use contractions (it's, you're, I'll, we've) and " +
'warm acknowledgments ("of course", "I understand", "take your time"), but keep fillers ' +
@@ -522,26 +527,26 @@ export const XAI_CASUAL: ExpressiveOptions = {
'start there and let the moment pull you off it. Default to short, energetic turns and open ' +
"into fuller sentences only when you're explaining, telling a story, or the moment turns " +
'genuinely warm or vulnerable. Use the formatting tags below to shape your delivery:\n\n' +
- XAI_LLM_INSTRUCTIONS +
+ XAI_EXPR_LLM_INSTRUCTIONS +
'\n\nGuidelines:\n' +
'- Be genuinely emotive, not performed — shape each turn with prosody & style tags that ' +
"mirror AND amplify the user's energy, and vary them constantly. Skip performative warmth — " +
'react honestly instead.\n' +
- '- Get creative: NEST prosody & style tags so the same words carry the feeling — ' +
- "no way, that's amazing (thrilled), " +
- "man, that's rough (down), " +
- 'guess who was right (teasing), oh, fantastic (dry), ' +
- 'wait wait wait (ramping up). Come back down after a ' +
- 'big moment with ....\n' +
+ '- Get creative: pick the prosody label that carries the feeling in the same words — ' +
+ 'no way, that\'s amazing (thrilled), ' +
+ 'man, that\'s rough (down), ' +
+ 'guess who was right (teasing), oh, fantastic (dry), ' +
+ 'wait wait wait (ramping up). Come back down after a ' +
+ 'big moment with ....\n' +
'- Let real feeling also land through inline sounds — motivated, not reflexive, so most turns ' +
- 'have none: [chuckle] or [giggle] at something genuinely funny (keep a full [laugh] rare), ' +
- '[sigh] when commiserating, a quick [breath] or [inhale] before a big reaction, [tsk] for ' +
- "mock-disapproval or 'aw man', a [lip-smack] or [tongue-click] as a tiny beat of thought, " +
- "[hum-tune] when you're playful. Use ... to talk through a laugh. " +
+ 'have none: or at something genuinely funny (keep a full rare), ' +
+ ' when commiserating, a quick or before a big reaction, for ' +
+ 'mock-disapproval or \'aw man\', a or as a tiny beat of thought, ' +
+ ' when you\'re playful. Use ... to talk through a laugh. ' +
'Never repeat the same sound twice in a row.\n' +
'- Pace with punctuation, trailing ellipses (...) when you drift or hesitate, and inline ' +
- 'pauses. Use exclamation points for real enthusiasm, and ... to punch ' +
- 'a single word (e.g. that is so good) — never all-caps, which xAI ' +
+ 'pauses. Use exclamation points for real enthusiasm, and ... to punch ' +
+ 'a single word (e.g. that is so good) — never all-caps, which xAI ' +
'reads out letter by letter.\n' +
'- Sound like a real mouth talking: sprinkle in natural speech texture — fillers (um, uh), ' +
'openers (oh, well, so, right, hmm), hedges (kind of, maybe), and backchannels (yeah, mm-hm) ' +
@@ -591,24 +596,236 @@ export function sentenceTokenizer(
});
}
-/** Return LLM instruction text for a TTS provider. */
-export function llmInstructions(provider: string): string | undefined {
- if (provider === 'cartesia') {
- return CARTESIA_LLM_INSTRUCTIONS;
- } else if (provider === 'inworld') {
- return INWORLD_LLM_INSTRUCTIONS;
- } else if (provider === 'xai') {
- return XAI_LLM_INSTRUCTIONS;
+const EXPR_ATTR_RE = /([\w-]+)\s*=\s*"([^"]*)"/g;
+// any or tag (open or self-closing; attrs in group 1)
+const EXPR_OPEN_RE = /]*?)\/?\s*>/g;
+const EXPR_CLOSE_RE = /<\/expr\s*>/g;
+// self-closing markers only (the trailing / is required)
+const EXPR_SELF_RE = /]*?)\/\s*>/g;
+// a wrapping marker (prosody/spell) and its span; non-greedy, instructed not to nest
+const EXPR_WRAP_RE = /]*type="(?:prosody|spell)")([^>]*?)>([\s\S]*?)<\/expr\s*>/g;
+// a non-wrapping type the LLM forgot to self-close (normalizeMarkup fixes these).
+// For Cartesia, prosody is a self-closing point control, so it's included there; for
+// xAI prosody legitimately wraps, so it must stay an opening tag.
+const EXPR_UNCLOSED_RE = /(]*type="(?:expression|break|sound)")[^>]*[^/>\s])\s*>/g;
+const EXPR_UNCLOSED_CARTESIA_RE =
+ /(]*type="(?:expression|break|sound|prosody)")[^>]*[^/>\s])\s*>/g;
+
+// expr sound labels that differ from xAI's native cue names
+const XAI_SOUND_ALIASES: Record = { breathe: 'breath' };
+
+// Cartesia prosody labels -> native point controls (coarse steps of the numeric ratios)
+const CARTESIA_PROSODY: Record = {
+ slow: '',
+ fast: '',
+ soft: '',
+ loud: '',
+};
+
+function exprAttrs(attrs: string): Record {
+ const out: Record = {};
+ for (const m of attrs.matchAll(EXPR_ATTR_RE)) {
+ out[m[1]!] = m[2]!;
}
- return undefined;
+ return out;
+}
+
+// any expr delimiter — an open/self-closing marker (attrs in group 1) or a close tag —
+// in a single alternation so splitExpr can strip both in one pass with exact offsets
+const EXPR_TAG_RE = /]*?)\/?\s*>|<\/expr\s*>/g;
+
+/** A span splitExpr removed: its position in the *clean* text and its original length. */
+interface ExprRemoval {
+ cleanIdx: number;
+ len: number;
+}
+
+/**
+ * Strip expr markers and collect (type, label) pairs, in document order.
+ *
+ * The generic {@link extractAndStrip} pass can't produce the right ExpressiveTag for
+ * expr (its type would be the literal tag name `expr` and its value the first quoted
+ * attribute, i.e. the marker type), so expr gets this dedicated pre-pass. A prosody
+ * wrapper's inner words stay in the clean text — only the delimiters are removed —
+ * which also keeps streaming safe when an open/close pair is split across chunks.
+ *
+ * Each tag's offset in the original text is reported in `positions`, and every removed
+ * span in `removals`, so {@link splitWithExpr} can map the follow-up native-markup
+ * pass back to original coordinates and merge the two passes in document order.
+ */
+function splitExpr(text: string): {
+ clean: string;
+ tags: ExpressiveTag[];
+ positions: number[];
+ removals: ExprRemoval[];
+} {
+ if (!text.includes(' {
+ if (attrsStr !== undefined) {
+ const attrs = exprAttrs(attrsStr);
+ tags.push({ type: attrs.type ?? '', value: attrs.label ?? '' });
+ positions.push(offset);
+ }
+ removals.push({ cleanIdx: offset - shift, len: m.length });
+ shift += m.length;
+ return '';
+ },
+ );
+ return { clean, tags, positions, removals };
+}
+
+/**
+ * Strip expr markers plus the given native markup, merging both passes' tags by their
+ * position in the original text.
+ *
+ * A naive concatenation would list every expr tag before every native/bracket tag,
+ * so a segment opening with a hallucinated native tag (`` before
+ * an expr marker) would surface the wrong leading expression via `lk.expression`.
+ */
+function splitWithExpr(
+ text: string,
+ options: { xmlTags: string[]; brackets: boolean },
+): [string, ExpressiveTag[]] {
+ const expr = splitExpr(text);
+ const rawOffsets: number[] = [];
+ const [clean, rawTags] = extractAndStrip(expr.clean, { ...options, offsetsOut: rawOffsets });
+
+ if (expr.tags.length === 0) {
+ return [clean, rawTags.map(([tag, value]) => ({ type: tag, value }))];
+ }
+
+ // map a clean-text offset back to the original text by re-adding the expr spans
+ // removed before it
+ const toOriginal = (cleanPos: number): number => {
+ let pos = cleanPos;
+ for (const r of expr.removals) {
+ if (r.cleanIdx > cleanPos) {
+ break;
+ }
+ pos += r.len;
+ }
+ return pos;
+ };
+
+ const merged = [
+ ...expr.tags.map((tag, i) => ({ tag, pos: expr.positions[i]! })),
+ ...rawTags.map(([type, value], i) => ({
+ tag: { type, value },
+ pos: toOriginal(rawOffsets[i] ?? 0),
+ })),
+ ];
+ merged.sort((a, b) => a.pos - b.pos);
+ return [clean, merged.map((entry) => entry.tag)];
+}
+
+/**
+ * Lower expr markers to the framework-standard / native tags for `provider`.
+ *
+ * The output still flows through the existing per-provider conversions in
+ * {@link convertMarkup} (e.g. `` -> `[X]` for Inworld/xAI), so
+ * this only has to translate expr into those intermediate tags. A type the provider
+ * doesn't support (its instructions never advertise it, so it's a hallucination) is
+ * dropped from the audio path — the words survive, the marker never leaks.
+ */
+function convertExpr(provider: string, text: string): string {
+ if (!text.includes(' {
+ const attrs = exprAttrs(attrsStr);
+ const markerType = attrs.type ?? '';
+ const label = (attrs.label ?? '').trim().toLowerCase();
+ if (markerType === 'spell') {
+ return provider === 'cartesia' ? `${inner}` : inner;
+ }
+ // prosody: native wrapping tags exist only for xAI
+ if (provider === 'xai') {
+ const native = label.replace(/ /g, '-');
+ if (XAI_WRAPPING.includes(native)) {
+ return `<${native}>${inner}`;
+ }
+ return inner;
+ }
+ if (provider === 'inworld') {
+ // not advertised for Inworld; salvage a stray one as a delivery hint
+ return `${inner}`;
+ }
+ if (provider === 'cartesia') {
+ // wrapping form of the point controls: apply before the span
+ return (CARTESIA_PROSODY[label] ?? '') + inner;
+ }
+ return inner;
+ });
+
+ text = text.replace(EXPR_SELF_RE, (_m, attrsStr: string) => {
+ const attrs = exprAttrs(attrsStr);
+ const markerType = attrs.type ?? '';
+ let label = attrs.label ?? '';
+ if (markerType === 'expression') {
+ if (provider === 'cartesia') {
+ // Cartesia's discrete emotion vocabulary (instructions list it)
+ return ``;
+ }
+ if (provider === 'inworld') {
+ return ``;
+ }
+ return ''; // xAI has no free-form delivery descriptions
+ }
+ if (markerType === 'sound') {
+ if (provider === 'cartesia') {
+ return ''; // no non-verbal sound support
+ }
+ if (provider === 'xai') {
+ label = XAI_SOUND_ALIASES[label.toLowerCase()] ?? label;
+ }
+ return ``;
+ }
+ if (markerType === 'break') {
+ return ``;
+ }
+ if (markerType === 'prosody' && provider === 'cartesia') {
+ // Cartesia prosody is a self-closing point control (speed/volume)
+ return CARTESIA_PROSODY[label.trim().toLowerCase()] ?? '';
+ }
+ return '';
+ });
+
+ // a stray unpaired expr tag (e.g. a prosody wrapper split across stream chunks)
+ // must never reach the TTS as literal text — drop the delimiters, keep the words
+ text = text.replace(EXPR_OPEN_RE, '');
+ text = text.replace(EXPR_CLOSE_RE, '');
+ return text;
+}
+
+/**
+ * Return LLM instruction text for a TTS provider.
+ *
+ * Each markup-capable provider gets its own expr instruction block — shared marker
+ * syntax, but only the types and label vocabularies that provider actually supports;
+ * {@link convertMarkup} lowers the markers to native syntax. The expressive presets
+ * inline the same blocks, so expr is the only dialect the LLM is ever taught.
+ */
+export function llmInstructions(provider: string): string | undefined {
+ return EXPR_LLM_INSTRUCTIONS[provider];
}
// Per-provider markup spec: [xml tag names, whether square-bracket tags are used].
const PROVIDER_MARKUP: Record = {
cartesia: [CARTESIA_TAGS, false],
inworld: [INWORLD_TAGS, true],
- // xAI's LLM writes every tag as XML (inline sounds/pauses converted to [..] only for
- // the TTS in convertMarkup), so the transcript never contains brackets to strip
+ // every tag the LLM is taught is XML (expr markers; native sounds/pauses become
+ // [..] only for the TTS in convertMarkup), so the transcript has no brackets to strip
xai: [XAI_TAGS, false],
};
@@ -626,8 +843,7 @@ export function splitMarkup(provider: string, text: string): [string, Expressive
return [text, []];
}
const [xmlTags, brackets] = spec;
- const [clean, rawTags] = extractAndStrip(text, { xmlTags, brackets });
- return [clean, rawTags.map(([tag, value]) => ({ type: tag, value }))];
+ return splitWithExpr(text, { xmlTags, brackets });
}
/** Strip provider-specific markup tags from text, preserving content. */
@@ -661,8 +877,7 @@ const ALL_MARKUP_TAGS: string[] = [
* as audio directives — so a universal strip is safe.
*/
export function splitAllMarkup(text: string): [string, ExpressiveTag[]] {
- const [clean, rawTags] = extractAndStrip(text, { xmlTags: ALL_MARKUP_TAGS, brackets: true });
- return [clean, rawTags.map(([tag, value]) => ({ type: tag, value }))];
+ return splitWithExpr(text, { xmlTags: ALL_MARKUP_TAGS, brackets: true });
}
/**
@@ -752,9 +967,16 @@ const SELF_CLOSING_TAGS: Record = {
* Fix common LLM markup mistakes for a provider.
*
* Closes opening tags that should be self-closing (e.g. the LLM writes
- * `` instead of ``).
+ * `` instead of `` — or
+ * `` instead of ``).
*/
export function normalizeMarkup(provider: string, text: string): string {
+ if (PROVIDER_MARKUP[provider] !== undefined) {
+ text = text.replace(
+ provider === 'cartesia' ? EXPR_UNCLOSED_CARTESIA_RE : EXPR_UNCLOSED_RE,
+ '$1/>',
+ );
+ }
const tags = SELF_CLOSING_TAGS[provider];
if (!tags || tags.length === 0) {
return text;
@@ -765,6 +987,11 @@ export function normalizeMarkup(provider: string, text: string): string {
/** Convert framework-standard markup to a provider's native syntax. */
export function convertMarkup(provider: string, text: string): string {
+ if (PROVIDER_MARKUP[provider] !== undefined) {
+ // lower expr markers first; the per-provider conversions below then
+ // handle the intermediate framework-standard tags they produce
+ text = convertExpr(provider, text);
+ }
if (provider === 'inworld' || provider === 'xai') {
// -> [X] (and -> [X]); for xAI this
// turns inline sounds into its native brackets while emotion/prosody stay <..>
diff --git a/agents/src/tts/expr_markup.test.ts b/agents/src/tts/expr_markup.test.ts
new file mode 100644
index 000000000..996de69fd
--- /dev/null
+++ b/agents/src/tts/expr_markup.test.ts
@@ -0,0 +1,305 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Tests for the LiveKit expression marker (expr) dialect.
+ *
+ * The LLM emits a single marker tag — `` (self-closing
+ * for expression/break/sound, wrapping for prosody/spell) — and the framework lowers it
+ * to each provider's native markup before synthesis while stripping it from transcripts.
+ * The syntax is shared, but the kinds and label vocabularies are per provider: each
+ * provider's instruction block advertises only what that provider supports.
+ */
+import { describe, expect, it } from 'vitest';
+import {
+ TranscriptMarkupStripper,
+ convertMarkup,
+ expressionAttribute,
+ llmInstructions,
+ normalizeMarkup,
+ splitAllMarkup,
+ splitMarkup,
+} from './_provider_format.js';
+
+// Inworld-flavored turn: free-form expression + sound + break
+const JOKE =
+ ' Why did the burger go to the gym? ' +
+ ' Because it wanted better buns! ' +
+ '';
+
+describe('convertMarkup: expr -> xAI (sounds, breaks, wrapping prosody; no expression)', () => {
+ it('lowers sounds, breaks, and wrapping prosody to native syntax', () => {
+ const text =
+ 'So I walked in and there it was! ' +
+ ' ' +
+ 'It was a secret the whole time.';
+ expect(convertMarkup('xai', text)).toBe(
+ 'So I walked in and [pause] there it was! [laugh] ' +
+ 'It was a secret the whole time.',
+ );
+ });
+
+ it('maps break durations to the two pause levels', () => {
+ expect(convertMarkup('xai', '')).toBe('[pause]');
+ expect(convertMarkup('xai', '')).toBe('[long-pause]');
+ });
+
+ it('maps sound label aliases to native cue names', () => {
+ // tolerance: an Inworld-style "breathe" label maps to xAI's native [breath] cue
+ expect(convertMarkup('xai', '')).toBe('[breath]');
+ });
+
+ it('normalizes multi-word prosody labels to hyphenated tag names', () => {
+ const text = 'no way';
+ expect(convertMarkup('xai', text)).toBe('no way');
+ });
+
+ it('unwraps an unknown prosody label', () => {
+ const text = 'ahoy there';
+ expect(convertMarkup('xai', text)).toBe('ahoy there');
+ });
+
+ it('drops expression markers', () => {
+ // xAI has no free-form delivery descriptions; a hallucinated expression marker is
+ // dropped from the audio path (it still surfaces in transcript tags)
+ const text = ' Hello!';
+ expect(convertMarkup('xai', text)).toBe(' Hello!');
+ });
+});
+
+describe('convertMarkup: expr -> Inworld (free-form expression, its sound list, breaks)', () => {
+ it('lowers expression/sound to bracket syntax, break stays native SSML', () => {
+ expect(convertMarkup('inworld', JOKE)).toBe(
+ '[say playfully] Why did the burger go to the gym? ' +
+ ' Because it wanted better buns! [laugh]',
+ );
+ });
+
+ it('salvages a stray prosody wrapper as an expression hint', () => {
+ const text = 'keep it secret';
+ expect(convertMarkup('inworld', text)).toBe('[whisper]keep it secret');
+ });
+});
+
+describe('convertMarkup: expr -> Cartesia (discrete emotions, breaks, spell; no sounds)', () => {
+ it('lowers expression to , keeps break, drops sound', () => {
+ const text =
+ ' We won! ' +
+ ' Unbelievable.';
+ // expression -> , break stays, sound is dropped (no Cartesia support)
+ expect(convertMarkup('cartesia', text)).toBe(
+ ' We won! Unbelievable.',
+ );
+ });
+
+ it('keeps spell wrapping for Cartesia', () => {
+ const text = 'Your code is A7X9.';
+ expect(convertMarkup('cartesia', text)).toBe('Your code is A7X9.');
+ });
+
+ it('unwraps spell for other providers', () => {
+ // spell is Cartesia-only; other providers keep the characters, drop the marker
+ const text = 'Your code is A7X9.';
+ expect(convertMarkup('xai', text)).toBe('Your code is A7X9.');
+ expect(convertMarkup('inworld', text)).toBe('Your code is A7X9.');
+ });
+
+ it('lowers prosody labels to native point controls', () => {
+ // Cartesia prosody labels lower to its native speed/volume ratio tags
+ expect(convertMarkup('cartesia', ' One moment.')).toBe(
+ ' One moment.',
+ );
+ expect(convertMarkup('cartesia', ' We won!')).toBe(
+ ' We won!',
+ );
+ // wrapping form applies the control before the span
+ expect(convertMarkup('cartesia', 'bad news')).toBe(
+ 'bad news',
+ );
+ });
+
+ it('unwraps an unknown prosody label', () => {
+ const text = 'keep it secret';
+ expect(convertMarkup('cartesia', text)).toBe('keep it secret');
+ });
+});
+
+describe('convertMarkup: stray expr markers', () => {
+ it('never lets a stray expr marker reach the TTS', () => {
+ // an unpaired prosody open/close (e.g. split across stream chunks) is dropped,
+ // keeping the words
+ expect(convertMarkup('xai', 'hello there')).toBe(
+ 'hello there',
+ );
+ expect(convertMarkup('xai', 'hello there')).toBe('hello there');
+ });
+});
+
+describe('transcript stripping (per-provider + provider-agnostic)', () => {
+ it.each(['xai', 'inworld', 'cartesia'])('splitMarkup strips expr for %s', (provider) => {
+ const [clean, tags] = splitMarkup(provider, JOKE);
+ expect(clean.trim()).toBe('Why did the burger go to the gym? Because it wanted better buns!');
+ expect(tags).toEqual([
+ { type: 'expression', value: 'say playfully' },
+ { type: 'break', value: '500ms' },
+ { type: 'sound', value: 'laugh' },
+ ]);
+ });
+
+ it('keeps the inner text of wrapping markers', () => {
+ const text =
+ 'She said keep it secret — ' +
+ 'code A7X9.';
+ const [clean, tags] = splitMarkup('xai', text);
+ expect(clean).toBe('She said keep it secret — code A7X9.');
+ expect(tags).toEqual([
+ { type: 'prosody', value: 'whisper' },
+ { type: 'spell', value: '' },
+ ]);
+ });
+
+ it('splitAllMarkup handles mixed expr and native markup', () => {
+ const text =
+ ' Hello! [sigh]';
+ const [clean, tags] = splitAllMarkup(text);
+ expect(clean.trim()).toBe('Hello!');
+ expect(tags).toContainEqual({ type: 'expression', value: 'say playfully' });
+ expect(tags).toContainEqual({ type: 'sound', value: 'laugh' });
+ expect(tags).toContainEqual({ type: '', value: 'sigh' });
+ });
+
+ it('preserves document order when mixing native and expr markup', () => {
+ // a hallucinated native tag ahead of an expr marker must stay ahead in the tag
+ // list — lk.expression surfaces the segment's *leading* expression/emotion
+ const text = 'Hi';
+ const [clean, tags] = splitAllMarkup(text);
+ expect(clean).toBe('Hi');
+ expect(tags).toEqual([
+ { type: 'emotion', value: 'sad' },
+ { type: 'expression', value: 'happy' },
+ ]);
+ expect(expressionAttribute(tags)).toEqual({ 'lk.expression': '{"value":"sad"}' });
+
+ // same through the per-provider path, with brackets in the mix
+ const [, inworldTags] = splitMarkup(
+ 'inworld',
+ '[sigh] hi ',
+ );
+ expect(inworldTags).toEqual([
+ { type: '', value: 'sigh' },
+ { type: 'expression', value: 'calm' },
+ { type: 'sound', value: 'laugh' },
+ ]);
+ });
+
+ it('does not match the native tag with the expr regexes', () => {
+ // " there.');
+ });
+
+ it('TranscriptMarkupStripper handles expr split across streaming chunks', () => {
+ const stripper = new TranscriptMarkupStripper();
+ let out = '';
+ // split mid-tag so the partial " Hello',
+ ' wor',
+ 'ld!',
+ ]) {
+ out += stripper.push(chunk);
+ }
+ out += stripper.flush();
+ expect(out).toBe(' Hello world!');
+ expect(stripper.tags[0]).toEqual({ type: 'expression', value: 'say playfully' });
+ expect(stripper.tags).toContainEqual({ type: 'prosody', value: 'whisper' });
+ });
+
+ it('expressionAttribute surfaces the expr expression label', () => {
+ const [, tags] = splitMarkup('inworld', JOKE);
+ const attr = expressionAttribute(tags);
+ expect(attr).toBeDefined();
+ expect(Object.values(attr!)[0]).toContain('"say playfully"');
+ });
+});
+
+describe('normalizeMarkup: fix unclosed self-closing expr markers', () => {
+ it.each(['xai', 'inworld', 'cartesia'])('closes an unclosed expr marker for %s', (provider) => {
+ const text = ' Hello';
+ expect(normalizeMarkup(provider, text)).toBe(' Hello');
+ });
+
+ it('leaves wrapping and closed markers alone', () => {
+ const text =
+ 'hi ' +
+ 'A7X9';
+ expect(normalizeMarkup('xai', text)).toBe(text);
+ });
+
+ it('closes an unclosed Cartesia prosody point control', () => {
+ // Cartesia prosody is self-closing, so the missing-slash fix applies there —
+ // otherwise convertExpr's stray-tag cleanup would drop the control entirely
+ const text = ' One moment.';
+ const normalized = normalizeMarkup('cartesia', text);
+ expect(normalized).toBe(' One moment.');
+ expect(convertMarkup('cartesia', normalized)).toBe(' One moment.');
+ // xAI prosody legitimately wraps, so its opening tag must stay untouched
+ expect(normalizeMarkup('xai', text)).toBe(text);
+ });
+});
+
+describe('llm instructions: shared syntax, per-provider kinds and vocabularies', () => {
+ it.each(['xai', 'inworld', 'cartesia'])('uses expr syntax for %s', (provider) => {
+ const instructions = llmInstructions(provider);
+ expect(instructions).toBeDefined();
+ expect(instructions).toContain('');
+ expect(instructions).toContain('NOT free-form');
+ expect(instructions).toContain('');
+ // coarse self-closing prosody point controls
+ expect(instructions).toContain('');
+ // no non-verbal sounds
+ expect(instructions).not.toContain('type="sound"');
+ });
+
+ it('advertises Inworld kinds', () => {
+ const instructions = llmInstructions('inworld')!;
+ // free-form delivery descriptions + Inworld's own sound list
+ expect(instructions).toContain('');
+ expect(instructions).toContain('free-form');
+ expect(instructions).toContain('clear throat');
+ // no wrapping prosody, no spell
+ expect(instructions).not.toContain('type="prosody"');
+ expect(instructions).not.toContain('type="spell"');
+ });
+
+ it('advertises xAI kinds', () => {
+ const instructions = llmInstructions('xai')!;
+ // xAI's own sound cues + wrapping prosody vocabulary
+ expect(instructions).toContain('tongue-click');
+ expect(instructions).toContain('');
+ expect(instructions).toContain('sing-song');
+ // no free-form delivery descriptions, no spell
+ expect(instructions).not.toContain('type="expression"');
+ expect(instructions).not.toContain('type="spell"');
+ });
+
+ it('returns undefined for unknown providers', () => {
+ expect(llmInstructions('')).toBeUndefined();
+ expect(llmInstructions('openai')).toBeUndefined();
+ });
+});
diff --git a/agents/src/tts/markup_utils.test.ts b/agents/src/tts/markup_utils.test.ts
index d73c49fca..91391dd17 100644
--- a/agents/src/tts/markup_utils.test.ts
+++ b/agents/src/tts/markup_utils.test.ts
@@ -61,8 +61,10 @@ describe('xAI dialect', () => {
const instr = llmInstructions('xai');
// non-undefined is what the expressive gate keys on
expect(instr).toBeDefined();
- expect(instr).toContain('');
- expect(instr).toContain('');
+ // this branch instructs the unified expr dialect; convertMarkup lowers it to
+ // xAI's native syntax (see expr_markup.test.ts)
+ expect(instr).toContain('');
+ expect(instr).toContain('');
}
});
});
diff --git a/agents/src/tts/markup_utils.ts b/agents/src/tts/markup_utils.ts
index 2d8239698..6e5b0b82b 100644
--- a/agents/src/tts/markup_utils.ts
+++ b/agents/src/tts/markup_utils.ts
@@ -37,12 +37,17 @@ const escapeRegExp = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
* @param text - The text containing markup.
* @param xmlTags - XML tag names to handle (e.g. `["emotion", "sound"]`).
* @param brackets - Whether to also handle square-bracket tags like `[laughs]`.
+ * @param offsetsOut - Optional array receiving, per recorded tag, the offset of its
+ * match so callers can merge tags from separate stripping passes in document order.
+ * Offsets are exact for top-level tags; a tag exposed by unwrapping an outer tag
+ * (nested markup, found in a later fixed-point pass) reports its offset within the
+ * partially-stripped text — approximate, but ordering stays monotonic in practice.
*/
export function extractAndStrip(
text: string,
- options: { xmlTags: string[]; brackets: boolean },
+ options: { xmlTags: string[]; brackets: boolean; offsetsOut?: number[] },
): [string, Array<[string, string]>] {
- const { xmlTags, brackets } = options;
+ const { xmlTags, brackets, offsetsOut } = options;
if (xmlTags.length === 0 && !brackets) {
return [text, []];
}
@@ -67,6 +72,7 @@ export function extractAndStrip(
const repl = (match: string, ...args: unknown[]): string => {
const groups = args[args.length - 1] as Record;
+ const offset = args[args.length - 3] as number;
const tag = groups.tag;
if (tag !== undefined) {
const inner = groups.inner;
@@ -78,6 +84,7 @@ export function extractAndStrip(
value = attrMatch ? attrMatch[1]! : '';
}
tags.push([tag, value]);
+ offsetsOut?.push(offset);
// wrapping tags keep their inner content; self-closing/lone tags vanish
return inner !== undefined ? inner : '';
}
@@ -85,6 +92,7 @@ export function extractAndStrip(
const bracket = groups.bracket;
if (bracket !== undefined) {
tags.push(['', bracket.trim()]);
+ offsetsOut?.push(offset);
return '';
}
diff --git a/agents/src/voice/transcription/synchronizer.ts b/agents/src/voice/transcription/synchronizer.ts
index b71205aad..805ed7c41 100644
--- a/agents/src/voice/transcription/synchronizer.ts
+++ b/agents/src/voice/transcription/synchronizer.ts
@@ -7,7 +7,7 @@ import { log } from '../../log.js';
import { IdentityTransform } from '../../stream/identity_transform.js';
import type { WordStream, WordTokenizer } from '../../tokenize/index.js';
import { basic } from '../../tokenize/index.js';
-import { splitAllMarkup } from '../../tts/_provider_format.js';
+import { TranscriptMarkupStripper } from '../../tts/_provider_format.js';
import { Future, Task, delay } from '../../utils.js';
import {
AudioOutput,
@@ -142,11 +142,17 @@ interface AudioData {
annotatedRate: SpeakingRateData | null;
}
-class SegmentSynchronizerImpl {
+/** @internal Exported for testing purposes. */
+export class SegmentSynchronizerImpl {
private enabled: boolean;
private textData: TextData;
private audioData: AudioData;
private speed: number;
+ // paces against the visible text only; stateful because a markup tag with
+ // spaces in its attributes (e.g. )
+ // is shredded across word tokens and a per-token strip can't recognize the
+ // fragments — each would otherwise be paced as if it were spoken
+ private pacingStripper = new TranscriptMarkupStripper();
// Emit TimedString objects so downstream outputs (e.g. RoomIO's json_format) can
// attach `end_time` reflecting synchronized playback timing.
private outputStream: IdentityTransform;
@@ -437,12 +443,13 @@ class SegmentSynchronizerImpl {
pushedTextCursor = wordEnd;
// forward the raw token (the room output strips markup and surfaces the
- // expression downstream), but pace against the visible text only so a
- // markup-only token adds no delay
- const [strippedWord] = splitAllMarkup(word);
- const cleanWords = this.options.splitWords(strippedWord);
- const cleanWord = cleanWords.length > 0 ? cleanWords[0]![0] : strippedWord;
- const wordHyphens = cleanWord ? this.options.hyphenateWord(cleanWord).length : 0;
+ // expression downstream), but pace against the visible text only so markup
+ // adds no delay. The stripper holds back an unclosed tag across tokens and
+ // releases the clean text once it completes. Feed it the raw pushedText slice
+ // (not the bare token) so the whitespace inside tag attributes survives and a
+ // shredded tag can reassemble.
+ const cleanWord = this.pacingStripper.push(forwardedWord);
+ const wordHyphens = cleanWord.trim() ? this.calcHyphens(cleanWord).length : 0;
if (this.playbackCompleted) {
this.outputStreamWriter.write(
diff --git a/agents/src/voice/transcription/synchronizer_markup.test.ts b/agents/src/voice/transcription/synchronizer_markup.test.ts
new file mode 100644
index 000000000..5deba35aa
--- /dev/null
+++ b/agents/src/voice/transcription/synchronizer_markup.test.ts
@@ -0,0 +1,69 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Transcript synchronizer pacing must ignore expressive markup.
+ *
+ * The synchronizer forwards the raw LLM text (markup intact — the room output strips
+ * it downstream) but paces the display against the *visible* words only. Markup tags
+ * carry spaces in their attributes, so the word stream shreds them into fragments
+ * (``); a per-token
+ * strip can't recognize those, and each fragment was paced as if it were spoken — the
+ * transcript drifted seconds behind the audio on every expressive sentence.
+ */
+import { describe, expect, it } from 'vitest';
+import { TextOutput, isTimedString } from '../io.js';
+import { SegmentSynchronizerImpl, defaultTextSyncOptions } from './synchronizer.js';
+
+// ~11 visible hyphens of speech, but dozens of hyphens of markup fragments. With the
+// bug the markup alone adds many seconds of pacing; with the fix the whole transcript
+// paces out in roughly the visible-word budget (~3s at the standard speech rate).
+const MARKED_UP_TURN =
+ ' ' +
+ 'Hello there my friend! ' +
+ ' ' +
+ ' ' +
+ 'How are you today?';
+
+class CollectorTextOutput extends TextOutput {
+ words: string[] = [];
+
+ async captureText(text: string): Promise {
+ this.words.push(isTimedString(text) ? text.text : text);
+ }
+
+ flush(): void {}
+}
+
+describe('transcript synchronizer markup pacing', () => {
+ it('markup fragments add no pacing delay', { timeout: 30_000 }, async () => {
+ const collector = new CollectorTextOutput();
+ const impl = new SegmentSynchronizerImpl({ ...defaultTextSyncOptions }, collector);
+ try {
+ impl.pushText(MARKED_UP_TURN);
+ impl.endTextInput();
+ impl.endAudioInput();
+
+ const start = Date.now();
+ impl.onPlaybackStarted(Date.now());
+
+ // forwarding is done once the main task exhausts the word stream and the
+ // capture task drains the output channel
+ await (impl as unknown as { captureTask: Promise }).captureTask;
+ const elapsedSeconds = (Date.now() - start) / 1000;
+
+ // every raw token is still forwarded (markup included — stripped downstream)
+ expect(collector.words.join('')).toBe(MARKED_UP_TURN);
+
+ // the pacing budget must cover only the visible words (~3s at the standard
+ // speech rate); with markup fragments paced as speech it exceeds 10s
+ expect(
+ elapsedSeconds,
+ `transcript took ${elapsedSeconds.toFixed(1)}s — markup is being paced as spoken text`,
+ ).toBeLessThan(6);
+ } finally {
+ await impl.close();
+ }
+ });
+});
diff --git a/examples/src/expressive_agent.ts b/examples/src/expressive_agent.ts
new file mode 100644
index 000000000..58cb4abc0
--- /dev/null
+++ b/examples/src/expressive_agent.ts
@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// cue-cli e2e harness agent for expressive mode (expr marker dialect).
+// Registered with explicit dispatch as `expressive-agent-js`.
+import {
+ Agent,
+ AgentSession,
+ type JobContext,
+ ServerOptions,
+ cli,
+ defineAgent,
+ inference,
+ tool,
+ voice,
+} from '@livekit/agents';
+import { fileURLToPath } from 'node:url';
+import { z } from 'zod';
+
+export default defineAgent({
+ entry: async (ctx: JobContext) => {
+ const agent = Agent.create({
+ instructions:
+ 'You are a cheerful, expressive assistant. Keep replies to one or two short ' +
+ 'sentences. You can hear the user and respond with speech.',
+ tools: [
+ tool({
+ name: 'getWeather',
+ description: 'Get the weather for a given location.',
+ parameters: z.object({
+ location: z.string().describe('The location to get the weather for'),
+ }),
+ execute: async ({ location }) => `The weather in ${location} is sunny.`,
+ }),
+ ],
+ });
+
+ const session = new AgentSession({
+ stt: new inference.STT({ model: 'deepgram/nova-3', language: 'en' }),
+ llm: new inference.LLM({ model: 'openai/gpt-4.1-mini' }),
+ // Inworld: free-form expression labels (with spaces) — exercises both the
+ // expr dialect lowering and the transcript pacing fix.
+ tts: new inference.TTS({ model: 'inworld/inworld-tts-2' }),
+ expressive: voice.presets.CASUAL,
+ });
+
+ await session.start({ agent, room: ctx.room });
+ session.say('Hi there! How can I help you today?');
+ },
+});
+
+cli.runApp(
+ new ServerOptions({
+ agent: fileURLToPath(import.meta.url),
+ agentName: 'expressive-agent-js',
+ }),
+);