fix(voice): ignore markup tags for transcript pacing

.changeset/markup-pacing-tags.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents': patch
+---
+
+Ignore expressive markup tags when pacing synchronized transcription output, while still forwarding the raw text for downstream stripping.

agents/src/voice/transcription/synchronizer.test.ts

@@ -4,7 +4,7 @@
 import { AudioFrame } from '@livekit/rtc-node';
 import { afterEach, describe, expect, it, vi } from 'vitest';
 import * as logModule from '../../log.js';
-import { AudioOutput, TextOutput } from '../io.js';
+import { AudioOutput, TextOutput, type TimedString, isTimedString } from '../io.js';
 import { SpeakingRateData, TranscriptionSynchronizer } from './synchronizer.js';
 
 describe('SpeakingRateData', () => {
@@ -223,13 +223,51 @@ class MockAudioOutput extends AudioOutput {
 class MockTextOutput extends TextOutput {
   captured: string[] = [];
 
-  async captureText(text: string): Promise<void> {
-    this.captured.push(text);
+  async captureText(text: string | TimedString): Promise<void> {
+    this.captured.push(isTimedString(text) ? text.text : text);
   }
 
   flush(): void {}
 }
 
+describe('TranscriptionSynchronizer markup pacing', () => {
+  const markedUpTurn =
+    '<expr type="expression" label="speak with warm surprise and bright energy"/> ' +
+    'Hello there my friend! ' +
+    '<expr type="sound" label="laugh"/> ' +
+    '<expr type="expression" label="speak calmly and evenly, unhurried"/> ' +
+    'How are you today?';
+
+  it('does not pace expressive markup fragments as spoken text', async () => {
+    const textOutput = new MockTextOutput();
+    const synchronizer = new TranscriptionSynchronizer(new MockAudioOutput(), textOutput);
+    const frame = new AudioFrame(new Int16Array(160), 8000, 1, 160);
+
+    try {
+      await synchronizer.audioOutput.captureFrame(frame);
+      synchronizer.audioOutput.flush();
+      await synchronizer.textOutput.captureText(markedUpTurn);
+      synchronizer.textOutput.flush();
+
+      const start = Date.now();
+      synchronizer.audioOutput.onPlaybackStarted(start);
+
+      while (textOutput.captured.join('').length < markedUpTurn.length) {
+        if (Date.now() - start > 6000) {
+          break;
+        }
+        await new Promise((resolve) => setTimeout(resolve, 25));
+      }
+
+      const elapsed = (Date.now() - start) / 1000;
+      expect(textOutput.captured.join('')).toBe(markedUpTurn);
+      expect(elapsed).toBeLessThan(6);
+    } finally {
+      await synchronizer.close();
+    }
+  }, 10000);
+});
+
 describe('TranscriptionSynchronizer attachment warnings', () => {
   const textDetachedWarning =
     'TranscriptSynchronizer text output was detached while audio output is still active; ' +

agents/src/voice/transcription/synchronizer.ts

@@ -19,6 +19,77 @@ import {
 
 const STANDARD_SPEECH_RATE = 3.83; // hyphens (syllables) per second
 
+const XML_MARKUP_TAGS = [
+  'expr',
+  'emotion',
+  'speed',
+  'volume',
+  'break',
+  'spell',
+  'expression',
+  'sound',
+  'happy',
+  'sad',
+  'angry',
+  'excited',
+  'calm',
+  'surprised',
+  'sympathetic',
+  'curious',
+  'sarcastic',
+  'confident',
+  'playful',
+  'nervous',
+  'emphasis',
+  'whisper',
+  'soft',
+  'loud',
+  'build-intensity',
+  'decrease-intensity',
+  'higher-pitch',
+  'lower-pitch',
+  'slow',
+  'fast',
+  'sing-song',
+  'singing',
+  'laugh-speak',
+];
+const XML_MARKUP_TAG_RE = new RegExp(
+  `<\\/?(?:${XML_MARKUP_TAGS.join('|')})(?:\\s[^>]*)?\\/?>`,
+  'gi',
+);
+const BRACKET_MARKUP_RE = /\[[^\]]+\]/g;
+
+class TranscriptMarkupStripper {
+  private buffer = '';
+
+  push(text: string): string {
+    this.buffer += text;
+    if (this.hasOpenTag()) {
+      return '';
+    }
+
+    const clean = this.stripMarkup(this.buffer);
+    this.buffer = '';
+    return clean;
+  }
+
+  private hasOpenTag(): boolean {
+    const lastLt = this.buffer.lastIndexOf('<');
+    if (lastLt > this.buffer.lastIndexOf('>')) {
+      const next = this.buffer.slice(lastLt + 1, lastLt + 2);
+      if (!next || next === '/' || /[a-z]/i.test(next)) {
+        return true;
+      }
+    }
+    return this.buffer.lastIndexOf('[') > this.buffer.lastIndexOf(']');
+  }
+
+  private stripMarkup(text: string): string {
+    return text.replace(XML_MARKUP_TAG_RE, '').replace(BRACKET_MARKUP_RE, '');
+  }
+}
+
 interface TextSyncOptions {
   speed: number;
   hyphenateWord: (word: string) => string[];
@@ -152,6 +223,7 @@ class SegmentSynchronizerImpl {
   private outputStreamWriter: WritableStreamDefaultWriter<string | TimedString>;
   private captureTask: Promise<void>;
   private startWallTime?: number;
+  private pacingStripper = new TranscriptMarkupStripper();
 
   private startFuture: Future = new Future();
   private closedFuture: Future = new Future();
@@ -442,16 +514,14 @@ class SegmentSynchronizerImpl {
             endTime: this.synchronizedElapsedSeconds(),
           }),
         );
-        const cleanWords = this.options.splitWords(word);
-        const cleanWord = cleanWords.length > 0 ? cleanWords[0]![0] : word;
-        this.textData.forwardedHyphens += this.options.hyphenateWord(cleanWord).length;
+        const cleanWord = this.pacingStripper.push(forwardedWord);
+        this.textData.forwardedHyphens += cleanWord.trim() ? this.calcHyphens(cleanWord).length : 0;
         this.textData.forwardedText += forwardedWord;
         continue;
       }
 
-      const cleanWords = this.options.splitWords(word);
-      const cleanWord = cleanWords.length > 0 ? cleanWords[0]![0] : word;
-      const wordHyphens = this.options.hyphenateWord(cleanWord).length;
+      const cleanWord = this.pacingStripper.push(forwardedWord);
+      const wordHyphens = cleanWord.trim() ? this.calcHyphens(cleanWord).length : 0;
       const elapsedSeconds = this.synchronizedElapsedSeconds()!;
 
       let dHyphens = 0;