diff --git a/.changeset/markup-pacing-tags.md b/.changeset/markup-pacing-tags.md new file mode 100644 index 000000000..7ba99c646 --- /dev/null +++ b/.changeset/markup-pacing-tags.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Ignore expressive markup tags when pacing synchronized transcription output, while still forwarding the raw text for downstream stripping. diff --git a/agents/src/voice/transcription/synchronizer.test.ts b/agents/src/voice/transcription/synchronizer.test.ts index d3b100d3f..3652ecd2b 100644 --- a/agents/src/voice/transcription/synchronizer.test.ts +++ b/agents/src/voice/transcription/synchronizer.test.ts @@ -4,7 +4,7 @@ import { AudioFrame } from '@livekit/rtc-node'; import { afterEach, describe, expect, it, vi } from 'vitest'; import * as logModule from '../../log.js'; -import { AudioOutput, TextOutput } from '../io.js'; +import { AudioOutput, TextOutput, type TimedString, isTimedString } from '../io.js'; import { SpeakingRateData, TranscriptionSynchronizer } from './synchronizer.js'; describe('SpeakingRateData', () => { @@ -223,13 +223,51 @@ class MockAudioOutput extends AudioOutput { class MockTextOutput extends TextOutput { captured: string[] = []; - async captureText(text: string): Promise { - this.captured.push(text); + async captureText(text: string | TimedString): Promise { + this.captured.push(isTimedString(text) ? text.text : text); } flush(): void {} } +describe('TranscriptionSynchronizer markup pacing', () => { + const markedUpTurn = + ' ' + + 'Hello there my friend! ' + + ' ' + + ' ' + + 'How are you today?'; + + it('does not pace expressive markup fragments as spoken text', async () => { + const textOutput = new MockTextOutput(); + const synchronizer = new TranscriptionSynchronizer(new MockAudioOutput(), textOutput); + const frame = new AudioFrame(new Int16Array(160), 8000, 1, 160); + + try { + await synchronizer.audioOutput.captureFrame(frame); + synchronizer.audioOutput.flush(); + await synchronizer.textOutput.captureText(markedUpTurn); + synchronizer.textOutput.flush(); + + const start = Date.now(); + synchronizer.audioOutput.onPlaybackStarted(start); + + while (textOutput.captured.join('').length < markedUpTurn.length) { + if (Date.now() - start > 6000) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 25)); + } + + const elapsed = (Date.now() - start) / 1000; + expect(textOutput.captured.join('')).toBe(markedUpTurn); + expect(elapsed).toBeLessThan(6); + } finally { + await synchronizer.close(); + } + }, 10000); +}); + describe('TranscriptionSynchronizer attachment warnings', () => { const textDetachedWarning = 'TranscriptSynchronizer text output was detached while audio output is still active; ' + diff --git a/agents/src/voice/transcription/synchronizer.ts b/agents/src/voice/transcription/synchronizer.ts index dc5c081f2..711fa70bb 100644 --- a/agents/src/voice/transcription/synchronizer.ts +++ b/agents/src/voice/transcription/synchronizer.ts @@ -19,6 +19,77 @@ import { const STANDARD_SPEECH_RATE = 3.83; // hyphens (syllables) per second +const XML_MARKUP_TAGS = [ + 'expr', + 'emotion', + 'speed', + 'volume', + 'break', + 'spell', + 'expression', + 'sound', + 'happy', + 'sad', + 'angry', + 'excited', + 'calm', + 'surprised', + 'sympathetic', + 'curious', + 'sarcastic', + 'confident', + 'playful', + 'nervous', + 'emphasis', + 'whisper', + 'soft', + 'loud', + 'build-intensity', + 'decrease-intensity', + 'higher-pitch', + 'lower-pitch', + 'slow', + 'fast', + 'sing-song', + 'singing', + 'laugh-speak', +]; +const XML_MARKUP_TAG_RE = new RegExp( + `<\\/?(?:${XML_MARKUP_TAGS.join('|')})(?:\\s[^>]*)?\\/?>`, + 'gi', +); +const BRACKET_MARKUP_RE = /\[[^\]]+\]/g; + +class TranscriptMarkupStripper { + private buffer = ''; + + push(text: string): string { + this.buffer += text; + if (this.hasOpenTag()) { + return ''; + } + + const clean = this.stripMarkup(this.buffer); + this.buffer = ''; + return clean; + } + + private hasOpenTag(): boolean { + const lastLt = this.buffer.lastIndexOf('<'); + if (lastLt > this.buffer.lastIndexOf('>')) { + const next = this.buffer.slice(lastLt + 1, lastLt + 2); + if (!next || next === '/' || /[a-z]/i.test(next)) { + return true; + } + } + return this.buffer.lastIndexOf('[') > this.buffer.lastIndexOf(']'); + } + + private stripMarkup(text: string): string { + return text.replace(XML_MARKUP_TAG_RE, '').replace(BRACKET_MARKUP_RE, ''); + } +} + interface TextSyncOptions { speed: number; hyphenateWord: (word: string) => string[]; @@ -152,6 +223,7 @@ class SegmentSynchronizerImpl { private outputStreamWriter: WritableStreamDefaultWriter; private captureTask: Promise; private startWallTime?: number; + private pacingStripper = new TranscriptMarkupStripper(); private startFuture: Future = new Future(); private closedFuture: Future = new Future(); @@ -442,16 +514,14 @@ class SegmentSynchronizerImpl { endTime: this.synchronizedElapsedSeconds(), }), ); - const cleanWords = this.options.splitWords(word); - const cleanWord = cleanWords.length > 0 ? cleanWords[0]![0] : word; - this.textData.forwardedHyphens += this.options.hyphenateWord(cleanWord).length; + const cleanWord = this.pacingStripper.push(forwardedWord); + this.textData.forwardedHyphens += cleanWord.trim() ? this.calcHyphens(cleanWord).length : 0; this.textData.forwardedText += forwardedWord; continue; } - const cleanWords = this.options.splitWords(word); - const cleanWord = cleanWords.length > 0 ? cleanWords[0]![0] : word; - const wordHyphens = this.options.hyphenateWord(cleanWord).length; + const cleanWord = this.pacingStripper.push(forwardedWord); + const wordHyphens = cleanWord.trim() ? this.calcHyphens(cleanWord).length : 0; const elapsedSeconds = this.synchronizedElapsedSeconds()!; let dHyphens = 0;