From be783f20eacd96417f784ab9504ab711deabd81e Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 17:03:56 +0000 Subject: [PATCH 1/2] fix(test): avoid false PCM failures in TTS tests --- plugins/test/src/tts.ts | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/plugins/test/src/tts.ts b/plugins/test/src/tts.ts index 3692b1202..b2d1aae3a 100644 --- a/plugins/test/src/tts.ts +++ b/plugins/test/src/tts.ts @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { stt } from '@livekit/agents'; import { type AudioBuffer, initializeLogger, tokenize, tts as ttslib } from '@livekit/agents'; -import type { AudioFrame } from '@livekit/rtc-node'; +import { type AudioFrame, combineAudioFrames } from '@livekit/rtc-node'; import { distance } from 'fastest-levenshtein'; import { ReadableStream } from 'stream/web'; import { describe, expect, it } from 'vitest'; @@ -11,6 +11,42 @@ import { describe, expect, it } from 'vitest'; const TEXT = 'The people who are crazy enough to think they can change the world are the ones who do.'; +const detectCompressedContainer = (data: Uint8Array): string | undefined => { + const startsWith = (signature: string, offset = 0) => + data.length >= offset + signature.length && + signature.split('').every((char, i) => data[offset + i] === char.charCodeAt(0)); + + if (data[0] === 0xff && data.length > 1 && (data[1]! & 0xf6) === 0xf0) { + return 'aac'; + } + if ( + startsWith('ID3') || + (data[0] === 0xff && data.length > 1 && (data[1]! & 0xe0) === 0xe0 && (data[1]! & 0x06) !== 0) + ) { + return 'mp3'; + } + if (startsWith('OggS')) return 'ogg'; + if (startsWith('fLaC')) return 'flac'; + if (startsWith('RIFF') && startsWith('WAVE', 8)) return 'wav'; + if (startsWith('ftyp', 4)) return 'mov,mp4,m4a,3gp,3g2,mj2'; + if (data[0] === 0x1a && data[1] === 0x45 && data[2] === 0xdf && data[3] === 0xa3) { + return 'matroska,webm'; + } + if (data[0] === 0x00 && data[1] === 0x00 && data[2] === 0x01 && data[3] === 0xba) { + return 'mpeg'; + } +}; + +const assertPCM = (frames: AudioFrame[]) => { + const frame = combineAudioFrames(frames); + const data = new Uint8Array(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength); + const container = detectCompressedContainer(data); + + if (container) { + throw new Error(`Audio data isn't PCM (detected ${container})`); + } +}; + const validate = async (frames: AudioBuffer, stt: stt.STT, text: string, threshold: number) => { const event = await stt.recognize(frames); const eventText = event.alternatives![0].text.toLowerCase().replace(/\s/g, ' ').trim(); @@ -64,6 +100,7 @@ export const tts = async ( frames.push(event.frame); } + assertPCM(frames); await validate(frames, stt, TEXT, 0.2); stream.close(); }); From 3eaf6b5aa3992a2fdbe9dc6671630aa2c1aba172 Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 17:18:45 +0000 Subject: [PATCH 2/2] fix(test): probe TTS audio with FFmpeg --- plugins/test/package.json | 1 + plugins/test/src/tts.ts | 69 +++++++++++++++++++++++---------------- pnpm-lock.yaml | 3 ++ 3 files changed, 44 insertions(+), 29 deletions(-) diff --git a/plugins/test/package.json b/plugins/test/package.json index 81d4ba098..d0daa2e2e 100644 --- a/plugins/test/package.json +++ b/plugins/test/package.json @@ -39,6 +39,7 @@ "zod": "^3.25.76" }, "dependencies": { + "@ffmpeg-installer/ffmpeg": "^1.1.0", "fastest-levenshtein": "^1.0.16", "vitest": "^4.0.17" }, diff --git a/plugins/test/src/tts.ts b/plugins/test/src/tts.ts index b2d1aae3a..6c7caf141 100644 --- a/plugins/test/src/tts.ts +++ b/plugins/test/src/tts.ts @@ -1,48 +1,59 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +import ffmpegInstaller from '@ffmpeg-installer/ffmpeg'; import type { stt } from '@livekit/agents'; import { type AudioBuffer, initializeLogger, tokenize, tts as ttslib } from '@livekit/agents'; import { type AudioFrame, combineAudioFrames } from '@livekit/rtc-node'; import { distance } from 'fastest-levenshtein'; +import { spawn } from 'node:child_process'; import { ReadableStream } from 'stream/web'; import { describe, expect, it } from 'vitest'; const TEXT = 'The people who are crazy enough to think they can change the world are the ones who do.'; -const detectCompressedContainer = (data: Uint8Array): string | undefined => { - const startsWith = (signature: string, offset = 0) => - data.length >= offset + signature.length && - signature.split('').every((char, i) => data[offset + i] === char.charCodeAt(0)); +const compressedFormats = new Set([ + 'mp3', + 'aac', + 'ogg', + 'flac', + 'wav', + 'mov,mp4,m4a,3gp,3g2,mj2', + 'matroska,webm', + 'mpeg', +]); - if (data[0] === 0xff && data.length > 1 && (data[1]! & 0xf6) === 0xf0) { - return 'aac'; - } - if ( - startsWith('ID3') || - (data[0] === 0xff && data.length > 1 && (data[1]! & 0xe0) === 0xe0 && (data[1]! & 0x06) !== 0) - ) { - return 'mp3'; - } - if (startsWith('OggS')) return 'ogg'; - if (startsWith('fLaC')) return 'flac'; - if (startsWith('RIFF') && startsWith('WAVE', 8)) return 'wav'; - if (startsWith('ftyp', 4)) return 'mov,mp4,m4a,3gp,3g2,mj2'; - if (data[0] === 0x1a && data[1] === 0x45 && data[2] === 0xdf && data[3] === 0xa3) { - return 'matroska,webm'; - } - if (data[0] === 0x00 && data[1] === 0x00 && data[2] === 0x01 && data[3] === 0xba) { - return 'mpeg'; - } -}; - -const assertPCM = (frames: AudioFrame[]) => { +const assertPCM = async (frames: AudioFrame[]) => { const frame = combineAudioFrames(frames); const data = new Uint8Array(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength); - const container = detectCompressedContainer(data); - if (container) { + const container = await new Promise((resolve, reject) => { + const ffmpeg = spawn(ffmpegInstaller.path, [ + '-hide_banner', + '-probesize', + '32', + '-analyzeduration', + '0', + '-i', + 'pipe:0', + ]); + let stderr = ''; + + ffmpeg.stderr.setEncoding('utf8'); + ffmpeg.stderr.on('data', (chunk: string) => { + stderr += chunk; + }); + ffmpeg.stdin.on('error', () => {}); + ffmpeg.on('error', reject); + ffmpeg.on('close', () => { + resolve(stderr.match(/Input #0, (.*), from 'pipe:0':/)?.[1]); + }); + + ffmpeg.stdin.end(data); + }); + + if (container && compressedFormats.has(container)) { throw new Error(`Audio data isn't PCM (detected ${container})`); } }; @@ -100,7 +111,7 @@ export const tts = async ( frames.push(event.frame); } - assertPCM(frames); + await assertPCM(frames); await validate(frames, stt, TEXT, 0.2); stream.close(); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3f01e7345..d600520d7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1354,6 +1354,9 @@ importers: plugins/test: dependencies: + '@ffmpeg-installer/ffmpeg': + specifier: ^1.1.0 + version: 1.1.0 fastest-levenshtein: specifier: ^1.0.16 version: 1.0.16