diff --git a/plugins/test/package.json b/plugins/test/package.json index 81d4ba098..d0daa2e2e 100644 --- a/plugins/test/package.json +++ b/plugins/test/package.json @@ -39,6 +39,7 @@ "zod": "^3.25.76" }, "dependencies": { + "@ffmpeg-installer/ffmpeg": "^1.1.0", "fastest-levenshtein": "^1.0.16", "vitest": "^4.0.17" }, diff --git a/plugins/test/src/tts.ts b/plugins/test/src/tts.ts index 3692b1202..6c7caf141 100644 --- a/plugins/test/src/tts.ts +++ b/plugins/test/src/tts.ts @@ -1,16 +1,63 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +import ffmpegInstaller from '@ffmpeg-installer/ffmpeg'; import type { stt } from '@livekit/agents'; import { type AudioBuffer, initializeLogger, tokenize, tts as ttslib } from '@livekit/agents'; -import type { AudioFrame } from '@livekit/rtc-node'; +import { type AudioFrame, combineAudioFrames } from '@livekit/rtc-node'; import { distance } from 'fastest-levenshtein'; +import { spawn } from 'node:child_process'; import { ReadableStream } from 'stream/web'; import { describe, expect, it } from 'vitest'; const TEXT = 'The people who are crazy enough to think they can change the world are the ones who do.'; +const compressedFormats = new Set([ + 'mp3', + 'aac', + 'ogg', + 'flac', + 'wav', + 'mov,mp4,m4a,3gp,3g2,mj2', + 'matroska,webm', + 'mpeg', +]); + +const assertPCM = async (frames: AudioFrame[]) => { + const frame = combineAudioFrames(frames); + const data = new Uint8Array(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength); + + const container = await new Promise((resolve, reject) => { + const ffmpeg = spawn(ffmpegInstaller.path, [ + '-hide_banner', + '-probesize', + '32', + '-analyzeduration', + '0', + '-i', + 'pipe:0', + ]); + let stderr = ''; + + ffmpeg.stderr.setEncoding('utf8'); + ffmpeg.stderr.on('data', (chunk: string) => { + stderr += chunk; + }); + ffmpeg.stdin.on('error', () => {}); + ffmpeg.on('error', reject); + ffmpeg.on('close', () => { + resolve(stderr.match(/Input #0, (.*), from 'pipe:0':/)?.[1]); + }); + + ffmpeg.stdin.end(data); + }); + + if (container && compressedFormats.has(container)) { + throw new Error(`Audio data isn't PCM (detected ${container})`); + } +}; + const validate = async (frames: AudioBuffer, stt: stt.STT, text: string, threshold: number) => { const event = await stt.recognize(frames); const eventText = event.alternatives![0].text.toLowerCase().replace(/\s/g, ' ').trim(); @@ -64,6 +111,7 @@ export const tts = async ( frames.push(event.frame); } + await assertPCM(frames); await validate(frames, stt, TEXT, 0.2); stream.close(); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3f01e7345..d600520d7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1354,6 +1354,9 @@ importers: plugins/test: dependencies: + '@ffmpeg-installer/ffmpeg': + specifier: ^1.1.0 + version: 1.1.0 fastest-levenshtein: specifier: ^1.0.16 version: 1.0.16