diff --git a/.changeset/azure-stt-cancellation-recovery.md b/.changeset/azure-stt-cancellation-recovery.md new file mode 100644 index 000000000..0c5d789f8 --- /dev/null +++ b/.changeset/azure-stt-cancellation-recovery.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-azure': minor +--- + +Add Azure streaming STT with cancellation error recovery. diff --git a/plugins/azure/README.md b/plugins/azure/README.md new file mode 100644 index 000000000..a1c1606ae --- /dev/null +++ b/plugins/azure/README.md @@ -0,0 +1,18 @@ + + +# Azure plugin for LiveKit Agents + +The Agents Framework is designed for building realtime, programmable +participants that run on servers. Use it to create conversational, multi-modal +voice agents that can see, hear, and understand. + +This package contains the Azure plugin, which allows for speech recognition. +Refer to the [documentation](https://docs.livekit.io/agents/overview/) for +information on how to use it, or browse the [API +reference](https://docs.livekit.io/agents-js/). See the +[repository](https://github.com/livekit/agents-js) for more information about +the framework as a whole. diff --git a/plugins/azure/api-extractor.json b/plugins/azure/api-extractor.json new file mode 100644 index 000000000..baa041649 --- /dev/null +++ b/plugins/azure/api-extractor.json @@ -0,0 +1,8 @@ +/** + * Config file for API Extractor. For more info, please visit: https://api-extractor.com + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + "extends": "../../api-extractor-shared.json", + "mainEntryPointFilePath": "./dist/index.d.ts" +} diff --git a/plugins/azure/etc/agents-plugin-azure.api.md b/plugins/azure/etc/agents-plugin-azure.api.md new file mode 100644 index 000000000..4d6f3ad71 --- /dev/null +++ b/plugins/azure/etc/agents-plugin-azure.api.md @@ -0,0 +1,155 @@ +## API Report File for "@livekit/agents-plugin-azure" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import { APIConnectOptions } from '@livekit/agents'; +import { AudioBuffer as AudioBuffer_2 } from '@livekit/agents'; +import * as speechsdk from 'microsoft-cognitiveservices-speech-sdk'; +import { stt } from '@livekit/agents'; + +// @internal (undocumented) +export interface _CanceledEvent { + // (undocumented) + errorCode?: speechsdk.CancellationErrorCode; + // (undocumented) + errorDetails?: string; + // (undocumented) + reason: speechsdk.CancellationReason; +} + +export { speechsdk } + +// @public (undocumented) +export class SpeechStream extends stt.SpeechStream { + constructor(stt: STT, opts: STTOptions, connOptions: APIConnectOptions); + // @internal (undocumented) + _cancellationError: _CanceledEvent | null; + // (undocumented) + label: string; + // @internal (undocumented) + _onCanceled(evt: _CanceledEvent): void; + // @internal (undocumented) + _onRecognized(evt: speechsdk.SpeechRecognitionEventArgs): void; + // @internal (undocumented) + _onRecognizing(evt: speechsdk.SpeechRecognitionEventArgs): void; + // @internal (undocumented) + _onSessionStarted(_evt: speechsdk.SessionEventArgs): void; + // @internal (undocumented) + _onSessionStopped(_evt: speechsdk.SessionEventArgs): void; + // @internal (undocumented) + _onSpeechEnd(_evt: speechsdk.RecognitionEventArgs): void; + // @internal (undocumented) + _onSpeechStart(_evt: speechsdk.RecognitionEventArgs): void; + // @internal (undocumented) + _opts: STTOptions; + // @internal (undocumented) + _reconnectEvent: _WaitableEvent; + // (undocumented) + protected run(): Promise; + // @internal (undocumented) + _sessionStartedEvent: _WaitableEvent; + // @internal (undocumented) + _sessionStoppedEvent: _WaitableEvent; + // @internal (undocumented) + _speaking: boolean; + // (undocumented) + updateOptions(opts: STTUpdateOptions): void; +} + +// @public (undocumented) +export class STT extends stt.STT { + constructor(opts?: STTConstructorOptions); + // (undocumented) + label: string; + // (undocumented) + get model(): string; + // (undocumented) + get provider(): string; + // (undocumented) + _recognize(_frame: AudioBuffer_2): Promise; + // (undocumented) + stream(options?: { + language?: string; + connOptions?: APIConnectOptions; + }): SpeechStream; + // (undocumented) + updateOptions(opts: STTUpdateOptions): void; +} + +// @public (undocumented) +export interface STTConstructorOptions extends STTUpdateOptions { + // (undocumented) + explicitPunctuation?: boolean; + // (undocumented) + numChannels?: number; + // (undocumented) + phraseList?: string[] | null; + // (undocumented) + profanity?: speechsdk.ProfanityOption; + // (undocumented) + sampleRate?: number; + // (undocumented) + speechAuthToken?: string; + // (undocumented) + speechEndpoint?: string; + // (undocumented) + speechHost?: string; + // (undocumented) + speechKey?: string; + // (undocumented) + speechRegion?: string; + // (undocumented) + trueTextPostProcessing?: boolean; +} + +// @public (undocumented) +export interface STTOptions { + // (undocumented) + explicitPunctuation: boolean; + // (undocumented) + language: string[]; + // (undocumented) + numChannels: number; + // (undocumented) + phraseList?: string[] | null; + // (undocumented) + profanity?: speechsdk.ProfanityOption; + // (undocumented) + sampleRate: number; + // (undocumented) + segmentationMaxTimeMs?: number; + // (undocumented) + segmentationSilenceTimeoutMs?: number; + // (undocumented) + segmentationStrategy?: string; + speechAuthToken?: string; + speechEndpoint?: string; + speechHost?: string; + speechKey?: string; + speechRegion?: string; + // (undocumented) + trueTextPostProcessing: boolean; +} + +// @public (undocumented) +export type STTUpdateOptions = Partial> & { + language?: string | string[]; +}; + +// @internal (undocumented) +export interface _WaitableEvent { + // (undocumented) + clear(): void; + // (undocumented) + isSet: boolean; + // (undocumented) + set(): void; + // (undocumented) + wait(): Promise; +} + +// (No @packageDocumentation comment for this package) + +``` diff --git a/plugins/azure/package.json b/plugins/azure/package.json new file mode 100644 index 000000000..0ffae04d6 --- /dev/null +++ b/plugins/azure/package.json @@ -0,0 +1,50 @@ +{ + "name": "@livekit/agents-plugin-azure", + "version": "1.5.0", + "description": "Azure plugin for LiveKit Agents for Node.js", + "main": "dist/index.js", + "require": "dist/index.cjs", + "types": "dist/index.d.ts", + "exports": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, + "author": "LiveKit", + "type": "module", + "repository": "git@github.com:livekit/agents-js.git", + "license": "Apache-2.0", + "files": [ + "dist", + "src", + "README.md" + ], + "scripts": { + "build": "tsup --onSuccess \"pnpm build:types\"", + "build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js", + "clean": "rm -rf dist", + "clean:build": "pnpm clean && pnpm build", + "lint": "eslint -f unix \"src/**/*.{ts,js}\"", + "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript", + "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose" + }, + "devDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/rtc-node": "catalog:", + "@microsoft/api-extractor": "^7.35.0", + "tsup": "^8.3.5", + "typescript": "^5.0.0" + }, + "dependencies": { + "microsoft-cognitiveservices-speech-sdk": "^1.50.0" + }, + "peerDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/rtc-node": "catalog:" + } +} diff --git a/plugins/azure/src/index.ts b/plugins/azure/src/index.ts new file mode 100644 index 000000000..bbc675974 --- /dev/null +++ b/plugins/azure/src/index.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { Plugin } from '@livekit/agents'; + +export * from './stt.js'; + +class AzurePlugin extends Plugin { + constructor() { + super({ + title: 'azure', + version: __PACKAGE_VERSION__, + package: __PACKAGE_NAME__, + }); + } +} + +Plugin.registerPlugin(new AzurePlugin()); diff --git a/plugins/azure/src/stt.test.ts b/plugins/azure/src/stt.test.ts new file mode 100644 index 000000000..30addd5ff --- /dev/null +++ b/plugins/azure/src/stt.test.ts @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { describe, expect, it } from 'vitest'; +import { SpeechStream, speechsdk } from './stt.js'; + +function canceledEvent( + reason: speechsdk.CancellationReason, + errorCode?: speechsdk.CancellationErrorCode, + errorDetails = '', +) { + return { reason, errorCode, errorDetails }; +} + +describe('Azure STT cancellation handling', () => { + it('unblocks run on canceled error', () => { + const stream = SpeechStream.prototype as SpeechStream; + const testStream = Object.create(stream) as SpeechStream; + testStream._sessionStoppedEvent = { + isSet: false, + set() { + this.isSet = true; + }, + clear() { + this.isSet = false; + }, + wait: () => Promise.resolve(), + } as SpeechStream['_sessionStoppedEvent']; + testStream._cancellationError = null; + + const event = canceledEvent( + speechsdk.CancellationReason.Error, + speechsdk.CancellationErrorCode.ServiceTimeout, + 'timeout', + ); + testStream._onCanceled(event); + + expect(testStream._sessionStoppedEvent.isSet).toBe(true); + expect(testStream._cancellationError).toBe(event); + }); + + it('ignores cancellations without error', () => { + const stream = SpeechStream.prototype as SpeechStream; + const testStream = Object.create(stream) as SpeechStream; + testStream._sessionStoppedEvent = { + isSet: false, + set() { + this.isSet = true; + }, + clear() { + this.isSet = false; + }, + wait: () => Promise.resolve(), + } as SpeechStream['_sessionStoppedEvent']; + testStream._cancellationError = null; + + testStream._onCanceled(canceledEvent(speechsdk.CancellationReason.EndOfStream)); + + expect(testStream._sessionStoppedEvent.isSet).toBe(false); + expect(testStream._cancellationError).toBeNull(); + }); +}); diff --git a/plugins/azure/src/stt.ts b/plugins/azure/src/stt.ts new file mode 100644 index 000000000..59c595ad9 --- /dev/null +++ b/plugins/azure/src/stt.ts @@ -0,0 +1,566 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { + type APIConnectOptions, + APIConnectionError, + type AudioBuffer, + DEFAULT_API_CONNECT_OPTIONS, + asLanguageCode, + log, + stt, +} from '@livekit/agents'; +import type { AudioFrame } from '@livekit/rtc-node'; +import * as speechsdk from 'microsoft-cognitiveservices-speech-sdk'; + +export { speechsdk }; + +/** @public */ +export interface STTOptions { + /** Azure Speech subscription key. Defaults to `AZURE_SPEECH_KEY`. */ + speechKey?: string; + /** Azure Speech region. Defaults to `AZURE_SPEECH_REGION`. */ + speechRegion?: string; + /** Azure Speech container host. Defaults to `AZURE_SPEECH_HOST`. */ + speechHost?: string; + /** Ephemeral Microsoft Entra auth token. */ + speechAuthToken?: string; + /** Azure Speech endpoint URL. */ + speechEndpoint?: string; + sampleRate: number; + numChannels: number; + segmentationSilenceTimeoutMs?: number; + segmentationMaxTimeMs?: number; + segmentationStrategy?: string; + language: string[]; + profanity?: speechsdk.ProfanityOption; + phraseList?: string[] | null; + explicitPunctuation: boolean; + trueTextPostProcessing: boolean; +} + +/** @public */ +export type STTUpdateOptions = Partial< + Pick< + STTOptions, + 'segmentationSilenceTimeoutMs' | 'segmentationMaxTimeMs' | 'segmentationStrategy' + > +> & { + language?: string | string[]; +}; + +/** @public */ +export interface STTConstructorOptions extends STTUpdateOptions { + speechKey?: string; + speechRegion?: string; + speechHost?: string; + speechAuthToken?: string; + speechEndpoint?: string; + sampleRate?: number; + numChannels?: number; + profanity?: speechsdk.ProfanityOption; + phraseList?: string[] | null; + explicitPunctuation?: boolean; + trueTextPostProcessing?: boolean; +} + +/** @internal */ +export interface _CanceledEvent { + errorDetails?: string; + reason: speechsdk.CancellationReason; + errorCode?: speechsdk.CancellationErrorCode; +} + +/** @internal */ +export interface _WaitableEvent { + isSet: boolean; + wait(): Promise; + set(): void; + clear(): void; +} + +class DeferredEvent implements _WaitableEvent { + #resolve?: () => void; + #promise: Promise; + isSet = false; + + constructor() { + this.#promise = new Promise((resolve) => { + this.#resolve = resolve; + }); + } + + wait(): Promise { + return this.#promise; + } + + set(): void { + if (this.isSet) return; + this.isSet = true; + this.#resolve?.(); + } + + clear(): void { + if (!this.isSet) return; + this.isSet = false; + this.#promise = new Promise((resolve) => { + this.#resolve = resolve; + }); + } +} + +const defaultSTTOptions = { + sampleRate: 16000, + numChannels: 1, + language: ['en-US'], + explicitPunctuation: false, + trueTextPostProcessing: false, +} satisfies Pick< + STTOptions, + 'sampleRate' | 'numChannels' | 'language' | 'explicitPunctuation' | 'trueTextPostProcessing' +>; + +/** @public */ +export class STT extends stt.STT { + #opts: STTOptions; + #streams = new Set>(); + label = 'azure.STT'; + + get model(): string { + return 'unknown'; + } + + get provider(): string { + return 'Azure STT'; + } + + constructor(opts: STTConstructorOptions = {}) { + super({ streaming: true, interimResults: true, alignedTranscript: 'chunk' }); + + const speechHost = opts.speechHost ?? process.env.AZURE_SPEECH_HOST; + const speechKey = opts.speechKey ?? process.env.AZURE_SPEECH_KEY; + const speechRegion = opts.speechRegion ?? process.env.AZURE_SPEECH_REGION; + const speechAuthToken = opts.speechAuthToken; + const speechEndpoint = opts.speechEndpoint; + + if ( + !speechHost && + !(speechKey && speechRegion) && + !(speechAuthToken && speechRegion) && + !(speechKey && speechEndpoint) + ) { + throw new Error( + 'AZURE_SPEECH_HOST or AZURE_SPEECH_KEY and AZURE_SPEECH_REGION or speechAuthToken and AZURE_SPEECH_REGION or AZURE_SPEECH_KEY and speechEndpoint must be set', + ); + } + + if (speechRegion && speechEndpoint) { + log().warn('speechRegion and speechEndpoint both are set, using speechEndpoint'); + } + + this.#opts = { + ...defaultSTTOptions, + ...opts, + speechHost, + speechKey, + speechRegion: speechEndpoint ? undefined : speechRegion, + speechAuthToken, + speechEndpoint, + language: normalizeLanguages(opts.language), + }; + } + + async _recognize(_frame: AudioBuffer): Promise { + throw new Error('Azure STT does not support single frame recognition'); + } + + updateOptions(opts: STTUpdateOptions): void { + this.#opts = { + ...this.#opts, + ...opts, + language: + opts.language === undefined ? this.#opts.language : normalizeLanguages(opts.language), + }; + + for (const ref of this.#streams) { + const stream = ref.deref(); + if (stream) { + stream.updateOptions(opts); + } else { + this.#streams.delete(ref); + } + } + } + + stream(options: { language?: string; connOptions?: APIConnectOptions } = {}): SpeechStream { + const opts = { + ...this.#opts, + language: options.language ? [options.language] : [...this.#opts.language], + }; + const stream = new SpeechStream(this, opts, options.connOptions ?? DEFAULT_API_CONNECT_OPTIONS); + this.#streams.add(new WeakRef(stream)); + return stream; + } +} + +/** @public */ +export class SpeechStream extends stt.SpeechStream { + /** @internal */ + _opts: STTOptions; + /** @internal */ + _speaking = false; + /** @internal */ + _sessionStoppedEvent: _WaitableEvent = new DeferredEvent(); + /** @internal */ + _sessionStartedEvent: _WaitableEvent = new DeferredEvent(); + /** @internal */ + _reconnectEvent: _WaitableEvent = new DeferredEvent(); + /** @internal */ + _cancellationError: _CanceledEvent | null = null; + #connOptions: APIConnectOptions; + #audioDuration = 0; + #lastAudioDurationReportTime = performance.now(); + label = 'azure.SpeechStream'; + + constructor(stt: STT, opts: STTOptions, connOptions: APIConnectOptions) { + super(stt, opts.sampleRate, connOptions); + this._opts = opts; + this.#connOptions = connOptions; + } + + updateOptions(opts: STTUpdateOptions): void { + this._opts = { + ...this._opts, + ...opts, + language: + opts.language === undefined ? this._opts.language : normalizeLanguages(opts.language), + }; + this._reconnectEvent.set(); + } + + protected async run(): Promise { + while (!this.input.closed && !this.closed) { + this._sessionStoppedEvent.clear(); + this._sessionStartedEvent.clear(); + this._cancellationError = null; + + const pushStream = speechsdk.AudioInputStream.createPushStream( + speechsdk.AudioStreamFormat.getWaveFormatPCM( + this._opts.sampleRate, + 16, + this._opts.numChannels, + ), + ); + const recognizer = createSpeechRecognizer(this._opts, pushStream); + this.#connectRecognizerEvents(recognizer); + + await new Promise((resolve, reject) => { + recognizer.startContinuousRecognitionAsync(resolve, (error) => reject(new Error(error))); + }); + + try { + await withTimeout(this._sessionStartedEvent.wait(), this.#connOptions.timeoutMs); + + const inputTask = this.#processInput(pushStream); + const completed = await Promise.race([ + inputTask.then(() => 'input' as const), + this._reconnectEvent.wait().then(() => 'reconnect' as const), + this._sessionStoppedEvent.wait().then(() => 'stopped' as const), + ]); + + if (completed === 'stopped') { + const details = this._cancellationError as _CanceledEvent | null; + if (details !== null) { + throw new APIConnectionError({ + message: + `Azure STT canceled: ${details.errorDetails || details.reason} ` + + `(${details.errorCode})`, + }); + } + throw new APIConnectionError({ message: 'SpeechRecognition session stopped' }); + } + + if (completed === 'reconnect') { + this._reconnectEvent.clear(); + } + + if (completed === 'input') { + pushStream.close(); + await this._sessionStoppedEvent.wait(); + break; + } + } finally { + await new Promise((resolve) => { + recognizer.stopContinuousRecognitionAsync(resolve, () => resolve()); + }); + recognizer.close(); + } + } + } + + async #processInput(pushStream: speechsdk.PushAudioInputStream): Promise { + try { + while (!this.closed) { + const result = await this.input.next(); + if (result.done) break; + + const input = result.value; + if (input === SpeechStream.FLUSH_SENTINEL) { + this.#emitRecognitionUsage(); + continue; + } + + this.#audioDuration += input.samplesPerChannel / input.sampleRate; + this.#maybeEmitRecognitionUsage(); + pushStream.write(toArrayBuffer(input)); + } + } finally { + this.#emitRecognitionUsage(); + } + } + + #connectRecognizerEvents(recognizer: speechsdk.SpeechRecognizer): void { + recognizer.recognizing = (_sender, evt) => this._onRecognizing(evt); + recognizer.recognized = (_sender, evt) => this._onRecognized(evt); + recognizer.speechStartDetected = (_sender, evt) => this._onSpeechStart(evt); + recognizer.speechEndDetected = (_sender, evt) => this._onSpeechEnd(evt); + recognizer.sessionStarted = (_sender, evt) => this._onSessionStarted(evt); + recognizer.sessionStopped = (_sender, evt) => this._onSessionStopped(evt); + recognizer.canceled = (_sender, evt) => this._onCanceled(evt); + } + + /** @internal */ + _onRecognized(evt: speechsdk.SpeechRecognitionEventArgs): void { + const text = evt.result.text.trim(); + if (!text) return; + + this.queue.put({ + type: stt.SpeechEventType.FINAL_TRANSCRIPT, + alternatives: [this.#speechData(evt, 1)], + }); + } + + /** @internal */ + _onRecognizing(evt: speechsdk.SpeechRecognitionEventArgs): void { + const text = evt.result.text.trim(); + if (!text) return; + + this.queue.put({ + type: stt.SpeechEventType.INTERIM_TRANSCRIPT, + alternatives: [this.#speechData(evt, 0)], + }); + } + + /** @internal */ + _onSpeechStart(_evt: speechsdk.RecognitionEventArgs): void { + if (this._speaking) return; + this._speaking = true; + this.queue.put({ type: stt.SpeechEventType.START_OF_SPEECH }); + } + + /** @internal */ + _onSpeechEnd(_evt: speechsdk.RecognitionEventArgs): void { + if (!this._speaking) return; + this._speaking = false; + this.queue.put({ type: stt.SpeechEventType.END_OF_SPEECH }); + } + + /** @internal */ + _onSessionStarted(_evt: speechsdk.SessionEventArgs): void { + this._sessionStartedEvent.set(); + } + + /** @internal */ + _onSessionStopped(_evt: speechsdk.SessionEventArgs): void { + this._sessionStoppedEvent.set(); + } + + /** @internal */ + _onCanceled(evt: _CanceledEvent): void { + if (evt.reason === speechsdk.CancellationReason.Error) { + log().warn( + { + code: evt.errorCode, + reason: evt.reason, + errorDetails: evt.errorDetails, + }, + `Speech recognition canceled: ${evt.errorDetails || evt.reason}`, + ); + this._cancellationError = evt; + this._sessionStoppedEvent.set(); + } + } + + #speechData(evt: speechsdk.SpeechRecognitionEventArgs, confidence: number): stt.SpeechData { + const result = speechsdk.AutoDetectSourceLanguageResult.fromResult(evt.result); + const language = result.language || this._opts.language[0] || ''; + return { + language: asLanguageCode(language), + confidence, + text: evt.result.text, + startTime: evt.result.offset / 10 ** 7 + this.startTimeOffset, + endTime: (evt.result.offset + evt.result.duration) / 10 ** 7 + this.startTimeOffset, + }; + } + + #maybeEmitRecognitionUsage(): void { + if (performance.now() - this.#lastAudioDurationReportTime >= 5000) { + this.#emitRecognitionUsage(); + } + } + + #emitRecognitionUsage(): void { + if (this.#audioDuration <= 0) return; + + const audioDuration = this.#audioDuration; + this.#audioDuration = 0; + this.#lastAudioDurationReportTime = performance.now(); + this.queue.put({ + type: stt.SpeechEventType.RECOGNITION_USAGE, + recognitionUsage: { audioDuration }, + }); + } +} + +function normalizeLanguages(language?: string | string[]): string[] { + if (language === undefined) return [...defaultSTTOptions.language]; + return Array.isArray(language) ? [...language] : [language]; +} + +function createSpeechRecognizer( + config: STTOptions, + stream: speechsdk.PushAudioInputStream, +): speechsdk.SpeechRecognizer { + const speechConfig = createSpeechConfig(config); + + if (config.segmentationSilenceTimeoutMs !== undefined) { + speechConfig.setProperty( + speechsdk.PropertyId.Speech_SegmentationSilenceTimeoutMs, + String(config.segmentationSilenceTimeoutMs), + ); + } + if (config.segmentationMaxTimeMs !== undefined) { + speechConfig.setProperty( + speechsdk.PropertyId.Speech_SegmentationMaximumTimeMs, + String(config.segmentationMaxTimeMs), + ); + } + if (config.segmentationStrategy !== undefined) { + speechConfig.setProperty( + speechsdk.PropertyId.Speech_SegmentationStrategy, + config.segmentationStrategy, + ); + } + if (config.profanity !== undefined) { + speechConfig.setProfanity(config.profanity); + } + if (config.explicitPunctuation) { + speechConfig.setServiceProperty( + 'punctuation', + 'explicit', + speechsdk.ServicePropertyChannel.UriQueryParameter, + ); + } + if (config.trueTextPostProcessing) { + speechConfig.setProperty( + speechsdk.PropertyId.SpeechServiceResponse_PostProcessingOption, + 'TrueText', + ); + } + + const audioConfig = speechsdk.AudioConfig.fromStreamInput(stream); + const recognizer = + config.language.length > 1 + ? createMultiLanguageRecognizer(speechConfig, audioConfig, config.language) + : new speechsdk.SpeechRecognizer(speechConfig, audioConfig); + + if (config.phraseList?.length) { + const phraseListGrammar = speechsdk.PhraseListGrammar.fromRecognizer(recognizer); + for (const phrase of config.phraseList) { + phraseListGrammar.addPhrase(phrase); + } + } + + return recognizer; +} + +function createSpeechConfig(config: STTOptions): speechsdk.SpeechConfig { + if (config.speechEndpoint) { + const speechConfig = speechsdk.SpeechConfig.fromEndpoint( + new URL(config.speechEndpoint), + config.speechKey, + ); + if (config.speechAuthToken) speechConfig.authorizationToken = config.speechAuthToken; + speechConfig.speechRecognitionLanguage = firstLanguage(config); + return speechConfig; + } + if (config.speechHost) { + const speechConfig = speechsdk.SpeechConfig.fromHost( + new URL(config.speechHost), + config.speechKey, + ); + if (config.speechAuthToken) speechConfig.authorizationToken = config.speechAuthToken; + speechConfig.speechRecognitionLanguage = firstLanguage(config); + return speechConfig; + } + if (config.speechAuthToken) { + const speechConfig = speechsdk.SpeechConfig.fromAuthorizationToken( + config.speechAuthToken, + config.speechRegion!, + ); + speechConfig.speechRecognitionLanguage = firstLanguage(config); + return speechConfig; + } + const speechConfig = speechsdk.SpeechConfig.fromSubscription( + config.speechKey!, + config.speechRegion!, + ); + speechConfig.speechRecognitionLanguage = firstLanguage(config); + return speechConfig; +} + +function firstLanguage(config: STTOptions): string { + return config.language[0] ?? 'en-US'; +} + +function createMultiLanguageRecognizer( + speechConfig: speechsdk.SpeechConfig, + audioConfig: speechsdk.AudioConfig, + languages: string[], +): speechsdk.SpeechRecognizer { + speechConfig.setProperty( + speechsdk.PropertyId.SpeechServiceConnection_LanguageIdMode, + 'Continuous', + ); + const autoDetectSourceLanguageConfig = + speechsdk.AutoDetectSourceLanguageConfig.fromLanguages(languages); + return speechsdk.SpeechRecognizer.FromConfig( + speechConfig, + autoDetectSourceLanguageConfig, + audioConfig, + ); +} + +function toArrayBuffer(frame: AudioFrame): ArrayBuffer { + const view = new Uint8Array(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength); + const buffer = new ArrayBuffer(view.byteLength); + new Uint8Array(buffer).set(view); + return buffer; +} + +async function withTimeout(promise: Promise, timeoutMs: number): Promise { + let timeout: NodeJS.Timeout | undefined; + try { + await Promise.race([ + promise, + new Promise((_, reject) => { + timeout = setTimeout( + () => reject(new APIConnectionError({ message: 'Request timed out.' })), + timeoutMs, + ); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } +} diff --git a/plugins/azure/tsconfig.json b/plugins/azure/tsconfig.json new file mode 100644 index 000000000..b8a84c3ed --- /dev/null +++ b/plugins/azure/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.json", + "include": ["./src"], + "compilerOptions": { + "rootDir": "./src", + "declarationDir": "./dist", + "outDir": "./dist" + }, + "typedocOptions": { + "name": "plugins/agents-plugin-azure", + "entryPointStrategy": "resolve", + "readme": "none", + "entryPoints": ["src/index.ts"] + } +} diff --git a/plugins/azure/tsup.config.ts b/plugins/azure/tsup.config.ts new file mode 100644 index 000000000..b491713a4 --- /dev/null +++ b/plugins/azure/tsup.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from 'tsup'; +import defaults from '../../tsup.config'; + +export default defineConfig({ + ...defaults, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f00f5059b..986624a6a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -60,7 +60,7 @@ importers: version: 8.10.0(eslint@8.57.0) eslint-config-standard: specifier: ^17.1.0 - version: 17.1.0(eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint@8.57.0))(eslint-plugin-n@16.6.2(eslint@8.57.0))(eslint-plugin-promise@6.1.1(eslint@8.57.0))(eslint@8.57.0) + version: 17.1.0(eslint-plugin-import@2.29.1)(eslint-plugin-n@16.6.2(eslint@8.57.0))(eslint-plugin-promise@6.1.1(eslint@8.57.0))(eslint@8.57.0) eslint-config-turbo: specifier: ^2.9.14 version: 2.10.1(eslint@8.57.0)(turbo@2.9.14) @@ -478,6 +478,28 @@ importers: specifier: ^5.0.0 version: 5.9.3 + plugins/azure: + dependencies: + microsoft-cognitiveservices-speech-sdk: + specifier: ^1.50.0 + version: 1.50.0 + devDependencies: + '@livekit/agents': + specifier: workspace:* + version: link:../../agents + '@livekit/rtc-node': + specifier: 'catalog:' + version: 0.13.31 + '@microsoft/api-extractor': + specifier: ^7.35.0 + version: 7.43.7(@types/node@25.6.0) + tsup: + specifier: ^8.3.5 + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@25.6.0))(postcss@8.5.9)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.0.0 + version: 5.9.3 + plugins/baseten: dependencies: dotenv: @@ -1450,6 +1472,18 @@ packages: '@anthropic-ai/sdk@0.33.1': resolution: {integrity: sha512-VrlbxiAdVRGuKP2UQlCnsShDHJKWepzvfRCkZMpU+oaUdKLpOfmylLMRojGrAgebV+kDtPjewCVP0laHXg+vsA==} + '@azure/abort-controller@2.1.2': + resolution: {integrity: sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==} + engines: {node: '>=18.0.0'} + + '@azure/core-auth@1.10.1': + resolution: {integrity: sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==} + engines: {node: '>=20.0.0'} + + '@azure/core-util@1.13.1': + resolution: {integrity: sha512-XPArKLzsvl0Hf0CaGyKHUyVgF7oDnhKoP85Xv6M4StF/1AhfORhZudHtOyf2s+FcbuQ9dPRAjB8J2KvRRMUK2A==} + engines: {node: '>=20.0.0'} + '@babel/code-frame@7.24.2': resolution: {integrity: sha512-y5+tLQyV8pg3fsiln67BVLD1P13Eg4lh5RW9mF0zUuvLrv9uIQ4MCL+CRT+FTsBlBjcIan6PGsLcBN0m3ClUyQ==} engines: {node: '>=6.9.0'} @@ -2809,6 +2843,9 @@ packages: '@types/tapable@1.0.6': resolution: {integrity: sha512-W+bw9ds02rAQaMvaLYxAbJ6cvguW/iJXNT6lTssS1ps6QdrMKttqEAMEG/b5CR8TZl3/L7/lH0ZV5nNR1LXikA==} + '@types/webrtc@0.0.37': + resolution: {integrity: sha512-JGAJC/ZZDhcrrmepU4sPLQLIOIAgs5oIK+Ieq90K8fdaNMhfdfqmYatJdgif1NDQtvrSlTOGJDUYHIDunuufOg==} + '@types/ws@8.18.1': resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} @@ -2870,6 +2907,10 @@ packages: resolution: {integrity: sha512-JJtkDduxLi9bivAB+cYOVMtbkqdPOhZ+ZI5LC47MIRrDV4Yn2o+ZnW10Nkmr28xRpSpdJ6Sm42Hjf2+REYXm0A==} engines: {node: ^16.0.0 || >=18.0.0} + '@typespec/ts-http-runtime@0.3.6': + resolution: {integrity: sha512-jIXhD0eWQ1JA6ln/5Dltyx22UxWNrw0hZmhy2rlv6m6KgF7kplHx3g0fzi09lNmTJQRR91OlemYp3xFnvDK9og==} + engines: {node: '>=20.0.0'} + '@ungap/structured-clone@1.2.0': resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} deprecated: Potential CWE-502 - Update to 1.3.1 or higher @@ -2935,6 +2976,14 @@ packages: resolution: {integrity: sha512-+Ut8d9LLqwEvHHJl1+PIHqoyDxFgVN847JTVM3Izi3xHDWPE4UtzzXysMZQs64DMcrJfBeS/uoEP4AD3HQHnQQ==} engines: {node: '>=12.0'} + agent-base@5.1.1: + resolution: {integrity: sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==} + engines: {node: '>= 6.0.0'} + + agent-base@6.0.2: + resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} + engines: {node: '>= 6.0.0'} + agent-base@7.1.4: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} @@ -3060,6 +3109,9 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + bent@7.3.12: + resolution: {integrity: sha512-T3yrKnVGB63zRuoco/7Ybl7BwwGZR0lceoVG5XmQyMIH9s19SV5m+a8qam4if0zQuAmOQTyPTPmsQBdAorGK3w==} + better-path-resolve@1.0.0: resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==} engines: {node: '>=4'} @@ -3104,6 +3156,9 @@ packages: peerDependencies: esbuild: '>=0.18' + bytesish@0.4.4: + resolution: {integrity: sha512-i4uu6M4zuMUiyfZN4RU2+i9+peJh//pXhd9x1oSe1LBkZ3LEbCoygu8W0bXTukU1Jme2txKuotpCZRaC3FLxcQ==} + cac@6.7.14: resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} engines: {node: '>=8'} @@ -3128,6 +3183,9 @@ packages: resolution: {integrity: sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==} engines: {node: '>=16'} + caseless@0.12.0: + resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==} + chai@6.2.2: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} @@ -3848,6 +3906,14 @@ packages: html-escaper@2.0.2: resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} + http-proxy-agent@7.0.2: + resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} + engines: {node: '>= 14'} + + https-proxy-agent@4.0.0: + resolution: {integrity: sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==} + engines: {node: '>= 6.0.0'} + https-proxy-agent@7.0.6: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} @@ -3985,6 +4051,10 @@ packages: resolution: {integrity: sha512-nA2hv5XIhLR3uVzDDfCIknerhx8XUKnstuOERPNNIinXG7v9u+ohXF67vxm4TPTEPU6lm61ZkwP3c9PCB97rhg==} engines: {node: '>= 0.4'} + is-stream@2.0.1: + resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} + engines: {node: '>=8'} + is-string@1.0.7: resolution: {integrity: sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==} engines: {node: '>= 0.4'} @@ -4239,6 +4309,9 @@ packages: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} + microsoft-cognitiveservices-speech-sdk@1.50.0: + resolution: {integrity: sha512-h2OE1YYokwwM0mrBSmOp7NxNr6LlzsT7hISocKxWgwogfJ5TI6SjFKI5Na1gETeJPcuLxnw2tmExB/E/0tCZ/g==} + mime-db@1.52.0: resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} engines: {node: '>= 0.6'} @@ -5071,6 +5144,11 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + uuid@9.0.1: + resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). + hasBin: true + validator@13.12.0: resolution: {integrity: sha512-c1Q0mCiPlgdTVVVIJIrBuxNicYE+t/7oKeI9MWLj3fh/uq2Pxh/3eeWbVZ4OcGW1TUf53At0njHw5SMdA3tmMg==} engines: {node: '>= 0.10'} @@ -5277,6 +5355,26 @@ snapshots: transitivePeerDependencies: - encoding + '@azure/abort-controller@2.1.2': + dependencies: + tslib: 2.6.2 + + '@azure/core-auth@1.10.1': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-util': 1.13.1 + tslib: 2.6.2 + transitivePeerDependencies: + - supports-color + + '@azure/core-util@1.13.1': + dependencies: + '@azure/abort-controller': 2.1.2 + '@typespec/ts-http-runtime': 0.3.6 + tslib: 2.6.2 + transitivePeerDependencies: + - supports-color + '@babel/code-frame@7.24.2': dependencies: '@babel/highlight': 7.24.5 @@ -6604,6 +6702,8 @@ snapshots: '@types/tapable@1.0.6': {} + '@types/webrtc@0.0.37': {} + '@types/ws@8.18.1': dependencies: '@types/node': 22.19.1 @@ -6694,6 +6794,14 @@ snapshots: '@typescript-eslint/types': 6.21.0 eslint-visitor-keys: 3.4.3 + '@typespec/ts-http-runtime@0.3.6': + dependencies: + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + tslib: 2.6.2 + transitivePeerDependencies: + - supports-color + '@ungap/structured-clone@1.2.0': {} '@vitest/coverage-v8@4.0.17(vitest@4.0.17(@opentelemetry/api@1.9.0)(@types/node@22.19.1)(tsx@4.21.0))': @@ -6727,14 +6835,6 @@ snapshots: optionalDependencies: vite: 7.3.2(@types/node@22.19.1)(tsx@4.21.0) - '@vitest/mocker@4.0.17(vite@7.3.2(@types/node@25.6.0)(tsx@4.21.0))': - dependencies: - '@vitest/spy': 4.0.17 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 7.3.2(@types/node@25.6.0)(tsx@4.21.0) - '@vitest/pretty-format@4.0.17': dependencies: tinyrainbow: 3.0.3 @@ -6773,6 +6873,14 @@ snapshots: adm-zip@0.5.17: {} + agent-base@5.1.1: {} + + agent-base@6.0.2: + dependencies: + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + agent-base@7.1.4: {} agentkeepalive@4.6.0: @@ -6920,6 +7028,12 @@ snapshots: base64-js@1.5.1: {} + bent@7.3.12: + dependencies: + bytesish: 0.4.4 + caseless: 0.12.0 + is-stream: 2.0.1 + better-path-resolve@1.0.0: dependencies: is-windows: 1.0.2 @@ -6963,6 +7077,8 @@ snapshots: esbuild: 0.25.2 load-tsconfig: 0.2.5 + bytesish@0.4.4: {} + cac@6.7.14: {} call-bind-apply-helpers@1.0.2: @@ -6989,6 +7105,8 @@ snapshots: camelcase@8.0.0: {} + caseless@0.12.0: {} + chai@6.2.2: {} chalk@2.4.2: @@ -7340,7 +7458,7 @@ snapshots: dependencies: eslint: 8.57.0 - eslint-config-standard@17.1.0(eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint@8.57.0))(eslint-plugin-n@16.6.2(eslint@8.57.0))(eslint-plugin-promise@6.1.1(eslint@8.57.0))(eslint@8.57.0): + eslint-config-standard@17.1.0(eslint-plugin-import@2.29.1)(eslint-plugin-n@16.6.2(eslint@8.57.0))(eslint-plugin-promise@6.1.1(eslint@8.57.0))(eslint@8.57.0): dependencies: eslint: 8.57.0 eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) @@ -7366,7 +7484,7 @@ snapshots: debug: 4.4.1 enhanced-resolve: 5.16.1 eslint: 8.57.0 - eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) + eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) fast-glob: 3.3.2 get-tsconfig: 4.7.5 @@ -7378,7 +7496,7 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-module-utils@2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0): + eslint-module-utils@2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0): dependencies: debug: 3.2.7 optionalDependencies: @@ -7406,7 +7524,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) + eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) hasown: 2.0.2 is-core-module: 2.13.1 is-glob: 4.0.3 @@ -7888,6 +8006,20 @@ snapshots: html-escaper@2.0.2: {} + http-proxy-agent@7.0.2: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + https-proxy-agent@4.0.0: + dependencies: + agent-base: 5.1.1 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + https-proxy-agent@7.0.6: dependencies: agent-base: 7.1.4 @@ -8015,6 +8147,8 @@ snapshots: dependencies: call-bind: 1.0.7 + is-stream@2.0.1: {} + is-string@1.0.7: dependencies: has-tostringtag: 1.0.2 @@ -8247,6 +8381,20 @@ snapshots: braces: 3.0.3 picomatch: 2.3.2 + microsoft-cognitiveservices-speech-sdk@1.50.0: + dependencies: + '@azure/core-auth': 1.10.1 + '@types/webrtc': 0.0.37 + agent-base: 6.0.2 + bent: 7.3.12 + https-proxy-agent: 4.0.0 + uuid: 9.0.1 + ws: 8.20.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + mime-db@1.52.0: {} mime-types@2.1.35: @@ -9222,6 +9370,8 @@ snapshots: dependencies: punycode: 2.3.1 + uuid@9.0.1: {} + validator@13.12.0: {} vite@7.3.2(@types/node@22.19.1)(tsx@4.21.0): @@ -9291,7 +9441,7 @@ snapshots: vitest@4.0.17(@opentelemetry/api@1.9.0)(@types/node@25.6.0)(tsx@4.21.0): dependencies: '@vitest/expect': 4.0.17 - '@vitest/mocker': 4.0.17(vite@7.3.2(@types/node@25.6.0)(tsx@4.21.0)) + '@vitest/mocker': 4.0.17(vite@7.3.2(@types/node@22.19.1)(tsx@4.21.0)) '@vitest/pretty-format': 4.0.17 '@vitest/runner': 4.0.17 '@vitest/snapshot': 4.0.17 diff --git a/turbo.json b/turbo.json index e3d0866ae..747dcace3 100644 --- a/turbo.json +++ b/turbo.json @@ -7,6 +7,9 @@ "AZURE_OPENAI_DEPLOYMENT", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_ENTRA_TOKEN", + "AZURE_SPEECH_HOST", + "AZURE_SPEECH_KEY", + "AZURE_SPEECH_REGION", "BEY_API_KEY", "BEY_API_URL", "BEY_AVATAR_ID",