From ee097ad27264667865d14afaa5630156185a1748 Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 05:17:23 +0000 Subject: [PATCH] feat(plugins): add gnani plugin --- .changeset/green-gnani-speak.md | 5 + plugins/gnani/README.md | 174 +++++++ plugins/gnani/api-extractor.json | 5 + plugins/gnani/etc/agents-plugin-gnani.api.md | 118 +++++ plugins/gnani/package.json | 51 ++ plugins/gnani/src/index.ts | 19 + plugins/gnani/src/stt.test.ts | 179 +++++++ plugins/gnani/src/stt.ts | 406 +++++++++++++++ plugins/gnani/src/tts.test.ts | 252 +++++++++ plugins/gnani/src/tts.ts | 516 +++++++++++++++++++ plugins/gnani/tsconfig.json | 14 + plugins/gnani/tsup.config.ts | 6 + pnpm-lock.yaml | 45 +- turbo.json | 1 + 14 files changed, 1777 insertions(+), 14 deletions(-) create mode 100644 .changeset/green-gnani-speak.md create mode 100644 plugins/gnani/README.md create mode 100644 plugins/gnani/api-extractor.json create mode 100644 plugins/gnani/etc/agents-plugin-gnani.api.md create mode 100644 plugins/gnani/package.json create mode 100644 plugins/gnani/src/index.ts create mode 100644 plugins/gnani/src/stt.test.ts create mode 100644 plugins/gnani/src/stt.ts create mode 100644 plugins/gnani/src/tts.test.ts create mode 100644 plugins/gnani/src/tts.ts create mode 100644 plugins/gnani/tsconfig.json create mode 100644 plugins/gnani/tsup.config.ts diff --git a/.changeset/green-gnani-speak.md b/.changeset/green-gnani-speak.md new file mode 100644 index 000000000..2a2830772 --- /dev/null +++ b/.changeset/green-gnani-speak.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-gnani': minor +--- + +Add the Gnani Vachana STT/TTS plugin for LiveKit Agents JS. diff --git a/plugins/gnani/README.md b/plugins/gnani/README.md new file mode 100644 index 000000000..387ed3ee6 --- /dev/null +++ b/plugins/gnani/README.md @@ -0,0 +1,174 @@ +# @livekit/agents-plugin-gnani + +[LiveKit Agents](https://github.com/livekit/agents-js) plugin for **[Gnani](https://gnani.ai/)**: high-accuracy Speech-to-Text (Prisma) and low-latency Text-to-Speech (Timbre) for Indian languages. + +> [Gnani.ai](https://gnani.ai) featuring **Prisma** (STT) and **Timbre** (TTS) models, supporting 10+ Indian languages with real-time streaming, multilingual transcription, and code-switching capabilities. + +## Installation + +```bash +pnpm add @livekit/agents-plugin-gnani +``` + +## Prerequisites + +You need a Gnani API key. Email **[speechstack@gnani.ai](mailto:speechstack@gnani.ai)** to get started; all new accounts receive free credits, no credit card required. + +### Authentication + +All APIs require a single API key: no `organizationId` or `userId` needed. + +**Option 1: Environment variable (recommended):** + +```bash +export GNANI_API_KEY="your-api-key" +``` + +**Option 2: Constructor argument:** + +```ts +const stt = new gnani.STT({ apiKey: 'your-api-key', language: 'hi-IN' }); +const tts = new gnani.TTS({ apiKey: 'your-api-key' }); +``` + +> **Migration note:** If upgrading from an earlier version, remove any `organizationId` and `userId` parameters; they are no longer accepted. + +## Quick Start + +### Speech-to-Text (REST + Streaming) + +```ts +import * as gnani from '@livekit/agents-plugin-gnani'; + +const stt = new gnani.STT({ language: 'hi-IN' }); + +// REST STT (file-based transcription) +const speechEvent = await stt.recognize(audioBuffer); + +// Streaming STT (real-time WebSocket) +const speechStream = stt.stream(); +``` + +### Text-to-Speech + +```ts +import * as gnani from '@livekit/agents-plugin-gnani'; + +// REST (default): single-request batch synthesis +const ttsRest = new gnani.TTS({ voice: 'Karan' }); + +// SSE: chunked synthesis via Server-Sent Events (lower latency) +const ttsSse = new gnani.TTS({ voice: 'Karan', synthesizeMethod: 'sse' }); + +// WebSocket: chunked synthesis over WS (lowest latency) +const ttsWs = new gnani.TTS({ voice: 'Karan', synthesizeMethod: 'websocket' }); +``` + +All three modes work with the standard LiveKit voice agent pipeline. The `synthesizeMethod` controls which transport `synthesize()` uses (REST, SSE, or WebSocket). The `stream()` method always uses WebSocket regardless of this setting. + +## Full Constructor Reference + +### STT: All Parameters + +```ts +const stt = new gnani.STT({ + language: 'en-IN', // Default: 'en-IN' + sampleRate: 16000, // Default: 16000 (also: 8000) + format: 'verbatim', // Default: 'verbatim' (also: 'transcribe') + preferredLanguage: undefined, // Default: undefined + itnNativeNumerals: false, // Default: false + apiKey: undefined, // Default: reads GNANI_API_KEY env var + baseURL: 'https://api.vachana.ai', // Default +}); +``` + +### TTS: All Parameters + +```ts +const tts = new gnani.TTS({ + voice: 'Karan', // Default: 'Karan' (also: Simran, Nara, Riya, Viraj, Raju) + model: 'vachana-voice-v3', // Default: 'vachana-voice-v3' + sampleRate: 16000, // Default: 16000 (also: 8000, 22050, 44100) + encoding: 'linear_pcm', // Default: 'linear_pcm' (also: 'oggopus') + container: 'wav', // Default: 'wav' (also: 'raw', 'mp3', 'mulaw', 'ogg') + numChannels: 1, // Default: 1 + bitrate: undefined, // Default: undefined (also: '96k', '128k', '192k') + synthesizeMethod: 'rest', // Default: 'rest' (also: 'sse', 'websocket') + apiKey: undefined, // Default: reads GNANI_API_KEY env var + baseURL: 'https://api.vachana.ai', // Default +}); +``` + +## Features + +### STT (Prisma) + +- **REST recognition**: REST API (`POST /stt/v3`) for file-based transcription +- **Real-time streaming**: WebSocket API (`wss://api.vachana.ai/stt/v3/stream`) for live audio transcription with VAD +- **10+ Indian languages**: see [supported language codes](https://docs.gnani.ai/api/STT/stt-websocket#supported-languages) +- **Code-switching**: supports multilingual and code-mixed audio +- **Sample rates**: 8 kHz and 16 kHz +- **ITN support**: Inverse Text Normalization via `format: 'transcribe'` + +#### Streaming PCM Specification + +All streaming audio must be sent as **raw PCM binary frames**: no container format (WAV, MP3) mid-stream. + +| Property | 16 kHz | 8 kHz | +| ------------------- | --------------------------------------- | --------------------------------------- | +| Encoding | PCM signed 16-bit little-endian | PCM signed 16-bit little-endian | +| Sample Rate | 16,000 Hz | 8,000 Hz | +| Channels | 1 (mono) | 1 (mono) | +| Samples per chunk | 512 | 512 | +| **Bytes per frame** | **1,024 bytes** (512 samples x 2 bytes) | **1,024 bytes** (512 samples x 2 bytes) | +| Frame duration | 32 ms | 64 ms | + +Frames must be sent at **real-time cadence**. See **[STT Realtime: PCM Specification](https://docs.gnani.ai/api/STT/stt-websocket#pcm-specification)** for full details. + +### TTS (Timbre) + +- **REST synthesis**: single-request batch audio generation (`synthesizeMethod: 'rest'`) +- **SSE streaming**: lower-latency chunked synthesis via Server-Sent Events (`synthesizeMethod: 'sse'`) +- **WebSocket synthesis**: lowest-latency synthesis via `synthesizeMethod: 'websocket'` or the `stream()` method +- **6 voices**: Karan, Simran, Nara, Riya, Viraj, Raju +- **Configurable output**: sample rate (8000-44100), encoding (linear_pcm, oggopus), container (raw, mp3, wav, mulaw, ogg) +- **Runtime updates**: change voice or model via `updateOptions()` + +## Supported Languages + +### STT Languages (Prisma) + +Prisma uses BCP-47 locale codes (e.g. `hi-IN`). Supported: + +- **[STT REST: Supported Languages](https://docs.gnani.ai/api/STT/speech-to-text#supported-languages)** +- **[STT Realtime: Supported Languages](https://docs.gnani.ai/api/STT/stt-websocket#supported-languages)** + +### TTS Languages (Timbre) + +For the full list of supported languages, see **[TTS: Supported Languages](https://docs.gnani.ai/api/TTS/tts-inference#supported-languages)**. + +## Available Voices + +| Voice | ID | Gender | Description | +| ------ | -------- | ------ | ------------------------ | +| Karan | `Karan` | Male | Bold, Trustworthy | +| Simran | `Simran` | Female | Confident, Bright | +| Nara | `Nara` | Female | Gentle, Expressive | +| Riya | `Riya` | Female | Cheerful, Energetic | +| Viraj | `Viraj` | Male | Commanding, Dynamic | +| Raju | `Raju` | Male | Grounded, Conversational | + +## Architecture + +This plugin directly implements the Gnani REST, SSE, and WebSocket APIs using `fetch` and `ws`, adapting them into LiveKit's `stt.STT` and `tts.TTS` base classes. It uses the **Prisma** model for speech-to-text and the **Timbre** model for text-to-speech. No external SDK is required; all connection logic, authentication, and audio format handling is self-contained. Authentication uses a single `apiKey` passed via the `X-API-Key-ID` header. + +## Documentation + +- [Gnani API Docs](https://docs.gnani.ai/) +- [LiveKit Agents Docs](https://docs.livekit.io/agents/) +- [Gnani STT Plugin Guide](https://docs.livekit.io/agents/integrations/stt/gnani/) +- [Gnani TTS Plugin Guide](https://docs.livekit.io/agents/integrations/tts/gnani/) + +## License + +Apache-2.0 diff --git a/plugins/gnani/api-extractor.json b/plugins/gnani/api-extractor.json new file mode 100644 index 000000000..32c90f0fa --- /dev/null +++ b/plugins/gnani/api-extractor.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + "extends": "../../api-extractor-shared.json", + "mainEntryPointFilePath": "./dist/index.d.ts" +} diff --git a/plugins/gnani/etc/agents-plugin-gnani.api.md b/plugins/gnani/etc/agents-plugin-gnani.api.md new file mode 100644 index 000000000..f728ec139 --- /dev/null +++ b/plugins/gnani/etc/agents-plugin-gnani.api.md @@ -0,0 +1,118 @@ +## API Report File for "@livekit/agents-plugin-gnani" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import { APIConnectOptions } from '@livekit/agents'; +import { AudioBuffer as AudioBuffer_2 } from '@livekit/agents'; +import { stt } from '@livekit/agents'; +import { tts } from '@livekit/agents'; + +// @public (undocumented) +export class SpeechStream extends stt.SpeechStream { + constructor(sttInstance: STT, opts: ResolvedSTTOptions, connOptions?: APIConnectOptions); + // (undocumented) + buildWsUrl(): string; + // (undocumented) + label: string; + // Warning: (ae-forgotten-export) The symbol "ResolvedSTTOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + _opts: ResolvedSTTOptions; + // (undocumented) + protected run(): Promise; +} + +// @public (undocumented) +export class STT extends stt.STT { + constructor(opts?: STTOptions & Record); + // (undocumented) + label: string; + // (undocumented) + get model(): string; + // (undocumented) + _opts: ResolvedSTTOptions; + // (undocumented) + get provider(): string; + // (undocumented) + protected _recognize(buffer: AudioBuffer_2, abortSignal?: AbortSignal): Promise; + // (undocumented) + stream(options?: { + language?: string; + connOptions?: APIConnectOptions; + }): SpeechStream; +} + +// @public (undocumented) +export interface STTOptions { + apiKey?: string; + baseURL?: string; + // Warning: (ae-forgotten-export) The symbol "GnaniSTTFormat" needs to be exported by the entry point index.d.ts + format?: GnaniSTTFormat; + itnNativeNumerals?: boolean; + // Warning: (ae-forgotten-export) The symbol "GnaniSTTLanguages" needs to be exported by the entry point index.d.ts + language?: GnaniSTTLanguages | string; + preferredLanguage?: string; + // Warning: (ae-forgotten-export) The symbol "SAMPLE_RATE_8K" needs to be exported by the entry point index.d.ts + // Warning: (ae-forgotten-export) The symbol "SAMPLE_RATE_16K" needs to be exported by the entry point index.d.ts + sampleRate?: typeof SAMPLE_RATE_8K | typeof SAMPLE_RATE_16K | number; +} + +// @public (undocumented) +export class SynthesizeStream extends tts.SynthesizeStream { + // Warning: (ae-forgotten-export) The symbol "ResolvedTTSOptions" needs to be exported by the entry point index.d.ts + constructor(ttsInstance: TTS, opts: ResolvedTTSOptions, connOptions?: APIConnectOptions); + // (undocumented) + buildWsUrl(): string; + // (undocumented) + label: string; + // (undocumented) + protected run(): Promise; +} + +// @public (undocumented) +export class TTS extends tts.TTS { + constructor(opts?: TTSOptions & Record); + // (undocumented) + label: string; + // (undocumented) + get model(): string; + // (undocumented) + _opts: ResolvedTTSOptions; + // (undocumented) + get provider(): string; + // (undocumented) + stream(options?: { + connOptions?: APIConnectOptions; + }): SynthesizeStream; + // Warning: (ae-forgotten-export) The symbol "ChunkedStream" needs to be exported by the entry point index.d.ts + // + // (undocumented) + synthesize(text: string, connOptions?: APIConnectOptions, abortSignal?: AbortSignal): ChunkedStream; + // (undocumented) + updateOptions(opts: Partial & Record): void; +} + +// @public (undocumented) +export interface TTSOptions { + apiKey?: string; + baseURL?: string; + // Warning: (ae-forgotten-export) The symbol "GnaniTTSBitrates" needs to be exported by the entry point index.d.ts + bitrate?: GnaniTTSBitrates | string; + // Warning: (ae-forgotten-export) The symbol "GnaniTTSContainers" needs to be exported by the entry point index.d.ts + container?: GnaniTTSContainers | string; + // Warning: (ae-forgotten-export) The symbol "GnaniTTSEncodings" needs to be exported by the entry point index.d.ts + encoding?: GnaniTTSEncodings | string; + model?: string; + numChannels?: number; + sampleRate?: number; + // Warning: (ae-forgotten-export) The symbol "GnaniTTSSynthesizeMethod" needs to be exported by the entry point index.d.ts + synthesizeMethod?: GnaniTTSSynthesizeMethod; + // Warning: (ae-forgotten-export) The symbol "GnaniTTSVoices" needs to be exported by the entry point index.d.ts + voice?: GnaniTTSVoices | string; +} + +// (No @packageDocumentation comment for this package) + +``` diff --git a/plugins/gnani/package.json b/plugins/gnani/package.json new file mode 100644 index 000000000..68ea44a14 --- /dev/null +++ b/plugins/gnani/package.json @@ -0,0 +1,51 @@ +{ + "name": "@livekit/agents-plugin-gnani", + "version": "1.5.0", + "description": "Gnani Vachana plugin for LiveKit Node Agents", + "main": "dist/index.js", + "require": "dist/index.cjs", + "types": "dist/index.d.ts", + "exports": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, + "author": "LiveKit", + "type": "module", + "repository": "git@github.com:livekit/agents-js.git", + "license": "Apache-2.0", + "files": [ + "dist", + "src", + "README.md" + ], + "scripts": { + "build": "tsup --onSuccess \"pnpm build:types\"", + "build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js", + "clean": "rm -rf dist", + "clean:build": "pnpm clean && pnpm build", + "lint": "eslint -f unix \"src/**/*.{ts,js}\"", + "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript", + "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose" + }, + "devDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/rtc-node": "catalog:", + "@microsoft/api-extractor": "^7.35.0", + "@types/ws": "catalog:", + "tsup": "^8.3.5", + "typescript": "^5.0.0" + }, + "dependencies": { + "ws": "catalog:" + }, + "peerDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/rtc-node": "catalog:" + } +} diff --git a/plugins/gnani/src/index.ts b/plugins/gnani/src/index.ts new file mode 100644 index 000000000..03dfdc0ff --- /dev/null +++ b/plugins/gnani/src/index.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { Plugin } from '@livekit/agents'; + +export { SpeechStream, STT, type STTOptions } from './stt.js'; +export { SynthesizeStream, TTS, type TTSOptions } from './tts.js'; + +class GnaniPlugin extends Plugin { + constructor() { + super({ + title: 'gnani', + version: __PACKAGE_VERSION__, + package: __PACKAGE_NAME__, + }); + } +} + +Plugin.registerPlugin(new GnaniPlugin()); diff --git a/plugins/gnani/src/stt.test.ts b/plugins/gnani/src/stt.test.ts new file mode 100644 index 000000000..9899ed50b --- /dev/null +++ b/plugins/gnani/src/stt.test.ts @@ -0,0 +1,179 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { STT, SpeechStream } from './stt.js'; + +vi.mock('ws', async () => { + const { EventEmitter } = await import('node:events'); + return { + WebSocket: class MockWebSocket extends EventEmitter { + static OPEN = 1; + readyState = 1; + + constructor() { + super(); + queueMicrotask(() => this.emit('open')); + } + + send() {} + + close() { + this.emit('close', 1000); + } + }, + }; +}); + +describe('Gnani STT', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('requires an API key', () => { + vi.stubEnv('GNANI_API_KEY', ''); + expect(() => new STT({ apiKey: undefined })).toThrow(/API key/i); + }); + + it('accepts apiKey directly', () => { + const stt = new STT({ apiKey: 'test-key' }); + expect(stt._opts.apiKey).toBe('test-key'); + }); + + it('accepts apiKey from env', () => { + vi.stubEnv('GNANI_API_KEY', 'env-key'); + const stt = new STT(); + expect(stt._opts.apiKey).toBe('env-key'); + }); + + it('defaults to en-IN', () => { + const stt = new STT({ apiKey: 'test-key' }); + expect(stt._opts.language).toBe('en-IN'); + }); + + it('accepts custom language', () => { + const stt = new STT({ apiKey: 'test-key', language: 'hi-IN' }); + expect(stt._opts.language).toBe('hi-IN'); + }); + + it('defaults to 16000 Hz sample rate', () => { + const stt = new STT({ apiKey: 'test-key' }); + expect(stt._opts.sampleRate).toBe(16000); + }); + + it('accepts 8000 Hz sample rate', () => { + const stt = new STT({ apiKey: 'test-key', sampleRate: 8000 }); + expect(stt._opts.sampleRate).toBe(8000); + }); + + it('rejects invalid sample rates', () => { + expect(() => new STT({ apiKey: 'test-key', sampleRate: 44100 })).toThrow(/sampleRate/i); + }); + + it('reports streaming=true and interimResults=false', () => { + const stt = new STT({ apiKey: 'test-key' }); + expect(stt.capabilities.streaming).toBe(true); + expect(stt.capabilities.interimResults).toBe(false); + }); + + it('returns model and provider properties', () => { + const stt = new STT({ apiKey: 'test-key' }); + expect(stt.model).toBe('vachana-stt-v3'); + expect(stt.provider).toBe('Gnani'); + }); + + it('defaults to Vachana API base URL', () => { + const stt = new STT({ apiKey: 'test-key' }); + expect(stt._opts.baseURL).toBe('https://api.vachana.ai'); + }); + + it('accepts custom base URL', () => { + const stt = new STT({ apiKey: 'test-key', baseURL: 'https://custom.api.com' }); + expect(stt._opts.baseURL).toBe('https://custom.api.com'); + }); + + it('uses only apiKey for authentication', () => { + const stt = new STT({ apiKey: 'test-key' }); + expect('organizationId' in stt._opts).toBe(false); + expect('userId' in stt._opts).toBe(false); + }); + + it('builds wss URL from https base', () => { + const stt = new STT({ apiKey: 'test-key' }); + const stream = new SpeechStream(stt, { + apiKey: 'test-key', + language: 'en-IN', + sampleRate: 16000, + baseURL: 'https://api.vachana.ai', + format: 'verbatim', + itnNativeNumerals: false, + }); + stream.close(); + expect(stream.buildWsUrl()).toBe('wss://api.vachana.ai/stt/v3/stream'); + }); + + it('builds ws URL from http base', () => { + const stt = new STT({ apiKey: 'test-key' }); + const stream = new SpeechStream(stt, { + apiKey: 'test-key', + language: 'en-IN', + sampleRate: 16000, + baseURL: 'http://localhost:8080', + format: 'verbatim', + itnNativeNumerals: false, + }); + stream.close(); + expect(stream.buildWsUrl()).toBe('ws://localhost:8080/stt/v3/stream'); + }); + + it('defaults format to verbatim', () => { + const stt = new STT({ apiKey: 'test-key' }); + expect(stt._opts.format).toBe('verbatim'); + }); + + it('accepts transcribe format for ITN', () => { + const stt = new STT({ apiKey: 'test-key', format: 'transcribe' }); + expect(stt._opts.format).toBe('transcribe'); + }); + + it('defaults preferredLanguage to undefined', () => { + const stt = new STT({ apiKey: 'test-key' }); + expect(stt._opts.preferredLanguage).toBeUndefined(); + }); + + it('accepts custom preferredLanguage', () => { + const stt = new STT({ apiKey: 'test-key', preferredLanguage: 'hi-IN' }); + expect(stt._opts.preferredLanguage).toBe('hi-IN'); + }); + + it('defaults itnNativeNumerals to false', () => { + const stt = new STT({ apiKey: 'test-key' }); + expect(stt._opts.itnNativeNumerals).toBe(false); + }); + + it('accepts itnNativeNumerals=true', () => { + const stt = new STT({ apiKey: 'test-key', format: 'transcribe', itnNativeNumerals: true }); + expect(stt._opts.itnNativeNumerals).toBe(true); + }); + + it('warns about deprecated auth kwargs without raising', () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const stt = new STT({ apiKey: 'test-key', organizationId: 'old', userId: 'old' }); + expect(stt._opts.apiKey).toBe('test-key'); + warn.mockRestore(); + }); + + it('stream() returns a SpeechStream instance', () => { + const stt = new STT({ apiKey: 'test-key' }); + const stream = stt.stream(); + stream.close(); + expect(stream).toBeInstanceOf(SpeechStream); + }); + + it('stream() uses the configured language', () => { + const stt = new STT({ apiKey: 'test-key', language: 'hi-IN' }); + const stream = stt.stream(); + stream.close(); + expect(stream._opts.language).toBe('hi-IN'); + }); +}); diff --git a/plugins/gnani/src/stt.ts b/plugins/gnani/src/stt.ts new file mode 100644 index 000000000..8abcbfd4b --- /dev/null +++ b/plugins/gnani/src/stt.ts @@ -0,0 +1,406 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { + type APIConnectOptions, + APIConnectionError, + APIStatusError, + APITimeoutError, + type AudioBuffer, + AudioByteStream, + DEFAULT_API_CONNECT_OPTIONS, + log, + mergeFrames, + normalizeLanguage, + stt, +} from '@livekit/agents'; +import type { AudioFrame } from '@livekit/rtc-node'; +import { type RawData, WebSocket } from 'ws'; + +export const GNANI_STT_BASE_URL = 'https://api.vachana.ai'; + +/** @public */ +export type GnaniSTTFormat = 'verbatim' | 'transcribe'; +/** @public */ +export type GnaniSTTLanguages = + | 'bn-IN' + | 'en-IN' + | 'gu-IN' + | 'hi-IN' + | 'kn-IN' + | 'ml-IN' + | 'mr-IN' + | 'pa-IN' + | 'ta-IN' + | 'te-IN' + | 'en-IN,hi-IN'; + +export const SUPPORTED_LANGUAGES = new Set([ + 'bn-IN', + 'en-IN', + 'gu-IN', + 'hi-IN', + 'kn-IN', + 'ml-IN', + 'mr-IN', + 'pa-IN', + 'ta-IN', + 'te-IN', + 'en-IN,hi-IN', +]); + +export const STREAM_SUPPORTED_LANGUAGES = new Set([ + 'bn-IN', + 'en-IN', + 'gu-IN', + 'hi-IN', + 'kn-IN', + 'ml-IN', + 'mr-IN', + 'pa-IN', + 'ta-IN', + 'te-IN', +]); + +export const SAMPLE_RATE_16K = 16000; +export const SAMPLE_RATE_8K = 8000; +export const STREAM_CHUNK_BYTES = 1024; +const NUM_CHANNELS = 1; + +const DEPRECATED_STT_OPTIONS = new Set(['organizationId', 'organization_id', 'userId', 'user_id']); + +/** @public */ +export interface STTOptions { + /** BCP-47 language code (for example, `hi-IN` or `en-IN`). Default: `en-IN`. */ + language?: GnaniSTTLanguages | string; + /** Gnani API key. Defaults to `$GNANI_API_KEY`. */ + apiKey?: string; + /** Streaming audio sample rate. Must be 8000 or 16000. Default: 16000. */ + sampleRate?: typeof SAMPLE_RATE_8K | typeof SAMPLE_RATE_16K | number; + /** Vachana API base URL. */ + baseURL?: string; + /** Force single-language model for this code. */ + preferredLanguage?: string; + /** `verbatim` (default) or `transcribe` to enable ITN. */ + format?: GnaniSTTFormat; + /** Render digits in native script when `format` is `transcribe`. */ + itnNativeNumerals?: boolean; +} + +interface ResolvedSTTOptions { + apiKey: string; + language: string; + sampleRate: number; + baseURL: string; + preferredLanguage?: string; + format: GnaniSTTFormat; + itnNativeNumerals: boolean; +} + +function warnDeprecatedOptions(opts: Record, caller: string) { + for (const name of DEPRECATED_STT_OPTIONS) { + if (name in opts) { + log().warn(`\`${name}\` is deprecated and no longer used by ${caller}`); + } + } +} + +function resolveOptions(opts: STTOptions & Record): ResolvedSTTOptions { + warnDeprecatedOptions(opts, 'STT'); + + const apiKey = opts.apiKey ?? process.env.GNANI_API_KEY; + if (!apiKey) { + throw new Error('Gnani API key is required. Provide it directly or set GNANI_API_KEY.'); + } + + const sampleRate = opts.sampleRate ?? SAMPLE_RATE_16K; + if (sampleRate !== SAMPLE_RATE_8K && sampleRate !== SAMPLE_RATE_16K) { + throw new Error('sampleRate must be 8000 or 16000'); + } + + return { + apiKey, + language: opts.language ?? 'en-IN', + sampleRate, + baseURL: opts.baseURL ?? GNANI_STT_BASE_URL, + preferredLanguage: opts.preferredLanguage, + format: opts.format ?? 'verbatim', + itnNativeNumerals: opts.itnNativeNumerals ?? false, + }; +} + +function websocketURL(baseURL: string, path: string): string { + if (baseURL.startsWith('https://')) return `wss://${baseURL.slice('https://'.length)}${path}`; + if (baseURL.startsWith('http://')) return `ws://${baseURL.slice('http://'.length)}${path}`; + return `wss://${baseURL}${path}`; +} + +function createWav(frame: AudioFrame): Buffer { + const bitsPerSample = 16; + const byteRate = (frame.sampleRate * frame.channels * bitsPerSample) / 8; + const blockAlign = (frame.channels * bitsPerSample) / 8; + + const header = Buffer.alloc(44); + header.write('RIFF', 0); + header.writeUInt32LE(36 + frame.data.byteLength, 4); + header.write('WAVE', 8); + header.write('fmt ', 12); + header.writeUInt32LE(16, 16); + header.writeUInt16LE(1, 20); + header.writeUInt16LE(frame.channels, 22); + header.writeUInt32LE(frame.sampleRate, 24); + header.writeUInt32LE(byteRate, 28); + header.writeUInt16LE(blockAlign, 32); + header.writeUInt16LE(bitsPerSample, 34); + header.write('data', 36); + header.writeUInt32LE(frame.data.byteLength, 40); + + const pcm = Buffer.from(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength); + return Buffer.concat([header, pcm]); +} + +function buildFormData(wavBlob: Blob, opts: ResolvedSTTOptions, language?: string): FormData { + const formData = new FormData(); + formData.append('audio_file', wavBlob, 'audio.wav'); + formData.append('language_code', language ?? opts.language); + formData.append('format', opts.format); + if (opts.preferredLanguage != null) { + formData.append('preferred_language', opts.preferredLanguage); + } + if (opts.itnNativeNumerals) { + formData.append('itn_native_numerals', 'true'); + } + return formData; +} + +function withTimeout(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal { + const timeoutSignal = AbortSignal.timeout(timeoutMs); + if (!signal) return timeoutSignal; + return AbortSignal.any([signal, timeoutSignal]); +} + +/** @public */ +export class STT extends stt.STT { + _opts: ResolvedSTTOptions; + label = 'gnani.STT'; + + /** Create a new Gnani Vachana Speech-to-Text instance. */ + constructor(opts: STTOptions & Record = {}) { + const resolved = resolveOptions(opts); + super({ streaming: true, interimResults: false, alignedTranscript: false }); + this._opts = resolved; + } + + get model(): string { + return 'vachana-stt-v3'; + } + + get provider(): string { + return 'Gnani'; + } + + protected async _recognize( + buffer: AudioBuffer, + abortSignal?: AbortSignal, + ): Promise { + const frame = mergeFrames(buffer); + const wavBuffer = createWav(frame); + const wavBlob = new Blob([new Uint8Array(wavBuffer)], { type: 'audio/wav' }); + + const response = await fetch(`${this._opts.baseURL}/stt/v3`, { + method: 'POST', + headers: { 'X-API-Key-ID': this._opts.apiKey }, + body: buildFormData(wavBlob, this._opts), + signal: withTimeout(abortSignal, DEFAULT_API_CONNECT_OPTIONS.timeoutMs), + }).catch((error: unknown) => { + if (error instanceof DOMException && error.name === 'TimeoutError') { + throw new APITimeoutError({ message: 'Gnani STT API request timed out' }); + } + throw new APIConnectionError({ message: `Gnani STT error: ${String(error)}` }); + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new APIStatusError({ + message: `Gnani STT API Error (${response.status}): ${errorText}`, + options: { statusCode: response.status, body: { error: errorText } }, + }); + } + + const data = (await response.json()) as { transcript?: string; request_id?: string }; + return { + type: stt.SpeechEventType.FINAL_TRANSCRIPT, + requestId: data.request_id ?? '', + alternatives: [ + { + language: normalizeLanguage(this._opts.language), + text: data.transcript ?? '', + confidence: 1, + startTime: 0, + endTime: 0, + }, + ], + }; + } + + stream(options?: { language?: string; connOptions?: APIConnectOptions }): SpeechStream { + return new SpeechStream( + this, + { ...this._opts, language: options?.language ?? this._opts.language }, + options?.connOptions, + ); + } +} + +/** @public */ +export class SpeechStream extends stt.SpeechStream { + _opts: ResolvedSTTOptions; + label = 'gnani.SpeechStream'; + + constructor(sttInstance: STT, opts: ResolvedSTTOptions, connOptions?: APIConnectOptions) { + super(sttInstance, opts.sampleRate, connOptions); + this._opts = opts; + } + + buildWsUrl(): string { + return websocketURL(this._opts.baseURL, '/stt/v3/stream'); + } + + protected async run() { + const ws = new WebSocket(this.buildWsUrl(), { + headers: this.buildHeaders(), + handshakeTimeout: DEFAULT_API_CONNECT_OPTIONS.timeoutMs, + }); + + await new Promise((resolve, reject) => { + const onOpen = () => { + cleanup(); + resolve(); + }; + const onError = (error: Error) => { + cleanup(); + reject(new APIConnectionError({ message: `Gnani STT WebSocket error: ${error.message}` })); + }; + const onClose = (code: number) => { + cleanup(); + reject(new APIConnectionError({ message: `Gnani STT WebSocket closed: ${code}` })); + }; + const cleanup = () => { + ws.removeListener('open', onOpen); + ws.removeListener('error', onError); + ws.removeListener('close', onClose); + }; + ws.on('open', onOpen); + ws.on('error', onError); + ws.on('close', onClose); + }); + + try { + await Promise.race([this.sendAudio(ws), this.receiveMessages(ws)]); + } finally { + ws.close(); + } + } + + private buildHeaders(): Record { + const headers: Record = { + 'x-api-key-id': this._opts.apiKey, + lang_code: this._opts.language, + 'x-sample-rate': String(this._opts.sampleRate), + }; + if (this._opts.format !== 'verbatim') { + headers['x-format'] = this._opts.format; + } + if (this._opts.preferredLanguage != null) { + headers.preferred_language = this._opts.preferredLanguage; + } + if (this._opts.itnNativeNumerals) { + headers.itn_native_numerals = 'true'; + } + return headers; + } + + private async sendAudio(ws: WebSocket) { + const stream = new AudioByteStream(this._opts.sampleRate, NUM_CHANNELS, STREAM_CHUNK_BYTES / 2); + try { + for await (const data of this.input) { + const frames = + data === SpeechStream.FLUSH_SENTINEL + ? stream.flush() + : stream.write( + data.data.buffer.slice( + data.data.byteOffset, + data.data.byteOffset + data.data.byteLength, + ) as ArrayBuffer, + ); + + for (const frame of frames) { + const chunk = Buffer.from( + frame.data.buffer, + frame.data.byteOffset, + frame.data.byteLength, + ); + ws.send(chunk); + } + } + for (const frame of stream.flush()) { + ws.send(Buffer.from(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength)); + } + } catch (error) { + if (!this.abortSignal.aborted) throw error; + } + } + + private async receiveMessages(ws: WebSocket) { + return new Promise((resolve, reject) => { + ws.on('message', (msg: RawData) => { + if (Buffer.isBuffer(msg)) return; + + try { + const data = JSON.parse(msg.toString()) as Record; + const msgType = data.type; + + if (msgType === 'connected' || msgType === 'processing') return; + + if (msgType === 'transcript') { + const text = typeof data.text === 'string' ? data.text : ''; + if (!text) return; + this.queue.put({ + type: stt.SpeechEventType.FINAL_TRANSCRIPT, + requestId: typeof data.segment_id === 'string' ? data.segment_id : '', + alternatives: [ + { + language: normalizeLanguage(this._opts.language), + text, + confidence: 1, + startTime: 0, + endTime: 0, + }, + ], + }); + } else if (msgType === 'speech_start' || msgType === 'vad_start') { + this.queue.put({ type: stt.SpeechEventType.START_OF_SPEECH }); + } else if (msgType === 'speech_end' || msgType === 'vad_end') { + this.queue.put({ type: stt.SpeechEventType.END_OF_SPEECH }); + } else if (msgType === 'error') { + const message = typeof data.message === 'string' ? data.message : 'Unknown error'; + reject( + new APIStatusError({ + message: `Gnani STT stream error: ${message}`, + options: { statusCode: 500, body: { error: message } }, + }), + ); + } + } catch (error) { + reject( + new APIConnectionError({ message: `Error receiving Gnani STT messages: ${error}` }), + ); + } + }); + ws.on('close', () => resolve()); + ws.on('error', (error) => + reject(new APIConnectionError({ message: `Gnani STT WebSocket error: ${error.message}` })), + ); + }); + } +} diff --git a/plugins/gnani/src/tts.test.ts b/plugins/gnani/src/tts.test.ts new file mode 100644 index 000000000..0c91587ff --- /dev/null +++ b/plugins/gnani/src/tts.test.ts @@ -0,0 +1,252 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + RESTChunkedStream, + SSEChunkedStream, + SynthesizeStream, + TTS, + WebSocketChunkedStream, +} from './tts.js'; + +vi.mock('ws', async () => { + const { EventEmitter } = await import('node:events'); + return { + WebSocket: class MockWebSocket extends EventEmitter { + static OPEN = 1; + readyState = 1; + + constructor() { + super(); + queueMicrotask(() => this.emit('open')); + } + + send() {} + + close() { + this.emit('close', 1000); + } + }, + }; +}); + +describe('Gnani TTS', () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + }); + + beforeEach(() => { + vi.stubGlobal( + 'fetch', + vi.fn(() => new Promise(() => {})), + ); + }); + + it('requires an API key', () => { + vi.stubEnv('GNANI_API_KEY', ''); + expect(() => new TTS({ apiKey: undefined })).toThrow(/API key/i); + }); + + it('accepts apiKey directly', () => { + const tts = new TTS({ apiKey: 'test-key' }); + expect(tts._opts.apiKey).toBe('test-key'); + }); + + it('accepts apiKey from env', () => { + vi.stubEnv('GNANI_API_KEY', 'env-key'); + const tts = new TTS(); + expect(tts._opts.apiKey).toBe('env-key'); + }); + + it('defaults to Karan voice', () => { + const tts = new TTS({ apiKey: 'test-key' }); + expect(tts._opts.voice).toBe('Karan'); + }); + + it('accepts custom voice', () => { + const tts = new TTS({ apiKey: 'test-key', voice: 'Raju' }); + expect(tts._opts.voice).toBe('Raju'); + }); + + it('accepts all documented voices', () => { + for (const voice of ['Karan', 'Simran', 'Nara', 'Riya', 'Viraj', 'Raju']) { + const tts = new TTS({ apiKey: 'test-key', voice }); + expect(tts._opts.voice).toBe(voice); + } + }); + + it('rejects unsupported voices', () => { + expect(() => new TTS({ apiKey: 'test-key', voice: 'nonexistent' })).toThrow(/not supported/i); + }); + + it('defaults to vachana-voice-v3', () => { + const tts = new TTS({ apiKey: 'test-key' }); + expect(tts._opts.model).toBe('vachana-voice-v3'); + }); + + it('uses vachana-voice-v3 for v3 voices', () => { + const tts = new TTS({ apiKey: 'test-key', voice: 'Simran' }); + expect(tts._opts.model).toBe('vachana-voice-v3'); + expect(tts.model).toBe('vachana-voice-v3'); + }); + + it('accepts explicit model override', () => { + const tts = new TTS({ apiKey: 'test-key', voice: 'Karan', model: 'custom-model' }); + expect(tts._opts.model).toBe('custom-model'); + }); + + it('returns model and provider properties', () => { + const tts = new TTS({ apiKey: 'test-key' }); + expect(tts.model).toBe('vachana-voice-v3'); + expect(tts.provider).toBe('Gnani'); + }); + + it('reports streaming=true', () => { + const tts = new TTS({ apiKey: 'test-key' }); + expect(tts.capabilities.streaming).toBe(true); + }); + + it('defaults to 16000 Hz sample rate', () => { + const tts = new TTS({ apiKey: 'test-key' }); + expect(tts.sampleRate).toBe(16000); + }); + + it('accepts custom sample rate', () => { + const tts = new TTS({ apiKey: 'test-key', sampleRate: 44100 }); + expect(tts.sampleRate).toBe(44100); + }); + + it('defaults encoding and container', () => { + const tts = new TTS({ apiKey: 'test-key' }); + expect(tts._opts.encoding).toBe('linear_pcm'); + expect(tts._opts.container).toBe('wav'); + }); + + it('accepts custom audio config', () => { + const tts = new TTS({ apiKey: 'test-key', encoding: 'oggopus', container: 'ogg' }); + expect(tts._opts.encoding).toBe('oggopus'); + expect(tts._opts.container).toBe('ogg'); + }); + + it('updateOptions can change voice', () => { + const tts = new TTS({ apiKey: 'test-key', voice: 'Karan' }); + tts.updateOptions({ voice: 'Simran' }); + expect(tts._opts.voice).toBe('Simran'); + }); + + it('updateOptions can change voice and model', () => { + const tts = new TTS({ apiKey: 'test-key', voice: 'Karan' }); + tts.updateOptions({ voice: 'Riya', model: 'custom-model' }); + expect(tts._opts.voice).toBe('Riya'); + expect(tts._opts.model).toBe('custom-model'); + }); + + it('updateOptions rejects unsupported voices', () => { + const tts = new TTS({ apiKey: 'test-key' }); + expect(() => tts.updateOptions({ voice: 'nonexistent' })).toThrow(/not supported/i); + }); + + it('updateOptions can change model', () => { + const tts = new TTS({ apiKey: 'test-key' }); + tts.updateOptions({ model: 'custom-model' }); + expect(tts._opts.model).toBe('custom-model'); + }); + + it('stores synthesizeMethod options', () => { + expect(new TTS({ apiKey: 'test-key' })._opts.synthesizeMethod).toBe('rest'); + expect(new TTS({ apiKey: 'test-key', synthesizeMethod: 'sse' })._opts.synthesizeMethod).toBe( + 'sse', + ); + expect( + new TTS({ apiKey: 'test-key', synthesizeMethod: 'websocket' })._opts.synthesizeMethod, + ).toBe('websocket'); + }); + + it('synthesize() routes by synthesizeMethod', () => { + const rest = new TTS({ apiKey: 'test-key', synthesizeMethod: 'rest' }).synthesize('hello'); + rest.close(); + expect(rest).toBeInstanceOf(RESTChunkedStream); + + const sse = new TTS({ apiKey: 'test-key', synthesizeMethod: 'sse' }).synthesize('hello'); + sse.close(); + expect(sse).toBeInstanceOf(SSEChunkedStream); + + const websocket = new TTS({ apiKey: 'test-key', synthesizeMethod: 'websocket' }).synthesize( + 'hello', + ); + websocket.close(); + expect(websocket).toBeInstanceOf(WebSocketChunkedStream); + }); + + it('defaults to Vachana API base URL', () => { + const tts = new TTS({ apiKey: 'test-key' }); + expect(tts._opts.baseURL).toBe('https://api.vachana.ai'); + }); + + it('builds wss URL from https base', () => { + const tts = new TTS({ apiKey: 'test-key' }); + const stream = new SynthesizeStream(tts, tts._opts); + stream.close(); + expect(stream.buildWsUrl()).toBe('wss://api.vachana.ai/api/v1/tts'); + }); + + it('builds ws URL from http base', () => { + const tts = new TTS({ apiKey: 'test-key', baseURL: 'http://localhost:9090' }); + const stream = new SynthesizeStream(tts, tts._opts); + stream.close(); + expect(stream.buildWsUrl()).toBe('ws://localhost:9090/api/v1/tts'); + }); + + it('defaults and accepts numChannels', () => { + const defaults = new TTS({ apiKey: 'test-key' }); + expect(defaults._opts.numChannels).toBe(1); + expect(defaults.numChannels).toBe(1); + + const custom = new TTS({ apiKey: 'test-key', numChannels: 2 }); + expect(custom._opts.numChannels).toBe(2); + }); + + it('defaults and accepts bitrate', () => { + const defaults = new TTS({ apiKey: 'test-key' }); + expect(defaults._opts.bitrate).toBeUndefined(); + + const custom = new TTS({ apiKey: 'test-key', bitrate: '128k' }); + expect(custom._opts.bitrate).toBe('128k'); + }); + + it('rejects unsupported sample rates', () => { + expect(() => new TTS({ apiKey: 'test-key', sampleRate: 48000 })).toThrow(/sampleRate/i); + }); + + it('accepts all documented sample rates', () => { + for (const sampleRate of [8000, 16000, 22050, 44100]) { + const tts = new TTS({ apiKey: 'test-key', sampleRate }); + expect(tts.sampleRate).toBe(sampleRate); + } + }); + + it('stream() returns a SynthesizeStream instance', () => { + const tts = new TTS({ apiKey: 'test-key' }); + const stream = tts.stream(); + stream.close(); + expect(stream).toBeInstanceOf(SynthesizeStream); + }); + + it('updateOptions preserves other fields', () => { + const tts = new TTS({ apiKey: 'test-key', voice: 'Karan' }); + tts.updateOptions({ voice: 'Raju' }); + expect(tts._opts.voice).toBe('Raju'); + expect(tts._opts.model).toBe('vachana-voice-v3'); + expect(tts._opts.encoding).toBe('linear_pcm'); + }); + + it('WebSocketChunkedStream builds correct WS URL', () => { + const tts = new TTS({ apiKey: 'test-key', synthesizeMethod: 'websocket' }); + const stream = tts.synthesize('hello'); + stream.close(); + expect(stream).toBeInstanceOf(WebSocketChunkedStream); + expect((stream as WebSocketChunkedStream).buildWsUrl()).toBe('wss://api.vachana.ai/api/v1/tts'); + }); +}); diff --git a/plugins/gnani/src/tts.ts b/plugins/gnani/src/tts.ts new file mode 100644 index 000000000..180abbc3d --- /dev/null +++ b/plugins/gnani/src/tts.ts @@ -0,0 +1,516 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { + type APIConnectOptions, + APIConnectionError, + APIStatusError, + APITimeoutError, + AudioByteStream, + DEFAULT_API_CONNECT_OPTIONS, + log, + shortuuid, + tts, +} from '@livekit/agents'; +import type { AudioFrame } from '@livekit/rtc-node'; +import { type RawData, WebSocket } from 'ws'; + +export const GNANI_TTS_BASE_URL = 'https://api.vachana.ai'; + +/** @public */ +export type GnaniTTSVoices = 'Karan' | 'Simran' | 'Nara' | 'Riya' | 'Viraj' | 'Raju'; +/** @public */ +export type GnaniTTSEncodings = 'linear_pcm' | 'oggopus'; +/** @public */ +export type GnaniTTSContainers = 'raw' | 'mp3' | 'wav' | 'mulaw' | 'ogg'; +/** @public */ +export type GnaniTTSBitrates = '96k' | '128k' | '192k'; +/** @public */ +export type GnaniTTSSynthesizeMethod = 'rest' | 'sse' | 'websocket'; + +export const SUPPORTED_VOICES = new Set([ + 'Karan', + 'Simran', + 'Nara', + 'Riya', + 'Viraj', + 'Raju', +]); +export const SUPPORTED_SAMPLE_RATES = [8000, 16000, 22050, 44100] as const; + +const WAV_HEADER_SIZE = 44; +const DEFAULT_SAMPLE_WIDTH = 2; +const DEPRECATED_TTS_OPTIONS = new Set(['language', 'httpSession', 'http_session']); + +/** @public */ +export interface TTSOptions { + /** Voice to use for synthesis. Default: `Karan`. */ + voice?: GnaniTTSVoices | string; + /** TTS model name. Default: `vachana-voice-v3`. */ + model?: string; + /** Audio output sample rate. Default: 16000. */ + sampleRate?: number; + /** Number of audio channels. Default: 1. */ + numChannels?: number; + /** Audio encoding. Default: `linear_pcm`. */ + encoding?: GnaniTTSEncodings | string; + /** Audio container. Default: `wav`. */ + container?: GnaniTTSContainers | string; + /** Optional audio bitrate. */ + bitrate?: GnaniTTSBitrates | string; + /** Gnani API key. Defaults to `$GNANI_API_KEY`. */ + apiKey?: string; + /** Vachana API base URL. */ + baseURL?: string; + /** Synthesis transport used by `synthesize()`. Default: `rest`. */ + synthesizeMethod?: GnaniTTSSynthesizeMethod; +} + +interface ResolvedTTSOptions { + apiKey: string; + voice: string; + model: string; + sampleRate: number; + encoding: string; + container: string; + numChannels: number; + sampleWidth: number; + bitrate?: string; + baseURL: string; + synthesizeMethod: GnaniTTSSynthesizeMethod; +} + +function warnDeprecatedOptions(opts: Record, caller: string) { + for (const name of DEPRECATED_TTS_OPTIONS) { + if (name in opts) { + log().warn(`\`${name}\` is deprecated and no longer used by ${caller}`); + } + } +} + +function resolveOptions(opts: TTSOptions & Record): ResolvedTTSOptions { + warnDeprecatedOptions(opts, 'TTS'); + + const apiKey = opts.apiKey ?? process.env.GNANI_API_KEY; + if (!apiKey) { + throw new Error('Gnani API key is required. Provide it directly or set GNANI_API_KEY.'); + } + + const sampleRate = opts.sampleRate ?? 16000; + if (!SUPPORTED_SAMPLE_RATES.includes(sampleRate as (typeof SUPPORTED_SAMPLE_RATES)[number])) { + throw new Error(`sampleRate must be one of ${SUPPORTED_SAMPLE_RATES.join(', ')}`); + } + + const voice = opts.voice ?? 'Karan'; + if (!SUPPORTED_VOICES.has(voice)) { + throw new Error( + `Voice '${voice}' not supported. Supported voices: ${[...SUPPORTED_VOICES].sort().join(', ')}`, + ); + } + + return { + apiKey, + voice, + model: opts.model ?? 'vachana-voice-v3', + sampleRate, + encoding: opts.encoding ?? 'linear_pcm', + container: opts.container ?? 'wav', + numChannels: opts.numChannels ?? 1, + sampleWidth: DEFAULT_SAMPLE_WIDTH, + bitrate: opts.bitrate, + baseURL: opts.baseURL ?? GNANI_TTS_BASE_URL, + synthesizeMethod: opts.synthesizeMethod ?? 'rest', + }; +} + +function websocketURL(baseURL: string, path: string): string { + if (baseURL.startsWith('https://')) return `wss://${baseURL.slice('https://'.length)}${path}`; + if (baseURL.startsWith('http://')) return `ws://${baseURL.slice('http://'.length)}${path}`; + return `wss://${baseURL}${path}`; +} + +function buildPayload(opts: ResolvedTTSOptions, text: string): Record { + const audioConfig: Record = { + sample_rate: opts.sampleRate, + encoding: opts.encoding, + num_channels: opts.numChannels, + sample_width: opts.sampleWidth, + container: opts.container, + }; + if (opts.bitrate != null) { + audioConfig.bitrate = opts.bitrate; + } + + return { + text, + voice: opts.voice, + model: opts.model, + audio_config: audioConfig, + }; +} + +function buildHeaders(opts: ResolvedTTSOptions): Record { + return { + 'X-API-Key-ID': opts.apiKey, + 'Content-Type': 'application/json', + }; +} + +function stripWavHeader(data: Buffer): Buffer { + if (data.length > WAV_HEADER_SIZE && data.subarray(0, 4).toString() === 'RIFF') { + return data.subarray(WAV_HEADER_SIZE); + } + return data; +} + +function decodeAudioChunk(data: Buffer, opts: ResolvedTTSOptions): Buffer { + return opts.container === 'wav' ? stripWavHeader(data) : data; +} + +function framesFromAudio(data: Buffer, opts: ResolvedTTSOptions): AudioFrame[] { + const stream = new AudioByteStream(opts.sampleRate, opts.numChannels); + return [...stream.write(data), ...stream.flush()]; +} + +function putFrames( + queue: { put: (audio: tts.SynthesizedAudio) => void }, + frames: AudioFrame[], + requestId: string, + segmentId: string, +) { + let lastFrame: AudioFrame | undefined; + const sendLastFrame = (final: boolean) => { + if (lastFrame) { + queue.put({ requestId, segmentId, frame: lastFrame, final }); + lastFrame = undefined; + } + }; + + for (const frame of frames) { + sendLastFrame(false); + lastFrame = frame; + } + sendLastFrame(true); +} + +function apiError(message: string, statusCode = 500): APIStatusError { + return new APIStatusError({ + message, + options: { statusCode, body: { error: message } }, + }); +} + +/** @public */ +export class TTS extends tts.TTS { + _opts: ResolvedTTSOptions; + label = 'gnani.TTS'; + + /** Create a new Gnani Vachana Text-to-Speech instance. */ + constructor(opts: TTSOptions & Record = {}) { + const resolved = resolveOptions(opts); + super(resolved.sampleRate, resolved.numChannels, { streaming: true }); + this._opts = resolved; + } + + get model(): string { + return this._opts.model; + } + + get provider(): string { + return 'Gnani'; + } + + synthesize( + text: string, + connOptions?: APIConnectOptions, + abortSignal?: AbortSignal, + ): ChunkedStream { + if (this._opts.synthesizeMethod === 'sse') { + return new SSEChunkedStream(this, text, this._opts, connOptions, abortSignal); + } + if (this._opts.synthesizeMethod === 'websocket') { + return new WebSocketChunkedStream(this, text, this._opts, connOptions, abortSignal); + } + return new RESTChunkedStream(this, text, this._opts, connOptions, abortSignal); + } + + stream(options?: { connOptions?: APIConnectOptions }): SynthesizeStream { + return new SynthesizeStream(this, this._opts, options?.connOptions); + } + + updateOptions(opts: Partial & Record) { + warnDeprecatedOptions(opts, 'TTS.updateOptions'); + this._opts = resolveOptions({ ...this._opts, ...opts }); + } +} + +/** @public */ +export type ChunkedStream = RESTChunkedStream | SSEChunkedStream | WebSocketChunkedStream; + +/** @public */ +export class RESTChunkedStream extends tts.ChunkedStream { + private opts: ResolvedTTSOptions; + label = 'gnani.RESTChunkedStream'; + + constructor( + ttsInstance: TTS, + text: string, + opts: ResolvedTTSOptions, + connOptions?: APIConnectOptions, + abortSignal?: AbortSignal, + ) { + super(text, ttsInstance, connOptions, abortSignal); + this.opts = { ...opts }; + } + + protected async run() { + const signal = AbortSignal.any([ + this.abortSignal, + AbortSignal.timeout(DEFAULT_API_CONNECT_OPTIONS.timeoutMs), + ]); + const response = await fetch(`${this.opts.baseURL}/api/v1/tts/inference`, { + method: 'POST', + headers: buildHeaders(this.opts), + body: JSON.stringify(buildPayload(this.opts, this.inputText)), + signal, + }).catch((error: unknown) => { + if (error instanceof DOMException && error.name === 'TimeoutError') { + throw new APITimeoutError({ message: 'Gnani TTS REST request timed out' }); + } + throw new APIConnectionError({ message: `Gnani TTS REST error: ${String(error)}` }); + }); + + if (!response.ok) { + const errorText = await response.text(); + throw apiError(`Gnani TTS API Error (${response.status}): ${errorText}`, response.status); + } + + const audioBytes = Buffer.from(await response.arrayBuffer()); + const requestId = shortuuid(); + putFrames(this.queue, framesFromAudio(audioBytes, this.opts), requestId, requestId); + } +} + +/** @public */ +export class SSEChunkedStream extends tts.ChunkedStream { + private opts: ResolvedTTSOptions; + label = 'gnani.SSEChunkedStream'; + + constructor( + ttsInstance: TTS, + text: string, + opts: ResolvedTTSOptions, + connOptions?: APIConnectOptions, + abortSignal?: AbortSignal, + ) { + super(text, ttsInstance, connOptions, abortSignal); + this.opts = { ...opts }; + } + + protected async run() { + const response = await fetch(`${this.opts.baseURL}/api/v1/tts/sse`, { + method: 'POST', + headers: buildHeaders(this.opts), + body: JSON.stringify(buildPayload(this.opts, this.inputText)), + signal: this.abortSignal, + }).catch((error: unknown) => { + throw new APIConnectionError({ message: `Gnani TTS SSE error: ${String(error)}` }); + }); + + if (!response.ok) { + const errorText = await response.text(); + throw apiError(`Gnani TTS SSE Error (${response.status}): ${errorText}`, response.status); + } + if (!response.body) { + throw new APIConnectionError({ message: 'Gnani TTS SSE returned no response body' }); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + const requestId = shortuuid(); + const segmentId = requestId; + const audioFrames: AudioFrame[] = []; + + const handlePayload = (payload: Record) => { + if (payload.status === 'error' || 'error' in payload) { + throw apiError(String(payload.message ?? payload.error ?? 'Gnani TTS SSE error')); + } + if (payload.status === 'streaming_started') return false; + const audio = typeof payload.audio === 'string' ? payload.audio : ''; + if (audio) { + const chunk = decodeAudioChunk(Buffer.from(audio, 'base64'), this.opts); + audioFrames.push(...framesFromAudio(chunk, this.opts)); + } + return payload.is_final === true; + }; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + + let separator = buffer.indexOf('\n\n'); + while (separator !== -1) { + const event = buffer.slice(0, separator); + buffer = buffer.slice(separator + 2); + const dataLines = event + .split(/\r?\n/) + .filter((line) => line.startsWith('data:')) + .map((line) => line.slice(5).trim()); + if (dataLines.length > 0) { + const isFinal = handlePayload(JSON.parse(dataLines.join('')) as Record); + if (isFinal) { + putFrames(this.queue, audioFrames, requestId, segmentId); + return; + } + } + separator = buffer.indexOf('\n\n'); + } + } + + putFrames(this.queue, audioFrames, requestId, segmentId); + } +} + +/** @public */ +export class WebSocketChunkedStream extends tts.ChunkedStream { + private opts: ResolvedTTSOptions; + label = 'gnani.WebSocketChunkedStream'; + + constructor( + ttsInstance: TTS, + text: string, + opts: ResolvedTTSOptions, + connOptions?: APIConnectOptions, + abortSignal?: AbortSignal, + ) { + super(text, ttsInstance, connOptions, abortSignal); + this.opts = { ...opts }; + } + + buildWsUrl(): string { + return websocketURL(this.opts.baseURL, '/api/v1/tts'); + } + + protected async run() { + const requestId = shortuuid(); + const segmentId = requestId; + const frames = await synthesizeViaWebSocket(this.buildWsUrl(), this.inputText, this.opts); + putFrames(this.queue, frames, requestId, segmentId); + } +} + +/** @public */ +export class SynthesizeStream extends tts.SynthesizeStream { + private opts: ResolvedTTSOptions; + label = 'gnani.SynthesizeStream'; + + constructor(ttsInstance: TTS, opts: ResolvedTTSOptions, connOptions?: APIConnectOptions) { + super(ttsInstance, connOptions); + this.opts = { ...opts }; + } + + buildWsUrl(): string { + return websocketURL(this.opts.baseURL, '/api/v1/tts'); + } + + protected async run() { + const textParts: string[] = []; + for await (const data of this.input) { + if (data === SynthesizeStream.FLUSH_SENTINEL) break; + textParts.push(data); + } + + const text = textParts.join('').trim(); + if (!text) return; + + const requestId = shortuuid(); + const segmentId = shortuuid(); + const frames = await synthesizeViaWebSocket(this.buildWsUrl(), text, this.opts, () => + this.markStarted(), + ); + putFrames(this.queue, frames, requestId, segmentId); + this.queue.put(SynthesizeStream.END_OF_STREAM); + } +} + +async function synthesizeViaWebSocket( + url: string, + text: string, + opts: ResolvedTTSOptions, + onStarted?: () => void, +): Promise { + const ws = new WebSocket(url, { + headers: buildHeaders(opts), + handshakeTimeout: DEFAULT_API_CONNECT_OPTIONS.timeoutMs, + }); + + await new Promise((resolve, reject) => { + const onOpen = () => { + cleanup(); + resolve(); + }; + const onError = (error: Error) => { + cleanup(); + reject(new APIConnectionError({ message: `Gnani TTS WebSocket error: ${error.message}` })); + }; + const onClose = (code: number) => { + cleanup(); + reject(new APIConnectionError({ message: `Gnani TTS WebSocket closed: ${code}` })); + }; + const cleanup = () => { + ws.removeListener('open', onOpen); + ws.removeListener('error', onError); + ws.removeListener('close', onClose); + }; + ws.on('open', onOpen); + ws.on('error', onError); + ws.on('close', onClose); + }); + + ws.send(JSON.stringify(buildPayload(opts, text))); + onStarted?.(); + + try { + return await new Promise((resolve, reject) => { + const audioFrames: AudioFrame[] = []; + ws.on('message', (msg: RawData) => { + try { + if (Buffer.isBuffer(msg)) { + audioFrames.push(...framesFromAudio(decodeAudioChunk(msg, opts), opts)); + return; + } + + const payload = JSON.parse(msg.toString()) as Record; + const msgType = payload.type; + const data = (payload.data ?? {}) as Record; + const audio = typeof data.audio === 'string' ? data.audio : ''; + + if (msgType === 'audio') { + if (audio) + audioFrames.push( + ...framesFromAudio(decodeAudioChunk(Buffer.from(audio, 'base64'), opts), opts), + ); + } else if (msgType === 'complete') { + if (audio) + audioFrames.push( + ...framesFromAudio(decodeAudioChunk(Buffer.from(audio, 'base64'), opts), opts), + ); + resolve(audioFrames); + } else if (msgType === 'error') { + reject(apiError(String(payload.message ?? data.message ?? 'Gnani TTS stream error'))); + } + } catch (error) { + reject(new APIConnectionError({ message: `Gnani TTS WebSocket error: ${error}` })); + } + }); + ws.on('close', () => resolve(audioFrames)); + ws.on('error', (error) => + reject(new APIConnectionError({ message: `Gnani TTS WebSocket error: ${error.message}` })), + ); + }); + } finally { + ws.close(); + } +} diff --git a/plugins/gnani/tsconfig.json b/plugins/gnani/tsconfig.json new file mode 100644 index 000000000..d3d3e25b3 --- /dev/null +++ b/plugins/gnani/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.json", + "include": ["./src"], + "compilerOptions": { + "rootDir": "./src", + "declarationDir": "./dist", + "outDir": "./dist" + }, + "typedocOptions": { + "name": "plugins/agents-plugin-gnani", + "entryPointStrategy": "resolve", + "entryPoints": ["src/index.ts"] + } +} diff --git a/plugins/gnani/tsup.config.ts b/plugins/gnani/tsup.config.ts new file mode 100644 index 000000000..b491713a4 --- /dev/null +++ b/plugins/gnani/tsup.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from 'tsup'; +import defaults from '../../tsup.config'; + +export default defineConfig({ + ...defaults, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f00f5059b..8856e5515 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -60,7 +60,7 @@ importers: version: 8.10.0(eslint@8.57.0) eslint-config-standard: specifier: ^17.1.0 - version: 17.1.0(eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint@8.57.0))(eslint-plugin-n@16.6.2(eslint@8.57.0))(eslint-plugin-promise@6.1.1(eslint@8.57.0))(eslint@8.57.0) + version: 17.1.0(eslint-plugin-import@2.29.1)(eslint-plugin-n@16.6.2(eslint@8.57.0))(eslint-plugin-promise@6.1.1(eslint@8.57.0))(eslint@8.57.0) eslint-config-turbo: specifier: ^2.9.14 version: 2.10.1(eslint@8.57.0)(turbo@2.9.14) @@ -741,6 +741,31 @@ importers: specifier: ^5.0.0 version: 5.9.3 + plugins/gnani: + dependencies: + ws: + specifier: 'catalog:' + version: 8.20.0 + devDependencies: + '@livekit/agents': + specifier: workspace:* + version: link:../../agents + '@livekit/rtc-node': + specifier: 'catalog:' + version: 0.13.31 + '@microsoft/api-extractor': + specifier: ^7.35.0 + version: 7.43.7(@types/node@25.6.0) + '@types/ws': + specifier: 'catalog:' + version: 8.18.1 + tsup: + specifier: ^8.3.5 + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@25.6.0))(postcss@8.5.9)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.0.0 + version: 5.9.3 + plugins/google: dependencies: '@google/genai': @@ -6727,14 +6752,6 @@ snapshots: optionalDependencies: vite: 7.3.2(@types/node@22.19.1)(tsx@4.21.0) - '@vitest/mocker@4.0.17(vite@7.3.2(@types/node@25.6.0)(tsx@4.21.0))': - dependencies: - '@vitest/spy': 4.0.17 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 7.3.2(@types/node@25.6.0)(tsx@4.21.0) - '@vitest/pretty-format@4.0.17': dependencies: tinyrainbow: 3.0.3 @@ -7340,7 +7357,7 @@ snapshots: dependencies: eslint: 8.57.0 - eslint-config-standard@17.1.0(eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint@8.57.0))(eslint-plugin-n@16.6.2(eslint@8.57.0))(eslint-plugin-promise@6.1.1(eslint@8.57.0))(eslint@8.57.0): + eslint-config-standard@17.1.0(eslint-plugin-import@2.29.1)(eslint-plugin-n@16.6.2(eslint@8.57.0))(eslint-plugin-promise@6.1.1(eslint@8.57.0))(eslint@8.57.0): dependencies: eslint: 8.57.0 eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) @@ -7366,7 +7383,7 @@ snapshots: debug: 4.4.1 enhanced-resolve: 5.16.1 eslint: 8.57.0 - eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) + eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) fast-glob: 3.3.2 get-tsconfig: 4.7.5 @@ -7378,7 +7395,7 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-module-utils@2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0): + eslint-module-utils@2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0): dependencies: debug: 3.2.7 optionalDependencies: @@ -7406,7 +7423,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) + eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) hasown: 2.0.2 is-core-module: 2.13.1 is-glob: 4.0.3 @@ -9291,7 +9308,7 @@ snapshots: vitest@4.0.17(@opentelemetry/api@1.9.0)(@types/node@25.6.0)(tsx@4.21.0): dependencies: '@vitest/expect': 4.0.17 - '@vitest/mocker': 4.0.17(vite@7.3.2(@types/node@25.6.0)(tsx@4.21.0)) + '@vitest/mocker': 4.0.17(vite@7.3.2(@types/node@22.19.1)(tsx@4.21.0)) '@vitest/pretty-format': 4.0.17 '@vitest/runner': 4.0.17 '@vitest/snapshot': 4.0.17 diff --git a/turbo.json b/turbo.json index e3d0866ae..16d7fb5a1 100644 --- a/turbo.json +++ b/turbo.json @@ -24,6 +24,7 @@ "ELEVEN_API_KEY", "FIREWORKS_API_KEY", "FISH_API_KEY", + "GNANI_API_KEY", "GROQ_API_KEY", "HEDRA_API_KEY", "HEDRA_API_URL",