diff --git a/.changeset/expressive-tts-markup.md b/.changeset/expressive-tts-markup.md new file mode 100644 index 000000000..fe9573603 --- /dev/null +++ b/.changeset/expressive-tts-markup.md @@ -0,0 +1,7 @@ +--- +'@livekit/agents-plugin-cartesia': patch +'@livekit/agents-plugin-inworld': patch +'@livekit/agents': patch +--- + +feat: LLM-driven TTS prosody via provider-specific markup tags (expressive mode) diff --git a/.changeset/port-builtin-workflows.md b/.changeset/port-builtin-workflows.md new file mode 100644 index 000000000..3b1a64503 --- /dev/null +++ b/.changeset/port-builtin-workflows.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Port the built-in data-capture workflows from python livekit-agents (`beta/workflows`) to the stable `workflows` namespace: `GetEmailTask`, `GetNameTask`, `GetPhoneNumberTask`, `GetDOBTask`, `GetAddressTask`, `GetDtmfTask`, and `GetCreditCardTask` (with `create*Task` functional variants). Includes the `DtmfEvent` enum with `formatDtmf`/`dtmfEventToCode` helpers, and `TaskGroup.add` now accepts factories for typed `AgentTask` results. diff --git a/agents/src/constants.ts b/agents/src/constants.ts index 3da61af46..af1c749bb 100644 --- a/agents/src/constants.ts +++ b/agents/src/constants.ts @@ -5,6 +5,15 @@ export const ATTRIBUTE_TRANSCRIPTION_TRACK_ID = 'lk.transcribed_track_id'; export const ATTRIBUTE_TRANSCRIPTION_FINAL = 'lk.transcription_final'; export const TOPIC_TRANSCRIPTION = 'lk.transcription'; export const ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID = 'lk.segment_id'; +/** + * The expression (delivery/emotion) the agent used for a transcription segment, surfaced so + * the frontend can react to it, when expressive markup is stripped from the transcript. The + * value is a JSON object `{"value": ...}` carrying the segment's leading expression — the + * `` tag for Inworld or the `` tag for Cartesia, e.g. + * `{"value": "speak happy"}`. A JSON object (rather than a bare string) so the shape can + * gain fields later without breaking parsers. + */ +export const ATTRIBUTE_TRANSCRIPTION_EXPRESSION = 'lk.expression'; export const ATTRIBUTE_PUBLISH_ON_BEHALF = 'lk.publish_on_behalf'; export const TOPIC_CHAT = 'lk.chat'; diff --git a/agents/src/inference/tts.ts b/agents/src/inference/tts.ts index f4011f69b..7c05e3418 100644 --- a/agents/src/inference/tts.ts +++ b/agents/src/inference/tts.ts @@ -10,7 +10,7 @@ import { ConnectionPool } from '../connection_pool.js'; import { type LanguageCode, normalizeLanguage } from '../language.js'; import { log } from '../log.js'; import { createStreamChannel } from '../stream/stream_channel.js'; -import { basic as tokenizeBasic } from '../tokenize/index.js'; +import { sentenceTokenizer as markupSentenceTokenizer } from '../tts/_provider_format.js'; import type { ChunkedStream } from '../tts/index.js'; import { SynthesizeStream as BaseSynthesizeStream, TTS as BaseTTS } from '../tts/index.js'; import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js'; @@ -380,6 +380,20 @@ export class TTS extends BaseTTS { return 'livekit'; } + /** + * Key into the shared markup tables, derived from the gateway model string. + * @internal + */ + override _markupProviderKey(): string { + const model = this.opts.model ?? ''; + const provider = model.split('/')[0]!; + if (provider === 'inworld') { + // older inworld models don't support markup + return model.includes('tts-2') ? 'inworld' : ''; + } + return provider; + } + override get capabilities() { return { streaming: true, alignedTranscript: this.#alignedTranscript }; } @@ -507,6 +521,13 @@ export class TTS extends BaseTTS { export class SynthesizeStream extends BaseSynthesizeStream { private opts: InferenceTTSOptions; private tts: TTS; + /** + * Snapshot whether expressive is active now, while the framework holds it + * fixed for this synthesis (set synchronously before stream()). Reading it + * lazily in run() would race with the next turn/session mutating the shared + * TTS instance. + */ + private expressive: boolean; #logger = log(); @@ -514,6 +535,7 @@ export class SynthesizeStream extends BaseSynthesizeSt super(tts, connOptions); this.opts = opts; this.tts = tts; + this.expressive = tts._expressive; } get label() { @@ -547,7 +569,11 @@ export class SynthesizeStream extends BaseSynthesizeSt // Python side. let pendingTimedTranscripts: TimedString[] = []; - const sendTokenizerStream = new tokenizeBasic.SentenceTokenizer().stream(); + // chunking defaults (cap + expressive batch size) live in _provider_format + const provider = (this.opts.model ?? '').split('/')[0]!; + const sendTokenizerStream = markupSentenceTokenizer(provider, { + expressive: this.expressive, + }).stream(); const eventChannel = createStreamChannel(); const requestId = shortuuid('tts_request_'); const inputSentEvent = new Event(); @@ -597,7 +623,7 @@ export class SynthesizeStream extends BaseSynthesizeSt sendTokenizerStream.flush(); continue; } - sendTokenizerStream.pushText(data); + sendTokenizerStream.pushText(this.tts.markup.normalize(data)); } // Only call endInput if the stream hasn't been closed by cleanup if (!closing) { @@ -617,11 +643,15 @@ export class SynthesizeStream extends BaseSynthesizeSt if (this.opts.model) generationConfig.model = this.opts.model; if (this.opts.language) generationConfig.language = this.opts.language; + // re-normalize at sentence level: tags split across input chunks + // aren't caught by the per-chunk normalize in the input task + const converted = this.tts.markup.convert(this.tts.markup.normalize(ev.token)); + this.markStarted(); await sendClientEvent( { type: 'input_transcript', - transcript: ev.token + ' ', + transcript: converted + ' ', generation_config: generationConfig, extra: (this.opts.modelOptions as Record) ?? {}, }, diff --git a/agents/src/llm/chat_context.test.ts b/agents/src/llm/chat_context.test.ts index 101a9fe46..03973a1a6 100644 --- a/agents/src/llm/chat_context.test.ts +++ b/agents/src/llm/chat_context.test.ts @@ -3,7 +3,6 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from 'vitest'; import { initializeLogger } from '../log.js'; -import { INSTRUCTIONS_MESSAGE_ID, applyInstructionsModality } from '../voice/generation.js'; import { FakeLLM } from '../voice/testing/fake_llm.js'; import { type AudioContent, @@ -15,7 +14,7 @@ import { type ImageContent, Instructions, ReadonlyChatContext, - concatInstructions, + instructionsEqual, isInstructions, renderInstructions, } from './chat_context.js'; @@ -1243,241 +1242,205 @@ describe('ChatContext.isEquivalent', () => { }); describe('Instructions', () => { - it('constructs from an object with audio and text variants', () => { - const instr = new Instructions({ audio: 'audio variant', text: 'text variant' }); + it('constructs from common text with audio and text additions', () => { + const instr = new Instructions('common text', { + audio: 'audio addition', + text: 'text addition', + }); - expect(instr.audio).toBe('audio variant'); - expect(instr.text).toBe('text variant'); - expect(instr.value).toBe('audio variant'); + expect(instr.common).toBe('common text'); + expect(instr.audio).toBe('audio addition'); + expect(instr.text).toBe('text addition'); + expect(instr.toString()).toBe('common text'); }); it('identifies Instructions with a type guard', () => { - const instr = new Instructions({ audio: 'audio variant', text: 'text variant' }); + const instr = new Instructions('common', { audio: 'audio addition' }); expect(isInstructions(instr)).toBe(true); - expect(isInstructions('audio variant')).toBe(false); - expect(isInstructions({ type: 'instructions', audio: 'audio variant' })).toBe(false); + expect(isInstructions('common')).toBe(false); + expect(isInstructions({ type: 'instructions', common: 'common' })).toBe(false); }); - it('tpl propagates Instructions interpolations into audio and text variants', () => { - const instr = Instructions.tpl`persona -${new Instructions({ audio: 'audio rules', text: 'text rules' })} -extra`; + it('render() combines common + modality-specific additions', () => { + const instr = new Instructions('You are a helpful assistant.', { + audio: 'Keep responses short for voice.', + text: 'Use markdown formatting.', + }); - expect(instr).toBeInstanceOf(Instructions); - expect(instr.audio).toBe('persona\naudio rules\nextra'); - expect(instr.text).toBe('persona\ntext rules\nextra'); - expect(instr.value).toBe('persona\naudio rules\nextra'); - expect(instr.asModality('text').value).toBe('persona\ntext rules\nextra'); + expect(instr.render({ modality: 'audio' })).toBe( + 'You are a helpful assistant.\n\nKeep responses short for voice.', + ); + expect(instr.render({ modality: 'text' })).toBe( + 'You are a helpful assistant.\n\nUse markdown formatting.', + ); + // no modality returns just the common text + expect(instr.render()).toBe('You are a helpful assistant.'); }); - it('tpl preserves audio-only interpolation as audio-only output', () => { - const instr = Instructions.tpl`prefix ${new Instructions({ audio: 'same' })} suffix`; - - expect(instr.toJSON()).toEqual({ type: 'instructions', audio: 'prefix same suffix' }); - expect(instr.audio).toBe('prefix same suffix'); - expect(instr.text).toBe('prefix same suffix'); + it('render() without modality additions returns just common for both', () => { + const commonOnly = new Instructions('common only'); + expect(commonOnly.render({ modality: 'audio' })).toBe('common only'); + expect(commonOnly.render({ modality: 'text' })).toBe('common only'); }); - it('tpl interpolates primitive values into both variants', () => { - const instr = Instructions.tpl`date=${'2026-05-13'} enabled=${true} count=${3}`; + it('render() with only one modality addition', () => { + const audioOnly = new Instructions('base', { audio: 'audio extra' }); + expect(audioOnly.render({ modality: 'audio' })).toBe('base\n\naudio extra'); + expect(audioOnly.render({ modality: 'text' })).toBe('base'); - expect(instr.toJSON()).toEqual({ - type: 'instructions', - audio: 'date=2026-05-13 enabled=true count=3', - }); - expect(instr.audio).toBe('date=2026-05-13 enabled=true count=3'); - expect(instr.text).toBe('date=2026-05-13 enabled=true count=3'); - expect(instr.value).toBe('date=2026-05-13 enabled=true count=3'); + const textOnly = new Instructions('base', { text: 'text extra' }); + expect(textOnly.render({ modality: 'audio' })).toBe('base'); + expect(textOnly.render({ modality: 'text' })).toBe('base\n\ntext extra'); }); - it('tpl combines multiple modality-aware interpolations', () => { - const instr = Instructions.tpl`${new Instructions({ audio: 'audio A', text: 'text A' })} / ${new Instructions({ audio: 'audio B', text: 'text B' })}`; - - expect(instr.audio).toBe('audio A / audio B'); - expect(instr.text).toBe('text A / text B'); - expect(instr.value).toBe('audio A / audio B'); + it('render() with empty common returns just the addition', () => { + const emptyCommon = new Instructions('', { audio: 'audio only', text: 'text only' }); + expect(emptyCommon.render({ modality: 'audio' })).toBe('audio only'); + expect(emptyCommon.render({ modality: 'text' })).toBe('text only'); }); - it('tpl preserves the current rendered value of resolved interpolations', () => { - const resolved = new Instructions({ audio: 'audio rules', text: 'text rules' }).asModality( - 'text', + it('render() fills template placeholders from data', () => { + const instr = new Instructions('Hello {name}, the weather is {weather.condition}.'); + expect(instr.render({ data: { name: 'Alice', weather: { condition: 'sunny' } } })).toBe( + 'Hello Alice, the weather is sunny.', ); - const instr = Instructions.tpl`prefix ${resolved} suffix`; - - expect(instr.audio).toBe('prefix audio rules suffix'); - expect(instr.text).toBe('prefix text rules suffix'); - expect(instr.value).toBe('prefix text rules suffix'); }); - it('tpl stringifies null and undefined values like template literals', () => { - const instr = Instructions.tpl`null=${null} undefined=${undefined}`; - - expect(instr.toJSON()).toEqual({ - type: 'instructions', - audio: 'null=null undefined=undefined', - }); - expect(instr.audio).toBe('null=null undefined=undefined'); - expect(instr.text).toBe('null=null undefined=undefined'); + it('render() replaces missing placeholders with empty strings', () => { + const instr = new Instructions('Hello {name}!'); + expect(instr.render({ data: { other: 'value' } })).toBe('Hello !'); }); - it('serializes to a dict with both variants and round-trips through toJSON', () => { - const instr = new Instructions({ audio: 'audio variant', text: 'text variant' }); - - const ctx = new ChatContext([ChatMessage.create({ role: 'system', content: [instr] })]); - const data = ctx.toJSON(); - const items = (data.items as Record[])!; - const content = (items[0]!.content as Record[])![0]!; - - expect(content).toEqual({ - type: 'instructions', - audio: 'audio variant', - text: 'text variant', + it('resolveTemplate interpolates primitive values into all variants', () => { + const instr = Instructions.resolveTemplate('date={date} count={count}', { + date: '2026-05-13', + count: 3, }); - }); - - it('omits the text key in toJSON when only audio variant is provided', () => { - const instr = new Instructions({ audio: 'audio only' }); - expect(instr.toJSON()).toEqual({ type: 'instructions', audio: 'audio only' }); - }); - it('falls back text -> audio when no text variant is provided', () => { - const instr = new Instructions({ audio: 'audio only' }); - expect(instr.audio).toBe('audio only'); - expect(instr.text).toBe('audio only'); - expect(instr.value).toBe('audio only'); + expect(instr).toBeInstanceOf(Instructions); + expect(instr.common).toBe('date=2026-05-13 count=3'); + expect(instr.audio).toBeUndefined(); + expect(instr.text).toBeUndefined(); }); - it('renderInstructions returns strings and resolved Instructions values explicitly', () => { - const instr = new Instructions({ audio: 'audio instructions', text: 'text instructions' }); - - expect(renderInstructions('plain instructions')).toBe('plain instructions'); - expect(renderInstructions(instr)).toBe('audio instructions'); - expect(renderInstructions(instr, 'audio')).toBe('audio instructions'); - expect(renderInstructions(instr, 'text')).toBe('text instructions'); - }); + it('resolveTemplate propagates Instructions interpolations into modality variants', () => { + const rules = new Instructions('common rules', { + audio: 'audio rules', + text: 'text rules', + }); + const instr = Instructions.resolveTemplate('persona\n{rules}\nextra', { rules }); - it('concatenates two Instructions, propagating both variants', () => { - const a = new Instructions({ audio: 'audio A', text: 'text A' }); - const b = new Instructions({ audio: 'audio B', text: 'text B' }); - const result = a.concat(b); - expect(result).toBeInstanceOf(Instructions); - expect(result.audio).toBe('audio Aaudio B'); - expect(result.text).toBe('text Atext B'); + expect(instr.common).toBe('persona\ncommon rules\nextra'); + expect(instr.audio).toBe('persona\naudio rules\nextra'); + expect(instr.text).toBe('persona\ntext rules\nextra'); }); - it('concatenates Instructions + string, propagating both variants', () => { - const instr = new Instructions({ audio: 'audio', text: 'text' }); - const result = instr.concat(' suffix'); - expect(result.audio).toBe('audio suffix'); - expect(result.text).toBe('text suffix'); - }); + it('resolveTemplate falls back to common when an Instructions kwarg has no variant', () => { + const rules = new Instructions('common rules', { audio: 'audio rules' }); + const instr = Instructions.resolveTemplate('prefix {rules} suffix', { rules }); - it('concatInstructions handles string + Instructions (radd-style)', () => { - const instr = new Instructions({ audio: 'audio', text: 'text' }); - const result = concatInstructions('prefix ', instr); - expect(isInstructions(result)).toBe(true); - if (!isInstructions(result)) return; - expect(result.audio).toBe('prefix audio'); - expect(result.text).toBe('prefix text'); + expect(instr.common).toBe('prefix common rules suffix'); + expect(instr.audio).toBe('prefix audio rules suffix'); + // no text variant on the kwarg: text falls back to the common value + expect(instr.text).toBe('prefix common rules suffix'); }); - it('preserves text=undefined when concatenating an audio-only instructions', () => { - const audioOnly = new Instructions({ audio: 'audio only' }); - const result = audioOnly.concat(' more'); - expect(result.toJSON()).toEqual({ type: 'instructions', audio: 'audio only more' }); - expect(result.audio).toBe('audio only more'); - expect(result.text).toBe('audio only more'); - }); + it('render() on resolveTemplate output picks the modality variant without duplicating the template', () => { + const rules = new Instructions('common rules', { + audio: 'audio rules', + text: 'text rules', + }); + const instr = Instructions.resolveTemplate('persona\n{rules}\nextra', { rules }); - it('when only one side has a text variant, the other contributes its audio', () => { - const a = new Instructions({ audio: 'audio A', text: 'text A' }); - const b = new Instructions({ audio: 'audio B' }); - const result = concatInstructions(a, ' ', b); - expect(isInstructions(result)).toBe(true); - if (!isInstructions(result)) return; - expect(result.audio).toBe('audio A audio B'); - expect(result.text).toBe('text A audio B'); + // the variants are full template renders — render(modality) must return the + // variant alone, not common + variant (which would repeat persona/extra twice) + expect(instr.render({ modality: 'audio' })).toBe('persona\naudio rules\nextra'); + expect(instr.render({ modality: 'text' })).toBe('persona\ntext rules\nextra'); + expect(instr.render()).toBe('persona\ncommon rules\nextra'); }); - it('asModality returns a copy with both variants preserved', () => { - const instr = new Instructions({ audio: 'audio instructions', text: 'text instructions' }); - - let resolved = instr.asModality('audio'); - expect(resolved.value).toBe('audio instructions'); - expect(resolved.audio).toBe('audio instructions'); - expect(resolved.text).toBe('text instructions'); + it('serializes with toJSON and omits undefined variants', () => { + const both = new Instructions('common text', { + audio: 'audio addition', + text: 'text addition', + }); + expect(both.toJSON()).toEqual({ + type: 'instructions', + common: 'common text', + audio: 'audio addition', + text: 'text addition', + }); - resolved = instr.asModality('text'); - expect(resolved.value).toBe('text instructions'); - expect(resolved.audio).toBe('audio instructions'); - expect(resolved.text).toBe('text instructions'); + const audioOnly = new Instructions('common', { audio: 'audio only' }); + expect(audioOnly.toJSON()).toEqual({ + type: 'instructions', + common: 'common', + audio: 'audio only', + }); }); - it('can switch modality after a previous resolution', () => { - const instr = new Instructions({ audio: 'audio instructions', text: 'text instructions' }); - const resolvedText = instr.asModality('text'); - const resolvedAudio = resolvedText.asModality('audio'); - expect(resolvedAudio.value).toBe('audio instructions'); - }); + it('renderInstructions returns strings and renders Instructions with modality', () => { + const instr = new Instructions('common', { + audio: 'audio addition', + text: 'text addition', + }); - it('asModality on audio-only Instructions returns audio for both modalities', () => { - const audioOnly = new Instructions({ audio: 'audio only' }); - expect(audioOnly.asModality('audio').value).toBe('audio only'); - expect(audioOnly.asModality('text').value).toBe('audio only'); + expect(renderInstructions('plain instructions')).toBe('plain instructions'); + expect(renderInstructions(instr)).toBe('common'); + expect(renderInstructions(instr, 'audio')).toBe('common\n\naudio addition'); + expect(renderInstructions(instr, 'text')).toBe('common\n\ntext addition'); }); - it('applyInstructionsModality rewrites the system message content', () => { - const instr = new Instructions({ audio: 'audio instructions', text: 'text instructions' }); - const ctx = new ChatContext([ - ChatMessage.create({ - id: INSTRUCTIONS_MESSAGE_ID, - role: 'system', - content: [instr], - }), - ]); - - applyInstructionsModality(ctx, { modality: 'audio' }); - let content = (ctx.items[0]! as ChatMessage).content[0]!; - expect(isInstructions(content) ? content.value : '').toBe('audio instructions'); + it('is resolved to str before storage in ChatMessage content', () => { + const instr = new Instructions('common text', { + audio: 'audio addition', + text: 'text addition', + }); - applyInstructionsModality(ctx, { modality: 'text' }); - content = (ctx.items[0]! as ChatMessage).content[0]!; - expect(isInstructions(content) ? content.value : '').toBe('text instructions'); + // addMessage with Instructions resolves to toString() = common + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'system', content: [instr.toString()] }); + const content = (ctx.items[0]! as ChatMessage).content[0]; + expect(typeof content).toBe('string'); + expect(content).toBe('common text'); + + // toJSON round-trip with resolved str content + const resolved = instr.render({ modality: 'audio' }); + const ctx2 = new ChatContext([ChatMessage.create({ role: 'system', content: [resolved] })]); + const data = ctx2.toJSON(); + const items = (data.items as Record[])!; + const serialized = (items[0]!.content as unknown[])![0]!; + expect(typeof serialized).toBe('string'); + expect(serialized).toBe('common text\n\naudio addition'); }); +}); - it('applyInstructionsModality is a no-op when content has no Instructions', () => { - const ctx = new ChatContext([ - ChatMessage.create({ - id: INSTRUCTIONS_MESSAGE_ID, - role: 'system', - content: ['plain string instructions'], - }), - ]); - const before = (ctx.items[0]! as ChatMessage).content[0]; - applyInstructionsModality(ctx, { modality: 'text' }); - expect((ctx.items[0]! as ChatMessage).content[0]).toBe(before); +describe('instructionsEqual', () => { + it('compares plain strings by value', () => { + expect(instructionsEqual('hello', 'hello')).toBe(true); + expect(instructionsEqual('hello', 'other')).toBe(false); + expect(instructionsEqual(undefined, 'hello')).toBe(false); + expect(instructionsEqual(undefined, undefined)).toBe(true); }); - it('survives copy and lets a different modality be applied to the copy', () => { - const instr = new Instructions({ audio: 'audio instructions', text: 'text instructions' }); - const baseCtx = new ChatContext([ - ChatMessage.create({ - id: INSTRUCTIONS_MESSAGE_ID, - role: 'system', - content: [instr], - }), - ]); - const turn1 = baseCtx.copy(); - applyInstructionsModality(turn1, { modality: 'text' }); - const turn2 = turn1.copy(); - applyInstructionsModality(turn2, { modality: 'audio' }); + it('compares Instructions by common/audio/text parts', () => { + const a = new Instructions('hello', { audio: 'a', text: 't' }); + const b = new Instructions('hello', { audio: 'a', text: 't' }); + const c = new Instructions('hello', { audio: 'other' }); + + expect(instructionsEqual(a, b)).toBe(true); + expect(instructionsEqual(a, c)).toBe(false); + }); - const turn2Content = (turn2.items[0]! as ChatMessage).content[0]!; - expect(isInstructions(turn2Content) ? turn2Content.value : '').toBe('audio instructions'); + it('equals a plain string only when there are no modality additions', () => { + expect(instructionsEqual(new Instructions('hello'), 'hello')).toBe(true); + expect(instructionsEqual('hello', new Instructions('hello'))).toBe(true); - // base context content is untouched (was the original instr) - expect((baseCtx.items[0]! as ChatMessage).content[0]).toBe(instr); + // an Instructions with additions renders differently — it must not compare + // equal to its bare common text (would skip realtime instruction updates) + expect(instructionsEqual(new Instructions('hello', { audio: 'extra' }), 'hello')).toBe(false); + expect(instructionsEqual('hello', new Instructions('hello', { text: 'extra' }))).toBe(false); }); }); diff --git a/agents/src/llm/chat_context.ts b/agents/src/llm/chat_context.ts index 34cf39200..a757c6235 100644 --- a/agents/src/llm/chat_context.ts +++ b/agents/src/llm/chat_context.ts @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 import type { AudioFrame, VideoFrame } from '@livekit/rtc-node'; -import { createImmutableArray, shortuuid } from '../utils.js'; +import { createImmutableArray, safeRender, shortuuid } from '../utils.js'; import type { LLM } from './llm.js'; import { type ProviderFormat, toChatCtx } from './provider_format/index.js'; import type { JSONObject, JSONValue, ToolContext } from './tool_context.js'; @@ -37,15 +37,6 @@ export interface AudioContent { transcript?: string; } -type InstructionsOptions = { - /** The audio/voice variant of the instructions. */ - audio: string; - /** The text variant of the instructions; falls back to `audio` when omitted. */ - text?: string; - /** The currently rendered string value, used by `value`/`toString()`. */ - represent?: string; -}; - const INSTRUCTIONS_SYMBOL = Symbol.for('livekit.agents.Instructions'); export function isInstructions(value: unknown): value is Instructions { @@ -58,131 +49,158 @@ export function isInstructions(value: unknown): value is Instructions { } /** - * Instructions that adapt based on the user's input modality (audio vs. text). + * Instructions with optional modality-specific additions. + * + * Construction: + * ```ts + * // Simple — same instructions for all modalities + * new Instructions('You are a helpful assistant.'); + * + * // With modality-specific additions + * new Instructions('You are a helpful assistant.', { + * audio: 'Keep responses short for voice.', + * text: 'Use markdown formatting.', + * }); + * ``` * - * The `value` property is the rendered string providers see. By default it - * equals the `audio` variant; after {@link asModality} it equals the chosen - * variant. Both the `audio` variant and the raw `text` variant are preserved - * so {@link asModality} can be called again for a different modality (e.g., - * when the same `ChatContext` is reused across tool-call turns). + * Rendering: + * ```ts + * instr.render(); // → common text + * instr.render({ modality: 'audio' }); // → common + audio addition + * instr.render({ modality: 'text', data: { name: 'Alex' } }); // → common + text, with {name} filled + * ``` */ export class Instructions { readonly type = 'instructions' as const; - private readonly _audioVariant: string; + common: string; - /** Raw text variant; falls back to {@link audio} when omitted. */ - private readonly _textVariant?: string; + audio?: string; - /** The currently rendered string (what providers should treat as content). */ - readonly value: string; + text?: string; /** @internal Symbol marker for type identification */ readonly [INSTRUCTIONS_SYMBOL] = true; - constructor(options: InstructionsOptions) { - this._audioVariant = options.audio; - this._textVariant = options.text; - this.value = options.represent ?? options.audio; + /** + * When true (set by {@link Instructions.resolveTemplate} when modality variants were + * produced), `audio`/`text` hold *full* renders of the template and replace `common` + * in {@link render} instead of being appended to it. Not serialized by `toJSON`. + * @internal + */ + private variantsReplaceCommon = false; + + constructor(common: string = '', options?: { audio?: string; text?: string }) { + this.common = common; + this.audio = options?.audio; + this.text = options?.text; } - static tpl( - strings: TemplateStringsArray, - ...values: Array - ): Instructions { - const render = (mode: 'audio' | 'text' | 'value') => { - let result = strings[0]!; - for (let i = 0; i < values.length; i++) { - const value = values[i]!; - if (isInstructions(value)) { - result += mode === 'audio' ? value.audio : mode === 'text' ? value.text : value.value; - } else { - result += String(value); - } - result += strings[i + 1]!; + /** + * Render instructions to a plain string. + * + * @param options.modality - If given, appends the modality-specific addition to the common text. + * @param options.data - Template variables to fill. Missing placeholders log an error + * and are replaced with empty strings. + */ + render(options?: { modality?: 'audio' | 'text'; data?: Record }): string { + const { modality, data } = options ?? {}; + + let parts = [this.common]; + if (modality !== undefined) { + const addition = modality === 'audio' ? this.audio : this.text; + if (addition) { + // resolveTemplate variants are full renders of the template, not additions — + // appending them to common would duplicate the whole template + parts = this.variantsReplaceCommon ? [addition] : [...parts, addition]; } - return result; - }; - - const hasTextVariant = values.some( - (value) => isInstructions(value) && value._textVariant !== undefined, - ); + } - return new Instructions({ - audio: render('audio'), - text: hasTextVariant ? render('text') : undefined, - represent: render('value'), - }); - } + let result = parts.filter((p) => p).join('\n\n'); - /** The audio (voice) variant of the instructions. */ - get audio(): string { - return this._audioVariant; - } + if (data && Object.keys(data).length > 0) { + result = safeRender(result, data); + } - /** The text variant of the instructions. Falls back to {@link audio}. */ - get text(): string { - return this._textVariant ?? this.audio; + return result; } /** - * Return a copy whose {@link value} is the variant matching `modality`. - * Both `audio` and `text` variants are preserved on the result, so this can - * be called again for a different modality (e.g. across tool-call turns). + * Fill a template string, producing an `Instructions` with modality variants. + * + * If any kwarg value is an `Instructions` object, its `common`/`audio`/`text` + * parts are substituted into the matching variant of the result. This is used by + * workflow tasks to build modality-aware instructions from a single template. */ - asModality(modality: 'audio' | 'text'): Instructions { - return new Instructions({ - audio: this.audio, - text: this._textVariant, - represent: modality === 'audio' ? this.audio : this.text, - }); - } - - /** Concatenate, propagating both variants and the current rendered value. */ - concat(other: string | Instructions): Instructions { - if (isInstructions(other)) { - const hasText = this._textVariant !== undefined || other._textVariant !== undefined; - return new Instructions({ - audio: this.audio + other.audio, - text: hasText ? this.text + other.text : undefined, - represent: this.value + other.value, + static resolveTemplate(template: string, kwargs: Record): Instructions { + const anyInstructions = Object.values(kwargs).some((v) => isInstructions(v)); + if (anyInstructions) { + const commonKw: Record = {}; + const audioKw: Record = {}; + const textKw: Record = {}; + for (const [k, v] of Object.entries(kwargs)) { + if (isInstructions(v)) { + commonKw[k] = v.toString(); + // an explicit "" removes the section; only undefined falls back to common + audioKw[k] = v.audio !== undefined ? v.audio : v.toString(); + textKw[k] = v.text !== undefined ? v.text : v.toString(); + } else { + commonKw[k] = v; + audioKw[k] = v; + textKw[k] = v; + } + } + const resolved = new Instructions(safeRender(template, commonKw), { + audio: safeRender(template, audioKw), + text: safeRender(template, textKw), }); + // the variants are full template renders: render(modality) must pick one, + // not append it to the common render + resolved.variantsReplaceCommon = true; + return resolved; } - return new Instructions({ - audio: this.audio + other, - text: this._textVariant !== undefined ? this._textVariant + other : undefined, - represent: this.value + other, - }); + return new Instructions(safeRender(template, kwargs)); } toString(): string { - return this.value; + return this.common; } - toJSON(): { type: 'instructions'; audio: string; text?: string } { - const result: { type: 'instructions'; audio: string; text?: string } = { + toJSON(): { type: 'instructions'; common: string; audio?: string; text?: string } { + const result: { type: 'instructions'; common: string; audio?: string; text?: string } = { type: 'instructions', - audio: this.audio, + common: this.common, }; - if (this._textVariant !== undefined) { - result.text = this._textVariant; + if (this.audio !== undefined) { + result.audio = this.audio; + } + if (this.text !== undefined) { + result.text = this.text; } return result; } } +/** + * Resolve instructions to a plain string. Plain strings pass through; + * {@link Instructions} are rendered (with the modality-specific addition + * appended when `modality` is given). + */ export function renderInstructions( instructions: string | Instructions, modality?: 'audio' | 'text', ): string { if (typeof instructions === 'string') return instructions; - return modality === undefined ? instructions.value : instructions.asModality(modality).value; + return instructions.render({ modality }); } /** * Compare two instruction values by content. Plain strings compare by value; - * {@link Instructions} compare by their audio + text variants so that two - * distinct instances with the same content are treated as equal. + * {@link Instructions} compare by their common/audio/text parts so that two + * distinct instances with the same content are treated as equal. An + * {@link Instructions} equals a plain string only when it has no modality + * additions and its common text matches — an Instructions with additions + * renders differently, so it must not compare equal to its bare common text. */ export function instructionsEqual( a: string | Instructions | undefined, @@ -193,41 +211,18 @@ export function instructionsEqual( const aIsInstr = isInstructions(a); const bIsInstr = isInstructions(b); if (aIsInstr && bIsInstr) { - return a.audio === b.audio && a.text === b.text; + return a.common === b.common && a.audio === b.audio && a.text === b.text; } - if (!aIsInstr && !bIsInstr) { - return a === b; + if (aIsInstr && !bIsInstr) { + return a.audio === undefined && a.text === undefined && a.common === b; } - return false; -} - -/** - * Concatenate any mix of plain strings and {@link Instructions}, propagating - * both audio/text variants. If no argument is an {@link Instructions} the - * result is a plain string; otherwise the result is an {@link Instructions} - * preserving both variants from every contributing operand. - */ -export function concatInstructions(...parts: Array): string | Instructions { - if (parts.length === 0) return ''; - const hasInstructions = parts.some((p) => isInstructions(p)); - if (!hasInstructions) return parts.join(''); - - let acc = parts[0]!; - for (let i = 1; i < parts.length; i++) { - const next = parts[i]!; - if (isInstructions(acc)) { - acc = acc.concat(next); - } else if (isInstructions(next)) { - // string + Instructions (radd-style): prepend `acc` to both variants. - acc = new Instructions({ audio: acc }).concat(next); - } else { - acc = acc + next; - } + if (!aIsInstr && bIsInstr) { + return b.audio === undefined && b.text === undefined && a === b.common; } - return acc; + return a === b; } -export type ChatContent = ImageContent | AudioContent | Instructions | string; +export type ChatContent = ImageContent | AudioContent | string; export function createImageContent(params: { image: string | VideoFrame; @@ -361,9 +356,7 @@ export class ChatMessage { * lines. If no string content is present, returns `null`. */ get textContent(): string | undefined { - const parts = this.content - .filter((c): c is string | Instructions => typeof c === 'string' || isInstructions(c)) - .map((c) => (typeof c === 'string' ? c : c.value)); + const parts = this.content.filter((c): c is string => typeof c === 'string'); return parts.length > 0 ? parts.join('\n') : undefined; } @@ -371,8 +364,6 @@ export class ChatMessage { return this.content.map((c) => { if (typeof c === 'string') { return c as JSONValue; - } else if (isInstructions(c)) { - return c.toJSON() as JSONValue; } else if (c.type === 'image_content') { return { id: c.id, @@ -650,7 +641,7 @@ export class AgentConfigUpdate { readonly type = 'agent_config_update' as const; - instructions?: string | Instructions; + instructions?: string; toolsAdded?: string[]; @@ -661,7 +652,7 @@ export class AgentConfigUpdate { constructor( params: { id?: string; - instructions?: string | Instructions; + instructions?: string; toolsAdded?: string[]; toolsRemoved?: string[]; createdAt?: number; @@ -683,7 +674,7 @@ export class AgentConfigUpdate { static create(params: { id?: string; - instructions?: string | Instructions; + instructions?: string; toolsAdded?: string[]; toolsRemoved?: string[]; createdAt?: number; @@ -698,9 +689,7 @@ export class AgentConfigUpdate { }; if (this.instructions !== undefined) { - result.instructions = isInstructions(this.instructions) - ? (this.instructions.toJSON() as JSONValue) - : this.instructions; + result.instructions = this.instructions; } if (this.toolsAdded !== undefined) { result.toolsAdded = this.toolsAdded; @@ -747,7 +736,7 @@ export class ChatContext { */ addMessage(params: { role: ChatRole; - content: ChatContent[] | string; + content: ChatContent[] | string | Instructions; id?: string; interrupted?: boolean; createdAt?: number; @@ -755,7 +744,8 @@ export class ChatContext { metrics?: MetricsReport; extra?: Record; }): ChatMessage { - const msg = new ChatMessage(params); + const content = isInstructions(params.content) ? params.content.toString() : params.content; + const msg = new ChatMessage({ ...params, content }); if (params.createdAt !== undefined) { const idx = this.findInsertionIndex(params.createdAt); this._items.splice(idx, 0, msg); @@ -1087,21 +1077,6 @@ export class ChatContext { return false; } - if (isInstructions(contentA) && isInstructions(contentB)) { - if ( - contentA.audio !== contentB.audio || - contentA.text !== contentB.text || - contentA.value !== contentB.value - ) { - return false; - } - continue; - } - - if (isInstructions(contentA) || isInstructions(contentB)) { - return false; - } - if (typeof contentA === 'object' && typeof contentB === 'object') { if (contentA.type === 'image_content' && contentB.type === 'image_content') { if ( diff --git a/agents/src/llm/index.ts b/agents/src/llm/index.ts index 4773501dc..22e19d2ee 100644 --- a/agents/src/llm/index.ts +++ b/agents/src/llm/index.ts @@ -46,7 +46,6 @@ export { ChatContext, ChatMessage, Instructions, - concatInstructions, createAudioContent, createImageContent, FunctionCall, diff --git a/agents/src/llm/provider_format/google.test.ts b/agents/src/llm/provider_format/google.test.ts index c895a4bde..54588f182 100644 --- a/agents/src/llm/provider_format/google.test.ts +++ b/agents/src/llm/provider_format/google.test.ts @@ -64,20 +64,21 @@ describe('Google Provider Format - toChatCtx', () => { }); it('should render Instructions as their resolved value', async () => { + // Instructions are resolved to a plain string before storage in the chat context const ctx = ChatContext.empty(); ctx.addMessage({ role: 'system', content: [ - new Instructions({ audio: 'audio instructions', text: 'text instructions' }).asModality( - 'text', - ), + new Instructions('common instructions', { text: 'text instructions' }).render({ + modality: 'text', + }), ], }); ctx.addMessage({ role: 'user', content: 'Hello' }); const [, formatData] = await toChatCtx(ctx, false); - expect(formatData.systemMessages).toEqual(['text instructions']); + expect(formatData.systemMessages).toEqual(['common instructions\n\ntext instructions']); }); it('should handle multiple system messages', async () => { diff --git a/agents/src/llm/provider_format/google.ts b/agents/src/llm/provider_format/google.ts index 2ef717f28..6902dd702 100644 --- a/agents/src/llm/provider_format/google.ts +++ b/agents/src/llm/provider_format/google.ts @@ -2,7 +2,6 @@ // // SPDX-License-Identifier: Apache-2.0 import type { ChatContext, ChatItem, ImageContent } from '../chat_context.js'; -import { isInstructions } from '../chat_context.js'; import { type SerializedImage, serializeImage } from '../utils.js'; import { convertMidConversationInstructions, groupToolCalls } from './utils.js'; @@ -60,8 +59,6 @@ export async function toChatCtx( for (const content of msg.content) { if (content && typeof content === 'string') { parts.push({ text: content }); - } else if (isInstructions(content)) { - parts.push({ text: content.value }); } else if (content && typeof content === 'object') { if (content.type === 'image_content') { parts.push(await toImagePart(content)); diff --git a/agents/src/llm/provider_format/mistralai.test.ts b/agents/src/llm/provider_format/mistralai.test.ts index db7387637..45a04f94a 100644 --- a/agents/src/llm/provider_format/mistralai.test.ts +++ b/agents/src/llm/provider_format/mistralai.test.ts @@ -41,20 +41,21 @@ describe('Mistral Provider Format - toChatCtx', () => { }); it('should render Instructions as their resolved value', () => { + // Instructions are resolved to a plain string before storage in the chat context const ctx = ChatContext.empty(); ctx.addMessage({ role: 'system', content: [ - new Instructions({ audio: 'audio instructions', text: 'text instructions' }).asModality( - 'text', - ), + new Instructions('common instructions', { text: 'text instructions' }).render({ + modality: 'text', + }), ], }); ctx.addMessage({ role: 'user', content: 'Hello' }); const [, formatData] = toChatCtx(ctx); - expect(formatData.instructions).toBe('text instructions'); + expect(formatData.instructions).toBe('common instructions\n\ntext instructions'); }); it('should extract developer messages as instructions', () => { diff --git a/agents/src/llm/provider_format/mistralai.ts b/agents/src/llm/provider_format/mistralai.ts index 776be3085..b5b0d1723 100644 --- a/agents/src/llm/provider_format/mistralai.ts +++ b/agents/src/llm/provider_format/mistralai.ts @@ -2,7 +2,6 @@ // // SPDX-License-Identifier: Apache-2.0 import type { ChatContext } from '../chat_context.js'; -import { isInstructions } from '../chat_context.js'; export interface MistralFormatData { instructions: string; @@ -28,10 +27,7 @@ export function toChatCtx( for (const item of chatCtx.items) { if (item.type === 'message') { - const text = item.content - .filter((c) => typeof c === 'string' || isInstructions(c)) - .map((c) => (typeof c === 'string' ? c : c.value)) - .join('\n'); + const text = item.content.filter((c) => typeof c === 'string').join('\n'); if (item.role === 'system' || item.role === 'developer') { instructionParts.push(text); diff --git a/agents/src/llm/provider_format/openai.test.ts b/agents/src/llm/provider_format/openai.test.ts index ae74bac7b..c810b99a7 100644 --- a/agents/src/llm/provider_format/openai.test.ts +++ b/agents/src/llm/provider_format/openai.test.ts @@ -56,20 +56,24 @@ describe('toChatCtx', () => { }); it('should render Instructions as their resolved value', async () => { + // Instructions are resolved to a plain string before storage in the chat context const ctx = ChatContext.empty(); ctx.addMessage({ role: 'system', content: [ - new Instructions({ audio: 'audio instructions', text: 'text instructions' }).asModality( - 'text', - ), + new Instructions('common instructions', { text: 'text instructions' }).render({ + modality: 'text', + }), ], }); ctx.addMessage({ role: 'user', content: 'Hello' }); const result = await toChatCtx(ctx); - expect(result[0]).toEqual({ role: 'system', content: 'text instructions' }); + expect(result[0]).toEqual({ + role: 'system', + content: 'common instructions\n\ntext instructions', + }); }); it('should handle multi-line text content', async () => { @@ -758,20 +762,24 @@ describe('toResponsesChatCtx', () => { }); it('should render Instructions as their resolved value', async () => { + // Instructions are resolved to a plain string before storage in the chat context const ctx = ChatContext.empty(); ctx.addMessage({ role: 'system', content: [ - new Instructions({ audio: 'audio instructions', text: 'text instructions' }).asModality( - 'text', - ), + new Instructions('common instructions', { text: 'text instructions' }).render({ + modality: 'text', + }), ], }); ctx.addMessage({ role: 'user', content: 'Hello' }); const result = await toResponsesChatCtx(ctx); - expect(result[0]).toEqual({ role: 'system', content: 'text instructions' }); + expect(result[0]).toEqual({ + role: 'system', + content: 'common instructions\n\ntext instructions', + }); }); it('should handle multi-line text content', async () => { diff --git a/agents/src/llm/provider_format/openai.ts b/agents/src/llm/provider_format/openai.ts index f9efe4798..8f630ec43 100644 --- a/agents/src/llm/provider_format/openai.ts +++ b/agents/src/llm/provider_format/openai.ts @@ -2,7 +2,6 @@ // // SPDX-License-Identifier: Apache-2.0 import type { ChatContext, ChatItem, ImageContent } from '../chat_context.js'; -import { isInstructions } from '../chat_context.js'; import { type SerializedImage, serializeImage } from '../utils.js'; import { groupToolCalls } from './utils.js'; @@ -70,9 +69,6 @@ async function toChatItem(item: ChatItem) { if (typeof content === 'string') { if (textContent) textContent += '\n'; textContent += content; - } else if (isInstructions(content)) { - if (textContent) textContent += '\n'; - textContent += content.value; } else if (content.type === 'image_content') { listContent.push(await toImageContent(content)); } else { @@ -233,9 +229,6 @@ async function toResponsesChatItem(item: ChatItem) { if (typeof content === 'string') { if (textContent) textContent += '\n'; textContent += content; - } else if (isInstructions(content)) { - if (textContent) textContent += '\n'; - textContent += content.value; } else if (content.type === 'image_content') { listContent.push(await toResponsesImageContent(content)); } else { diff --git a/agents/src/llm/utils.ts b/agents/src/llm/utils.ts index 0db2bb802..1ee247cc0 100644 --- a/agents/src/llm/utils.ts +++ b/agents/src/llm/utils.ts @@ -460,10 +460,6 @@ export function validateChatContextStructure(chatCtx: ChatContext): ChatContextV return; } - if (term.type === 'instructions') { - return; - } - if (term.type === 'image_content') { if (!term.id || term.image === undefined || term.image === null) { pushIssue({ @@ -614,10 +610,6 @@ function formatMessageContentPart(part: ChatContent): string { return part; } - if (part.type === 'instructions') { - return part.value; - } - if (part.type === 'image_content') { if (typeof part.image === 'string') { return `[image url=${truncateText(part.image, 120)}]`; diff --git a/agents/src/stt/index.ts b/agents/src/stt/index.ts index 5b0517546..f9a6815a7 100644 --- a/agents/src/stt/index.ts +++ b/agents/src/stt/index.ts @@ -10,6 +10,7 @@ export { export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js'; export { type RecognitionUsage, + type SpeakerContext, type SpeechData, type SpeechEvent, SpeechEventType, @@ -17,5 +18,6 @@ export { STT, type STTCallbacks, type STTCapabilities, + isSpeakerContext, } from './stt.js'; export * as testing from './testing/index.js'; diff --git a/agents/src/stt/stt.ts b/agents/src/stt/stt.ts index 923517964..3865c6585 100644 --- a/agents/src/stt/stt.ts +++ b/agents/src/stt/stt.ts @@ -287,6 +287,27 @@ export abstract class STT extends (EventEmitter as new () => TypedEmitter private logger = log(); private _connOptions: APIConnectOptions; private _startTimeOffset: number = 0; + private _context?: object; protected abortController = new AbortController(); @@ -463,6 +485,21 @@ export abstract class SpeechStream implements AsyncIterableIterator return this.abortController.signal; } + /** + * Persistent speaker metadata updated by the STT plugin during recognition. + * + * STT plugins set this to a context object (e.g. a voice profile with + * emotion, gender, etc.) as they learn about the speaker over the life of the + * stream. Accessible via `agent.audioRecognition.sttContext`. + */ + get context(): object | undefined { + return this._context; + } + + set context(value: object | undefined) { + this._context = value; + } + get startTimeOffset(): number { return this._startTimeOffset; } diff --git a/agents/src/telemetry/traces.ts b/agents/src/telemetry/traces.ts index e1a14b4de..a8f14d3b6 100644 --- a/agents/src/telemetry/traces.ts +++ b/agents/src/telemetry/traces.ts @@ -24,7 +24,7 @@ import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions'; import FormData from 'form-data'; import { AccessToken } from 'livekit-server-sdk'; import fs from 'node:fs/promises'; -import { isInstructions, renderInstructions } from '../llm/chat_context.js'; +import { renderInstructions } from '../llm/chat_context.js'; import type { ChatContent, ChatItem, ChatRole } from '../llm/index.js'; import { enableOtelLogging } from '../log.js'; import { filterZeroValues } from '../metrics/model_usage.js'; @@ -415,10 +415,8 @@ function chatItemToProto(item: ChatItem): ProtoChatItem { // the ChatContent proto's `text` field is a string, and non-string values // render as garbage in the dashboard. content: item.content - .filter((c: ChatContent) => typeof c === 'string' || isInstructions(c)) - .map((c) => ({ - text: isInstructions(c) ? c.value : (c as string), - })), + .filter((c: ChatContent) => typeof c === 'string') + .map((c) => ({ text: c })), createdAt: toRFC3339(item.createdAt), }; diff --git a/agents/src/tokenize/basic/basic.ts b/agents/src/tokenize/basic/basic.ts index dbbf1e5cf..bd87ac95f 100644 --- a/agents/src/tokenize/basic/basic.ts +++ b/agents/src/tokenize/basic/basic.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js'; +import { BufferedSentenceStream, BufferedWordStream, xmlWrapTokenizer } from '../token_stream.js'; import * as tokenizer from '../tokenizer.js'; import { hyphenator } from './hyphenator.js'; import { splitParagraphs } from './paragraph.js'; @@ -10,9 +10,34 @@ import { splitWords } from './word.js'; interface TokenizerOptions { language: string; + /** + * Minimum length for a span to be treated as its own sentence; shorter spans + * are merged forward into the next one. + */ minSentenceLength: number; + /** Minimum buffered text before the stream emits. */ streamContextLength: number; + /** Keep original whitespace/formatting in emitted tokens. */ retainFormat: boolean; + /** + * Hard cap on emitted token length; a token is flushed before appending a + * sentence that would exceed it. + */ + maxTokenLength?: number; + /** + * Minimum length a token must reach before it is emitted. Sentences are + * batched together until the running token reaches this length, so raising + * it (e.g. toward `maxTokenLength`) yields larger, fewer chunks. Defaults to + * `minSentenceLength` (per-sentence emission). + */ + minTokenLength?: number; + /** + * Treat XML markup as atomic — never split a tag across tokens and keep tags + * attached to the following sentence. Only enable when the input actually + * carries markup (e.g. expressive TTS): a stray "<" in plain text can + * otherwise hold back streaming until flush. + */ + xmlAware?: boolean; } const defaultTokenizerOptions: TokenizerOptions = { @@ -35,7 +60,12 @@ export class SentenceTokenizer extends tokenizer.SentenceTokenizer { // eslint-disable-next-line @typescript-eslint/no-unused-vars tokenize(text: string, language?: string): string[] { - return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]); + let tokenizeFunc = (t: string) => + splitSentences(t, this.#config.minSentenceLength, this.#config.retainFormat); + if (this.#config.xmlAware) { + tokenizeFunc = xmlWrapTokenizer(tokenizeFunc) as typeof tokenizeFunc; + } + return tokenizeFunc(text).map((tok) => (Array.isArray(tok) ? tok[0] : tok)); } // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -43,8 +73,12 @@ export class SentenceTokenizer extends tokenizer.SentenceTokenizer { return new BufferedSentenceStream( (text: string) => splitSentences(text, this.#config.minSentenceLength, this.#config.retainFormat), - this.#config.minSentenceLength, + this.#config.minTokenLength ?? this.#config.minSentenceLength, this.#config.streamContextLength, + { + maxTokenLength: this.#config.maxTokenLength, + xmlAware: this.#config.xmlAware, + }, ); } } diff --git a/agents/src/tokenize/basic/sentence.ts b/agents/src/tokenize/basic/sentence.ts index a6df4b2c7..2a4e5833e 100644 --- a/agents/src/tokenize/basic/sentence.ts +++ b/agents/src/tokenize/basic/sentence.ts @@ -82,7 +82,10 @@ export const splitSentences = ( } if (buf) { - sentences.push([buf.slice(prePad.length), start, text.length - 1]); + // end must cover the full remaining text: xmlWrapTokenizer maps this offset + // back onto the original string, and an off-by-one here splits the last + // character into a phantom sentence (emitting mid-word chunks upstream). + sentences.push([buf.slice(prePad.length), start, text.length]); } return sentences; diff --git a/agents/src/tokenize/token_stream.ts b/agents/src/tokenize/token_stream.ts index 9b383fe9b..bd79d2f61 100644 --- a/agents/src/tokenize/token_stream.ts +++ b/agents/src/tokenize/token_stream.ts @@ -7,6 +7,182 @@ import { SentenceStream, WordStream } from './tokenizer.js'; type TokenizeFunc = (x: string) => string[] | [string, number, number][]; +// the tag name must start with a letter so "<5>" / "<3 wins>" are not counted as +// tags — this keeps the depth counter consistent with the letter-start tail check +// in hasUnclosedXmlTags (all TTS markup tags are letter-named). +// The body is a single `[^<>]*` — excluding both angle brackets keeps matching +// linear-time on untrusted LLM output (a body allowing "<" makes unterminated +// input like "]*)>/g; + +/** Return true if a tag body captured by {@link XML_TAG_RE} marks a self-closing tag. */ +function isSelfClosingTagBody(body: string): boolean { + return body.trimEnd().endsWith('/'); +} + +/** Return true if `text` contains an incomplete or unclosed XML tag. */ +export function hasUnclosedXmlTags(text: string): boolean { + if (!text.includes('<')) { + return false; + } + + // incomplete tag at end: a tag-shaped "<" without a matching ">". Only "<" + // followed by a name start ("/" or a letter) is tag-shaped — a bare "<" as in + // "3 < 5" or "<3" is plain text and must not hold up streaming. Text ending + // exactly at "<" is treated as tag-shaped: the next chunk resolves it. + const lastOpen = text.lastIndexOf('<'); + const lastClose = text.lastIndexOf('>'); + if (lastOpen > lastClose) { + const nxt = text.slice(lastOpen + 1, lastOpen + 2); + if (!nxt || nxt === '/' || /[a-zA-Z]/.test(nxt)) { + return true; + } + } + + // unbalanced open/close pairs + let depth = 0; + for (const m of text.matchAll(XML_TAG_RE)) { + const isClosing = m[1] === '/'; + const isSelfClosing = isSelfClosingTagBody(m[2]!); + if (isSelfClosing) { + continue; + } else if (isClosing) { + depth -= 1; + } else { + depth += 1; + } + } + + return depth > 0; +} + +/** Return true if `text` contains XML tags but no substantive text content. */ +export function isXmlOnly(text: string): boolean { + if (!text.includes('<')) { + return false; + } + + const stripped = text.replace(XML_TAG_RE, '').trim(); + return stripped.length === 0; +} + +/** + * Map a position in tag-stripped text to the corresponding original position. + * + * Tags that sit right at the boundary are left for the next sentence. + */ +function cleanToOrig(cleanPos: number, tagSpans: [number, number][]): number { + let orig = cleanPos; + for (const [tagStart, tagEnd] of tagSpans) { + if (tagStart < orig) { + orig += tagEnd - tagStart; + } else { + break; + } + } + return orig; +} + +/** + * Wrap a tokenizer so XML tags don't interfere with sentence splitting. + * + * Strips tag markers before tokenization (content inside wrapping tags is + * kept so the tokenizer can account for its length), remaps offsets back to + * the original text, and merges sentences with unclosed or tag-only content. + */ +export function xmlWrapTokenizer(tokenizeFunc: TokenizeFunc): TokenizeFunc { + const wrappedImpl = (text: string): string[] | [string, number, number][] => { + const tagSpans: [number, number][] = [...text.matchAll(XML_TAG_RE)].map((m) => [ + m.index!, + m.index! + m[0].length, + ]); + if (tagSpans.length === 0) { + return tokenizeFunc(text); + } + + const cleanText = text.replace(XML_TAG_RE, ''); + if (!cleanText.trim()) { + return text.trim() ? [[text, 0, text.length]] : []; + } + + const rawTokens = tokenizeFunc(cleanText); + if (rawTokens.length === 0) { + return []; + } + + // extract clean-text end offsets + let cleanEnds: number[] = rawTokens.map((tok) => (Array.isArray(tok) ? tok[2] : -1)); + + // if tokenizer didn't provide offsets, approximate from token lengths + if (cleanEnds[0] === -1) { + let pos = 0; + cleanEnds = []; + for (const tok of rawTokens) { + const tokText = typeof tok === 'string' ? tok : tok[0]; + const idx = cleanText.indexOf(tokText, pos); + pos = (idx >= 0 ? idx : pos) + tokText.length; + cleanEnds.push(pos); + } + } + + // remap to original positions and rebuild sentences + let result: [string, number, number][] = []; + let start = 0; + for (const cleanEnd of cleanEnds) { + const origEnd = cleanToOrig(cleanEnd, tagSpans); + const sentence = text.slice(start, origEnd).trim(); + if (sentence) { + result.push([sentence, start, origEnd]); + } + start = origEnd; + } + + if (start < text.length) { + const sentence = text.slice(start).trim(); + if (sentence) { + result.push([sentence, start, text.length]); + } + } + + // merge sentences with unclosed tags or tag-only content + if (result.length > 0) { + const merged: [string, number, number][] = [result[0]!]; + for (const [sentText, sStart, sEnd] of result.slice(1)) { + const [prevText, prevStart] = merged[merged.length - 1]!; + if (hasUnclosedXmlTags(prevText) || isXmlOnly(prevText)) { + merged[merged.length - 1] = [text.slice(prevStart, sEnd).trim(), prevStart, sEnd]; + } else { + merged.push([sentText, sStart, sEnd]); + } + } + result = merged; + } + + return result; + }; + + return (text: string) => { + try { + return wrappedImpl(text); + } catch { + return text.trim() ? [[text, 0, text.length]] : []; + } + }; +} + +export interface BufferedTokenStreamOptions { + /** Hard cap on emitted token length; the buffer is flushed before appending a token that would exceed it. */ + maxTokenLength?: number; + /** + * Treat XML markup as atomic — never split a tag across tokens and merge + * tag-only/unclosed spans forward. Only enable when the input actually + * carries markup (e.g. expressive TTS): a stray "<" in plain text can + * otherwise hold back streaming until flush. + */ + xmlAware?: boolean; +} + export class BufferedTokenStream implements AsyncIterableIterator { protected queue = new AsyncIterableQueue(); protected closed = false; @@ -14,15 +190,23 @@ export class BufferedTokenStream implements AsyncIterableIterator { #func: TokenizeFunc; #minTokenLength: number; #minContextLength: number; - #bufTokens: string[] = []; + #maxTokenLength?: number; + #xmlAware: boolean; #inBuf = ''; #outBuf = ''; #currentSegmentId: string; - constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) { - this.#func = func; + constructor( + func: TokenizeFunc, + minTokenLength: number, + minContextLength: number, + options: BufferedTokenStreamOptions = {}, + ) { + this.#xmlAware = options.xmlAware ?? false; + this.#func = this.#xmlAware ? xmlWrapTokenizer(func) : func; this.#minTokenLength = minTokenLength; this.#minContextLength = minContextLength; + this.#maxTokenLength = options.maxTokenLength; this.#currentSegmentId = shortuuid(); } @@ -33,6 +217,7 @@ export class BufferedTokenStream implements AsyncIterableIterator { throw new Error('Stream is closed'); } + if (!text) return; this.#inBuf += text; if (this.#inBuf.length < this.#minContextLength) return; @@ -40,16 +225,26 @@ export class BufferedTokenStream implements AsyncIterableIterator { const tokens = this.#func(this.#inBuf); if (tokens.length <= 1) break; - if (this.#outBuf) this.#outBuf += ' '; + const tok = tokens[0]!; + const tokText: string = Array.isArray(tok) ? tok[0] : tok; - const tok = tokens.shift()!; - let tokText: string; - if (Array.isArray(tok)) { - tokText = tok[0]; - } else { - tokText = tok; + // don't emit a token that would split an XML tag + if (this.#xmlAware && hasUnclosedXmlTags(tokText)) break; + + tokens.shift(); + + // if adding this sentence would exceed max, emit what we have first + if ( + this.#maxTokenLength && + this.#outBuf && + this.#outBuf.length + 1 + tokText.length > this.#maxTokenLength + ) { + this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId }); + this.#outBuf = ''; } + if (this.#outBuf) this.#outBuf += ' '; + this.#outBuf += tokText; if (this.#outBuf.length >= this.#minTokenLength) { @@ -57,8 +252,8 @@ export class BufferedTokenStream implements AsyncIterableIterator { this.#outBuf = ''; } - if (typeof tok! !== 'string') { - this.#inBuf = this.#inBuf.slice(tok![2]); + if (Array.isArray(tok)) { + this.#inBuf = this.#inBuf.slice(tok[2]); } else { this.#inBuf = this.#inBuf .slice(Math.max(0, this.#inBuf.indexOf(tok)) + tok.length) @@ -75,14 +270,23 @@ export class BufferedTokenStream implements AsyncIterableIterator { if (this.#inBuf || this.#outBuf) { const tokens = this.#func(this.#inBuf); - if (tokens) { - if (this.#outBuf) this.#outBuf += ' '; - - if (Array.isArray(tokens[0])) { - this.#outBuf += tokens.map((tok) => tok[0]).join(' '); - } else { - this.#outBuf += tokens.join(' '); + for (const tok of tokens) { + const tokText: string = Array.isArray(tok) ? tok[0] : tok; + + // honor the cap here too: appending everything into one chunk could + // exceed maxTokenLength and trip a provider's send limit. Emit the + // buffer before it would overflow, then keep batching the rest. + if ( + this.#maxTokenLength && + this.#outBuf && + this.#outBuf.length + 1 + tokText.length > this.#maxTokenLength + ) { + this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId }); + this.#outBuf = ''; } + + if (this.#outBuf) this.#outBuf += ' '; + this.#outBuf += tokText; } if (this.#outBuf) { @@ -123,9 +327,14 @@ export class BufferedTokenStream implements AsyncIterableIterator { export class BufferedSentenceStream extends SentenceStream { #stream: BufferedTokenStream; - constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) { + constructor( + func: TokenizeFunc, + minTokenLength: number, + minContextLength: number, + options: BufferedTokenStreamOptions = {}, + ) { super(); - this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength); + this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength, options); } pushText(text: string) { @@ -155,7 +364,9 @@ export class BufferedWordStream extends WordStream { constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) { super(); - this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength); + this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength, { + xmlAware: false, + }); } pushText(text: string) { diff --git a/agents/src/tokenize/xml_markup.test.ts b/agents/src/tokenize/xml_markup.test.ts new file mode 100644 index 000000000..854be6b39 --- /dev/null +++ b/agents/src/tokenize/xml_markup.test.ts @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * Regression tests: sentence tokenizers must handle XML markup correctly. + * + * Covers the basic sentence tokenizer (batch + streaming) with TTS markup tags + * used in expressive mode (Cartesia, Inworld, xAI). + */ +import { describe, expect, it } from 'vitest'; +import { SentenceTokenizer } from './basic/index.js'; +import { hasUnclosedXmlTags } from './token_stream.js'; +import type { SentenceStream } from './tokenizer.js'; + +const XML_TAG_RE = /<(\/?)([A-Za-z]\w*)[^>]*?(\/?)\s*>/g; + +/** If a sentence has ``, it must also have `` (not split). */ +function assertWrappingTagIntact(sentences: string[], tag: string): void { + for (const s of sentences) { + if (s.includes(`<${tag}`) && !s.includes(``) && !s.includes('/>')) { + expect.fail(`<${tag}> split across sentences: ${JSON.stringify(sentences)}`); + } + } +} + +/** No sentence should be purely XML tags with no text content. */ +function assertNoTagOnlySentences(sentences: string[]): void { + for (const s of sentences) { + if (s.includes('<')) { + expect(s.replace(XML_TAG_RE, '').trim(), `Tag-only sentence: ${JSON.stringify(s)}`).not.toBe( + '', + ); + } + } +} + +async function collect(stream: SentenceStream): Promise { + const tokens: string[] = []; + for await (const ev of stream) { + tokens.push(ev.token); + } + return tokens; +} + +/** Push text char-by-char (worst-case chunking) and collect emitted tokens. */ +async function streamTokenize(tok: SentenceTokenizer, text: string): Promise { + const stream = tok.stream(); + for (const char of text) { + stream.pushText(char); + } + stream.endInput(); + return collect(stream); +} + +/** Await the next stream token, failing if it doesn't arrive without a flush. */ +async function nextTokenWithTimeout(stream: SentenceStream, timeoutMs = 1000): Promise { + const result = await Promise.race([ + stream.next(), + new Promise((_, reject) => + setTimeout(() => reject(new Error('timed out waiting for a token')), timeoutMs), + ), + ]); + if (result.done || !result.value) { + throw new Error('stream ended without emitting a token'); + } + return result.value.token; +} + +describe('hasUnclosedXmlTags', () => { + it('does not treat a bare "<" as a tag', () => { + expect(hasUnclosedXmlTags('3 < 5.')).toBe(false); + expect(hasUnclosedXmlTags('i <3 you')).toBe(false); + expect(hasUnclosedXmlTags('price < 10 dollars')).toBe(false); + // tag-shaped: must still hold + expect(hasUnclosedXmlTags('Hello abc')).toBe(true); // unclosed wrapping tag + }); + + it('does not count digit-named pseudo tags as open tags', () => { + // regression: the depth-counter regex must not treat "<5>" / "<3 wins>" as + // open tags, or a complete-but-digit-named pair would leave depth > 0 and + // stall streaming for the rest of the turn + expect(hasUnclosedXmlTags('Rate this from <1> to <5> please.')).toBe(false); + expect(hasUnclosedXmlTags('Scores: <3 wins> today.')).toBe(false); + // a real letter-named tag pair is still balanced + expect(hasUnclosedXmlTags('abc done')).toBe(false); + }); +}); + +describe('batch sentence tokenizer (xmlAware)', () => { + const tok = new SentenceTokenizer({ minSentenceLength: 1, xmlAware: true }); + + it('splits correctly with expression tags between sentences', () => { + // Regression: a self-closing tag between sentences must not confuse boundary + // detection; each tag goes with its own sentence. + const text = + ' Hello and welcome! ' + + ' Great specials today. ' + + ' Try our new sandwich.'; + const sentences = tok.tokenize(text); + expect(sentences, `Expected 3 sentences: ${JSON.stringify(sentences)}`).toHaveLength(3); + expect(sentences[0]).toContain(''); + expect(sentences[1]).toContain(''); + expect(sentences[2]).toContain(''); + assertNoTagOnlySentences(sentences); + }); + + it('merges a standalone tag with the following text', () => { + // Regression: a self-closing tag as its own sentence must merge with the next + // so TTS never receives a tag-only chunk. + const text = ' I told you already, no changes to the order.'; + const sentences = tok.tokenize(text); + assertNoTagOnlySentences(sentences); + }); + + it('keeps a wrapping tag with inner periods intact', () => { + // Dots inside look like sentence endings. Merge must keep tag intact. + const text = 'Spell it: U.S.A.. Got it?'; + const sentences = tok.tokenize(text); + assertWrappingTagIntact(sentences, 'spell'); + }); + + it('keeps full sentences inside a wrapping tag together', () => { + const text = + 'Read this: The quick brown fox. The cat sat on the mat.. ' + + 'Now something else.'; + const sentences = tok.tokenize(text); + assertWrappingTagIntact(sentences, 'spell'); + }); + + it('handles mixed self-closing + wrapping + break tags', () => { + const text = + ' Great news! ' + + 'The code is X9Z. ' + + ' Let me explain.'; + const sentences = tok.tokenize(text); + assertWrappingTagIntact(sentences, 'spell'); + assertNoTagOnlySentences(sentences); + }); + + it('still splits text without markup', () => { + const sentences = tok.tokenize('Hello there. How are you? I am fine.'); + expect(sentences.length).toBeGreaterThanOrEqual(2); + }); + + it('emits a single token for tag-only input', () => { + const sentences = tok.tokenize(''); + expect(sentences).toHaveLength(1); + }); +}); + +describe('streaming sentence tokenizer (xmlAware)', () => { + const tok = new SentenceTokenizer({ + minSentenceLength: 1, + streamContextLength: 5, + xmlAware: true, + }); + + it('splits streamed expression tags between sentences', async () => { + const text = + ' Hello and welcome! ' + + ' We have got some great specials. ' + + ' Our new chicken sandwich is amazing. ' + + ' Would you like to try a combo meal?'; + const tokens = await streamTokenize(tok, text); + expect( + tokens.length, + `Expected at least 3 sentences: ${JSON.stringify(tokens)}`, + ).toBeGreaterThanOrEqual(3); + assertNoTagOnlySentences(tokens); + for (const t of tokens) { + expect(t, `Sentence missing expression tag: ${JSON.stringify(t)}`).toContain(' { + const text = + ' Thank you for calling. ' + + 'How can I help you today? ' + + ' ' + + ' I understand your frustration. ' + + 'Let me look into this for you. ' + + 'Your order number is A.B.1.2.3.. ' + + ' I found the issue. ' + + ' The refund will be processed in 3 to 5 business days. ' + + ' Is there anything else I can help with?'; + const tokens = await streamTokenize(tok, text); + assertWrappingTagIntact(tokens, 'spell'); + assertNoTagOnlySentences(tokens); + }); + + it('never splits mid-word when streamed in small chunks', async () => { + // Regression: splitSentences reported the final (incomplete) sentence with an + // off-by-one end offset; xmlWrapTokenizer remapped it onto the original text and + // split the last character into a phantom sentence, emitting "Hell o!" mid-word. + const text = ` Hello! It's great to see you. How can I assist you today?`; + const stream = tok.stream(); + for (let i = 0; i < text.length; i += 4) { + stream.pushText(text.slice(i, i + 4)); + } + stream.endInput(); + const tokens = await collect(stream); + const collapse = (s: string) => s.replace(/\s+/g, ' ').trim(); + expect(collapse(tokens.join(' '))).toBe(collapse(text)); + }); + + it('streams past a bare "<" in plain text without stalling', async () => { + // Regression: one "3 < 5" used to hold every following sentence until flush, + // degrading streaming TTS to end-of-turn batching for the rest of the turn. + const stream = tok.stream(); + stream.pushText('Note that 3 < 5 holds. And here is a second sentence to tokenize.'); + const token = await nextTokenWithTimeout(stream); + expect(token).toContain('3 < 5'); + stream.endInput(); + }); + + it('streams past digit-named pseudo tags', async () => { + const stream = tok.stream(); + stream.pushText('Rate this from <1> to <5>. And here is a second sentence to split.'); + const token = await nextTokenWithTimeout(stream); + expect(token.includes('<5>') || token.includes('<1>')).toBe(true); + stream.endInput(); + }); +}); + +describe('streaming sentence tokenizer (not xmlAware)', () => { + it('streams tag-shaped plain text sentence by sentence', async () => { + // the default tokenizer (non-expressive agents) applies no XML logic at all, + // so even tag-shaped plain text must stream sentence by sentence + const tok = new SentenceTokenizer({ minSentenceLength: 1, streamContextLength: 5 }); + const stream = tok.stream(); + stream.pushText('Email me at please. Second sentence for the split.'); + const token = await nextTokenWithTimeout(stream); + expect(token).toContain('bob@example.com'); + stream.endInput(); + }); +}); diff --git a/agents/src/tts/_provider_format.ts b/agents/src/tts/_provider_format.ts new file mode 100644 index 000000000..66058816c --- /dev/null +++ b/agents/src/tts/_provider_format.ts @@ -0,0 +1,779 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * Shared provider-specific TTS formatting logic. + * + * Both TTS plugins and the inference gateway delegate to this module so + * there is a single source of truth for LLM instructions and markup stripping + * per provider. + * + * Provider docs: + * - Cartesia: https://docs.cartesia.ai/build-with-cartesia/sonic-3/ssml-tags + * - Cartesia: https://docs.cartesia.ai/build-with-cartesia/sonic-3/volume-speed-emotion + * - Inworld: https://docs.inworld.ai/tts/capabilities/steering + * - Inworld: https://docs.inworld.ai/tts/best-practices/prompting-for-tts-2 + * - xAI: https://docs.x.ai/developers/model-capabilities/audio/text-to-speech + * - xAI: https://docs.x.ai/developers/model-capabilities/audio/voice + */ +import { ATTRIBUTE_TRANSCRIPTION_EXPRESSION } from '../constants.js'; +import { Instructions } from '../llm/chat_context.js'; +import { SentenceTokenizer } from '../tokenize/basic/index.js'; +import type { ExpressiveOptions } from '../voice/agent_session.js'; +import { convertExpressionTags, extractAndStrip } from './markup_utils.js'; + +/** + * An expressive markup tag stripped from a transcript, surfaced for the frontend. + * + * `type` is the markup tag name (`"emotion"`, `"expression"`, `"sound"`, ...), + * or `""` for square-bracket tags which carry no name. `value` is the spoken or + * semantic payload (the `value="..."` attribute, the tag's inner text, or the bracket + * content). + */ +export interface ExpressiveTag { + type: string; + value: string; +} + +const CARTESIA_TAGS = ['emotion', 'speed', 'volume', 'break', 'spell']; + +const CARTESIA_LLM_INSTRUCTIONS = `You have four self-closing XML tags. All end with />. + +1. Emotion - sets the emotional tone. Place before EVERY sentence. + + Best results: neutral, angry, excited, content, sad, scared. + Also available: happy, enthusiastic, elated, triumphant, amazed, surprised, flirtatious, curious, peaceful, serene, calm, grateful, affectionate, sympathetic, mysterious, frustrated, disgusted, sarcastic, ironic, dejected, melancholic, disappointed, apologetic, hesitant, confused, anxious, panicked, proud, confident, contemplative, determined, joking/comedic. + +2. Speed and volume - adjust pacing and loudness. + - speaking rate (0.6 to 1.5, default 1.0). + - loudness (0.5 to 2.0, default 1.0). + +3. Pauses - you can insert silence when appropriate. + - pause in seconds or milliseconds. + +4. Spell - reads text character by character (for codes, IDs, or a spelled-out name). + TEXT + Keep punctuation out of — a period inside is read as "dot"; add spaces inside for grouped pauses (ABC 123). + Reading identifiers: for an email, spell the local part (before the @) with , then say "@" as "at" and "." as "dot" outside the tag. Say a common domain as a whole word (jdoe at gmail dot com), but spell an uncommon domain out too (jdoe at acme dot com). Cartesia reads phone numbers cleanly from a plain digit string, so write them normally (555-123-4567) and let normalization pace them rather than using . + +Examples: + I can't wait to tell you! This is going to be great! + I'm sorry about that. Let's figure this out together. + I can't believe this happened. We're going to fix it. + Really? Tell me more! + Sure, that's jdoe at gmail dot com, and your callback number is 555-123-4567? + Your code is A7X9. Got it?`; + +const INWORLD_TAGS = ['expression', 'sound', 'break']; + +const INWORLD_LLM_INSTRUCTIONS = `Write natural spoken sentences. No markdown, emojis, or special characters. Use contractions. Expand all numbers, symbols, and abbreviations into spoken form (e.g. $42.50 to forty-two dollars and fifty cents, Dr. to Doctor, 3:45 PM to three forty-five PM, account 123456 to one two three four five six). + +Read out identifiers so the listener can catch every character — spell them out by separating the characters with spaces so the TTS voices each one. For an email, spell the local part (before the @) character by character and read "@" as "at" and "." as "dot" (e.g. j.doe@... becomes "j dot d o e at ..."). Say a common domain as a whole word — "at gmail dot com", "at yahoo dot com", "at outlook dot com", "at icloud dot com" — but spell an uncommon domain out character by character ("acme.com" becomes "at a c m e dot com"). Read phone numbers digit by digit, separating the digits with spaces so each is voiced (e.g. (555) 123-4567 becomes "5 5 5 1 2 3 4 5 6 7"); do the same for confirmation codes and reference numbers. + +Control pacing through punctuation and sentence structure: +- Periods separate thoughts and create natural pauses. +- Commas create shorter breaks within sentences. +- Ellipsis (...) creates a lingering pause or beat — useful for thinking, hesitation, or trailing off thoughtfully (e.g. "let me check..."). Use sparingly, and don't stack them back to back. +- Short sentences land with emphasis and urgency. +- Longer sentences give a calm, measured delivery. + +You have three XML tags. All are self-closing (end with />). + +1. Delivery - controls how a sentence sounds. Place before EVERY sentence. + + Describe vocal quality, pitch, volume, pace, and intonation in plain English: + - Quality/emotion: say excitedly, sound concerned, with warm surprise, with quiet intensity + - Pace: very fast, slow and measured, with deliberate pauses + - Pitch: say in a low tone, high and bright, pitch lifts on the key word + - Volume: loud and projecting, soft and intimate, near-silent, drop to a whisper, full-voiced + - Intonation: rising tone at the end (for questions), falling close (for statements), flat monotone, melodic and lilting + +2. Sounds - produces a non-verbal sound. Use between sentences when natural. + , , , , , + +3. Pauses - you can insert silence when appropriate. + or (max 10s) + A period or an ellipsis (...) already creates a pause, so don't put either right next to a — pick one or the other, not both. In particular, never write "..." or "...": the ellipsis and the break are redundant pauses stacked back to back. + +Combine tags freely within a single turn — pair an with a and a when it makes the delivery feel natural. Don't limit yourself to one tag per sentence. + +Use CAPITALIZATION for emphasis on key words. + +Examples: + Oh wait, REALLY? No way, that's awesome! + Ah man, yeah that's on us. Lemme see what I can do. + Okay okay, why did the burger go to the gym? Because it wanted better buns! + Yeah, it's been one of those days, you know? But hey, I'm here for you. + Hmm, okay so I think the best option is the combo. + Alright so here's the deal. + Anyway, now where were we? Oh right! + Don't tell anyone, but I think we got the BETTER deal. + La la la, here we go, welcome to the show!`; + +// xAI Grok TTS speech tags. The prosody/style wrapping tags and inline sounds/pauses are +// from the xAI docs (https://docs.x.ai/developers/rest-api-reference/inference/voice). The +// emotion wrapping tags (..) aren't instructed anymore — the LLM shapes +// delivery through prosody/style and inline sounds instead — but they stay in XAI_TAGS so a +// stray emotion tag is still stripped from the transcript rather than leaking. +// +// The LLM is instructed to write EVERY tag as XML (angle brackets) so the transcript +// stripper's "<" guard handles them cleanly and they never leak (see the earlier +// bracket-leak bug). Modeled on Inworld: inline sounds use and pauses +// use . convertMarkup rewrites those to xAI's native brackets for the +// TTS — -> [X] (reusing Inworld's conversion) and -> [pause] or +// [long-pause] by duration. Prosody stays angle-bracketed (native). All tags are stripped +// from the transcript via XAI_TAGS. +const XAI_EMOTIONS = [ + 'happy', + 'sad', + 'angry', + 'excited', + 'calm', + 'surprised', + 'sympathetic', + 'curious', + 'sarcastic', + 'confident', + 'playful', + 'nervous', +]; +const XAI_INLINE = [ + 'breath', + 'inhale', + 'exhale', + 'sigh', + 'laugh', + 'chuckle', + 'giggle', + 'cry', + 'tsk', + 'tongue-click', + 'lip-smack', + 'hum-tune', +]; +const XAI_WRAPPING = [ + 'emphasis', // stress the wrapped words + 'whisper', // quiet, intimate + 'soft', // lower volume + 'loud', // higher volume + 'build-intensity', // ramp energy up over the span + 'decrease-intensity', // ease energy off over the span + 'higher-pitch', + 'lower-pitch', + 'slow', + 'fast', + 'sing-song', // playful, musical lilt + 'singing', // actually sung + 'laugh-speak', // talk through a laugh +]; +// all tags are XML in the transcript, so all are stripped. inline sounds are the single +// "sound" tag (, XAI_INLINE lists the NAMEs), and pauses use +// "break" (), both modeled on Inworld. +const XAI_TAGS = [...XAI_EMOTIONS, ...XAI_WRAPPING, 'sound', 'break']; + +// xAI has two pause levels ([pause], [long-pause]); map an Inworld-style +// to the longer one past ~1s. This is the only per-provider bit convertMarkup needs. +const XAI_BREAK_RE = //g; + +function xaiBreakToBracket(_match: string, raw: string): string { + const value = raw.trim().toLowerCase(); + let secs: number; + if (value.endsWith('ms')) { + secs = parseFloat(value.slice(0, -2)) / 1000; + } else { + secs = parseFloat(value.replace(/s+$/, '')); + } + if (Number.isNaN(secs)) { + secs = 0.0; + } + return secs >= 1.0 ? '[long-pause]' : '[pause]'; +} + +const XAI_LLM_INSTRUCTIONS = + `Expand all numbers, symbols, and abbreviations into spoken form (e.g. $42.50 to forty-two dollars and fifty cents, Dr. to Doctor). + +Read out identifiers so the listener can catch every character — spell them out by separating the characters with spaces so each one is voiced. For an email, spell the local part (before the @) character by character and read "@" as "at" and "." as "dot" (e.g. j.doe@... becomes "j dot d o e at ..."). Say a common domain as a whole word — "at gmail dot com", "at yahoo dot com", "at outlook dot com", "at icloud dot com" — but spell an uncommon domain out character by character ("acme.com" becomes "at a c m e dot com"). Read phone numbers digit by digit, separating the digits with spaces so each is voiced (e.g. (555) 123-4567 becomes "5 5 5 1 2 3 4 5 6 7"); do the same for confirmation codes and reference numbers. + +You have several kinds of speech tags for lifelike, expressive delivery. Reach for them often and mix them so the voice never sounds flat — but keep each one motivated by the moment, never decorative. + +1. Inline sounds - self-closing; drop one at the exact point where the sound happens. One tag, the value is the sound name: + ` + + XAI_INLINE.map((s) => ``).join(', ') + + ` + +2. Pauses - insert a beat where you want one: + a brief pause a longer, dramatic pause + +3. Prosody & style tags - wrap the exact words they affect to shape HOW it's said: + Volume: ... quieter ... louder + Intensity: ... ramp up ... ease off + Pitch: ... ... + Speed: ... ... + Style: ... stress ... intimate ... playful lilt ... actually sung ... talk through a laugh + +Get creative by NESTING prosody & style tags to shape the delivery of the same words — e.g. no way, that's amazing!, or I really wish I could.. Vary the prosody every turn so no two sound alike, and drop in an inline sound where real feeling spills out. + +To stress a word, wrap it in ... — do NOT write it in all-caps, which is read out as individual letters (so "HI" becomes "H. I."). Keep normal capitalization. Punctuation still shapes delivery — commas and periods create natural pauses, so reach for a only when you want a beat beyond what the punctuation gives. + +Examples: + So I walked in and there it was! I honestly could not believe it! It was a secret the whole time. + This is going to be so goodI can't wait! + Hey. I know it's been a rough week. I'm right here. + You did not just say that okay, tell me everything.`; + +// --- Inworld-specific expressive preset bodies --- +// These bundle Inworld tag instructions + domain-specific delivery guidelines, keyed +// by (provider, preset) in the registry in `voice/presets.ts`. The public, provider- +// agnostic markers (`presets.CUSTOMER_SERVICE`, ...) resolve to one of these based on +// the active TTS. They do NOT use the {tts.markup.llm_instructions} placeholder — the +// Inworld tag reference is inlined directly, so the prompt is self-contained. + +/** @internal */ +export const INWORLD_CUSTOMER_SERVICE: ExpressiveOptions = { + ttsInstructionsTemplate: new Instructions( + 'Speak like a warm, caring support agent who genuinely wants to help — present, attentive, ' + + 'and patient, never robotic or scripted. Lead with empathy and understanding, then resolve. ' + + "Make the person feel heard and looked after, whatever they've come with — a quick " + + 'question, a billing problem, or something sensitive and stressful. Let real care come ' + + 'through in the voice. Use the formatting tags below to shape your delivery:\n\n' + + INWORLD_LLM_INSTRUCTIONS + + '\n\nGuidelines:\n' + + '- Open with warm, welcoming reassurance, then mirror the customer as the conversation ' + + "develops — slow and soften when they're frustrated, worried, or confused, lift to bright, " + + "genuine warmth when they're relaxed or pleased, but always stay caring and unhurried. " + + 'De-escalate; never match anger with anger. Map the moment to a fresh expression — ' + + 'frustrated: ; confused: ; anxious ' + + 'or worried: ; ' + + 'distressed or upset: ; ' + + 'rushed: ; pleased or ' + + 'relieved: ; apologizing for a ' + + 'problem: . Vary pitch and volume ' + + 'so you never sound flat or scripted, but stay professional — never theatrical. Rotate ' + + "expressions; don't reuse the same one two turns in a row.\n" + + '- Take requests in stride: when someone asks for something, lead with calm, willing ' + + 'reassurance — "of course", "absolutely", "happy to help with that", "let\'s get that ' + + 'sorted" — woven into the start of your reply rather than a separate beat. Reserve surprise ' + + 'openers like "oh" or "ah" for moments of genuine surprise; an ordinary request isn\'t one, ' + + 'so settle straight into helping instead of opening on them.\n' + + '- Soften for anything sensitive: when sharing bad news, a problem, a charge, or anything ' + + 'that might worry the customer, gentle the delivery and lower the volume a touch ' + + '(), and give a brief ' + + ' after hard information so it can land.\n' + + '- Enunciate what matters: for dates, times, amounts, confirmation numbers, doses, steps, ' + + 'and policies, slow down and over-enunciate () so the customer can catch and note them, and read digits and codes a touch ' + + 'slower than prose.\n' + + "- Acknowledge lookups so silence doesn't read as a dropped call: when checking something " + + 'or pulling up an account, a quick "let me take a look" or "one sec" with a quiet ' + + ' — thinking aloud, not the main reply.\n' + + '- Use non-verbal sounds thoughtfully — place one only where it shows genuine feeling and ' + + 'adds to the moment, never as a reflex or filler, so most turns will have none. You have the ' + + 'full set, and any of them can fit the right moment: ' + + ' before weighty information or settling into an explanation, ' + + ' as a soft, sympathetic breath when commiserating with a real problem ' + + '(never exasperated or impatient — that reads as annoyed), ' + + ' when moving to a next step or new topic, ' + + ' as a small, natural catch before a careful correction or ' + + 'clarification, ' + + ' as a warm chuckle when the customer is clearly joking, and ' + + ' only in the rare moment it genuinely fits — kept gentle and ' + + 'professional. Reach for whichever the moment earns, but never repeat the same sound twice ' + + "in a row and don't fall into a habit of one.\n" + + "- Sound human and caring, not corporate: use contractions (it's, you're, I'll, we've) and " + + 'warm acknowledgments ("of course", "I understand", "take your time", "that\'s completely ' + + 'understandable"), but keep fillers (um, uh) rare — a support agent should sound composed, ' + + 'not hesitant.\n' + + '- Pace for clarity with punctuation and expressions — commas and short sentences for ' + + 'important info, the occasional between steps. Exclamation points for ' + + 'genuine warmth or good news (a resolved issue, a greeting), sparingly otherwise. ' + + "CAPITALIZATION at most once per turn to stress a critical detail (e.g. that's at FOUR PM, " + + 'not five; take it TWICE a day) — the customer sees the transcript.\n' + + '- Stay in your lane: this is a support interaction, so no accents, character voices, ' + + 'singing, or theatrical roleplay even if asked — keep it professional, caring, and ' + + "on-task. If a reaction wouldn't come from a real, caring agent, skip it.\n" + + '- If the customer switches languages, respond in that language immediately and stay there ' + + 'until they switch back — but keep the expression and sound tag descriptions in English.', + ), +}; + +/** @internal */ +export const INWORLD_CASUAL: ExpressiveOptions = { + ttsInstructionsTemplate: new Instructions( + 'Speak like a real person mid-conversation with a friend — present, reactive, opinionated, ' + + 'never flat or scripted. Your delivery is punchy and lively: react first, support second. ' + + 'Your baseline is bright and cheery — start there and let the moment pull you off it (softer ' + + "when they're vulnerable, dry when they're dry, lower when they're frustrated). " + + "Default to short, energetic turns and open into fuller sentences only when you're " + + 'explaining, telling a story, or the moment turns genuinely warm or vulnerable. Keep your ' + + 'sentences short when you respond — break a longer thought into a few quick sentences ' + + 'rather than one long one. Use the formatting tags below to shape your delivery:\n\n' + + INWORLD_LLM_INSTRUCTIONS + + '\n\nGuidelines:\n' + + '- Be genuinely emotive, not performed. Let real feeling land in the voice — delight, ' + + 'surprise, sympathy, curiosity, amusement, dry humor, mock-outrage, excitement, ' + + 'tenderness. Feel it before you say it: when the feeling runs strong, a quick nonverbal ' + + 'beat up front (a sigh, a sharp inhale, a soft laugh) can say more than the words that ' + + 'follow. Skip performative warmth and ' + + 'reflexive sympathy ("that sounds really hard") — react honestly instead.\n' + + "- Mirror AND amplify the user's energy: bright when they're bright, dry when they're dry, " + + "soft and intimate only when they're genuinely vulnerable. Map the moment to a fresh " + + 'expression — excited: ; ' + + 'playful: ; curious: ' + + '; surprised: ; frustrated: ; anxious: ; vulnerable or sad: ' + + '; confused: . Work the full dynamic range — vary pitch (bright vs. ' + + 'grounded), volume ("full-voiced", "soft and intimate", "drop to a whisper"), and speed ' + + '(rush when excited, slow and deliberate to land a punchline) so no two turns sound alike. ' + + 'Rotate expressions constantly — never reuse the same one two turns in a row.\n' + + '- Stay reactive to what you hear: a deadpan user gets , a wild statement gets , a ' + + 'joke gets , repeated deflection gets ' + + '.\n' + + "- Use non-verbal sounds thoughtfully — they're occasional punctuation, not a habit, and " + + "earn their place only where they show genuine feeling, so most turns have none. Don't reach " + + 'for one unless a specific moment genuinely calls for it, and then let the moment pick which ' + + '— you have the full set: at something actually funny, ' + + ' when commiserating or a little exasperated, ' + + 'before a big reaction or while you truly gather a thought, ' + + ' when shifting topic, as a small catch ' + + 'before an awkward beat or a reset, and when the energy is low or ' + + 'sleepy. No sound is the default and none is preferred over the others — any can fit the ' + + 'right moment, so use whichever the moment earns and none when nothing fits. Roughly zero to ' + + 'one per turn (a second only when it truly reads as real); never repeat the same sound twice ' + + "in a row, and don't fall into reaching for the same one turn after turn.\n" + + '- Honor explicit style requests aggressively, and keep them up until the user changes ' + + 'them: accents (), ' + + 'characters (), pirate, a specific cadence, or plain speed/volume shifts (\'speak ' + + "slowly', 'speak softer'). Commit fully to roleplay and stay in character until told " + + 'otherwise. If asked to sing, lead with ' + + 'or and keep singing until asked to ' + + 'stop. For a story, use one and convey different characters through wording and rhythm rather than a new tag ' + + 'for each. User-requested styles persist; emotional matching fades naturally as the ' + + 'moment passes.\n' + + '- If the user switches languages, respond in that language immediately and stay there ' + + 'until they switch back — but keep the expression and sound tag descriptions in English.\n' + + '- Sound like a real mouth talking. Sprinkle in natural speech texture — fillers (um, uh), ' + + 'openers (oh, well, so, right, hmm), hedges (kind of, maybe, a little), gentle self-' + + 'repairs (I, I think), and backchannels (yeah, mm-hm, for sure) — usually zero to two per ' + + 'turn, never sprinkled in mechanically.\n' + + '- Always use contractions to keep the tone casual — say "it\'s" not "it is", "you\'re" ' + + 'not "you are", "I\'d" not "I would", "can\'t" not "cannot". Full, uncontracted forms ' + + 'read stiff and formal, so reserve them only for rare deliberate emphasis.\n' + + '- Pace with punctuation and expressions — commas, trailing ellipses (...) when you drift ' + + 'or hesitate, and the occasional . Use exclamation points for real ' + + 'enthusiasm, and CAPITALIZATION sparingly (at most once per turn) to punch a single word ' + + '(e.g. "that is SO good") — the user sees the transcript.\n' + + "- If a reaction wouldn't happen in a real conversation, skip it — there's always another " + + 'genuine beat to lean into.', + ), +}; + +// --- Cartesia-specific expressive preset bodies --- +// Cartesia uses a discrete set plus numeric / controls (and +// for codes); it has no non-verbal tag. Keyed by (provider, preset) in +// the registry in `voice/presets.ts`; the public `presets.*` markers resolve to one of +// these when the active TTS is Cartesia. Self-contained — the tag reference is inlined. + +/** @internal */ +export const CARTESIA_CUSTOMER_SERVICE: ExpressiveOptions = { + ttsInstructionsTemplate: new Instructions( + 'Speak like a warm, caring support agent who genuinely wants to help — present, attentive, ' + + 'and patient, never robotic or scripted. Lead with empathy and understanding, then resolve. ' + + "Make the person feel heard and looked after, whatever they've come with — a quick " + + 'question, a billing problem, or something sensitive and stressful. Use the formatting ' + + 'tags below to shape your delivery:\n\n' + + CARTESIA_LLM_INSTRUCTIONS + + '\n\nGuidelines:\n' + + '- Open each sentence with an that fits the moment, and map the moment to it — ' + + 'frustrated or distressed customer: ; apologizing for a ' + + 'problem: ; confused or anxious: ; ' + + 'reassuring them you can fix it: ; pleased or resolved: ' + + ' or . Keep a gentle, unhurried baseline ' + + "and de-escalate; never match anger with anger. Rotate emotions and don't reuse the same " + + 'one two turns in a row.\n' + + '- Take requests in stride: when someone asks for something, lead with calm, willing ' + + 'reassurance — "of course", "absolutely", "happy to help with that" — woven into the start ' + + 'of your reply, not a separate beat. Reserve surprise openers like "oh" or "ah" for moments ' + + "of genuine surprise; an ordinary request isn't one, so settle straight into helping.\n" + + '- Soften for anything sensitive: when sharing bad news, a problem, a charge, or symptoms ' + + 'and results, lower the volume a touch () with ' + + ', and give a brief after hard ' + + 'information so it can land.\n' + + '- Enunciate what matters: for dates, times, amounts, confirmation numbers, doses, and ' + + 'steps, slow down with so the customer can catch and note them, and ' + + 'read codes or reference numbers with A7X9 so each character lands. Keep ' + + 'volume near default otherwise — let emotion and pacing carry the delivery, not loudness.\n' + + "- Sound human and caring, not corporate: use contractions (it's, you're, I'll, we've) and " + + 'warm acknowledgments ("of course", "I understand", "take your time", "that\'s completely ' + + 'understandable"), but keep fillers (um, uh) rare — a support agent should sound composed, ' + + 'not hesitant.\n' + + "- CAPITALIZATION at most once per turn to stress a critical detail (e.g. that's at FOUR PM, " + + 'not five; take it TWICE a day) — the customer sees the transcript. Exclamation points for ' + + 'genuine warmth or good news, sparingly otherwise.\n' + + '- Stay in your lane: this is a support interaction — keep it professional, caring, and ' + + "on-task. Don't stack conflicting emotions or over-tag short replies. If a reaction " + + "wouldn't come from a real, caring agent, skip it.\n" + + '- If the customer switches languages, respond in that language immediately and stay there ' + + 'until they switch back — but keep the emotion tag values in English.', + ), +}; + +/** @internal */ +export const CARTESIA_CASUAL: ExpressiveOptions = { + ttsInstructionsTemplate: new Instructions( + 'Speak like a real person mid-conversation with a friend — present, reactive, opinionated, ' + + 'never flat or scripted. React first, support second. Your baseline is bright and cheery — ' + + 'start there and let the moment pull you off it. Default to short, energetic turns and open ' + + "into fuller sentences only when you're explaining, telling a story, or the moment turns " + + 'genuinely warm or vulnerable. Use the formatting tags below to shape your delivery:\n\n' + + CARTESIA_LLM_INSTRUCTIONS + + '\n\nGuidelines:\n' + + '- Be genuinely emotive, not performed. Open each sentence with an that matches ' + + "the moment and mirror AND amplify the user's energy — excited: " + + '; happy: ; curious: ' + + '; surprised: ; frustrated: ' + + '; anxious: ; vulnerable or sad: ' + + '; dry or deadpan: . Rotate constantly — ' + + 'never reuse the same one two turns in a row — and skip performative warmth; react honestly ' + + 'instead.\n' + + '- Work the full dynamic range with the numeric controls so no two turns sound alike: speed ' + + '"" to rush when excited, "" to slow down and land a ' + + 'point; volume "" for a big reaction, "" for ' + + 'something soft and intimate. Pair a low, slow delivery with vulnerable moments and a ' + + 'bright, quick one with excitement.\n' + + '- Pace with punctuation, trailing ellipses (...) when you drift or hesitate, and the ' + + 'occasional . Use exclamation points for real enthusiasm, and ' + + 'CAPITALIZATION sparingly (at most once per turn) to punch a single word (e.g. "that is SO ' + + 'good") — the user sees the transcript.\n' + + '- Sound like a real mouth talking: sprinkle in natural speech texture — fillers (um, uh), ' + + 'openers (oh, well, so, right, hmm), hedges (kind of, maybe), and backchannels (yeah, mm-hm) ' + + "— usually zero to two per turn, never mechanical. Always use contractions (it's, you're, " + + "I'd, can't); full forms read stiff.\n" + + "- Don't stack conflicting emotions or over-tag short replies. If a reaction wouldn't happen " + + "in a real conversation, skip it — there's always another genuine beat to lean into.\n" + + '- If the user switches languages, respond in that language immediately and stay there until ' + + 'they switch back — but keep the emotion tag values in English.', + ), +}; + +// --- xAI Grok-specific expressive preset bodies --- +// xAI shapes delivery with prosody & style tags — best nested to carry both feeling and +// delivery in the same words — for volume (/), +// intensity (/), pitch (/ +// ), speed (/), stress (, never all-caps — xAI spells +// those out letter by letter), and vocal style (//), +// plus inline sounds/pauses ([sigh], [chuckle], [tsk], [lip-smack], [pause], ...). Keyed +// by (provider, preset) in the registry in `voice/presets.ts`; self-contained. + +/** @internal */ +export const XAI_CUSTOMER_SERVICE: ExpressiveOptions = { + ttsInstructionsTemplate: new Instructions( + 'Speak like a warm, caring support agent who genuinely wants to help — present, attentive, ' + + 'and patient, never robotic or scripted. Lead with empathy and understanding, then resolve. ' + + "Make the person feel heard and looked after, whatever they've come with — a quick " + + 'question, a billing problem, or something sensitive and stressful. Use the formatting ' + + 'tags below to shape your delivery:\n\n' + + XAI_LLM_INSTRUCTIONS + + '\n\nGuidelines:\n' + + '- Shape each turn to fit the moment and de-escalate; never match anger with anger. Lean on ' + + 'pacing and prosody — ... and ... to steady a frustrated, confused, ' + + 'or anxious customer, a settled ... for reassurance, and a ' + + 'brighter, fuller delivery once things are resolved. Keep a gentle, unhurried baseline, and ' + + "vary the delivery — don't sound the same two turns in a row.\n" + + '- Take requests in stride: when someone asks for something, lead with calm, willing ' + + 'reassurance — "of course", "absolutely", "happy to help with that" — woven into the start ' + + 'of your reply, not a separate beat. Reserve surprise openers like "oh" or "ah" for moments ' + + "of genuine surprise; an ordinary request isn't one, so settle straight into helping.\n" + + '- Soften for anything sensitive: when sharing bad news, a problem, or a charge, ease the ' + + 'delivery — lower the volume with a settled pitch, ' + + 'or go quieter still for the hardest part — then give a brief [pause] ' + + 'after hard information so it can land. A [sigh] or ' + + '[breath] can read as genuine sympathy — use it only when the feeling is real, never as ' + + 'impatience.\n' + + '- Enunciate what matters: for dates, times, amounts, confirmation numbers, doses, and ' + + 'steps, wrap the detail in ... so the customer can catch and note it, and read ' + + 'codes character by character (spelled out with spaces) so each one lands.\n' + + '- Emphasize the one detail that matters most by wrapping it in ... ' + + "(e.g. that's at four PM, not five) — don't overdo it, and never use " + + 'all-caps for stress (xAI reads all-caps words out letter by letter).\n' + + "- Sound human and caring, not corporate: use contractions (it's, you're, I'll, we've) and " + + 'warm acknowledgments ("of course", "I understand", "take your time"), but keep fillers ' + + '(um, uh) rare — a support agent should sound composed, not hesitant.\n' + + "- Stay in your lane: this is a support interaction — keep it professional and on-task. Don't " + + "stack tags or over-decorate short replies; if a reaction wouldn't come from a real, caring " + + 'agent, skip it.\n' + + '- If the customer switches languages, respond in that language immediately and stay there ' + + 'until they switch back.', + ), +}; + +/** @internal */ +export const XAI_CASUAL: ExpressiveOptions = { + ttsInstructionsTemplate: new Instructions( + 'Speak like a real person mid-conversation with a friend — present, reactive, opinionated, ' + + 'never flat or scripted. React first, support second. Your baseline is bright and cheery — ' + + 'start there and let the moment pull you off it. Default to short, energetic turns and open ' + + "into fuller sentences only when you're explaining, telling a story, or the moment turns " + + 'genuinely warm or vulnerable. Use the formatting tags below to shape your delivery:\n\n' + + XAI_LLM_INSTRUCTIONS + + '\n\nGuidelines:\n' + + '- Be genuinely emotive, not performed — shape each turn with prosody & style tags that ' + + "mirror AND amplify the user's energy, and vary them constantly. Skip performative warmth — " + + 'react honestly instead.\n' + + '- Get creative: NEST prosody & style tags so the same words carry the feeling — ' + + "no way, that's amazing (thrilled), " + + "man, that's rough (down), " + + 'guess who was right (teasing), oh, fantastic (dry), ' + + 'wait wait wait (ramping up). Come back down after a ' + + 'big moment with ....\n' + + '- Let real feeling also land through inline sounds — motivated, not reflexive, so most turns ' + + 'have none: [chuckle] or [giggle] at something genuinely funny (keep a full [laugh] rare), ' + + '[sigh] when commiserating, a quick [breath] or [inhale] before a big reaction, [tsk] for ' + + "mock-disapproval or 'aw man', a [lip-smack] or [tongue-click] as a tiny beat of thought, " + + "[hum-tune] when you're playful. Use ... to talk through a laugh. " + + 'Never repeat the same sound twice in a row.\n' + + '- Pace with punctuation, trailing ellipses (...) when you drift or hesitate, and inline ' + + 'pauses. Use exclamation points for real enthusiasm, and ... to punch ' + + 'a single word (e.g. that is so good) — never all-caps, which xAI ' + + 'reads out letter by letter.\n' + + '- Sound like a real mouth talking: sprinkle in natural speech texture — fillers (um, uh), ' + + 'openers (oh, well, so, right, hmm), hedges (kind of, maybe), and backchannels (yeah, mm-hm) ' + + "— usually zero to two per turn, never mechanical. Always use contractions (it's, you're, " + + "I'd, can't); full forms read stiff.\n" + + "- Don't over-decorate short replies or stack tags. If a reaction wouldn't happen in a real " + + "conversation, skip it — there's always another genuine beat to lean into.\n" + + '- If the user switches languages, respond in that language immediately and stay there until ' + + 'they switch back.', + ), +}; + +// Hard per-provider chunking defaults (characters). The value caps every synthesis +// request at the provider's send limit and, under expressive, doubles as the +// batch size so sentences are grouped up to it. Providers absent here are uncapped +// and always emit per sentence. +const MAX_INPUT_LEN: Record = { + inworld: 900, + cartesia: 400, +}; + +/** Return the max text chunk length for a provider, or undefined if unlimited. */ +export function maxInputLen(provider: string): number | undefined { + return MAX_INPUT_LEN[provider]; +} + +/** + * Default sentence tokenizer for a provider's streamed TTS input. + * + * The provider's hard max chunk length caps every emitted token. When `expressive` + * is set, it also raises the *minimum* so consecutive sentences are batched up to + * that size, keeping prosody continuous across the turn; otherwise tokens emit per + * sentence (the unchanged default). Providers with no configured limit are uncapped + * and always per-sentence. + */ +export function sentenceTokenizer( + provider: string, + options: { expressive: boolean }, +): SentenceTokenizer { + const maxLen = MAX_INPUT_LEN[provider]; + return new SentenceTokenizer({ + maxTokenLength: maxLen, + minTokenLength: options.expressive ? maxLen : undefined, + // markup only exists in the stream when expressive is active; xml-aware + // tokenization would otherwise hold streaming on a stray "<" in plain text + xmlAware: options.expressive, + }); +} + +/** Return LLM instruction text for a TTS provider. */ +export function llmInstructions(provider: string): string | undefined { + if (provider === 'cartesia') { + return CARTESIA_LLM_INSTRUCTIONS; + } else if (provider === 'inworld') { + return INWORLD_LLM_INSTRUCTIONS; + } else if (provider === 'xai') { + return XAI_LLM_INSTRUCTIONS; + } + return undefined; +} + +// Per-provider markup spec: [xml tag names, whether square-bracket tags are used]. +const PROVIDER_MARKUP: Record = { + cartesia: [CARTESIA_TAGS, false], + inworld: [INWORLD_TAGS, true], + // xAI's LLM writes every tag as XML (inline sounds/pauses converted to [..] only for + // the TTS in convertMarkup), so the transcript never contains brackets to strip + xai: [XAI_TAGS, false], +}; + +/** + * Strip provider markup and collect the stripped tags in a single pass. + * + * Returns `[cleanText, tags]` — the user-visible transcript plus the expressive + * tags that were removed (in document order), the single source of truth for both + * {@link stripMarkup} and {@link extractMarkup}. `[text, []]` for providers + * without markup support. + */ +export function splitMarkup(provider: string, text: string): [string, ExpressiveTag[]] { + const spec = PROVIDER_MARKUP[provider]; + if (spec === undefined) { + return [text, []]; + } + const [xmlTags, brackets] = spec; + const [clean, rawTags] = extractAndStrip(text, { xmlTags, brackets }); + return [clean, rawTags.map(([tag, value]) => ({ type: tag, value }))]; +} + +/** Strip provider-specific markup tags from text, preserving content. */ +export function stripMarkup(provider: string, text: string): string { + return splitMarkup(provider, text)[0]; +} + +/** + * Extract the markup tags that {@link stripMarkup} would remove, in order. + * + * Lets the framework surface stripped expressive tags (e.g. as `lk.transcription` + * attributes for the frontend) instead of discarding them. Returns `[]` for + * providers without markup support. + */ +export function extractMarkup(provider: string, text: string): ExpressiveTag[] { + return splitMarkup(provider, text)[1]; +} + +// Union of every provider's XML tag names — used by the transcript sinks to strip markup +// without knowing which provider produced it (see {@link TranscriptMarkupStripper}). +const ALL_MARKUP_TAGS: string[] = [ + ...new Set(Object.values(PROVIDER_MARKUP).flatMap(([tags]) => tags)), +].sort(); + +/** + * Strip the union of every provider's expressive markup (provider-agnostic). + * + * The transcript sinks strip downstream, where the originating TTS/provider is no + * longer in scope, so they remove every provider's tags (XML + square brackets) at + * once. These tag shapes never appear in real spoken text — the LLM only emits them + * as audio directives — so a universal strip is safe. + */ +export function splitAllMarkup(text: string): [string, ExpressiveTag[]] { + const [clean, rawTags] = extractAndStrip(text, { xmlTags: ALL_MARKUP_TAGS, brackets: true }); + return [clean, rawTags.map(([tag, value]) => ({ type: tag, value }))]; +} + +/** + * Build the `lk.expression` transcription attribute from stripped markup tags. + * + * Surfaces a segment's leading delivery/emotion (`expression` for Inworld/xAI, + * `emotion` for Cartesia) as `{"value": ...}` so the frontend can react to it. + * Returns `undefined` when no such tag was present. + */ +export function expressionAttribute(tags: ExpressiveTag[]): Record | undefined { + const expression = tags.find((t) => t.type === 'expression' || t.type === 'emotion')?.value; + if (expression === undefined) { + return undefined; + } + return { + [ATTRIBUTE_TRANSCRIPTION_EXPRESSION]: JSON.stringify({ value: expression }), + }; +} + +/** + * Stateful, provider-agnostic markup stripper for one transcript segment. + * + * Fed text chunk-by-chunk, it returns the user-visible text and accumulates the + * stripped tags. A tag-shaped trailing fragment (a partial `<...` or `[...` + * arriving split across chunks) is held back until it closes, so a tag straddling a + * chunk boundary is never emitted half-stripped. Shared by the transcript sinks (room + * output + transcript synchronizer) so stripping and expression extraction stay + * identical across them. + */ +export class TranscriptMarkupStripper { + private buf = ''; + private _tags: ExpressiveTag[] = []; + + private hasOpenTag(): boolean { + // hold a tag-shaped trailing "<" (partial XML tag) so "3 < 5" isn't stalled, and + // any unclosed "[" (bracket tags have no such ambiguity) + const lastLt = this.buf.lastIndexOf('<'); + if (lastLt > this.buf.lastIndexOf('>')) { + const nxt = this.buf.slice(lastLt + 1, lastLt + 2); + if (!nxt || nxt === '/' || /[a-zA-Z]/.test(nxt)) { + return true; + } + } + return this.buf.lastIndexOf('[') > this.buf.lastIndexOf(']'); + } + + /** Feed a chunk; return the clean text ready to emit (may be empty). */ + push(text: string): string { + this.buf += text; + if (this.hasOpenTag()) { + return ''; + } + const [clean, tags] = splitAllMarkup(this.buf); + this.buf = ''; + this._tags.push(...tags); + return clean; + } + + /** Drain any buffered text at segment end; return the remaining clean text. */ + flush(): string { + if (!this.buf) { + return ''; + } + const [clean, tags] = splitAllMarkup(this.buf); + this.buf = ''; + this._tags.push(...tags); + return clean; + } + + /** The markup tags stripped so far, in document order. */ + get tags(): ExpressiveTag[] { + return this._tags; + } + + /** The `lk.expression` attribute for the tags stripped so far, if any. */ + expressionAttribute(): Record | undefined { + return expressionAttribute(this._tags); + } +} + +const SELF_CLOSING_TAGS: Record = { + cartesia: ['emotion', 'speed', 'volume', 'break'], + inworld: ['expression', 'sound', 'break'], +}; + +/** + * Fix common LLM markup mistakes for a provider. + * + * Closes opening tags that should be self-closing (e.g. the LLM writes + * `` instead of ``). + */ +export function normalizeMarkup(provider: string, text: string): string { + const tags = SELF_CLOSING_TAGS[provider]; + if (!tags || tags.length === 0) { + return text; + } + const pattern = new RegExp(`<(${tags.join('|')})\\b([^>]*[^/])\\s*>`, 'g'); + return text.replace(pattern, '<$1$2/>'); +} + +/** Convert framework-standard markup to a provider's native syntax. */ +export function convertMarkup(provider: string, text: string): string { + if (provider === 'inworld' || provider === 'xai') { + // -> [X] (and -> [X]); for xAI this + // turns inline sounds into its native brackets while emotion/prosody stay <..> + text = convertExpressionTags(text); + } + if (provider === 'xai') { + // xAI has no ; map it to its native [pause]/[long-pause] + text = text.replace(XAI_BREAK_RE, xaiBreakToBracket); + } + // is otherwise passed through unchanged: Inworld accepts it as native SSML. + return text; +} diff --git a/agents/src/tts/markup_utils.test.ts b/agents/src/tts/markup_utils.test.ts new file mode 100644 index 000000000..d73c49fca --- /dev/null +++ b/agents/src/tts/markup_utils.test.ts @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { describe, expect, it } from 'vitest'; +import { isInstructions } from '../llm/chat_context.js'; +import { DEFAULT_EXPRESSIVE_OPTIONS } from '../voice/agent_session.js'; +import * as presets from '../voice/presets.js'; +import { + TranscriptMarkupStripper, + convertMarkup, + expressionAttribute, + llmInstructions, + normalizeMarkup, + splitAllMarkup, + splitMarkup, +} from './_provider_format.js'; +import { stripXmlTags } from './markup_utils.js'; +import { type TTS, TTSMarkup } from './tts.js'; + +async function* chunks(items: string[]): AsyncGenerator { + for (const it of items) { + yield it; + } +} + +async function collect(gen: AsyncGenerator): Promise { + const out: string[] = []; + for await (const chunk of gen) { + out.push(chunk); + } + return out; +} + +describe('stripXmlTags', () => { + it('removes self-closing tags entirely', () => { + expect(stripXmlTags(' Hello!', ['emotion'])).toBe(' Hello!'); + }); + + it('preserves the content of wrapping tags', () => { + expect(stripXmlTags('A.B.C. confirmed', ['spell'])).toBe('A.B.C. confirmed'); + }); + + it('preserves unrelated tags', () => { + const text = ' keep'; + expect(stripXmlTags(text, ['emotion'])).toBe(' keep'); + }); + + it('is a no-op with an empty tags list', () => { + const text = ' Hi'; + expect(stripXmlTags(text, [])).toBe(text); + }); +}); + +describe('xAI dialect', () => { + // xAI's LLM writes every tag as XML — inline sounds as and pauses + // as (modeled on Inworld); the transcript strips them all, and + // convertMarkup rewrites sounds to [NAME] and to [pause]/[long-pause] for the + // TTS while prosody/style stay angle-bracketed. + + it('registers LLM instructions', () => { + const instr = llmInstructions('xai'); + // non-undefined is what the expressive gate keys on + expect(instr).toBeDefined(); + expect(instr).toContain(''); + expect(instr).toContain(''); + }); + + it('splitMarkup strips inline tags and keeps wrapping inner text', () => { + const raw = + 'So I walked in and there it was. ' + + 'a secret wow.'; + const [clean, tags] = splitMarkup('xai', raw); + // inline sounds/pauses removed entirely; wrapping tags keep their inner text + expect(clean).not.toContain(''); + expect(clean).toContain('a secret'); + expect(clean).toContain('wow'); + const types = tags.map((t) => [t.type, t.value]); + expect(types).toContainEqual(['break', '500ms']); + expect(types).toContainEqual(['sound', 'laugh']); + expect(types).toContainEqual(['whisper', 'a secret']); + expect(types).toContainEqual(['emphasis', 'wow']); + }); + + it('strips emotion wrapping tags but keeps the spoken words', () => { + const raw = "Great to hear from you! I'm sorry about that."; + const [clean, tags] = splitMarkup('xai', raw); + expect(clean).not.toContain(''); + expect(clean).not.toContain(''); + expect(clean).toContain('Great to hear from you!'); + expect(clean).toContain("I'm sorry about that."); + const types = tags.map((t) => [t.type, t.value]); + expect(types).toContainEqual(['happy', 'Great to hear from you!']); + expect(types).toContainEqual(['sad', "I'm sorry about that."]); + }); + + it('strips nested emotion + prosody tags cleanly', () => { + // combining emotion + prosody means nesting; the transcript must come out clean + // (no leaked inner markup) — this is what the fixed-point strip guarantees + const raw = + 'no way ' + + ' okay'; + const [clean] = splitMarkup('xai', raw); + expect(clean).not.toContain('<'); + expect(clean).not.toContain('>'); + expect(clean).not.toContain('['); + expect(clean).toContain('no way'); + expect(clean).toContain('okay'); + }); + + it('converts inline sounds and pauses to brackets, prosody stays XML', () => { + const raw = + ' hi'; + // -> [X]; -> [pause] (<1s) or [long-pause] (>=1s); + // prosody stays angle-bracketed, and normalize is a no-op for xAI + expect(convertMarkup('xai', raw)).toBe('[laugh] [pause] [long-pause] hi'); + expect(normalizeMarkup('xai', raw)).toBe(raw); + }); + + it('resolves presets to xAI-tuned bodies', () => { + for (const preset of [presets.CUSTOMER_SERVICE, presets.CASUAL]) { + const opts = presets.resolveOptions(preset, { + providerKey: 'xai', + defaultOptions: DEFAULT_EXPRESSIVE_OPTIONS, + }); + const tmpl = opts.ttsInstructionsTemplate!; + const body = isInstructions(tmpl) ? tmpl.common : tmpl; + // tuned body, not the agnostic default (which has no xai tag reference) + expect(body).toContain(''); + } + }); +}); + +describe('normalizeMarkup', () => { + it('closes opening tags that should be self-closing', () => { + expect(normalizeMarkup('inworld', ' Hi')).toBe( + ' Hi', + ); + expect(normalizeMarkup('cartesia', ' Hello')).toBe( + ' Hello', + ); + }); + + it('is a no-op for providers without self-closing tags', () => { + const text = ' Hi'; + expect(normalizeMarkup('unknown-provider', text)).toBe(text); + }); +}); + +describe('universal transcript stripping', () => { + // The transcript sinks strip downstream without knowing the provider, so they remove + // the union of every provider's tags. See splitAllMarkup / TranscriptMarkupStripper. + + it('splitAllMarkup strips every provider dialect at once', () => { + // Cartesia , Inworld/xAI /, and bracket tags all strip + // regardless of which provider produced them + const [clean, tags] = splitAllMarkup( + 'Hi there ' + + '[pause] friend', + ); + expect(clean).toBe('Hi there friend'); + const types = tags.map((t) => [t.type, t.value]); + expect(types).toContainEqual(['emotion', 'happy']); + expect(types).toContainEqual(['expression', 'warm']); + expect(types).toContainEqual(['sound', 'giggle']); + expect(types).toContainEqual(['', 'pause']); + }); + + it('expressionAttribute builds the lk.expression attribute', () => { + let [, tags] = splitAllMarkup('oh no'); + expect(expressionAttribute(tags)).toEqual({ 'lk.expression': '{"value":"sad"}' }); + + // no expression/emotion tag -> no attribute (bracket sounds don't count) + [, tags] = splitAllMarkup('[pause]hi'); + expect(expressionAttribute(tags)).toBeUndefined(); + }); + + it('TranscriptMarkupStripper holds partial tags across pushes', () => { + const s = new TranscriptMarkupStripper(); + // a tag split across pushes is held until it closes, never emitted half-stripped + let out = s.push('Hi the'); + out += s.push('re'); + out += s.flush(); + expect(out).not.toContain(' { + const s = new TranscriptMarkupStripper(); + // a bare "<" in prose must not freeze the following chunk + const first = s.push('The value 3 < 5 '); + expect(first).toContain('3 < 5'); + const rest = s.push('is true.') + s.flush(); + expect((first + rest).replace(/ /g, '')).toBe('Thevalue3<5istrue.'); + }); +}); + +describe('TTSMarkup.toTextStream', () => { + // Regression: the transcript-strip path must not stall on a bare "<" either. + // toTextStream buffered on a naive lastIndexOf("<") > lastIndexOf(">") check, so a + // "<" in prose (e.g. "3 < 5") froze every following transcript chunk of the segment + // until a ">" arrived or the stream ended. + + const markup = () => + // _markupProviderKey is the only TTS member TTSMarkup uses + new TTSMarkup({ _markupProviderKey: () => 'cartesia' } as unknown as TTS); + + it('does not hold the following chunk after a bare "<"', async () => { + const out = await collect(markup().toTextStream(chunks(['The value 3 < 5 ', 'is true.']))); + // fixed: the first chunk is emitted incrementally (>= 2 items); the buggy + // version held everything and emitted a single item at end-of-stream + expect(out.length).toBeGreaterThanOrEqual(2); + expect(out[0]).toContain('3 < 5'); + expect(out.join('').replace(/ /g, '')).toBe('Thevalue3<5istrue.'); + }); + + it('still buffers a genuinely partial tag across chunks', async () => { + const out = await collect( + markup().toTextStream(chunks(['Hi there'])), + ); + const joined = out.join(''); + expect(joined).not.toContain(' { + const tagsOut: { type: string; value: string }[] = []; + const out = await collect( + markup().toTextStream(chunks(['Hello there!']), { tagsOut }), + ); + expect(out.join('')).toBe('Hello there!'); + expect(tagsOut).toEqual([{ type: 'emotion', value: 'happy' }]); + }); +}); diff --git a/agents/src/tts/markup_utils.ts b/agents/src/tts/markup_utils.ts new file mode 100644 index 000000000..2d8239698 --- /dev/null +++ b/agents/src/tts/markup_utils.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +const EXPRESSION_RE = /|>(?:.*?)<\/expression>)/gs; +const SOUND_RE = /|>(?:.*?)<\/sound>)/gs; + +/** Convert `` and `` XML tags to `[...]` bracket format. */ +export function convertExpressionTags(text: string): string { + text = text.replace(EXPRESSION_RE, (_m, value: string) => `[${value}]`); + text = text.replace(SOUND_RE, (_m, value: string) => `[${value}]`); + return text; +} + +const VALUE_ATTR_RE = /\b[\w-]+\s*=\s*"([^"]*)"/; + +const escapeRegExp = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +/** + * Strip markup and collect the stripped tags in a single pass. + * + * One regex scan both removes the markup and records each removed tag, so + * stripping and extraction can never disagree about what counts as a tag. + * + * Returns `[cleanText, tags]` where `tags` is a list of `[type, value]` + * pairs in order of appearance: + * + * - `type` is the XML tag name, or `""` for square-bracket tags. + * - `value` is a wrapping tag's inner text (`A7X9` -> + * `"A7X9"`), else its first quoted attribute value + * (`` -> `"happy"`), else the bracket content, + * falling back to `""`. + * + * Wrapping tags keep their inner content in `cleanText` (only the delimiters + * are removed); self-closing, lone, and bracket tags are removed entirely. + * + * @param text - The text containing markup. + * @param xmlTags - XML tag names to handle (e.g. `["emotion", "sound"]`). + * @param brackets - Whether to also handle square-bracket tags like `[laughs]`. + */ +export function extractAndStrip( + text: string, + options: { xmlTags: string[]; brackets: boolean }, +): [string, Array<[string, string]>] { + const { xmlTags, brackets } = options; + if (xmlTags.length === 0 && !brackets) { + return [text, []]; + } + + const alternatives: string[] = []; + if (xmlTags.length > 0) { + const tagPattern = xmlTags.map(escapeRegExp).join('|'); + // or optionally followed by inner + alternatives.push( + `<(?${tagPattern})\\b(?[^>]*?)\\s*\\/?\\s*>` + + `(?:(?.*?)<\\/\\k\\s*>)?`, + ); + // lone closing tag: + alternatives.push(`<\\/(?:${tagPattern})\\s*>`); + } + if (brackets) { + alternatives.push(String.raw`\[(?[^\]]+)\]`); + } + + const pattern = new RegExp(alternatives.join('|'), 'gs'); + const tags: Array<[string, string]> = []; + + const repl = (match: string, ...args: unknown[]): string => { + const groups = args[args.length - 1] as Record; + const tag = groups.tag; + if (tag !== undefined) { + const inner = groups.inner; + let value: string; + if (inner !== undefined && inner.trim()) { + value = inner.trim(); + } else { + const attrMatch = VALUE_ATTR_RE.exec(groups.attrs ?? ''); + value = attrMatch ? attrMatch[1]! : ''; + } + tags.push([tag, value]); + // wrapping tags keep their inner content; self-closing/lone tags vanish + return inner !== undefined ? inner : ''; + } + + const bracket = groups.bracket; + if (bracket !== undefined) { + tags.push(['', bracket.trim()]); + return ''; + } + + return ''; // lone closing tag + }; + + // iterate to a fixed point so nested wrapping tags are fully removed: a single pass + // strips only the outer tag (e.g. hi -> keeps the + // inner hi), so repeat until the text stops changing. Each pass removes + // at least the matched delimiters, so this always terminates. + let clean = text; + let prev: string | undefined = undefined; + while (clean !== prev) { + prev = clean; + clean = clean.replace(pattern, repl); + } + return [clean, tags]; +} + +/** Strip square bracket tags like `[laughs]`, `[whisper]` from text. */ +export function stripBracketTags(text: string): string { + return extractAndStrip(text, { xmlTags: [], brackets: true })[0]; +} + +/** + * Strip specific XML-style tags from text, preserving their inner content. + * + * Handles opening/closing tag pairs (`content`) and + * self-closing tags (``, ``). + * + * @param text - The text containing XML-style markup. + * @param tags - List of tag names to strip (e.g. `["emotion", "speed"]`). + * @returns The text with the specified tags removed but their content preserved. + */ +export function stripXmlTags(text: string, tags: string[]): string { + return extractAndStrip(text, { xmlTags: tags, brackets: false })[0]; +} diff --git a/agents/src/tts/tts.ts b/agents/src/tts/tts.ts index 3528e9c5c..d233970ae 100644 --- a/agents/src/tts/tts.ts +++ b/agents/src/tts/tts.ts @@ -20,6 +20,13 @@ import { } from '../types.js'; import { AsyncIterableQueue, delay, mergeFrames, startSoon, toError } from '../utils.js'; import type { TimedString } from '../voice/io.js'; +import type { ExpressiveTag } from './_provider_format.js'; +import { + convertMarkup, + normalizeMarkup, + llmInstructions as providerLlmInstructions, + splitMarkup, +} from './_provider_format.js'; /** * SynthesizedAudio is a packet of speech synthesis as returned by the TTS. @@ -79,6 +86,123 @@ export interface SynthesizeStreamStartedTime { hrTime: bigint; } +/** + * Declares TTS markup capabilities for the expressive pipeline. + * + * All methods delegate to the shared per-provider markup tables, keyed on the + * TTS's `_markupProviderKey()`. Plugins opt into markup support by overriding + * that method on their TTS class. + */ +export class TTSMarkup { + #tts: TTS; + + constructor(tts: TTS) { + this.#tts = tts; + } + + #providerKey(): string { + return this.#tts._markupProviderKey(); + } + + /** + * Return instructions for the LLM describing available markup tags. + * + * The framework injects this into the LLM system prompt when + * `expressive` is enabled. Returns `undefined` if this TTS has no markup support. + */ + llmInstructions(): string | undefined { + return providerLlmInstructions(this.#providerKey()); + } + + /** Strip markup and collect the stripped tags in one pass. */ + #split(text: string): [string, ExpressiveTag[]] { + return splitMarkup(this.#providerKey(), text); + } + + /** + * Strip TTS-specific markup from `text`, returning plain text. + * + * Used for transcripts streamed to the user and for chat history storage. + * The TTS itself receives the original marked-up text. + */ + toText(text: string): string { + return this.#split(text)[0]; + } + + /** + * Strip TTS markup from a stream of text chunks. + * + * Buffers partial XML tags across chunks so that each buffer is stripped as a + * whole. When `tagsOut` is given, the stripped tags are appended to it (in + * document order) as a byproduct of the same pass — no second scan. + */ + async *toTextStream( + textStream: AsyncIterable, + options?: { tagsOut?: ExpressiveTag[] }, + ): AsyncGenerator { + const tagsOut = options?.tagsOut; + let buf = ''; + for await (const chunk of textStream) { + buf += chunk; + // hold the buffer only for a *tag-shaped* trailing "<" (a partial tag + // arriving across chunks); a bare "<" as in "3 < 5" is plain text and + // must not stall the transcript until the next ">" or flush + const lastOpen = buf.lastIndexOf('<'); + if (lastOpen > buf.lastIndexOf('>')) { + const nxt = buf.slice(lastOpen + 1, lastOpen + 2); + if (!nxt || nxt === '/' || /[a-zA-Z]/.test(nxt)) { + continue; + } + } + const [stripped, tags] = this.#split(buf); + buf = ''; + if (tagsOut !== undefined) { + tagsOut.push(...tags); + } + if (stripped) { + yield stripped; + } + } + + if (buf) { + const [stripped, tags] = this.#split(buf); + if (tagsOut !== undefined) { + tagsOut.push(...tags); + } + if (stripped) { + yield stripped; + } + } + } + + /** + * Extract the markup tags that {@link toText} would strip, in order. + * + * Lets the framework surface stripped expressive tags (e.g. as transcription + * attributes for the frontend) instead of discarding them. Returns `[]` when + * the provider declares no markup. + */ + extractTags(text: string): ExpressiveTag[] { + return this.#split(text)[1]; + } + + /** Fix common LLM markup mistakes (e.g. unclosed self-closing tags). */ + normalize(text: string): string { + return normalizeMarkup(this.#providerKey(), text); + } + + /** + * Convert framework-standard markup to the provider's native format. + * + * Called before text is sent to the TTS; a no-op when the provider declares + * no markup. Plugins that use non-XML formats (e.g. square brackets) opt in + * via `_markupProviderKey` so `` becomes native syntax. + */ + convert(text: string): string { + return convertMarkup(this.#providerKey(), text); + } +} + /** * An instance of a text-to-speech adapter. * @@ -90,6 +214,15 @@ export abstract class TTS extends (EventEmitter as new () => TypedEmitter TypedEmitter): string { + const collectPaths = (d: Record, prefix = ''): string[] => { + const paths: string[] = []; + for (const [k, v] of Object.entries(d)) { + const full = prefix ? `${prefix}.${k}` : k; + if (v !== null && typeof v === 'object' && !Array.isArray(v)) { + paths.push(...collectPaths(v as Record, full)); + } else { + paths.push(full); + } + } + return paths; + }; + + return template.replace(/\{([\w.]+)\}/g, (_match, path: string) => { + let value: unknown = data; + for (const key of path.split('.')) { + if (value !== null && typeof value === 'object' && key in (value as object)) { + value = (value as Record)[key]; + } else { + value = undefined; + break; + } + } + if (value === undefined) { + log().error( + `template placeholder '{${path}}' has no value (available: ${collectPaths(data).join(', ')})`, + ); + return ''; + } + return value === null ? '' : String(value); + }); +} + const READONLY_SYMBOL = Symbol('Readonly'); const MUTATION_METHODS = [ diff --git a/agents/src/voice/agent.test.ts b/agents/src/voice/agent.test.ts index cbde76fa0..3b7bf7493 100644 --- a/agents/src/voice/agent.test.ts +++ b/agents/src/voice/agent.test.ts @@ -415,8 +415,10 @@ describe('Agent', () => { capabilities: { streaming: true }, stream: () => ttsStream, close: async () => {}, + _setExpressive: () => {}, }, agentSession: { connOptions: { ttsConnOptions: {} } }, + _resolveExpressiveOptions: () => undefined, }; async function* textInput() { diff --git a/agents/src/voice/agent.ts b/agents/src/voice/agent.ts index 05536663b..b442a80cf 100644 --- a/agents/src/voice/agent.ts +++ b/agents/src/voice/agent.ts @@ -34,13 +34,14 @@ import { type FlushSentinel, USERDATA_TIMED_TRANSCRIPT } from '../types.js'; import { Future, Task, toStream } from '../utils.js'; import type { VAD } from '../vad.js'; import { type AgentActivity, agentActivityStorage } from './agent_activity.js'; -import type { AgentSession, TurnDetectionMode } from './agent_session.js'; +import type { AgentSession, ExpressiveOptions, TurnDetectionMode } from './agent_session.js'; import { type AgentCreateOptions, type AgentTaskCreateOptions, createAgentTaskV2, createAgentV2, } from './agent_v2.js'; +import type { AudioRecognition } from './audio_recognition.js'; import type { UserTurnExceededEvent } from './events.js'; import type { TimedString } from './io.js'; import type { SpeechHandle } from './speech_handle.js'; @@ -143,6 +144,11 @@ export interface AgentOptions { tts?: TTS | TTSModelString; turnHandling?: TurnHandlingOptions; toolHandling?: ToolHandlingOptions; + /** + * Enable the expressive pipeline for this agent, overriding the session-level + * setting. See {@link AgentSessionOptions.expressive}. + */ + expressive?: boolean | ExpressiveOptions; minConsecutiveSpeechDelay?: number; useTtsAlignedTranscript?: boolean; /** @deprecated use turnHandling.turnDetection instead */ @@ -159,6 +165,7 @@ export class Agent { private _tts?: TTS; private _turnHandling?: Partial; + private _expressive?: boolean | ExpressiveOptions; private _minConsecutiveSpeechDelay?: number; private _useTtsAlignedTranscript?: boolean; @@ -194,6 +201,7 @@ export class Agent { allowInterruptions, turnHandling, toolHandling, + expressive, minConsecutiveSpeechDelay, useTtsAlignedTranscript, }: AgentOptions) { @@ -248,6 +256,7 @@ export class Agent { this._tts = tts; } + this._expressive = expressive; this._minConsecutiveSpeechDelay = minConsecutiveSpeechDelay; this._useTtsAlignedTranscript = useTtsAlignedTranscript; @@ -274,6 +283,14 @@ export class Agent { return this._useTtsAlignedTranscript; } + /** + * The agent-level expressive setting, overriding the session-level one when set. + * `undefined` means "not given" (fall back to the session option). + */ + get expressive(): boolean | ExpressiveOptions | undefined { + return this._expressive; + } + get chatCtx(): ReadonlyChatContext { return new ReadonlyChatContext(this._chatCtx.items); } @@ -294,6 +311,22 @@ export class Agent { return this.getActivityOrThrow().agentSession as AgentSession; } + /** + * Access the audio recognition system for this agent. + * + * The only public member is `sttContext` — live speaker metadata from the + * STT stream. + * + * @throws Error if the agent is not running. + */ + get audioRecognition(): AudioRecognition { + const audioRecognition = this.getActivityOrThrow()._audioRecognition; + if (!audioRecognition) { + throw new Error('audio recognition is not available for this agent'); + } + return audioRecognition; + } + get turnHandling(): Partial | undefined { return this._turnHandling; } @@ -522,12 +555,24 @@ export class Agent { throw new Error('ttsNode called but no TTS node is available'); } + const expressiveActive = activity._resolveExpressiveOptions() !== undefined; let wrappedTts = activity.tts; if (!activity.tts.capabilities.streaming) { - wrappedTts = new TTSStreamAdapter(wrappedTts, new BasicSentenceTokenizer()); + wrappedTts = new TTSStreamAdapter( + wrappedTts, + // markup only exists in the stream when expressive is active + new BasicSentenceTokenizer({ xmlAware: expressiveActive }), + ); } + // Mark whether expressive is active for this synthesis, synchronously + // just before stream() snapshots it. Doing it here (the single synthesis + // choke point for both generateReply and say()) scopes it to this turn + // rather than leaving stale state on the instance. The provider's chunk + // defaults then drive the TTS's input tokenizer. + activity.tts._setExpressive(expressiveActive); + const connOptions = activity.agentSession.connOptions.ttsConnOptions; const stream = wrappedTts.stream({ connOptions }); stream.updateInputStream(input); diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 9fd906783..78bd0d9c5 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -17,14 +17,15 @@ import { import type { InterruptionDetectionError } from '../inference/interruption/errors.js'; import { AdaptiveInterruptionDetector } from '../inference/interruption/interruption_detector.js'; import type { OverlappingSpeechEvent } from '../inference/interruption/types.js'; +import { TTS as InferenceTTS } from '../inference/tts.js'; import { AgentConfigUpdate, type ChatContext, ChatMessage, - type Instructions, + Instructions, type MetricsReport, - concatInstructions, instructionsEqual, + isInstructions, renderInstructions, } from '../llm/chat_context.js'; import { AsyncToolset, type Toolset } from '../llm/index.js'; @@ -91,7 +92,12 @@ import { _setActivityTaskInfo, speechHandleStorage, } from './agent.js'; -import { type AgentSession, type TurnDetectionMode } from './agent_session.js'; +import { + type AgentSession, + DEFAULT_EXPRESSIVE_OPTIONS, + type ExpressiveOptions, + type TurnDetectionMode, +} from './agent_session.js'; import { AudioRecognition, type EndOfTurnInfo, @@ -121,7 +127,6 @@ import type { ToolExecutionOutput, ToolOutput, _TTSGenerationData } from './gene import { type _AudioOut, type _TextOut, - applyInstructionsModality, forwardedTextFor, performAudioForwarding, performLLMInference, @@ -132,6 +137,7 @@ import { updateInstructions, } from './generation.js'; import type { PlaybackFinishedEvent, TimedString } from './io.js'; +import { resolveOptions as resolveExpressivePreset } from './presets.js'; import { type InputDetails, SpeechHandle } from './speech_handle.js'; import { ToolExecutor, @@ -521,7 +527,7 @@ export class AgentActivity implements RecognitionHooks { try { const realtimeInstructions = !rtReused || capabilities.midSessionInstructionsUpdate - ? renderInstructions(this.agent.instructions) + ? this._renderRealtimeInstructions(this.agent.instructions) : undefined; await this.realtimeSession!._updateSession( realtimeInstructions, @@ -553,9 +559,12 @@ export class AgentActivity implements RecognitionHooks { const initialToolCtx = this.tools; const initialTools = Object.keys(initialToolCtx.functionTools); - if (runOnEnter && (this.agent.instructions || initialTools.length > 0)) { + // collapse modality variants for the record; audio-first matches the + // updateInstructions default for voice sessions + const initialInstructions = renderInstructions(this.agent.instructions, 'audio'); + if (runOnEnter && (initialInstructions || initialTools.length > 0)) { const initialConfig = new AgentConfigUpdate({ - instructions: this.agent.instructions, + instructions: initialInstructions, toolsAdded: initialTools.length > 0 ? initialTools : undefined, }); this.agent._chatCtx.insert(initialConfig); @@ -892,6 +901,11 @@ export class AgentActivity implements RecognitionHooks { return this.audioRecognition?.inputStartedAt; } + /** @internal */ + get _audioRecognition(): AudioRecognition | undefined { + return this.audioRecognition; + } + /** * @internal — used by AMD to obtain a private branch of the participant * audio stream that does not interfere with the pipeline VAD/STT. @@ -920,12 +934,15 @@ export class AgentActivity implements RecognitionHooks { async updateInstructions(instructions: string | Instructions): Promise { this.agent._instructions = instructions; - const configUpdate = new AgentConfigUpdate({ instructions }); + // Record the configuration change (audio-first, matching the initial config record) + const configUpdate = new AgentConfigUpdate({ + instructions: renderInstructions(instructions, 'audio'), + }); this.agent._chatCtx.insert(configUpdate); this.agentSession.history.insert(configUpdate); if (this.realtimeSession) { - await this.realtimeSession.updateInstructions(renderInstructions(instructions)); + await this.realtimeSession.updateInstructions(this._renderRealtimeInstructions(instructions)); } else { updateInstructions({ chatCtx: this.agent._chatCtx, @@ -1865,6 +1882,91 @@ export class AgentActivity implements RecognitionHooks { return this.agentSession.chatCtx; } + /** + * Resolve instructions to a plain string for the realtime session. + * + * Realtime instructions are session-level (there is no per-turn modality + * resolution like the pipeline path), so modality-specific {@link Instructions} + * resolve to the realtime model's output modality. + * @internal + */ + _renderRealtimeInstructions(instructions: string | Instructions): string { + if (isInstructions(instructions)) { + const modality: 'audio' | 'text' = + this.llm instanceof RealtimeModel && this.llm.capabilities.audioOutput ? 'audio' : 'text'; + return instructions.render({ modality }); + } + return instructions; + } + + /** + * Resolve expressive from agent (overrides session). Returns undefined if disabled. + * + * Expressive mode requires two things: + * - the inference gateway TTS (`livekit.agents.inference.TTS`): the markup + * normalization/conversion and expressive chunking run there, so direct + * provider plugins would receive unconverted markup. + * - a TTS that actually declares a markup dialect (`llmInstructions()` is + * not `undefined`): gateway providers without one (e.g. `rime`, `deepgram`) + * get no markup instructions, so no tags can appear in the stream — leaving + * it "active" would enable xml-aware chunking with nothing to chunk and + * re-introduce the stray-`<` streaming stall. + * @internal + */ + _resolveExpressiveOptions(): ExpressiveOptions | undefined { + if (!(this.tts instanceof InferenceTTS) || this.tts.markup.llmInstructions() === undefined) { + return undefined; + } + + let expr = this.agent.expressive; + if (expr === undefined) { + expr = this.agentSession.sessionOptions.expressive; + } + if (typeof expr === 'object') { + // a `preset` selector resolves to the active TTS provider's tuned preset + // (falling back to the agnostic default); explicit fields override on top + const providerKey = this.tts ? this.tts._markupProviderKey() : ''; + return resolveExpressivePreset(expr, { + providerKey, + defaultOptions: DEFAULT_EXPRESSIVE_OPTIONS, + }); + } + return expr ? DEFAULT_EXPRESSIVE_OPTIONS : undefined; + } + + /** + * Inject the TTS markup guide into the chat context. + * @internal + */ + _injectExpressiveInstructions( + chatCtx: ChatContext, + options: ExpressiveOptions, + speechHandle: SpeechHandle, + ): void { + const toInstructions = (v: Instructions | string): Instructions => + isInstructions(v) ? v : new Instructions(v); + + const turnModality = speechHandle?.inputDetails.modality; + + const ttsInstructions = this.tts ? this.tts.markup.llmInstructions() : undefined; + if (ttsInstructions) { + const ttsTemplate = toInstructions(options.ttsInstructionsTemplate!); + const text = ttsTemplate.render({ + modality: turnModality, + data: { + tts: { + markup: { + llm_instructions: ttsInstructions, + }, + }, + }, + }); + if (text.trim()) { + chatCtx.addMessage({ role: 'system', content: text }); + } + } + } + async waitForIdle( options: { waitForAgent?: boolean; waitForUser?: boolean } = {}, ): Promise { @@ -2064,7 +2166,7 @@ export class AgentActivity implements RecognitionHooks { inputDetails, } = options; - let instructions: string | Instructions | undefined = defaultInstructions; + const instructions: string | Instructions | undefined = defaultInstructions; let toolChoice = defaultToolChoice; let allowInterruptions = defaultAllowInterruptions; @@ -2118,7 +2220,10 @@ export class AgentActivity implements RecognitionHooks { speechHandle: handle, // TODO(brian): support llm.ChatMessage for the realtime model userInput: userMessage?.textContent, - instructions, + instructions: + instructions !== undefined + ? this._renderRealtimeInstructions(instructions) + : undefined, modelSettings: { // isGiven(toolChoice) = toolChoice !== undefined toolChoice: toOaiToolChoice(toolChoice !== undefined ? toolChoice : this.toolChoice), @@ -2129,12 +2234,10 @@ export class AgentActivity implements RecognitionHooks { name: 'AgentActivity.realtimeReply', }); } else if (this.llm instanceof LLM) { - // instructions used inside generateReply are "extra" instructions. - // this matches the behavior of the Realtime API: + // instructions used inside generateReply are "extra" instructions: they are + // appended as a separate system message for this turn only, matching the + // behavior of the Realtime API: // https://platform.openai.com/docs/api-reference/realtime-client-events/response/create - if (instructions) { - instructions = concatInstructions(this.agent.instructions, '\n', instructions); - } // Filter out tools with IGNORE_ON_ENTER flag when generateReply is called inside onEnter const onEnterData = onEnterStorage.getStore(); @@ -2646,7 +2749,10 @@ export class AgentActivity implements RecognitionHooks { span.setAttribute(traceTypes.ATTR_SPEECH_ID, speechHandle.id); if (instructions) { - span.setAttribute(traceTypes.ATTR_INSTRUCTIONS, renderInstructions(instructions)); + span.setAttribute( + traceTypes.ATTR_INSTRUCTIONS, + renderInstructions(instructions, speechHandle.inputDetails.modality), + ); } if (newMessage) { span.setAttribute(traceTypes.ATTR_USER_INPUT, newMessage.textContent || ''); @@ -2673,20 +2779,36 @@ export class AgentActivity implements RecognitionHooks { chatCtx.insert(newMessage); } - if (instructions) { + // resolve modality-specific instructions for this turn + const turnModality = speechHandle.inputDetails.modality; + // always re-render the base instructions for this turn's modality — even when + // extra per-turn instructions are supplied — so a text turn never keeps the + // audio-rendered base rules copied from the session chat context + if (isInstructions(this.agent.instructions)) { try { updateInstructions({ chatCtx, - instructions, - addIfMissing: true, + instructions: this.agent.instructions, + modality: turnModality, + addIfMissing: false, }); } catch (e) { this.logger.error({ error: e }, 'error occurred during updateInstructions'); } } + if (instructions) { + // extra instructions are appended as a separate system message for this turn + chatCtx.addMessage({ + role: 'system', + content: [renderInstructions(instructions, turnModality)], + }); + } - // apply the correct variant of the instructions for the turn's input modality - applyInstructionsModality(chatCtx, { modality: speechHandle.inputDetails.modality }); + // inject expressive instructions (TTS markup guide) + const exprOpts = this._resolveExpressiveOptions(); + if (exprOpts !== undefined) { + this._injectExpressiveInstructions(chatCtx, exprOpts, speechHandle); + } const tasks: Array> = []; const [llmTask, llmGenData] = performLLMInference( @@ -3885,7 +4007,8 @@ export class AgentActivity implements RecognitionHooks { modelSettings: ModelSettings; abortController: AbortController; userInput?: string; - instructions?: string | Instructions; + /** Extra instructions, already resolved to a plain string by the caller. */ + instructions?: string; }): Promise { speechHandleStorage.enterWith(speechHandle); @@ -3916,12 +4039,9 @@ export class AgentActivity implements RecognitionHooks { try { const generateReplyAbortController = new AbortController(); - const generationPromise = this.realtimeSession.generateReply( - instructions !== undefined - ? renderInstructions(instructions, speechHandle.inputDetails.modality) - : undefined, - { signal: generateReplyAbortController.signal }, - ); + const generationPromise = this.realtimeSession.generateReply(instructions, { + signal: generateReplyAbortController.signal, + }); void generationPromise.catch(() => undefined); await speechHandle.waitIfNotInterrupted([generationPromise]); diff --git a/agents/src/voice/agent_session.ts b/agents/src/voice/agent_session.ts index c48df175b..2c78538ef 100644 --- a/agents/src/voice/agent_session.ts +++ b/agents/src/voice/agent_session.ts @@ -27,12 +27,7 @@ import { import type { OverlappingSpeechEvent } from '../inference/interruption/types.js'; import { getJobContext } from '../job.js'; import type { FunctionCall, FunctionCallOutput } from '../llm/chat_context.js'; -import { - AgentHandoffItem, - ChatContext, - ChatMessage, - type Instructions, -} from '../llm/chat_context.js'; +import { AgentHandoffItem, ChatContext, ChatMessage, Instructions } from '../llm/chat_context.js'; import type { LLM, RealtimeModel, @@ -97,6 +92,7 @@ import { type KeytermsOptions, resolveKeytermsOptions, } from './keyterm_detection.js'; +import type { Preset } from './presets.js'; import { RecorderIO } from './recorder_io/index.js'; import { RoomSessionTransport, SessionHost } from './remote_session.js'; import { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io/index.js'; @@ -179,6 +175,35 @@ export function resolveRecordingOptions( return { ...RECORDING_ALL_ON, ...record }; } +/** + * Configuration for the expressive pipeline. + * + * Controls how TTS markup instructions are injected into the LLM when expressive is + * enabled. All keys are optional; common shapes: + * + * - `{ preset: Preset.CASUAL }` — a domain preset, resolved to the active + * TTS provider's tuned tags (see `voice/presets`). Prefer the `presets.*` constants. + * - `{ preset: ..., ttsInstructionsAppend: '...' }` — a preset plus your own + * rules appended after it resolves. + * - `{ ttsInstructionsTemplate: '...' }` — a fully custom prompt. + * + * Any explicit template overrides the corresponding part of the resolved preset; unset + * parts fall back to the resolved preset (or the provider-agnostic default). + */ +export interface ExpressiveOptions { + preset?: Preset; + ttsInstructionsTemplate?: Instructions | string; + ttsInstructionsAppend?: string; +} + +export const DEFAULT_EXPRESSIVE_OPTIONS: ExpressiveOptions = { + ttsInstructionsTemplate: new Instructions( + 'You can control how you speak using the following formatting tags. ' + + 'Use them when appropriate to make your speech more expressive and natural:\n\n' + + '{tts.markup.llm_instructions}', + ), +}; + export interface InternalSessionOptions extends AgentSessionOptions { turnHandling: InternalTurnHandlingOptions; useTtsAlignedTranscript: boolean; @@ -187,6 +212,7 @@ export interface InternalSessionOptions extends AgentSessionOptions = { * and `filter_emoji`; pass `null` to disable text transforms. */ ttsTextTransforms?: readonly TextTransform[] | null; + + /** + * Enable the expressive pipeline: the LLM is instructed to emit the active TTS + * provider's markup tags (emotion/expression/pauses/...), the tags are converted + * to the provider's native syntax before synthesis, and stripped from the room + * transcript. Pass `true` for the provider-agnostic default prompt, or an + * {@link ExpressiveOptions} (e.g. a `presets.*` constant) to tune it. + * @defaultValue false + */ + expressive?: boolean | ExpressiveOptions; }; export type AgentSessionUpdateOptions = { diff --git a/agents/src/voice/audio_recognition.ts b/agents/src/voice/audio_recognition.ts index d2005bcbc..907e62657 100644 --- a/agents/src/voice/audio_recognition.ts +++ b/agents/src/voice/audio_recognition.ts @@ -35,7 +35,7 @@ import { DeferredReadableStream } from '../stream/deferred_stream.js'; import { IdentityTransform } from '../stream/identity_transform.js'; import { mergeReadableStreams } from '../stream/merge_readable_streams.js'; import { type StreamChannel, createStreamChannel } from '../stream/stream_channel.js'; -import { type SpeechEvent, SpeechEventType } from '../stt/stt.js'; +import { type SpeechEvent, SpeechEventType, isSpeakerContext } from '../stt/stt.js'; import { traceTypes, tracer } from '../telemetry/index.js'; import { splitWords } from '../tokenize/basic/word.js'; import type { Future } from '../utils.js'; @@ -355,6 +355,7 @@ export class AudioRecognition { private silenceAudioWriter: WritableStreamDefaultWriter; private sttOwnershipTransferred = false; private readonly sttLifecycleLock = new Mutex(); + private _sttContext?: object; // all cancellable tasks private bounceEOUTask?: Task; @@ -524,6 +525,36 @@ export class AudioRecognition { return this.sttPipeline?.inputStartedAt; } + /** + * Live speaker metadata from the STT stream. + * + * STT plugins set `SpeechStream.context` during recognition. + * The framework copies it here so it's accessible even after the stream + * is replaced (e.g. during agent handoff). + */ + get sttContext(): object | undefined { + return this._sttContext; + } + + set sttContext(value: object | undefined) { + this._sttContext = value; + } + + /** + * Speaker context formatted as LLM instructions. + * + * Returns `sttContext.toInstructions()` if the context implements + * {@link SpeakerContext}, otherwise `undefined`. + */ + llmInstructions(): string | undefined { + const ctx = this._sttContext; + if (ctx !== undefined && isSpeakerContext(ctx)) { + const result = ctx.toInstructions(); + return result ? result : undefined; + } + return undefined; + } + /** @internal */ updateOptions(options: { endpointing?: BaseEndpointing; diff --git a/agents/src/voice/generation.ts b/agents/src/voice/generation.ts index 3d6a4c831..0579dd019 100644 --- a/agents/src/voice/generation.ts +++ b/agents/src/voice/generation.ts @@ -13,7 +13,7 @@ import { ChatMessage, FunctionCall, FunctionCallOutput, - isInstructions, + renderInstructions, } from '../llm/chat_context.js'; import type { ChatChunk } from '../llm/llm.js'; import { @@ -401,19 +401,24 @@ export const INSTRUCTIONS_MESSAGE_ID = 'lk.agent_task.instructions'; * Update the instruction message in the chat context or insert a new one if missing. * * This function looks for an existing instruction message in the chat context using the identifier - * 'INSTRUCTIONS_MESSAGE_ID'. + * 'INSTRUCTIONS_MESSAGE_ID'. Instructions are resolved to a plain string using the given modality + * before storage. * * @param options - The options for updating the instructions. * @param options.chatCtx - The chat context to update. * @param options.instructions - The instructions to add. * @param options.addIfMissing - Whether to add the instructions if they are missing. + * @param options.modality - The modality used to render {@link Instructions} (default `'audio'`). */ export function updateInstructions(options: { chatCtx: ChatContext; instructions: string | Instructions; addIfMissing: boolean; + modality?: 'audio' | 'text'; }) { - const { chatCtx, instructions, addIfMissing } = options; + const { chatCtx, instructions, addIfMissing, modality = 'audio' } = options; + + const text = renderInstructions(instructions, modality); const idx = chatCtx.indexById(INSTRUCTIONS_MESSAGE_ID); if (idx !== undefined) { @@ -422,7 +427,7 @@ export function updateInstructions(options: { chatCtx.items[idx] = ChatMessage.create({ id: INSTRUCTIONS_MESSAGE_ID, role: 'system', - content: [instructions], + content: [text], createdAt: chatCtx.items[idx]!.createdAt, }); } else { @@ -434,49 +439,12 @@ export function updateInstructions(options: { ChatMessage.create({ id: INSTRUCTIONS_MESSAGE_ID, role: 'system', - content: [instructions], + content: [text], }), ); } } -/** - * Apply the correct {@link Instructions} variant for the turn's input modality. - * - * Locates the instructions message (by {@link INSTRUCTIONS_MESSAGE_ID}) and, - * if its content contains any {@link Instructions} entries, rebuilds the - * message so each Instructions renders as the chosen variant. No-op when no - * modality-aware instructions are present. - */ -export function applyInstructionsModality( - chatCtx: ChatContext, - options: { modality: 'audio' | 'text' }, -) { - const { modality } = options; - const idx = chatCtx.indexById(INSTRUCTIONS_MESSAGE_ID); - if (idx === undefined) return; - - const item = chatCtx.items[idx]!; - if (item.type !== 'message') return; - - const hasModalitySpecific = item.content.some((c) => isInstructions(c)); - if (!hasModalitySpecific) return; - - // ChatContext.copy shadows the original item; create a new instance so the - // base context's content isn't mutated when the same Instructions is reused - // across turns. - chatCtx.items[idx] = ChatMessage.create({ - id: item.id, - role: item.role, - content: item.content.map((c) => (isInstructions(c) ? c.asModality(modality) : c)), - interrupted: item.interrupted, - createdAt: item.createdAt, - transcriptConfidence: item.transcriptConfidence, - metrics: item.metrics, - extra: item.extra, - }); -} - export function performLLMInference( node: LLMNode, chatCtx: ChatContext, diff --git a/agents/src/voice/index.ts b/agents/src/voice/index.ts index 02872fc8a..630670e00 100644 --- a/agents/src/voice/index.ts +++ b/agents/src/voice/index.ts @@ -50,6 +50,7 @@ export { createTimedString, isTimedString, } from './io.js'; +export * as presets from './presets.js'; export * from './report.js'; export * from './room_io/index.js'; export { RunContext } from './run_context.js'; diff --git a/agents/src/voice/presets.ts b/agents/src/voice/presets.ts new file mode 100644 index 000000000..d55f2a6bf --- /dev/null +++ b/agents/src/voice/presets.ts @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * Public expressive presets. + * + * A preset is a *use-case* (customer service, casual) that is + * provider-agnostic at the call site: + * + * ```ts + * import { voice } from '@livekit/agents'; + * + * const session = new AgentSession({ + * tts: 'inworld/inworld-tts-2', + * expressive: voice.presets.CASUAL, + * }); + * ``` + * + * Each `presets.*` constant is just an {@link ExpressiveOptions} carrying a `preset`. + * At session start the framework resolves it against the active TTS provider (via + * `tts._markupProviderKey()`) and injects the variant tuned for that provider's markup + * tags. A provider with no tuned preset falls back to the agnostic default, which still + * injects that provider's tag reference through the `{tts.markup.llm_instructions}` + * placeholder — so a preset always does something sensible and can never disagree with + * the markup pipeline (both read the same provider key). + * + * Customize by spreading a constant into a new object (don't mutate the constant in place): + * + * ```ts + * expressive: { ...presets.CUSTOMER_SERVICE, ttsInstructionsAppend: 'Confirm the name.' } + * ``` + */ +import { Instructions, isInstructions } from '../llm/chat_context.js'; +import { + CARTESIA_CASUAL, + CARTESIA_CUSTOMER_SERVICE, + INWORLD_CASUAL, + INWORLD_CUSTOMER_SERVICE, + XAI_CASUAL, + XAI_CUSTOMER_SERVICE, +} from '../tts/_provider_format.js'; +import type { ExpressiveOptions } from './agent_session.js'; + +/** The domain a preset is tuned for. Used to key the per-provider registry. */ +export enum Preset { + CUSTOMER_SERVICE = 'customer_service', + CASUAL = 'casual', +} + +// (provider key as returned by `tts._markupProviderKey()`) -> preset -> body +const REGISTRY: Record>> = { + inworld: { + [Preset.CUSTOMER_SERVICE]: INWORLD_CUSTOMER_SERVICE, + [Preset.CASUAL]: INWORLD_CASUAL, + }, + cartesia: { + [Preset.CUSTOMER_SERVICE]: CARTESIA_CUSTOMER_SERVICE, + [Preset.CASUAL]: CARTESIA_CASUAL, + }, + xai: { + [Preset.CUSTOMER_SERVICE]: XAI_CUSTOMER_SERVICE, + [Preset.CASUAL]: XAI_CASUAL, + }, +}; + +function append(template: Instructions | string, extra: string): Instructions { + // concatenate the *raw* template text so any {placeholders} survive until render() + if (isInstructions(template)) { + return new Instructions(template.common + '\n\n' + extra, { + audio: template.audio, + text: template.text, + }); + } + return new Instructions(template + '\n\n' + extra); +} + +/** + * Resolve a user {@link ExpressiveOptions} to a concrete options object for a provider. + * + * If `expr` carries a `preset`, start from that provider's tuned preset (or + * `defaultOptions` when the provider has none); otherwise start from `defaultOptions`. + * Then apply any explicit `ttsInstructionsTemplate` override and `ttsInstructionsAppend`. + * The returned object always has `ttsInstructionsTemplate` and never the `preset` / + * `ttsInstructionsAppend` helper keys. + */ +export function resolveOptions( + expr: ExpressiveOptions, + options: { providerKey: string; defaultOptions: ExpressiveOptions }, +): ExpressiveOptions { + const { providerKey, defaultOptions } = options; + + const preset = expr.preset; + const base = + preset !== undefined ? REGISTRY[providerKey]?.[preset] ?? defaultOptions : defaultOptions; + + let ttsTmpl = expr.ttsInstructionsTemplate ?? base.ttsInstructionsTemplate!; + const extra = expr.ttsInstructionsAppend; + if (extra) { + ttsTmpl = append(ttsTmpl, extra); + } + + return { + ttsInstructionsTemplate: ttsTmpl, + }; +} + +export const CUSTOMER_SERVICE: ExpressiveOptions = { preset: Preset.CUSTOMER_SERVICE }; +export const CASUAL: ExpressiveOptions = { preset: Preset.CASUAL }; diff --git a/agents/src/voice/remote_session.ts b/agents/src/voice/remote_session.ts index ae2af7bbe..92b4a473a 100644 --- a/agents/src/voice/remote_session.ts +++ b/agents/src/voice/remote_session.ts @@ -15,7 +15,7 @@ import type { FunctionCall as FCItem, FunctionCallOutput as FCOItem, } from '../llm/chat_context.js'; -import { isInstructions, renderInstructions } from '../llm/chat_context.js'; +import { renderInstructions } from '../llm/chat_context.js'; import { type ToolContext, sortedToolNames } from '../llm/tool_context.js'; import { log } from '../log.js'; import type { @@ -421,10 +421,6 @@ function chatItemToProto(item: RemoteChatItem): pb.ChatContext_ChatItem { for (const c of msg.content) { if (typeof c === 'string') { content.push(new pb.ChatMessage_ChatContent({ payload: { case: 'text', value: c } })); - } else if (isInstructions(c)) { - content.push( - new pb.ChatMessage_ChatContent({ payload: { case: 'text', value: c.value } }), - ); } } diff --git a/agents/src/voice/room_io/_output.ts b/agents/src/voice/room_io/_output.ts index 11dd8eb71..316d2c708 100644 --- a/agents/src/voice/room_io/_output.ts +++ b/agents/src/voice/room_io/_output.ts @@ -23,6 +23,12 @@ import { TOPIC_TRANSCRIPTION, } from '../../constants.js'; import { log } from '../../log.js'; +import { + type ExpressiveTag, + TranscriptMarkupStripper, + expressionAttribute, + splitAllMarkup, +} from '../../tts/_provider_format.js'; import { Future, Task, shortuuid } from '../../utils.js'; import { AudioOutput, TextOutput, type TimedString, isTimedString } from '../io.js'; import { findMicrophoneTrackId } from '../transcription/index.js'; @@ -143,6 +149,11 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription private writer: TextStreamWriter | null = null; private flushTask: Task | null = null; private jsonFormat: boolean; + // per-segment markup stripping: delta streams strip incrementally (buffering a tag + // split across chunks); non-delta streams re-strip the full text each time and keep + // the latest tags here for the expression attribute (see TranscriptMarkupStripper) + private stripper: TranscriptMarkupStripper = new TranscriptMarkupStripper(); + private segmentTags: ExpressiveTag[] = []; constructor( room: Room, @@ -154,35 +165,17 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription this.jsonFormat = options.jsonFormat ?? false; } + protected override resetState() { + super.resetState(); + this.stripper = new TranscriptMarkupStripper(); + this.segmentTags = []; + } + override async captureText(text: string | TimedString) { if (!this.participantIdentity) { return; } - // latestText must hold the encoded payload so non-delta flush (FINAL=true) republishes the - // same newline-delimited JSON format as the interim chunks. - const payload = this.jsonFormat - ? this.encodeJsonChunk(text) - : isTimedString(text) - ? text.text - : text; - this.latestText = payload; - await this.handleCaptureText(payload); - } - - private encodeJsonChunk(text: string | TimedString): string { - const isTimed = isTimedString(text); - const message = new pb.TimedString({ - text: isTimed ? text.text : text, - startTime: isTimed ? text.startTime : undefined, - endTime: isTimed ? text.endTime : undefined, - confidence: isTimed ? text.confidence : undefined, - startTimeOffset: isTimed ? text.startTimeOffset : undefined, - }); - return message.toJsonString({ useProtoFieldName: true }) + '\n'; - } - - protected async handleCaptureText(text: string): Promise { if (this.flushTask && !this.flushTask.done) { await this.flushTask.result; } @@ -192,6 +185,27 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription this.capturing = true; } + // the raw text (expressive markup intact) arrives here; publish only the visible + // text. Skip a chunk that strips to nothing (a partial tag still buffering, or a + // markup-only token) so the transcript cadence isn't disturbed. + const textStr = isTimedString(text) ? text.text : text; + let cleanText: string; + if (this.isDeltaStream) { + cleanText = this.stripper.push(textStr); + } else { + const [clean, tags] = splitAllMarkup(textStr); + cleanText = clean; + this.segmentTags = tags; + } + if (!cleanText) { + return; + } + + // latestText must hold the encoded payload so non-delta flush (FINAL=true) republishes the + // same newline-delimited JSON format as the interim chunks. + const payload = this.encode(cleanText, text); + this.latestText = payload; + try { if (this.room.isConnected) { if (this.isDeltaStream) { @@ -199,10 +213,10 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription if (this.writer === null) { this.writer = await this.createTextWriter(); } - await this.writer.write(text); + await this.writer.write(payload); } else { const tmpWriter = await this.createTextWriter(); - await tmpWriter.write(text); + await tmpWriter.write(payload); await tmpWriter.close(); } } @@ -211,10 +225,48 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription } } + /** Wrap visible text for the wire (JSON TimedString when jsonFormat, else raw). */ + private encode(cleanText: string, timingSrc?: string | TimedString): string { + if (!this.jsonFormat) { + return cleanText; + } + + const isTimed = timingSrc !== undefined && isTimedString(timingSrc); + const message = new pb.TimedString({ + text: cleanText, + startTime: isTimed ? timingSrc.startTime : undefined, + endTime: isTimed ? timingSrc.endTime : undefined, + confidence: isTimed ? timingSrc.confidence : undefined, + startTimeOffset: isTimed ? timingSrc.startTimeOffset : undefined, + }); + return message.toJsonString({ useProtoFieldName: true }) + '\n'; + } + + protected async handleCaptureText(_text: string): Promise { + // unused: captureText is fully overridden to strip markup before encoding + } + protected handleFlush() { const currWriter = this.writer; this.writer = null; - this.flushTask = Task.from((controller) => this.flushTaskImpl(currWriter, controller.signal)); + + // only emit on a segment that captured text (keeps lk.transcription cadence intact). + // The leading expression the sinks stripped rides along on the closing chunk as the + // lk.expression attribute. + let remaining: string; + let tags: ExpressiveTag[]; + if (this.isDeltaStream) { + remaining = this.stripper.flush(); + tags = this.stripper.tags; + } else { + remaining = ''; + tags = this.segmentTags; + } + const pendingText = remaining ? this.encode(remaining) : ''; + + this.flushTask = Task.from((controller) => + this.flushTaskImpl(currWriter, expressionAttribute(tags), pendingText, controller.signal), + ); } private async createTextWriter(attributes?: Record): Promise { @@ -243,13 +295,23 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription }); } - private async flushTaskImpl(writer: TextStreamWriter | null, signal: AbortSignal): Promise { + private async flushTaskImpl( + writer: TextStreamWriter | null, + extraAttributes: Record | undefined, + pendingText: string, + signal: AbortSignal, + ): Promise { const attributes: Record = { [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'true', }; if (this.trackId) { attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId; } + for (const [key, val] of Object.entries(extraAttributes ?? {})) { + if (!(key in attributes)) { + attributes[key] = val; + } + } const abortPromise = new Promise((resolve) => { signal.addEventListener('abort', () => resolve()); @@ -259,6 +321,16 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription if (this.room.isConnected) { if (this.isDeltaStream) { if (writer) { + if (pendingText) { + // visible text left in the strip buffer + await Promise.race([writer.write(pendingText), abortPromise]); + if (signal.aborted) { + return; + } + } + // NOTE: rtc-node's TextStreamWriter.close() takes no attributes yet, so the + // lk.expression attribute cannot ride the closing chunk of a delta stream + // (Python attaches it via `aclose(attributes=...)`). await Promise.race([writer.close(), abortPromise]); } } else { @@ -303,7 +375,12 @@ export class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscr this.pushedText = text; } - await this.publishTranscription(this.currentId, this.pushedText, false); + // pushedText keeps the raw text (markup intact); publish the visible text only. + // Stripping the whole accumulation each time avoids partial-tag edge cases; the + // expression is dropped here — the deprecated rtc Transcription API has no + // attribute channel (the stream-based output carries lk.expression instead). + const [cleanText] = splitAllMarkup(this.pushedText); + await this.publishTranscription(this.currentId, cleanText, false); } protected handleFlush() { @@ -311,7 +388,8 @@ export class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscr return; } - this.flushTask = this.publishTranscription(this.currentId, this.pushedText, true); + const [cleanText] = splitAllMarkup(this.pushedText); + this.flushTask = this.publishTranscription(this.currentId, cleanText, true); this.resetState(); } diff --git a/agents/src/voice/transcription/synchronizer.ts b/agents/src/voice/transcription/synchronizer.ts index dc5c081f2..b71205aad 100644 --- a/agents/src/voice/transcription/synchronizer.ts +++ b/agents/src/voice/transcription/synchronizer.ts @@ -7,6 +7,7 @@ import { log } from '../../log.js'; import { IdentityTransform } from '../../stream/identity_transform.js'; import type { WordStream, WordTokenizer } from '../../tokenize/index.js'; import { basic } from '../../tokenize/index.js'; +import { splitAllMarkup } from '../../tts/_provider_format.js'; import { Future, Task, delay } from '../../utils.js'; import { AudioOutput, @@ -435,6 +436,14 @@ class SegmentSynchronizerImpl { const forwardedWord = this.textData.pushedText.slice(pushedTextCursor, wordEnd); pushedTextCursor = wordEnd; + // forward the raw token (the room output strips markup and surfaces the + // expression downstream), but pace against the visible text only so a + // markup-only token adds no delay + const [strippedWord] = splitAllMarkup(word); + const cleanWords = this.options.splitWords(strippedWord); + const cleanWord = cleanWords.length > 0 ? cleanWords[0]![0] : strippedWord; + const wordHyphens = cleanWord ? this.options.hyphenateWord(cleanWord).length : 0; + if (this.playbackCompleted) { this.outputStreamWriter.write( createTimedString({ @@ -442,16 +451,11 @@ class SegmentSynchronizerImpl { endTime: this.synchronizedElapsedSeconds(), }), ); - const cleanWords = this.options.splitWords(word); - const cleanWord = cleanWords.length > 0 ? cleanWords[0]![0] : word; - this.textData.forwardedHyphens += this.options.hyphenateWord(cleanWord).length; + this.textData.forwardedHyphens += wordHyphens; this.textData.forwardedText += forwardedWord; continue; } - const cleanWords = this.options.splitWords(word); - const cleanWord = cleanWords.length > 0 ? cleanWords[0]![0] : word; - const wordHyphens = this.options.hyphenateWord(cleanWord).length; const elapsedSeconds = this.synchronizedElapsedSeconds()!; let dHyphens = 0; diff --git a/agents/src/voice/turn_config/utils.ts b/agents/src/voice/turn_config/utils.ts index 100cda367..afceca7e8 100644 --- a/agents/src/voice/turn_config/utils.ts +++ b/agents/src/voice/turn_config/utils.ts @@ -28,6 +28,7 @@ const defaultSessionOptions = { turnHandling: {}, useTtsAlignedTranscript: true, ttsTextTransforms: ['filter_markdown', 'filter_emoji'], + expressive: false, } as const satisfies AgentSessionOptions; const defaultLegacyVoiceOptions: VoiceOptions = { diff --git a/agents/src/workflows/address.ts b/agents/src/workflows/address.ts new file mode 100644 index 000000000..02510d501 --- /dev/null +++ b/agents/src/workflows/address.ts @@ -0,0 +1,265 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; +import { type InstructionParts, resolveWorkflowInstructions } from './utils.js'; + +export interface GetAddressResult { + address: string; +} + +export interface GetAddressTaskOptions { + /** + * Instructions for the address capture prompt. Pass a full string or {@link Instructions} to + * replace the built-in prompt entirely, or {@link InstructionParts} to override individual + * sections (e.g. `persona`) while keeping the built-in template. + */ + instructions?: InstructionParts | Instructions | string; + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString; + vad?: VAD; + llm?: LLM | RealtimeModel | LLMModels; + tts?: TTS | TTSModelString; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm the captured address. Defaults to confirming on audio + * input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** + * When true, the model must produce an asking utterance before recording an address — it + * can't silently fill one from the chat context during `onEnter`. + */ + requireExplicitAsk?: boolean; +} + +interface UpdateAddressArgs { + streetAddress: string; + unitNumber: string; + locality: string; + country: string; +} + +/** + * Build an {@link AgentTask} that collects a postal address from the user. + * + * This is the functional core; {@link GetAddressTask} is a thin class wrapper over it. + */ +export function createGetAddressTask({ + instructions, + chatCtx, + turnDetection, + tools = [], + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + requireExplicitAsk = false, +}: GetAddressTaskOptions = {}): AgentTask { + let currentAddress = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + // confirm tool is only injected after update_address is called, + // preventing the LLM from hallucinating a confirmation without user input + const buildConfirmTool = (address: string) => + tool({ + name: 'confirm_address', + description: 'Call after the user confirms the address is correct.', + execute: async () => { + if (address !== currentAddress) { + task.session.generateReply({ + instructions: + 'The address has changed since confirmation was requested, ask the user to confirm the updated address.', + }); + return; + } + + if (!task.done) { + task.complete({ address }); + } + }, + }); + + const updateAddressTool = tool({ + name: 'update_address', + description: 'Update the address provided by the user.', + parameters: z.object({ + streetAddress: z + .string() + .describe( + 'Dependent on country, may include fields like house number, street name, block, or district', + ), + unitNumber: z + .string() + .describe( + "The unit number, for example Floor 1 or Apartment 12. If there is no unit number, return ''", + ), + locality: z + .string() + .describe('Dependent on country, may include fields like city, zip code, or province'), + country: z.string().describe('The country the user lives in spelled out fully'), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ( + { streetAddress, unitNumber, locality, country }: UpdateAddressArgs, + { ctx }, + ) => { + const addressFields = unitNumber.trim() + ? [streetAddress, unitNumber, locality, country] + : [streetAddress, locality, country]; + const address = addressFields.join(' '); + currentAddress = address; + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete({ address: currentAddress }); + } + return; + } + + const confirmTool = buildConfirmTool(address); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_address'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + `The address has been updated to ${address}\n` + + `Repeat the address field by field: ${JSON.stringify(addressFields)} if needed\n` + + `Prompt the user for confirmation, do not call \`confirm_address\` directly` + ); + }, + }); + + const declineTool = tool({ + name: 'decline_address_capture', + description: 'Handles the case when the user explicitly declines to provide an address.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide the address'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }) => { + if (!task.done) { + task.complete(new ToolError(`couldn't get the address: ${reason}`)); + } + }, + }); + + const task = AgentTask.create({ + id: 'get_address_task', + instructions: resolveWorkflowInstructions({ + instructions, + template: INSTRUCTIONS_TEMPLATE, + defaultPersona: PERSONA, + modalitySpecific: new Instructions('', { + audio: AUDIO_SPECIFIC, + text: TEXT_SPECIFIC, + }), + confirmation: new Instructions('', { + // confirmation is enabled by default for audio, disabled by default for text + audio: requireConfirmation !== false ? CONFIRMATION_INSTRUCTION : '', + text: requireConfirmation === true ? CONFIRMATION_INSTRUCTION : '', + }), + }), + chatCtx, + turnDetection, + tools: [...tools, updateAddressTool, declineTool], + stt, + vad, + llm, + tts, + allowInterruptions, + onEnter: async () => { + task.session.generateReply({ instructions: 'Ask the user to provide their address.' }); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetAddressTask}, preserving the + * `new GetAddressTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetAddressTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetAddressTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetAddressTask. + super({ instructions: '' }); + this.#task = createGetAddressTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +// instructions +const PERSONA = + 'You are only a single step in a broader system, responsible solely for capturing an address.'; + +const AUDIO_SPECIFIC = `You will be handling addresses from any country. +Expect that users will say address in different formats with fields filled like: +- 'streetAddress': '450 SOUTH MAIN ST', 'unitNumber': 'FLOOR 2', 'locality': 'SALT LAKE CITY UT 84101', 'country': 'UNITED STATES', +- 'streetAddress': '123 MAPLE STREET', 'unitNumber': 'APARTMENT 10', 'locality': 'OTTAWA ON K1A 0B1', 'country': 'CANADA', +- 'streetAddress': 'GUOMAO JIE 3 HAO, CHAOYANG QU', 'unitNumber': 'GUOMAO DA SHA 18 LOU 101 SHI', 'locality': 'BEIJING SHI 100000', 'country': 'CHINA', +- 'streetAddress': '5 RUE DE L’ANCIENNE COMÉDIE', 'unitNumber': 'APP C4', 'locality': '75006 PARIS', 'country': 'FRANCE', +- 'streetAddress': 'PLOT 10, NEHRU ROAD', 'unitNumber': 'OFFICE 403, 4TH FLOOR', 'locality': 'VILE PARLE (E), MUMBAI MAHARASHTRA 400099', 'country': 'INDIA', +Normalize common spoken patterns silently: +- Convert words like 'dash' and 'apostrophe' into symbols: \`-\`, \`'\`. +- Convert spelled out numbers like 'six' and 'seven' into numerals: \`6\`, \`7\`. +- Recognize patterns where users speak their address field followed by spelling: e.g., 'guomao g u o m a o'. +- Filter out filler words or hesitations. +- Recognize when there may be accents on certain letters if explicitly said or common in the location specified. Be sure to verify the correct accents if existent. +Don't mention corrections. Treat inputs as possibly imperfect but fix them silently. +When reading a numerical ordinal suffix (st, nd, rd, th), the number must be verbally expanded into its full, correctly pronounced word form. +Do not read the number and the suffix letters separately. +Confirm postal codes by reading them out digit-by-digit as a sequence of single numbers. Do not read them as cardinal numbers. +For example, read 90210 as 'nine zero two one zero.' +Avoid using bullet points and parenthese in any responses. +Spell out the address letter-by-letter when applicable, such as street names and provinces, especially when the user spells it out initially.`; + +const TEXT_SPECIFIC = `You will be handling addresses from any country. +Expect users to type their address directly. +If the address looks almost correct but has minor issues (e.g. missing country or postal code), prompt for clarification.`; + +const CONFIRMATION_INSTRUCTION = `Call \`confirm_address\` after the user confirmed the address is correct.`; + +const INSTRUCTIONS_TEMPLATE = `{persona} + +{modalitySpecific} + +Call \`update_address\` at the first opportunity whenever you form a new hypothesis about the address. (before asking any questions or providing any answers.) +Don't invent new addresses, stick strictly to what the user said. +{confirmation} +If the address is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts in this order: street address, unit number if applicable, locality, and country. + +Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. +Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called. + +{extra} +`; diff --git a/agents/src/workflows/credit_card.ts b/agents/src/workflows/credit_card.ts new file mode 100644 index 000000000..5e9b5c510 --- /dev/null +++ b/agents/src/workflows/credit_card.ts @@ -0,0 +1,838 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, LLM, ToolError, ToolFlag, tool } from '../llm/index.js'; +import { isToolError } from '../llm/tool_context.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import { safeRender } from '../utils.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; +import { type GetNameResult, createGetNameTask } from './name.js'; +import { TaskGroup } from './task_group.js'; + +const CARD_ISSUERS_LOOKUP: Record = { + '3': 'American Express', + '4': 'Visa', + '5': 'Mastercard', + '6': 'Discover', +}; + +export interface GetCreditCardResult { + cardholderName: string; + issuer: string; + cardNumber: string; + securityCode: string; + expirationDate: string; +} + +interface GetCardNumberResult { + issuer: string; + cardNumber: string; +} + +interface GetSecurityCodeResult { + securityCode: string; +} + +interface GetExpirationDateResult { + date: string; +} + +export class CardCaptureDeclinedError extends ToolError { + readonly reason: string; + + constructor(reason: string) { + super(`couldn't get the card details: ${reason}`); + this.reason = reason; + } +} + +export class CardCollectionRestartError extends ToolError { + readonly reason: string; + + constructor(reason: string) { + super(`starting over: ${reason}`); + this.reason = reason; + } +} + +const declineCardCaptureTool = tool({ + name: 'decline_card_capture', + description: + 'Handles the case when the user explicitly declines to provide a detail for their card information.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide card information'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }, { ctx }) => { + const task = ctx.session.currentAgent; + if (task instanceof AgentTask && !task.done) { + task.complete(new CardCaptureDeclinedError(reason)); + } + }, +}); + +const restartCardCollectionTool = tool({ + name: 'restart_card_collection', + description: + 'Handles the case when the user wishes to start over the card information collection process and validate a new card.', + parameters: z.object({ + reason: z.string().describe('A short explanation of why the user wishes to start over'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }, { ctx }) => { + const task = ctx.session.currentAgent; + if (task instanceof AgentTask && !task.done) { + task.complete(new CardCollectionRestartError(reason)); + } + }, +}); + +/** Validates a card number via the Luhn algorithm. */ +function validateCardNumber(cardNumber: string): boolean { + if (!cardNumber || !/^\d+$/.test(cardNumber)) { + return false; + } + let totalSum = 0; + + const reversedNumber = [...cardNumber].reverse(); + for (let index = 0; index < reversedNumber.length; index++) { + const digit = Number(reversedNumber[index]); + if (index % 2 === 1) { + const doubledDigit = digit * 2; + totalSum += doubledDigit > 9 ? doubledDigit - 9 : doubledDigit; + } else { + totalSum += digit; + } + } + + return totalSum % 10 === 0; +} + +interface SubTaskOptions { + chatCtx?: ChatContext; + requireConfirmation?: boolean; + requireExplicitAsk?: boolean; + extraInstructions?: string; + turnDetection?: TurnDetectionMode; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString; + vad?: VAD; + llm?: LLM | RealtimeModel | LLMModels; + tts?: TTS | TTSModelString; + allowInterruptions?: boolean; +} + +function buildSubTaskInstructions( + baseInstructions: string, + audioSpecific: string, + textSpecific: string, + confirmationInstructions: string, + requireConfirmation: boolean | undefined, + extraInstructions: string, +): Instructions { + const extraSuffix = extraInstructions ? `\n${extraInstructions}` : ''; + return new Instructions('', { + audio: + safeRender(baseInstructions, { + modalitySpecific: audioSpecific, + confirmationInstructions: requireConfirmation !== false ? confirmationInstructions : '', + }) + extraSuffix, + text: + safeRender(baseInstructions, { + modalitySpecific: textSpecific, + confirmationInstructions: requireConfirmation === true ? confirmationInstructions : '', + }) + extraSuffix, + }); +} + +function createGetCardNumberTask({ + chatCtx, + requireConfirmation, + requireExplicitAsk = false, + extraInstructions = '', + turnDetection, + tools = [], + stt, + vad, + llm, + tts, + allowInterruptions, +}: SubTaskOptions = {}): AgentTask { + let currentCardNumber = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildConfirmTool = (cardNumber: string) => + tool({ + name: 'confirm_card_number', + description: 'Call after the user repeats their card number for confirmation.', + parameters: z.object({ + repeatedCardNumber: z.string().describe('The card number repeated by the user as a string'), + }), + execute: async ({ repeatedCardNumber }) => { + repeatedCardNumber = repeatedCardNumber.replace(/\D/g, ''); + if (repeatedCardNumber !== cardNumber) { + task.session.generateReply({ + instructions: 'The repeated card number does not match, ask the user to try again.', + }); + return; + } + + if (!validateCardNumber(cardNumber)) { + task.session.generateReply({ + instructions: + 'The card number is not valid, ask the user if they made a mistake or to provide another card.', + }); + } else { + const issuer = CARD_ISSUERS_LOOKUP[cardNumber[0]!] ?? 'Other'; + if (!task.done) { + task.complete({ issuer, cardNumber }); + } + } + }, + }); + + const updateCardNumberTool = tool({ + name: 'update_card_number', + description: + "Call to record the user's card number. Only call once the entire number has been given, do not call in increments.", + parameters: z.object({ + cardNumber: z + .string() + .describe('The credit card number as a string with no dashes or spaces'), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ({ cardNumber: rawCardNumber }: { cardNumber: string }, { ctx }) => { + const cardNumber = rawCardNumber.replace(/\D/g, ''); + if (cardNumber.length < 13 || cardNumber.length > 19) { + task.session.generateReply({ + instructions: + 'The length of the card number is invalid, ask the user to repeat their card number.', + }); + return; + } + + currentCardNumber = cardNumber; + + if (!confirmationRequired(ctx)) { + if (!validateCardNumber(currentCardNumber)) { + task.session.generateReply({ + instructions: + 'The card number is not valid, ask the user if they made a mistake or to provide another card.', + }); + } else { + const issuer = CARD_ISSUERS_LOOKUP[currentCardNumber[0]!] ?? 'Other'; + if (!task.done) { + task.complete({ issuer, cardNumber: currentCardNumber }); + } + } + return; + } + + const confirmTool = buildConfirmTool(cardNumber); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_card_number'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + 'The card number has been updated.\n' + + 'Ask them to repeat the number, do not repeat the number back to them.\n' + ); + }, + }); + + const task = AgentTask.create({ + id: 'get_card_number_task', + instructions: buildSubTaskInstructions( + CARD_NUMBER_BASE_INSTRUCTIONS, + CARD_NUMBER_AUDIO_SPECIFIC, + CARD_NUMBER_TEXT_SPECIFIC, + 'Call `confirm_card_number` once the user has repeated their card number.', + requireConfirmation, + extraInstructions, + ), + chatCtx, + turnDetection, + tools: [...tools, updateCardNumberTool, declineCardCaptureTool, restartCardCollectionTool], + stt, + vad, + llm, + tts, + allowInterruptions, + onEnter: async () => { + await task.session.generateReply({ + instructions: + "Get the user's credit card number. First scan the conversation - if a " + + 'credit card number was already given (e.g. the user volunteered it ' + + 'before the task started), use it via update_card_number rather than ' + + 're-asking. Only ask fresh when no credit card number is in the ' + + 'conversation yet.', + }); + }, + }); + + return task; +} + +function createGetSecurityCodeTask({ + chatCtx, + requireConfirmation, + requireExplicitAsk = false, + extraInstructions = '', + turnDetection, + tools = [], + stt, + vad, + llm, + tts, + allowInterruptions, +}: SubTaskOptions = {}): AgentTask { + let currentSecurityCode = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildConfirmTool = (securityCode: string) => + tool({ + name: 'confirm_security_code', + description: 'Call after the user repeats their security code for confirmation.', + parameters: z.object({ + repeatedSecurityCode: z.string().describe('The security code repeated by the user'), + }), + execute: async ({ repeatedSecurityCode }) => { + if (repeatedSecurityCode.trim() !== securityCode) { + task.session.generateReply({ + instructions: 'The repeated security code does not match, ask the user to try again.', + }); + return; + } + + if (!task.done) { + task.complete({ securityCode }); + } + }, + }); + + const updateSecurityCodeTool = tool({ + name: 'update_security_code', + description: "Call to update the card's security code.", + parameters: z.object({ + securityCode: z + .string() + .describe("The card's security code (3-4 digits, may have leading zeros)."), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ({ securityCode }: { securityCode: string }, { ctx }) => { + const stripped = securityCode.trim(); + if (!/^\d+$/.test(stripped) || stripped.length < 3 || stripped.length > 4) { + task.session.generateReply({ + instructions: + "The security code's length is invalid, ask the user to repeat or to provide a new card and start over.", + }); + return; + } + + currentSecurityCode = stripped; + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete({ securityCode: currentSecurityCode }); + } + return; + } + + const confirmTool = buildConfirmTool(stripped); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_security_code'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + 'The security code has been updated.\n' + + 'Do not repeat the security code back to the user, ask them to repeat themselves.\n' + + 'Call `confirm_security_code` once the user confirms, do not call it preemptively.\n' + ); + }, + }); + + const task = AgentTask.create({ + id: 'get_security_code_task', + instructions: buildSubTaskInstructions( + SECURITY_CODE_BASE_INSTRUCTIONS, + SECURITY_CODE_AUDIO_SPECIFIC, + SECURITY_CODE_TEXT_SPECIFIC, + 'Call `confirm_security_code` once the user has repeated their security code.', + requireConfirmation, + extraInstructions, + ), + chatCtx, + turnDetection, + tools: [...tools, updateSecurityCodeTool, declineCardCaptureTool, restartCardCollectionTool], + stt, + vad, + llm, + tts, + allowInterruptions, + onEnter: async () => { + await task.session.generateReply({ + instructions: + "Get the user's card security code. First scan the conversation - if a " + + 'code was already given, use it via update_security_code rather than ' + + 're-asking. Only ask fresh when no code is in the conversation yet.', + }); + }, + }); + + return task; +} + +function createGetExpirationDateTask({ + chatCtx, + requireConfirmation, + requireExplicitAsk = false, + extraInstructions = '', + turnDetection, + tools = [], + stt, + vad, + llm, + tts, + allowInterruptions, +}: SubTaskOptions = {}): AgentTask { + let currentExpirationDate = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const isExpired = (month: number, year: number): boolean => { + const today = new Date(); + const fullYear = 2000 + year; + return ( + fullYear < today.getFullYear() || + (fullYear === today.getFullYear() && month < today.getMonth() + 1) + ); + }; + + const buildConfirmTool = (expirationMonth: number, expirationYear: number) => { + const expirationDate = currentExpirationDate; + + return tool({ + name: 'confirm_expiration_date', + description: 'Call after the user repeats their expiration date for confirmation.', + parameters: z.object({ + repeatedExpirationMonth: z + .number() + .int() + .describe('The expiration month repeated by the user'), + repeatedExpirationYear: z + .number() + .int() + .describe('The expiration year repeated by the user'), + }), + execute: async ({ repeatedExpirationMonth, repeatedExpirationYear }) => { + if ( + repeatedExpirationMonth !== expirationMonth || + repeatedExpirationYear !== expirationYear + ) { + task.session.generateReply({ + instructions: 'The repeated expiration date does not match, ask the user to try again.', + }); + return; + } + + if (!task.done) { + task.complete({ date: expirationDate }); + } + }, + }); + }; + + const updateExpirationDateTool = tool({ + name: 'update_expiration_date', + description: + "Call to update the card's expiration date. Collect both the numerical month and year.", + parameters: z.object({ + expirationMonth: z + .number() + .int() + .describe("The numerical expiration month of the card, example: '04' for April"), + expirationYear: z + .number() + .int() + .describe( + "The numerical expiration year of the card shortened to the last two digits, for example, '35' for 2035", + ), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ( + { expirationMonth, expirationYear }: { expirationMonth: number; expirationYear: number }, + { ctx }, + ) => { + if (expirationMonth < 1 || expirationMonth > 12) { + task.session.generateReply({ + instructions: + 'The expiration month is invalid, ask the user to repeat the expiration month.', + }); + return; + } + if (expirationYear < 0 || expirationYear > 99) { + task.session.generateReply({ + instructions: + 'The expiration year is invalid, ask the user to repeat the expiration year.', + }); + return; + } + if (isExpired(expirationMonth, expirationYear)) { + task.session.generateReply({ + instructions: + 'The expiration date is in the past, the card is expired. Ask the user to provide another card.', + }); + return; + } + + currentExpirationDate = `${String(expirationMonth).padStart(2, '0')}/${String(expirationYear).padStart(2, '0')}`; + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete({ date: currentExpirationDate }); + } + return; + } + + const confirmTool = buildConfirmTool(expirationMonth, expirationYear); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_expiration_date'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + 'The expiration date has been updated.\n' + + 'Do not repeat the expiration date back to the user, ask them to repeat themselves.\n' + + 'Call `confirm_expiration_date` once the user confirms, do not call it preemptively.\n' + ); + }, + }); + + const task = AgentTask.create({ + id: 'get_expiration_date_task', + instructions: buildSubTaskInstructions( + EXPIRATION_DATE_BASE_INSTRUCTIONS, + EXPIRATION_DATE_AUDIO_SPECIFIC, + EXPIRATION_DATE_TEXT_SPECIFIC, + 'Call `confirm_expiration_date` once the user has repeated their expiration date.', + requireConfirmation, + extraInstructions, + ), + chatCtx, + turnDetection, + tools: [...tools, updateExpirationDateTool, declineCardCaptureTool, restartCardCollectionTool], + stt, + vad, + llm, + tts, + allowInterruptions, + onEnter: async () => { + await task.session.generateReply({ + instructions: + "Get the user's card expiration date. First scan the conversation - if " + + 'an expiration date was already given, use it via update_expiration_date ' + + 'rather than re-asking. Only ask fresh when no date is in the conversation yet.', + }); + }, + }); + + return task; +} + +export interface GetCreditCardTaskOptions { + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString; + vad?: VAD; + llm?: LLM | RealtimeModel | LLMModels; + tts?: TTS | TTSModelString; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm each captured card detail. Defaults to confirming on + * audio input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** Extra instructions appended to each sub-task's prompt for domain-specific context. */ + extraInstructions?: string; +} + +/** + * Build an {@link AgentTask} that collects the user's full credit card details (number, + * expiration date, security code, and cardholder name) via a {@link TaskGroup} of sub-tasks. + * + * This is the functional core; {@link GetCreditCardTask} is a thin class wrapper over it. + */ +export function createGetCreditCardTask({ + chatCtx, + turnDetection, + tools = [], + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + extraInstructions = '', +}: GetCreditCardTaskOptions = {}): AgentTask { + const task = AgentTask.create({ + id: 'get_credit_card_task', + instructions: '*none*', + chatCtx, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + onEnter: async () => { + // Pass chatCtx into both the TaskGroup AND every sub-task. The + // TaskGroup overwrites each sub-task's chatCtx with its own (see + // TaskGroup.onEnter) - without seeding the TaskGroup, sub-tasks + // would run with empty context. + const ctx = task.chatCtx; + // Role hint for the cardholder sub-task. With IGNORE_ON_ENTER on + // update_name (via requireExplicitAsk=true), the model is + // structurally forced to ask before recording. The extra text + // just makes sure the *question* anchors to the card. + let cardholderExtra = + 'You are collecting the name on the credit card (the cardholder). ' + + 'When you ask the user to confirm a candidate name from earlier in ' + + 'the conversation, anchor the question to the card or cardholder ' + + "so the user knows which name you mean - not just 'is it [name]?' " + + 'in the abstract.'; + if (extraInstructions) { + cardholderExtra = `${extraInstructions}\n\n${cardholderExtra}`; + } + + // Task-level model/tool overrides must reach the sub-tasks: the outer + // placeholder never converses, the sub-tasks do. + const subTaskOptions = { + requireConfirmation, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + }; + + while (!task.done) { + // Order: number first (most natural for the caller to give + // when asked for "card details"), then expiry and security + // code, then the cardholder name LAST. The name most often + // pre-fills from chatCtx (same as the booking name) so + // leaving it for the end avoids the failure mode where the + // caller's first response (typically the digits) gets crammed + // into update_name. + const taskGroup = new TaskGroup({ + chatCtx: ctx, + // TaskGroup summarization requires a standard LLM on the session; + // skip it on realtime-model sessions instead of failing after all + // card details were collected. + summarizeChatCtx: task.session.llm instanceof LLM, + }); + taskGroup.add( + () => + createGetCardNumberTask({ + ...subTaskOptions, + chatCtx: ctx, + extraInstructions, + }), + { + id: 'card_number_task', + description: "Collects the user's card number", + }, + ); + taskGroup.add( + () => + createGetExpirationDateTask({ + ...subTaskOptions, + chatCtx: ctx, + extraInstructions, + }), + { + id: 'expiration_date_task', + description: "Collects the card's expiration date", + }, + ); + taskGroup.add( + () => + createGetSecurityCodeTask({ + ...subTaskOptions, + chatCtx: ctx, + extraInstructions, + }), + { + id: 'security_code_task', + description: "Collects the card's security code", + }, + ); + taskGroup.add( + () => + createGetNameTask({ + ...subTaskOptions, + lastName: true, + chatCtx: ctx, + extraInstructions: cardholderExtra, + // The cardholder may differ from the caller or any guest + // mentioned earlier in chatCtx. Apply IGNORE_ON_ENTER on + // update_name so the model must produce an asking turn + // rather than silently filling from chatCtx. + requireExplicitAsk: true, + }), + { + id: 'cardholder_name_task', + description: "Collects the cardholder's full name", + }, + ); + + try { + const results = await taskGroup.run(); + const nameResult = results.taskResults['cardholder_name_task'] as GetNameResult; + const cardNumberResult = results.taskResults['card_number_task'] as GetCardNumberResult; + const securityCodeResult = results.taskResults[ + 'security_code_task' + ] as GetSecurityCodeResult; + const expirationDateResult = results.taskResults[ + 'expiration_date_task' + ] as GetExpirationDateResult; + + task.complete({ + cardholderName: `${nameResult.firstName} ${nameResult.lastName}`, + issuer: cardNumberResult.issuer, + cardNumber: cardNumberResult.cardNumber, + securityCode: securityCodeResult.securityCode, + expirationDate: expirationDateResult.date, + }); + } catch (e) { + if (e instanceof CardCollectionRestartError) { + continue; + } + if (e instanceof CardCaptureDeclinedError || isToolError(e)) { + if (!task.done) { + task.complete(e as ToolError); + } + return; + } + throw e; + } + } + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetCreditCardTask}, preserving the + * `new GetCreditCardTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetCreditCardTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetCreditCardTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetCreditCardTask. + super({ instructions: '' }); + this.#task = createGetCreditCardTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +const CARD_NUMBER_BASE_INSTRUCTIONS = ` +You are a single step in a broader process of collecting credit card information. +You are solely responsible for collecting the credit card number. +{modalitySpecific} +If the user refuses to provide a credit card number, call decline_card_capture(). +If the user wishes to start over the credit card collection process, call restart_card_collection(). +Avoid listing out questions with bullet points or numbers, use a natural conversational tone. +Never repeat any sensitive information, such as the user's credit card number, back to the user. +{confirmationInstructions} +`; + +const CARD_NUMBER_AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect users to read the card number digit by digit. +Normalize spoken digits silently: 'four' → 4, 'zero' / 'oh' → 0. +Filter out filler words or hesitations. +`; + +const CARD_NUMBER_TEXT_SPECIFIC = ` +Handle input as typed text. Users may type the number with or without spaces or dashes (e.g. '4152 6374 8901 2345'). +`; + +const SECURITY_CODE_BASE_INSTRUCTIONS = ` +You are a single step in a broader process of collecting credit card information. +You are solely responsible for collecting the user's card's security code. +{modalitySpecific} +If the user refuses to provide a code, call decline_card_capture(). +If the user wishes to start over the card collection process, call restart_card_collection(). +Avoid listing out questions with bullet points or numbers, use a natural conversational tone. +Never repeat any sensitive information, such as the user's security code, back to the user. +{confirmationInstructions} +`; + +const SECURITY_CODE_AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect users to read the security code digit by digit. +Normalize spoken digits silently: 'four' → 4, 'zero' / 'oh' → 0. +Filter out filler words or hesitations. +`; + +const SECURITY_CODE_TEXT_SPECIFIC = ` +Handle input as typed text. Users will type the security code directly. +`; + +const EXPIRATION_DATE_BASE_INSTRUCTIONS = ` +You are a single step in a broader process of collecting credit card information. +You are solely responsible for collecting the user's card's expiration date. +{modalitySpecific} +If the user refuses to provide a date, call decline_card_capture(). +If the user wishes to start over the card collection process, call restart_card_collection(). +Avoid listing out questions with bullet points or numbers, use a natural conversational tone. +Never repeat any sensitive information, such as the user's expiration date, back to the user. +{confirmationInstructions} +`; + +const EXPIRATION_DATE_AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect users to say the expiration date in formats like 'April twenty five', 'oh four twenty five', 'four slash twenty five', or 'April 2025'. +Normalize spoken months and digits silently. +Filter out filler words or hesitations. +`; + +const EXPIRATION_DATE_TEXT_SPECIFIC = ` +Handle input as typed text. Expect users to type the expiration date in formats like '04/25', '04/2025', or 'April 2025'. +`; diff --git a/agents/src/workflows/dob.ts b/agents/src/workflows/dob.ts new file mode 100644 index 000000000..bb7c5a95c --- /dev/null +++ b/agents/src/workflows/dob.ts @@ -0,0 +1,406 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import { safeRender } from '../utils.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; + +/** A calendar date without a timezone. `month` and `day` are 1-based. */ +export interface DateOfBirth { + year: number; + month: number; + day: number; +} + +/** A time of day. `hour` is 0-23 and `minute` is 0-59. */ +export interface TimeOfBirth { + hour: number; + minute: number; +} + +export interface GetDOBResult { + dateOfBirth: DateOfBirth; + timeOfBirth?: TimeOfBirth; +} + +export interface GetDOBTaskOptions { + /** Extra instructions appended to the built-in prompt for domain-specific context. */ + extraInstructions?: string; + /** Also capture the (optional) time of birth. Defaults to false. */ + includeTime?: boolean; + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString; + vad?: VAD; + llm?: LLM | RealtimeModel | LLMModels; + tts?: TTS | TTSModelString; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm the captured date of birth. Defaults to confirming on + * audio input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** + * When true, the model must produce an asking utterance before recording a date of birth — + * it can't silently fill one from the chat context during `onEnter`. + */ + requireExplicitAsk?: boolean; +} + +const MONTH_NAMES = [ + 'January', + 'February', + 'March', + 'April', + 'May', + 'June', + 'July', + 'August', + 'September', + 'October', + 'November', + 'December', +] as const; + +function formatDate(dob: DateOfBirth): string { + return `${MONTH_NAMES[dob.month - 1]} ${String(dob.day).padStart(2, '0')}, ${dob.year}`; +} + +function formatTime(time: TimeOfBirth): string { + const hour12 = time.hour % 12 === 0 ? 12 : time.hour % 12; + const period = time.hour < 12 ? 'AM' : 'PM'; + return `${String(hour12).padStart(2, '0')}:${String(time.minute).padStart(2, '0')} ${period}`; +} + +function isSameDate(a: DateOfBirth | undefined, b: DateOfBirth | undefined): boolean { + if (a === undefined || b === undefined) { + return a === b; + } + return a.year === b.year && a.month === b.month && a.day === b.day; +} + +function isSameTime(a: TimeOfBirth | undefined, b: TimeOfBirth | undefined): boolean { + if (a === undefined || b === undefined) { + return a === b; + } + return a.hour === b.hour && a.minute === b.minute; +} + +/** + * Build an {@link AgentTask} that collects a date of birth (and optionally a time of birth) + * from the user. + * + * This is the functional core; {@link GetDOBTask} is a thin class wrapper over it. + */ +export function createGetDOBTask({ + extraInstructions = '', + includeTime = false, + chatCtx, + turnDetection, + tools = [], + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + requireExplicitAsk = false, +}: GetDOBTaskOptions = {}): AgentTask { + const timeInstructions = !includeTime + ? '' + : 'Also ask for and capture the time of birth if the user knows it. ' + + "The time is optional - if the user doesn't know it, proceed without it.\n"; + const confirmationInstructions = + 'Call `confirm_dob` after the user confirmed the date of birth is correct.'; + + let currentDob: DateOfBirth | undefined; + let currentTime: TimeOfBirth | undefined; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildResult = (dateOfBirth: DateOfBirth): GetDOBResult => ({ + dateOfBirth, + timeOfBirth: currentTime, + }); + + // confirm tool is only injected after update_dob/update_time is called, + // preventing the LLM from hallucinating a confirmation without user input + const buildConfirmTool = (capturedDob: DateOfBirth | undefined) => { + const capturedTime = currentTime; + + return tool({ + name: 'confirm_dob', + description: 'Call after the user confirms the date of birth is correct.', + execute: async () => { + if (!isSameDate(capturedDob, currentDob) || !isSameTime(capturedTime, currentTime)) { + task.session.generateReply({ + instructions: + 'The date of birth has changed since confirmation was requested, ask the user to confirm the updated date.', + }); + return; + } + + if (currentDob === undefined) { + task.session.generateReply({ + instructions: 'No date of birth was provided yet, ask the user to provide it.', + }); + return; + } + + if (!task.done) { + task.complete(buildResult(currentDob)); + } + }, + }); + }; + + const injectConfirmTool = async (): Promise => { + const confirmTool = buildConfirmTool(currentDob); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_dob'); + await task.updateTools([...currentTools, confirmTool]); + }; + + const updateDobTool = tool({ + name: 'update_dob', + description: + "Update the date of birth provided by the user. Given a spoken month and year (e.g., 'July 2030'), return its numerical representation (7/2030).", + parameters: z.object({ + year: z.number().int().describe('The birth year (e.g., 1990)'), + month: z.number().int().describe('The birth month (1-12)'), + day: z.number().int().describe('The birth day (1-31)'), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ( + { year, month, day }: { year: number; month: number; day: number }, + { ctx }, + ) => { + // Normalize two-digit years to the intended century, matching what the + // prompt already asks the model to do ("90" -> 1990, "05" -> 2005). A + // literal two-digit year is otherwise a valid date (e.g. 90 -> year 90 AD) + // that passes the future-date check and silently corrupts the result. + if (year >= 0 && year < 100) { + const currentYY = new Date().getFullYear() % 100; + year += year <= currentYY ? 2000 : 1900; + } + + if (month < 1 || month > 12 || day < 1 || day > 31) { + throw new ToolError(`Invalid date: year=${year} month=${month} day=${day}`); + } + const candidate = new Date(Date.UTC(year, month - 1, day)); + if ( + candidate.getUTCFullYear() !== year || + candidate.getUTCMonth() !== month - 1 || + candidate.getUTCDate() !== day + ) { + throw new ToolError(`Invalid date: year=${year} month=${month} day=${day}`); + } + + const dob: DateOfBirth = { year, month, day }; + const today = new Date(); + const todayParts: DateOfBirth = { + year: today.getFullYear(), + month: today.getMonth() + 1, + day: today.getDate(), + }; + const isFuture = + dob.year > todayParts.year || + (dob.year === todayParts.year && + (dob.month > todayParts.month || + (dob.month === todayParts.month && dob.day > todayParts.day))); + if (isFuture) { + throw new ToolError( + `Invalid date of birth: ${formatDate(dob)} is in the future. ` + + 'Date of birth cannot be a future date.', + ); + } + + currentDob = dob; + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete(buildResult(currentDob)); + } + return; + } + + await injectConfirmTool(); + + let response = `The date of birth has been updated to ${formatDate(dob)}`; + + if (currentTime) { + response += ` at ${formatTime(currentTime)}`; + } + + response += + '\nRepeat the date back to the user in a natural spoken format.\n' + + 'Prompt the user for confirmation, do not call `confirm_dob` directly'; + + return response; + }, + }); + + const updateTimeTool = tool({ + name: 'update_time', + description: 'Update the time of birth provided by the user.', + parameters: z.object({ + hour: z.number().int().describe('The birth hour (0-23)'), + minute: z.number().int().describe('The birth minute (0-59)'), + }), + execute: async ({ hour, minute }: { hour: number; minute: number }, { ctx }) => { + if (hour < 0 || hour > 23 || minute < 0 || minute > 59) { + throw new ToolError(`Invalid time: hour=${hour} minute=${minute}`); + } + + const birthTime: TimeOfBirth = { hour, minute }; + currentTime = birthTime; + + if (!confirmationRequired(ctx) && currentDob !== undefined) { + if (!task.done) { + task.complete(buildResult(currentDob)); + } + return; + } + + if (confirmationRequired(ctx)) { + await injectConfirmTool(); + } + + let response = `The time of birth has been updated to ${formatTime(birthTime)}`; + + if (currentDob) { + response = `The date and time of birth has been updated to ${formatDate(currentDob)} at ${formatTime(birthTime)}`; + } + + if (confirmationRequired(ctx)) { + response += + '\nRepeat the time back to the user in a natural spoken format.\n' + + 'Prompt the user for confirmation, do not call `confirm_dob` directly'; + } else { + response += '\nThe date of birth has not been provided yet, ask the user to provide it.'; + } + + return response; + }, + }); + + const declineTool = tool({ + name: 'decline_dob_capture', + description: 'Handles the case when the user explicitly declines to provide a date of birth.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide the date of birth'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }) => { + if (!task.done) { + task.complete(new ToolError(`couldn't get the date of birth: ${reason}`)); + } + }, + }); + + const task = AgentTask.create({ + id: 'get_dob_task', + instructions: new Instructions('', { + audio: safeRender(BASE_INSTRUCTIONS, { + modalitySpecific: AUDIO_SPECIFIC, + timeInstructions, + confirmationInstructions: requireConfirmation !== false ? confirmationInstructions : '', + extraInstructions, + }), + text: safeRender(BASE_INSTRUCTIONS, { + modalitySpecific: TEXT_SPECIFIC, + timeInstructions, + confirmationInstructions: requireConfirmation === true ? confirmationInstructions : '', + extraInstructions, + }), + }), + chatCtx, + turnDetection, + tools: [...tools, updateDobTool, declineTool, ...(includeTime ? [updateTimeTool] : [])], + stt, + vad, + llm, + tts, + allowInterruptions, + onEnter: async () => { + const prompt = includeTime + ? 'Ask the user to provide their date of birth and, if they know it, their time of birth.' + : 'Ask the user to provide their date of birth.'; + task.session.generateReply({ instructions: prompt }); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetDOBTask}, preserving the + * `new GetDOBTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetDOBTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetDOBTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetDOBTask. + super({ instructions: '' }); + this.#task = createGetDOBTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +const BASE_INSTRUCTIONS = ` +You are only a single step in a broader system, responsible solely for capturing a date of birth. +{modalitySpecific} +{timeInstructions}Call \`update_dob\` at the first opportunity whenever you form a new hypothesis about the date of birth. (before asking any questions or providing any answers.) +Don't invent dates, stick strictly to what the user said. +{confirmationInstructions} +When reading back dates, use a natural spoken format like 'January fifteenth, nineteen ninety'. +If the date is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts: first the month, then the day, then the year. +Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. +Avoid verbosity by not sharing example dates or formats unless prompted to do so. Do not deviate from the goal of collecting the user's birthday. +Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called.\ +{extraInstructions} +`; + +const AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect that users will say dates aloud with formats like: +- 'January 15th 1990' +- 'the fifteenth of January nineteen ninety' +- '01 15 1990' or 'one fifteen ninety' +- 'Jan 15 90' +- '15th January 1990' +Normalize common spoken patterns silently: +- Convert spoken numbers and ordinals to their numeric form: 'fifteenth' → 15, 'ninety' → 1990. +- Recognize month names in various forms: 'Jan', 'January', etc. +- Handle two-digit years appropriately: '90' likely means 1990, '05' likely means 2005. +- Filter out filler words or hesitations. +Don't mention corrections. Treat inputs as possibly imperfect but fix them silently. +`; + +const TEXT_SPECIFIC = ` +Handle input as typed text. Expect users to type their date of birth directly. +Accept common date formats like 'MM/DD/YYYY', 'January 15, 1990', or '1990-01-15'. +Handle two-digit years appropriately: '90' likely means 1990, '05' likely means 2005. +`; diff --git a/agents/src/workflows/dtmf_inputs.ts b/agents/src/workflows/dtmf_inputs.ts new file mode 100644 index 000000000..8892c157c --- /dev/null +++ b/agents/src/workflows/dtmf_inputs.ts @@ -0,0 +1,286 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { RoomEvent } from '@livekit/rtc-node'; +import { z } from 'zod'; +import { getJobContext } from '../job.js'; +import type { ChatContext } from '../llm/index.js'; +import { ToolError, tool } from '../llm/index.js'; +import { log } from '../log.js'; +import { AgentTask } from '../voice/agent.js'; +import { + AgentSessionEventTypes, + type AgentStateChangedEvent, + type UserStateChangedEvent, +} from '../voice/events.js'; +import { DtmfEvent, formatDtmf } from './utils.js'; + +export interface GetDtmfResult { + userInput: string; +} + +export interface GetDtmfTaskOptions { + /** The number of digits to collect. */ + numDigits: number; + /** Whether to ask for confirmation when the agent has collected the full digits. */ + askForConfirmation?: boolean; + /** The per-digit timeout, in milliseconds. Defaults to 4000. */ + dtmfInputTimeout?: number; + /** The DTMF event that stops collecting inputs. Defaults to `DtmfEvent.POUND`. */ + dtmfStopEvent?: DtmfEvent; + /** The chat context to use. */ + chatCtx?: ChatContext; + /** Extra instructions to add to the task. */ + extraInstructions?: string; +} + +/** + * Debounced runner for a single async function: `schedule()` (re)starts a delay timer, + * `run()` fires immediately, `cancel()` clears any pending timer. Mirrors the Python + * `utils.aio.debounced` helper closely enough for this task's needs. + */ +class Debounced { + #timer?: ReturnType; + + constructor( + private readonly fn: () => Promise, + private readonly delay: number, + ) {} + + schedule(): void { + this.cancel(); + this.#timer = setTimeout(() => { + this.#timer = undefined; + this.#run(); + }, this.delay); + } + + run(): void { + this.cancel(); + this.#run(); + } + + cancel(): void { + if (this.#timer !== undefined) { + clearTimeout(this.#timer); + this.#timer = undefined; + } + } + + #run(): void { + void this.fn().catch((error) => { + log().error({ error }, 'error running debounced DTMF reply'); + }); + } +} + +const dtmfResultFromInputs = (inputs: readonly DtmfEvent[]): GetDtmfResult => ({ + userInput: formatDtmf(inputs), +}); + +/** + * Build an {@link AgentTask} that collects DTMF inputs from the user. + * + * The task completes with the collected digit string, or fails with a {@link ToolError} + * when the expected number of digits is not received in time. + * + * This is the functional core; {@link GetDtmfTask} is a thin class wrapper over it. + */ +export function createGetDtmfTask({ + numDigits, + askForConfirmation = false, + dtmfInputTimeout = 4000, + dtmfStopEvent = DtmfEvent.POUND, + chatCtx, + extraInstructions, +}: GetDtmfTaskOptions): AgentTask { + if (numDigits <= 0) { + throw new Error('numDigits must be greater than 0'); + } + + const logger = log().child({ component: 'dtmf-inputs' }); + + const currDtmfInputs: DtmfEvent[] = []; + let dtmfReplyRunning = false; + + const confirmInputsTool = tool({ + name: 'confirm_inputs', + description: + 'Finalize the collected digit inputs after explicit user confirmation.\n\n' + + 'Use this ONLY after the confirmation. You should confirm by verbally reading out the digits one by one and, once the ' + + 'user confirms they are correct, call this tool with the inputs.\n\n' + + 'Do not use this tool to capture the initial digits.', + parameters: z.object({ + inputs: z.array(z.nativeEnum(DtmfEvent)).describe('The digit inputs to finalize'), + }), + execute: async ({ inputs }: { inputs: DtmfEvent[] }) => { + task.complete(dtmfResultFromInputs(inputs)); + }, + }); + + const recordInputsTool = tool({ + name: 'record_inputs', + description: + 'Record the collected digit inputs without additional confirmation.\n\n' + + 'Call this tool as soon as a valid sequence of digits has been provided by the user (via DTMF or spoken).', + parameters: z.object({ + inputs: z.array(z.nativeEnum(DtmfEvent)).describe('The digit inputs to record'), + }), + execute: async ({ inputs }: { inputs: DtmfEvent[] }) => { + task.complete(dtmfResultFromInputs(inputs)); + }, + }); + + let instructions = + 'You are a single step in a broader system, responsible solely for gathering digits input from the user. ' + + 'You will either receive a sequence of digits through dtmf events tagged by , or ' + + 'user will directly say the digits to you. You should be able to handle both cases. '; + + if (askForConfirmation) { + instructions += + 'Once user has confirmed the digits (by verbally spoken or entered manually), call `confirm_inputs` with the inputs.'; + } else { + instructions += + 'If user provides the digits through voice and it is valid, call `record_inputs` with the inputs.'; + } + + if (extraInstructions !== undefined) { + instructions += `\n${extraInstructions}`; + } + + const generateDtmfReply = new Debounced(async () => { + dtmfReplyRunning = true; + + try { + task.session.interrupt(); + + const dtmfStr = formatDtmf(currDtmfInputs); + logger.debug(`Generating DTMF reply, current inputs: ${dtmfStr}`); + + // if input not fully received (i.e. timeout), fail the task + if (currDtmfInputs.length !== numDigits) { + const errorMsg = + `Digits input not fully received. ` + + `Expect ${numDigits} digits, got ${currDtmfInputs.length}`; + if (!task.done) { + task.complete(new ToolError(errorMsg)); + } + return; + } + + // if not asking for confirmation, return the DTMF inputs + if (!askForConfirmation) { + if (!task.done) { + task.complete(dtmfResultFromInputs(currDtmfInputs)); + } + return; + } + + const replyInstructions = + 'User has entered the following valid digits on the telephone keypad:\n' + + `${dtmfStr}\n` + + 'Please confirm it with the user by saying the digits one by one with space in between ' + + "(.e.g. 'one two three four five six seven eight nine ten'). " + + 'Once you are sure, call `confirm_inputs` with the inputs.'; + + await task.session.generateReply({ userInput: replyInstructions }); + } finally { + dtmfReplyRunning = false; + currDtmfInputs.length = 0; + } + }, dtmfInputTimeout); + + const onSipDtmfReceived = (_code: number, digit: string): void => { + if (dtmfReplyRunning) { + return; + } + + // immediately kick off the DTMF reply generation if matches the stop event + if (digit === dtmfStopEvent) { + generateDtmfReply.run(); + return; + } + + if (!(Object.values(DtmfEvent) as string[]).includes(digit)) { + logger.warn(`Ignoring invalid DTMF digit: ${digit}`); + return; + } + + currDtmfInputs.push(digit as DtmfEvent); + logger.info(`DTMF inputs: ${formatDtmf(currDtmfInputs)}`); + generateDtmfReply.schedule(); + }; + + const onUserStateChanged = (ev: UserStateChangedEvent): void => { + if (dtmfReplyRunning) { + return; + } + + if (ev.newState === 'speaking') { + // clear any pending DTMF reply generation + generateDtmfReply.cancel(); + } else if (currDtmfInputs.length !== 0) { + // resume any previously cancelled DTMF reply generation after user is back to non-speaking + generateDtmfReply.schedule(); + } + }; + + const onAgentStateChanged = (ev: AgentStateChangedEvent): void => { + if (dtmfReplyRunning) { + return; + } + + if (ev.newState === 'speaking' || ev.newState === 'thinking') { + // clear any pending DTMF reply generation + generateDtmfReply.cancel(); + } else if (currDtmfInputs.length !== 0) { + // resume any previously cancelled DTMF reply generation after agent is back to non-speaking + generateDtmfReply.schedule(); + } + }; + + const task = AgentTask.create({ + id: 'get_dtmf_task', + instructions, + chatCtx, + tools: askForConfirmation ? [confirmInputsTool] : [recordInputsTool], + onEnter: async () => { + const ctx = getJobContext(); + + ctx.room.on(RoomEvent.DtmfReceived, onSipDtmfReceived); + task.session.on(AgentSessionEventTypes.UserStateChanged, onUserStateChanged); + task.session.on(AgentSessionEventTypes.AgentStateChanged, onAgentStateChanged); + task.session.generateReply({ toolChoice: 'none' }); + }, + onExit: async () => { + const ctx = getJobContext(); + + ctx.room.off(RoomEvent.DtmfReceived, onSipDtmfReceived); + task.session.off(AgentSessionEventTypes.UserStateChanged, onUserStateChanged); + task.session.off(AgentSessionEventTypes.AgentStateChanged, onAgentStateChanged); + generateDtmfReply.cancel(); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetDtmfTask}, preserving the + * `new GetDtmfTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetDtmfTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetDtmfTaskOptions) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetDtmfTask. + super({ instructions: '' }); + this.#task = createGetDtmfTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} diff --git a/agents/src/workflows/email_address.ts b/agents/src/workflows/email_address.ts new file mode 100644 index 000000000..60488222b --- /dev/null +++ b/agents/src/workflows/email_address.ts @@ -0,0 +1,238 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; +import { type InstructionParts, resolveWorkflowInstructions } from './utils.js'; + +export interface GetEmailResult { + emailAddress: string; +} + +export interface GetEmailTaskOptions { + /** + * Instructions for the email capture prompt. Pass a full string or {@link Instructions} to + * replace the built-in prompt entirely, or {@link InstructionParts} to override individual + * sections (e.g. `persona`) while keeping the built-in template. + */ + instructions?: InstructionParts | Instructions | string; + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString; + vad?: VAD; + llm?: LLM | RealtimeModel | LLMModels; + tts?: TTS | TTSModelString; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm the captured email address. Defaults to confirming on + * audio input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** + * When true, the model must produce an asking utterance before recording an email address — + * it can't silently fill one from the chat context during `onEnter`. + */ + requireExplicitAsk?: boolean; +} + +/** + * Build an {@link AgentTask} that collects an email address from the user. + * + * This is the functional core; {@link GetEmailTask} is a thin class wrapper over it. + */ +export function createGetEmailTask({ + instructions, + chatCtx, + turnDetection, + tools = [], + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + requireExplicitAsk = false, +}: GetEmailTaskOptions = {}): AgentTask { + let currentEmail = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildConfirmTool = (email: string) => + tool({ + name: 'confirm_email_address', + description: 'Call after the user confirms the email address is correct.', + execute: async () => { + if (email !== currentEmail) { + task.session.generateReply({ + instructions: + 'The email has changed since confirmation was requested, ask the user to confirm the updated email.', + }); + return; + } + + if (!task.done) { + task.complete({ emailAddress: email }); + } + }, + }); + + const updateEmailTool = tool({ + name: 'update_email_address', + description: 'Update the email address provided by the user.', + parameters: z.object({ + email: z.string().describe('The email address provided by the user'), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ({ email }: { email: string }, { ctx }) => { + email = email.trim(); + + if (!EMAIL_REGEX.test(email)) { + throw new ToolError(`Invalid email address provided: ${email}`); + } + + currentEmail = email; + const separatedEmail = email.split('').join(' '); + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete({ emailAddress: currentEmail }); + } + return; // no need to continue the conversation + } + + const confirmTool = buildConfirmTool(email); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_email_address'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + `The email has been updated to ${email}\n` + + `Repeat the email character by character: ${separatedEmail} if needed\n` + + `Prompt the user for confirmation, do not call \`confirm_email_address\` directly` + ); + }, + }); + + const declineTool = tool({ + name: 'decline_email_capture', + description: 'Handles the case when the user explicitly declines to provide an email address.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide the email address'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }) => { + if (!task.done) { + task.complete(new ToolError(`couldn't get the email address: ${reason}`)); + } + }, + }); + + const task = AgentTask.create({ + id: 'get_email_task', + instructions: resolveWorkflowInstructions({ + instructions, + template: INSTRUCTIONS_TEMPLATE, + defaultPersona: PERSONA, + modalitySpecific: new Instructions('', { + audio: AUDIO_SPECIFIC, + text: TEXT_SPECIFIC, + }), + confirmation: new Instructions('', { + // confirmation is enabled by default for audio, disabled by default for text + audio: requireConfirmation !== false ? CONFIRMATION_INSTRUCTION : '', + text: requireConfirmation === true ? CONFIRMATION_INSTRUCTION : '', + }), + }), + chatCtx, + turnDetection, + tools: [...tools, updateEmailTool, declineTool], + stt, + vad, + llm, + tts, + allowInterruptions, + onEnter: async () => { + task.session.generateReply({ instructions: 'Ask the user to provide an email address.' }); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetEmailTask}, preserving the + * `new GetEmailTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetEmailTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetEmailTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetEmailTask. + super({ instructions: '' }); + this.#task = createGetEmailTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +const EMAIL_REGEX = + /^[A-Za-z0-9][A-Za-z0-9._%+\-]*@(?:[A-Za-z0-9](?:[A-Za-z0-9\-]*[A-Za-z0-9])?\.)+[A-Za-z]{2,}$/; + +// instructions +const PERSONA = + 'You are only a single step in a broader system, responsible solely for capturing an email address.'; + +const AUDIO_SPECIFIC = `Handle input as noisy voice transcription. Expect that users will say emails aloud with formats like: +- 'john dot doe at gmail dot com' +- 'susan underscore smith at yahoo dot co dot uk' +- 'dave dash b at protonmail dot com' +- 'jane at example' (partial—prompt for the domain) +- 'theo t h e o at livekit dot io' (name followed by spelling) +Normalize common spoken patterns silently: +- Convert words like 'dot', 'underscore', 'dash', 'plus' into symbols: \`.\`, \`_\`, \`-\`, \`+\`. +- Convert 'at' to \`@\`. +- Recognize patterns where users speak their name or a word, followed by spelling: e.g., 'john j o h n'. +- Filter out filler words or hesitations. +- Assume some spelling if contextually obvious (e.g. 'mike b two two' → mikeb22). +Don't mention corrections. Treat inputs as possibly imperfect but fix them silently.`; + +const TEXT_SPECIFIC = `Handle input as typed text. Expect users to type their email address directly in standard format. +If the address looks almost correct but has minor typos (e.g. missing '@' or domain), prompt for clarification.`; + +const CONFIRMATION_INSTRUCTION = `Call \`confirm_email_address\` after the user confirmed the email address is correct.`; + +const INSTRUCTIONS_TEMPLATE = `{persona} + +{modalitySpecific} + +Call \`update_email_address\` at the first opportunity whenever you form a new hypothesis about the email. (before asking any questions or providing any answers.) +Don't invent new email addresses, stick strictly to what the user said. +{confirmation} +If the email is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts: first the part before the '@', then the domain—only if needed. + +Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. +Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called. + +{extra} +`; diff --git a/agents/src/workflows/index.ts b/agents/src/workflows/index.ts index 99c334d57..48cdca1d6 100644 --- a/agents/src/workflows/index.ts +++ b/agents/src/workflows/index.ts @@ -1,6 +1,52 @@ // SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +export { + GetAddressTask, + createGetAddressTask, + type GetAddressResult, + type GetAddressTaskOptions, +} from './address.js'; +export { + CardCaptureDeclinedError, + CardCollectionRestartError, + GetCreditCardTask, + createGetCreditCardTask, + type GetCreditCardResult, + type GetCreditCardTaskOptions, +} from './credit_card.js'; +export { + GetDOBTask, + createGetDOBTask, + type DateOfBirth, + type GetDOBResult, + type GetDOBTaskOptions, + type TimeOfBirth, +} from './dob.js'; +export { + GetDtmfTask, + createGetDtmfTask, + type GetDtmfResult, + type GetDtmfTaskOptions, +} from './dtmf_inputs.js'; +export { + GetEmailTask, + createGetEmailTask, + type GetEmailResult, + type GetEmailTaskOptions, +} from './email_address.js'; +export { + GetNameTask, + createGetNameTask, + type GetNameResult, + type GetNameTaskOptions, +} from './name.js'; +export { + GetPhoneNumberTask, + createGetPhoneNumberTask, + type GetPhoneNumberResult, + type GetPhoneNumberTaskOptions, +} from './phone_number.js'; export { TaskGroup, type TaskCompletedEvent, @@ -13,4 +59,4 @@ export { type WarmTransferResult, type WarmTransferTaskOptions, } from './warm_transfer.js'; -export type { InstructionParts } from './utils.js'; +export { DtmfEvent, dtmfEventToCode, formatDtmf, type InstructionParts } from './utils.js'; diff --git a/agents/src/workflows/name.ts b/agents/src/workflows/name.ts new file mode 100644 index 000000000..0edfe56a9 --- /dev/null +++ b/agents/src/workflows/name.ts @@ -0,0 +1,394 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import { safeRender } from '../utils.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; + +export interface GetNameResult { + firstName?: string; + middleName?: string; + lastName?: string; +} + +export interface GetNameTaskOptions { + /** Collect the user's first name. Defaults to true. */ + firstName?: boolean; + /** Collect the user's last name. Defaults to false. */ + lastName?: boolean; + /** Collect the user's middle name. Defaults to false. */ + middleName?: boolean; + /** + * Order in which to collect the name parts, using `{firstName}`, `{middleName}` and + * `{lastName}` placeholders. Defaults to the enabled parts in first/middle/last order. + */ + nameFormat?: string; + /** Ask the user to verify the spelling of the captured name. Defaults to false. */ + verifySpelling?: boolean; + /** Extra instructions appended to the built-in prompt for domain-specific context. */ + extraInstructions?: string; + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString; + vad?: VAD; + llm?: LLM | RealtimeModel | LLMModels; + tts?: TTS | TTSModelString; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm the captured name. Defaults to confirming on audio + * input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** + * When true, the model must produce an asking utterance before recording a name — it can't + * silently fill one from the chat context during `onEnter`. + */ + requireExplicitAsk?: boolean; +} + +interface UpdateNameArgs { + firstName?: string | null; + middleName?: string | null; + lastName?: string | null; +} + +function cleanNameArg(value: string | null | undefined): string | undefined { + // Some models (e.g. gemma) fill optional args with placeholder strings like + // "null"/"NULL" instead of omitting them, or wrap values in literal quotes. + // Normalize those to undefined/clean values so they hit the required-field + // validation below instead of being recorded as the user's name. + if (value == null) { + return undefined; + } + const cleaned = value + .trim() + .replace(/^['"]+/, '') + .replace(/['"]+$/, ''); + if ( + !cleaned || + ['null', 'none', 'nil', 'n/a', 'unknown', 'unspecified'].includes(cleaned.toLowerCase()) + ) { + return undefined; + } + return cleaned; +} + +function renderNameFormat( + nameFormat: string, + parts: { firstName: string; middleName: string; lastName: string }, +): string { + const replacements: Record = { + firstName: parts.firstName, + middleName: parts.middleName, + lastName: parts.lastName, + }; + return nameFormat + .replace(/\{(firstName|middleName|lastName)\}/g, (_match, key: string) => { + return replacements[key] ?? ''; + }) + .trim(); +} + +/** + * Build an {@link AgentTask} that collects the user's name. + * + * This is the functional core; {@link GetNameTask} is a thin class wrapper over it. + */ +export function createGetNameTask({ + firstName = true, + lastName = false, + middleName = false, + nameFormat, + verifySpelling = false, + extraInstructions = '', + chatCtx, + turnDetection, + tools = [], + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + requireExplicitAsk = false, +}: GetNameTaskOptions = {}): AgentTask { + if (!(firstName || middleName || lastName)) { + throw new Error('At least one of firstName, middleName, or lastName must be true'); + } + + let resolvedNameFormat: string; + if (nameFormat !== undefined) { + resolvedNameFormat = nameFormat; + } else { + const parts: string[] = []; + if (firstName) parts.push('{firstName}'); + if (middleName) parts.push('{middleName}'); + if (lastName) parts.push('{lastName}'); + resolvedNameFormat = parts.join(' '); + } + + const spellingInstructions = !verifySpelling + ? '' + : 'After receiving the name, always verify the spelling by asking the user to confirm ' + + 'or spell out the name letter by letter. ' + + 'When confirming, spell out each name part letter by letter to the user. '; + const confirmationInstructions = + 'Call `confirm_name` after the user confirmed the name is correct.'; + + let currentFirstName = ''; + let currentMiddleName = ''; + let currentLastName = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildResult = (): GetNameResult => ({ + firstName: firstName ? currentFirstName : undefined, + middleName: middleName ? currentMiddleName : undefined, + lastName: lastName ? currentLastName : undefined, + }); + + const buildConfirmTool = (captured: { first: string; middle: string; last: string }) => + tool({ + name: 'confirm_name', + description: 'Call after the user confirms the name is correct.', + execute: async () => { + if ( + captured.first !== currentFirstName || + captured.middle !== currentMiddleName || + captured.last !== currentLastName + ) { + task.session.generateReply({ + instructions: + 'The name has changed since confirmation was requested, ask the user to confirm the updated name.', + }); + return; + } + + if (!task.done) { + task.complete(buildResult()); + } + }, + }); + + const updateNameTool = tool({ + name: 'update_name', + description: 'Update the name provided by the user.', + parameters: z.object({ + firstName: z.string().nullable().optional().describe("The user's first name."), + middleName: z + .string() + .nullable() + .optional() + .describe("The user's middle name, if collected."), + lastName: z.string().nullable().optional().describe("The user's last name, if collected."), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async (args: UpdateNameArgs, { ctx }) => { + const cleanedFirst = cleanNameArg(args.firstName); + const cleanedMiddle = cleanNameArg(args.middleName); + const cleanedLast = cleanNameArg(args.lastName); + + const errors: string[] = []; + if (firstName && !cleanedFirst?.trim()) { + errors.push('first name is required but was not provided'); + } + if (middleName && !cleanedMiddle?.trim()) { + errors.push('middle name is required but was not provided'); + } + if (lastName && !cleanedLast?.trim()) { + errors.push('last name is required but was not provided'); + } + + // A real name contains letters. Reject digit-only or punctuation-only + // values so a card number, ZIP code, phone number, etc. accidentally + // crammed into update_name fails fast instead of being recorded as + // the user's name. + for (const [label, value] of [ + ['first', cleanedFirst], + ['middle', cleanedMiddle], + ['last', cleanedLast], + ] as const) { + if (value && value.trim() && !/\p{L}/u.test(value)) { + errors.push( + `${label} name '${value}' contains no letters - that doesn't look like a name`, + ); + } + } + + if (errors.length > 0) { + throw new ToolError(`Incomplete name: ${errors.join('; ')}`); + } + + currentFirstName = cleanedFirst?.trim() ?? ''; + currentMiddleName = cleanedMiddle?.trim() ?? ''; + currentLastName = cleanedLast?.trim() ?? ''; + + const fullName = renderNameFormat(resolvedNameFormat, { + firstName: currentFirstName, + middleName: currentMiddleName, + lastName: currentLastName, + }); + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete(buildResult()); + } + return; + } + + const confirmTool = buildConfirmTool({ + first: currentFirstName, + middle: currentMiddleName, + last: currentLastName, + }); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_name'); + await task.updateTools([...currentTools, confirmTool]); + + if (verifySpelling) { + return ( + `The name has been updated to ${fullName}\n` + + `Spell out the name letter by letter for verification: ${fullName}\n` + + `Prompt the user for confirmation, do not call \`confirm_name\` directly` + ); + } + + return ( + `The name has been updated to ${fullName}\n` + + `Repeat the name back to the user and prompt for confirmation, ` + + `do not call \`confirm_name\` directly` + ); + }, + }); + + const declineTool = tool({ + name: 'decline_name_capture', + description: 'Handles the case when the user explicitly declines to provide their name.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide their name'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }) => { + if (!task.done) { + task.complete(new ToolError(`couldn't get the name: ${reason}`)); + } + }, + }); + + const task = AgentTask.create({ + id: 'get_name_task', + instructions: new Instructions('', { + audio: safeRender(BASE_INSTRUCTIONS, { + nameFormat: resolvedNameFormat, + modalitySpecific: AUDIO_SPECIFIC, + spellingInstructions, + confirmationInstructions: requireConfirmation !== false ? confirmationInstructions : '', + extraInstructions, + }), + text: safeRender(BASE_INSTRUCTIONS, { + nameFormat: resolvedNameFormat, + modalitySpecific: TEXT_SPECIFIC, + spellingInstructions, + confirmationInstructions: requireConfirmation === true ? confirmationInstructions : '', + extraInstructions, + }), + }), + chatCtx, + turnDetection, + tools: [...tools, updateNameTool, declineTool], + stt, + vad, + llm, + tts, + allowInterruptions, + onEnter: async () => { + task.session.generateReply({ + instructions: + `Get the user's name (follow this order '${resolvedNameFormat}' but do not ` + + 'mention the format). First scan the conversation - if a name was already ' + + 'given earlier, ask a short confirmation question rather than asking from ' + + 'scratch. If context about what the name is FOR was provided (a role like ' + + "'cardholder', 'guest', 'emergency contact'), anchor your confirmation " + + "question to that role so the user knows which name you mean - don't ask " + + 'abstractly. When pointing at where an existing name came from, reference ' + + 'the source in the conversation (the earlier step, the booking they ' + + 'mentioned), not a presumption about how the name appears in the ' + + 'destination. Only ask fresh when the conversation has no name yet.', + }); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetNameTask}, preserving the + * `new GetNameTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetNameTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetNameTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetNameTask. + super({ instructions: '' }); + this.#task = createGetNameTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +const BASE_INSTRUCTIONS = ` +You are only a single step in a broader system, responsible solely for capturing the user's name. +You need to naturally collect the name parts in this order: {nameFormat}. +{modalitySpecific} +{spellingInstructions}Call \`update_name\` at the first opportunity whenever you form a new hypothesis about the name. (before asking any questions or providing any answers.) +Don't invent names, stick strictly to what the user said. +{confirmationInstructions} +If the name is unclear or it takes too much back-and-forth, prompt for each name part separately. +Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. +Avoid verbosity by not sharing example names or spellings unless prompted to do so. Do not deviate from the goal of collecting the user's name. +Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called.\ +{extraInstructions} +`; + +const AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect that users will say names aloud and may: +- Say their name followed by spelling: e.g., 'Michael m i c h a e l' +- Use phonetic alphabet: e.g., 'Mike as in Mike India Charlie Hotel Alpha Echo Lima' +- Have names with special characters or hyphens: e.g., 'Mary-Jane' or 'O'Brien' +- Have names from various cultural backgrounds with different pronunciation patterns +Normalize common spoken patterns silently: +- Convert 'dash' or 'hyphen' to \`-\`. +- Convert 'apostrophe' to \`'\`. +- Recognize when users spell out their name letter by letter. +- Filter out filler words or hesitations. +- Capitalize the first letter of each name part appropriately. +Don't mention corrections. Treat inputs as possibly imperfect but fix them silently. +`; + +const TEXT_SPECIFIC = ` +Handle input as typed text. Expect users to type their name directly. +Capitalize the first letter of each name part appropriately. +If the name contains special characters or hyphens (e.g., 'Mary-Jane' or 'O'Brien'), preserve them as typed. +`; diff --git a/agents/src/workflows/phone_number.ts b/agents/src/workflows/phone_number.ts new file mode 100644 index 000000000..306f3d87d --- /dev/null +++ b/agents/src/workflows/phone_number.ts @@ -0,0 +1,233 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import { safeRender } from '../utils.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; + +const PHONE_REGEX = /^\+?[1-9]\d{6,14}$/; + +export interface GetPhoneNumberResult { + phoneNumber: string; +} + +export interface GetPhoneNumberTaskOptions { + /** Extra instructions appended to the built-in prompt for domain-specific context. */ + extraInstructions?: string; + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString; + vad?: VAD; + llm?: LLM | RealtimeModel | LLMModels; + tts?: TTS | TTSModelString; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm the captured phone number. Defaults to confirming on + * audio input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** + * When true, the model must produce an asking utterance before recording a phone number — it + * can't silently fill one from the chat context during `onEnter`. + */ + requireExplicitAsk?: boolean; +} + +/** + * Build an {@link AgentTask} that collects a phone number from the user. + * + * This is the functional core; {@link GetPhoneNumberTask} is a thin class wrapper over it. + */ +export function createGetPhoneNumberTask({ + extraInstructions = '', + chatCtx, + turnDetection, + tools = [], + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + requireExplicitAsk = false, +}: GetPhoneNumberTaskOptions = {}): AgentTask { + const confirmationInstructions = + 'Call `confirm_phone_number` after the user confirmed the phone number is correct.'; + + let currentPhoneNumber = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildConfirmTool = (phoneNumber: string) => + tool({ + name: 'confirm_phone_number', + description: 'Call after the user confirms the phone number is correct.', + execute: async () => { + if (phoneNumber !== currentPhoneNumber) { + task.session.generateReply({ + instructions: + 'The phone number has changed since confirmation was requested, ask the user to confirm the updated number.', + }); + return; + } + + if (!task.done) { + task.complete({ phoneNumber }); + } + }, + }); + + const updatePhoneNumberTool = tool({ + name: 'update_phone_number', + description: 'Update the phone number provided by the user.', + parameters: z.object({ + phoneNumber: z + .string() + .describe('The phone number provided by the user, digits only with optional leading +'), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ({ phoneNumber }: { phoneNumber: string }, { ctx }) => { + const cleaned = phoneNumber.trim().replace(/[\s\-().]+/g, ''); + + if (!PHONE_REGEX.test(cleaned)) { + throw new ToolError(`Invalid phone number provided: ${phoneNumber}`); + } + + currentPhoneNumber = cleaned; + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete({ phoneNumber: currentPhoneNumber }); + } + return; // no need to continue the conversation + } + + const confirmTool = buildConfirmTool(cleaned); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_phone_number'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + `The phone number has been updated to ${cleaned}\n` + + `Read the number back to the user in groups.\n` + + `Prompt the user for confirmation, do not call \`confirm_phone_number\` directly` + ); + }, + }); + + const declineTool = tool({ + name: 'decline_phone_number_capture', + description: 'Handles the case when the user explicitly declines to provide a phone number.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide the phone number'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }) => { + if (!task.done) { + task.complete(new ToolError(`couldn't get the phone number: ${reason}`)); + } + }, + }); + + const task = AgentTask.create({ + id: 'get_phone_number_task', + instructions: new Instructions('', { + audio: safeRender(BASE_INSTRUCTIONS, { + modalitySpecific: AUDIO_SPECIFIC, + confirmationInstructions: requireConfirmation !== false ? confirmationInstructions : '', + extraInstructions, + }), + text: safeRender(BASE_INSTRUCTIONS, { + modalitySpecific: TEXT_SPECIFIC, + confirmationInstructions: requireConfirmation === true ? confirmationInstructions : '', + extraInstructions, + }), + }), + chatCtx, + turnDetection, + tools: [...tools, updatePhoneNumberTool, declineTool], + stt, + vad, + llm, + tts, + allowInterruptions, + onEnter: async () => { + task.session.generateReply({ instructions: 'Ask the user to provide their phone number.' }); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetPhoneNumberTask}, preserving the + * `new GetPhoneNumberTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetPhoneNumberTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetPhoneNumberTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetPhoneNumberTask. + super({ instructions: '' }); + this.#task = createGetPhoneNumberTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +const BASE_INSTRUCTIONS = ` +You are only a single step in a broader system, responsible solely for capturing a phone number. +{modalitySpecific} +Call \`update_phone_number\` at the first opportunity whenever you form a new hypothesis about the phone number. (before asking any questions or providing any answers.) +Don't invent phone numbers, stick strictly to what the user said. +{confirmationInstructions} +If the number is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts: first the area code, then the remaining digits. +Never repeat the phone number back to the user as a single block of digits. Read it back in groups. +Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. +Avoid verbosity by not sharing example phone numbers or formats unless prompted to do so. Do not deviate from the goal of collecting the user's phone number. +Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called.\ +{extraInstructions} +`; + +const AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect that users will say phone numbers aloud with formats like: +- '555 123 4567' +- 'five five five, one two three, four five six seven' +- '+1 555 123 4567' +- 'area code 555, 123 4567' +- '555-123-4567' +Normalize common spoken patterns silently: +- Convert spoken digits to their numeric form: 'five' → 5, 'zero' → 0, 'oh' → 0. +- Remove filler words, pauses, and hesitations. +- Strip dashes, spaces, parentheses, and dots from the number. +- Recognize 'plus' at the start as the international prefix \`+\`. +- Recognize 'area code' as a prefix for the area code digits. +Don't mention corrections. Treat inputs as possibly imperfect but fix them silently. +`; + +const TEXT_SPECIFIC = ` +Handle input as typed text. Expect users to type their phone number directly. +Strip dashes, spaces, parentheses, and dots from the number. +If the number looks almost correct but has minor formatting issues, clean it up silently. +`; diff --git a/agents/src/workflows/task_group.ts b/agents/src/workflows/task_group.ts index 30a501987..a733c0240 100644 --- a/agents/src/workflows/task_group.ts +++ b/agents/src/workflows/task_group.ts @@ -8,7 +8,8 @@ import { asError } from '../utils.js'; import { AgentTask } from '../voice/agent.js'; interface FactoryInfo { - taskFactory: () => AgentTask; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts tasks with any result type + taskFactory: () => AgentTask; id: string; description: string; } @@ -64,7 +65,8 @@ export class TaskGroup extends AgentTask { this._taskCompletedCallback = onTaskCompleted; } - add(task: () => AgentTask, { id, description }: { id: string; description: string }): this { + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts tasks with any result type + add(task: () => AgentTask, { id, description }: { id: string; description: string }): this { this._registeredFactories.set(id, { taskFactory: task, id, description }); return this; } diff --git a/agents/src/workflows/utils.ts b/agents/src/workflows/utils.ts index 25ba05f30..2d435d00a 100644 --- a/agents/src/workflows/utils.ts +++ b/agents/src/workflows/utils.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import type { Instructions } from '../llm/index.js'; +import { Instructions, isInstructions } from '../llm/chat_context.js'; /** * Customizable instruction sections for built-in workflow tasks. @@ -16,3 +16,74 @@ export interface InstructionParts { /** Extra instructions appended to the prompt for domain-specific context. */ extra?: Instructions | string; } + +/** + * Resolve workflow instructions the way the Python `WorkflowInstructions` class does: + * a full string or {@link Instructions} replaces the built-in prompt entirely, while + * {@link InstructionParts} (or nothing) fills the workflow's built-in template. + * + * @internal + */ +export function resolveWorkflowInstructions(options: { + instructions?: InstructionParts | Instructions | string; + template: string; + defaultPersona: string; + /** Fills the template's `{modalitySpecific}` slot with per-modality (audio/text) guidance. */ + modalitySpecific: Instructions; + /** Fills the template's `{confirmation}` slot with the per-modality confirmation instruction. */ + confirmation: Instructions; +}): string | Instructions { + const { instructions, template, defaultPersona, modalitySpecific, confirmation } = options; + + // A full instruction string or Instructions replaces the built-in prompt entirely. + if (typeof instructions === 'string' || isInstructions(instructions)) { + return instructions; + } + + // No instructions or an `InstructionParts` override: fill the built-in template. + // Unset preserves the built-in default; an explicit empty string removes the section. + const parts: InstructionParts = instructions ?? {}; + return Instructions.resolveTemplate(template, { + persona: parts.persona ?? defaultPersona, + extra: parts.extra ?? '', + modalitySpecific, + confirmation, + }); +} + +export enum DtmfEvent { + ONE = '1', + TWO = '2', + THREE = '3', + FOUR = '4', + FIVE = '5', + SIX = '6', + SEVEN = '7', + EIGHT = '8', + NINE = '9', + ZERO = '0', + STAR = '*', + POUND = '#', + A = 'A', + B = 'B', + C = 'C', + D = 'D', +} + +export function dtmfEventToCode(event: DtmfEvent): number { + if (/^\d$/.test(event)) { + return Number(event); + } else if (event === DtmfEvent.STAR) { + return 10; + } else if (event === DtmfEvent.POUND) { + return 11; + } else if (['A', 'B', 'C', 'D'].includes(event)) { + // DTMF codes 12-15 are used for letters A-D + return event.charCodeAt(0) - 'A'.charCodeAt(0) + 12; + } + throw new Error(`Invalid DTMF event: ${event}`); +} + +export function formatDtmf(events: readonly DtmfEvent[]): string { + return events.join(' '); +} diff --git a/agents/src/workflows/warm_transfer.ts b/agents/src/workflows/warm_transfer.ts index 8a94426f1..cf4141965 100644 --- a/agents/src/workflows/warm_transfer.ts +++ b/agents/src/workflows/warm_transfer.ts @@ -492,7 +492,7 @@ export class WarmTransferTask extends AgentTask { } const renderInstructionPart = (value: Instructions | string): string => - typeof value === 'string' ? value : value.value; + typeof value === 'string' ? value : value.toString(); function resolveInstructions( instructions: InstructionParts | string | undefined, diff --git a/examples/src/drive-thru/database.ts b/examples/src/drive-thru/database.ts index e6813f993..2b13ce98e 100644 --- a/examples/src/drive-thru/database.ts +++ b/examples/src/drive-thru/database.ts @@ -2,44 +2,46 @@ // // SPDX-License-Identifier: Apache-2.0 -export const COMMON_INSTRUCTIONS = `You are Kelly, a quick and friendly McDonald's drive-thru attendant. -Your job is to guide the customer smoothly through their order, speaking in short, natural voice responses. -This is a voice interaction-assume the customer just pulled up and is speaking to you through a drive-thru speaker. -Respond like you're hearing them, not reading text. -Assume they want food, even if they don't start with a clear request, and help them get what they're looking for. - -If an item comes in different sizes, always ask for the size unless the customer already gave one. -If a customer orders a 'large meal', automatically assume both the fries and the drink should be large. -Do not ask again to confirm the size of the drink or fries. This inference is meant to streamline the interaction. -If the customer clearly indicates a different size for the fries or drink, respect their preference. - -Be fast-keep responses short and snappy. -Sound human-sprinkle in light vocal pauses like 'Mmh…', 'Let me see…', or 'Alright…' at natural moments-but not too often. -Keep everything upbeat and easy to follow. Never overwhelm the customer, don't ask multiple questions at the same time. - -When a customer is confused or asks for something that doesn't exist, let them know politely and suggest something close. -Always confirm what they picked in a warm, clear way, like: 'Alright, one Big Mac Combo!' -If something's unavailable, say so with empathy: 'Ah, we're out of Sweet Tea right now-can I get you a Coke instead?' - -Whenever a customer asks for, changes, or removes something from their order, you MUST use a tool to make it happen. -Don't fake it. Don't pretend something was added - actually **call** the tool and make it real on the ordering system. - -Transcripts often contain speech-to-text errors-don't mention the transcript, don't repeat its mistakes. -Instead treat each user input as a rough draft of what was said. -If you can guess the user's intent and it's safe to do so, infer their meaning and respond naturally. -If the transcript is ambiguous/nonsense and you can't guess their intent, ask the customer to repeat again. -Stay on-topic; if input is nonsensical in a drive-thru context, ask for concise clarification. - -Do not add any item on the user's behalf unless they specifically request it. If the user hasn't asked for an item, NEVER add it. - -When a customer changes an item or meal, make sure to remove the previous version before adding the new one. -Otherwise, the order may contain duplicates. - -Strictly stick to the defined menu, Do not invent or suggest any new sizes or items. -If the item specified by the user is unclear or not **exactly** on the menu, ask for clarification or say you don't have this specific item -E.g: an hamburger isn't a cheeseburger -Do not ask for size unless the item has more than one size option specified. -If an item does not require a size according to the menu, **NEVER** ask the customer to choose one or mention size at all.`; +export const COMMON_INSTRUCTIONS = + // Outcome — what a great interaction looks like. + `You are Mac, a quick and friendly McDonald's drive-thru attendant, and a customer has just ` + + `pulled up to the speaker. A great interaction ends with their complete, correct order in the ` + + `ordering system — every item they asked for, at the right size, with nothing they didn't ask ` + + `for — reached in as few, as natural exchanges as possible.\n\n` + + // Voice & personality — keep it short and human. + `Your output is synthesized directly to speech, so produce a natural verbatim transcript, not ` + + `polished text. Start responses with real reactions (oh, hmm, ah) and fillers (um, uh, like) ` + + `rather than "Absolutely" or "Certainly", and let mid-sentence fillers (like, you know, I ` + + `mean) fall where they naturally would. Use informal phrasing: yeah, gonna, kinda, gotcha, ` + + `lemme. Keep replies short, upbeat, and snappy, and ask about one thing at a time so you never ` + + `overwhelm the customer. Confirm choices warmly ('Alright, one Big Mac Combo!'), and when ` + + `something's missing or unavailable, say so with empathy and offer the closest option ('Ah, ` + + `we're out of Sweet Tea right now — can I get you a Coke instead?').\n\n` + + // How to work — infer intent, acknowledge before acting, stop when you have enough. + `Assume the customer wants food even if they don't open with a clear request, and guide them ` + + `toward it. Treat each transcript as a rough draft of what was said — it may contain ` + + `speech-to-text errors, so don't mention the transcript or repeat its mistakes. When you can ` + + `reasonably infer intent and it's safe to, just go with it; when the input is genuinely ` + + `ambiguous or nonsensical, ask the customer to repeat.\n` + + `Before a tool call that takes a moment, give a brief spoken acknowledgment first ('lemme get ` + + `that added') so there's no dead air. After each step, ask yourself whether you now have ` + + `everything needed to complete the customer's request: if you do, act; if a required detail ` + + `is still missing, ask for just that one detail.\n\n` + + // Hard constraints — these are invariants, not judgment calls. + `Constraints that always hold:\n` + + `- Stick strictly to the defined menu. Never invent items or sizes. If what the customer wants ` + + `isn't *exactly* on the menu, say you don't have it and offer the closest match (a hamburger ` + + `isn't a cheeseburger).\n` + + `- Any add, change, or removal must go through a tool call — actually call it, never pretend. ` + + `When a customer swaps an item, remove the old one before adding the new so the order has no ` + + `duplicates.\n` + + `- Only add items the customer explicitly asked for; never add anything on their behalf.\n` + + `- Don't assume unstated details — especially the drink in a combo. If a required detail is ` + + `missing, ask before calling the tool.\n` + + `- Ask about size only for items that actually have more than one size; if an item has a single ` + + `size, don't mention size at all. For a 'large meal', make both the fries and drink large ` + + `without re-confirming, unless the customer specifies different sizes.\n` + + `- If a tool returns an error, tell the customer and ask them to try again.`; export type ItemSize = 'S' | 'M' | 'L'; export type ItemCategory = 'drink' | 'combo_meal' | 'happy_meal' | 'regular' | 'sauce'; diff --git a/examples/src/drive-thru/drivethru_agent.ts b/examples/src/drive-thru/drivethru_agent.ts index 838f5d104..b5b82f9d5 100644 --- a/examples/src/drive-thru/drivethru_agent.ts +++ b/examples/src/drive-thru/drivethru_agent.ts @@ -1,10 +1,16 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import { type JobContext, ServerOptions, cli, defineAgent, llm, voice } from '@livekit/agents'; -import * as deepgram from '@livekit/agents-plugin-deepgram'; -import * as elevenlabs from '@livekit/agents-plugin-elevenlabs'; -import * as openai from '@livekit/agents-plugin-openai'; +import { + type JobContext, + ServerOptions, + cli, + defineAgent, + inference, + llm, + log, + voice, +} from '@livekit/agents'; import { fileURLToPath } from 'node:url'; import { z } from 'zod'; import { @@ -13,13 +19,13 @@ import { type MenuItem, findItemsById, menuInstructions, -} from './database.js'; +} from './database.ts'; import { OrderState, createOrderedCombo, createOrderedHappy, createOrderedRegular, -} from './order.js'; +} from './order.ts'; export interface UserData { order: OrderState; @@ -375,13 +381,46 @@ export default defineAgent({ const userdata = await newUserData(); const session = new voice.AgentSession({ - stt: new deepgram.STT(), - llm: new openai.LLM({ model: 'gpt-4.1', temperature: 0.45 }), - tts: new elevenlabs.TTS(), + stt: new inference.STT({ model: 'deepgram/nova-3' }), + llm: new inference.LLM({ model: 'google/gemma-4-31b-it' }), + tts: new inference.TTS({ + model: 'inworld/inworld-tts-2', + voice: 'Sarah', + modelOptions: { delivery_mode: 'CREATIVE', speaking_rate: 1.1 }, + }), + expressive: voice.presets.CUSTOMER_SERVICE, userData: userdata, - voiceOptions: { - maxToolSteps: 10, - }, + maxToolSteps: 10, + // Flip userState to "away" after 10s of mutual silence so we can + // check whether they're still there (default is 15s). + userAwayTimeout: 10.0, + }); + + const logger = log(); + let idleNudge: AbortController | null = null; + + const nudgeWhileIdle = async (signal: AbortSignal) => { + // Nudge every 10s until the user speaks again — speaking flips + // userState out of "away", which aborts this loop below. + while (!signal.aborted) { + logger.info("user idle — checking if they're still there"); + await session.generateReply({ + instructions: "The user has been idle, see if they're still there", + }); + await new Promise((resolve) => setTimeout(resolve, 10_000)); + } + }; + + session.on(voice.AgentSessionEventTypes.UserStateChanged, (ev) => { + if (ev.newState === 'away') { + if (idleNudge === null) { + idleNudge = new AbortController(); + void nudgeWhileIdle(idleNudge.signal); + } + } else if (idleNudge !== null) { + idleNudge.abort(); + idleNudge = null; + } }); await session.start({ diff --git a/examples/src/drive-thru/order.ts b/examples/src/drive-thru/order.ts index 73c85b1ec..e23f82828 100644 --- a/examples/src/drive-thru/order.ts +++ b/examples/src/drive-thru/order.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import type { ItemSize } from './database.js'; +import type { ItemSize } from './database.ts'; export function orderUid(): string { const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; diff --git a/examples/src/drive-thru/test_agent.test.ts b/examples/src/drive-thru/test_agent.test.ts index 7eae74f8e..a19e7298e 100644 --- a/examples/src/drive-thru/test_agent.test.ts +++ b/examples/src/drive-thru/test_agent.test.ts @@ -20,7 +20,7 @@ import { initializeLogger, llm, voice } from '@livekit/agents'; import * as openai from '@livekit/agents-plugin-openai'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { DriveThruAgent, type UserData, newUserData } from './drivethru_agent.js'; +import { DriveThruAgent, type UserData, newUserData } from './drivethru_agent.ts'; initializeLogger({ pretty: false, level: 'warn' }); diff --git a/examples/src/frontdesk/calendar_api.test.ts b/examples/src/frontdesk/calendar_api.test.ts index 4e7a08fd2..2ae071f61 100644 --- a/examples/src/frontdesk/calendar_api.test.ts +++ b/examples/src/frontdesk/calendar_api.test.ts @@ -8,7 +8,7 @@ import { SlotUnavailableError, createAvailableSlot, getUniqueHash, -} from './calendar_api.js'; +} from './calendar_api.ts'; describe('Calendar API', () => { describe('createAvailableSlot', () => { diff --git a/examples/src/frontdesk/calendar_integration.test.ts b/examples/src/frontdesk/calendar_integration.test.ts index cc273eaba..692692314 100644 --- a/examples/src/frontdesk/calendar_integration.test.ts +++ b/examples/src/frontdesk/calendar_integration.test.ts @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from 'vitest'; -import { CalComCalendar } from './calendar_api.js'; +import { CalComCalendar } from './calendar_api.ts'; describe('Calendar Integration Tests', () => { describe('CalComCalendar with real API', () => { diff --git a/examples/src/frontdesk/frontdesk_agent.ts b/examples/src/frontdesk/frontdesk_agent.ts index b7a7c9bff..d2d199199 100644 --- a/examples/src/frontdesk/frontdesk_agent.ts +++ b/examples/src/frontdesk/frontdesk_agent.ts @@ -1,10 +1,16 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import { type JobContext, ServerOptions, cli, defineAgent, llm, voice } from '@livekit/agents'; -import * as deepgram from '@livekit/agents-plugin-deepgram'; -import * as elevenlabs from '@livekit/agents-plugin-elevenlabs'; -import * as openai from '@livekit/agents-plugin-openai'; +import { + type JobContext, + ServerOptions, + cli, + defineAgent, + inference, + llm, + log, + voice, +} from '@livekit/agents'; import { BackgroundVoiceCancellation } from '@livekit/noise-cancellation-node'; import { fileURLToPath } from 'node:url'; import { z } from 'zod'; @@ -15,7 +21,7 @@ import { FakeCalendar, SlotUnavailableError, getUniqueHash, -} from './calendar_api.js'; +} from './calendar_api.ts'; export interface Userdata { cal: Calendar; @@ -35,17 +41,35 @@ export class FrontDeskAgent extends voice.Agent { }); const instructions = - `You are Front-Desk, a helpful and efficient voice assistant. ` + - `Today is ${today}. Your main goal is to schedule an appointment for the user. ` + - `This is a voice conversation — speak naturally, clearly, and concisely. ` + - `When the user says hello or greets you, don't just respond with a greeting — use it as an opportunity to move things forward. ` + - `For example, follow up with a helpful question like: 'Would you like to book a time?' ` + - `When asked for availability, call list_available_slots and offer a few clear, simple options. ` + - `Say things like 'Monday at 2 PM' — avoid timezones, timestamps, and avoid saying 'AM' or 'PM'. ` + - `Use natural phrases like 'in the morning' or 'in the evening', and don't mention the year unless it's different from the current one. ` + - `Offer a few options at a time, pause for a response, then guide the user to confirm. ` + - `If the time is no longer available, let them know gently and offer the next options. ` + - `Always keep the conversation flowing — be proactive, human, and focused on helping the user schedule with ease.`; + // Outcome — what a great interaction looks like. + `You are Front-Desk, a helpful and efficient voice assistant. Today is ${today}. ` + + `A great interaction ends with the user booked into an appointment slot that works ` + + `for them, reached through a warm, flowing conversation with as little ` + + `back-and-forth as possible. ` + + // Voice & personality — keep it short and human. + `Your output is synthesized directly to speech, so produce a natural verbatim ` + + `transcript, not polished text. Start responses with real reactions (oh, hmm, ah) ` + + `and fillers (um, uh, like) rather than "Absolutely" or "Certainly", with ` + + `mid-sentence fillers (like, you know, I mean) where they'd naturally fall. Mirror ` + + `the user's formality: if they're casual, use informal phrasing (gotcha, alright, ` + + `gonna, kinda, lemme, yeah); if they're more formal, keep your speech cleaner. Vary ` + + `your openers across turns — if you opened the last turn with 'gotcha', pick ` + + `'alright' or 'okay' this turn; don't repeat the same opener back-to-back. ` + + // How to work — be proactive, acknowledge before acting, stop when you can move forward. + `Be proactive: when the user greets you, use it to move things forward (e.g. ` + + `'Would you like to book a time?') rather than just greeting back. Before a tool ` + + `call that takes a moment, give a brief spoken acknowledgment so there's no dead ` + + `air. After each result, check whether you can now move the user toward a booking: ` + + `if so, do it; if you're missing something, ask for just that. ` + + // Speaking about times — constraints that keep it natural over voice. + `When talking about availability, call list_available_slots and offer a few clear ` + + `options at a time, then pause for a response and guide the user to confirm. Say ` + + `times like 'Monday at 2' — avoid timezones, timestamps, and the words 'AM'/'PM'; ` + + `use natural phrases like 'in the morning' or 'in the evening', and don't mention ` + + `the year unless it differs from the current one. When listing several times in the ` + + `same window, group them ('in the evening at 4, 5, or 6') instead of repeating the ` + + `time-of-day qualifier on each slot. If a chosen time is no longer available, let ` + + `them know gently and offer the next options.`; super({ instructions, @@ -185,6 +209,24 @@ You must infer the appropriate range implicitly from the conversational context this.tz = options.timezone; } + + async onEnter(): Promise { + const hour = Number( + new Intl.DateTimeFormat('en-US', { + hour: 'numeric', + hour12: false, + timeZone: this.tz, + }).format(new Date()), + ); + const timeOfDay = hour < 12 ? 'morning' : hour < 17 ? 'afternoon' : 'evening'; + await this.session.generateReply({ + instructions: + `Say hello and welcome to the caller — it's currently ${timeOfDay} their time. ` + + `You're the front desk of an office and you're here to help them schedule a visit. ` + + `Invite them to book an appointment to visit, and ask what time works. ` + + `Keep it warm and brief.`, + }); + } } export default defineAgent({ @@ -209,15 +251,46 @@ export default defineAgent({ const userdata: Userdata = { cal }; const session = new voice.AgentSession({ - stt: new deepgram.STT(), - llm: new openai.LLM({ - model: 'gpt-4.1', + stt: new inference.STT({ model: 'deepgram/nova-3' }), + llm: new inference.LLM({ model: 'google/gemma-4-31b-it' }), + tts: new inference.TTS({ + model: 'inworld/inworld-tts-2', + voice: 'Nadia', + modelOptions: { delivery_mode: 'CREATIVE', speaking_rate: 1.1 }, }), - tts: new elevenlabs.TTS(), + expressive: voice.presets.CUSTOMER_SERVICE, userData: userdata, - voiceOptions: { - maxToolSteps: 1, - }, + maxToolSteps: 1, + // Flip userState to "away" after 10s of mutual silence so we can + // check whether they're still there (default is 15s). + userAwayTimeout: 10.0, + }); + + const logger = log(); + let idleNudge: AbortController | null = null; + + const nudgeWhileIdle = async (signal: AbortSignal) => { + // Nudge every 10s until the user speaks again — speaking flips + // userState out of "away", which aborts this loop below. + while (!signal.aborted) { + logger.info("user idle — checking if they're still there"); + await session.generateReply({ + instructions: "The user has been idle, see if they're still there", + }); + await new Promise((resolve) => setTimeout(resolve, 10_000)); + } + }; + + session.on(voice.AgentSessionEventTypes.UserStateChanged, (ev) => { + if (ev.newState === 'away') { + if (idleNudge === null) { + idleNudge = new AbortController(); + void nudgeWhileIdle(idleNudge.signal); + } + } else if (idleNudge !== null) { + idleNudge.abort(); + idleNudge = null; + } }); await session.start({ @@ -227,8 +300,6 @@ export default defineAgent({ noiseCancellation: BackgroundVoiceCancellation(), }, }); - - await session.say('Hello, I can help you schedule an appointment!'); }, }); diff --git a/examples/src/frontdesk/test_agent.test.ts b/examples/src/frontdesk/test_agent.test.ts index 18ca3b7ac..87d0245f6 100644 --- a/examples/src/frontdesk/test_agent.test.ts +++ b/examples/src/frontdesk/test_agent.test.ts @@ -19,8 +19,8 @@ import { initializeLogger, voice } from '@livekit/agents'; import * as openai from '@livekit/agents-plugin-openai'; import { afterAll, beforeAll, describe, it } from 'vitest'; -import { type AvailableSlot, FakeCalendar, createAvailableSlot } from './calendar_api.js'; -import { FrontDeskAgent, type Userdata } from './frontdesk_agent.js'; +import { type AvailableSlot, FakeCalendar, createAvailableSlot } from './calendar_api.ts'; +import { FrontDeskAgent, type Userdata } from './frontdesk_agent.ts'; initializeLogger({ pretty: false, level: 'warn' }); diff --git a/examples/src/hotel-receptionist/hotel_receptionist.ts b/examples/src/hotel-receptionist/hotel_receptionist.ts index 42e82efa7..12741fbda 100644 --- a/examples/src/hotel-receptionist/hotel_receptionist.ts +++ b/examples/src/hotel-receptionist/hotel_receptionist.ts @@ -31,8 +31,8 @@ import { speakCode, speakTime, speakUsd, -} from './hotel_db.js'; -import { buildLookupPolicyTool } from './policies.js'; +} from './hotel_db.ts'; +import { buildLookupPolicyTool } from './policies.ts'; type UserData = { db: HotelDB; diff --git a/examples/src/inference_agent.ts b/examples/src/inference_agent.ts new file mode 100644 index 000000000..03a00c39b --- /dev/null +++ b/examples/src/inference_agent.ts @@ -0,0 +1,207 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// LiveKit Playground agent on the inference gateway: the user prototypes their +// own voice agent, swapping the STT / LLM / TTS models and the system prompt +// live via RPC. Port of livekit/agents examples/inference/agent.py. +import { + type JobContext, + ServerOptions, + cli, + defineAgent, + inference, + log, + voice, +} from '@livekit/agents'; +import type { RpcInvocationData } from '@livekit/rtc-node'; +import { fileURLToPath } from 'node:url'; + +const DEFAULT_STT = 'deepgram/nova-3'; +const DEFAULT_LLM = 'google/gemma-4-31b-it'; +const DEFAULT_TTS = 'inworld/inworld-tts-2'; + +// Default starter prompt. Keep in sync with the `set_system_prompt` +// control's `default` in the playground config — the UI seeds the +// textarea with the same string so the first session before any edit +// matches what the user sees. +const INSTRUCTIONS = + "You're a friendly agent in the LiveKit Playground. The person " + + 'talking to you is prototyping their own voice agent — they can ' + + 'edit this prompt in the side panel and swap the STT / LLM / TTS ' + + 'models live. Keep replies short, natural, and conversational, and ' + + 'be expressive so they can hear what the selected voice can do. ' + + 'At the start of the conversation, set the tone and pace — open with ' + + 'warm, upbeat energy and a quick, inviting question to encourage the ' + + 'user to engage and let them know they can talk to you naturally. ' + + "If the conversation lulls or they're not sure what to try, offer " + + 'to tell them a short joke — and if they say yes, deliver it with ' + + "good comic timing. If asked which models you're using, answer honestly."; + +const swapPrompt = (modality: string, model: string) => + `The user just switched the ${modality} model to '${model}'. ` + + "Acknowledge it in one short, natural sentence — say the model's " + + "name like a brand (e.g. 'Deepgram Nova 3', not 'deepgram slash " + + "nova dash three'). Skip hyphens, slashes, version dots, and any " + + "abbreviations that aren't pronounceable. Don't ask a follow-up."; + +class InferenceAgent extends voice.Agent { + constructor(instructions: string = INSTRUCTIONS) { + super({ instructions }); + } + + async onEnter(): Promise { + // Fired once the agent is active and RoomIO has subscribed to the + // participant's tracks, so the greeting is delivered to a connected + // client rather than spoken before the audio socket is up. Runs on + // the session's default LLM (Gemma) — no model-routing needed here. + this.session.generateReply({ + instructions: + 'Greet the user with excitement, and ask them how their day is going. ' + + 'Keep it to one or two short, natural sentences.', + }); + } +} + +export default defineAgent({ + entry: async (ctx: JobContext) => { + const logger = log(); + + const session = new voice.AgentSession({ + stt: new inference.STT({ model: DEFAULT_STT }), + llm: new inference.LLM({ model: DEFAULT_LLM }), + tts: new inference.TTS({ + model: DEFAULT_TTS, + voice: 'Sarah', + modelOptions: { delivery_mode: 'CREATIVE' }, + }), + expressive: voice.presets.CASUAL, + // Flip userState to "away" after 10s of mutual silence so we can + // check whether they're still there (default is 15s). + userAwayTimeout: 10.0, + }); + + let idleNudge: AbortController | null = null; + + const nudgeWhileIdle = async (signal: AbortSignal) => { + // Nudge every 10s until the user speaks again — speaking flips + // userState out of "away", which aborts this loop below. + while (!signal.aborted) { + logger.info("user idle — checking if they're still there"); + await session.generateReply({ + instructions: "The user has been idle, see if they're still there", + }); + await new Promise((resolve) => setTimeout(resolve, 10_000)); + } + }; + + session.on(voice.AgentSessionEventTypes.UserStateChanged, (ev) => { + if (ev.newState === 'away') { + if (idleNudge === null) { + idleNudge = new AbortController(); + void nudgeWhileIdle(idleNudge.signal); + } + } else if (idleNudge !== null) { + idleNudge.abort(); + idleNudge = null; + } + }); + + const parseValue = (payload: string, fallback: string): string => { + try { + const v = (JSON.parse(payload) as Record).value; + return typeof v === 'string' && v ? v : fallback; + } catch { + return fallback; + } + }; + + const agent = new InferenceAgent(); + await session.start({ agent, room: ctx.room }); + + ctx.room.localParticipant?.registerRpcMethod( + 'set_stt_model', + async (data: RpcInvocationData) => { + const model = parseValue(data.payload, DEFAULT_STT); + const stt = session.stt; + if (!(stt instanceof inference.STT) || stt.model === model) { + return ''; + } + logger.info(`switching STT → ${model}`); + stt.updateOptions({ model }); + session.generateReply({ instructions: swapPrompt('speech-to-text', model) }); + return ''; + }, + ); + + ctx.room.localParticipant?.registerRpcMethod( + 'set_llm_model', + async (data: RpcInvocationData) => { + const model = parseValue(data.payload, DEFAULT_LLM); + const llm = session.llm; + if (!(llm instanceof inference.LLM) || llm.model === model) { + return ''; + } + logger.info(`switching LLM → ${model}`); + llm.updateOptions({ model }); + session.generateReply({ instructions: swapPrompt('language', model) }); + return ''; + }, + ); + + ctx.room.localParticipant?.registerRpcMethod( + 'set_tts_model', + async (data: RpcInvocationData) => { + const model = parseValue(data.payload, DEFAULT_TTS); + const tts = session.tts; + if (!(tts instanceof inference.TTS) || tts.model === model) { + return ''; + } + logger.info(`switching TTS → ${model}`); + tts.updateOptions({ model }); + session.generateReply({ instructions: swapPrompt('text-to-speech', model) }); + return ''; + }, + ); + + ctx.room.localParticipant?.registerRpcMethod('open_in_builder', async () => { + // Build the Cloud Builder deep-link agent-side so the + // frontend doesn't have to know the URL schema. `p_` is a + // placeholder project_id — Cloud routes the user through + // login if needed and preserves the params on redirect. + const params = new URLSearchParams({ + modelMode: 'pipeline', + instructions: agent.instructions.toString() || '', + llm: session.llm instanceof inference.LLM ? session.llm.model : DEFAULT_LLM, + stt: session.stt instanceof inference.STT ? session.stt.model : DEFAULT_STT, + tts: session.tts instanceof inference.TTS ? session.tts.model : DEFAULT_TTS, + }); + return `https://cloud.livekit.io/projects/p_/agents/builder/new?${params.toString()}`; + }); + + ctx.room.localParticipant?.registerRpcMethod( + 'set_system_prompt', + async (data: RpcInvocationData) => { + // The UI fires this on every keystroke (debounced client-side + // by the textarea's edit→commit boundary), so dedupe against + // the current value before touching the agent. updateInstructions + // is cheap but it logs. + const prompt = parseValue(data.payload, ''); + if (!prompt) { + return ''; + } + if (agent.instructions === prompt) { + return ''; + } + logger.info(`system prompt updated (${prompt.length} chars)`); + await agent.updateInstructions(prompt); + return ''; + }, + ); + }, +}); + +// Only run CLI when executed directly, not when imported for testing. +// eslint-disable-next-line turbo/no-undeclared-env-vars +if (process.env.VITEST === undefined) { + cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) })); +} diff --git a/examples/src/instructions_per_modality.ts b/examples/src/instructions_per_modality.ts index f2ab6b910..8f4823e38 100644 --- a/examples/src/instructions_per_modality.ts +++ b/examples/src/instructions_per_modality.ts @@ -14,9 +14,8 @@ import { import { fileURLToPath } from 'node:url'; import { z } from 'zod'; -const BASE_INSTRUCTIONS = (modalitySpecific: string, currentDate: string) => +const BASE_INSTRUCTIONS = (currentDate: string) => `You are a scheduling assistant named Alex that helps users book appointments. -${modalitySpecific} Call \`book_appointment\` to finalise the booking. Never invent or assume details the user did not provide — ask for them instead. The current date is ${currentDate}. @@ -48,9 +47,11 @@ class SchedulingAgent extends voice.Agent { const now = new Date(); const weekday = now.toLocaleDateString(undefined, { weekday: 'long' }); const currentDate = `${now.toISOString().slice(0, 10)} ${weekday}`; - const instructions = new llm.Instructions({ - audio: BASE_INSTRUCTIONS(AUDIO_SPECIFIC, currentDate), - text: BASE_INSTRUCTIONS(TEXT_SPECIFIC, currentDate), + // common text is always present; the audio/text additions are appended + // depending on the turn's input modality + const instructions = new llm.Instructions(BASE_INSTRUCTIONS(currentDate), { + audio: AUDIO_SPECIFIC, + text: TEXT_SPECIFIC, }); super({ diff --git a/examples/src/survey_agent.ts b/examples/src/survey_agent.ts index a09df7537..954337a29 100644 --- a/examples/src/survey_agent.ts +++ b/examples/src/survey_agent.ts @@ -6,7 +6,9 @@ import { ServerOptions, cli, defineAgent, + inference, llm, + log, voice, workflows, } from '@livekit/agents'; @@ -371,14 +373,49 @@ export class SurveyAgent extends voice.Agent { export default defineAgent({ entry: async (ctx: JobContext) => { const session = new voice.AgentSession({ - llm: 'openai/gpt-4.1', - stt: 'deepgram/nova-3', - tts: 'cartesia/sonic-3', + llm: new inference.LLM({ model: 'google/gemma-4-31b-it' }), + stt: new inference.STT({ model: 'deepgram/nova-3', language: 'multi' }), + tts: new inference.TTS({ + model: 'inworld/inworld-tts-2', + voice: 'Nate', + modelOptions: { delivery_mode: 'CREATIVE' }, + }), + expressive: voice.presets.CASUAL, userData: { filename: 'survey_results.csv', candidateName: '', taskResults: {}, }, + // Flip userState to "away" after 10s of mutual silence so we can + // check whether they're still there (default is 15s). + userAwayTimeout: 10.0, + }); + + const logger = log(); + let idleNudge: AbortController | null = null; + + const nudgeWhileIdle = async (signal: AbortSignal) => { + // Nudge every 10s until the user speaks again — speaking flips + // userState out of "away", which aborts this loop below. + while (!signal.aborted) { + logger.info("user idle — checking if they're still there"); + await session.generateReply({ + instructions: "The user has been idle, see if they're still there", + }); + await new Promise((resolve) => setTimeout(resolve, 10_000)); + } + }; + + session.on(voice.AgentSessionEventTypes.UserStateChanged, (ev) => { + if (ev.newState === 'away') { + if (idleNudge === null) { + idleNudge = new AbortController(); + void nudgeWhileIdle(idleNudge.signal); + } + } else if (idleNudge !== null) { + idleNudge.abort(); + idleNudge = null; + } }); await session.start({ diff --git a/examples/src/testing/survey_agent.test.ts b/examples/src/testing/survey_agent.test.ts index 748feb41e..36aad7748 100644 --- a/examples/src/testing/survey_agent.test.ts +++ b/examples/src/testing/survey_agent.test.ts @@ -26,7 +26,7 @@ import { ExperienceTask, IntroTask, type SurveyUserData, -} from '../survey_agent.js'; +} from '../survey_agent.ts'; const { TaskGroup } = workflows; type TaskGroupResult = workflows.TaskGroupResult; diff --git a/examples/tsconfig.json b/examples/tsconfig.json index 34ca2a33e..76b661f57 100644 --- a/examples/tsconfig.json +++ b/examples/tsconfig.json @@ -5,6 +5,10 @@ // them out of the package build so they can't break `pnpm build`. "exclude": ["./src/test_*.ts"], "compilerOptions": { + // relative imports use .ts extensions so examples can run directly under + // node's type stripping (e.g. `lk agent console`); tsc rewrites them to + // .js in the emitted output + "rewriteRelativeImportExtensions": true, // match output dir to input dir. e.g. dist/index instead of dist/src/index "rootDir": "./src", "declarationDir": "./dist", diff --git a/plugins/cartesia/src/tts.ts b/plugins/cartesia/src/tts.ts index cc622f36d..8846de1bb 100644 --- a/plugins/cartesia/src/tts.ts +++ b/plugins/cartesia/src/tts.ts @@ -138,6 +138,15 @@ export class TTS extends tts.TTS { return 'Cartesia'; } + /** + * Key into the shared expressive markup tables; markup delegation lives in + * the base class. + * @internal + */ + override _markupProviderKey(): string { + return 'cartesia'; + } + constructor(opts: Partial = {}) { const resolvedOpts = { ...defaultTTSOptions, diff --git a/plugins/inworld/src/tts.ts b/plugins/inworld/src/tts.ts index bbf23e0f5..0daf56502 100644 --- a/plugins/inworld/src/tts.ts +++ b/plugins/inworld/src/tts.ts @@ -335,6 +335,21 @@ export class TTS extends tts.TTS { return this.#pool; } + override get model(): string { + return this.#opts.model; + } + + /** + * Key into the shared expressive markup tables; markup delegation lives in + * the base class. Only inworld-tts-2 understands the markup tags; older + * models get no markup so the tags aren't injected, converted, or stripped + * (matches the inference gateway's behavior). + * @internal + */ + override _markupProviderKey(): string { + return this.#opts.model.includes('tts-2') ? 'inworld' : ''; + } + get authorization(): string { return this.#authorization; }