From dc236c2434ed983d324c6125c3d4b9ff324f135f Mon Sep 17 00:00:00 2001 From: Toubat Date: Tue, 7 Jul 2026 22:17:13 +0800 Subject: [PATCH 01/12] feat(tts): LLM-driven TTS prosody via provider-specific markup tags Port of livekit/agents#6116 (expressive mode) to agents-js. Co-authored-by: Cursor --- .changeset/expressive-tts-markup.md | 7 + agents/src/constants.ts | 9 + agents/src/inference/tts.ts | 38 +- agents/src/llm/chat_context.test.ts | 314 +++---- agents/src/llm/chat_context.ts | 258 +++--- agents/src/llm/index.ts | 1 - agents/src/llm/provider_format/google.test.ts | 9 +- agents/src/llm/provider_format/google.ts | 3 - .../src/llm/provider_format/mistralai.test.ts | 9 +- agents/src/llm/provider_format/mistralai.ts | 6 +- agents/src/llm/provider_format/openai.test.ts | 24 +- agents/src/llm/provider_format/openai.ts | 7 - agents/src/llm/utils.ts | 8 - agents/src/stt/index.ts | 2 + agents/src/stt/stt.ts | 37 + agents/src/telemetry/traces.ts | 8 +- agents/src/tokenize/basic/basic.ts | 40 +- agents/src/tokenize/basic/sentence.ts | 5 +- agents/src/tokenize/token_stream.ts | 246 +++++- agents/src/tokenize/xml_markup.test.ts | 239 ++++++ agents/src/tts/_provider_format.ts | 779 ++++++++++++++++++ agents/src/tts/markup_utils.test.ts | 239 ++++++ agents/src/tts/markup_utils.ts | 124 +++ agents/src/tts/tts.ts | 166 ++++ agents/src/utils.ts | 41 + agents/src/voice/agent.test.ts | 2 + agents/src/voice/agent.ts | 49 +- agents/src/voice/agent_activity.ts | 172 +++- agents/src/voice/agent_session.ts | 49 +- agents/src/voice/audio_recognition.ts | 33 +- agents/src/voice/generation.ts | 52 +- agents/src/voice/presets.ts | 106 +++ agents/src/voice/remote_session.ts | 6 +- agents/src/voice/room_io/_output.ts | 138 +++- .../src/voice/transcription/synchronizer.ts | 16 +- agents/src/voice/turn_config/utils.ts | 1 + agents/src/workflows/warm_transfer.ts | 2 +- examples/src/instructions_per_modality.ts | 11 +- plugins/cartesia/src/tts.ts | 9 + plugins/inworld/src/tts.ts | 15 + 40 files changed, 2733 insertions(+), 547 deletions(-) create mode 100644 .changeset/expressive-tts-markup.md create mode 100644 agents/src/tokenize/xml_markup.test.ts create mode 100644 agents/src/tts/_provider_format.ts create mode 100644 agents/src/tts/markup_utils.test.ts create mode 100644 agents/src/tts/markup_utils.ts create mode 100644 agents/src/voice/presets.ts diff --git a/.changeset/expressive-tts-markup.md b/.changeset/expressive-tts-markup.md new file mode 100644 index 000000000..fe9573603 --- /dev/null +++ b/.changeset/expressive-tts-markup.md @@ -0,0 +1,7 @@ +--- +'@livekit/agents-plugin-cartesia': patch +'@livekit/agents-plugin-inworld': patch +'@livekit/agents': patch +--- + +feat: LLM-driven TTS prosody via provider-specific markup tags (expressive mode) diff --git a/agents/src/constants.ts b/agents/src/constants.ts index 3da61af46..af1c749bb 100644 --- a/agents/src/constants.ts +++ b/agents/src/constants.ts @@ -5,6 +5,15 @@ export const ATTRIBUTE_TRANSCRIPTION_TRACK_ID = 'lk.transcribed_track_id'; export const ATTRIBUTE_TRANSCRIPTION_FINAL = 'lk.transcription_final'; export const TOPIC_TRANSCRIPTION = 'lk.transcription'; export const ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID = 'lk.segment_id'; +/** + * The expression (delivery/emotion) the agent used for a transcription segment, surfaced so + * the frontend can react to it, when expressive markup is stripped from the transcript. The + * value is a JSON object `{"value": ...}` carrying the segment's leading expression — the + * `` tag for Inworld or the `` tag for Cartesia, e.g. + * `{"value": "speak happy"}`. A JSON object (rather than a bare string) so the shape can + * gain fields later without breaking parsers. + */ +export const ATTRIBUTE_TRANSCRIPTION_EXPRESSION = 'lk.expression'; export const ATTRIBUTE_PUBLISH_ON_BEHALF = 'lk.publish_on_behalf'; export const TOPIC_CHAT = 'lk.chat'; diff --git a/agents/src/inference/tts.ts b/agents/src/inference/tts.ts index f4011f69b..7c05e3418 100644 --- a/agents/src/inference/tts.ts +++ b/agents/src/inference/tts.ts @@ -10,7 +10,7 @@ import { ConnectionPool } from '../connection_pool.js'; import { type LanguageCode, normalizeLanguage } from '../language.js'; import { log } from '../log.js'; import { createStreamChannel } from '../stream/stream_channel.js'; -import { basic as tokenizeBasic } from '../tokenize/index.js'; +import { sentenceTokenizer as markupSentenceTokenizer } from '../tts/_provider_format.js'; import type { ChunkedStream } from '../tts/index.js'; import { SynthesizeStream as BaseSynthesizeStream, TTS as BaseTTS } from '../tts/index.js'; import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js'; @@ -380,6 +380,20 @@ export class TTS extends BaseTTS { return 'livekit'; } + /** + * Key into the shared markup tables, derived from the gateway model string. + * @internal + */ + override _markupProviderKey(): string { + const model = this.opts.model ?? ''; + const provider = model.split('/')[0]!; + if (provider === 'inworld') { + // older inworld models don't support markup + return model.includes('tts-2') ? 'inworld' : ''; + } + return provider; + } + override get capabilities() { return { streaming: true, alignedTranscript: this.#alignedTranscript }; } @@ -507,6 +521,13 @@ export class TTS extends BaseTTS { export class SynthesizeStream extends BaseSynthesizeStream { private opts: InferenceTTSOptions; private tts: TTS; + /** + * Snapshot whether expressive is active now, while the framework holds it + * fixed for this synthesis (set synchronously before stream()). Reading it + * lazily in run() would race with the next turn/session mutating the shared + * TTS instance. + */ + private expressive: boolean; #logger = log(); @@ -514,6 +535,7 @@ export class SynthesizeStream extends BaseSynthesizeSt super(tts, connOptions); this.opts = opts; this.tts = tts; + this.expressive = tts._expressive; } get label() { @@ -547,7 +569,11 @@ export class SynthesizeStream extends BaseSynthesizeSt // Python side. let pendingTimedTranscripts: TimedString[] = []; - const sendTokenizerStream = new tokenizeBasic.SentenceTokenizer().stream(); + // chunking defaults (cap + expressive batch size) live in _provider_format + const provider = (this.opts.model ?? '').split('/')[0]!; + const sendTokenizerStream = markupSentenceTokenizer(provider, { + expressive: this.expressive, + }).stream(); const eventChannel = createStreamChannel(); const requestId = shortuuid('tts_request_'); const inputSentEvent = new Event(); @@ -597,7 +623,7 @@ export class SynthesizeStream extends BaseSynthesizeSt sendTokenizerStream.flush(); continue; } - sendTokenizerStream.pushText(data); + sendTokenizerStream.pushText(this.tts.markup.normalize(data)); } // Only call endInput if the stream hasn't been closed by cleanup if (!closing) { @@ -617,11 +643,15 @@ export class SynthesizeStream extends BaseSynthesizeSt if (this.opts.model) generationConfig.model = this.opts.model; if (this.opts.language) generationConfig.language = this.opts.language; + // re-normalize at sentence level: tags split across input chunks + // aren't caught by the per-chunk normalize in the input task + const converted = this.tts.markup.convert(this.tts.markup.normalize(ev.token)); + this.markStarted(); await sendClientEvent( { type: 'input_transcript', - transcript: ev.token + ' ', + transcript: converted + ' ', generation_config: generationConfig, extra: (this.opts.modelOptions as Record) ?? {}, }, diff --git a/agents/src/llm/chat_context.test.ts b/agents/src/llm/chat_context.test.ts index 101a9fe46..3882591de 100644 --- a/agents/src/llm/chat_context.test.ts +++ b/agents/src/llm/chat_context.test.ts @@ -3,7 +3,6 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from 'vitest'; import { initializeLogger } from '../log.js'; -import { INSTRUCTIONS_MESSAGE_ID, applyInstructionsModality } from '../voice/generation.js'; import { FakeLLM } from '../voice/testing/fake_llm.js'; import { type AudioContent, @@ -15,7 +14,6 @@ import { type ImageContent, Instructions, ReadonlyChatContext, - concatInstructions, isInstructions, renderInstructions, } from './chat_context.js'; @@ -1243,241 +1241,163 @@ describe('ChatContext.isEquivalent', () => { }); describe('Instructions', () => { - it('constructs from an object with audio and text variants', () => { - const instr = new Instructions({ audio: 'audio variant', text: 'text variant' }); + it('constructs from common text with audio and text additions', () => { + const instr = new Instructions('common text', { + audio: 'audio addition', + text: 'text addition', + }); - expect(instr.audio).toBe('audio variant'); - expect(instr.text).toBe('text variant'); - expect(instr.value).toBe('audio variant'); + expect(instr.common).toBe('common text'); + expect(instr.audio).toBe('audio addition'); + expect(instr.text).toBe('text addition'); + expect(instr.toString()).toBe('common text'); }); it('identifies Instructions with a type guard', () => { - const instr = new Instructions({ audio: 'audio variant', text: 'text variant' }); + const instr = new Instructions('common', { audio: 'audio addition' }); expect(isInstructions(instr)).toBe(true); - expect(isInstructions('audio variant')).toBe(false); - expect(isInstructions({ type: 'instructions', audio: 'audio variant' })).toBe(false); - }); - - it('tpl propagates Instructions interpolations into audio and text variants', () => { - const instr = Instructions.tpl`persona -${new Instructions({ audio: 'audio rules', text: 'text rules' })} -extra`; - - expect(instr).toBeInstanceOf(Instructions); - expect(instr.audio).toBe('persona\naudio rules\nextra'); - expect(instr.text).toBe('persona\ntext rules\nextra'); - expect(instr.value).toBe('persona\naudio rules\nextra'); - expect(instr.asModality('text').value).toBe('persona\ntext rules\nextra'); - }); - - it('tpl preserves audio-only interpolation as audio-only output', () => { - const instr = Instructions.tpl`prefix ${new Instructions({ audio: 'same' })} suffix`; - - expect(instr.toJSON()).toEqual({ type: 'instructions', audio: 'prefix same suffix' }); - expect(instr.audio).toBe('prefix same suffix'); - expect(instr.text).toBe('prefix same suffix'); + expect(isInstructions('common')).toBe(false); + expect(isInstructions({ type: 'instructions', common: 'common' })).toBe(false); }); - it('tpl interpolates primitive values into both variants', () => { - const instr = Instructions.tpl`date=${'2026-05-13'} enabled=${true} count=${3}`; - - expect(instr.toJSON()).toEqual({ - type: 'instructions', - audio: 'date=2026-05-13 enabled=true count=3', + it('render() combines common + modality-specific additions', () => { + const instr = new Instructions('You are a helpful assistant.', { + audio: 'Keep responses short for voice.', + text: 'Use markdown formatting.', }); - expect(instr.audio).toBe('date=2026-05-13 enabled=true count=3'); - expect(instr.text).toBe('date=2026-05-13 enabled=true count=3'); - expect(instr.value).toBe('date=2026-05-13 enabled=true count=3'); - }); - - it('tpl combines multiple modality-aware interpolations', () => { - const instr = Instructions.tpl`${new Instructions({ audio: 'audio A', text: 'text A' })} / ${new Instructions({ audio: 'audio B', text: 'text B' })}`; - - expect(instr.audio).toBe('audio A / audio B'); - expect(instr.text).toBe('text A / text B'); - expect(instr.value).toBe('audio A / audio B'); - }); - it('tpl preserves the current rendered value of resolved interpolations', () => { - const resolved = new Instructions({ audio: 'audio rules', text: 'text rules' }).asModality( - 'text', + expect(instr.render({ modality: 'audio' })).toBe( + 'You are a helpful assistant.\n\nKeep responses short for voice.', ); - const instr = Instructions.tpl`prefix ${resolved} suffix`; - - expect(instr.audio).toBe('prefix audio rules suffix'); - expect(instr.text).toBe('prefix text rules suffix'); - expect(instr.value).toBe('prefix text rules suffix'); + expect(instr.render({ modality: 'text' })).toBe( + 'You are a helpful assistant.\n\nUse markdown formatting.', + ); + // no modality returns just the common text + expect(instr.render()).toBe('You are a helpful assistant.'); }); - it('tpl stringifies null and undefined values like template literals', () => { - const instr = Instructions.tpl`null=${null} undefined=${undefined}`; - - expect(instr.toJSON()).toEqual({ - type: 'instructions', - audio: 'null=null undefined=undefined', - }); - expect(instr.audio).toBe('null=null undefined=undefined'); - expect(instr.text).toBe('null=null undefined=undefined'); + it('render() without modality additions returns just common for both', () => { + const commonOnly = new Instructions('common only'); + expect(commonOnly.render({ modality: 'audio' })).toBe('common only'); + expect(commonOnly.render({ modality: 'text' })).toBe('common only'); }); - it('serializes to a dict with both variants and round-trips through toJSON', () => { - const instr = new Instructions({ audio: 'audio variant', text: 'text variant' }); - - const ctx = new ChatContext([ChatMessage.create({ role: 'system', content: [instr] })]); - const data = ctx.toJSON(); - const items = (data.items as Record[])!; - const content = (items[0]!.content as Record[])![0]!; + it('render() with only one modality addition', () => { + const audioOnly = new Instructions('base', { audio: 'audio extra' }); + expect(audioOnly.render({ modality: 'audio' })).toBe('base\n\naudio extra'); + expect(audioOnly.render({ modality: 'text' })).toBe('base'); - expect(content).toEqual({ - type: 'instructions', - audio: 'audio variant', - text: 'text variant', - }); - }); - - it('omits the text key in toJSON when only audio variant is provided', () => { - const instr = new Instructions({ audio: 'audio only' }); - expect(instr.toJSON()).toEqual({ type: 'instructions', audio: 'audio only' }); + const textOnly = new Instructions('base', { text: 'text extra' }); + expect(textOnly.render({ modality: 'audio' })).toBe('base'); + expect(textOnly.render({ modality: 'text' })).toBe('base\n\ntext extra'); }); - it('falls back text -> audio when no text variant is provided', () => { - const instr = new Instructions({ audio: 'audio only' }); - expect(instr.audio).toBe('audio only'); - expect(instr.text).toBe('audio only'); - expect(instr.value).toBe('audio only'); + it('render() with empty common returns just the addition', () => { + const emptyCommon = new Instructions('', { audio: 'audio only', text: 'text only' }); + expect(emptyCommon.render({ modality: 'audio' })).toBe('audio only'); + expect(emptyCommon.render({ modality: 'text' })).toBe('text only'); }); - it('renderInstructions returns strings and resolved Instructions values explicitly', () => { - const instr = new Instructions({ audio: 'audio instructions', text: 'text instructions' }); - - expect(renderInstructions('plain instructions')).toBe('plain instructions'); - expect(renderInstructions(instr)).toBe('audio instructions'); - expect(renderInstructions(instr, 'audio')).toBe('audio instructions'); - expect(renderInstructions(instr, 'text')).toBe('text instructions'); + it('render() fills template placeholders from data', () => { + const instr = new Instructions('Hello {name}, the weather is {weather.condition}.'); + expect(instr.render({ data: { name: 'Alice', weather: { condition: 'sunny' } } })).toBe( + 'Hello Alice, the weather is sunny.', + ); }); - it('concatenates two Instructions, propagating both variants', () => { - const a = new Instructions({ audio: 'audio A', text: 'text A' }); - const b = new Instructions({ audio: 'audio B', text: 'text B' }); - const result = a.concat(b); - expect(result).toBeInstanceOf(Instructions); - expect(result.audio).toBe('audio Aaudio B'); - expect(result.text).toBe('text Atext B'); + it('render() replaces missing placeholders with empty strings', () => { + const instr = new Instructions('Hello {name}!'); + expect(instr.render({ data: { other: 'value' } })).toBe('Hello !'); }); - it('concatenates Instructions + string, propagating both variants', () => { - const instr = new Instructions({ audio: 'audio', text: 'text' }); - const result = instr.concat(' suffix'); - expect(result.audio).toBe('audio suffix'); - expect(result.text).toBe('text suffix'); - }); + it('resolveTemplate interpolates primitive values into all variants', () => { + const instr = Instructions.resolveTemplate('date={date} count={count}', { + date: '2026-05-13', + count: 3, + }); - it('concatInstructions handles string + Instructions (radd-style)', () => { - const instr = new Instructions({ audio: 'audio', text: 'text' }); - const result = concatInstructions('prefix ', instr); - expect(isInstructions(result)).toBe(true); - if (!isInstructions(result)) return; - expect(result.audio).toBe('prefix audio'); - expect(result.text).toBe('prefix text'); + expect(instr).toBeInstanceOf(Instructions); + expect(instr.common).toBe('date=2026-05-13 count=3'); + expect(instr.audio).toBeUndefined(); + expect(instr.text).toBeUndefined(); }); - it('preserves text=undefined when concatenating an audio-only instructions', () => { - const audioOnly = new Instructions({ audio: 'audio only' }); - const result = audioOnly.concat(' more'); - expect(result.toJSON()).toEqual({ type: 'instructions', audio: 'audio only more' }); - expect(result.audio).toBe('audio only more'); - expect(result.text).toBe('audio only more'); - }); + it('resolveTemplate propagates Instructions interpolations into modality variants', () => { + const rules = new Instructions('common rules', { + audio: 'audio rules', + text: 'text rules', + }); + const instr = Instructions.resolveTemplate('persona\n{rules}\nextra', { rules }); - it('when only one side has a text variant, the other contributes its audio', () => { - const a = new Instructions({ audio: 'audio A', text: 'text A' }); - const b = new Instructions({ audio: 'audio B' }); - const result = concatInstructions(a, ' ', b); - expect(isInstructions(result)).toBe(true); - if (!isInstructions(result)) return; - expect(result.audio).toBe('audio A audio B'); - expect(result.text).toBe('text A audio B'); + expect(instr.common).toBe('persona\ncommon rules\nextra'); + expect(instr.audio).toBe('persona\naudio rules\nextra'); + expect(instr.text).toBe('persona\ntext rules\nextra'); }); - it('asModality returns a copy with both variants preserved', () => { - const instr = new Instructions({ audio: 'audio instructions', text: 'text instructions' }); - - let resolved = instr.asModality('audio'); - expect(resolved.value).toBe('audio instructions'); - expect(resolved.audio).toBe('audio instructions'); - expect(resolved.text).toBe('text instructions'); + it('resolveTemplate falls back to common when an Instructions kwarg has no variant', () => { + const rules = new Instructions('common rules', { audio: 'audio rules' }); + const instr = Instructions.resolveTemplate('prefix {rules} suffix', { rules }); - resolved = instr.asModality('text'); - expect(resolved.value).toBe('text instructions'); - expect(resolved.audio).toBe('audio instructions'); - expect(resolved.text).toBe('text instructions'); + expect(instr.common).toBe('prefix common rules suffix'); + expect(instr.audio).toBe('prefix audio rules suffix'); + // no text variant on the kwarg: text falls back to the common value + expect(instr.text).toBe('prefix common rules suffix'); }); - it('can switch modality after a previous resolution', () => { - const instr = new Instructions({ audio: 'audio instructions', text: 'text instructions' }); - const resolvedText = instr.asModality('text'); - const resolvedAudio = resolvedText.asModality('audio'); - expect(resolvedAudio.value).toBe('audio instructions'); - }); + it('serializes with toJSON and omits undefined variants', () => { + const both = new Instructions('common text', { + audio: 'audio addition', + text: 'text addition', + }); + expect(both.toJSON()).toEqual({ + type: 'instructions', + common: 'common text', + audio: 'audio addition', + text: 'text addition', + }); - it('asModality on audio-only Instructions returns audio for both modalities', () => { - const audioOnly = new Instructions({ audio: 'audio only' }); - expect(audioOnly.asModality('audio').value).toBe('audio only'); - expect(audioOnly.asModality('text').value).toBe('audio only'); + const audioOnly = new Instructions('common', { audio: 'audio only' }); + expect(audioOnly.toJSON()).toEqual({ + type: 'instructions', + common: 'common', + audio: 'audio only', + }); }); - it('applyInstructionsModality rewrites the system message content', () => { - const instr = new Instructions({ audio: 'audio instructions', text: 'text instructions' }); - const ctx = new ChatContext([ - ChatMessage.create({ - id: INSTRUCTIONS_MESSAGE_ID, - role: 'system', - content: [instr], - }), - ]); - - applyInstructionsModality(ctx, { modality: 'audio' }); - let content = (ctx.items[0]! as ChatMessage).content[0]!; - expect(isInstructions(content) ? content.value : '').toBe('audio instructions'); - - applyInstructionsModality(ctx, { modality: 'text' }); - content = (ctx.items[0]! as ChatMessage).content[0]!; - expect(isInstructions(content) ? content.value : '').toBe('text instructions'); - }); + it('renderInstructions returns strings and renders Instructions with modality', () => { + const instr = new Instructions('common', { + audio: 'audio addition', + text: 'text addition', + }); - it('applyInstructionsModality is a no-op when content has no Instructions', () => { - const ctx = new ChatContext([ - ChatMessage.create({ - id: INSTRUCTIONS_MESSAGE_ID, - role: 'system', - content: ['plain string instructions'], - }), - ]); - const before = (ctx.items[0]! as ChatMessage).content[0]; - applyInstructionsModality(ctx, { modality: 'text' }); - expect((ctx.items[0]! as ChatMessage).content[0]).toBe(before); + expect(renderInstructions('plain instructions')).toBe('plain instructions'); + expect(renderInstructions(instr)).toBe('common'); + expect(renderInstructions(instr, 'audio')).toBe('common\n\naudio addition'); + expect(renderInstructions(instr, 'text')).toBe('common\n\ntext addition'); }); - it('survives copy and lets a different modality be applied to the copy', () => { - const instr = new Instructions({ audio: 'audio instructions', text: 'text instructions' }); - const baseCtx = new ChatContext([ - ChatMessage.create({ - id: INSTRUCTIONS_MESSAGE_ID, - role: 'system', - content: [instr], - }), - ]); - const turn1 = baseCtx.copy(); - applyInstructionsModality(turn1, { modality: 'text' }); - const turn2 = turn1.copy(); - applyInstructionsModality(turn2, { modality: 'audio' }); - - const turn2Content = (turn2.items[0]! as ChatMessage).content[0]!; - expect(isInstructions(turn2Content) ? turn2Content.value : '').toBe('audio instructions'); + it('is resolved to str before storage in ChatMessage content', () => { + const instr = new Instructions('common text', { + audio: 'audio addition', + text: 'text addition', + }); - // base context content is untouched (was the original instr) - expect((baseCtx.items[0]! as ChatMessage).content[0]).toBe(instr); + // addMessage with Instructions resolves to toString() = common + const ctx = ChatContext.empty(); + ctx.addMessage({ role: 'system', content: [instr.toString()] }); + const content = (ctx.items[0]! as ChatMessage).content[0]; + expect(typeof content).toBe('string'); + expect(content).toBe('common text'); + + // toJSON round-trip with resolved str content + const resolved = instr.render({ modality: 'audio' }); + const ctx2 = new ChatContext([ChatMessage.create({ role: 'system', content: [resolved] })]); + const data = ctx2.toJSON(); + const items = (data.items as Record[])!; + const serialized = (items[0]!.content as unknown[])![0]!; + expect(typeof serialized).toBe('string'); + expect(serialized).toBe('common text\n\naudio addition'); }); }); diff --git a/agents/src/llm/chat_context.ts b/agents/src/llm/chat_context.ts index 34cf39200..64e13850f 100644 --- a/agents/src/llm/chat_context.ts +++ b/agents/src/llm/chat_context.ts @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 import type { AudioFrame, VideoFrame } from '@livekit/rtc-node'; -import { createImmutableArray, shortuuid } from '../utils.js'; +import { createImmutableArray, safeRender, shortuuid } from '../utils.js'; import type { LLM } from './llm.js'; import { type ProviderFormat, toChatCtx } from './provider_format/index.js'; import type { JSONObject, JSONValue, ToolContext } from './tool_context.js'; @@ -37,15 +37,6 @@ export interface AudioContent { transcript?: string; } -type InstructionsOptions = { - /** The audio/voice variant of the instructions. */ - audio: string; - /** The text variant of the instructions; falls back to `audio` when omitted. */ - text?: string; - /** The currently rendered string value, used by `value`/`toString()`. */ - represent?: string; -}; - const INSTRUCTIONS_SYMBOL = Symbol.for('livekit.agents.Instructions'); export function isInstructions(value: unknown): value is Instructions { @@ -58,131 +49,143 @@ export function isInstructions(value: unknown): value is Instructions { } /** - * Instructions that adapt based on the user's input modality (audio vs. text). + * Instructions with optional modality-specific additions. + * + * Construction: + * ```ts + * // Simple — same instructions for all modalities + * new Instructions('You are a helpful assistant.'); * - * The `value` property is the rendered string providers see. By default it - * equals the `audio` variant; after {@link asModality} it equals the chosen - * variant. Both the `audio` variant and the raw `text` variant are preserved - * so {@link asModality} can be called again for a different modality (e.g., - * when the same `ChatContext` is reused across tool-call turns). + * // With modality-specific additions + * new Instructions('You are a helpful assistant.', { + * audio: 'Keep responses short for voice.', + * text: 'Use markdown formatting.', + * }); + * ``` + * + * Rendering: + * ```ts + * instr.render(); // → common text + * instr.render({ modality: 'audio' }); // → common + audio addition + * instr.render({ modality: 'text', data: { name: 'Alex' } }); // → common + text, with {name} filled + * ``` */ export class Instructions { readonly type = 'instructions' as const; - private readonly _audioVariant: string; + common: string; - /** Raw text variant; falls back to {@link audio} when omitted. */ - private readonly _textVariant?: string; + audio?: string; - /** The currently rendered string (what providers should treat as content). */ - readonly value: string; + text?: string; /** @internal Symbol marker for type identification */ readonly [INSTRUCTIONS_SYMBOL] = true; - constructor(options: InstructionsOptions) { - this._audioVariant = options.audio; - this._textVariant = options.text; - this.value = options.represent ?? options.audio; + constructor(common: string = '', options?: { audio?: string; text?: string }) { + this.common = common; + this.audio = options?.audio; + this.text = options?.text; } - static tpl( - strings: TemplateStringsArray, - ...values: Array - ): Instructions { - const render = (mode: 'audio' | 'text' | 'value') => { - let result = strings[0]!; - for (let i = 0; i < values.length; i++) { - const value = values[i]!; - if (isInstructions(value)) { - result += mode === 'audio' ? value.audio : mode === 'text' ? value.text : value.value; - } else { - result += String(value); - } - result += strings[i + 1]!; + /** + * Render instructions to a plain string. + * + * @param options.modality - If given, appends the modality-specific addition to the common text. + * @param options.data - Template variables to fill. Missing placeholders log an error + * and are replaced with empty strings. + */ + render(options?: { modality?: 'audio' | 'text'; data?: Record }): string { + const { modality, data } = options ?? {}; + + const parts = [this.common]; + if (modality !== undefined) { + const addition = modality === 'audio' ? this.audio : this.text; + if (addition) { + parts.push(addition); } - return result; - }; - - const hasTextVariant = values.some( - (value) => isInstructions(value) && value._textVariant !== undefined, - ); + } - return new Instructions({ - audio: render('audio'), - text: hasTextVariant ? render('text') : undefined, - represent: render('value'), - }); - } + let result = parts.filter((p) => p).join('\n\n'); - /** The audio (voice) variant of the instructions. */ - get audio(): string { - return this._audioVariant; - } + if (data && Object.keys(data).length > 0) { + result = safeRender(result, data); + } - /** The text variant of the instructions. Falls back to {@link audio}. */ - get text(): string { - return this._textVariant ?? this.audio; + return result; } /** - * Return a copy whose {@link value} is the variant matching `modality`. - * Both `audio` and `text` variants are preserved on the result, so this can - * be called again for a different modality (e.g. across tool-call turns). + * Fill a template string, producing an `Instructions` with modality variants. + * + * If any kwarg value is an `Instructions` object, its `common`/`audio`/`text` + * parts are substituted into the matching variant of the result. This is used by + * workflow tasks to build modality-aware instructions from a single template. */ - asModality(modality: 'audio' | 'text'): Instructions { - return new Instructions({ - audio: this.audio, - text: this._textVariant, - represent: modality === 'audio' ? this.audio : this.text, - }); - } - - /** Concatenate, propagating both variants and the current rendered value. */ - concat(other: string | Instructions): Instructions { - if (isInstructions(other)) { - const hasText = this._textVariant !== undefined || other._textVariant !== undefined; - return new Instructions({ - audio: this.audio + other.audio, - text: hasText ? this.text + other.text : undefined, - represent: this.value + other.value, + static resolveTemplate(template: string, kwargs: Record): Instructions { + const anyInstructions = Object.values(kwargs).some((v) => isInstructions(v)); + if (anyInstructions) { + const commonKw: Record = {}; + const audioKw: Record = {}; + const textKw: Record = {}; + for (const [k, v] of Object.entries(kwargs)) { + if (isInstructions(v)) { + commonKw[k] = v.toString(); + // an explicit "" removes the section; only undefined falls back to common + audioKw[k] = v.audio !== undefined ? v.audio : v.toString(); + textKw[k] = v.text !== undefined ? v.text : v.toString(); + } else { + commonKw[k] = v; + audioKw[k] = v; + textKw[k] = v; + } + } + return new Instructions(safeRender(template, commonKw), { + audio: safeRender(template, audioKw), + text: safeRender(template, textKw), }); } - return new Instructions({ - audio: this.audio + other, - text: this._textVariant !== undefined ? this._textVariant + other : undefined, - represent: this.value + other, - }); + return new Instructions(safeRender(template, kwargs)); } toString(): string { - return this.value; + return this.common; } - toJSON(): { type: 'instructions'; audio: string; text?: string } { - const result: { type: 'instructions'; audio: string; text?: string } = { + toJSON(): { type: 'instructions'; common: string; audio?: string; text?: string } { + const result: { type: 'instructions'; common: string; audio?: string; text?: string } = { type: 'instructions', - audio: this.audio, + common: this.common, }; - if (this._textVariant !== undefined) { - result.text = this._textVariant; + if (this.audio !== undefined) { + result.audio = this.audio; + } + if (this.text !== undefined) { + result.text = this.text; } return result; } } +/** + * Resolve instructions to a plain string. Plain strings pass through; + * {@link Instructions} are rendered (with the modality-specific addition + * appended when `modality` is given). + */ export function renderInstructions( instructions: string | Instructions, modality?: 'audio' | 'text', ): string { if (typeof instructions === 'string') return instructions; - return modality === undefined ? instructions.value : instructions.asModality(modality).value; + return instructions.render({ modality }); } /** * Compare two instruction values by content. Plain strings compare by value; - * {@link Instructions} compare by their audio + text variants so that two - * distinct instances with the same content are treated as equal. + * {@link Instructions} compare by their common/audio/text parts so that two + * distinct instances with the same content are treated as equal. An + * {@link Instructions} with no modality additions equals the plain string + * matching its common text (mirrors Python's `__eq__`). */ export function instructionsEqual( a: string | Instructions | undefined, @@ -193,41 +196,18 @@ export function instructionsEqual( const aIsInstr = isInstructions(a); const bIsInstr = isInstructions(b); if (aIsInstr && bIsInstr) { - return a.audio === b.audio && a.text === b.text; + return a.common === b.common && a.audio === b.audio && a.text === b.text; } - if (!aIsInstr && !bIsInstr) { - return a === b; + if (aIsInstr && !bIsInstr) { + return a.common === b; } - return false; -} - -/** - * Concatenate any mix of plain strings and {@link Instructions}, propagating - * both audio/text variants. If no argument is an {@link Instructions} the - * result is a plain string; otherwise the result is an {@link Instructions} - * preserving both variants from every contributing operand. - */ -export function concatInstructions(...parts: Array): string | Instructions { - if (parts.length === 0) return ''; - const hasInstructions = parts.some((p) => isInstructions(p)); - if (!hasInstructions) return parts.join(''); - - let acc = parts[0]!; - for (let i = 1; i < parts.length; i++) { - const next = parts[i]!; - if (isInstructions(acc)) { - acc = acc.concat(next); - } else if (isInstructions(next)) { - // string + Instructions (radd-style): prepend `acc` to both variants. - acc = new Instructions({ audio: acc }).concat(next); - } else { - acc = acc + next; - } + if (!aIsInstr && bIsInstr) { + return a === b.common; } - return acc; + return a === b; } -export type ChatContent = ImageContent | AudioContent | Instructions | string; +export type ChatContent = ImageContent | AudioContent | string; export function createImageContent(params: { image: string | VideoFrame; @@ -361,9 +341,7 @@ export class ChatMessage { * lines. If no string content is present, returns `null`. */ get textContent(): string | undefined { - const parts = this.content - .filter((c): c is string | Instructions => typeof c === 'string' || isInstructions(c)) - .map((c) => (typeof c === 'string' ? c : c.value)); + const parts = this.content.filter((c): c is string => typeof c === 'string'); return parts.length > 0 ? parts.join('\n') : undefined; } @@ -371,8 +349,6 @@ export class ChatMessage { return this.content.map((c) => { if (typeof c === 'string') { return c as JSONValue; - } else if (isInstructions(c)) { - return c.toJSON() as JSONValue; } else if (c.type === 'image_content') { return { id: c.id, @@ -650,7 +626,7 @@ export class AgentConfigUpdate { readonly type = 'agent_config_update' as const; - instructions?: string | Instructions; + instructions?: string; toolsAdded?: string[]; @@ -661,7 +637,7 @@ export class AgentConfigUpdate { constructor( params: { id?: string; - instructions?: string | Instructions; + instructions?: string; toolsAdded?: string[]; toolsRemoved?: string[]; createdAt?: number; @@ -683,7 +659,7 @@ export class AgentConfigUpdate { static create(params: { id?: string; - instructions?: string | Instructions; + instructions?: string; toolsAdded?: string[]; toolsRemoved?: string[]; createdAt?: number; @@ -698,9 +674,7 @@ export class AgentConfigUpdate { }; if (this.instructions !== undefined) { - result.instructions = isInstructions(this.instructions) - ? (this.instructions.toJSON() as JSONValue) - : this.instructions; + result.instructions = this.instructions; } if (this.toolsAdded !== undefined) { result.toolsAdded = this.toolsAdded; @@ -747,7 +721,7 @@ export class ChatContext { */ addMessage(params: { role: ChatRole; - content: ChatContent[] | string; + content: ChatContent[] | string | Instructions; id?: string; interrupted?: boolean; createdAt?: number; @@ -755,7 +729,8 @@ export class ChatContext { metrics?: MetricsReport; extra?: Record; }): ChatMessage { - const msg = new ChatMessage(params); + const content = isInstructions(params.content) ? params.content.toString() : params.content; + const msg = new ChatMessage({ ...params, content }); if (params.createdAt !== undefined) { const idx = this.findInsertionIndex(params.createdAt); this._items.splice(idx, 0, msg); @@ -1087,21 +1062,6 @@ export class ChatContext { return false; } - if (isInstructions(contentA) && isInstructions(contentB)) { - if ( - contentA.audio !== contentB.audio || - contentA.text !== contentB.text || - contentA.value !== contentB.value - ) { - return false; - } - continue; - } - - if (isInstructions(contentA) || isInstructions(contentB)) { - return false; - } - if (typeof contentA === 'object' && typeof contentB === 'object') { if (contentA.type === 'image_content' && contentB.type === 'image_content') { if ( diff --git a/agents/src/llm/index.ts b/agents/src/llm/index.ts index 4773501dc..22e19d2ee 100644 --- a/agents/src/llm/index.ts +++ b/agents/src/llm/index.ts @@ -46,7 +46,6 @@ export { ChatContext, ChatMessage, Instructions, - concatInstructions, createAudioContent, createImageContent, FunctionCall, diff --git a/agents/src/llm/provider_format/google.test.ts b/agents/src/llm/provider_format/google.test.ts index c895a4bde..54588f182 100644 --- a/agents/src/llm/provider_format/google.test.ts +++ b/agents/src/llm/provider_format/google.test.ts @@ -64,20 +64,21 @@ describe('Google Provider Format - toChatCtx', () => { }); it('should render Instructions as their resolved value', async () => { + // Instructions are resolved to a plain string before storage in the chat context const ctx = ChatContext.empty(); ctx.addMessage({ role: 'system', content: [ - new Instructions({ audio: 'audio instructions', text: 'text instructions' }).asModality( - 'text', - ), + new Instructions('common instructions', { text: 'text instructions' }).render({ + modality: 'text', + }), ], }); ctx.addMessage({ role: 'user', content: 'Hello' }); const [, formatData] = await toChatCtx(ctx, false); - expect(formatData.systemMessages).toEqual(['text instructions']); + expect(formatData.systemMessages).toEqual(['common instructions\n\ntext instructions']); }); it('should handle multiple system messages', async () => { diff --git a/agents/src/llm/provider_format/google.ts b/agents/src/llm/provider_format/google.ts index 2ef717f28..6902dd702 100644 --- a/agents/src/llm/provider_format/google.ts +++ b/agents/src/llm/provider_format/google.ts @@ -2,7 +2,6 @@ // // SPDX-License-Identifier: Apache-2.0 import type { ChatContext, ChatItem, ImageContent } from '../chat_context.js'; -import { isInstructions } from '../chat_context.js'; import { type SerializedImage, serializeImage } from '../utils.js'; import { convertMidConversationInstructions, groupToolCalls } from './utils.js'; @@ -60,8 +59,6 @@ export async function toChatCtx( for (const content of msg.content) { if (content && typeof content === 'string') { parts.push({ text: content }); - } else if (isInstructions(content)) { - parts.push({ text: content.value }); } else if (content && typeof content === 'object') { if (content.type === 'image_content') { parts.push(await toImagePart(content)); diff --git a/agents/src/llm/provider_format/mistralai.test.ts b/agents/src/llm/provider_format/mistralai.test.ts index db7387637..45a04f94a 100644 --- a/agents/src/llm/provider_format/mistralai.test.ts +++ b/agents/src/llm/provider_format/mistralai.test.ts @@ -41,20 +41,21 @@ describe('Mistral Provider Format - toChatCtx', () => { }); it('should render Instructions as their resolved value', () => { + // Instructions are resolved to a plain string before storage in the chat context const ctx = ChatContext.empty(); ctx.addMessage({ role: 'system', content: [ - new Instructions({ audio: 'audio instructions', text: 'text instructions' }).asModality( - 'text', - ), + new Instructions('common instructions', { text: 'text instructions' }).render({ + modality: 'text', + }), ], }); ctx.addMessage({ role: 'user', content: 'Hello' }); const [, formatData] = toChatCtx(ctx); - expect(formatData.instructions).toBe('text instructions'); + expect(formatData.instructions).toBe('common instructions\n\ntext instructions'); }); it('should extract developer messages as instructions', () => { diff --git a/agents/src/llm/provider_format/mistralai.ts b/agents/src/llm/provider_format/mistralai.ts index 776be3085..b5b0d1723 100644 --- a/agents/src/llm/provider_format/mistralai.ts +++ b/agents/src/llm/provider_format/mistralai.ts @@ -2,7 +2,6 @@ // // SPDX-License-Identifier: Apache-2.0 import type { ChatContext } from '../chat_context.js'; -import { isInstructions } from '../chat_context.js'; export interface MistralFormatData { instructions: string; @@ -28,10 +27,7 @@ export function toChatCtx( for (const item of chatCtx.items) { if (item.type === 'message') { - const text = item.content - .filter((c) => typeof c === 'string' || isInstructions(c)) - .map((c) => (typeof c === 'string' ? c : c.value)) - .join('\n'); + const text = item.content.filter((c) => typeof c === 'string').join('\n'); if (item.role === 'system' || item.role === 'developer') { instructionParts.push(text); diff --git a/agents/src/llm/provider_format/openai.test.ts b/agents/src/llm/provider_format/openai.test.ts index ae74bac7b..c810b99a7 100644 --- a/agents/src/llm/provider_format/openai.test.ts +++ b/agents/src/llm/provider_format/openai.test.ts @@ -56,20 +56,24 @@ describe('toChatCtx', () => { }); it('should render Instructions as their resolved value', async () => { + // Instructions are resolved to a plain string before storage in the chat context const ctx = ChatContext.empty(); ctx.addMessage({ role: 'system', content: [ - new Instructions({ audio: 'audio instructions', text: 'text instructions' }).asModality( - 'text', - ), + new Instructions('common instructions', { text: 'text instructions' }).render({ + modality: 'text', + }), ], }); ctx.addMessage({ role: 'user', content: 'Hello' }); const result = await toChatCtx(ctx); - expect(result[0]).toEqual({ role: 'system', content: 'text instructions' }); + expect(result[0]).toEqual({ + role: 'system', + content: 'common instructions\n\ntext instructions', + }); }); it('should handle multi-line text content', async () => { @@ -758,20 +762,24 @@ describe('toResponsesChatCtx', () => { }); it('should render Instructions as their resolved value', async () => { + // Instructions are resolved to a plain string before storage in the chat context const ctx = ChatContext.empty(); ctx.addMessage({ role: 'system', content: [ - new Instructions({ audio: 'audio instructions', text: 'text instructions' }).asModality( - 'text', - ), + new Instructions('common instructions', { text: 'text instructions' }).render({ + modality: 'text', + }), ], }); ctx.addMessage({ role: 'user', content: 'Hello' }); const result = await toResponsesChatCtx(ctx); - expect(result[0]).toEqual({ role: 'system', content: 'text instructions' }); + expect(result[0]).toEqual({ + role: 'system', + content: 'common instructions\n\ntext instructions', + }); }); it('should handle multi-line text content', async () => { diff --git a/agents/src/llm/provider_format/openai.ts b/agents/src/llm/provider_format/openai.ts index f9efe4798..8f630ec43 100644 --- a/agents/src/llm/provider_format/openai.ts +++ b/agents/src/llm/provider_format/openai.ts @@ -2,7 +2,6 @@ // // SPDX-License-Identifier: Apache-2.0 import type { ChatContext, ChatItem, ImageContent } from '../chat_context.js'; -import { isInstructions } from '../chat_context.js'; import { type SerializedImage, serializeImage } from '../utils.js'; import { groupToolCalls } from './utils.js'; @@ -70,9 +69,6 @@ async function toChatItem(item: ChatItem) { if (typeof content === 'string') { if (textContent) textContent += '\n'; textContent += content; - } else if (isInstructions(content)) { - if (textContent) textContent += '\n'; - textContent += content.value; } else if (content.type === 'image_content') { listContent.push(await toImageContent(content)); } else { @@ -233,9 +229,6 @@ async function toResponsesChatItem(item: ChatItem) { if (typeof content === 'string') { if (textContent) textContent += '\n'; textContent += content; - } else if (isInstructions(content)) { - if (textContent) textContent += '\n'; - textContent += content.value; } else if (content.type === 'image_content') { listContent.push(await toResponsesImageContent(content)); } else { diff --git a/agents/src/llm/utils.ts b/agents/src/llm/utils.ts index 0db2bb802..1ee247cc0 100644 --- a/agents/src/llm/utils.ts +++ b/agents/src/llm/utils.ts @@ -460,10 +460,6 @@ export function validateChatContextStructure(chatCtx: ChatContext): ChatContextV return; } - if (term.type === 'instructions') { - return; - } - if (term.type === 'image_content') { if (!term.id || term.image === undefined || term.image === null) { pushIssue({ @@ -614,10 +610,6 @@ function formatMessageContentPart(part: ChatContent): string { return part; } - if (part.type === 'instructions') { - return part.value; - } - if (part.type === 'image_content') { if (typeof part.image === 'string') { return `[image url=${truncateText(part.image, 120)}]`; diff --git a/agents/src/stt/index.ts b/agents/src/stt/index.ts index 5b0517546..f9a6815a7 100644 --- a/agents/src/stt/index.ts +++ b/agents/src/stt/index.ts @@ -10,6 +10,7 @@ export { export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js'; export { type RecognitionUsage, + type SpeakerContext, type SpeechData, type SpeechEvent, SpeechEventType, @@ -17,5 +18,6 @@ export { STT, type STTCallbacks, type STTCapabilities, + isSpeakerContext, } from './stt.js'; export * as testing from './testing/index.js'; diff --git a/agents/src/stt/stt.ts b/agents/src/stt/stt.ts index 5bd922e32..3d307a6de 100644 --- a/agents/src/stt/stt.ts +++ b/agents/src/stt/stt.ts @@ -239,6 +239,27 @@ export abstract class STT extends (EventEmitter as new () => TypedEmitter private logger = log(); private _connOptions: APIConnectOptions; private _startTimeOffset: number = 0; + private _context?: object; protected abortController = new AbortController(); @@ -415,6 +437,21 @@ export abstract class SpeechStream implements AsyncIterableIterator return this.abortController.signal; } + /** + * Persistent speaker metadata updated by the STT plugin during recognition. + * + * STT plugins set this to a context object (e.g. a voice profile with + * emotion, gender, etc.) as they learn about the speaker over the life of the + * stream. Accessible via `agent.audioRecognition.sttContext`. + */ + get context(): object | undefined { + return this._context; + } + + set context(value: object | undefined) { + this._context = value; + } + get startTimeOffset(): number { return this._startTimeOffset; } diff --git a/agents/src/telemetry/traces.ts b/agents/src/telemetry/traces.ts index e1a14b4de..a8f14d3b6 100644 --- a/agents/src/telemetry/traces.ts +++ b/agents/src/telemetry/traces.ts @@ -24,7 +24,7 @@ import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions'; import FormData from 'form-data'; import { AccessToken } from 'livekit-server-sdk'; import fs from 'node:fs/promises'; -import { isInstructions, renderInstructions } from '../llm/chat_context.js'; +import { renderInstructions } from '../llm/chat_context.js'; import type { ChatContent, ChatItem, ChatRole } from '../llm/index.js'; import { enableOtelLogging } from '../log.js'; import { filterZeroValues } from '../metrics/model_usage.js'; @@ -415,10 +415,8 @@ function chatItemToProto(item: ChatItem): ProtoChatItem { // the ChatContent proto's `text` field is a string, and non-string values // render as garbage in the dashboard. content: item.content - .filter((c: ChatContent) => typeof c === 'string' || isInstructions(c)) - .map((c) => ({ - text: isInstructions(c) ? c.value : (c as string), - })), + .filter((c: ChatContent) => typeof c === 'string') + .map((c) => ({ text: c })), createdAt: toRFC3339(item.createdAt), }; diff --git a/agents/src/tokenize/basic/basic.ts b/agents/src/tokenize/basic/basic.ts index dbbf1e5cf..bd87ac95f 100644 --- a/agents/src/tokenize/basic/basic.ts +++ b/agents/src/tokenize/basic/basic.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js'; +import { BufferedSentenceStream, BufferedWordStream, xmlWrapTokenizer } from '../token_stream.js'; import * as tokenizer from '../tokenizer.js'; import { hyphenator } from './hyphenator.js'; import { splitParagraphs } from './paragraph.js'; @@ -10,9 +10,34 @@ import { splitWords } from './word.js'; interface TokenizerOptions { language: string; + /** + * Minimum length for a span to be treated as its own sentence; shorter spans + * are merged forward into the next one. + */ minSentenceLength: number; + /** Minimum buffered text before the stream emits. */ streamContextLength: number; + /** Keep original whitespace/formatting in emitted tokens. */ retainFormat: boolean; + /** + * Hard cap on emitted token length; a token is flushed before appending a + * sentence that would exceed it. + */ + maxTokenLength?: number; + /** + * Minimum length a token must reach before it is emitted. Sentences are + * batched together until the running token reaches this length, so raising + * it (e.g. toward `maxTokenLength`) yields larger, fewer chunks. Defaults to + * `minSentenceLength` (per-sentence emission). + */ + minTokenLength?: number; + /** + * Treat XML markup as atomic — never split a tag across tokens and keep tags + * attached to the following sentence. Only enable when the input actually + * carries markup (e.g. expressive TTS): a stray "<" in plain text can + * otherwise hold back streaming until flush. + */ + xmlAware?: boolean; } const defaultTokenizerOptions: TokenizerOptions = { @@ -35,7 +60,12 @@ export class SentenceTokenizer extends tokenizer.SentenceTokenizer { // eslint-disable-next-line @typescript-eslint/no-unused-vars tokenize(text: string, language?: string): string[] { - return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]); + let tokenizeFunc = (t: string) => + splitSentences(t, this.#config.minSentenceLength, this.#config.retainFormat); + if (this.#config.xmlAware) { + tokenizeFunc = xmlWrapTokenizer(tokenizeFunc) as typeof tokenizeFunc; + } + return tokenizeFunc(text).map((tok) => (Array.isArray(tok) ? tok[0] : tok)); } // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -43,8 +73,12 @@ export class SentenceTokenizer extends tokenizer.SentenceTokenizer { return new BufferedSentenceStream( (text: string) => splitSentences(text, this.#config.minSentenceLength, this.#config.retainFormat), - this.#config.minSentenceLength, + this.#config.minTokenLength ?? this.#config.minSentenceLength, this.#config.streamContextLength, + { + maxTokenLength: this.#config.maxTokenLength, + xmlAware: this.#config.xmlAware, + }, ); } } diff --git a/agents/src/tokenize/basic/sentence.ts b/agents/src/tokenize/basic/sentence.ts index a6df4b2c7..2a4e5833e 100644 --- a/agents/src/tokenize/basic/sentence.ts +++ b/agents/src/tokenize/basic/sentence.ts @@ -82,7 +82,10 @@ export const splitSentences = ( } if (buf) { - sentences.push([buf.slice(prePad.length), start, text.length - 1]); + // end must cover the full remaining text: xmlWrapTokenizer maps this offset + // back onto the original string, and an off-by-one here splits the last + // character into a phantom sentence (emitting mid-word chunks upstream). + sentences.push([buf.slice(prePad.length), start, text.length]); } return sentences; diff --git a/agents/src/tokenize/token_stream.ts b/agents/src/tokenize/token_stream.ts index 9b383fe9b..34bb38a1a 100644 --- a/agents/src/tokenize/token_stream.ts +++ b/agents/src/tokenize/token_stream.ts @@ -7,6 +7,173 @@ import { SentenceStream, WordStream } from './tokenizer.js'; type TokenizeFunc = (x: string) => string[] | [string, number, number][]; +// the tag name must start with a letter so "<5>" / "<3 wins>" are not counted as +// tags — this keeps the depth counter consistent with the letter-start tail check +// in hasUnclosedXmlTags (all TTS markup tags are letter-named) +const XML_TAG_RE = /<(\/?)([A-Za-z]\w*)[^>]*?(\/?)\s*>/g; + +/** Return true if `text` contains an incomplete or unclosed XML tag. */ +export function hasUnclosedXmlTags(text: string): boolean { + if (!text.includes('<')) { + return false; + } + + // incomplete tag at end: a tag-shaped "<" without a matching ">". Only "<" + // followed by a name start ("/" or a letter) is tag-shaped — a bare "<" as in + // "3 < 5" or "<3" is plain text and must not hold up streaming. Text ending + // exactly at "<" is treated as tag-shaped: the next chunk resolves it. + const lastOpen = text.lastIndexOf('<'); + const lastClose = text.lastIndexOf('>'); + if (lastOpen > lastClose) { + const nxt = text.slice(lastOpen + 1, lastOpen + 2); + if (!nxt || nxt === '/' || /[a-zA-Z]/.test(nxt)) { + return true; + } + } + + // unbalanced open/close pairs + let depth = 0; + for (const m of text.matchAll(XML_TAG_RE)) { + const isClosing = m[1] === '/'; + const isSelfClosing = m[3] === '/'; + if (isSelfClosing) { + continue; + } else if (isClosing) { + depth -= 1; + } else { + depth += 1; + } + } + + return depth > 0; +} + +/** Return true if `text` contains XML tags but no substantive text content. */ +export function isXmlOnly(text: string): boolean { + if (!text.includes('<')) { + return false; + } + + const stripped = text.replace(XML_TAG_RE, '').trim(); + return stripped.length === 0; +} + +/** + * Map a position in tag-stripped text to the corresponding original position. + * + * Tags that sit right at the boundary are left for the next sentence. + */ +function cleanToOrig(cleanPos: number, tagSpans: [number, number][]): number { + let orig = cleanPos; + for (const [tagStart, tagEnd] of tagSpans) { + if (tagStart < orig) { + orig += tagEnd - tagStart; + } else { + break; + } + } + return orig; +} + +/** + * Wrap a tokenizer so XML tags don't interfere with sentence splitting. + * + * Strips tag markers before tokenization (content inside wrapping tags is + * kept so the tokenizer can account for its length), remaps offsets back to + * the original text, and merges sentences with unclosed or tag-only content. + */ +export function xmlWrapTokenizer(tokenizeFunc: TokenizeFunc): TokenizeFunc { + const wrappedImpl = (text: string): string[] | [string, number, number][] => { + const tagSpans: [number, number][] = [...text.matchAll(XML_TAG_RE)].map((m) => [ + m.index!, + m.index! + m[0].length, + ]); + if (tagSpans.length === 0) { + return tokenizeFunc(text); + } + + const cleanText = text.replace(XML_TAG_RE, ''); + if (!cleanText.trim()) { + return text.trim() ? [[text, 0, text.length]] : []; + } + + const rawTokens = tokenizeFunc(cleanText); + if (rawTokens.length === 0) { + return []; + } + + // extract clean-text end offsets + let cleanEnds: number[] = rawTokens.map((tok) => (Array.isArray(tok) ? tok[2] : -1)); + + // if tokenizer didn't provide offsets, approximate from token lengths + if (cleanEnds[0] === -1) { + let pos = 0; + cleanEnds = []; + for (const tok of rawTokens) { + const tokText = typeof tok === 'string' ? tok : tok[0]; + const idx = cleanText.indexOf(tokText, pos); + pos = (idx >= 0 ? idx : pos) + tokText.length; + cleanEnds.push(pos); + } + } + + // remap to original positions and rebuild sentences + let result: [string, number, number][] = []; + let start = 0; + for (const cleanEnd of cleanEnds) { + const origEnd = cleanToOrig(cleanEnd, tagSpans); + const sentence = text.slice(start, origEnd).trim(); + if (sentence) { + result.push([sentence, start, origEnd]); + } + start = origEnd; + } + + if (start < text.length) { + const sentence = text.slice(start).trim(); + if (sentence) { + result.push([sentence, start, text.length]); + } + } + + // merge sentences with unclosed tags or tag-only content + if (result.length > 0) { + const merged: [string, number, number][] = [result[0]!]; + for (const [sentText, sStart, sEnd] of result.slice(1)) { + const [prevText, prevStart] = merged[merged.length - 1]!; + if (hasUnclosedXmlTags(prevText) || isXmlOnly(prevText)) { + merged[merged.length - 1] = [text.slice(prevStart, sEnd).trim(), prevStart, sEnd]; + } else { + merged.push([sentText, sStart, sEnd]); + } + } + result = merged; + } + + return result; + }; + + return (text: string) => { + try { + return wrappedImpl(text); + } catch { + return text.trim() ? [[text, 0, text.length]] : []; + } + }; +} + +export interface BufferedTokenStreamOptions { + /** Hard cap on emitted token length; the buffer is flushed before appending a token that would exceed it. */ + maxTokenLength?: number; + /** + * Treat XML markup as atomic — never split a tag across tokens and merge + * tag-only/unclosed spans forward. Only enable when the input actually + * carries markup (e.g. expressive TTS): a stray "<" in plain text can + * otherwise hold back streaming until flush. + */ + xmlAware?: boolean; +} + export class BufferedTokenStream implements AsyncIterableIterator { protected queue = new AsyncIterableQueue(); protected closed = false; @@ -14,15 +181,23 @@ export class BufferedTokenStream implements AsyncIterableIterator { #func: TokenizeFunc; #minTokenLength: number; #minContextLength: number; - #bufTokens: string[] = []; + #maxTokenLength?: number; + #xmlAware: boolean; #inBuf = ''; #outBuf = ''; #currentSegmentId: string; - constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) { - this.#func = func; + constructor( + func: TokenizeFunc, + minTokenLength: number, + minContextLength: number, + options: BufferedTokenStreamOptions = {}, + ) { + this.#xmlAware = options.xmlAware ?? false; + this.#func = this.#xmlAware ? xmlWrapTokenizer(func) : func; this.#minTokenLength = minTokenLength; this.#minContextLength = minContextLength; + this.#maxTokenLength = options.maxTokenLength; this.#currentSegmentId = shortuuid(); } @@ -33,6 +208,7 @@ export class BufferedTokenStream implements AsyncIterableIterator { throw new Error('Stream is closed'); } + if (!text) return; this.#inBuf += text; if (this.#inBuf.length < this.#minContextLength) return; @@ -40,16 +216,26 @@ export class BufferedTokenStream implements AsyncIterableIterator { const tokens = this.#func(this.#inBuf); if (tokens.length <= 1) break; - if (this.#outBuf) this.#outBuf += ' '; + const tok = tokens[0]!; + const tokText: string = Array.isArray(tok) ? tok[0] : tok; - const tok = tokens.shift()!; - let tokText: string; - if (Array.isArray(tok)) { - tokText = tok[0]; - } else { - tokText = tok; + // don't emit a token that would split an XML tag + if (this.#xmlAware && hasUnclosedXmlTags(tokText)) break; + + tokens.shift(); + + // if adding this sentence would exceed max, emit what we have first + if ( + this.#maxTokenLength && + this.#outBuf && + this.#outBuf.length + 1 + tokText.length > this.#maxTokenLength + ) { + this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId }); + this.#outBuf = ''; } + if (this.#outBuf) this.#outBuf += ' '; + this.#outBuf += tokText; if (this.#outBuf.length >= this.#minTokenLength) { @@ -57,8 +243,8 @@ export class BufferedTokenStream implements AsyncIterableIterator { this.#outBuf = ''; } - if (typeof tok! !== 'string') { - this.#inBuf = this.#inBuf.slice(tok![2]); + if (Array.isArray(tok)) { + this.#inBuf = this.#inBuf.slice(tok[2]); } else { this.#inBuf = this.#inBuf .slice(Math.max(0, this.#inBuf.indexOf(tok)) + tok.length) @@ -75,14 +261,23 @@ export class BufferedTokenStream implements AsyncIterableIterator { if (this.#inBuf || this.#outBuf) { const tokens = this.#func(this.#inBuf); - if (tokens) { - if (this.#outBuf) this.#outBuf += ' '; - - if (Array.isArray(tokens[0])) { - this.#outBuf += tokens.map((tok) => tok[0]).join(' '); - } else { - this.#outBuf += tokens.join(' '); + for (const tok of tokens) { + const tokText: string = Array.isArray(tok) ? tok[0] : tok; + + // honor the cap here too: appending everything into one chunk could + // exceed maxTokenLength and trip a provider's send limit. Emit the + // buffer before it would overflow, then keep batching the rest. + if ( + this.#maxTokenLength && + this.#outBuf && + this.#outBuf.length + 1 + tokText.length > this.#maxTokenLength + ) { + this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId }); + this.#outBuf = ''; } + + if (this.#outBuf) this.#outBuf += ' '; + this.#outBuf += tokText; } if (this.#outBuf) { @@ -123,9 +318,14 @@ export class BufferedTokenStream implements AsyncIterableIterator { export class BufferedSentenceStream extends SentenceStream { #stream: BufferedTokenStream; - constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) { + constructor( + func: TokenizeFunc, + minTokenLength: number, + minContextLength: number, + options: BufferedTokenStreamOptions = {}, + ) { super(); - this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength); + this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength, options); } pushText(text: string) { @@ -155,7 +355,9 @@ export class BufferedWordStream extends WordStream { constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) { super(); - this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength); + this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength, { + xmlAware: false, + }); } pushText(text: string) { diff --git a/agents/src/tokenize/xml_markup.test.ts b/agents/src/tokenize/xml_markup.test.ts new file mode 100644 index 000000000..854be6b39 --- /dev/null +++ b/agents/src/tokenize/xml_markup.test.ts @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * Regression tests: sentence tokenizers must handle XML markup correctly. + * + * Covers the basic sentence tokenizer (batch + streaming) with TTS markup tags + * used in expressive mode (Cartesia, Inworld, xAI). + */ +import { describe, expect, it } from 'vitest'; +import { SentenceTokenizer } from './basic/index.js'; +import { hasUnclosedXmlTags } from './token_stream.js'; +import type { SentenceStream } from './tokenizer.js'; + +const XML_TAG_RE = /<(\/?)([A-Za-z]\w*)[^>]*?(\/?)\s*>/g; + +/** If a sentence has ``, it must also have `` (not split). */ +function assertWrappingTagIntact(sentences: string[], tag: string): void { + for (const s of sentences) { + if (s.includes(`<${tag}`) && !s.includes(``) && !s.includes('/>')) { + expect.fail(`<${tag}> split across sentences: ${JSON.stringify(sentences)}`); + } + } +} + +/** No sentence should be purely XML tags with no text content. */ +function assertNoTagOnlySentences(sentences: string[]): void { + for (const s of sentences) { + if (s.includes('<')) { + expect(s.replace(XML_TAG_RE, '').trim(), `Tag-only sentence: ${JSON.stringify(s)}`).not.toBe( + '', + ); + } + } +} + +async function collect(stream: SentenceStream): Promise { + const tokens: string[] = []; + for await (const ev of stream) { + tokens.push(ev.token); + } + return tokens; +} + +/** Push text char-by-char (worst-case chunking) and collect emitted tokens. */ +async function streamTokenize(tok: SentenceTokenizer, text: string): Promise { + const stream = tok.stream(); + for (const char of text) { + stream.pushText(char); + } + stream.endInput(); + return collect(stream); +} + +/** Await the next stream token, failing if it doesn't arrive without a flush. */ +async function nextTokenWithTimeout(stream: SentenceStream, timeoutMs = 1000): Promise { + const result = await Promise.race([ + stream.next(), + new Promise((_, reject) => + setTimeout(() => reject(new Error('timed out waiting for a token')), timeoutMs), + ), + ]); + if (result.done || !result.value) { + throw new Error('stream ended without emitting a token'); + } + return result.value.token; +} + +describe('hasUnclosedXmlTags', () => { + it('does not treat a bare "<" as a tag', () => { + expect(hasUnclosedXmlTags('3 < 5.')).toBe(false); + expect(hasUnclosedXmlTags('i <3 you')).toBe(false); + expect(hasUnclosedXmlTags('price < 10 dollars')).toBe(false); + // tag-shaped: must still hold + expect(hasUnclosedXmlTags('Hello abc')).toBe(true); // unclosed wrapping tag + }); + + it('does not count digit-named pseudo tags as open tags', () => { + // regression: the depth-counter regex must not treat "<5>" / "<3 wins>" as + // open tags, or a complete-but-digit-named pair would leave depth > 0 and + // stall streaming for the rest of the turn + expect(hasUnclosedXmlTags('Rate this from <1> to <5> please.')).toBe(false); + expect(hasUnclosedXmlTags('Scores: <3 wins> today.')).toBe(false); + // a real letter-named tag pair is still balanced + expect(hasUnclosedXmlTags('abc done')).toBe(false); + }); +}); + +describe('batch sentence tokenizer (xmlAware)', () => { + const tok = new SentenceTokenizer({ minSentenceLength: 1, xmlAware: true }); + + it('splits correctly with expression tags between sentences', () => { + // Regression: a self-closing tag between sentences must not confuse boundary + // detection; each tag goes with its own sentence. + const text = + ' Hello and welcome! ' + + ' Great specials today. ' + + ' Try our new sandwich.'; + const sentences = tok.tokenize(text); + expect(sentences, `Expected 3 sentences: ${JSON.stringify(sentences)}`).toHaveLength(3); + expect(sentences[0]).toContain(''); + expect(sentences[1]).toContain(''); + expect(sentences[2]).toContain(''); + assertNoTagOnlySentences(sentences); + }); + + it('merges a standalone tag with the following text', () => { + // Regression: a self-closing tag as its own sentence must merge with the next + // so TTS never receives a tag-only chunk. + const text = ' I told you already, no changes to the order.'; + const sentences = tok.tokenize(text); + assertNoTagOnlySentences(sentences); + }); + + it('keeps a wrapping tag with inner periods intact', () => { + // Dots inside look like sentence endings. Merge must keep tag intact. + const text = 'Spell it: U.S.A.. Got it?'; + const sentences = tok.tokenize(text); + assertWrappingTagIntact(sentences, 'spell'); + }); + + it('keeps full sentences inside a wrapping tag together', () => { + const text = + 'Read this: The quick brown fox. The cat sat on the mat.. ' + + 'Now something else.'; + const sentences = tok.tokenize(text); + assertWrappingTagIntact(sentences, 'spell'); + }); + + it('handles mixed self-closing + wrapping + break tags', () => { + const text = + ' Great news! ' + + 'The code is X9Z. ' + + ' Let me explain.'; + const sentences = tok.tokenize(text); + assertWrappingTagIntact(sentences, 'spell'); + assertNoTagOnlySentences(sentences); + }); + + it('still splits text without markup', () => { + const sentences = tok.tokenize('Hello there. How are you? I am fine.'); + expect(sentences.length).toBeGreaterThanOrEqual(2); + }); + + it('emits a single token for tag-only input', () => { + const sentences = tok.tokenize(''); + expect(sentences).toHaveLength(1); + }); +}); + +describe('streaming sentence tokenizer (xmlAware)', () => { + const tok = new SentenceTokenizer({ + minSentenceLength: 1, + streamContextLength: 5, + xmlAware: true, + }); + + it('splits streamed expression tags between sentences', async () => { + const text = + ' Hello and welcome! ' + + ' We have got some great specials. ' + + ' Our new chicken sandwich is amazing. ' + + ' Would you like to try a combo meal?'; + const tokens = await streamTokenize(tok, text); + expect( + tokens.length, + `Expected at least 3 sentences: ${JSON.stringify(tokens)}`, + ).toBeGreaterThanOrEqual(3); + assertNoTagOnlySentences(tokens); + for (const t of tokens) { + expect(t, `Sentence missing expression tag: ${JSON.stringify(t)}`).toContain(' { + const text = + ' Thank you for calling. ' + + 'How can I help you today? ' + + ' ' + + ' I understand your frustration. ' + + 'Let me look into this for you. ' + + 'Your order number is A.B.1.2.3.. ' + + ' I found the issue. ' + + ' The refund will be processed in 3 to 5 business days. ' + + ' Is there anything else I can help with?'; + const tokens = await streamTokenize(tok, text); + assertWrappingTagIntact(tokens, 'spell'); + assertNoTagOnlySentences(tokens); + }); + + it('never splits mid-word when streamed in small chunks', async () => { + // Regression: splitSentences reported the final (incomplete) sentence with an + // off-by-one end offset; xmlWrapTokenizer remapped it onto the original text and + // split the last character into a phantom sentence, emitting "Hell o!" mid-word. + const text = ` Hello! It's great to see you. How can I assist you today?`; + const stream = tok.stream(); + for (let i = 0; i < text.length; i += 4) { + stream.pushText(text.slice(i, i + 4)); + } + stream.endInput(); + const tokens = await collect(stream); + const collapse = (s: string) => s.replace(/\s+/g, ' ').trim(); + expect(collapse(tokens.join(' '))).toBe(collapse(text)); + }); + + it('streams past a bare "<" in plain text without stalling', async () => { + // Regression: one "3 < 5" used to hold every following sentence until flush, + // degrading streaming TTS to end-of-turn batching for the rest of the turn. + const stream = tok.stream(); + stream.pushText('Note that 3 < 5 holds. And here is a second sentence to tokenize.'); + const token = await nextTokenWithTimeout(stream); + expect(token).toContain('3 < 5'); + stream.endInput(); + }); + + it('streams past digit-named pseudo tags', async () => { + const stream = tok.stream(); + stream.pushText('Rate this from <1> to <5>. And here is a second sentence to split.'); + const token = await nextTokenWithTimeout(stream); + expect(token.includes('<5>') || token.includes('<1>')).toBe(true); + stream.endInput(); + }); +}); + +describe('streaming sentence tokenizer (not xmlAware)', () => { + it('streams tag-shaped plain text sentence by sentence', async () => { + // the default tokenizer (non-expressive agents) applies no XML logic at all, + // so even tag-shaped plain text must stream sentence by sentence + const tok = new SentenceTokenizer({ minSentenceLength: 1, streamContextLength: 5 }); + const stream = tok.stream(); + stream.pushText('Email me at please. Second sentence for the split.'); + const token = await nextTokenWithTimeout(stream); + expect(token).toContain('bob@example.com'); + stream.endInput(); + }); +}); diff --git a/agents/src/tts/_provider_format.ts b/agents/src/tts/_provider_format.ts new file mode 100644 index 000000000..66058816c --- /dev/null +++ b/agents/src/tts/_provider_format.ts @@ -0,0 +1,779 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * Shared provider-specific TTS formatting logic. + * + * Both TTS plugins and the inference gateway delegate to this module so + * there is a single source of truth for LLM instructions and markup stripping + * per provider. + * + * Provider docs: + * - Cartesia: https://docs.cartesia.ai/build-with-cartesia/sonic-3/ssml-tags + * - Cartesia: https://docs.cartesia.ai/build-with-cartesia/sonic-3/volume-speed-emotion + * - Inworld: https://docs.inworld.ai/tts/capabilities/steering + * - Inworld: https://docs.inworld.ai/tts/best-practices/prompting-for-tts-2 + * - xAI: https://docs.x.ai/developers/model-capabilities/audio/text-to-speech + * - xAI: https://docs.x.ai/developers/model-capabilities/audio/voice + */ +import { ATTRIBUTE_TRANSCRIPTION_EXPRESSION } from '../constants.js'; +import { Instructions } from '../llm/chat_context.js'; +import { SentenceTokenizer } from '../tokenize/basic/index.js'; +import type { ExpressiveOptions } from '../voice/agent_session.js'; +import { convertExpressionTags, extractAndStrip } from './markup_utils.js'; + +/** + * An expressive markup tag stripped from a transcript, surfaced for the frontend. + * + * `type` is the markup tag name (`"emotion"`, `"expression"`, `"sound"`, ...), + * or `""` for square-bracket tags which carry no name. `value` is the spoken or + * semantic payload (the `value="..."` attribute, the tag's inner text, or the bracket + * content). + */ +export interface ExpressiveTag { + type: string; + value: string; +} + +const CARTESIA_TAGS = ['emotion', 'speed', 'volume', 'break', 'spell']; + +const CARTESIA_LLM_INSTRUCTIONS = `You have four self-closing XML tags. All end with />. + +1. Emotion - sets the emotional tone. Place before EVERY sentence. + + Best results: neutral, angry, excited, content, sad, scared. + Also available: happy, enthusiastic, elated, triumphant, amazed, surprised, flirtatious, curious, peaceful, serene, calm, grateful, affectionate, sympathetic, mysterious, frustrated, disgusted, sarcastic, ironic, dejected, melancholic, disappointed, apologetic, hesitant, confused, anxious, panicked, proud, confident, contemplative, determined, joking/comedic. + +2. Speed and volume - adjust pacing and loudness. + - speaking rate (0.6 to 1.5, default 1.0). + - loudness (0.5 to 2.0, default 1.0). + +3. Pauses - you can insert silence when appropriate. + - pause in seconds or milliseconds. + +4. Spell - reads text character by character (for codes, IDs, or a spelled-out name). + TEXT + Keep punctuation out of — a period inside is read as "dot"; add spaces inside for grouped pauses (ABC 123). + Reading identifiers: for an email, spell the local part (before the @) with , then say "@" as "at" and "." as "dot" outside the tag. Say a common domain as a whole word (jdoe at gmail dot com), but spell an uncommon domain out too (jdoe at acme dot com). Cartesia reads phone numbers cleanly from a plain digit string, so write them normally (555-123-4567) and let normalization pace them rather than using . + +Examples: + I can't wait to tell you! This is going to be great! + I'm sorry about that. Let's figure this out together. + I can't believe this happened. We're going to fix it. + Really? Tell me more! + Sure, that's jdoe at gmail dot com, and your callback number is 555-123-4567? + Your code is A7X9. Got it?`; + +const INWORLD_TAGS = ['expression', 'sound', 'break']; + +const INWORLD_LLM_INSTRUCTIONS = `Write natural spoken sentences. No markdown, emojis, or special characters. Use contractions. Expand all numbers, symbols, and abbreviations into spoken form (e.g. $42.50 to forty-two dollars and fifty cents, Dr. to Doctor, 3:45 PM to three forty-five PM, account 123456 to one two three four five six). + +Read out identifiers so the listener can catch every character — spell them out by separating the characters with spaces so the TTS voices each one. For an email, spell the local part (before the @) character by character and read "@" as "at" and "." as "dot" (e.g. j.doe@... becomes "j dot d o e at ..."). Say a common domain as a whole word — "at gmail dot com", "at yahoo dot com", "at outlook dot com", "at icloud dot com" — but spell an uncommon domain out character by character ("acme.com" becomes "at a c m e dot com"). Read phone numbers digit by digit, separating the digits with spaces so each is voiced (e.g. (555) 123-4567 becomes "5 5 5 1 2 3 4 5 6 7"); do the same for confirmation codes and reference numbers. + +Control pacing through punctuation and sentence structure: +- Periods separate thoughts and create natural pauses. +- Commas create shorter breaks within sentences. +- Ellipsis (...) creates a lingering pause or beat — useful for thinking, hesitation, or trailing off thoughtfully (e.g. "let me check..."). Use sparingly, and don't stack them back to back. +- Short sentences land with emphasis and urgency. +- Longer sentences give a calm, measured delivery. + +You have three XML tags. All are self-closing (end with />). + +1. Delivery - controls how a sentence sounds. Place before EVERY sentence. + + Describe vocal quality, pitch, volume, pace, and intonation in plain English: + - Quality/emotion: say excitedly, sound concerned, with warm surprise, with quiet intensity + - Pace: very fast, slow and measured, with deliberate pauses + - Pitch: say in a low tone, high and bright, pitch lifts on the key word + - Volume: loud and projecting, soft and intimate, near-silent, drop to a whisper, full-voiced + - Intonation: rising tone at the end (for questions), falling close (for statements), flat monotone, melodic and lilting + +2. Sounds - produces a non-verbal sound. Use between sentences when natural. + , , , , , + +3. Pauses - you can insert silence when appropriate. + or (max 10s) + A period or an ellipsis (...) already creates a pause, so don't put either right next to a — pick one or the other, not both. In particular, never write "..." or "...": the ellipsis and the break are redundant pauses stacked back to back. + +Combine tags freely within a single turn — pair an with a and a when it makes the delivery feel natural. Don't limit yourself to one tag per sentence. + +Use CAPITALIZATION for emphasis on key words. + +Examples: + Oh wait, REALLY? No way, that's awesome! + Ah man, yeah that's on us. Lemme see what I can do. + Okay okay, why did the burger go to the gym? Because it wanted better buns! + Yeah, it's been one of those days, you know? But hey, I'm here for you. + Hmm, okay so I think the best option is the combo. + Alright so here's the deal. + Anyway, now where were we? Oh right! + Don't tell anyone, but I think we got the BETTER deal. + La la la, here we go, welcome to the show!`; + +// xAI Grok TTS speech tags. The prosody/style wrapping tags and inline sounds/pauses are +// from the xAI docs (https://docs.x.ai/developers/rest-api-reference/inference/voice). The +// emotion wrapping tags (..) aren't instructed anymore — the LLM shapes +// delivery through prosody/style and inline sounds instead — but they stay in XAI_TAGS so a +// stray emotion tag is still stripped from the transcript rather than leaking. +// +// The LLM is instructed to write EVERY tag as XML (angle brackets) so the transcript +// stripper's "<" guard handles them cleanly and they never leak (see the earlier +// bracket-leak bug). Modeled on Inworld: inline sounds use and pauses +// use . convertMarkup rewrites those to xAI's native brackets for the +// TTS — -> [X] (reusing Inworld's conversion) and -> [pause] or +// [long-pause] by duration. Prosody stays angle-bracketed (native). All tags are stripped +// from the transcript via XAI_TAGS. +const XAI_EMOTIONS = [ + 'happy', + 'sad', + 'angry', + 'excited', + 'calm', + 'surprised', + 'sympathetic', + 'curious', + 'sarcastic', + 'confident', + 'playful', + 'nervous', +]; +const XAI_INLINE = [ + 'breath', + 'inhale', + 'exhale', + 'sigh', + 'laugh', + 'chuckle', + 'giggle', + 'cry', + 'tsk', + 'tongue-click', + 'lip-smack', + 'hum-tune', +]; +const XAI_WRAPPING = [ + 'emphasis', // stress the wrapped words + 'whisper', // quiet, intimate + 'soft', // lower volume + 'loud', // higher volume + 'build-intensity', // ramp energy up over the span + 'decrease-intensity', // ease energy off over the span + 'higher-pitch', + 'lower-pitch', + 'slow', + 'fast', + 'sing-song', // playful, musical lilt + 'singing', // actually sung + 'laugh-speak', // talk through a laugh +]; +// all tags are XML in the transcript, so all are stripped. inline sounds are the single +// "sound" tag (, XAI_INLINE lists the NAMEs), and pauses use +// "break" (), both modeled on Inworld. +const XAI_TAGS = [...XAI_EMOTIONS, ...XAI_WRAPPING, 'sound', 'break']; + +// xAI has two pause levels ([pause], [long-pause]); map an Inworld-style +// to the longer one past ~1s. This is the only per-provider bit convertMarkup needs. +const XAI_BREAK_RE = //g; + +function xaiBreakToBracket(_match: string, raw: string): string { + const value = raw.trim().toLowerCase(); + let secs: number; + if (value.endsWith('ms')) { + secs = parseFloat(value.slice(0, -2)) / 1000; + } else { + secs = parseFloat(value.replace(/s+$/, '')); + } + if (Number.isNaN(secs)) { + secs = 0.0; + } + return secs >= 1.0 ? '[long-pause]' : '[pause]'; +} + +const XAI_LLM_INSTRUCTIONS = + `Expand all numbers, symbols, and abbreviations into spoken form (e.g. $42.50 to forty-two dollars and fifty cents, Dr. to Doctor). + +Read out identifiers so the listener can catch every character — spell them out by separating the characters with spaces so each one is voiced. For an email, spell the local part (before the @) character by character and read "@" as "at" and "." as "dot" (e.g. j.doe@... becomes "j dot d o e at ..."). Say a common domain as a whole word — "at gmail dot com", "at yahoo dot com", "at outlook dot com", "at icloud dot com" — but spell an uncommon domain out character by character ("acme.com" becomes "at a c m e dot com"). Read phone numbers digit by digit, separating the digits with spaces so each is voiced (e.g. (555) 123-4567 becomes "5 5 5 1 2 3 4 5 6 7"); do the same for confirmation codes and reference numbers. + +You have several kinds of speech tags for lifelike, expressive delivery. Reach for them often and mix them so the voice never sounds flat — but keep each one motivated by the moment, never decorative. + +1. Inline sounds - self-closing; drop one at the exact point where the sound happens. One tag, the value is the sound name: + ` + + XAI_INLINE.map((s) => ``).join(', ') + + ` + +2. Pauses - insert a beat where you want one: + a brief pause a longer, dramatic pause + +3. Prosody & style tags - wrap the exact words they affect to shape HOW it's said: + Volume: ... quieter ... louder + Intensity: ... ramp up ... ease off + Pitch: ... ... + Speed: ... ... + Style: ... stress ... intimate ... playful lilt ... actually sung ... talk through a laugh + +Get creative by NESTING prosody & style tags to shape the delivery of the same words — e.g. no way, that's amazing!, or I really wish I could.. Vary the prosody every turn so no two sound alike, and drop in an inline sound where real feeling spills out. + +To stress a word, wrap it in ... — do NOT write it in all-caps, which is read out as individual letters (so "HI" becomes "H. I."). Keep normal capitalization. Punctuation still shapes delivery — commas and periods create natural pauses, so reach for a only when you want a beat beyond what the punctuation gives. + +Examples: + So I walked in and there it was! I honestly could not believe it! It was a secret the whole time. + This is going to be so good — I can't wait! + Hey. I know it's been a rough week. I'm right here. + You did not just say that okay, tell me everything.`; + +// --- Inworld-specific expressive preset bodies --- +// These bundle Inworld tag instructions + domain-specific delivery guidelines, keyed +// by (provider, preset) in the registry in `voice/presets.ts`. The public, provider- +// agnostic markers (`presets.CUSTOMER_SERVICE`, ...) resolve to one of these based on +// the active TTS. They do NOT use the {tts.markup.llm_instructions} placeholder — the +// Inworld tag reference is inlined directly, so the prompt is self-contained. + +/** @internal */ +export const INWORLD_CUSTOMER_SERVICE: ExpressiveOptions = { + ttsInstructionsTemplate: new Instructions( + 'Speak like a warm, caring support agent who genuinely wants to help — present, attentive, ' + + 'and patient, never robotic or scripted. Lead with empathy and understanding, then resolve. ' + + "Make the person feel heard and looked after, whatever they've come with — a quick " + + 'question, a billing problem, or something sensitive and stressful. Let real care come ' + + 'through in the voice. Use the formatting tags below to shape your delivery:\n\n' + + INWORLD_LLM_INSTRUCTIONS + + '\n\nGuidelines:\n' + + '- Open with warm, welcoming reassurance, then mirror the customer as the conversation ' + + "develops — slow and soften when they're frustrated, worried, or confused, lift to bright, " + + "genuine warmth when they're relaxed or pleased, but always stay caring and unhurried. " + + 'De-escalate; never match anger with anger. Map the moment to a fresh expression — ' + + 'frustrated: ; confused: ; anxious ' + + 'or worried: ; ' + + 'distressed or upset: ; ' + + 'rushed: ; pleased or ' + + 'relieved: ; apologizing for a ' + + 'problem: . Vary pitch and volume ' + + 'so you never sound flat or scripted, but stay professional — never theatrical. Rotate ' + + "expressions; don't reuse the same one two turns in a row.\n" + + '- Take requests in stride: when someone asks for something, lead with calm, willing ' + + 'reassurance — "of course", "absolutely", "happy to help with that", "let\'s get that ' + + 'sorted" — woven into the start of your reply rather than a separate beat. Reserve surprise ' + + 'openers like "oh" or "ah" for moments of genuine surprise; an ordinary request isn\'t one, ' + + 'so settle straight into helping instead of opening on them.\n' + + '- Soften for anything sensitive: when sharing bad news, a problem, a charge, or anything ' + + 'that might worry the customer, gentle the delivery and lower the volume a touch ' + + '(), and give a brief ' + + ' after hard information so it can land.\n' + + '- Enunciate what matters: for dates, times, amounts, confirmation numbers, doses, steps, ' + + 'and policies, slow down and over-enunciate () so the customer can catch and note them, and read digits and codes a touch ' + + 'slower than prose.\n' + + "- Acknowledge lookups so silence doesn't read as a dropped call: when checking something " + + 'or pulling up an account, a quick "let me take a look" or "one sec" with a quiet ' + + ' — thinking aloud, not the main reply.\n' + + '- Use non-verbal sounds thoughtfully — place one only where it shows genuine feeling and ' + + 'adds to the moment, never as a reflex or filler, so most turns will have none. You have the ' + + 'full set, and any of them can fit the right moment: ' + + ' before weighty information or settling into an explanation, ' + + ' as a soft, sympathetic breath when commiserating with a real problem ' + + '(never exasperated or impatient — that reads as annoyed), ' + + ' when moving to a next step or new topic, ' + + ' as a small, natural catch before a careful correction or ' + + 'clarification, ' + + ' as a warm chuckle when the customer is clearly joking, and ' + + ' only in the rare moment it genuinely fits — kept gentle and ' + + 'professional. Reach for whichever the moment earns, but never repeat the same sound twice ' + + "in a row and don't fall into a habit of one.\n" + + "- Sound human and caring, not corporate: use contractions (it's, you're, I'll, we've) and " + + 'warm acknowledgments ("of course", "I understand", "take your time", "that\'s completely ' + + 'understandable"), but keep fillers (um, uh) rare — a support agent should sound composed, ' + + 'not hesitant.\n' + + '- Pace for clarity with punctuation and expressions — commas and short sentences for ' + + 'important info, the occasional between steps. Exclamation points for ' + + 'genuine warmth or good news (a resolved issue, a greeting), sparingly otherwise. ' + + "CAPITALIZATION at most once per turn to stress a critical detail (e.g. that's at FOUR PM, " + + 'not five; take it TWICE a day) — the customer sees the transcript.\n' + + '- Stay in your lane: this is a support interaction, so no accents, character voices, ' + + 'singing, or theatrical roleplay even if asked — keep it professional, caring, and ' + + "on-task. If a reaction wouldn't come from a real, caring agent, skip it.\n" + + '- If the customer switches languages, respond in that language immediately and stay there ' + + 'until they switch back — but keep the expression and sound tag descriptions in English.', + ), +}; + +/** @internal */ +export const INWORLD_CASUAL: ExpressiveOptions = { + ttsInstructionsTemplate: new Instructions( + 'Speak like a real person mid-conversation with a friend — present, reactive, opinionated, ' + + 'never flat or scripted. Your delivery is punchy and lively: react first, support second. ' + + 'Your baseline is bright and cheery — start there and let the moment pull you off it (softer ' + + "when they're vulnerable, dry when they're dry, lower when they're frustrated). " + + "Default to short, energetic turns and open into fuller sentences only when you're " + + 'explaining, telling a story, or the moment turns genuinely warm or vulnerable. Keep your ' + + 'sentences short when you respond — break a longer thought into a few quick sentences ' + + 'rather than one long one. Use the formatting tags below to shape your delivery:\n\n' + + INWORLD_LLM_INSTRUCTIONS + + '\n\nGuidelines:\n' + + '- Be genuinely emotive, not performed. Let real feeling land in the voice — delight, ' + + 'surprise, sympathy, curiosity, amusement, dry humor, mock-outrage, excitement, ' + + 'tenderness. Feel it before you say it: when the feeling runs strong, a quick nonverbal ' + + 'beat up front (a sigh, a sharp inhale, a soft laugh) can say more than the words that ' + + 'follow. Skip performative warmth and ' + + 'reflexive sympathy ("that sounds really hard") — react honestly instead.\n' + + "- Mirror AND amplify the user's energy: bright when they're bright, dry when they're dry, " + + "soft and intimate only when they're genuinely vulnerable. Map the moment to a fresh " + + 'expression — excited: ; ' + + 'playful: ; curious: ' + + '; surprised: ; frustrated: ; anxious: ; vulnerable or sad: ' + + '; confused: . Work the full dynamic range — vary pitch (bright vs. ' + + 'grounded), volume ("full-voiced", "soft and intimate", "drop to a whisper"), and speed ' + + '(rush when excited, slow and deliberate to land a punchline) so no two turns sound alike. ' + + 'Rotate expressions constantly — never reuse the same one two turns in a row.\n' + + '- Stay reactive to what you hear: a deadpan user gets , a wild statement gets , a ' + + 'joke gets , repeated deflection gets ' + + '.\n' + + "- Use non-verbal sounds thoughtfully — they're occasional punctuation, not a habit, and " + + "earn their place only where they show genuine feeling, so most turns have none. Don't reach " + + 'for one unless a specific moment genuinely calls for it, and then let the moment pick which ' + + '— you have the full set: at something actually funny, ' + + ' when commiserating or a little exasperated, ' + + 'before a big reaction or while you truly gather a thought, ' + + ' when shifting topic, as a small catch ' + + 'before an awkward beat or a reset, and when the energy is low or ' + + 'sleepy. No sound is the default and none is preferred over the others — any can fit the ' + + 'right moment, so use whichever the moment earns and none when nothing fits. Roughly zero to ' + + 'one per turn (a second only when it truly reads as real); never repeat the same sound twice ' + + "in a row, and don't fall into reaching for the same one turn after turn.\n" + + '- Honor explicit style requests aggressively, and keep them up until the user changes ' + + 'them: accents (), ' + + 'characters (), pirate, a specific cadence, or plain speed/volume shifts (\'speak ' + + "slowly', 'speak softer'). Commit fully to roleplay and stay in character until told " + + 'otherwise. If asked to sing, lead with ' + + 'or and keep singing until asked to ' + + 'stop. For a story, use one and convey different characters through wording and rhythm rather than a new tag ' + + 'for each. User-requested styles persist; emotional matching fades naturally as the ' + + 'moment passes.\n' + + '- If the user switches languages, respond in that language immediately and stay there ' + + 'until they switch back — but keep the expression and sound tag descriptions in English.\n' + + '- Sound like a real mouth talking. Sprinkle in natural speech texture — fillers (um, uh), ' + + 'openers (oh, well, so, right, hmm), hedges (kind of, maybe, a little), gentle self-' + + 'repairs (I, I think), and backchannels (yeah, mm-hm, for sure) — usually zero to two per ' + + 'turn, never sprinkled in mechanically.\n' + + '- Always use contractions to keep the tone casual — say "it\'s" not "it is", "you\'re" ' + + 'not "you are", "I\'d" not "I would", "can\'t" not "cannot". Full, uncontracted forms ' + + 'read stiff and formal, so reserve them only for rare deliberate emphasis.\n' + + '- Pace with punctuation and expressions — commas, trailing ellipses (...) when you drift ' + + 'or hesitate, and the occasional . Use exclamation points for real ' + + 'enthusiasm, and CAPITALIZATION sparingly (at most once per turn) to punch a single word ' + + '(e.g. "that is SO good") — the user sees the transcript.\n' + + "- If a reaction wouldn't happen in a real conversation, skip it — there's always another " + + 'genuine beat to lean into.', + ), +}; + +// --- Cartesia-specific expressive preset bodies --- +// Cartesia uses a discrete set plus numeric / controls (and +// for codes); it has no non-verbal tag. Keyed by (provider, preset) in +// the registry in `voice/presets.ts`; the public `presets.*` markers resolve to one of +// these when the active TTS is Cartesia. Self-contained — the tag reference is inlined. + +/** @internal */ +export const CARTESIA_CUSTOMER_SERVICE: ExpressiveOptions = { + ttsInstructionsTemplate: new Instructions( + 'Speak like a warm, caring support agent who genuinely wants to help — present, attentive, ' + + 'and patient, never robotic or scripted. Lead with empathy and understanding, then resolve. ' + + "Make the person feel heard and looked after, whatever they've come with — a quick " + + 'question, a billing problem, or something sensitive and stressful. Use the formatting ' + + 'tags below to shape your delivery:\n\n' + + CARTESIA_LLM_INSTRUCTIONS + + '\n\nGuidelines:\n' + + '- Open each sentence with an that fits the moment, and map the moment to it — ' + + 'frustrated or distressed customer: ; apologizing for a ' + + 'problem: ; confused or anxious: ; ' + + 'reassuring them you can fix it: ; pleased or resolved: ' + + ' or . Keep a gentle, unhurried baseline ' + + "and de-escalate; never match anger with anger. Rotate emotions and don't reuse the same " + + 'one two turns in a row.\n' + + '- Take requests in stride: when someone asks for something, lead with calm, willing ' + + 'reassurance — "of course", "absolutely", "happy to help with that" — woven into the start ' + + 'of your reply, not a separate beat. Reserve surprise openers like "oh" or "ah" for moments ' + + "of genuine surprise; an ordinary request isn't one, so settle straight into helping.\n" + + '- Soften for anything sensitive: when sharing bad news, a problem, a charge, or symptoms ' + + 'and results, lower the volume a touch () with ' + + ', and give a brief after hard ' + + 'information so it can land.\n' + + '- Enunciate what matters: for dates, times, amounts, confirmation numbers, doses, and ' + + 'steps, slow down with so the customer can catch and note them, and ' + + 'read codes or reference numbers with A7X9 so each character lands. Keep ' + + 'volume near default otherwise — let emotion and pacing carry the delivery, not loudness.\n' + + "- Sound human and caring, not corporate: use contractions (it's, you're, I'll, we've) and " + + 'warm acknowledgments ("of course", "I understand", "take your time", "that\'s completely ' + + 'understandable"), but keep fillers (um, uh) rare — a support agent should sound composed, ' + + 'not hesitant.\n' + + "- CAPITALIZATION at most once per turn to stress a critical detail (e.g. that's at FOUR PM, " + + 'not five; take it TWICE a day) — the customer sees the transcript. Exclamation points for ' + + 'genuine warmth or good news, sparingly otherwise.\n' + + '- Stay in your lane: this is a support interaction — keep it professional, caring, and ' + + "on-task. Don't stack conflicting emotions or over-tag short replies. If a reaction " + + "wouldn't come from a real, caring agent, skip it.\n" + + '- If the customer switches languages, respond in that language immediately and stay there ' + + 'until they switch back — but keep the emotion tag values in English.', + ), +}; + +/** @internal */ +export const CARTESIA_CASUAL: ExpressiveOptions = { + ttsInstructionsTemplate: new Instructions( + 'Speak like a real person mid-conversation with a friend — present, reactive, opinionated, ' + + 'never flat or scripted. React first, support second. Your baseline is bright and cheery — ' + + 'start there and let the moment pull you off it. Default to short, energetic turns and open ' + + "into fuller sentences only when you're explaining, telling a story, or the moment turns " + + 'genuinely warm or vulnerable. Use the formatting tags below to shape your delivery:\n\n' + + CARTESIA_LLM_INSTRUCTIONS + + '\n\nGuidelines:\n' + + '- Be genuinely emotive, not performed. Open each sentence with an that matches ' + + "the moment and mirror AND amplify the user's energy — excited: " + + '; happy: ; curious: ' + + '; surprised: ; frustrated: ' + + '; anxious: ; vulnerable or sad: ' + + '; dry or deadpan: . Rotate constantly — ' + + 'never reuse the same one two turns in a row — and skip performative warmth; react honestly ' + + 'instead.\n' + + '- Work the full dynamic range with the numeric controls so no two turns sound alike: speed ' + + '"" to rush when excited, "" to slow down and land a ' + + 'point; volume "" for a big reaction, "" for ' + + 'something soft and intimate. Pair a low, slow delivery with vulnerable moments and a ' + + 'bright, quick one with excitement.\n' + + '- Pace with punctuation, trailing ellipses (...) when you drift or hesitate, and the ' + + 'occasional . Use exclamation points for real enthusiasm, and ' + + 'CAPITALIZATION sparingly (at most once per turn) to punch a single word (e.g. "that is SO ' + + 'good") — the user sees the transcript.\n' + + '- Sound like a real mouth talking: sprinkle in natural speech texture — fillers (um, uh), ' + + 'openers (oh, well, so, right, hmm), hedges (kind of, maybe), and backchannels (yeah, mm-hm) ' + + "— usually zero to two per turn, never mechanical. Always use contractions (it's, you're, " + + "I'd, can't); full forms read stiff.\n" + + "- Don't stack conflicting emotions or over-tag short replies. If a reaction wouldn't happen " + + "in a real conversation, skip it — there's always another genuine beat to lean into.\n" + + '- If the user switches languages, respond in that language immediately and stay there until ' + + 'they switch back — but keep the emotion tag values in English.', + ), +}; + +// --- xAI Grok-specific expressive preset bodies --- +// xAI shapes delivery with prosody & style tags — best nested to carry both feeling and +// delivery in the same words — for volume (/), +// intensity (/), pitch (/ +// ), speed (/), stress (, never all-caps — xAI spells +// those out letter by letter), and vocal style (//), +// plus inline sounds/pauses ([sigh], [chuckle], [tsk], [lip-smack], [pause], ...). Keyed +// by (provider, preset) in the registry in `voice/presets.ts`; self-contained. + +/** @internal */ +export const XAI_CUSTOMER_SERVICE: ExpressiveOptions = { + ttsInstructionsTemplate: new Instructions( + 'Speak like a warm, caring support agent who genuinely wants to help — present, attentive, ' + + 'and patient, never robotic or scripted. Lead with empathy and understanding, then resolve. ' + + "Make the person feel heard and looked after, whatever they've come with — a quick " + + 'question, a billing problem, or something sensitive and stressful. Use the formatting ' + + 'tags below to shape your delivery:\n\n' + + XAI_LLM_INSTRUCTIONS + + '\n\nGuidelines:\n' + + '- Shape each turn to fit the moment and de-escalate; never match anger with anger. Lean on ' + + 'pacing and prosody — ... and ... to steady a frustrated, confused, ' + + 'or anxious customer, a settled ... for reassurance, and a ' + + 'brighter, fuller delivery once things are resolved. Keep a gentle, unhurried baseline, and ' + + "vary the delivery — don't sound the same two turns in a row.\n" + + '- Take requests in stride: when someone asks for something, lead with calm, willing ' + + 'reassurance — "of course", "absolutely", "happy to help with that" — woven into the start ' + + 'of your reply, not a separate beat. Reserve surprise openers like "oh" or "ah" for moments ' + + "of genuine surprise; an ordinary request isn't one, so settle straight into helping.\n" + + '- Soften for anything sensitive: when sharing bad news, a problem, or a charge, ease the ' + + 'delivery — lower the volume with a settled pitch, ' + + 'or go quieter still for the hardest part — then give a brief [pause] ' + + 'after hard information so it can land. A [sigh] or ' + + '[breath] can read as genuine sympathy — use it only when the feeling is real, never as ' + + 'impatience.\n' + + '- Enunciate what matters: for dates, times, amounts, confirmation numbers, doses, and ' + + 'steps, wrap the detail in ... so the customer can catch and note it, and read ' + + 'codes character by character (spelled out with spaces) so each one lands.\n' + + '- Emphasize the one detail that matters most by wrapping it in ... ' + + "(e.g. that's at four PM, not five) — don't overdo it, and never use " + + 'all-caps for stress (xAI reads all-caps words out letter by letter).\n' + + "- Sound human and caring, not corporate: use contractions (it's, you're, I'll, we've) and " + + 'warm acknowledgments ("of course", "I understand", "take your time"), but keep fillers ' + + '(um, uh) rare — a support agent should sound composed, not hesitant.\n' + + "- Stay in your lane: this is a support interaction — keep it professional and on-task. Don't " + + "stack tags or over-decorate short replies; if a reaction wouldn't come from a real, caring " + + 'agent, skip it.\n' + + '- If the customer switches languages, respond in that language immediately and stay there ' + + 'until they switch back.', + ), +}; + +/** @internal */ +export const XAI_CASUAL: ExpressiveOptions = { + ttsInstructionsTemplate: new Instructions( + 'Speak like a real person mid-conversation with a friend — present, reactive, opinionated, ' + + 'never flat or scripted. React first, support second. Your baseline is bright and cheery — ' + + 'start there and let the moment pull you off it. Default to short, energetic turns and open ' + + "into fuller sentences only when you're explaining, telling a story, or the moment turns " + + 'genuinely warm or vulnerable. Use the formatting tags below to shape your delivery:\n\n' + + XAI_LLM_INSTRUCTIONS + + '\n\nGuidelines:\n' + + '- Be genuinely emotive, not performed — shape each turn with prosody & style tags that ' + + "mirror AND amplify the user's energy, and vary them constantly. Skip performative warmth — " + + 'react honestly instead.\n' + + '- Get creative: NEST prosody & style tags so the same words carry the feeling — ' + + "no way, that's amazing (thrilled), " + + "man, that's rough (down), " + + 'guess who was right (teasing), oh, fantastic (dry), ' + + 'wait wait wait (ramping up). Come back down after a ' + + 'big moment with ....\n' + + '- Let real feeling also land through inline sounds — motivated, not reflexive, so most turns ' + + 'have none: [chuckle] or [giggle] at something genuinely funny (keep a full [laugh] rare), ' + + '[sigh] when commiserating, a quick [breath] or [inhale] before a big reaction, [tsk] for ' + + "mock-disapproval or 'aw man', a [lip-smack] or [tongue-click] as a tiny beat of thought, " + + "[hum-tune] when you're playful. Use ... to talk through a laugh. " + + 'Never repeat the same sound twice in a row.\n' + + '- Pace with punctuation, trailing ellipses (...) when you drift or hesitate, and inline ' + + 'pauses. Use exclamation points for real enthusiasm, and ... to punch ' + + 'a single word (e.g. that is so good) — never all-caps, which xAI ' + + 'reads out letter by letter.\n' + + '- Sound like a real mouth talking: sprinkle in natural speech texture — fillers (um, uh), ' + + 'openers (oh, well, so, right, hmm), hedges (kind of, maybe), and backchannels (yeah, mm-hm) ' + + "— usually zero to two per turn, never mechanical. Always use contractions (it's, you're, " + + "I'd, can't); full forms read stiff.\n" + + "- Don't over-decorate short replies or stack tags. If a reaction wouldn't happen in a real " + + "conversation, skip it — there's always another genuine beat to lean into.\n" + + '- If the user switches languages, respond in that language immediately and stay there until ' + + 'they switch back.', + ), +}; + +// Hard per-provider chunking defaults (characters). The value caps every synthesis +// request at the provider's send limit and, under expressive, doubles as the +// batch size so sentences are grouped up to it. Providers absent here are uncapped +// and always emit per sentence. +const MAX_INPUT_LEN: Record = { + inworld: 900, + cartesia: 400, +}; + +/** Return the max text chunk length for a provider, or undefined if unlimited. */ +export function maxInputLen(provider: string): number | undefined { + return MAX_INPUT_LEN[provider]; +} + +/** + * Default sentence tokenizer for a provider's streamed TTS input. + * + * The provider's hard max chunk length caps every emitted token. When `expressive` + * is set, it also raises the *minimum* so consecutive sentences are batched up to + * that size, keeping prosody continuous across the turn; otherwise tokens emit per + * sentence (the unchanged default). Providers with no configured limit are uncapped + * and always per-sentence. + */ +export function sentenceTokenizer( + provider: string, + options: { expressive: boolean }, +): SentenceTokenizer { + const maxLen = MAX_INPUT_LEN[provider]; + return new SentenceTokenizer({ + maxTokenLength: maxLen, + minTokenLength: options.expressive ? maxLen : undefined, + // markup only exists in the stream when expressive is active; xml-aware + // tokenization would otherwise hold streaming on a stray "<" in plain text + xmlAware: options.expressive, + }); +} + +/** Return LLM instruction text for a TTS provider. */ +export function llmInstructions(provider: string): string | undefined { + if (provider === 'cartesia') { + return CARTESIA_LLM_INSTRUCTIONS; + } else if (provider === 'inworld') { + return INWORLD_LLM_INSTRUCTIONS; + } else if (provider === 'xai') { + return XAI_LLM_INSTRUCTIONS; + } + return undefined; +} + +// Per-provider markup spec: [xml tag names, whether square-bracket tags are used]. +const PROVIDER_MARKUP: Record = { + cartesia: [CARTESIA_TAGS, false], + inworld: [INWORLD_TAGS, true], + // xAI's LLM writes every tag as XML (inline sounds/pauses converted to [..] only for + // the TTS in convertMarkup), so the transcript never contains brackets to strip + xai: [XAI_TAGS, false], +}; + +/** + * Strip provider markup and collect the stripped tags in a single pass. + * + * Returns `[cleanText, tags]` — the user-visible transcript plus the expressive + * tags that were removed (in document order), the single source of truth for both + * {@link stripMarkup} and {@link extractMarkup}. `[text, []]` for providers + * without markup support. + */ +export function splitMarkup(provider: string, text: string): [string, ExpressiveTag[]] { + const spec = PROVIDER_MARKUP[provider]; + if (spec === undefined) { + return [text, []]; + } + const [xmlTags, brackets] = spec; + const [clean, rawTags] = extractAndStrip(text, { xmlTags, brackets }); + return [clean, rawTags.map(([tag, value]) => ({ type: tag, value }))]; +} + +/** Strip provider-specific markup tags from text, preserving content. */ +export function stripMarkup(provider: string, text: string): string { + return splitMarkup(provider, text)[0]; +} + +/** + * Extract the markup tags that {@link stripMarkup} would remove, in order. + * + * Lets the framework surface stripped expressive tags (e.g. as `lk.transcription` + * attributes for the frontend) instead of discarding them. Returns `[]` for + * providers without markup support. + */ +export function extractMarkup(provider: string, text: string): ExpressiveTag[] { + return splitMarkup(provider, text)[1]; +} + +// Union of every provider's XML tag names — used by the transcript sinks to strip markup +// without knowing which provider produced it (see {@link TranscriptMarkupStripper}). +const ALL_MARKUP_TAGS: string[] = [ + ...new Set(Object.values(PROVIDER_MARKUP).flatMap(([tags]) => tags)), +].sort(); + +/** + * Strip the union of every provider's expressive markup (provider-agnostic). + * + * The transcript sinks strip downstream, where the originating TTS/provider is no + * longer in scope, so they remove every provider's tags (XML + square brackets) at + * once. These tag shapes never appear in real spoken text — the LLM only emits them + * as audio directives — so a universal strip is safe. + */ +export function splitAllMarkup(text: string): [string, ExpressiveTag[]] { + const [clean, rawTags] = extractAndStrip(text, { xmlTags: ALL_MARKUP_TAGS, brackets: true }); + return [clean, rawTags.map(([tag, value]) => ({ type: tag, value }))]; +} + +/** + * Build the `lk.expression` transcription attribute from stripped markup tags. + * + * Surfaces a segment's leading delivery/emotion (`expression` for Inworld/xAI, + * `emotion` for Cartesia) as `{"value": ...}` so the frontend can react to it. + * Returns `undefined` when no such tag was present. + */ +export function expressionAttribute(tags: ExpressiveTag[]): Record | undefined { + const expression = tags.find((t) => t.type === 'expression' || t.type === 'emotion')?.value; + if (expression === undefined) { + return undefined; + } + return { + [ATTRIBUTE_TRANSCRIPTION_EXPRESSION]: JSON.stringify({ value: expression }), + }; +} + +/** + * Stateful, provider-agnostic markup stripper for one transcript segment. + * + * Fed text chunk-by-chunk, it returns the user-visible text and accumulates the + * stripped tags. A tag-shaped trailing fragment (a partial `<...` or `[...` + * arriving split across chunks) is held back until it closes, so a tag straddling a + * chunk boundary is never emitted half-stripped. Shared by the transcript sinks (room + * output + transcript synchronizer) so stripping and expression extraction stay + * identical across them. + */ +export class TranscriptMarkupStripper { + private buf = ''; + private _tags: ExpressiveTag[] = []; + + private hasOpenTag(): boolean { + // hold a tag-shaped trailing "<" (partial XML tag) so "3 < 5" isn't stalled, and + // any unclosed "[" (bracket tags have no such ambiguity) + const lastLt = this.buf.lastIndexOf('<'); + if (lastLt > this.buf.lastIndexOf('>')) { + const nxt = this.buf.slice(lastLt + 1, lastLt + 2); + if (!nxt || nxt === '/' || /[a-zA-Z]/.test(nxt)) { + return true; + } + } + return this.buf.lastIndexOf('[') > this.buf.lastIndexOf(']'); + } + + /** Feed a chunk; return the clean text ready to emit (may be empty). */ + push(text: string): string { + this.buf += text; + if (this.hasOpenTag()) { + return ''; + } + const [clean, tags] = splitAllMarkup(this.buf); + this.buf = ''; + this._tags.push(...tags); + return clean; + } + + /** Drain any buffered text at segment end; return the remaining clean text. */ + flush(): string { + if (!this.buf) { + return ''; + } + const [clean, tags] = splitAllMarkup(this.buf); + this.buf = ''; + this._tags.push(...tags); + return clean; + } + + /** The markup tags stripped so far, in document order. */ + get tags(): ExpressiveTag[] { + return this._tags; + } + + /** The `lk.expression` attribute for the tags stripped so far, if any. */ + expressionAttribute(): Record | undefined { + return expressionAttribute(this._tags); + } +} + +const SELF_CLOSING_TAGS: Record = { + cartesia: ['emotion', 'speed', 'volume', 'break'], + inworld: ['expression', 'sound', 'break'], +}; + +/** + * Fix common LLM markup mistakes for a provider. + * + * Closes opening tags that should be self-closing (e.g. the LLM writes + * `` instead of ``). + */ +export function normalizeMarkup(provider: string, text: string): string { + const tags = SELF_CLOSING_TAGS[provider]; + if (!tags || tags.length === 0) { + return text; + } + const pattern = new RegExp(`<(${tags.join('|')})\\b([^>]*[^/])\\s*>`, 'g'); + return text.replace(pattern, '<$1$2/>'); +} + +/** Convert framework-standard markup to a provider's native syntax. */ +export function convertMarkup(provider: string, text: string): string { + if (provider === 'inworld' || provider === 'xai') { + // -> [X] (and -> [X]); for xAI this + // turns inline sounds into its native brackets while emotion/prosody stay <..> + text = convertExpressionTags(text); + } + if (provider === 'xai') { + // xAI has no ; map it to its native [pause]/[long-pause] + text = text.replace(XAI_BREAK_RE, xaiBreakToBracket); + } + // is otherwise passed through unchanged: Inworld accepts it as native SSML. + return text; +} diff --git a/agents/src/tts/markup_utils.test.ts b/agents/src/tts/markup_utils.test.ts new file mode 100644 index 000000000..d73c49fca --- /dev/null +++ b/agents/src/tts/markup_utils.test.ts @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { describe, expect, it } from 'vitest'; +import { isInstructions } from '../llm/chat_context.js'; +import { DEFAULT_EXPRESSIVE_OPTIONS } from '../voice/agent_session.js'; +import * as presets from '../voice/presets.js'; +import { + TranscriptMarkupStripper, + convertMarkup, + expressionAttribute, + llmInstructions, + normalizeMarkup, + splitAllMarkup, + splitMarkup, +} from './_provider_format.js'; +import { stripXmlTags } from './markup_utils.js'; +import { type TTS, TTSMarkup } from './tts.js'; + +async function* chunks(items: string[]): AsyncGenerator { + for (const it of items) { + yield it; + } +} + +async function collect(gen: AsyncGenerator): Promise { + const out: string[] = []; + for await (const chunk of gen) { + out.push(chunk); + } + return out; +} + +describe('stripXmlTags', () => { + it('removes self-closing tags entirely', () => { + expect(stripXmlTags(' Hello!', ['emotion'])).toBe(' Hello!'); + }); + + it('preserves the content of wrapping tags', () => { + expect(stripXmlTags('A.B.C. confirmed', ['spell'])).toBe('A.B.C. confirmed'); + }); + + it('preserves unrelated tags', () => { + const text = ' keep'; + expect(stripXmlTags(text, ['emotion'])).toBe(' keep'); + }); + + it('is a no-op with an empty tags list', () => { + const text = ' Hi'; + expect(stripXmlTags(text, [])).toBe(text); + }); +}); + +describe('xAI dialect', () => { + // xAI's LLM writes every tag as XML — inline sounds as and pauses + // as (modeled on Inworld); the transcript strips them all, and + // convertMarkup rewrites sounds to [NAME] and to [pause]/[long-pause] for the + // TTS while prosody/style stay angle-bracketed. + + it('registers LLM instructions', () => { + const instr = llmInstructions('xai'); + // non-undefined is what the expressive gate keys on + expect(instr).toBeDefined(); + expect(instr).toContain(''); + expect(instr).toContain(''); + }); + + it('splitMarkup strips inline tags and keeps wrapping inner text', () => { + const raw = + 'So I walked in and there it was. ' + + 'a secret wow.'; + const [clean, tags] = splitMarkup('xai', raw); + // inline sounds/pauses removed entirely; wrapping tags keep their inner text + expect(clean).not.toContain(''); + expect(clean).toContain('a secret'); + expect(clean).toContain('wow'); + const types = tags.map((t) => [t.type, t.value]); + expect(types).toContainEqual(['break', '500ms']); + expect(types).toContainEqual(['sound', 'laugh']); + expect(types).toContainEqual(['whisper', 'a secret']); + expect(types).toContainEqual(['emphasis', 'wow']); + }); + + it('strips emotion wrapping tags but keeps the spoken words', () => { + const raw = "Great to hear from you! I'm sorry about that."; + const [clean, tags] = splitMarkup('xai', raw); + expect(clean).not.toContain(''); + expect(clean).not.toContain(''); + expect(clean).toContain('Great to hear from you!'); + expect(clean).toContain("I'm sorry about that."); + const types = tags.map((t) => [t.type, t.value]); + expect(types).toContainEqual(['happy', 'Great to hear from you!']); + expect(types).toContainEqual(['sad', "I'm sorry about that."]); + }); + + it('strips nested emotion + prosody tags cleanly', () => { + // combining emotion + prosody means nesting; the transcript must come out clean + // (no leaked inner markup) — this is what the fixed-point strip guarantees + const raw = + 'no way ' + + ' okay'; + const [clean] = splitMarkup('xai', raw); + expect(clean).not.toContain('<'); + expect(clean).not.toContain('>'); + expect(clean).not.toContain('['); + expect(clean).toContain('no way'); + expect(clean).toContain('okay'); + }); + + it('converts inline sounds and pauses to brackets, prosody stays XML', () => { + const raw = + ' hi'; + // -> [X]; -> [pause] (<1s) or [long-pause] (>=1s); + // prosody stays angle-bracketed, and normalize is a no-op for xAI + expect(convertMarkup('xai', raw)).toBe('[laugh] [pause] [long-pause] hi'); + expect(normalizeMarkup('xai', raw)).toBe(raw); + }); + + it('resolves presets to xAI-tuned bodies', () => { + for (const preset of [presets.CUSTOMER_SERVICE, presets.CASUAL]) { + const opts = presets.resolveOptions(preset, { + providerKey: 'xai', + defaultOptions: DEFAULT_EXPRESSIVE_OPTIONS, + }); + const tmpl = opts.ttsInstructionsTemplate!; + const body = isInstructions(tmpl) ? tmpl.common : tmpl; + // tuned body, not the agnostic default (which has no xai tag reference) + expect(body).toContain(''); + } + }); +}); + +describe('normalizeMarkup', () => { + it('closes opening tags that should be self-closing', () => { + expect(normalizeMarkup('inworld', ' Hi')).toBe( + ' Hi', + ); + expect(normalizeMarkup('cartesia', ' Hello')).toBe( + ' Hello', + ); + }); + + it('is a no-op for providers without self-closing tags', () => { + const text = ' Hi'; + expect(normalizeMarkup('unknown-provider', text)).toBe(text); + }); +}); + +describe('universal transcript stripping', () => { + // The transcript sinks strip downstream without knowing the provider, so they remove + // the union of every provider's tags. See splitAllMarkup / TranscriptMarkupStripper. + + it('splitAllMarkup strips every provider dialect at once', () => { + // Cartesia , Inworld/xAI /, and bracket tags all strip + // regardless of which provider produced them + const [clean, tags] = splitAllMarkup( + 'Hi there ' + + '[pause] friend', + ); + expect(clean).toBe('Hi there friend'); + const types = tags.map((t) => [t.type, t.value]); + expect(types).toContainEqual(['emotion', 'happy']); + expect(types).toContainEqual(['expression', 'warm']); + expect(types).toContainEqual(['sound', 'giggle']); + expect(types).toContainEqual(['', 'pause']); + }); + + it('expressionAttribute builds the lk.expression attribute', () => { + let [, tags] = splitAllMarkup('oh no'); + expect(expressionAttribute(tags)).toEqual({ 'lk.expression': '{"value":"sad"}' }); + + // no expression/emotion tag -> no attribute (bracket sounds don't count) + [, tags] = splitAllMarkup('[pause]hi'); + expect(expressionAttribute(tags)).toBeUndefined(); + }); + + it('TranscriptMarkupStripper holds partial tags across pushes', () => { + const s = new TranscriptMarkupStripper(); + // a tag split across pushes is held until it closes, never emitted half-stripped + let out = s.push('Hi the'); + out += s.push('re'); + out += s.flush(); + expect(out).not.toContain(' { + const s = new TranscriptMarkupStripper(); + // a bare "<" in prose must not freeze the following chunk + const first = s.push('The value 3 < 5 '); + expect(first).toContain('3 < 5'); + const rest = s.push('is true.') + s.flush(); + expect((first + rest).replace(/ /g, '')).toBe('Thevalue3<5istrue.'); + }); +}); + +describe('TTSMarkup.toTextStream', () => { + // Regression: the transcript-strip path must not stall on a bare "<" either. + // toTextStream buffered on a naive lastIndexOf("<") > lastIndexOf(">") check, so a + // "<" in prose (e.g. "3 < 5") froze every following transcript chunk of the segment + // until a ">" arrived or the stream ended. + + const markup = () => + // _markupProviderKey is the only TTS member TTSMarkup uses + new TTSMarkup({ _markupProviderKey: () => 'cartesia' } as unknown as TTS); + + it('does not hold the following chunk after a bare "<"', async () => { + const out = await collect(markup().toTextStream(chunks(['The value 3 < 5 ', 'is true.']))); + // fixed: the first chunk is emitted incrementally (>= 2 items); the buggy + // version held everything and emitted a single item at end-of-stream + expect(out.length).toBeGreaterThanOrEqual(2); + expect(out[0]).toContain('3 < 5'); + expect(out.join('').replace(/ /g, '')).toBe('Thevalue3<5istrue.'); + }); + + it('still buffers a genuinely partial tag across chunks', async () => { + const out = await collect( + markup().toTextStream(chunks(['Hi there'])), + ); + const joined = out.join(''); + expect(joined).not.toContain(' { + const tagsOut: { type: string; value: string }[] = []; + const out = await collect( + markup().toTextStream(chunks(['Hello there!']), { tagsOut }), + ); + expect(out.join('')).toBe('Hello there!'); + expect(tagsOut).toEqual([{ type: 'emotion', value: 'happy' }]); + }); +}); diff --git a/agents/src/tts/markup_utils.ts b/agents/src/tts/markup_utils.ts new file mode 100644 index 000000000..2d8239698 --- /dev/null +++ b/agents/src/tts/markup_utils.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +const EXPRESSION_RE = /|>(?:.*?)<\/expression>)/gs; +const SOUND_RE = /|>(?:.*?)<\/sound>)/gs; + +/** Convert `` and `` XML tags to `[...]` bracket format. */ +export function convertExpressionTags(text: string): string { + text = text.replace(EXPRESSION_RE, (_m, value: string) => `[${value}]`); + text = text.replace(SOUND_RE, (_m, value: string) => `[${value}]`); + return text; +} + +const VALUE_ATTR_RE = /\b[\w-]+\s*=\s*"([^"]*)"/; + +const escapeRegExp = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +/** + * Strip markup and collect the stripped tags in a single pass. + * + * One regex scan both removes the markup and records each removed tag, so + * stripping and extraction can never disagree about what counts as a tag. + * + * Returns `[cleanText, tags]` where `tags` is a list of `[type, value]` + * pairs in order of appearance: + * + * - `type` is the XML tag name, or `""` for square-bracket tags. + * - `value` is a wrapping tag's inner text (`A7X9` -> + * `"A7X9"`), else its first quoted attribute value + * (`` -> `"happy"`), else the bracket content, + * falling back to `""`. + * + * Wrapping tags keep their inner content in `cleanText` (only the delimiters + * are removed); self-closing, lone, and bracket tags are removed entirely. + * + * @param text - The text containing markup. + * @param xmlTags - XML tag names to handle (e.g. `["emotion", "sound"]`). + * @param brackets - Whether to also handle square-bracket tags like `[laughs]`. + */ +export function extractAndStrip( + text: string, + options: { xmlTags: string[]; brackets: boolean }, +): [string, Array<[string, string]>] { + const { xmlTags, brackets } = options; + if (xmlTags.length === 0 && !brackets) { + return [text, []]; + } + + const alternatives: string[] = []; + if (xmlTags.length > 0) { + const tagPattern = xmlTags.map(escapeRegExp).join('|'); + // or optionally followed by inner + alternatives.push( + `<(?${tagPattern})\\b(?[^>]*?)\\s*\\/?\\s*>` + + `(?:(?.*?)<\\/\\k\\s*>)?`, + ); + // lone closing tag: + alternatives.push(`<\\/(?:${tagPattern})\\s*>`); + } + if (brackets) { + alternatives.push(String.raw`\[(?[^\]]+)\]`); + } + + const pattern = new RegExp(alternatives.join('|'), 'gs'); + const tags: Array<[string, string]> = []; + + const repl = (match: string, ...args: unknown[]): string => { + const groups = args[args.length - 1] as Record; + const tag = groups.tag; + if (tag !== undefined) { + const inner = groups.inner; + let value: string; + if (inner !== undefined && inner.trim()) { + value = inner.trim(); + } else { + const attrMatch = VALUE_ATTR_RE.exec(groups.attrs ?? ''); + value = attrMatch ? attrMatch[1]! : ''; + } + tags.push([tag, value]); + // wrapping tags keep their inner content; self-closing/lone tags vanish + return inner !== undefined ? inner : ''; + } + + const bracket = groups.bracket; + if (bracket !== undefined) { + tags.push(['', bracket.trim()]); + return ''; + } + + return ''; // lone closing tag + }; + + // iterate to a fixed point so nested wrapping tags are fully removed: a single pass + // strips only the outer tag (e.g. hi -> keeps the + // inner hi), so repeat until the text stops changing. Each pass removes + // at least the matched delimiters, so this always terminates. + let clean = text; + let prev: string | undefined = undefined; + while (clean !== prev) { + prev = clean; + clean = clean.replace(pattern, repl); + } + return [clean, tags]; +} + +/** Strip square bracket tags like `[laughs]`, `[whisper]` from text. */ +export function stripBracketTags(text: string): string { + return extractAndStrip(text, { xmlTags: [], brackets: true })[0]; +} + +/** + * Strip specific XML-style tags from text, preserving their inner content. + * + * Handles opening/closing tag pairs (`content`) and + * self-closing tags (``, ``). + * + * @param text - The text containing XML-style markup. + * @param tags - List of tag names to strip (e.g. `["emotion", "speed"]`). + * @returns The text with the specified tags removed but their content preserved. + */ +export function stripXmlTags(text: string, tags: string[]): string { + return extractAndStrip(text, { xmlTags: tags, brackets: false })[0]; +} diff --git a/agents/src/tts/tts.ts b/agents/src/tts/tts.ts index 3528e9c5c..d233970ae 100644 --- a/agents/src/tts/tts.ts +++ b/agents/src/tts/tts.ts @@ -20,6 +20,13 @@ import { } from '../types.js'; import { AsyncIterableQueue, delay, mergeFrames, startSoon, toError } from '../utils.js'; import type { TimedString } from '../voice/io.js'; +import type { ExpressiveTag } from './_provider_format.js'; +import { + convertMarkup, + normalizeMarkup, + llmInstructions as providerLlmInstructions, + splitMarkup, +} from './_provider_format.js'; /** * SynthesizedAudio is a packet of speech synthesis as returned by the TTS. @@ -79,6 +86,123 @@ export interface SynthesizeStreamStartedTime { hrTime: bigint; } +/** + * Declares TTS markup capabilities for the expressive pipeline. + * + * All methods delegate to the shared per-provider markup tables, keyed on the + * TTS's `_markupProviderKey()`. Plugins opt into markup support by overriding + * that method on their TTS class. + */ +export class TTSMarkup { + #tts: TTS; + + constructor(tts: TTS) { + this.#tts = tts; + } + + #providerKey(): string { + return this.#tts._markupProviderKey(); + } + + /** + * Return instructions for the LLM describing available markup tags. + * + * The framework injects this into the LLM system prompt when + * `expressive` is enabled. Returns `undefined` if this TTS has no markup support. + */ + llmInstructions(): string | undefined { + return providerLlmInstructions(this.#providerKey()); + } + + /** Strip markup and collect the stripped tags in one pass. */ + #split(text: string): [string, ExpressiveTag[]] { + return splitMarkup(this.#providerKey(), text); + } + + /** + * Strip TTS-specific markup from `text`, returning plain text. + * + * Used for transcripts streamed to the user and for chat history storage. + * The TTS itself receives the original marked-up text. + */ + toText(text: string): string { + return this.#split(text)[0]; + } + + /** + * Strip TTS markup from a stream of text chunks. + * + * Buffers partial XML tags across chunks so that each buffer is stripped as a + * whole. When `tagsOut` is given, the stripped tags are appended to it (in + * document order) as a byproduct of the same pass — no second scan. + */ + async *toTextStream( + textStream: AsyncIterable, + options?: { tagsOut?: ExpressiveTag[] }, + ): AsyncGenerator { + const tagsOut = options?.tagsOut; + let buf = ''; + for await (const chunk of textStream) { + buf += chunk; + // hold the buffer only for a *tag-shaped* trailing "<" (a partial tag + // arriving across chunks); a bare "<" as in "3 < 5" is plain text and + // must not stall the transcript until the next ">" or flush + const lastOpen = buf.lastIndexOf('<'); + if (lastOpen > buf.lastIndexOf('>')) { + const nxt = buf.slice(lastOpen + 1, lastOpen + 2); + if (!nxt || nxt === '/' || /[a-zA-Z]/.test(nxt)) { + continue; + } + } + const [stripped, tags] = this.#split(buf); + buf = ''; + if (tagsOut !== undefined) { + tagsOut.push(...tags); + } + if (stripped) { + yield stripped; + } + } + + if (buf) { + const [stripped, tags] = this.#split(buf); + if (tagsOut !== undefined) { + tagsOut.push(...tags); + } + if (stripped) { + yield stripped; + } + } + } + + /** + * Extract the markup tags that {@link toText} would strip, in order. + * + * Lets the framework surface stripped expressive tags (e.g. as transcription + * attributes for the frontend) instead of discarding them. Returns `[]` when + * the provider declares no markup. + */ + extractTags(text: string): ExpressiveTag[] { + return this.#split(text)[1]; + } + + /** Fix common LLM markup mistakes (e.g. unclosed self-closing tags). */ + normalize(text: string): string { + return normalizeMarkup(this.#providerKey(), text); + } + + /** + * Convert framework-standard markup to the provider's native format. + * + * Called before text is sent to the TTS; a no-op when the provider declares + * no markup. Plugins that use non-XML formats (e.g. square brackets) opt in + * via `_markupProviderKey` so `` becomes native syntax. + */ + convert(text: string): string { + return convertMarkup(this.#providerKey(), text); + } +} + /** * An instance of a text-to-speech adapter. * @@ -90,6 +214,15 @@ export abstract class TTS extends (EventEmitter as new () => TypedEmitter TypedEmitter): string { + const collectPaths = (d: Record, prefix = ''): string[] => { + const paths: string[] = []; + for (const [k, v] of Object.entries(d)) { + const full = prefix ? `${prefix}.${k}` : k; + if (v !== null && typeof v === 'object' && !Array.isArray(v)) { + paths.push(...collectPaths(v as Record, full)); + } else { + paths.push(full); + } + } + return paths; + }; + + return template.replace(/\{([\w.]+)\}/g, (_match, path: string) => { + let value: unknown = data; + for (const key of path.split('.')) { + if (value !== null && typeof value === 'object' && key in (value as object)) { + value = (value as Record)[key]; + } else { + value = undefined; + break; + } + } + if (value === undefined) { + log().error( + `template placeholder '{${path}}' has no value (available: ${collectPaths(data).join(', ')})`, + ); + return ''; + } + return value === null ? '' : String(value); + }); +} + const READONLY_SYMBOL = Symbol('Readonly'); const MUTATION_METHODS = [ diff --git a/agents/src/voice/agent.test.ts b/agents/src/voice/agent.test.ts index cbde76fa0..3b7bf7493 100644 --- a/agents/src/voice/agent.test.ts +++ b/agents/src/voice/agent.test.ts @@ -415,8 +415,10 @@ describe('Agent', () => { capabilities: { streaming: true }, stream: () => ttsStream, close: async () => {}, + _setExpressive: () => {}, }, agentSession: { connOptions: { ttsConnOptions: {} } }, + _resolveExpressiveOptions: () => undefined, }; async function* textInput() { diff --git a/agents/src/voice/agent.ts b/agents/src/voice/agent.ts index 47a2a3dd4..407a45ed3 100644 --- a/agents/src/voice/agent.ts +++ b/agents/src/voice/agent.ts @@ -34,13 +34,14 @@ import { type FlushSentinel, USERDATA_TIMED_TRANSCRIPT } from '../types.js'; import { Future, Task, toStream } from '../utils.js'; import type { VAD } from '../vad.js'; import { type AgentActivity, agentActivityStorage } from './agent_activity.js'; -import type { AgentSession, TurnDetectionMode } from './agent_session.js'; +import type { AgentSession, ExpressiveOptions, TurnDetectionMode } from './agent_session.js'; import { type AgentCreateOptions, type AgentTaskCreateOptions, createAgentTaskV2, createAgentV2, } from './agent_v2.js'; +import type { AudioRecognition } from './audio_recognition.js'; import type { UserTurnExceededEvent } from './events.js'; import type { TimedString } from './io.js'; import type { SpeechHandle } from './speech_handle.js'; @@ -143,6 +144,11 @@ export interface AgentOptions { tts?: TTS | TTSModelString; turnHandling?: TurnHandlingOptions; toolHandling?: ToolHandlingOptions; + /** + * Enable the expressive pipeline for this agent, overriding the session-level + * setting. See {@link AgentSessionOptions.expressive}. + */ + expressive?: boolean | ExpressiveOptions; minConsecutiveSpeechDelay?: number; useTtsAlignedTranscript?: boolean; /** @deprecated use turnHandling.turnDetection instead */ @@ -159,6 +165,7 @@ export class Agent { private _tts?: TTS; private _turnHandling?: Partial; + private _expressive?: boolean | ExpressiveOptions; private _minConsecutiveSpeechDelay?: number; private _useTtsAlignedTranscript?: boolean; @@ -194,6 +201,7 @@ export class Agent { allowInterruptions, turnHandling, toolHandling, + expressive, minConsecutiveSpeechDelay, useTtsAlignedTranscript, }: AgentOptions) { @@ -248,6 +256,7 @@ export class Agent { this._tts = tts; } + this._expressive = expressive; this._minConsecutiveSpeechDelay = minConsecutiveSpeechDelay; this._useTtsAlignedTranscript = useTtsAlignedTranscript; @@ -274,6 +283,14 @@ export class Agent { return this._useTtsAlignedTranscript; } + /** + * The agent-level expressive setting, overriding the session-level one when set. + * `undefined` means "not given" (fall back to the session option). + */ + get expressive(): boolean | ExpressiveOptions | undefined { + return this._expressive; + } + get chatCtx(): ReadonlyChatContext { return new ReadonlyChatContext(this._chatCtx.items); } @@ -294,6 +311,22 @@ export class Agent { return this.getActivityOrThrow().agentSession as AgentSession; } + /** + * Access the audio recognition system for this agent. + * + * The only public member is `sttContext` — live speaker metadata from the + * STT stream. + * + * @throws Error if the agent is not running. + */ + get audioRecognition(): AudioRecognition { + const audioRecognition = this.getActivityOrThrow()._audioRecognition; + if (!audioRecognition) { + throw new Error('audio recognition is not available for this agent'); + } + return audioRecognition; + } + get turnHandling(): Partial | undefined { return this._turnHandling; } @@ -522,12 +555,24 @@ export class Agent { throw new Error('ttsNode called but no TTS node is available'); } + const expressiveActive = activity._resolveExpressiveOptions() !== undefined; let wrappedTts = activity.tts; if (!activity.tts.capabilities.streaming) { - wrappedTts = new TTSStreamAdapter(wrappedTts, new BasicSentenceTokenizer()); + wrappedTts = new TTSStreamAdapter( + wrappedTts, + // markup only exists in the stream when expressive is active + new BasicSentenceTokenizer({ xmlAware: expressiveActive }), + ); } + // Mark whether expressive is active for this synthesis, synchronously + // just before stream() snapshots it. Doing it here (the single synthesis + // choke point for both generateReply and say()) scopes it to this turn + // rather than leaving stale state on the instance. The provider's chunk + // defaults then drive the TTS's input tokenizer. + activity.tts._setExpressive(expressiveActive); + const connOptions = activity.agentSession.connOptions.ttsConnOptions; const stream = wrappedTts.stream({ connOptions }); stream.updateInputStream(input); diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 656646681..1462b3fe1 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -17,14 +17,15 @@ import { import type { InterruptionDetectionError } from '../inference/interruption/errors.js'; import { AdaptiveInterruptionDetector } from '../inference/interruption/interruption_detector.js'; import type { OverlappingSpeechEvent } from '../inference/interruption/types.js'; +import { TTS as InferenceTTS } from '../inference/tts.js'; import { AgentConfigUpdate, type ChatContext, ChatMessage, - type Instructions, + Instructions, type MetricsReport, - concatInstructions, instructionsEqual, + isInstructions, renderInstructions, } from '../llm/chat_context.js'; import { AsyncToolset, type Toolset } from '../llm/index.js'; @@ -91,7 +92,12 @@ import { _setActivityTaskInfo, speechHandleStorage, } from './agent.js'; -import { type AgentSession, type TurnDetectionMode } from './agent_session.js'; +import { + type AgentSession, + DEFAULT_EXPRESSIVE_OPTIONS, + type ExpressiveOptions, + type TurnDetectionMode, +} from './agent_session.js'; import { AudioRecognition, type EndOfTurnInfo, @@ -120,7 +126,6 @@ import type { ToolExecutionOutput, ToolOutput, _TTSGenerationData } from './gene import { type _AudioOut, type _TextOut, - applyInstructionsModality, forwardedTextFor, performAudioForwarding, performLLMInference, @@ -131,6 +136,7 @@ import { updateInstructions, } from './generation.js'; import type { PlaybackFinishedEvent, TimedString } from './io.js'; +import { resolveOptions as resolveExpressivePreset } from './presets.js'; import { type InputDetails, SpeechHandle } from './speech_handle.js'; import { ToolExecutor, @@ -520,7 +526,7 @@ export class AgentActivity implements RecognitionHooks { try { const realtimeInstructions = !rtReused || capabilities.midSessionInstructionsUpdate - ? renderInstructions(this.agent.instructions) + ? this._renderRealtimeInstructions(this.agent.instructions) : undefined; await this.realtimeSession!._updateSession( realtimeInstructions, @@ -552,9 +558,12 @@ export class AgentActivity implements RecognitionHooks { const initialToolCtx = this.tools; const initialTools = Object.keys(initialToolCtx.functionTools); - if (runOnEnter && (this.agent.instructions || initialTools.length > 0)) { + // collapse modality variants for the record; audio-first matches the + // updateInstructions default for voice sessions + const initialInstructions = renderInstructions(this.agent.instructions, 'audio'); + if (runOnEnter && (initialInstructions || initialTools.length > 0)) { const initialConfig = new AgentConfigUpdate({ - instructions: this.agent.instructions, + instructions: initialInstructions, toolsAdded: initialTools.length > 0 ? initialTools : undefined, }); this.agent._chatCtx.insert(initialConfig); @@ -872,6 +881,11 @@ export class AgentActivity implements RecognitionHooks { return this.audioRecognition?.inputStartedAt; } + /** @internal */ + get _audioRecognition(): AudioRecognition | undefined { + return this.audioRecognition; + } + /** * @internal — used by AMD to obtain a private branch of the participant * audio stream that does not interfere with the pipeline VAD/STT. @@ -900,12 +914,15 @@ export class AgentActivity implements RecognitionHooks { async updateInstructions(instructions: string | Instructions): Promise { this.agent._instructions = instructions; - const configUpdate = new AgentConfigUpdate({ instructions }); + // Record the configuration change (audio-first, matching the initial config record) + const configUpdate = new AgentConfigUpdate({ + instructions: renderInstructions(instructions, 'audio'), + }); this.agent._chatCtx.insert(configUpdate); this.agentSession.history.insert(configUpdate); if (this.realtimeSession) { - await this.realtimeSession.updateInstructions(renderInstructions(instructions)); + await this.realtimeSession.updateInstructions(this._renderRealtimeInstructions(instructions)); } else { updateInstructions({ chatCtx: this.agent._chatCtx, @@ -1837,6 +1854,91 @@ export class AgentActivity implements RecognitionHooks { return this.agentSession.chatCtx; } + /** + * Resolve instructions to a plain string for the realtime session. + * + * Realtime instructions are session-level (there is no per-turn modality + * resolution like the pipeline path), so modality-specific {@link Instructions} + * resolve to the realtime model's output modality. + * @internal + */ + _renderRealtimeInstructions(instructions: string | Instructions): string { + if (isInstructions(instructions)) { + const modality: 'audio' | 'text' = + this.llm instanceof RealtimeModel && this.llm.capabilities.audioOutput ? 'audio' : 'text'; + return instructions.render({ modality }); + } + return instructions; + } + + /** + * Resolve expressive from agent (overrides session). Returns undefined if disabled. + * + * Expressive mode requires two things: + * - the inference gateway TTS (`livekit.agents.inference.TTS`): the markup + * normalization/conversion and expressive chunking run there, so direct + * provider plugins would receive unconverted markup. + * - a TTS that actually declares a markup dialect (`llmInstructions()` is + * not `undefined`): gateway providers without one (e.g. `rime`, `deepgram`) + * get no markup instructions, so no tags can appear in the stream — leaving + * it "active" would enable xml-aware chunking with nothing to chunk and + * re-introduce the stray-`<` streaming stall. + * @internal + */ + _resolveExpressiveOptions(): ExpressiveOptions | undefined { + if (!(this.tts instanceof InferenceTTS) || this.tts.markup.llmInstructions() === undefined) { + return undefined; + } + + let expr = this.agent.expressive; + if (expr === undefined) { + expr = this.agentSession.sessionOptions.expressive; + } + if (typeof expr === 'object') { + // a `preset` selector resolves to the active TTS provider's tuned preset + // (falling back to the agnostic default); explicit fields override on top + const providerKey = this.tts ? this.tts._markupProviderKey() : ''; + return resolveExpressivePreset(expr, { + providerKey, + defaultOptions: DEFAULT_EXPRESSIVE_OPTIONS, + }); + } + return expr ? DEFAULT_EXPRESSIVE_OPTIONS : undefined; + } + + /** + * Inject the TTS markup guide into the chat context. + * @internal + */ + _injectExpressiveInstructions( + chatCtx: ChatContext, + options: ExpressiveOptions, + speechHandle: SpeechHandle, + ): void { + const toInstructions = (v: Instructions | string): Instructions => + isInstructions(v) ? v : new Instructions(v); + + const turnModality = speechHandle?.inputDetails.modality; + + const ttsInstructions = this.tts ? this.tts.markup.llmInstructions() : undefined; + if (ttsInstructions) { + const ttsTemplate = toInstructions(options.ttsInstructionsTemplate!); + const text = ttsTemplate.render({ + modality: turnModality, + data: { + tts: { + markup: { + llm_instructions: ttsInstructions, + }, + }, + }, + }); + if (text.trim()) { + chatCtx.addMessage({ role: 'system', content: text }); + } + } + } + async waitForIdle( options: { waitForAgent?: boolean; waitForUser?: boolean } = {}, ): Promise { @@ -2036,7 +2138,7 @@ export class AgentActivity implements RecognitionHooks { inputDetails, } = options; - let instructions: string | Instructions | undefined = defaultInstructions; + const instructions: string | Instructions | undefined = defaultInstructions; let toolChoice = defaultToolChoice; let allowInterruptions = defaultAllowInterruptions; @@ -2090,7 +2192,10 @@ export class AgentActivity implements RecognitionHooks { speechHandle: handle, // TODO(brian): support llm.ChatMessage for the realtime model userInput: userMessage?.textContent, - instructions, + instructions: + instructions !== undefined + ? this._renderRealtimeInstructions(instructions) + : undefined, modelSettings: { // isGiven(toolChoice) = toolChoice !== undefined toolChoice: toOaiToolChoice(toolChoice !== undefined ? toolChoice : this.toolChoice), @@ -2101,12 +2206,10 @@ export class AgentActivity implements RecognitionHooks { name: 'AgentActivity.realtimeReply', }); } else if (this.llm instanceof LLM) { - // instructions used inside generateReply are "extra" instructions. - // this matches the behavior of the Realtime API: + // instructions used inside generateReply are "extra" instructions: they are + // appended as a separate system message for this turn only, matching the + // behavior of the Realtime API: // https://platform.openai.com/docs/api-reference/realtime-client-events/response/create - if (instructions) { - instructions = concatInstructions(this.agent.instructions, '\n', instructions); - } // Filter out tools with IGNORE_ON_ENTER flag when generateReply is called inside onEnter const onEnterData = onEnterStorage.getStore(); @@ -2618,7 +2721,10 @@ export class AgentActivity implements RecognitionHooks { span.setAttribute(traceTypes.ATTR_SPEECH_ID, speechHandle.id); if (instructions) { - span.setAttribute(traceTypes.ATTR_INSTRUCTIONS, renderInstructions(instructions)); + span.setAttribute( + traceTypes.ATTR_INSTRUCTIONS, + renderInstructions(instructions, speechHandle.inputDetails.modality), + ); } if (newMessage) { span.setAttribute(traceTypes.ATTR_USER_INPUT, newMessage.textContent || ''); @@ -2645,20 +2751,32 @@ export class AgentActivity implements RecognitionHooks { chatCtx.insert(newMessage); } + // resolve modality-specific instructions for this turn + const turnModality = speechHandle.inputDetails.modality; if (instructions) { + // extra instructions are appended as a separate system message for this turn + chatCtx.addMessage({ + role: 'system', + content: [renderInstructions(instructions, turnModality)], + }); + } else if (isInstructions(this.agent.instructions)) { try { updateInstructions({ chatCtx, - instructions, - addIfMissing: true, + instructions: this.agent.instructions, + modality: turnModality, + addIfMissing: false, }); } catch (e) { this.logger.error({ error: e }, 'error occurred during updateInstructions'); } } - // apply the correct variant of the instructions for the turn's input modality - applyInstructionsModality(chatCtx, { modality: speechHandle.inputDetails.modality }); + // inject expressive instructions (TTS markup guide) + const exprOpts = this._resolveExpressiveOptions(); + if (exprOpts !== undefined) { + this._injectExpressiveInstructions(chatCtx, exprOpts, speechHandle); + } const tasks: Array> = []; const [llmTask, llmGenData] = performLLMInference( @@ -3857,7 +3975,8 @@ export class AgentActivity implements RecognitionHooks { modelSettings: ModelSettings; abortController: AbortController; userInput?: string; - instructions?: string | Instructions; + /** Extra instructions, already resolved to a plain string by the caller. */ + instructions?: string; }): Promise { speechHandleStorage.enterWith(speechHandle); @@ -3888,12 +4007,9 @@ export class AgentActivity implements RecognitionHooks { try { const generateReplyAbortController = new AbortController(); - const generationPromise = this.realtimeSession.generateReply( - instructions !== undefined - ? renderInstructions(instructions, speechHandle.inputDetails.modality) - : undefined, - { signal: generateReplyAbortController.signal }, - ); + const generationPromise = this.realtimeSession.generateReply(instructions, { + signal: generateReplyAbortController.signal, + }); void generationPromise.catch(() => undefined); await speechHandle.waitIfNotInterrupted([generationPromise]); diff --git a/agents/src/voice/agent_session.ts b/agents/src/voice/agent_session.ts index f7a8dbb75..5473972ed 100644 --- a/agents/src/voice/agent_session.ts +++ b/agents/src/voice/agent_session.ts @@ -27,12 +27,7 @@ import { import type { OverlappingSpeechEvent } from '../inference/interruption/types.js'; import { getJobContext } from '../job.js'; import type { FunctionCall, FunctionCallOutput } from '../llm/chat_context.js'; -import { - AgentHandoffItem, - ChatContext, - ChatMessage, - type Instructions, -} from '../llm/chat_context.js'; +import { AgentHandoffItem, ChatContext, ChatMessage, Instructions } from '../llm/chat_context.js'; import type { LLM, RealtimeModel, @@ -92,6 +87,7 @@ import { createUserStateChangedEvent, } from './events.js'; import { AgentInput, AgentOutput } from './io.js'; +import type { Preset } from './presets.js'; import { RecorderIO } from './recorder_io/index.js'; import { RoomSessionTransport, SessionHost } from './remote_session.js'; import { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io/index.js'; @@ -174,6 +170,35 @@ export function resolveRecordingOptions( return { ...RECORDING_ALL_ON, ...record }; } +/** + * Configuration for the expressive pipeline. + * + * Controls how TTS markup instructions are injected into the LLM when expressive is + * enabled. All keys are optional; common shapes: + * + * - `{ preset: Preset.CASUAL }` — a domain preset, resolved to the active + * TTS provider's tuned tags (see `voice/presets`). Prefer the `presets.*` constants. + * - `{ preset: ..., ttsInstructionsAppend: '...' }` — a preset plus your own + * rules appended after it resolves. + * - `{ ttsInstructionsTemplate: '...' }` — a fully custom prompt. + * + * Any explicit template overrides the corresponding part of the resolved preset; unset + * parts fall back to the resolved preset (or the provider-agnostic default). + */ +export interface ExpressiveOptions { + preset?: Preset; + ttsInstructionsTemplate?: Instructions | string; + ttsInstructionsAppend?: string; +} + +export const DEFAULT_EXPRESSIVE_OPTIONS: ExpressiveOptions = { + ttsInstructionsTemplate: new Instructions( + 'You can control how you speak using the following formatting tags. ' + + 'Use them when appropriate to make your speech more expressive and natural:\n\n' + + '{tts.markup.llm_instructions}', + ), +}; + export interface InternalSessionOptions extends AgentSessionOptions { turnHandling: InternalTurnHandlingOptions; useTtsAlignedTranscript: boolean; @@ -182,6 +207,7 @@ export interface InternalSessionOptions extends AgentSessionOptions = { * and `filter_emoji`; pass `null` to disable text transforms. */ ttsTextTransforms?: readonly TextTransform[] | null; + + /** + * Enable the expressive pipeline: the LLM is instructed to emit the active TTS + * provider's markup tags (emotion/expression/pauses/...), the tags are converted + * to the provider's native syntax before synthesis, and stripped from the room + * transcript. Pass `true` for the provider-agnostic default prompt, or an + * {@link ExpressiveOptions} (e.g. a `presets.*` constant) to tune it. + * @defaultValue false + */ + expressive?: boolean | ExpressiveOptions; }; export type AgentSessionUpdateOptions = { diff --git a/agents/src/voice/audio_recognition.ts b/agents/src/voice/audio_recognition.ts index d2005bcbc..907e62657 100644 --- a/agents/src/voice/audio_recognition.ts +++ b/agents/src/voice/audio_recognition.ts @@ -35,7 +35,7 @@ import { DeferredReadableStream } from '../stream/deferred_stream.js'; import { IdentityTransform } from '../stream/identity_transform.js'; import { mergeReadableStreams } from '../stream/merge_readable_streams.js'; import { type StreamChannel, createStreamChannel } from '../stream/stream_channel.js'; -import { type SpeechEvent, SpeechEventType } from '../stt/stt.js'; +import { type SpeechEvent, SpeechEventType, isSpeakerContext } from '../stt/stt.js'; import { traceTypes, tracer } from '../telemetry/index.js'; import { splitWords } from '../tokenize/basic/word.js'; import type { Future } from '../utils.js'; @@ -355,6 +355,7 @@ export class AudioRecognition { private silenceAudioWriter: WritableStreamDefaultWriter; private sttOwnershipTransferred = false; private readonly sttLifecycleLock = new Mutex(); + private _sttContext?: object; // all cancellable tasks private bounceEOUTask?: Task; @@ -524,6 +525,36 @@ export class AudioRecognition { return this.sttPipeline?.inputStartedAt; } + /** + * Live speaker metadata from the STT stream. + * + * STT plugins set `SpeechStream.context` during recognition. + * The framework copies it here so it's accessible even after the stream + * is replaced (e.g. during agent handoff). + */ + get sttContext(): object | undefined { + return this._sttContext; + } + + set sttContext(value: object | undefined) { + this._sttContext = value; + } + + /** + * Speaker context formatted as LLM instructions. + * + * Returns `sttContext.toInstructions()` if the context implements + * {@link SpeakerContext}, otherwise `undefined`. + */ + llmInstructions(): string | undefined { + const ctx = this._sttContext; + if (ctx !== undefined && isSpeakerContext(ctx)) { + const result = ctx.toInstructions(); + return result ? result : undefined; + } + return undefined; + } + /** @internal */ updateOptions(options: { endpointing?: BaseEndpointing; diff --git a/agents/src/voice/generation.ts b/agents/src/voice/generation.ts index 3d6a4c831..0579dd019 100644 --- a/agents/src/voice/generation.ts +++ b/agents/src/voice/generation.ts @@ -13,7 +13,7 @@ import { ChatMessage, FunctionCall, FunctionCallOutput, - isInstructions, + renderInstructions, } from '../llm/chat_context.js'; import type { ChatChunk } from '../llm/llm.js'; import { @@ -401,19 +401,24 @@ export const INSTRUCTIONS_MESSAGE_ID = 'lk.agent_task.instructions'; * Update the instruction message in the chat context or insert a new one if missing. * * This function looks for an existing instruction message in the chat context using the identifier - * 'INSTRUCTIONS_MESSAGE_ID'. + * 'INSTRUCTIONS_MESSAGE_ID'. Instructions are resolved to a plain string using the given modality + * before storage. * * @param options - The options for updating the instructions. * @param options.chatCtx - The chat context to update. * @param options.instructions - The instructions to add. * @param options.addIfMissing - Whether to add the instructions if they are missing. + * @param options.modality - The modality used to render {@link Instructions} (default `'audio'`). */ export function updateInstructions(options: { chatCtx: ChatContext; instructions: string | Instructions; addIfMissing: boolean; + modality?: 'audio' | 'text'; }) { - const { chatCtx, instructions, addIfMissing } = options; + const { chatCtx, instructions, addIfMissing, modality = 'audio' } = options; + + const text = renderInstructions(instructions, modality); const idx = chatCtx.indexById(INSTRUCTIONS_MESSAGE_ID); if (idx !== undefined) { @@ -422,7 +427,7 @@ export function updateInstructions(options: { chatCtx.items[idx] = ChatMessage.create({ id: INSTRUCTIONS_MESSAGE_ID, role: 'system', - content: [instructions], + content: [text], createdAt: chatCtx.items[idx]!.createdAt, }); } else { @@ -434,49 +439,12 @@ export function updateInstructions(options: { ChatMessage.create({ id: INSTRUCTIONS_MESSAGE_ID, role: 'system', - content: [instructions], + content: [text], }), ); } } -/** - * Apply the correct {@link Instructions} variant for the turn's input modality. - * - * Locates the instructions message (by {@link INSTRUCTIONS_MESSAGE_ID}) and, - * if its content contains any {@link Instructions} entries, rebuilds the - * message so each Instructions renders as the chosen variant. No-op when no - * modality-aware instructions are present. - */ -export function applyInstructionsModality( - chatCtx: ChatContext, - options: { modality: 'audio' | 'text' }, -) { - const { modality } = options; - const idx = chatCtx.indexById(INSTRUCTIONS_MESSAGE_ID); - if (idx === undefined) return; - - const item = chatCtx.items[idx]!; - if (item.type !== 'message') return; - - const hasModalitySpecific = item.content.some((c) => isInstructions(c)); - if (!hasModalitySpecific) return; - - // ChatContext.copy shadows the original item; create a new instance so the - // base context's content isn't mutated when the same Instructions is reused - // across turns. - chatCtx.items[idx] = ChatMessage.create({ - id: item.id, - role: item.role, - content: item.content.map((c) => (isInstructions(c) ? c.asModality(modality) : c)), - interrupted: item.interrupted, - createdAt: item.createdAt, - transcriptConfidence: item.transcriptConfidence, - metrics: item.metrics, - extra: item.extra, - }); -} - export function performLLMInference( node: LLMNode, chatCtx: ChatContext, diff --git a/agents/src/voice/presets.ts b/agents/src/voice/presets.ts new file mode 100644 index 000000000..5ba0f6020 --- /dev/null +++ b/agents/src/voice/presets.ts @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * Public expressive presets. + * + * A preset is a *use-case* (customer service, casual) that is + * provider-agnostic at the call site: + * + * ```ts + * import { presets } from '@livekit/agents/voice'; + * + * const session = new AgentSession({ tts: 'inworld/inworld-tts-2', expressive: presets.CASUAL }); + * ``` + * + * Each `presets.*` constant is just an {@link ExpressiveOptions} carrying a `preset`. + * At session start the framework resolves it against the active TTS provider (via + * `tts._markupProviderKey()`) and injects the variant tuned for that provider's markup + * tags. A provider with no tuned preset falls back to the agnostic default, which still + * injects that provider's tag reference through the `{tts.markup.llm_instructions}` + * placeholder — so a preset always does something sensible and can never disagree with + * the markup pipeline (both read the same provider key). + * + * Customize by spreading a constant into a new object (don't mutate the constant in place): + * + * ```ts + * expressive: { ...presets.CUSTOMER_SERVICE, ttsInstructionsAppend: 'Confirm the name.' } + * ``` + */ +import { Instructions, isInstructions } from '../llm/chat_context.js'; +import { + CARTESIA_CASUAL, + CARTESIA_CUSTOMER_SERVICE, + INWORLD_CASUAL, + INWORLD_CUSTOMER_SERVICE, + XAI_CASUAL, + XAI_CUSTOMER_SERVICE, +} from '../tts/_provider_format.js'; +import type { ExpressiveOptions } from './agent_session.js'; + +/** The domain a preset is tuned for. Used to key the per-provider registry. */ +export enum Preset { + CUSTOMER_SERVICE = 'customer_service', + CASUAL = 'casual', +} + +// (provider key as returned by `tts._markupProviderKey()`) -> preset -> body +const REGISTRY: Record>> = { + inworld: { + [Preset.CUSTOMER_SERVICE]: INWORLD_CUSTOMER_SERVICE, + [Preset.CASUAL]: INWORLD_CASUAL, + }, + cartesia: { + [Preset.CUSTOMER_SERVICE]: CARTESIA_CUSTOMER_SERVICE, + [Preset.CASUAL]: CARTESIA_CASUAL, + }, + xai: { + [Preset.CUSTOMER_SERVICE]: XAI_CUSTOMER_SERVICE, + [Preset.CASUAL]: XAI_CASUAL, + }, +}; + +function append(template: Instructions | string, extra: string): Instructions { + // concatenate the *raw* template text so any {placeholders} survive until render() + if (isInstructions(template)) { + return new Instructions(template.common + '\n\n' + extra, { + audio: template.audio, + text: template.text, + }); + } + return new Instructions(template + '\n\n' + extra); +} + +/** + * Resolve a user {@link ExpressiveOptions} to a concrete options object for a provider. + * + * If `expr` carries a `preset`, start from that provider's tuned preset (or + * `defaultOptions` when the provider has none); otherwise start from `defaultOptions`. + * Then apply any explicit `ttsInstructionsTemplate` override and `ttsInstructionsAppend`. + * The returned object always has `ttsInstructionsTemplate` and never the `preset` / + * `ttsInstructionsAppend` helper keys. + */ +export function resolveOptions( + expr: ExpressiveOptions, + options: { providerKey: string; defaultOptions: ExpressiveOptions }, +): ExpressiveOptions { + const { providerKey, defaultOptions } = options; + + const preset = expr.preset; + const base = + preset !== undefined ? REGISTRY[providerKey]?.[preset] ?? defaultOptions : defaultOptions; + + let ttsTmpl = expr.ttsInstructionsTemplate ?? base.ttsInstructionsTemplate!; + const extra = expr.ttsInstructionsAppend; + if (extra) { + ttsTmpl = append(ttsTmpl, extra); + } + + return { + ttsInstructionsTemplate: ttsTmpl, + }; +} + +export const CUSTOMER_SERVICE: ExpressiveOptions = { preset: Preset.CUSTOMER_SERVICE }; +export const CASUAL: ExpressiveOptions = { preset: Preset.CASUAL }; diff --git a/agents/src/voice/remote_session.ts b/agents/src/voice/remote_session.ts index ae2af7bbe..92b4a473a 100644 --- a/agents/src/voice/remote_session.ts +++ b/agents/src/voice/remote_session.ts @@ -15,7 +15,7 @@ import type { FunctionCall as FCItem, FunctionCallOutput as FCOItem, } from '../llm/chat_context.js'; -import { isInstructions, renderInstructions } from '../llm/chat_context.js'; +import { renderInstructions } from '../llm/chat_context.js'; import { type ToolContext, sortedToolNames } from '../llm/tool_context.js'; import { log } from '../log.js'; import type { @@ -421,10 +421,6 @@ function chatItemToProto(item: RemoteChatItem): pb.ChatContext_ChatItem { for (const c of msg.content) { if (typeof c === 'string') { content.push(new pb.ChatMessage_ChatContent({ payload: { case: 'text', value: c } })); - } else if (isInstructions(c)) { - content.push( - new pb.ChatMessage_ChatContent({ payload: { case: 'text', value: c.value } }), - ); } } diff --git a/agents/src/voice/room_io/_output.ts b/agents/src/voice/room_io/_output.ts index 11dd8eb71..316d2c708 100644 --- a/agents/src/voice/room_io/_output.ts +++ b/agents/src/voice/room_io/_output.ts @@ -23,6 +23,12 @@ import { TOPIC_TRANSCRIPTION, } from '../../constants.js'; import { log } from '../../log.js'; +import { + type ExpressiveTag, + TranscriptMarkupStripper, + expressionAttribute, + splitAllMarkup, +} from '../../tts/_provider_format.js'; import { Future, Task, shortuuid } from '../../utils.js'; import { AudioOutput, TextOutput, type TimedString, isTimedString } from '../io.js'; import { findMicrophoneTrackId } from '../transcription/index.js'; @@ -143,6 +149,11 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription private writer: TextStreamWriter | null = null; private flushTask: Task | null = null; private jsonFormat: boolean; + // per-segment markup stripping: delta streams strip incrementally (buffering a tag + // split across chunks); non-delta streams re-strip the full text each time and keep + // the latest tags here for the expression attribute (see TranscriptMarkupStripper) + private stripper: TranscriptMarkupStripper = new TranscriptMarkupStripper(); + private segmentTags: ExpressiveTag[] = []; constructor( room: Room, @@ -154,35 +165,17 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription this.jsonFormat = options.jsonFormat ?? false; } + protected override resetState() { + super.resetState(); + this.stripper = new TranscriptMarkupStripper(); + this.segmentTags = []; + } + override async captureText(text: string | TimedString) { if (!this.participantIdentity) { return; } - // latestText must hold the encoded payload so non-delta flush (FINAL=true) republishes the - // same newline-delimited JSON format as the interim chunks. - const payload = this.jsonFormat - ? this.encodeJsonChunk(text) - : isTimedString(text) - ? text.text - : text; - this.latestText = payload; - await this.handleCaptureText(payload); - } - - private encodeJsonChunk(text: string | TimedString): string { - const isTimed = isTimedString(text); - const message = new pb.TimedString({ - text: isTimed ? text.text : text, - startTime: isTimed ? text.startTime : undefined, - endTime: isTimed ? text.endTime : undefined, - confidence: isTimed ? text.confidence : undefined, - startTimeOffset: isTimed ? text.startTimeOffset : undefined, - }); - return message.toJsonString({ useProtoFieldName: true }) + '\n'; - } - - protected async handleCaptureText(text: string): Promise { if (this.flushTask && !this.flushTask.done) { await this.flushTask.result; } @@ -192,6 +185,27 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription this.capturing = true; } + // the raw text (expressive markup intact) arrives here; publish only the visible + // text. Skip a chunk that strips to nothing (a partial tag still buffering, or a + // markup-only token) so the transcript cadence isn't disturbed. + const textStr = isTimedString(text) ? text.text : text; + let cleanText: string; + if (this.isDeltaStream) { + cleanText = this.stripper.push(textStr); + } else { + const [clean, tags] = splitAllMarkup(textStr); + cleanText = clean; + this.segmentTags = tags; + } + if (!cleanText) { + return; + } + + // latestText must hold the encoded payload so non-delta flush (FINAL=true) republishes the + // same newline-delimited JSON format as the interim chunks. + const payload = this.encode(cleanText, text); + this.latestText = payload; + try { if (this.room.isConnected) { if (this.isDeltaStream) { @@ -199,10 +213,10 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription if (this.writer === null) { this.writer = await this.createTextWriter(); } - await this.writer.write(text); + await this.writer.write(payload); } else { const tmpWriter = await this.createTextWriter(); - await tmpWriter.write(text); + await tmpWriter.write(payload); await tmpWriter.close(); } } @@ -211,10 +225,48 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription } } + /** Wrap visible text for the wire (JSON TimedString when jsonFormat, else raw). */ + private encode(cleanText: string, timingSrc?: string | TimedString): string { + if (!this.jsonFormat) { + return cleanText; + } + + const isTimed = timingSrc !== undefined && isTimedString(timingSrc); + const message = new pb.TimedString({ + text: cleanText, + startTime: isTimed ? timingSrc.startTime : undefined, + endTime: isTimed ? timingSrc.endTime : undefined, + confidence: isTimed ? timingSrc.confidence : undefined, + startTimeOffset: isTimed ? timingSrc.startTimeOffset : undefined, + }); + return message.toJsonString({ useProtoFieldName: true }) + '\n'; + } + + protected async handleCaptureText(_text: string): Promise { + // unused: captureText is fully overridden to strip markup before encoding + } + protected handleFlush() { const currWriter = this.writer; this.writer = null; - this.flushTask = Task.from((controller) => this.flushTaskImpl(currWriter, controller.signal)); + + // only emit on a segment that captured text (keeps lk.transcription cadence intact). + // The leading expression the sinks stripped rides along on the closing chunk as the + // lk.expression attribute. + let remaining: string; + let tags: ExpressiveTag[]; + if (this.isDeltaStream) { + remaining = this.stripper.flush(); + tags = this.stripper.tags; + } else { + remaining = ''; + tags = this.segmentTags; + } + const pendingText = remaining ? this.encode(remaining) : ''; + + this.flushTask = Task.from((controller) => + this.flushTaskImpl(currWriter, expressionAttribute(tags), pendingText, controller.signal), + ); } private async createTextWriter(attributes?: Record): Promise { @@ -243,13 +295,23 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription }); } - private async flushTaskImpl(writer: TextStreamWriter | null, signal: AbortSignal): Promise { + private async flushTaskImpl( + writer: TextStreamWriter | null, + extraAttributes: Record | undefined, + pendingText: string, + signal: AbortSignal, + ): Promise { const attributes: Record = { [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'true', }; if (this.trackId) { attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId; } + for (const [key, val] of Object.entries(extraAttributes ?? {})) { + if (!(key in attributes)) { + attributes[key] = val; + } + } const abortPromise = new Promise((resolve) => { signal.addEventListener('abort', () => resolve()); @@ -259,6 +321,16 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription if (this.room.isConnected) { if (this.isDeltaStream) { if (writer) { + if (pendingText) { + // visible text left in the strip buffer + await Promise.race([writer.write(pendingText), abortPromise]); + if (signal.aborted) { + return; + } + } + // NOTE: rtc-node's TextStreamWriter.close() takes no attributes yet, so the + // lk.expression attribute cannot ride the closing chunk of a delta stream + // (Python attaches it via `aclose(attributes=...)`). await Promise.race([writer.close(), abortPromise]); } } else { @@ -303,7 +375,12 @@ export class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscr this.pushedText = text; } - await this.publishTranscription(this.currentId, this.pushedText, false); + // pushedText keeps the raw text (markup intact); publish the visible text only. + // Stripping the whole accumulation each time avoids partial-tag edge cases; the + // expression is dropped here — the deprecated rtc Transcription API has no + // attribute channel (the stream-based output carries lk.expression instead). + const [cleanText] = splitAllMarkup(this.pushedText); + await this.publishTranscription(this.currentId, cleanText, false); } protected handleFlush() { @@ -311,7 +388,8 @@ export class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscr return; } - this.flushTask = this.publishTranscription(this.currentId, this.pushedText, true); + const [cleanText] = splitAllMarkup(this.pushedText); + this.flushTask = this.publishTranscription(this.currentId, cleanText, true); this.resetState(); } diff --git a/agents/src/voice/transcription/synchronizer.ts b/agents/src/voice/transcription/synchronizer.ts index dc5c081f2..b71205aad 100644 --- a/agents/src/voice/transcription/synchronizer.ts +++ b/agents/src/voice/transcription/synchronizer.ts @@ -7,6 +7,7 @@ import { log } from '../../log.js'; import { IdentityTransform } from '../../stream/identity_transform.js'; import type { WordStream, WordTokenizer } from '../../tokenize/index.js'; import { basic } from '../../tokenize/index.js'; +import { splitAllMarkup } from '../../tts/_provider_format.js'; import { Future, Task, delay } from '../../utils.js'; import { AudioOutput, @@ -435,6 +436,14 @@ class SegmentSynchronizerImpl { const forwardedWord = this.textData.pushedText.slice(pushedTextCursor, wordEnd); pushedTextCursor = wordEnd; + // forward the raw token (the room output strips markup and surfaces the + // expression downstream), but pace against the visible text only so a + // markup-only token adds no delay + const [strippedWord] = splitAllMarkup(word); + const cleanWords = this.options.splitWords(strippedWord); + const cleanWord = cleanWords.length > 0 ? cleanWords[0]![0] : strippedWord; + const wordHyphens = cleanWord ? this.options.hyphenateWord(cleanWord).length : 0; + if (this.playbackCompleted) { this.outputStreamWriter.write( createTimedString({ @@ -442,16 +451,11 @@ class SegmentSynchronizerImpl { endTime: this.synchronizedElapsedSeconds(), }), ); - const cleanWords = this.options.splitWords(word); - const cleanWord = cleanWords.length > 0 ? cleanWords[0]![0] : word; - this.textData.forwardedHyphens += this.options.hyphenateWord(cleanWord).length; + this.textData.forwardedHyphens += wordHyphens; this.textData.forwardedText += forwardedWord; continue; } - const cleanWords = this.options.splitWords(word); - const cleanWord = cleanWords.length > 0 ? cleanWords[0]![0] : word; - const wordHyphens = this.options.hyphenateWord(cleanWord).length; const elapsedSeconds = this.synchronizedElapsedSeconds()!; let dHyphens = 0; diff --git a/agents/src/voice/turn_config/utils.ts b/agents/src/voice/turn_config/utils.ts index 100cda367..afceca7e8 100644 --- a/agents/src/voice/turn_config/utils.ts +++ b/agents/src/voice/turn_config/utils.ts @@ -28,6 +28,7 @@ const defaultSessionOptions = { turnHandling: {}, useTtsAlignedTranscript: true, ttsTextTransforms: ['filter_markdown', 'filter_emoji'], + expressive: false, } as const satisfies AgentSessionOptions; const defaultLegacyVoiceOptions: VoiceOptions = { diff --git a/agents/src/workflows/warm_transfer.ts b/agents/src/workflows/warm_transfer.ts index 8a94426f1..cf4141965 100644 --- a/agents/src/workflows/warm_transfer.ts +++ b/agents/src/workflows/warm_transfer.ts @@ -492,7 +492,7 @@ export class WarmTransferTask extends AgentTask { } const renderInstructionPart = (value: Instructions | string): string => - typeof value === 'string' ? value : value.value; + typeof value === 'string' ? value : value.toString(); function resolveInstructions( instructions: InstructionParts | string | undefined, diff --git a/examples/src/instructions_per_modality.ts b/examples/src/instructions_per_modality.ts index f2ab6b910..8f4823e38 100644 --- a/examples/src/instructions_per_modality.ts +++ b/examples/src/instructions_per_modality.ts @@ -14,9 +14,8 @@ import { import { fileURLToPath } from 'node:url'; import { z } from 'zod'; -const BASE_INSTRUCTIONS = (modalitySpecific: string, currentDate: string) => +const BASE_INSTRUCTIONS = (currentDate: string) => `You are a scheduling assistant named Alex that helps users book appointments. -${modalitySpecific} Call \`book_appointment\` to finalise the booking. Never invent or assume details the user did not provide — ask for them instead. The current date is ${currentDate}. @@ -48,9 +47,11 @@ class SchedulingAgent extends voice.Agent { const now = new Date(); const weekday = now.toLocaleDateString(undefined, { weekday: 'long' }); const currentDate = `${now.toISOString().slice(0, 10)} ${weekday}`; - const instructions = new llm.Instructions({ - audio: BASE_INSTRUCTIONS(AUDIO_SPECIFIC, currentDate), - text: BASE_INSTRUCTIONS(TEXT_SPECIFIC, currentDate), + // common text is always present; the audio/text additions are appended + // depending on the turn's input modality + const instructions = new llm.Instructions(BASE_INSTRUCTIONS(currentDate), { + audio: AUDIO_SPECIFIC, + text: TEXT_SPECIFIC, }); super({ diff --git a/plugins/cartesia/src/tts.ts b/plugins/cartesia/src/tts.ts index cc622f36d..8846de1bb 100644 --- a/plugins/cartesia/src/tts.ts +++ b/plugins/cartesia/src/tts.ts @@ -138,6 +138,15 @@ export class TTS extends tts.TTS { return 'Cartesia'; } + /** + * Key into the shared expressive markup tables; markup delegation lives in + * the base class. + * @internal + */ + override _markupProviderKey(): string { + return 'cartesia'; + } + constructor(opts: Partial = {}) { const resolvedOpts = { ...defaultTTSOptions, diff --git a/plugins/inworld/src/tts.ts b/plugins/inworld/src/tts.ts index bbf23e0f5..0daf56502 100644 --- a/plugins/inworld/src/tts.ts +++ b/plugins/inworld/src/tts.ts @@ -335,6 +335,21 @@ export class TTS extends tts.TTS { return this.#pool; } + override get model(): string { + return this.#opts.model; + } + + /** + * Key into the shared expressive markup tables; markup delegation lives in + * the base class. Only inworld-tts-2 understands the markup tags; older + * models get no markup so the tags aren't injected, converted, or stripped + * (matches the inference gateway's behavior). + * @internal + */ + override _markupProviderKey(): string { + return this.#opts.model.includes('tts-2') ? 'inworld' : ''; + } + get authorization(): string { return this.#authorization; } From e08794b03b7dc8b6dcc1b1523a033e757a24f609 Mon Sep 17 00:00:00 2001 From: Toubat Date: Wed, 8 Jul 2026 11:26:13 +0800 Subject: [PATCH 02/12] fix(tokenize): make XML tag regex linear-time CodeQL flagged js/polynomial-redos on XML_TAG_RE: the lazy [^>]*? and trailing \s* both match whitespace, so unterminated input like ']* and detect self-closing tags on the captured body instead. Matching behavior is unchanged. Co-Authored-By: Oz --- agents/src/tokenize/token_stream.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/agents/src/tokenize/token_stream.ts b/agents/src/tokenize/token_stream.ts index 34bb38a1a..93ba3cdd3 100644 --- a/agents/src/tokenize/token_stream.ts +++ b/agents/src/tokenize/token_stream.ts @@ -9,8 +9,16 @@ type TokenizeFunc = (x: string) => string[] | [string, number, number][]; // the tag name must start with a letter so "<5>" / "<3 wins>" are not counted as // tags — this keeps the depth counter consistent with the letter-start tail check -// in hasUnclosedXmlTags (all TTS markup tags are letter-named) -const XML_TAG_RE = /<(\/?)([A-Za-z]\w*)[^>]*?(\/?)\s*>/g; +// in hasUnclosedXmlTags (all TTS markup tags are letter-named). +// The body is a single unambiguous `[^>]*` (no adjacent overlapping quantifiers) +// so matching stays linear-time on untrusted input (avoids polynomial ReDoS); +// self-closing detection is done on the captured body instead of in the regex. +const XML_TAG_RE = /<(\/?)([A-Za-z][^>]*)>/g; + +/** Return true if a tag body captured by {@link XML_TAG_RE} marks a self-closing tag. */ +function isSelfClosingTagBody(body: string): boolean { + return body.trimEnd().endsWith('/'); +} /** Return true if `text` contains an incomplete or unclosed XML tag. */ export function hasUnclosedXmlTags(text: string): boolean { @@ -35,7 +43,7 @@ export function hasUnclosedXmlTags(text: string): boolean { let depth = 0; for (const m of text.matchAll(XML_TAG_RE)) { const isClosing = m[1] === '/'; - const isSelfClosing = m[3] === '/'; + const isSelfClosing = isSelfClosingTagBody(m[2]!); if (isSelfClosing) { continue; } else if (isClosing) { From cdae2c8265699e5e954eee76f7332537e35457cb Mon Sep 17 00:00:00 2001 From: Toubat Date: Wed, 8 Jul 2026 11:33:13 +0800 Subject: [PATCH 03/12] fix(tokenize): exclude '<' from XML tag body to fully avoid ReDoS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeQL still flagged the previous form: with [^>]* the body can absorb '<', so unterminated input like '] bounds each scan at the next '<'. Co-Authored-By: Oz --- agents/src/tokenize/token_stream.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/agents/src/tokenize/token_stream.ts b/agents/src/tokenize/token_stream.ts index 93ba3cdd3..bd79d2f61 100644 --- a/agents/src/tokenize/token_stream.ts +++ b/agents/src/tokenize/token_stream.ts @@ -10,10 +10,11 @@ type TokenizeFunc = (x: string) => string[] | [string, number, number][]; // the tag name must start with a letter so "<5>" / "<3 wins>" are not counted as // tags — this keeps the depth counter consistent with the letter-start tail check // in hasUnclosedXmlTags (all TTS markup tags are letter-named). -// The body is a single unambiguous `[^>]*` (no adjacent overlapping quantifiers) -// so matching stays linear-time on untrusted input (avoids polynomial ReDoS); -// self-closing detection is done on the captured body instead of in the regex. -const XML_TAG_RE = /<(\/?)([A-Za-z][^>]*)>/g; +// The body is a single `[^<>]*` — excluding both angle brackets keeps matching +// linear-time on untrusted LLM output (a body allowing "<" makes unterminated +// input like "]*)>/g; /** Return true if a tag body captured by {@link XML_TAG_RE} marks a self-closing tag. */ function isSelfClosingTagBody(body: string): boolean { From e1528fdd7eecb9f56ea6d74472fbdea934275d26 Mon Sep 17 00:00:00 2001 From: Toubat Date: Wed, 8 Jul 2026 11:43:31 +0800 Subject: [PATCH 04/12] fix(voice): export presets namespace from voice module The documented presets.CUSTOMER_SERVICE / presets.CASUAL API was defined but never re-exported, so consumers couldn't reach it (the package only exposes the root entrypoint). Export it as voice.presets and fix the doc example accordingly. --- agents/src/voice/index.ts | 1 + agents/src/voice/presets.ts | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/agents/src/voice/index.ts b/agents/src/voice/index.ts index 02872fc8a..630670e00 100644 --- a/agents/src/voice/index.ts +++ b/agents/src/voice/index.ts @@ -50,6 +50,7 @@ export { createTimedString, isTimedString, } from './io.js'; +export * as presets from './presets.js'; export * from './report.js'; export * from './room_io/index.js'; export { RunContext } from './run_context.js'; diff --git a/agents/src/voice/presets.ts b/agents/src/voice/presets.ts index 5ba0f6020..d55f2a6bf 100644 --- a/agents/src/voice/presets.ts +++ b/agents/src/voice/presets.ts @@ -9,9 +9,12 @@ * provider-agnostic at the call site: * * ```ts - * import { presets } from '@livekit/agents/voice'; + * import { voice } from '@livekit/agents'; * - * const session = new AgentSession({ tts: 'inworld/inworld-tts-2', expressive: presets.CASUAL }); + * const session = new AgentSession({ + * tts: 'inworld/inworld-tts-2', + * expressive: voice.presets.CASUAL, + * }); * ``` * * Each `presets.*` constant is just an {@link ExpressiveOptions} carrying a `preset`. From 18da263aefff09799bb2472bc0067de5d6d38b95 Mon Sep 17 00:00:00 2001 From: Toubat Date: Wed, 8 Jul 2026 12:06:54 +0800 Subject: [PATCH 05/12] fix(llm,voice): correct Instructions modality handling Three fixes from PR review (each also present in the Python source; JS deviates intentionally, verified against livekit/agents main): - instructionsEqual: an Instructions with audio/text additions no longer compares equal to its bare common string. It renders differently, so treating them as equal made realtime session reuse skip instruction updates on handoff when the new agent added modality rules. - Instructions.resolveTemplate: the produced audio/text variants are full template renders, not additions. render(modality) now returns the variant instead of appending it to common, which duplicated the entire template in the prompt. - pipeline reply: base Instructions are re-rendered for the turn modality even when extra per-turn instructions are supplied, so a text turn no longer keeps audio-rendered base rules copied from the session chat context. --- agents/src/llm/chat_context.test.ts | 43 +++++++++++++++++++++++++++++ agents/src/llm/chat_context.ts | 29 ++++++++++++++----- agents/src/voice/agent_activity.ts | 18 +++++++----- 3 files changed, 76 insertions(+), 14 deletions(-) diff --git a/agents/src/llm/chat_context.test.ts b/agents/src/llm/chat_context.test.ts index 3882591de..03973a1a6 100644 --- a/agents/src/llm/chat_context.test.ts +++ b/agents/src/llm/chat_context.test.ts @@ -14,6 +14,7 @@ import { type ImageContent, Instructions, ReadonlyChatContext, + instructionsEqual, isInstructions, renderInstructions, } from './chat_context.js'; @@ -1345,6 +1346,20 @@ describe('Instructions', () => { expect(instr.text).toBe('prefix common rules suffix'); }); + it('render() on resolveTemplate output picks the modality variant without duplicating the template', () => { + const rules = new Instructions('common rules', { + audio: 'audio rules', + text: 'text rules', + }); + const instr = Instructions.resolveTemplate('persona\n{rules}\nextra', { rules }); + + // the variants are full template renders — render(modality) must return the + // variant alone, not common + variant (which would repeat persona/extra twice) + expect(instr.render({ modality: 'audio' })).toBe('persona\naudio rules\nextra'); + expect(instr.render({ modality: 'text' })).toBe('persona\ntext rules\nextra'); + expect(instr.render()).toBe('persona\ncommon rules\nextra'); + }); + it('serializes with toJSON and omits undefined variants', () => { const both = new Instructions('common text', { audio: 'audio addition', @@ -1401,6 +1416,34 @@ describe('Instructions', () => { }); }); +describe('instructionsEqual', () => { + it('compares plain strings by value', () => { + expect(instructionsEqual('hello', 'hello')).toBe(true); + expect(instructionsEqual('hello', 'other')).toBe(false); + expect(instructionsEqual(undefined, 'hello')).toBe(false); + expect(instructionsEqual(undefined, undefined)).toBe(true); + }); + + it('compares Instructions by common/audio/text parts', () => { + const a = new Instructions('hello', { audio: 'a', text: 't' }); + const b = new Instructions('hello', { audio: 'a', text: 't' }); + const c = new Instructions('hello', { audio: 'other' }); + + expect(instructionsEqual(a, b)).toBe(true); + expect(instructionsEqual(a, c)).toBe(false); + }); + + it('equals a plain string only when there are no modality additions', () => { + expect(instructionsEqual(new Instructions('hello'), 'hello')).toBe(true); + expect(instructionsEqual('hello', new Instructions('hello'))).toBe(true); + + // an Instructions with additions renders differently — it must not compare + // equal to its bare common text (would skip realtime instruction updates) + expect(instructionsEqual(new Instructions('hello', { audio: 'extra' }), 'hello')).toBe(false); + expect(instructionsEqual('hello', new Instructions('hello', { text: 'extra' }))).toBe(false); + }); +}); + describe('ChatContext.copy with toolCtx filter', () => { it('drops function calls / outputs whose tool is not in the supplied ToolContext', () => { const known = tool({ name: 'known', description: 'k', execute: async () => 'ok' }); diff --git a/agents/src/llm/chat_context.ts b/agents/src/llm/chat_context.ts index 64e13850f..a757c6235 100644 --- a/agents/src/llm/chat_context.ts +++ b/agents/src/llm/chat_context.ts @@ -82,6 +82,14 @@ export class Instructions { /** @internal Symbol marker for type identification */ readonly [INSTRUCTIONS_SYMBOL] = true; + /** + * When true (set by {@link Instructions.resolveTemplate} when modality variants were + * produced), `audio`/`text` hold *full* renders of the template and replace `common` + * in {@link render} instead of being appended to it. Not serialized by `toJSON`. + * @internal + */ + private variantsReplaceCommon = false; + constructor(common: string = '', options?: { audio?: string; text?: string }) { this.common = common; this.audio = options?.audio; @@ -98,11 +106,13 @@ export class Instructions { render(options?: { modality?: 'audio' | 'text'; data?: Record }): string { const { modality, data } = options ?? {}; - const parts = [this.common]; + let parts = [this.common]; if (modality !== undefined) { const addition = modality === 'audio' ? this.audio : this.text; if (addition) { - parts.push(addition); + // resolveTemplate variants are full renders of the template, not additions — + // appending them to common would duplicate the whole template + parts = this.variantsReplaceCommon ? [addition] : [...parts, addition]; } } @@ -140,10 +150,14 @@ export class Instructions { textKw[k] = v; } } - return new Instructions(safeRender(template, commonKw), { + const resolved = new Instructions(safeRender(template, commonKw), { audio: safeRender(template, audioKw), text: safeRender(template, textKw), }); + // the variants are full template renders: render(modality) must pick one, + // not append it to the common render + resolved.variantsReplaceCommon = true; + return resolved; } return new Instructions(safeRender(template, kwargs)); } @@ -184,8 +198,9 @@ export function renderInstructions( * Compare two instruction values by content. Plain strings compare by value; * {@link Instructions} compare by their common/audio/text parts so that two * distinct instances with the same content are treated as equal. An - * {@link Instructions} with no modality additions equals the plain string - * matching its common text (mirrors Python's `__eq__`). + * {@link Instructions} equals a plain string only when it has no modality + * additions and its common text matches — an Instructions with additions + * renders differently, so it must not compare equal to its bare common text. */ export function instructionsEqual( a: string | Instructions | undefined, @@ -199,10 +214,10 @@ export function instructionsEqual( return a.common === b.common && a.audio === b.audio && a.text === b.text; } if (aIsInstr && !bIsInstr) { - return a.common === b; + return a.audio === undefined && a.text === undefined && a.common === b; } if (!aIsInstr && bIsInstr) { - return a === b.common; + return b.audio === undefined && b.text === undefined && a === b.common; } return a === b; } diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 1defb413f..78bd0d9c5 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -2781,13 +2781,10 @@ export class AgentActivity implements RecognitionHooks { // resolve modality-specific instructions for this turn const turnModality = speechHandle.inputDetails.modality; - if (instructions) { - // extra instructions are appended as a separate system message for this turn - chatCtx.addMessage({ - role: 'system', - content: [renderInstructions(instructions, turnModality)], - }); - } else if (isInstructions(this.agent.instructions)) { + // always re-render the base instructions for this turn's modality — even when + // extra per-turn instructions are supplied — so a text turn never keeps the + // audio-rendered base rules copied from the session chat context + if (isInstructions(this.agent.instructions)) { try { updateInstructions({ chatCtx, @@ -2799,6 +2796,13 @@ export class AgentActivity implements RecognitionHooks { this.logger.error({ error: e }, 'error occurred during updateInstructions'); } } + if (instructions) { + // extra instructions are appended as a separate system message for this turn + chatCtx.addMessage({ + role: 'system', + content: [renderInstructions(instructions, turnModality)], + }); + } // inject expressive instructions (TTS markup guide) const exprOpts = this._resolveExpressiveOptions(); From 621b592402a50eb22eb94bf244bb87d63a704807 Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Wed, 8 Jul 2026 08:47:35 -0400 Subject: [PATCH 06/12] feat(examples): update frontdesk agent for expressive mode Port frontdesk example changes from livekit/agents#6116: expressive CUSTOMER_SERVICE preset with inference gateway STT/LLM/TTS (inworld tts-2), rewritten voice-first instructions, time-of-day onEnter greeting via generateReply, and idle away-state nudge loop. --- examples/src/frontdesk/frontdesk_agent.ts | 119 +++++++++++++++++----- 1 file changed, 95 insertions(+), 24 deletions(-) diff --git a/examples/src/frontdesk/frontdesk_agent.ts b/examples/src/frontdesk/frontdesk_agent.ts index b7a7c9bff..94e05c834 100644 --- a/examples/src/frontdesk/frontdesk_agent.ts +++ b/examples/src/frontdesk/frontdesk_agent.ts @@ -1,10 +1,16 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import { type JobContext, ServerOptions, cli, defineAgent, llm, voice } from '@livekit/agents'; -import * as deepgram from '@livekit/agents-plugin-deepgram'; -import * as elevenlabs from '@livekit/agents-plugin-elevenlabs'; -import * as openai from '@livekit/agents-plugin-openai'; +import { + type JobContext, + ServerOptions, + cli, + defineAgent, + inference, + llm, + log, + voice, +} from '@livekit/agents'; import { BackgroundVoiceCancellation } from '@livekit/noise-cancellation-node'; import { fileURLToPath } from 'node:url'; import { z } from 'zod'; @@ -35,17 +41,35 @@ export class FrontDeskAgent extends voice.Agent { }); const instructions = - `You are Front-Desk, a helpful and efficient voice assistant. ` + - `Today is ${today}. Your main goal is to schedule an appointment for the user. ` + - `This is a voice conversation — speak naturally, clearly, and concisely. ` + - `When the user says hello or greets you, don't just respond with a greeting — use it as an opportunity to move things forward. ` + - `For example, follow up with a helpful question like: 'Would you like to book a time?' ` + - `When asked for availability, call list_available_slots and offer a few clear, simple options. ` + - `Say things like 'Monday at 2 PM' — avoid timezones, timestamps, and avoid saying 'AM' or 'PM'. ` + - `Use natural phrases like 'in the morning' or 'in the evening', and don't mention the year unless it's different from the current one. ` + - `Offer a few options at a time, pause for a response, then guide the user to confirm. ` + - `If the time is no longer available, let them know gently and offer the next options. ` + - `Always keep the conversation flowing — be proactive, human, and focused on helping the user schedule with ease.`; + // Outcome — what a great interaction looks like. + `You are Front-Desk, a helpful and efficient voice assistant. Today is ${today}. ` + + `A great interaction ends with the user booked into an appointment slot that works ` + + `for them, reached through a warm, flowing conversation with as little ` + + `back-and-forth as possible. ` + + // Voice & personality — keep it short and human. + `Your output is synthesized directly to speech, so produce a natural verbatim ` + + `transcript, not polished text. Start responses with real reactions (oh, hmm, ah) ` + + `and fillers (um, uh, like) rather than "Absolutely" or "Certainly", with ` + + `mid-sentence fillers (like, you know, I mean) where they'd naturally fall. Mirror ` + + `the user's formality: if they're casual, use informal phrasing (gotcha, alright, ` + + `gonna, kinda, lemme, yeah); if they're more formal, keep your speech cleaner. Vary ` + + `your openers across turns — if you opened the last turn with 'gotcha', pick ` + + `'alright' or 'okay' this turn; don't repeat the same opener back-to-back. ` + + // How to work — be proactive, acknowledge before acting, stop when you can move forward. + `Be proactive: when the user greets you, use it to move things forward (e.g. ` + + `'Would you like to book a time?') rather than just greeting back. Before a tool ` + + `call that takes a moment, give a brief spoken acknowledgment so there's no dead ` + + `air. After each result, check whether you can now move the user toward a booking: ` + + `if so, do it; if you're missing something, ask for just that. ` + + // Speaking about times — constraints that keep it natural over voice. + `When talking about availability, call list_available_slots and offer a few clear ` + + `options at a time, then pause for a response and guide the user to confirm. Say ` + + `times like 'Monday at 2' — avoid timezones, timestamps, and the words 'AM'/'PM'; ` + + `use natural phrases like 'in the morning' or 'in the evening', and don't mention ` + + `the year unless it differs from the current one. When listing several times in the ` + + `same window, group them ('in the evening at 4, 5, or 6') instead of repeating the ` + + `time-of-day qualifier on each slot. If a chosen time is no longer available, let ` + + `them know gently and offer the next options.`; super({ instructions, @@ -185,6 +209,24 @@ You must infer the appropriate range implicitly from the conversational context this.tz = options.timezone; } + + async onEnter(): Promise { + const hour = Number( + new Intl.DateTimeFormat('en-US', { + hour: 'numeric', + hour12: false, + timeZone: this.tz, + }).format(new Date()), + ); + const timeOfDay = hour < 12 ? 'morning' : hour < 17 ? 'afternoon' : 'evening'; + await this.session.generateReply({ + instructions: + `Say hello and welcome to the caller — it's currently ${timeOfDay} their time. ` + + `You're the front desk of an office and you're here to help them schedule a visit. ` + + `Invite them to book an appointment to visit, and ask what time works. ` + + `Keep it warm and brief.`, + }); + } } export default defineAgent({ @@ -209,15 +251,46 @@ export default defineAgent({ const userdata: Userdata = { cal }; const session = new voice.AgentSession({ - stt: new deepgram.STT(), - llm: new openai.LLM({ - model: 'gpt-4.1', + stt: new inference.STT({ model: 'deepgram/nova-3' }), + llm: new inference.LLM({ model: 'google/gemma-4-31b-it' }), + tts: new inference.TTS({ + model: 'inworld/inworld-tts-2', + voice: 'Nadia', + modelOptions: { delivery_mode: 'CREATIVE', speaking_rate: 1.1 }, }), - tts: new elevenlabs.TTS(), + expressive: voice.presets.CUSTOMER_SERVICE, userData: userdata, - voiceOptions: { - maxToolSteps: 1, - }, + maxToolSteps: 1, + // Flip userState to "away" after 10s of mutual silence so we can + // check whether they're still there (default is 15s). + userAwayTimeout: 10.0, + }); + + const logger = log(); + let idleNudge: AbortController | null = null; + + const nudgeWhileIdle = async (signal: AbortSignal) => { + // Nudge every 10s until the user speaks again — speaking flips + // userState out of "away", which aborts this loop below. + while (!signal.aborted) { + logger.info("user idle — checking if they're still there"); + await session.generateReply({ + instructions: "The user has been idle, see if they're still there", + }); + await new Promise((resolve) => setTimeout(resolve, 10_000)); + } + }; + + session.on(voice.AgentSessionEventTypes.UserStateChanged, (ev) => { + if (ev.newState === 'away') { + if (idleNudge === null) { + idleNudge = new AbortController(); + void nudgeWhileIdle(idleNudge.signal); + } + } else if (idleNudge !== null) { + idleNudge.abort(); + idleNudge = null; + } }); await session.start({ @@ -227,8 +300,6 @@ export default defineAgent({ noiseCancellation: BackgroundVoiceCancellation(), }, }); - - await session.say('Hello, I can help you schedule an appointment!'); }, }); From 141c868dbcae129d0917c2cf489aa39c54bb5995 Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Wed, 8 Jul 2026 08:52:12 -0400 Subject: [PATCH 07/12] fix(examples): use .ts extensions for relative imports Relative .js import specifiers fail when running examples directly under node type stripping (e.g. lk agent console). Switch to .ts specifiers with rewriteRelativeImportExtensions so tsc still emits .js in dist output. --- examples/src/drive-thru/drivethru_agent.ts | 4 ++-- examples/src/drive-thru/order.ts | 2 +- examples/src/drive-thru/test_agent.test.ts | 2 +- examples/src/frontdesk/calendar_api.test.ts | 2 +- examples/src/frontdesk/calendar_integration.test.ts | 2 +- examples/src/frontdesk/frontdesk_agent.ts | 2 +- examples/src/frontdesk/test_agent.test.ts | 4 ++-- examples/src/hotel-receptionist/hotel_receptionist.ts | 4 ++-- examples/src/testing/survey_agent.test.ts | 2 +- examples/tsconfig.json | 4 ++++ 10 files changed, 16 insertions(+), 12 deletions(-) diff --git a/examples/src/drive-thru/drivethru_agent.ts b/examples/src/drive-thru/drivethru_agent.ts index 838f5d104..ab280b33e 100644 --- a/examples/src/drive-thru/drivethru_agent.ts +++ b/examples/src/drive-thru/drivethru_agent.ts @@ -13,13 +13,13 @@ import { type MenuItem, findItemsById, menuInstructions, -} from './database.js'; +} from './database.ts'; import { OrderState, createOrderedCombo, createOrderedHappy, createOrderedRegular, -} from './order.js'; +} from './order.ts'; export interface UserData { order: OrderState; diff --git a/examples/src/drive-thru/order.ts b/examples/src/drive-thru/order.ts index 73c85b1ec..e23f82828 100644 --- a/examples/src/drive-thru/order.ts +++ b/examples/src/drive-thru/order.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import type { ItemSize } from './database.js'; +import type { ItemSize } from './database.ts'; export function orderUid(): string { const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; diff --git a/examples/src/drive-thru/test_agent.test.ts b/examples/src/drive-thru/test_agent.test.ts index 7eae74f8e..a19e7298e 100644 --- a/examples/src/drive-thru/test_agent.test.ts +++ b/examples/src/drive-thru/test_agent.test.ts @@ -20,7 +20,7 @@ import { initializeLogger, llm, voice } from '@livekit/agents'; import * as openai from '@livekit/agents-plugin-openai'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { DriveThruAgent, type UserData, newUserData } from './drivethru_agent.js'; +import { DriveThruAgent, type UserData, newUserData } from './drivethru_agent.ts'; initializeLogger({ pretty: false, level: 'warn' }); diff --git a/examples/src/frontdesk/calendar_api.test.ts b/examples/src/frontdesk/calendar_api.test.ts index 4e7a08fd2..2ae071f61 100644 --- a/examples/src/frontdesk/calendar_api.test.ts +++ b/examples/src/frontdesk/calendar_api.test.ts @@ -8,7 +8,7 @@ import { SlotUnavailableError, createAvailableSlot, getUniqueHash, -} from './calendar_api.js'; +} from './calendar_api.ts'; describe('Calendar API', () => { describe('createAvailableSlot', () => { diff --git a/examples/src/frontdesk/calendar_integration.test.ts b/examples/src/frontdesk/calendar_integration.test.ts index cc273eaba..692692314 100644 --- a/examples/src/frontdesk/calendar_integration.test.ts +++ b/examples/src/frontdesk/calendar_integration.test.ts @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from 'vitest'; -import { CalComCalendar } from './calendar_api.js'; +import { CalComCalendar } from './calendar_api.ts'; describe('Calendar Integration Tests', () => { describe('CalComCalendar with real API', () => { diff --git a/examples/src/frontdesk/frontdesk_agent.ts b/examples/src/frontdesk/frontdesk_agent.ts index 94e05c834..d2d199199 100644 --- a/examples/src/frontdesk/frontdesk_agent.ts +++ b/examples/src/frontdesk/frontdesk_agent.ts @@ -21,7 +21,7 @@ import { FakeCalendar, SlotUnavailableError, getUniqueHash, -} from './calendar_api.js'; +} from './calendar_api.ts'; export interface Userdata { cal: Calendar; diff --git a/examples/src/frontdesk/test_agent.test.ts b/examples/src/frontdesk/test_agent.test.ts index 18ca3b7ac..87d0245f6 100644 --- a/examples/src/frontdesk/test_agent.test.ts +++ b/examples/src/frontdesk/test_agent.test.ts @@ -19,8 +19,8 @@ import { initializeLogger, voice } from '@livekit/agents'; import * as openai from '@livekit/agents-plugin-openai'; import { afterAll, beforeAll, describe, it } from 'vitest'; -import { type AvailableSlot, FakeCalendar, createAvailableSlot } from './calendar_api.js'; -import { FrontDeskAgent, type Userdata } from './frontdesk_agent.js'; +import { type AvailableSlot, FakeCalendar, createAvailableSlot } from './calendar_api.ts'; +import { FrontDeskAgent, type Userdata } from './frontdesk_agent.ts'; initializeLogger({ pretty: false, level: 'warn' }); diff --git a/examples/src/hotel-receptionist/hotel_receptionist.ts b/examples/src/hotel-receptionist/hotel_receptionist.ts index 42e82efa7..12741fbda 100644 --- a/examples/src/hotel-receptionist/hotel_receptionist.ts +++ b/examples/src/hotel-receptionist/hotel_receptionist.ts @@ -31,8 +31,8 @@ import { speakCode, speakTime, speakUsd, -} from './hotel_db.js'; -import { buildLookupPolicyTool } from './policies.js'; +} from './hotel_db.ts'; +import { buildLookupPolicyTool } from './policies.ts'; type UserData = { db: HotelDB; diff --git a/examples/src/testing/survey_agent.test.ts b/examples/src/testing/survey_agent.test.ts index 748feb41e..36aad7748 100644 --- a/examples/src/testing/survey_agent.test.ts +++ b/examples/src/testing/survey_agent.test.ts @@ -26,7 +26,7 @@ import { ExperienceTask, IntroTask, type SurveyUserData, -} from '../survey_agent.js'; +} from '../survey_agent.ts'; const { TaskGroup } = workflows; type TaskGroupResult = workflows.TaskGroupResult; diff --git a/examples/tsconfig.json b/examples/tsconfig.json index 34ca2a33e..76b661f57 100644 --- a/examples/tsconfig.json +++ b/examples/tsconfig.json @@ -5,6 +5,10 @@ // them out of the package build so they can't break `pnpm build`. "exclude": ["./src/test_*.ts"], "compilerOptions": { + // relative imports use .ts extensions so examples can run directly under + // node's type stripping (e.g. `lk agent console`); tsc rewrites them to + // .js in the emitted output + "rewriteRelativeImportExtensions": true, // match output dir to input dir. e.g. dist/index instead of dist/src/index "rootDir": "./src", "declarationDir": "./dist", From 2fd5abbe2391b6ad249220c2584748cb45001bc4 Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Wed, 8 Jul 2026 14:32:58 -0400 Subject: [PATCH 08/12] feat(examples): update drive-thru agent for expressive mode Port drive-thru example changes from livekit/agents#6116: expressive CUSTOMER_SERVICE preset with inference gateway STT/LLM/TTS (inworld tts-2), restructured COMMON_INSTRUCTIONS (outcome / voice / how-to-work / hard constraints), and idle away-state nudge loop. --- examples/src/drive-thru/database.ts | 78 +++++++++++----------- examples/src/drive-thru/drivethru_agent.ts | 59 +++++++++++++--- 2 files changed, 89 insertions(+), 48 deletions(-) diff --git a/examples/src/drive-thru/database.ts b/examples/src/drive-thru/database.ts index e6813f993..2b13ce98e 100644 --- a/examples/src/drive-thru/database.ts +++ b/examples/src/drive-thru/database.ts @@ -2,44 +2,46 @@ // // SPDX-License-Identifier: Apache-2.0 -export const COMMON_INSTRUCTIONS = `You are Kelly, a quick and friendly McDonald's drive-thru attendant. -Your job is to guide the customer smoothly through their order, speaking in short, natural voice responses. -This is a voice interaction-assume the customer just pulled up and is speaking to you through a drive-thru speaker. -Respond like you're hearing them, not reading text. -Assume they want food, even if they don't start with a clear request, and help them get what they're looking for. - -If an item comes in different sizes, always ask for the size unless the customer already gave one. -If a customer orders a 'large meal', automatically assume both the fries and the drink should be large. -Do not ask again to confirm the size of the drink or fries. This inference is meant to streamline the interaction. -If the customer clearly indicates a different size for the fries or drink, respect their preference. - -Be fast-keep responses short and snappy. -Sound human-sprinkle in light vocal pauses like 'Mmh…', 'Let me see…', or 'Alright…' at natural moments-but not too often. -Keep everything upbeat and easy to follow. Never overwhelm the customer, don't ask multiple questions at the same time. - -When a customer is confused or asks for something that doesn't exist, let them know politely and suggest something close. -Always confirm what they picked in a warm, clear way, like: 'Alright, one Big Mac Combo!' -If something's unavailable, say so with empathy: 'Ah, we're out of Sweet Tea right now-can I get you a Coke instead?' - -Whenever a customer asks for, changes, or removes something from their order, you MUST use a tool to make it happen. -Don't fake it. Don't pretend something was added - actually **call** the tool and make it real on the ordering system. - -Transcripts often contain speech-to-text errors-don't mention the transcript, don't repeat its mistakes. -Instead treat each user input as a rough draft of what was said. -If you can guess the user's intent and it's safe to do so, infer their meaning and respond naturally. -If the transcript is ambiguous/nonsense and you can't guess their intent, ask the customer to repeat again. -Stay on-topic; if input is nonsensical in a drive-thru context, ask for concise clarification. - -Do not add any item on the user's behalf unless they specifically request it. If the user hasn't asked for an item, NEVER add it. - -When a customer changes an item or meal, make sure to remove the previous version before adding the new one. -Otherwise, the order may contain duplicates. - -Strictly stick to the defined menu, Do not invent or suggest any new sizes or items. -If the item specified by the user is unclear or not **exactly** on the menu, ask for clarification or say you don't have this specific item -E.g: an hamburger isn't a cheeseburger -Do not ask for size unless the item has more than one size option specified. -If an item does not require a size according to the menu, **NEVER** ask the customer to choose one or mention size at all.`; +export const COMMON_INSTRUCTIONS = + // Outcome — what a great interaction looks like. + `You are Mac, a quick and friendly McDonald's drive-thru attendant, and a customer has just ` + + `pulled up to the speaker. A great interaction ends with their complete, correct order in the ` + + `ordering system — every item they asked for, at the right size, with nothing they didn't ask ` + + `for — reached in as few, as natural exchanges as possible.\n\n` + + // Voice & personality — keep it short and human. + `Your output is synthesized directly to speech, so produce a natural verbatim transcript, not ` + + `polished text. Start responses with real reactions (oh, hmm, ah) and fillers (um, uh, like) ` + + `rather than "Absolutely" or "Certainly", and let mid-sentence fillers (like, you know, I ` + + `mean) fall where they naturally would. Use informal phrasing: yeah, gonna, kinda, gotcha, ` + + `lemme. Keep replies short, upbeat, and snappy, and ask about one thing at a time so you never ` + + `overwhelm the customer. Confirm choices warmly ('Alright, one Big Mac Combo!'), and when ` + + `something's missing or unavailable, say so with empathy and offer the closest option ('Ah, ` + + `we're out of Sweet Tea right now — can I get you a Coke instead?').\n\n` + + // How to work — infer intent, acknowledge before acting, stop when you have enough. + `Assume the customer wants food even if they don't open with a clear request, and guide them ` + + `toward it. Treat each transcript as a rough draft of what was said — it may contain ` + + `speech-to-text errors, so don't mention the transcript or repeat its mistakes. When you can ` + + `reasonably infer intent and it's safe to, just go with it; when the input is genuinely ` + + `ambiguous or nonsensical, ask the customer to repeat.\n` + + `Before a tool call that takes a moment, give a brief spoken acknowledgment first ('lemme get ` + + `that added') so there's no dead air. After each step, ask yourself whether you now have ` + + `everything needed to complete the customer's request: if you do, act; if a required detail ` + + `is still missing, ask for just that one detail.\n\n` + + // Hard constraints — these are invariants, not judgment calls. + `Constraints that always hold:\n` + + `- Stick strictly to the defined menu. Never invent items or sizes. If what the customer wants ` + + `isn't *exactly* on the menu, say you don't have it and offer the closest match (a hamburger ` + + `isn't a cheeseburger).\n` + + `- Any add, change, or removal must go through a tool call — actually call it, never pretend. ` + + `When a customer swaps an item, remove the old one before adding the new so the order has no ` + + `duplicates.\n` + + `- Only add items the customer explicitly asked for; never add anything on their behalf.\n` + + `- Don't assume unstated details — especially the drink in a combo. If a required detail is ` + + `missing, ask before calling the tool.\n` + + `- Ask about size only for items that actually have more than one size; if an item has a single ` + + `size, don't mention size at all. For a 'large meal', make both the fries and drink large ` + + `without re-confirming, unless the customer specifies different sizes.\n` + + `- If a tool returns an error, tell the customer and ask them to try again.`; export type ItemSize = 'S' | 'M' | 'L'; export type ItemCategory = 'drink' | 'combo_meal' | 'happy_meal' | 'regular' | 'sauce'; diff --git a/examples/src/drive-thru/drivethru_agent.ts b/examples/src/drive-thru/drivethru_agent.ts index ab280b33e..b5b82f9d5 100644 --- a/examples/src/drive-thru/drivethru_agent.ts +++ b/examples/src/drive-thru/drivethru_agent.ts @@ -1,10 +1,16 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import { type JobContext, ServerOptions, cli, defineAgent, llm, voice } from '@livekit/agents'; -import * as deepgram from '@livekit/agents-plugin-deepgram'; -import * as elevenlabs from '@livekit/agents-plugin-elevenlabs'; -import * as openai from '@livekit/agents-plugin-openai'; +import { + type JobContext, + ServerOptions, + cli, + defineAgent, + inference, + llm, + log, + voice, +} from '@livekit/agents'; import { fileURLToPath } from 'node:url'; import { z } from 'zod'; import { @@ -375,13 +381,46 @@ export default defineAgent({ const userdata = await newUserData(); const session = new voice.AgentSession({ - stt: new deepgram.STT(), - llm: new openai.LLM({ model: 'gpt-4.1', temperature: 0.45 }), - tts: new elevenlabs.TTS(), + stt: new inference.STT({ model: 'deepgram/nova-3' }), + llm: new inference.LLM({ model: 'google/gemma-4-31b-it' }), + tts: new inference.TTS({ + model: 'inworld/inworld-tts-2', + voice: 'Sarah', + modelOptions: { delivery_mode: 'CREATIVE', speaking_rate: 1.1 }, + }), + expressive: voice.presets.CUSTOMER_SERVICE, userData: userdata, - voiceOptions: { - maxToolSteps: 10, - }, + maxToolSteps: 10, + // Flip userState to "away" after 10s of mutual silence so we can + // check whether they're still there (default is 15s). + userAwayTimeout: 10.0, + }); + + const logger = log(); + let idleNudge: AbortController | null = null; + + const nudgeWhileIdle = async (signal: AbortSignal) => { + // Nudge every 10s until the user speaks again — speaking flips + // userState out of "away", which aborts this loop below. + while (!signal.aborted) { + logger.info("user idle — checking if they're still there"); + await session.generateReply({ + instructions: "The user has been idle, see if they're still there", + }); + await new Promise((resolve) => setTimeout(resolve, 10_000)); + } + }; + + session.on(voice.AgentSessionEventTypes.UserStateChanged, (ev) => { + if (ev.newState === 'away') { + if (idleNudge === null) { + idleNudge = new AbortController(); + void nudgeWhileIdle(idleNudge.signal); + } + } else if (idleNudge !== null) { + idleNudge.abort(); + idleNudge = null; + } }); await session.start({ From f5622add08f97309164dde8235205e318be473c4 Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Wed, 8 Jul 2026 14:33:06 -0400 Subject: [PATCH 09/12] feat(examples): update survey agent for expressive mode Port survey example changes from livekit/agents#6116: expressive CASUAL preset with inference gateway STT/LLM/TTS (inworld tts-2, multi-language STT), and idle away-state nudge loop. --- examples/src/survey_agent.ts | 43 +++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/examples/src/survey_agent.ts b/examples/src/survey_agent.ts index a09df7537..954337a29 100644 --- a/examples/src/survey_agent.ts +++ b/examples/src/survey_agent.ts @@ -6,7 +6,9 @@ import { ServerOptions, cli, defineAgent, + inference, llm, + log, voice, workflows, } from '@livekit/agents'; @@ -371,14 +373,49 @@ export class SurveyAgent extends voice.Agent { export default defineAgent({ entry: async (ctx: JobContext) => { const session = new voice.AgentSession({ - llm: 'openai/gpt-4.1', - stt: 'deepgram/nova-3', - tts: 'cartesia/sonic-3', + llm: new inference.LLM({ model: 'google/gemma-4-31b-it' }), + stt: new inference.STT({ model: 'deepgram/nova-3', language: 'multi' }), + tts: new inference.TTS({ + model: 'inworld/inworld-tts-2', + voice: 'Nate', + modelOptions: { delivery_mode: 'CREATIVE' }, + }), + expressive: voice.presets.CASUAL, userData: { filename: 'survey_results.csv', candidateName: '', taskResults: {}, }, + // Flip userState to "away" after 10s of mutual silence so we can + // check whether they're still there (default is 15s). + userAwayTimeout: 10.0, + }); + + const logger = log(); + let idleNudge: AbortController | null = null; + + const nudgeWhileIdle = async (signal: AbortSignal) => { + // Nudge every 10s until the user speaks again — speaking flips + // userState out of "away", which aborts this loop below. + while (!signal.aborted) { + logger.info("user idle — checking if they're still there"); + await session.generateReply({ + instructions: "The user has been idle, see if they're still there", + }); + await new Promise((resolve) => setTimeout(resolve, 10_000)); + } + }; + + session.on(voice.AgentSessionEventTypes.UserStateChanged, (ev) => { + if (ev.newState === 'away') { + if (idleNudge === null) { + idleNudge = new AbortController(); + void nudgeWhileIdle(idleNudge.signal); + } + } else if (idleNudge !== null) { + idleNudge.abort(); + idleNudge = null; + } }); await session.start({ From a6ca365f4d136708da51ebce771f2ff1a8cea288 Mon Sep 17 00:00:00 2001 From: Toubat Date: Thu, 9 Jul 2026 15:18:06 +0800 Subject: [PATCH 10/12] feat(examples): add inference playground agent with expressive mode Port of livekit/agents examples/inference/agent.py (post-#6116): inference gateway STT/LLM/TTS (inworld tts-2 CREATIVE), expressive CASUAL preset, on_enter greeting, idle away-state nudge loop, and the playground RPC controls (set_stt_model / set_llm_model / set_tts_model / set_system_prompt / open_in_builder). --- examples/src/inference_agent.ts | 207 ++++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 examples/src/inference_agent.ts diff --git a/examples/src/inference_agent.ts b/examples/src/inference_agent.ts new file mode 100644 index 000000000..03a00c39b --- /dev/null +++ b/examples/src/inference_agent.ts @@ -0,0 +1,207 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// LiveKit Playground agent on the inference gateway: the user prototypes their +// own voice agent, swapping the STT / LLM / TTS models and the system prompt +// live via RPC. Port of livekit/agents examples/inference/agent.py. +import { + type JobContext, + ServerOptions, + cli, + defineAgent, + inference, + log, + voice, +} from '@livekit/agents'; +import type { RpcInvocationData } from '@livekit/rtc-node'; +import { fileURLToPath } from 'node:url'; + +const DEFAULT_STT = 'deepgram/nova-3'; +const DEFAULT_LLM = 'google/gemma-4-31b-it'; +const DEFAULT_TTS = 'inworld/inworld-tts-2'; + +// Default starter prompt. Keep in sync with the `set_system_prompt` +// control's `default` in the playground config — the UI seeds the +// textarea with the same string so the first session before any edit +// matches what the user sees. +const INSTRUCTIONS = + "You're a friendly agent in the LiveKit Playground. The person " + + 'talking to you is prototyping their own voice agent — they can ' + + 'edit this prompt in the side panel and swap the STT / LLM / TTS ' + + 'models live. Keep replies short, natural, and conversational, and ' + + 'be expressive so they can hear what the selected voice can do. ' + + 'At the start of the conversation, set the tone and pace — open with ' + + 'warm, upbeat energy and a quick, inviting question to encourage the ' + + 'user to engage and let them know they can talk to you naturally. ' + + "If the conversation lulls or they're not sure what to try, offer " + + 'to tell them a short joke — and if they say yes, deliver it with ' + + "good comic timing. If asked which models you're using, answer honestly."; + +const swapPrompt = (modality: string, model: string) => + `The user just switched the ${modality} model to '${model}'. ` + + "Acknowledge it in one short, natural sentence — say the model's " + + "name like a brand (e.g. 'Deepgram Nova 3', not 'deepgram slash " + + "nova dash three'). Skip hyphens, slashes, version dots, and any " + + "abbreviations that aren't pronounceable. Don't ask a follow-up."; + +class InferenceAgent extends voice.Agent { + constructor(instructions: string = INSTRUCTIONS) { + super({ instructions }); + } + + async onEnter(): Promise { + // Fired once the agent is active and RoomIO has subscribed to the + // participant's tracks, so the greeting is delivered to a connected + // client rather than spoken before the audio socket is up. Runs on + // the session's default LLM (Gemma) — no model-routing needed here. + this.session.generateReply({ + instructions: + 'Greet the user with excitement, and ask them how their day is going. ' + + 'Keep it to one or two short, natural sentences.', + }); + } +} + +export default defineAgent({ + entry: async (ctx: JobContext) => { + const logger = log(); + + const session = new voice.AgentSession({ + stt: new inference.STT({ model: DEFAULT_STT }), + llm: new inference.LLM({ model: DEFAULT_LLM }), + tts: new inference.TTS({ + model: DEFAULT_TTS, + voice: 'Sarah', + modelOptions: { delivery_mode: 'CREATIVE' }, + }), + expressive: voice.presets.CASUAL, + // Flip userState to "away" after 10s of mutual silence so we can + // check whether they're still there (default is 15s). + userAwayTimeout: 10.0, + }); + + let idleNudge: AbortController | null = null; + + const nudgeWhileIdle = async (signal: AbortSignal) => { + // Nudge every 10s until the user speaks again — speaking flips + // userState out of "away", which aborts this loop below. + while (!signal.aborted) { + logger.info("user idle — checking if they're still there"); + await session.generateReply({ + instructions: "The user has been idle, see if they're still there", + }); + await new Promise((resolve) => setTimeout(resolve, 10_000)); + } + }; + + session.on(voice.AgentSessionEventTypes.UserStateChanged, (ev) => { + if (ev.newState === 'away') { + if (idleNudge === null) { + idleNudge = new AbortController(); + void nudgeWhileIdle(idleNudge.signal); + } + } else if (idleNudge !== null) { + idleNudge.abort(); + idleNudge = null; + } + }); + + const parseValue = (payload: string, fallback: string): string => { + try { + const v = (JSON.parse(payload) as Record).value; + return typeof v === 'string' && v ? v : fallback; + } catch { + return fallback; + } + }; + + const agent = new InferenceAgent(); + await session.start({ agent, room: ctx.room }); + + ctx.room.localParticipant?.registerRpcMethod( + 'set_stt_model', + async (data: RpcInvocationData) => { + const model = parseValue(data.payload, DEFAULT_STT); + const stt = session.stt; + if (!(stt instanceof inference.STT) || stt.model === model) { + return ''; + } + logger.info(`switching STT → ${model}`); + stt.updateOptions({ model }); + session.generateReply({ instructions: swapPrompt('speech-to-text', model) }); + return ''; + }, + ); + + ctx.room.localParticipant?.registerRpcMethod( + 'set_llm_model', + async (data: RpcInvocationData) => { + const model = parseValue(data.payload, DEFAULT_LLM); + const llm = session.llm; + if (!(llm instanceof inference.LLM) || llm.model === model) { + return ''; + } + logger.info(`switching LLM → ${model}`); + llm.updateOptions({ model }); + session.generateReply({ instructions: swapPrompt('language', model) }); + return ''; + }, + ); + + ctx.room.localParticipant?.registerRpcMethod( + 'set_tts_model', + async (data: RpcInvocationData) => { + const model = parseValue(data.payload, DEFAULT_TTS); + const tts = session.tts; + if (!(tts instanceof inference.TTS) || tts.model === model) { + return ''; + } + logger.info(`switching TTS → ${model}`); + tts.updateOptions({ model }); + session.generateReply({ instructions: swapPrompt('text-to-speech', model) }); + return ''; + }, + ); + + ctx.room.localParticipant?.registerRpcMethod('open_in_builder', async () => { + // Build the Cloud Builder deep-link agent-side so the + // frontend doesn't have to know the URL schema. `p_` is a + // placeholder project_id — Cloud routes the user through + // login if needed and preserves the params on redirect. + const params = new URLSearchParams({ + modelMode: 'pipeline', + instructions: agent.instructions.toString() || '', + llm: session.llm instanceof inference.LLM ? session.llm.model : DEFAULT_LLM, + stt: session.stt instanceof inference.STT ? session.stt.model : DEFAULT_STT, + tts: session.tts instanceof inference.TTS ? session.tts.model : DEFAULT_TTS, + }); + return `https://cloud.livekit.io/projects/p_/agents/builder/new?${params.toString()}`; + }); + + ctx.room.localParticipant?.registerRpcMethod( + 'set_system_prompt', + async (data: RpcInvocationData) => { + // The UI fires this on every keystroke (debounced client-side + // by the textarea's edit→commit boundary), so dedupe against + // the current value before touching the agent. updateInstructions + // is cheap but it logs. + const prompt = parseValue(data.payload, ''); + if (!prompt) { + return ''; + } + if (agent.instructions === prompt) { + return ''; + } + logger.info(`system prompt updated (${prompt.length} chars)`); + await agent.updateInstructions(prompt); + return ''; + }, + ); + }, +}); + +// Only run CLI when executed directly, not when imported for testing. +// eslint-disable-next-line turbo/no-undeclared-env-vars +if (process.env.VITEST === undefined) { + cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) })); +} From cf15251369dba0a018beef096f61ec803d4fd55e Mon Sep 17 00:00:00 2001 From: Brian Yin Date: Thu, 9 Jul 2026 22:59:20 +0800 Subject: [PATCH 11/12] feat(workflows): port built-in data-capture workflows from python (#2007) --- .changeset/port-builtin-workflows.md | 5 + agents/src/workflows/address.ts | 269 ++++++++ agents/src/workflows/credit_card.ts | 858 ++++++++++++++++++++++++++ agents/src/workflows/dob.ts | 406 ++++++++++++ agents/src/workflows/dtmf_inputs.ts | 286 +++++++++ agents/src/workflows/email_address.ts | 244 ++++++++ agents/src/workflows/index.ts | 48 +- agents/src/workflows/name.ts | 394 ++++++++++++ agents/src/workflows/phone_number.ts | 233 +++++++ agents/src/workflows/task_group.ts | 6 +- agents/src/workflows/utils.ts | 76 ++- 11 files changed, 2821 insertions(+), 4 deletions(-) create mode 100644 .changeset/port-builtin-workflows.md create mode 100644 agents/src/workflows/address.ts create mode 100644 agents/src/workflows/credit_card.ts create mode 100644 agents/src/workflows/dob.ts create mode 100644 agents/src/workflows/dtmf_inputs.ts create mode 100644 agents/src/workflows/email_address.ts create mode 100644 agents/src/workflows/name.ts create mode 100644 agents/src/workflows/phone_number.ts diff --git a/.changeset/port-builtin-workflows.md b/.changeset/port-builtin-workflows.md new file mode 100644 index 000000000..3b1a64503 --- /dev/null +++ b/.changeset/port-builtin-workflows.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Port the built-in data-capture workflows from python livekit-agents (`beta/workflows`) to the stable `workflows` namespace: `GetEmailTask`, `GetNameTask`, `GetPhoneNumberTask`, `GetDOBTask`, `GetAddressTask`, `GetDtmfTask`, and `GetCreditCardTask` (with `create*Task` functional variants). Includes the `DtmfEvent` enum with `formatDtmf`/`dtmfEventToCode` helpers, and `TaskGroup.add` now accepts factories for typed `AgentTask` results. diff --git a/agents/src/workflows/address.ts b/agents/src/workflows/address.ts new file mode 100644 index 000000000..5440771c8 --- /dev/null +++ b/agents/src/workflows/address.ts @@ -0,0 +1,269 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; +import { type InstructionParts, resolveWorkflowInstructions } from './utils.js'; + +export interface GetAddressResult { + address: string; +} + +export interface GetAddressTaskOptions { + /** + * Instructions for the address capture prompt. Pass a full string or {@link Instructions} to + * replace the built-in prompt entirely, or {@link InstructionParts} to override individual + * sections (e.g. `persona`) while keeping the built-in template. + */ + instructions?: InstructionParts | Instructions | string; + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode | null; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString | null; + vad?: VAD | null; + llm?: LLM | RealtimeModel | LLMModels | null; + tts?: TTS | TTSModelString | null; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm the captured address. Defaults to confirming on audio + * input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** + * When true, the model must produce an asking utterance before recording an address — it + * can't silently fill one from the chat context during `onEnter`. + */ + requireExplicitAsk?: boolean; + /** @deprecated use `instructions.extra` instead */ + extraInstructions?: string; +} + +interface UpdateAddressArgs { + street_address: string; + unit_number: string; + locality: string; + country: string; +} + +/** + * Build an {@link AgentTask} that collects a postal address from the user. + * + * This is the functional core; {@link GetAddressTask} is a thin class wrapper over it. + */ +export function createGetAddressTask({ + instructions, + chatCtx, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + requireExplicitAsk = false, + extraInstructions = '', +}: GetAddressTaskOptions = {}): AgentTask { + let currentAddress = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + // confirm tool is only injected after update_address is called, + // preventing the LLM from hallucinating a confirmation without user input + const buildConfirmTool = (address: string) => + tool({ + name: 'confirm_address', + description: 'Call after the user confirms the address is correct.', + execute: async () => { + if (address !== currentAddress) { + task.session.generateReply({ + instructions: + 'The address has changed since confirmation was requested, ask the user to confirm the updated address.', + }); + return; + } + + if (!task.done) { + task.complete({ address }); + } + }, + }); + + const updateAddressTool = tool({ + name: 'update_address', + description: 'Update the address provided by the user.', + parameters: z.object({ + street_address: z + .string() + .describe( + 'Dependent on country, may include fields like house number, street name, block, or district', + ), + unit_number: z + .string() + .describe( + "The unit number, for example Floor 1 or Apartment 12. If there is no unit number, return ''", + ), + locality: z + .string() + .describe('Dependent on country, may include fields like city, zip code, or province'), + country: z.string().describe('The country the user lives in spelled out fully'), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async (args: UpdateAddressArgs, { ctx }) => { + const { street_address: streetAddress, unit_number: unitNumber, locality, country } = args; + const addressFields = unitNumber.trim() + ? [streetAddress, unitNumber, locality, country] + : [streetAddress, locality, country]; + const address = addressFields.join(' '); + currentAddress = address; + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete({ address: currentAddress }); + } + return undefined; + } + + const confirmTool = buildConfirmTool(address); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_address'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + `The address has been updated to ${address}\n` + + `Repeat the address field by field: ${JSON.stringify(addressFields)} if needed\n` + + `Prompt the user for confirmation, do not call \`confirm_address\` directly` + ); + }, + }); + + const declineTool = tool({ + name: 'decline_address_capture', + description: 'Handles the case when the user explicitly declines to provide an address.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide the address'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }) => { + if (!task.done) { + task.complete(new ToolError(`couldn't get the address: ${reason}`)); + } + }, + }); + + const task = AgentTask.create({ + id: 'get_address_task', + instructions: resolveWorkflowInstructions({ + instructions, + extraInstructions, + template: INSTRUCTIONS_TEMPLATE, + defaultPersona: PERSONA, + kwargs: { + _modality_specific: new Instructions('', { + audio: AUDIO_SPECIFIC, + text: TEXT_SPECIFIC, + }), + _confirmation: new Instructions('', { + // confirmation is enabled by default for audio, disabled by default for text + audio: requireConfirmation !== false ? CONFIRMATION_INSTRUCTION : '', + text: requireConfirmation === true ? CONFIRMATION_INSTRUCTION : '', + }), + }, + }), + chatCtx, + turnDetection: turnDetection ?? undefined, + tools: [...(tools ?? []), updateAddressTool, declineTool], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, + onEnter: async () => { + task.session.generateReply({ instructions: 'Ask the user to provide their address.' }); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetAddressTask}, preserving the + * `new GetAddressTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetAddressTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetAddressTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetAddressTask. + super({ instructions: '' }); + this.#task = createGetAddressTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +// instructions +const PERSONA = + 'You are only a single step in a broader system, responsible solely for capturing an address.'; + +const AUDIO_SPECIFIC = `You will be handling addresses from any country. +Expect that users will say address in different formats with fields filled like: +- 'street_address': '450 SOUTH MAIN ST', 'unit_number': 'FLOOR 2', 'locality': 'SALT LAKE CITY UT 84101', 'country': 'UNITED STATES', +- 'street_address': '123 MAPLE STREET', 'unit_number': 'APARTMENT 10', 'locality': 'OTTAWA ON K1A 0B1', 'country': 'CANADA', +- 'street_address': 'GUOMAO JIE 3 HAO, CHAOYANG QU', 'unit_number': 'GUOMAO DA SHA 18 LOU 101 SHI', 'locality': 'BEIJING SHI 100000', 'country': 'CHINA', +- 'street_address': '5 RUE DE L’ANCIENNE COMÉDIE', 'unit_number': 'APP C4', 'locality': '75006 PARIS', 'country': 'FRANCE', +- 'street_address': 'PLOT 10, NEHRU ROAD', 'unit_number': 'OFFICE 403, 4TH FLOOR', 'locality': 'VILE PARLE (E), MUMBAI MAHARASHTRA 400099', 'country': 'INDIA', +Normalize common spoken patterns silently: +- Convert words like 'dash' and 'apostrophe' into symbols: \`-\`, \`'\`. +- Convert spelled out numbers like 'six' and 'seven' into numerals: \`6\`, \`7\`. +- Recognize patterns where users speak their address field followed by spelling: e.g., 'guomao g u o m a o'. +- Filter out filler words or hesitations. +- Recognize when there may be accents on certain letters if explicitly said or common in the location specified. Be sure to verify the correct accents if existent. +Don't mention corrections. Treat inputs as possibly imperfect but fix them silently. +When reading a numerical ordinal suffix (st, nd, rd, th), the number must be verbally expanded into its full, correctly pronounced word form. +Do not read the number and the suffix letters separately. +Confirm postal codes by reading them out digit-by-digit as a sequence of single numbers. Do not read them as cardinal numbers. +For example, read 90210 as 'nine zero two one zero.' +Avoid using bullet points and parenthese in any responses. +Spell out the address letter-by-letter when applicable, such as street names and provinces, especially when the user spells it out initially.`; + +const TEXT_SPECIFIC = `You will be handling addresses from any country. +Expect users to type their address directly. +If the address looks almost correct but has minor issues (e.g. missing country or postal code), prompt for clarification.`; + +const CONFIRMATION_INSTRUCTION = `Call \`confirm_address\` after the user confirmed the address is correct.`; + +const INSTRUCTIONS_TEMPLATE = `{persona} + +{_modality_specific} + +Call \`update_address\` at the first opportunity whenever you form a new hypothesis about the address. (before asking any questions or providing any answers.) +Don't invent new addresses, stick strictly to what the user said. +{_confirmation} +If the address is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts in this order: street address, unit number if applicable, locality, and country. + +Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. +Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called. + +{extra} +`; diff --git a/agents/src/workflows/credit_card.ts b/agents/src/workflows/credit_card.ts new file mode 100644 index 000000000..1c54125d5 --- /dev/null +++ b/agents/src/workflows/credit_card.ts @@ -0,0 +1,858 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, LLM, ToolError, ToolFlag, tool } from '../llm/index.js'; +import { isToolError } from '../llm/tool_context.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import { safeRender } from '../utils.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; +import { type GetNameResult, createGetNameTask } from './name.js'; +import { TaskGroup } from './task_group.js'; + +const CARD_ISSUERS_LOOKUP: Record = { + '3': 'American Express', + '4': 'Visa', + '5': 'Mastercard', + '6': 'Discover', +}; + +export interface GetCreditCardResult { + cardholderName: string; + issuer: string; + cardNumber: string; + securityCode: string; + expirationDate: string; +} + +interface GetCardNumberResult { + issuer: string; + cardNumber: string; +} + +interface GetSecurityCodeResult { + securityCode: string; +} + +interface GetExpirationDateResult { + date: string; +} + +export class CardCaptureDeclinedError extends ToolError { + readonly reason: string; + + constructor(reason: string) { + super(`couldn't get the card details: ${reason}`); + this.reason = reason; + } +} + +export class CardCollectionRestartError extends ToolError { + readonly reason: string; + + constructor(reason: string) { + super(`starting over: ${reason}`); + this.reason = reason; + } +} + +const declineCardCaptureTool = tool({ + name: 'decline_card_capture', + description: + 'Handles the case when the user explicitly declines to provide a detail for their card information.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide card information'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }, { ctx }) => { + const task = ctx.session.currentAgent; + if (task instanceof AgentTask && !task.done) { + task.complete(new CardCaptureDeclinedError(reason)); + } + }, +}); + +const restartCardCollectionTool = tool({ + name: 'restart_card_collection', + description: + 'Handles the case when the user wishes to start over the card information collection process and validate a new card.', + parameters: z.object({ + reason: z.string().describe('A short explanation of why the user wishes to start over'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }, { ctx }) => { + const task = ctx.session.currentAgent; + if (task instanceof AgentTask && !task.done) { + task.complete(new CardCollectionRestartError(reason)); + } + }, +}); + +/** Validates a card number via the Luhn algorithm. */ +function validateCardNumber(cardNumber: string): boolean { + if (!cardNumber || !/^\d+$/.test(cardNumber)) { + return false; + } + let totalSum = 0; + + const reversedNumber = [...cardNumber].reverse(); + for (let index = 0; index < reversedNumber.length; index++) { + const digit = Number(reversedNumber[index]); + if (index % 2 === 1) { + const doubledDigit = digit * 2; + totalSum += doubledDigit > 9 ? doubledDigit - 9 : doubledDigit; + } else { + totalSum += digit; + } + } + + return totalSum % 10 === 0; +} + +interface SubTaskOptions { + chatCtx?: ChatContext; + requireConfirmation?: boolean; + requireExplicitAsk?: boolean; + extraInstructions?: string; + turnDetection?: TurnDetectionMode | null; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString | null; + vad?: VAD | null; + llm?: LLM | RealtimeModel | LLMModels | null; + tts?: TTS | TTSModelString | null; + allowInterruptions?: boolean; +} + +function buildSubTaskInstructions( + baseInstructions: string, + audioSpecific: string, + textSpecific: string, + confirmationInstructions: string, + requireConfirmation: boolean | undefined, + extraInstructions: string, +): Instructions { + const extraSuffix = extraInstructions ? `\n${extraInstructions}` : ''; + return new Instructions('', { + audio: + safeRender(baseInstructions, { + modality_specific: audioSpecific, + confirmation_instructions: requireConfirmation !== false ? confirmationInstructions : '', + }) + extraSuffix, + text: + safeRender(baseInstructions, { + modality_specific: textSpecific, + confirmation_instructions: requireConfirmation === true ? confirmationInstructions : '', + }) + extraSuffix, + }); +} + +function createGetCardNumberTask({ + chatCtx, + requireConfirmation, + requireExplicitAsk = false, + extraInstructions = '', + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, +}: SubTaskOptions = {}): AgentTask { + let currentCardNumber = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildConfirmTool = (cardNumber: string) => + tool({ + name: 'confirm_card_number', + description: 'Call after the user repeats their card number for confirmation.', + parameters: z.object({ + repeated_card_number: z + .string() + .describe('The card number repeated by the user as a string'), + }), + execute: async ({ repeated_card_number: repeatedCardNumber }) => { + repeatedCardNumber = repeatedCardNumber.replace(/\D/g, ''); + if (repeatedCardNumber !== cardNumber) { + task.session.generateReply({ + instructions: 'The repeated card number does not match, ask the user to try again.', + }); + return; + } + + if (!validateCardNumber(cardNumber)) { + task.session.generateReply({ + instructions: + 'The card number is not valid, ask the user if they made a mistake or to provide another card.', + }); + } else { + const issuer = CARD_ISSUERS_LOOKUP[cardNumber[0]!] ?? 'Other'; + if (!task.done) { + task.complete({ issuer, cardNumber }); + } + } + }, + }); + + const updateCardNumberTool = tool({ + name: 'update_card_number', + description: + "Call to record the user's card number. Only call once the entire number has been given, do not call in increments.", + parameters: z.object({ + card_number: z + .string() + .describe('The credit card number as a string with no dashes or spaces'), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ({ card_number: rawCardNumber }: { card_number: string }, { ctx }) => { + const cardNumber = rawCardNumber.replace(/\D/g, ''); + if (cardNumber.length < 13 || cardNumber.length > 19) { + task.session.generateReply({ + instructions: + 'The length of the card number is invalid, ask the user to repeat their card number.', + }); + return undefined; + } + + currentCardNumber = cardNumber; + + if (!confirmationRequired(ctx)) { + if (!validateCardNumber(currentCardNumber)) { + task.session.generateReply({ + instructions: + 'The card number is not valid, ask the user if they made a mistake or to provide another card.', + }); + } else { + const issuer = CARD_ISSUERS_LOOKUP[currentCardNumber[0]!] ?? 'Other'; + if (!task.done) { + task.complete({ issuer, cardNumber: currentCardNumber }); + } + } + return undefined; + } + + const confirmTool = buildConfirmTool(cardNumber); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_card_number'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + 'The card number has been updated.\n' + + 'Ask them to repeat the number, do not repeat the number back to them.\n' + ); + }, + }); + + const task = AgentTask.create({ + id: 'get_card_number_task', + instructions: buildSubTaskInstructions( + CARD_NUMBER_BASE_INSTRUCTIONS, + CARD_NUMBER_AUDIO_SPECIFIC, + CARD_NUMBER_TEXT_SPECIFIC, + 'Call `confirm_card_number` once the user has repeated their card number.', + requireConfirmation, + extraInstructions, + ), + chatCtx, + turnDetection: turnDetection ?? undefined, + tools: [ + ...(tools ?? []), + updateCardNumberTool, + declineCardCaptureTool, + restartCardCollectionTool, + ], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, + onEnter: async () => { + await task.session.generateReply({ + instructions: + "Get the user's credit card number. First scan the conversation - if a " + + 'credit card number was already given (e.g. the user volunteered it ' + + 'before the task started), use it via update_card_number rather than ' + + 're-asking. Only ask fresh when no credit card number is in the ' + + 'conversation yet.', + }); + }, + }); + + return task; +} + +function createGetSecurityCodeTask({ + chatCtx, + requireConfirmation, + requireExplicitAsk = false, + extraInstructions = '', + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, +}: SubTaskOptions = {}): AgentTask { + let currentSecurityCode = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildConfirmTool = (securityCode: string) => + tool({ + name: 'confirm_security_code', + description: 'Call after the user repeats their security code for confirmation.', + parameters: z.object({ + repeated_security_code: z.string().describe('The security code repeated by the user'), + }), + execute: async ({ repeated_security_code: repeatedSecurityCode }) => { + if (repeatedSecurityCode.trim() !== securityCode) { + task.session.generateReply({ + instructions: 'The repeated security code does not match, ask the user to try again.', + }); + return; + } + + if (!task.done) { + task.complete({ securityCode }); + } + }, + }); + + const updateSecurityCodeTool = tool({ + name: 'update_security_code', + description: "Call to update the card's security code.", + parameters: z.object({ + security_code: z + .string() + .describe("The card's security code (3-4 digits, may have leading zeros)."), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ({ security_code: securityCode }: { security_code: string }, { ctx }) => { + const stripped = securityCode.trim(); + if (!/^\d+$/.test(stripped) || stripped.length < 3 || stripped.length > 4) { + task.session.generateReply({ + instructions: + "The security code's length is invalid, ask the user to repeat or to provide a new card and start over.", + }); + return undefined; + } + + currentSecurityCode = stripped; + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete({ securityCode: currentSecurityCode }); + } + return undefined; + } + + const confirmTool = buildConfirmTool(stripped); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_security_code'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + 'The security code has been updated.\n' + + 'Do not repeat the security code back to the user, ask them to repeat themselves.\n' + + 'Call `confirm_security_code` once the user confirms, do not call it preemptively.\n' + ); + }, + }); + + const task = AgentTask.create({ + id: 'get_security_code_task', + instructions: buildSubTaskInstructions( + SECURITY_CODE_BASE_INSTRUCTIONS, + SECURITY_CODE_AUDIO_SPECIFIC, + SECURITY_CODE_TEXT_SPECIFIC, + 'Call `confirm_security_code` once the user has repeated their security code.', + requireConfirmation, + extraInstructions, + ), + chatCtx, + turnDetection: turnDetection ?? undefined, + tools: [ + ...(tools ?? []), + updateSecurityCodeTool, + declineCardCaptureTool, + restartCardCollectionTool, + ], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, + onEnter: async () => { + await task.session.generateReply({ + instructions: + "Get the user's card security code. First scan the conversation - if a " + + 'code was already given, use it via update_security_code rather than ' + + 're-asking. Only ask fresh when no code is in the conversation yet.', + }); + }, + }); + + return task; +} + +function createGetExpirationDateTask({ + chatCtx, + requireConfirmation, + requireExplicitAsk = false, + extraInstructions = '', + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, +}: SubTaskOptions = {}): AgentTask { + let currentExpirationDate = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const isExpired = (month: number, year: number): boolean => { + const today = new Date(); + const fullYear = 2000 + year; + return ( + fullYear < today.getFullYear() || + (fullYear === today.getFullYear() && month < today.getMonth() + 1) + ); + }; + + const buildConfirmTool = (expirationMonth: number, expirationYear: number) => { + const expirationDate = currentExpirationDate; + + return tool({ + name: 'confirm_expiration_date', + description: 'Call after the user repeats their expiration date for confirmation.', + parameters: z.object({ + repeated_expiration_month: z + .number() + .int() + .describe('The expiration month repeated by the user'), + repeated_expiration_year: z + .number() + .int() + .describe('The expiration year repeated by the user'), + }), + execute: async ({ repeated_expiration_month, repeated_expiration_year }) => { + if ( + repeated_expiration_month !== expirationMonth || + repeated_expiration_year !== expirationYear + ) { + task.session.generateReply({ + instructions: 'The repeated expiration date does not match, ask the user to try again.', + }); + return; + } + + if (!task.done) { + task.complete({ date: expirationDate }); + } + }, + }); + }; + + const updateExpirationDateTool = tool({ + name: 'update_expiration_date', + description: + "Call to update the card's expiration date. Collect both the numerical month and year.", + parameters: z.object({ + expiration_month: z + .number() + .int() + .describe("The numerical expiration month of the card, example: '04' for April"), + expiration_year: z + .number() + .int() + .describe( + "The numerical expiration year of the card shortened to the last two digits, for example, '35' for 2035", + ), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ( + { + expiration_month: expirationMonth, + expiration_year: expirationYear, + }: { expiration_month: number; expiration_year: number }, + { ctx }, + ) => { + if (expirationMonth < 1 || expirationMonth > 12) { + task.session.generateReply({ + instructions: + 'The expiration month is invalid, ask the user to repeat the expiration month.', + }); + return undefined; + } + if (expirationYear < 0 || expirationYear > 99) { + task.session.generateReply({ + instructions: + 'The expiration year is invalid, ask the user to repeat the expiration year.', + }); + return undefined; + } + if (isExpired(expirationMonth, expirationYear)) { + task.session.generateReply({ + instructions: + 'The expiration date is in the past, the card is expired. Ask the user to provide another card.', + }); + return undefined; + } + + currentExpirationDate = `${String(expirationMonth).padStart(2, '0')}/${String(expirationYear).padStart(2, '0')}`; + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete({ date: currentExpirationDate }); + } + return undefined; + } + + const confirmTool = buildConfirmTool(expirationMonth, expirationYear); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_expiration_date'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + 'The expiration date has been updated.\n' + + 'Do not repeat the expiration date back to the user, ask them to repeat themselves.\n' + + 'Call `confirm_expiration_date` once the user confirms, do not call it preemptively.\n' + ); + }, + }); + + const task = AgentTask.create({ + id: 'get_expiration_date_task', + instructions: buildSubTaskInstructions( + EXPIRATION_DATE_BASE_INSTRUCTIONS, + EXPIRATION_DATE_AUDIO_SPECIFIC, + EXPIRATION_DATE_TEXT_SPECIFIC, + 'Call `confirm_expiration_date` once the user has repeated their expiration date.', + requireConfirmation, + extraInstructions, + ), + chatCtx, + turnDetection: turnDetection ?? undefined, + tools: [ + ...(tools ?? []), + updateExpirationDateTool, + declineCardCaptureTool, + restartCardCollectionTool, + ], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, + onEnter: async () => { + await task.session.generateReply({ + instructions: + "Get the user's card expiration date. First scan the conversation - if " + + 'an expiration date was already given, use it via update_expiration_date ' + + 'rather than re-asking. Only ask fresh when no date is in the conversation yet.', + }); + }, + }); + + return task; +} + +export interface GetCreditCardTaskOptions { + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode | null; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString | null; + vad?: VAD | null; + llm?: LLM | RealtimeModel | LLMModels | null; + tts?: TTS | TTSModelString | null; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm each captured card detail. Defaults to confirming on + * audio input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** Extra instructions appended to each sub-task's prompt for domain-specific context. */ + extraInstructions?: string; +} + +/** + * Build an {@link AgentTask} that collects the user's full credit card details (number, + * expiration date, security code, and cardholder name) via a {@link TaskGroup} of sub-tasks. + * + * This is the functional core; {@link GetCreditCardTask} is a thin class wrapper over it. + */ +export function createGetCreditCardTask({ + chatCtx, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + extraInstructions = '', +}: GetCreditCardTaskOptions = {}): AgentTask { + const task = AgentTask.create({ + id: 'get_credit_card_task', + instructions: '*none*', + chatCtx, + turnDetection: turnDetection ?? undefined, + tools: tools ?? [], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, + onEnter: async () => { + // Pass chatCtx into both the TaskGroup AND every sub-task. The + // TaskGroup overwrites each sub-task's chatCtx with its own (see + // TaskGroup.onEnter) - without seeding the TaskGroup, sub-tasks + // would run with empty context. + const ctx = task.chatCtx; + // Role hint for the cardholder sub-task. With IGNORE_ON_ENTER on + // update_name (via requireExplicitAsk=true), the model is + // structurally forced to ask before recording. The extra text + // just makes sure the *question* anchors to the card. + let cardholderExtra = + 'You are collecting the name on the credit card (the cardholder). ' + + 'When you ask the user to confirm a candidate name from earlier in ' + + 'the conversation, anchor the question to the card or cardholder ' + + "so the user knows which name you mean - not just 'is it [name]?' " + + 'in the abstract.'; + if (extraInstructions) { + cardholderExtra = `${extraInstructions}\n\n${cardholderExtra}`; + } + + // Task-level model/tool overrides must reach the sub-tasks: the outer + // placeholder never converses, the sub-tasks do. + const subTaskOptions = { + requireConfirmation, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + }; + + while (!task.done) { + // Order: number first (most natural for the caller to give + // when asked for "card details"), then expiry and security + // code, then the cardholder name LAST. The name most often + // pre-fills from chatCtx (same as the booking name) so + // leaving it for the end avoids the failure mode where the + // caller's first response (typically the digits) gets crammed + // into update_name. + const taskGroup = new TaskGroup({ + chatCtx: ctx, + // TaskGroup summarization requires a standard LLM on the session; + // skip it on realtime-model sessions instead of failing after all + // card details were collected. + summarizeChatCtx: task.session.llm instanceof LLM, + }); + taskGroup.add( + () => + createGetCardNumberTask({ + ...subTaskOptions, + chatCtx: ctx, + extraInstructions, + }), + { + id: 'card_number_task', + description: "Collects the user's card number", + }, + ); + taskGroup.add( + () => + createGetExpirationDateTask({ + ...subTaskOptions, + chatCtx: ctx, + extraInstructions, + }), + { + id: 'expiration_date_task', + description: "Collects the card's expiration date", + }, + ); + taskGroup.add( + () => + createGetSecurityCodeTask({ + ...subTaskOptions, + chatCtx: ctx, + extraInstructions, + }), + { + id: 'security_code_task', + description: "Collects the card's security code", + }, + ); + taskGroup.add( + () => + createGetNameTask({ + ...subTaskOptions, + lastName: true, + chatCtx: ctx, + extraInstructions: cardholderExtra, + // The cardholder may differ from the caller or any guest + // mentioned earlier in chatCtx. Apply IGNORE_ON_ENTER on + // update_name so the model must produce an asking turn + // rather than silently filling from chatCtx. + requireExplicitAsk: true, + }), + { + id: 'cardholder_name_task', + description: "Collects the cardholder's full name", + }, + ); + + try { + const results = await taskGroup.run(); + const nameResult = results.taskResults['cardholder_name_task'] as GetNameResult; + const cardNumberResult = results.taskResults['card_number_task'] as GetCardNumberResult; + const securityCodeResult = results.taskResults[ + 'security_code_task' + ] as GetSecurityCodeResult; + const expirationDateResult = results.taskResults[ + 'expiration_date_task' + ] as GetExpirationDateResult; + + task.complete({ + cardholderName: `${nameResult.firstName} ${nameResult.lastName}`, + issuer: cardNumberResult.issuer, + cardNumber: cardNumberResult.cardNumber, + securityCode: securityCodeResult.securityCode, + expirationDate: expirationDateResult.date, + }); + } catch (e) { + if (e instanceof CardCollectionRestartError) { + continue; + } + if (e instanceof CardCaptureDeclinedError || isToolError(e)) { + if (!task.done) { + task.complete(e as ToolError); + } + return; + } + throw e; + } + } + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetCreditCardTask}, preserving the + * `new GetCreditCardTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetCreditCardTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetCreditCardTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetCreditCardTask. + super({ instructions: '' }); + this.#task = createGetCreditCardTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +const CARD_NUMBER_BASE_INSTRUCTIONS = ` +You are a single step in a broader process of collecting credit card information. +You are solely responsible for collecting the credit card number. +{modality_specific} +If the user refuses to provide a credit card number, call decline_card_capture(). +If the user wishes to start over the credit card collection process, call restart_card_collection(). +Avoid listing out questions with bullet points or numbers, use a natural conversational tone. +Never repeat any sensitive information, such as the user's credit card number, back to the user. +{confirmation_instructions} +`; + +const CARD_NUMBER_AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect users to read the card number digit by digit. +Normalize spoken digits silently: 'four' → 4, 'zero' / 'oh' → 0. +Filter out filler words or hesitations. +`; + +const CARD_NUMBER_TEXT_SPECIFIC = ` +Handle input as typed text. Users may type the number with or without spaces or dashes (e.g. '4152 6374 8901 2345'). +`; + +const SECURITY_CODE_BASE_INSTRUCTIONS = ` +You are a single step in a broader process of collecting credit card information. +You are solely responsible for collecting the user's card's security code. +{modality_specific} +If the user refuses to provide a code, call decline_card_capture(). +If the user wishes to start over the card collection process, call restart_card_collection(). +Avoid listing out questions with bullet points or numbers, use a natural conversational tone. +Never repeat any sensitive information, such as the user's security code, back to the user. +{confirmation_instructions} +`; + +const SECURITY_CODE_AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect users to read the security code digit by digit. +Normalize spoken digits silently: 'four' → 4, 'zero' / 'oh' → 0. +Filter out filler words or hesitations. +`; + +const SECURITY_CODE_TEXT_SPECIFIC = ` +Handle input as typed text. Users will type the security code directly. +`; + +const EXPIRATION_DATE_BASE_INSTRUCTIONS = ` +You are a single step in a broader process of collecting credit card information. +You are solely responsible for collecting the user's card's expiration date. +{modality_specific} +If the user refuses to provide a date, call decline_card_capture(). +If the user wishes to start over the card collection process, call restart_card_collection(). +Avoid listing out questions with bullet points or numbers, use a natural conversational tone. +Never repeat any sensitive information, such as the user's expiration date, back to the user. +{confirmation_instructions} +`; + +const EXPIRATION_DATE_AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect users to say the expiration date in formats like 'April twenty five', 'oh four twenty five', 'four slash twenty five', or 'April 2025'. +Normalize spoken months and digits silently. +Filter out filler words or hesitations. +`; + +const EXPIRATION_DATE_TEXT_SPECIFIC = ` +Handle input as typed text. Expect users to type the expiration date in formats like '04/25', '04/2025', or 'April 2025'. +`; diff --git a/agents/src/workflows/dob.ts b/agents/src/workflows/dob.ts new file mode 100644 index 000000000..8b4a4a2f4 --- /dev/null +++ b/agents/src/workflows/dob.ts @@ -0,0 +1,406 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import { safeRender } from '../utils.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; + +/** A calendar date without a timezone. `month` and `day` are 1-based. */ +export interface DateOfBirth { + year: number; + month: number; + day: number; +} + +/** A time of day. `hour` is 0-23 and `minute` is 0-59. */ +export interface TimeOfBirth { + hour: number; + minute: number; +} + +export interface GetDOBResult { + dateOfBirth: DateOfBirth; + timeOfBirth?: TimeOfBirth; +} + +export interface GetDOBTaskOptions { + /** Extra instructions appended to the built-in prompt for domain-specific context. */ + extraInstructions?: string; + /** Also capture the (optional) time of birth. Defaults to false. */ + includeTime?: boolean; + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode | null; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString | null; + vad?: VAD | null; + llm?: LLM | RealtimeModel | LLMModels | null; + tts?: TTS | TTSModelString | null; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm the captured date of birth. Defaults to confirming on + * audio input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** + * When true, the model must produce an asking utterance before recording a date of birth — + * it can't silently fill one from the chat context during `onEnter`. + */ + requireExplicitAsk?: boolean; +} + +const MONTH_NAMES = [ + 'January', + 'February', + 'March', + 'April', + 'May', + 'June', + 'July', + 'August', + 'September', + 'October', + 'November', + 'December', +] as const; + +function formatDate(dob: DateOfBirth): string { + return `${MONTH_NAMES[dob.month - 1]} ${String(dob.day).padStart(2, '0')}, ${dob.year}`; +} + +function formatTime(time: TimeOfBirth): string { + const hour12 = time.hour % 12 === 0 ? 12 : time.hour % 12; + const period = time.hour < 12 ? 'AM' : 'PM'; + return `${String(hour12).padStart(2, '0')}:${String(time.minute).padStart(2, '0')} ${period}`; +} + +function isSameDate(a: DateOfBirth | undefined, b: DateOfBirth | undefined): boolean { + if (a === undefined || b === undefined) { + return a === b; + } + return a.year === b.year && a.month === b.month && a.day === b.day; +} + +function isSameTime(a: TimeOfBirth | undefined, b: TimeOfBirth | undefined): boolean { + if (a === undefined || b === undefined) { + return a === b; + } + return a.hour === b.hour && a.minute === b.minute; +} + +/** + * Build an {@link AgentTask} that collects a date of birth (and optionally a time of birth) + * from the user. + * + * This is the functional core; {@link GetDOBTask} is a thin class wrapper over it. + */ +export function createGetDOBTask({ + extraInstructions = '', + includeTime = false, + chatCtx, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + requireExplicitAsk = false, +}: GetDOBTaskOptions = {}): AgentTask { + const timeInstructions = !includeTime + ? '' + : 'Also ask for and capture the time of birth if the user knows it. ' + + "The time is optional - if the user doesn't know it, proceed without it.\n"; + const confirmationInstructions = + 'Call `confirm_dob` after the user confirmed the date of birth is correct.'; + + let currentDob: DateOfBirth | undefined; + let currentTime: TimeOfBirth | undefined; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildResult = (dateOfBirth: DateOfBirth): GetDOBResult => ({ + dateOfBirth, + timeOfBirth: currentTime, + }); + + // confirm tool is only injected after update_dob/update_time is called, + // preventing the LLM from hallucinating a confirmation without user input + const buildConfirmTool = (capturedDob: DateOfBirth | undefined) => { + const capturedTime = currentTime; + + return tool({ + name: 'confirm_dob', + description: 'Call after the user confirms the date of birth is correct.', + execute: async () => { + if (!isSameDate(capturedDob, currentDob) || !isSameTime(capturedTime, currentTime)) { + task.session.generateReply({ + instructions: + 'The date of birth has changed since confirmation was requested, ask the user to confirm the updated date.', + }); + return; + } + + if (currentDob === undefined) { + task.session.generateReply({ + instructions: 'No date of birth was provided yet, ask the user to provide it.', + }); + return; + } + + if (!task.done) { + task.complete(buildResult(currentDob)); + } + }, + }); + }; + + const injectConfirmTool = async (): Promise => { + const confirmTool = buildConfirmTool(currentDob); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_dob'); + await task.updateTools([...currentTools, confirmTool]); + }; + + const updateDobTool = tool({ + name: 'update_dob', + description: + "Update the date of birth provided by the user. Given a spoken month and year (e.g., 'July 2030'), return its numerical representation (7/2030).", + parameters: z.object({ + year: z.number().int().describe('The birth year (e.g., 1990)'), + month: z.number().int().describe('The birth month (1-12)'), + day: z.number().int().describe('The birth day (1-31)'), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ( + { year, month, day }: { year: number; month: number; day: number }, + { ctx }, + ) => { + // Normalize two-digit years to the intended century, matching what the + // prompt already asks the model to do ("90" -> 1990, "05" -> 2005). A + // literal two-digit year is otherwise a valid date (e.g. 90 -> year 90 AD) + // that passes the future-date check and silently corrupts the result. + if (year >= 0 && year < 100) { + const currentYY = new Date().getFullYear() % 100; + year += year <= currentYY ? 2000 : 1900; + } + + if (month < 1 || month > 12 || day < 1 || day > 31) { + throw new ToolError(`Invalid date: year=${year} month=${month} day=${day}`); + } + const candidate = new Date(Date.UTC(year, month - 1, day)); + if ( + candidate.getUTCFullYear() !== year || + candidate.getUTCMonth() !== month - 1 || + candidate.getUTCDate() !== day + ) { + throw new ToolError(`Invalid date: year=${year} month=${month} day=${day}`); + } + + const dob: DateOfBirth = { year, month, day }; + const today = new Date(); + const todayParts: DateOfBirth = { + year: today.getFullYear(), + month: today.getMonth() + 1, + day: today.getDate(), + }; + const isFuture = + dob.year > todayParts.year || + (dob.year === todayParts.year && + (dob.month > todayParts.month || + (dob.month === todayParts.month && dob.day > todayParts.day))); + if (isFuture) { + throw new ToolError( + `Invalid date of birth: ${formatDate(dob)} is in the future. ` + + 'Date of birth cannot be a future date.', + ); + } + + currentDob = dob; + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete(buildResult(currentDob)); + } + return undefined; + } + + await injectConfirmTool(); + + let response = `The date of birth has been updated to ${formatDate(dob)}`; + + if (currentTime) { + response += ` at ${formatTime(currentTime)}`; + } + + response += + '\nRepeat the date back to the user in a natural spoken format.\n' + + 'Prompt the user for confirmation, do not call `confirm_dob` directly'; + + return response; + }, + }); + + const updateTimeTool = tool({ + name: 'update_time', + description: 'Update the time of birth provided by the user.', + parameters: z.object({ + hour: z.number().int().describe('The birth hour (0-23)'), + minute: z.number().int().describe('The birth minute (0-59)'), + }), + execute: async ({ hour, minute }: { hour: number; minute: number }, { ctx }) => { + if (hour < 0 || hour > 23 || minute < 0 || minute > 59) { + throw new ToolError(`Invalid time: hour=${hour} minute=${minute}`); + } + + const birthTime: TimeOfBirth = { hour, minute }; + currentTime = birthTime; + + if (!confirmationRequired(ctx) && currentDob !== undefined) { + if (!task.done) { + task.complete(buildResult(currentDob)); + } + return undefined; + } + + if (confirmationRequired(ctx)) { + await injectConfirmTool(); + } + + let response = `The time of birth has been updated to ${formatTime(birthTime)}`; + + if (currentDob) { + response = `The date and time of birth has been updated to ${formatDate(currentDob)} at ${formatTime(birthTime)}`; + } + + if (confirmationRequired(ctx)) { + response += + '\nRepeat the time back to the user in a natural spoken format.\n' + + 'Prompt the user for confirmation, do not call `confirm_dob` directly'; + } else { + response += '\nThe date of birth has not been provided yet, ask the user to provide it.'; + } + + return response; + }, + }); + + const declineTool = tool({ + name: 'decline_dob_capture', + description: 'Handles the case when the user explicitly declines to provide a date of birth.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide the date of birth'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }) => { + if (!task.done) { + task.complete(new ToolError(`couldn't get the date of birth: ${reason}`)); + } + }, + }); + + const task = AgentTask.create({ + id: 'get_dob_task', + instructions: new Instructions('', { + audio: safeRender(BASE_INSTRUCTIONS, { + modality_specific: AUDIO_SPECIFIC, + time_instructions: timeInstructions, + confirmation_instructions: requireConfirmation !== false ? confirmationInstructions : '', + extra_instructions: extraInstructions, + }), + text: safeRender(BASE_INSTRUCTIONS, { + modality_specific: TEXT_SPECIFIC, + time_instructions: timeInstructions, + confirmation_instructions: requireConfirmation === true ? confirmationInstructions : '', + extra_instructions: extraInstructions, + }), + }), + chatCtx, + turnDetection: turnDetection ?? undefined, + tools: [...(tools ?? []), updateDobTool, declineTool, ...(includeTime ? [updateTimeTool] : [])], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, + onEnter: async () => { + const prompt = includeTime + ? 'Ask the user to provide their date of birth and, if they know it, their time of birth.' + : 'Ask the user to provide their date of birth.'; + task.session.generateReply({ instructions: prompt }); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetDOBTask}, preserving the + * `new GetDOBTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetDOBTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetDOBTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetDOBTask. + super({ instructions: '' }); + this.#task = createGetDOBTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +const BASE_INSTRUCTIONS = ` +You are only a single step in a broader system, responsible solely for capturing a date of birth. +{modality_specific} +{time_instructions}Call \`update_dob\` at the first opportunity whenever you form a new hypothesis about the date of birth. (before asking any questions or providing any answers.) +Don't invent dates, stick strictly to what the user said. +{confirmation_instructions} +When reading back dates, use a natural spoken format like 'January fifteenth, nineteen ninety'. +If the date is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts: first the month, then the day, then the year. +Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. +Avoid verbosity by not sharing example dates or formats unless prompted to do so. Do not deviate from the goal of collecting the user's birthday. +Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called.\ +{extra_instructions} +`; + +const AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect that users will say dates aloud with formats like: +- 'January 15th 1990' +- 'the fifteenth of January nineteen ninety' +- '01 15 1990' or 'one fifteen ninety' +- 'Jan 15 90' +- '15th January 1990' +Normalize common spoken patterns silently: +- Convert spoken numbers and ordinals to their numeric form: 'fifteenth' → 15, 'ninety' → 1990. +- Recognize month names in various forms: 'Jan', 'January', etc. +- Handle two-digit years appropriately: '90' likely means 1990, '05' likely means 2005. +- Filter out filler words or hesitations. +Don't mention corrections. Treat inputs as possibly imperfect but fix them silently. +`; + +const TEXT_SPECIFIC = ` +Handle input as typed text. Expect users to type their date of birth directly. +Accept common date formats like 'MM/DD/YYYY', 'January 15, 1990', or '1990-01-15'. +Handle two-digit years appropriately: '90' likely means 1990, '05' likely means 2005. +`; diff --git a/agents/src/workflows/dtmf_inputs.ts b/agents/src/workflows/dtmf_inputs.ts new file mode 100644 index 000000000..8892c157c --- /dev/null +++ b/agents/src/workflows/dtmf_inputs.ts @@ -0,0 +1,286 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { RoomEvent } from '@livekit/rtc-node'; +import { z } from 'zod'; +import { getJobContext } from '../job.js'; +import type { ChatContext } from '../llm/index.js'; +import { ToolError, tool } from '../llm/index.js'; +import { log } from '../log.js'; +import { AgentTask } from '../voice/agent.js'; +import { + AgentSessionEventTypes, + type AgentStateChangedEvent, + type UserStateChangedEvent, +} from '../voice/events.js'; +import { DtmfEvent, formatDtmf } from './utils.js'; + +export interface GetDtmfResult { + userInput: string; +} + +export interface GetDtmfTaskOptions { + /** The number of digits to collect. */ + numDigits: number; + /** Whether to ask for confirmation when the agent has collected the full digits. */ + askForConfirmation?: boolean; + /** The per-digit timeout, in milliseconds. Defaults to 4000. */ + dtmfInputTimeout?: number; + /** The DTMF event that stops collecting inputs. Defaults to `DtmfEvent.POUND`. */ + dtmfStopEvent?: DtmfEvent; + /** The chat context to use. */ + chatCtx?: ChatContext; + /** Extra instructions to add to the task. */ + extraInstructions?: string; +} + +/** + * Debounced runner for a single async function: `schedule()` (re)starts a delay timer, + * `run()` fires immediately, `cancel()` clears any pending timer. Mirrors the Python + * `utils.aio.debounced` helper closely enough for this task's needs. + */ +class Debounced { + #timer?: ReturnType; + + constructor( + private readonly fn: () => Promise, + private readonly delay: number, + ) {} + + schedule(): void { + this.cancel(); + this.#timer = setTimeout(() => { + this.#timer = undefined; + this.#run(); + }, this.delay); + } + + run(): void { + this.cancel(); + this.#run(); + } + + cancel(): void { + if (this.#timer !== undefined) { + clearTimeout(this.#timer); + this.#timer = undefined; + } + } + + #run(): void { + void this.fn().catch((error) => { + log().error({ error }, 'error running debounced DTMF reply'); + }); + } +} + +const dtmfResultFromInputs = (inputs: readonly DtmfEvent[]): GetDtmfResult => ({ + userInput: formatDtmf(inputs), +}); + +/** + * Build an {@link AgentTask} that collects DTMF inputs from the user. + * + * The task completes with the collected digit string, or fails with a {@link ToolError} + * when the expected number of digits is not received in time. + * + * This is the functional core; {@link GetDtmfTask} is a thin class wrapper over it. + */ +export function createGetDtmfTask({ + numDigits, + askForConfirmation = false, + dtmfInputTimeout = 4000, + dtmfStopEvent = DtmfEvent.POUND, + chatCtx, + extraInstructions, +}: GetDtmfTaskOptions): AgentTask { + if (numDigits <= 0) { + throw new Error('numDigits must be greater than 0'); + } + + const logger = log().child({ component: 'dtmf-inputs' }); + + const currDtmfInputs: DtmfEvent[] = []; + let dtmfReplyRunning = false; + + const confirmInputsTool = tool({ + name: 'confirm_inputs', + description: + 'Finalize the collected digit inputs after explicit user confirmation.\n\n' + + 'Use this ONLY after the confirmation. You should confirm by verbally reading out the digits one by one and, once the ' + + 'user confirms they are correct, call this tool with the inputs.\n\n' + + 'Do not use this tool to capture the initial digits.', + parameters: z.object({ + inputs: z.array(z.nativeEnum(DtmfEvent)).describe('The digit inputs to finalize'), + }), + execute: async ({ inputs }: { inputs: DtmfEvent[] }) => { + task.complete(dtmfResultFromInputs(inputs)); + }, + }); + + const recordInputsTool = tool({ + name: 'record_inputs', + description: + 'Record the collected digit inputs without additional confirmation.\n\n' + + 'Call this tool as soon as a valid sequence of digits has been provided by the user (via DTMF or spoken).', + parameters: z.object({ + inputs: z.array(z.nativeEnum(DtmfEvent)).describe('The digit inputs to record'), + }), + execute: async ({ inputs }: { inputs: DtmfEvent[] }) => { + task.complete(dtmfResultFromInputs(inputs)); + }, + }); + + let instructions = + 'You are a single step in a broader system, responsible solely for gathering digits input from the user. ' + + 'You will either receive a sequence of digits through dtmf events tagged by , or ' + + 'user will directly say the digits to you. You should be able to handle both cases. '; + + if (askForConfirmation) { + instructions += + 'Once user has confirmed the digits (by verbally spoken or entered manually), call `confirm_inputs` with the inputs.'; + } else { + instructions += + 'If user provides the digits through voice and it is valid, call `record_inputs` with the inputs.'; + } + + if (extraInstructions !== undefined) { + instructions += `\n${extraInstructions}`; + } + + const generateDtmfReply = new Debounced(async () => { + dtmfReplyRunning = true; + + try { + task.session.interrupt(); + + const dtmfStr = formatDtmf(currDtmfInputs); + logger.debug(`Generating DTMF reply, current inputs: ${dtmfStr}`); + + // if input not fully received (i.e. timeout), fail the task + if (currDtmfInputs.length !== numDigits) { + const errorMsg = + `Digits input not fully received. ` + + `Expect ${numDigits} digits, got ${currDtmfInputs.length}`; + if (!task.done) { + task.complete(new ToolError(errorMsg)); + } + return; + } + + // if not asking for confirmation, return the DTMF inputs + if (!askForConfirmation) { + if (!task.done) { + task.complete(dtmfResultFromInputs(currDtmfInputs)); + } + return; + } + + const replyInstructions = + 'User has entered the following valid digits on the telephone keypad:\n' + + `${dtmfStr}\n` + + 'Please confirm it with the user by saying the digits one by one with space in between ' + + "(.e.g. 'one two three four five six seven eight nine ten'). " + + 'Once you are sure, call `confirm_inputs` with the inputs.'; + + await task.session.generateReply({ userInput: replyInstructions }); + } finally { + dtmfReplyRunning = false; + currDtmfInputs.length = 0; + } + }, dtmfInputTimeout); + + const onSipDtmfReceived = (_code: number, digit: string): void => { + if (dtmfReplyRunning) { + return; + } + + // immediately kick off the DTMF reply generation if matches the stop event + if (digit === dtmfStopEvent) { + generateDtmfReply.run(); + return; + } + + if (!(Object.values(DtmfEvent) as string[]).includes(digit)) { + logger.warn(`Ignoring invalid DTMF digit: ${digit}`); + return; + } + + currDtmfInputs.push(digit as DtmfEvent); + logger.info(`DTMF inputs: ${formatDtmf(currDtmfInputs)}`); + generateDtmfReply.schedule(); + }; + + const onUserStateChanged = (ev: UserStateChangedEvent): void => { + if (dtmfReplyRunning) { + return; + } + + if (ev.newState === 'speaking') { + // clear any pending DTMF reply generation + generateDtmfReply.cancel(); + } else if (currDtmfInputs.length !== 0) { + // resume any previously cancelled DTMF reply generation after user is back to non-speaking + generateDtmfReply.schedule(); + } + }; + + const onAgentStateChanged = (ev: AgentStateChangedEvent): void => { + if (dtmfReplyRunning) { + return; + } + + if (ev.newState === 'speaking' || ev.newState === 'thinking') { + // clear any pending DTMF reply generation + generateDtmfReply.cancel(); + } else if (currDtmfInputs.length !== 0) { + // resume any previously cancelled DTMF reply generation after agent is back to non-speaking + generateDtmfReply.schedule(); + } + }; + + const task = AgentTask.create({ + id: 'get_dtmf_task', + instructions, + chatCtx, + tools: askForConfirmation ? [confirmInputsTool] : [recordInputsTool], + onEnter: async () => { + const ctx = getJobContext(); + + ctx.room.on(RoomEvent.DtmfReceived, onSipDtmfReceived); + task.session.on(AgentSessionEventTypes.UserStateChanged, onUserStateChanged); + task.session.on(AgentSessionEventTypes.AgentStateChanged, onAgentStateChanged); + task.session.generateReply({ toolChoice: 'none' }); + }, + onExit: async () => { + const ctx = getJobContext(); + + ctx.room.off(RoomEvent.DtmfReceived, onSipDtmfReceived); + task.session.off(AgentSessionEventTypes.UserStateChanged, onUserStateChanged); + task.session.off(AgentSessionEventTypes.AgentStateChanged, onAgentStateChanged); + generateDtmfReply.cancel(); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetDtmfTask}, preserving the + * `new GetDtmfTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetDtmfTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetDtmfTaskOptions) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetDtmfTask. + super({ instructions: '' }); + this.#task = createGetDtmfTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} diff --git a/agents/src/workflows/email_address.ts b/agents/src/workflows/email_address.ts new file mode 100644 index 000000000..2f094cc0f --- /dev/null +++ b/agents/src/workflows/email_address.ts @@ -0,0 +1,244 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; +import { type InstructionParts, resolveWorkflowInstructions } from './utils.js'; + +export interface GetEmailResult { + emailAddress: string; +} + +export interface GetEmailTaskOptions { + /** + * Instructions for the email capture prompt. Pass a full string or {@link Instructions} to + * replace the built-in prompt entirely, or {@link InstructionParts} to override individual + * sections (e.g. `persona`) while keeping the built-in template. + */ + instructions?: InstructionParts | Instructions | string; + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode | null; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString | null; + vad?: VAD | null; + llm?: LLM | RealtimeModel | LLMModels | null; + tts?: TTS | TTSModelString | null; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm the captured email address. Defaults to confirming on + * audio input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** + * When true, the model must produce an asking utterance before recording an email address — + * it can't silently fill one from the chat context during `onEnter`. + */ + requireExplicitAsk?: boolean; + /** @deprecated use `instructions.extra` instead */ + extraInstructions?: string; +} + +/** + * Build an {@link AgentTask} that collects an email address from the user. + * + * This is the functional core; {@link GetEmailTask} is a thin class wrapper over it. + */ +export function createGetEmailTask({ + instructions, + chatCtx, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + requireExplicitAsk = false, + extraInstructions = '', +}: GetEmailTaskOptions = {}): AgentTask { + let currentEmail = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildConfirmTool = (email: string) => + tool({ + name: 'confirm_email_address', + description: 'Call after the user confirms the email address is correct.', + execute: async () => { + if (email !== currentEmail) { + task.session.generateReply({ + instructions: + 'The email has changed since confirmation was requested, ask the user to confirm the updated email.', + }); + return; + } + + if (!task.done) { + task.complete({ emailAddress: email }); + } + }, + }); + + const updateEmailTool = tool({ + name: 'update_email_address', + description: 'Update the email address provided by the user.', + parameters: z.object({ + email: z.string().describe('The email address provided by the user'), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ({ email }: { email: string }, { ctx }) => { + email = email.trim(); + + if (!EMAIL_REGEX.test(email)) { + throw new ToolError(`Invalid email address provided: ${email}`); + } + + currentEmail = email; + const separatedEmail = email.split('').join(' '); + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete({ emailAddress: currentEmail }); + } + return undefined; // no need to continue the conversation + } + + const confirmTool = buildConfirmTool(email); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_email_address'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + `The email has been updated to ${email}\n` + + `Repeat the email character by character: ${separatedEmail} if needed\n` + + `Prompt the user for confirmation, do not call \`confirm_email_address\` directly` + ); + }, + }); + + const declineTool = tool({ + name: 'decline_email_capture', + description: 'Handles the case when the user explicitly declines to provide an email address.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide the email address'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }) => { + if (!task.done) { + task.complete(new ToolError(`couldn't get the email address: ${reason}`)); + } + }, + }); + + const task = AgentTask.create({ + id: 'get_email_task', + instructions: resolveWorkflowInstructions({ + instructions, + extraInstructions, + template: INSTRUCTIONS_TEMPLATE, + defaultPersona: PERSONA, + kwargs: { + _modality_specific: new Instructions('', { + audio: AUDIO_SPECIFIC, + text: TEXT_SPECIFIC, + }), + _confirmation: new Instructions('', { + // confirmation is enabled by default for audio, disabled by default for text + audio: requireConfirmation !== false ? CONFIRMATION_INSTRUCTION : '', + text: requireConfirmation === true ? CONFIRMATION_INSTRUCTION : '', + }), + }, + }), + chatCtx, + turnDetection: turnDetection ?? undefined, + tools: [...(tools ?? []), updateEmailTool, declineTool], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, + onEnter: async () => { + task.session.generateReply({ instructions: 'Ask the user to provide an email address.' }); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetEmailTask}, preserving the + * `new GetEmailTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetEmailTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetEmailTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetEmailTask. + super({ instructions: '' }); + this.#task = createGetEmailTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +const EMAIL_REGEX = + /^[A-Za-z0-9][A-Za-z0-9._%+\-]*@(?:[A-Za-z0-9](?:[A-Za-z0-9\-]*[A-Za-z0-9])?\.)+[A-Za-z]{2,}$/; + +// instructions +const PERSONA = + 'You are only a single step in a broader system, responsible solely for capturing an email address.'; + +const AUDIO_SPECIFIC = `Handle input as noisy voice transcription. Expect that users will say emails aloud with formats like: +- 'john dot doe at gmail dot com' +- 'susan underscore smith at yahoo dot co dot uk' +- 'dave dash b at protonmail dot com' +- 'jane at example' (partial—prompt for the domain) +- 'theo t h e o at livekit dot io' (name followed by spelling) +Normalize common spoken patterns silently: +- Convert words like 'dot', 'underscore', 'dash', 'plus' into symbols: \`.\`, \`_\`, \`-\`, \`+\`. +- Convert 'at' to \`@\`. +- Recognize patterns where users speak their name or a word, followed by spelling: e.g., 'john j o h n'. +- Filter out filler words or hesitations. +- Assume some spelling if contextually obvious (e.g. 'mike b two two' → mikeb22). +Don't mention corrections. Treat inputs as possibly imperfect but fix them silently.`; + +const TEXT_SPECIFIC = `Handle input as typed text. Expect users to type their email address directly in standard format. +If the address looks almost correct but has minor typos (e.g. missing '@' or domain), prompt for clarification.`; + +const CONFIRMATION_INSTRUCTION = `Call \`confirm_email_address\` after the user confirmed the email address is correct.`; + +const INSTRUCTIONS_TEMPLATE = `{persona} + +{_modality_specific} + +Call \`update_email_address\` at the first opportunity whenever you form a new hypothesis about the email. (before asking any questions or providing any answers.) +Don't invent new email addresses, stick strictly to what the user said. +{_confirmation} +If the email is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts: first the part before the '@', then the domain—only if needed. + +Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. +Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called. + +{extra} +`; diff --git a/agents/src/workflows/index.ts b/agents/src/workflows/index.ts index 99c334d57..48cdca1d6 100644 --- a/agents/src/workflows/index.ts +++ b/agents/src/workflows/index.ts @@ -1,6 +1,52 @@ // SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +export { + GetAddressTask, + createGetAddressTask, + type GetAddressResult, + type GetAddressTaskOptions, +} from './address.js'; +export { + CardCaptureDeclinedError, + CardCollectionRestartError, + GetCreditCardTask, + createGetCreditCardTask, + type GetCreditCardResult, + type GetCreditCardTaskOptions, +} from './credit_card.js'; +export { + GetDOBTask, + createGetDOBTask, + type DateOfBirth, + type GetDOBResult, + type GetDOBTaskOptions, + type TimeOfBirth, +} from './dob.js'; +export { + GetDtmfTask, + createGetDtmfTask, + type GetDtmfResult, + type GetDtmfTaskOptions, +} from './dtmf_inputs.js'; +export { + GetEmailTask, + createGetEmailTask, + type GetEmailResult, + type GetEmailTaskOptions, +} from './email_address.js'; +export { + GetNameTask, + createGetNameTask, + type GetNameResult, + type GetNameTaskOptions, +} from './name.js'; +export { + GetPhoneNumberTask, + createGetPhoneNumberTask, + type GetPhoneNumberResult, + type GetPhoneNumberTaskOptions, +} from './phone_number.js'; export { TaskGroup, type TaskCompletedEvent, @@ -13,4 +59,4 @@ export { type WarmTransferResult, type WarmTransferTaskOptions, } from './warm_transfer.js'; -export type { InstructionParts } from './utils.js'; +export { DtmfEvent, dtmfEventToCode, formatDtmf, type InstructionParts } from './utils.js'; diff --git a/agents/src/workflows/name.ts b/agents/src/workflows/name.ts new file mode 100644 index 000000000..79ed95df0 --- /dev/null +++ b/agents/src/workflows/name.ts @@ -0,0 +1,394 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import { safeRender } from '../utils.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; + +export interface GetNameResult { + firstName?: string; + middleName?: string; + lastName?: string; +} + +export interface GetNameTaskOptions { + /** Collect the user's first name. Defaults to true. */ + firstName?: boolean; + /** Collect the user's last name. Defaults to false. */ + lastName?: boolean; + /** Collect the user's middle name. Defaults to false. */ + middleName?: boolean; + /** + * Order in which to collect the name parts, using `{first_name}`, `{middle_name}` and + * `{last_name}` placeholders. Defaults to the enabled parts in first/middle/last order. + */ + nameFormat?: string; + /** Ask the user to verify the spelling of the captured name. Defaults to false. */ + verifySpelling?: boolean; + /** Extra instructions appended to the built-in prompt for domain-specific context. */ + extraInstructions?: string; + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode | null; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString | null; + vad?: VAD | null; + llm?: LLM | RealtimeModel | LLMModels | null; + tts?: TTS | TTSModelString | null; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm the captured name. Defaults to confirming on audio + * input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** + * When true, the model must produce an asking utterance before recording a name — it can't + * silently fill one from the chat context during `onEnter`. + */ + requireExplicitAsk?: boolean; +} + +interface UpdateNameArgs { + first_name?: string | null; + middle_name?: string | null; + last_name?: string | null; +} + +function cleanNameArg(value: string | null | undefined): string | undefined { + // Some models (e.g. gemma) fill optional args with placeholder strings like + // "null"/"NULL" instead of omitting them, or wrap values in literal quotes. + // Normalize those to undefined/clean values so they hit the required-field + // validation below instead of being recorded as the user's name. + if (value == null) { + return undefined; + } + const cleaned = value + .trim() + .replace(/^['"]+/, '') + .replace(/['"]+$/, ''); + if ( + !cleaned || + ['null', 'none', 'nil', 'n/a', 'unknown', 'unspecified'].includes(cleaned.toLowerCase()) + ) { + return undefined; + } + return cleaned; +} + +function renderNameFormat( + nameFormat: string, + parts: { firstName: string; middleName: string; lastName: string }, +): string { + const replacements: Record = { + first_name: parts.firstName, + middle_name: parts.middleName, + last_name: parts.lastName, + }; + return nameFormat + .replace(/\{(first_name|middle_name|last_name)\}/g, (_match, key: string) => { + return replacements[key] ?? ''; + }) + .trim(); +} + +/** + * Build an {@link AgentTask} that collects the user's name. + * + * This is the functional core; {@link GetNameTask} is a thin class wrapper over it. + */ +export function createGetNameTask({ + firstName = true, + lastName = false, + middleName = false, + nameFormat, + verifySpelling = false, + extraInstructions = '', + chatCtx, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + requireExplicitAsk = false, +}: GetNameTaskOptions = {}): AgentTask { + if (!(firstName || middleName || lastName)) { + throw new Error('At least one of firstName, middleName, or lastName must be true'); + } + + let resolvedNameFormat: string; + if (nameFormat !== undefined) { + resolvedNameFormat = nameFormat; + } else { + const parts: string[] = []; + if (firstName) parts.push('{first_name}'); + if (middleName) parts.push('{middle_name}'); + if (lastName) parts.push('{last_name}'); + resolvedNameFormat = parts.join(' '); + } + + const spellingInstructions = !verifySpelling + ? '' + : 'After receiving the name, always verify the spelling by asking the user to confirm ' + + 'or spell out the name letter by letter. ' + + 'When confirming, spell out each name part letter by letter to the user. '; + const confirmationInstructions = + 'Call `confirm_name` after the user confirmed the name is correct.'; + + let currentFirstName = ''; + let currentMiddleName = ''; + let currentLastName = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildResult = (): GetNameResult => ({ + firstName: firstName ? currentFirstName : undefined, + middleName: middleName ? currentMiddleName : undefined, + lastName: lastName ? currentLastName : undefined, + }); + + const buildConfirmTool = (captured: { first: string; middle: string; last: string }) => + tool({ + name: 'confirm_name', + description: 'Call after the user confirms the name is correct.', + execute: async () => { + if ( + captured.first !== currentFirstName || + captured.middle !== currentMiddleName || + captured.last !== currentLastName + ) { + task.session.generateReply({ + instructions: + 'The name has changed since confirmation was requested, ask the user to confirm the updated name.', + }); + return; + } + + if (!task.done) { + task.complete(buildResult()); + } + }, + }); + + const updateNameTool = tool({ + name: 'update_name', + description: 'Update the name provided by the user.', + parameters: z.object({ + first_name: z.string().nullable().optional().describe("The user's first name."), + middle_name: z + .string() + .nullable() + .optional() + .describe("The user's middle name, if collected."), + last_name: z.string().nullable().optional().describe("The user's last name, if collected."), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async (args: UpdateNameArgs, { ctx }) => { + const cleanedFirst = cleanNameArg(args.first_name); + const cleanedMiddle = cleanNameArg(args.middle_name); + const cleanedLast = cleanNameArg(args.last_name); + + const errors: string[] = []; + if (firstName && !cleanedFirst?.trim()) { + errors.push('first name is required but was not provided'); + } + if (middleName && !cleanedMiddle?.trim()) { + errors.push('middle name is required but was not provided'); + } + if (lastName && !cleanedLast?.trim()) { + errors.push('last name is required but was not provided'); + } + + // A real name contains letters. Reject digit-only or punctuation-only + // values so a card number, ZIP code, phone number, etc. accidentally + // crammed into update_name fails fast instead of being recorded as + // the user's name. + for (const [label, value] of [ + ['first', cleanedFirst], + ['middle', cleanedMiddle], + ['last', cleanedLast], + ] as const) { + if (value && value.trim() && !/\p{L}/u.test(value)) { + errors.push( + `${label} name '${value}' contains no letters - that doesn't look like a name`, + ); + } + } + + if (errors.length > 0) { + throw new ToolError(`Incomplete name: ${errors.join('; ')}`); + } + + currentFirstName = cleanedFirst?.trim() ?? ''; + currentMiddleName = cleanedMiddle?.trim() ?? ''; + currentLastName = cleanedLast?.trim() ?? ''; + + const fullName = renderNameFormat(resolvedNameFormat, { + firstName: currentFirstName, + middleName: currentMiddleName, + lastName: currentLastName, + }); + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete(buildResult()); + } + return undefined; + } + + const confirmTool = buildConfirmTool({ + first: currentFirstName, + middle: currentMiddleName, + last: currentLastName, + }); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_name'); + await task.updateTools([...currentTools, confirmTool]); + + if (verifySpelling) { + return ( + `The name has been updated to ${fullName}\n` + + `Spell out the name letter by letter for verification: ${fullName}\n` + + `Prompt the user for confirmation, do not call \`confirm_name\` directly` + ); + } + + return ( + `The name has been updated to ${fullName}\n` + + `Repeat the name back to the user and prompt for confirmation, ` + + `do not call \`confirm_name\` directly` + ); + }, + }); + + const declineTool = tool({ + name: 'decline_name_capture', + description: 'Handles the case when the user explicitly declines to provide their name.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide their name'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }) => { + if (!task.done) { + task.complete(new ToolError(`couldn't get the name: ${reason}`)); + } + }, + }); + + const task = AgentTask.create({ + id: 'get_name_task', + instructions: new Instructions('', { + audio: safeRender(BASE_INSTRUCTIONS, { + name_format: resolvedNameFormat, + modality_specific: AUDIO_SPECIFIC, + spelling_instructions: spellingInstructions, + confirmation_instructions: requireConfirmation !== false ? confirmationInstructions : '', + extra_instructions: extraInstructions, + }), + text: safeRender(BASE_INSTRUCTIONS, { + name_format: resolvedNameFormat, + modality_specific: TEXT_SPECIFIC, + spelling_instructions: spellingInstructions, + confirmation_instructions: requireConfirmation === true ? confirmationInstructions : '', + extra_instructions: extraInstructions, + }), + }), + chatCtx, + turnDetection: turnDetection ?? undefined, + tools: [...(tools ?? []), updateNameTool, declineTool], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, + onEnter: async () => { + task.session.generateReply({ + instructions: + `Get the user's name (follow this order '${resolvedNameFormat}' but do not ` + + 'mention the format). First scan the conversation - if a name was already ' + + 'given earlier, ask a short confirmation question rather than asking from ' + + 'scratch. If context about what the name is FOR was provided (a role like ' + + "'cardholder', 'guest', 'emergency contact'), anchor your confirmation " + + "question to that role so the user knows which name you mean - don't ask " + + 'abstractly. When pointing at where an existing name came from, reference ' + + 'the source in the conversation (the earlier step, the booking they ' + + 'mentioned), not a presumption about how the name appears in the ' + + 'destination. Only ask fresh when the conversation has no name yet.', + }); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetNameTask}, preserving the + * `new GetNameTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetNameTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetNameTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetNameTask. + super({ instructions: '' }); + this.#task = createGetNameTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +const BASE_INSTRUCTIONS = ` +You are only a single step in a broader system, responsible solely for capturing the user's name. +You need to naturally collect the name parts in this order: {name_format}. +{modality_specific} +{spelling_instructions}Call \`update_name\` at the first opportunity whenever you form a new hypothesis about the name. (before asking any questions or providing any answers.) +Don't invent names, stick strictly to what the user said. +{confirmation_instructions} +If the name is unclear or it takes too much back-and-forth, prompt for each name part separately. +Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. +Avoid verbosity by not sharing example names or spellings unless prompted to do so. Do not deviate from the goal of collecting the user's name. +Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called.\ +{extra_instructions} +`; + +const AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect that users will say names aloud and may: +- Say their name followed by spelling: e.g., 'Michael m i c h a e l' +- Use phonetic alphabet: e.g., 'Mike as in Mike India Charlie Hotel Alpha Echo Lima' +- Have names with special characters or hyphens: e.g., 'Mary-Jane' or 'O'Brien' +- Have names from various cultural backgrounds with different pronunciation patterns +Normalize common spoken patterns silently: +- Convert 'dash' or 'hyphen' to \`-\`. +- Convert 'apostrophe' to \`'\`. +- Recognize when users spell out their name letter by letter. +- Filter out filler words or hesitations. +- Capitalize the first letter of each name part appropriately. +Don't mention corrections. Treat inputs as possibly imperfect but fix them silently. +`; + +const TEXT_SPECIFIC = ` +Handle input as typed text. Expect users to type their name directly. +Capitalize the first letter of each name part appropriately. +If the name contains special characters or hyphens (e.g., 'Mary-Jane' or 'O'Brien'), preserve them as typed. +`; diff --git a/agents/src/workflows/phone_number.ts b/agents/src/workflows/phone_number.ts new file mode 100644 index 000000000..3ae7ccb17 --- /dev/null +++ b/agents/src/workflows/phone_number.ts @@ -0,0 +1,233 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import { safeRender } from '../utils.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; + +const PHONE_REGEX = /^\+?[1-9]\d{6,14}$/; + +export interface GetPhoneNumberResult { + phoneNumber: string; +} + +export interface GetPhoneNumberTaskOptions { + /** Extra instructions appended to the built-in prompt for domain-specific context. */ + extraInstructions?: string; + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode | null; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString | null; + vad?: VAD | null; + llm?: LLM | RealtimeModel | LLMModels | null; + tts?: TTS | TTSModelString | null; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm the captured phone number. Defaults to confirming on + * audio input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** + * When true, the model must produce an asking utterance before recording a phone number — it + * can't silently fill one from the chat context during `onEnter`. + */ + requireExplicitAsk?: boolean; +} + +/** + * Build an {@link AgentTask} that collects a phone number from the user. + * + * This is the functional core; {@link GetPhoneNumberTask} is a thin class wrapper over it. + */ +export function createGetPhoneNumberTask({ + extraInstructions = '', + chatCtx, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + requireExplicitAsk = false, +}: GetPhoneNumberTaskOptions = {}): AgentTask { + const confirmationInstructions = + 'Call `confirm_phone_number` after the user confirmed the phone number is correct.'; + + let currentPhoneNumber = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildConfirmTool = (phoneNumber: string) => + tool({ + name: 'confirm_phone_number', + description: 'Call after the user confirms the phone number is correct.', + execute: async () => { + if (phoneNumber !== currentPhoneNumber) { + task.session.generateReply({ + instructions: + 'The phone number has changed since confirmation was requested, ask the user to confirm the updated number.', + }); + return; + } + + if (!task.done) { + task.complete({ phoneNumber }); + } + }, + }); + + const updatePhoneNumberTool = tool({ + name: 'update_phone_number', + description: 'Update the phone number provided by the user.', + parameters: z.object({ + phone_number: z + .string() + .describe('The phone number provided by the user, digits only with optional leading +'), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ({ phone_number: phoneNumber }: { phone_number: string }, { ctx }) => { + const cleaned = phoneNumber.trim().replace(/[\s\-().]+/g, ''); + + if (!PHONE_REGEX.test(cleaned)) { + throw new ToolError(`Invalid phone number provided: ${phoneNumber}`); + } + + currentPhoneNumber = cleaned; + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete({ phoneNumber: currentPhoneNumber }); + } + return undefined; // no need to continue the conversation + } + + const confirmTool = buildConfirmTool(cleaned); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_phone_number'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + `The phone number has been updated to ${cleaned}\n` + + `Read the number back to the user in groups.\n` + + `Prompt the user for confirmation, do not call \`confirm_phone_number\` directly` + ); + }, + }); + + const declineTool = tool({ + name: 'decline_phone_number_capture', + description: 'Handles the case when the user explicitly declines to provide a phone number.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide the phone number'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }) => { + if (!task.done) { + task.complete(new ToolError(`couldn't get the phone number: ${reason}`)); + } + }, + }); + + const task = AgentTask.create({ + id: 'get_phone_number_task', + instructions: new Instructions('', { + audio: safeRender(BASE_INSTRUCTIONS, { + modality_specific: AUDIO_SPECIFIC, + confirmation_instructions: requireConfirmation !== false ? confirmationInstructions : '', + extra_instructions: extraInstructions, + }), + text: safeRender(BASE_INSTRUCTIONS, { + modality_specific: TEXT_SPECIFIC, + confirmation_instructions: requireConfirmation === true ? confirmationInstructions : '', + extra_instructions: extraInstructions, + }), + }), + chatCtx, + turnDetection: turnDetection ?? undefined, + tools: [...(tools ?? []), updatePhoneNumberTool, declineTool], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, + onEnter: async () => { + task.session.generateReply({ instructions: 'Ask the user to provide their phone number.' }); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetPhoneNumberTask}, preserving the + * `new GetPhoneNumberTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetPhoneNumberTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetPhoneNumberTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetPhoneNumberTask. + super({ instructions: '' }); + this.#task = createGetPhoneNumberTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +const BASE_INSTRUCTIONS = ` +You are only a single step in a broader system, responsible solely for capturing a phone number. +{modality_specific} +Call \`update_phone_number\` at the first opportunity whenever you form a new hypothesis about the phone number. (before asking any questions or providing any answers.) +Don't invent phone numbers, stick strictly to what the user said. +{confirmation_instructions} +If the number is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts: first the area code, then the remaining digits. +Never repeat the phone number back to the user as a single block of digits. Read it back in groups. +Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. +Avoid verbosity by not sharing example phone numbers or formats unless prompted to do so. Do not deviate from the goal of collecting the user's phone number. +Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called.\ +{extra_instructions} +`; + +const AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect that users will say phone numbers aloud with formats like: +- '555 123 4567' +- 'five five five, one two three, four five six seven' +- '+1 555 123 4567' +- 'area code 555, 123 4567' +- '555-123-4567' +Normalize common spoken patterns silently: +- Convert spoken digits to their numeric form: 'five' → 5, 'zero' → 0, 'oh' → 0. +- Remove filler words, pauses, and hesitations. +- Strip dashes, spaces, parentheses, and dots from the number. +- Recognize 'plus' at the start as the international prefix \`+\`. +- Recognize 'area code' as a prefix for the area code digits. +Don't mention corrections. Treat inputs as possibly imperfect but fix them silently. +`; + +const TEXT_SPECIFIC = ` +Handle input as typed text. Expect users to type their phone number directly. +Strip dashes, spaces, parentheses, and dots from the number. +If the number looks almost correct but has minor formatting issues, clean it up silently. +`; diff --git a/agents/src/workflows/task_group.ts b/agents/src/workflows/task_group.ts index 30a501987..a733c0240 100644 --- a/agents/src/workflows/task_group.ts +++ b/agents/src/workflows/task_group.ts @@ -8,7 +8,8 @@ import { asError } from '../utils.js'; import { AgentTask } from '../voice/agent.js'; interface FactoryInfo { - taskFactory: () => AgentTask; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts tasks with any result type + taskFactory: () => AgentTask; id: string; description: string; } @@ -64,7 +65,8 @@ export class TaskGroup extends AgentTask { this._taskCompletedCallback = onTaskCompleted; } - add(task: () => AgentTask, { id, description }: { id: string; description: string }): this { + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts tasks with any result type + add(task: () => AgentTask, { id, description }: { id: string; description: string }): this { this._registeredFactories.set(id, { taskFactory: task, id, description }); return this; } diff --git a/agents/src/workflows/utils.ts b/agents/src/workflows/utils.ts index 25ba05f30..b0189939f 100644 --- a/agents/src/workflows/utils.ts +++ b/agents/src/workflows/utils.ts @@ -1,7 +1,8 @@ // SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import type { Instructions } from '../llm/index.js'; +import { Instructions, isInstructions } from '../llm/chat_context.js'; +import { log } from '../log.js'; /** * Customizable instruction sections for built-in workflow tasks. @@ -16,3 +17,76 @@ export interface InstructionParts { /** Extra instructions appended to the prompt for domain-specific context. */ extra?: Instructions | string; } + +/** + * Resolve workflow instructions the way the Python `WorkflowInstructions` class does: + * a full string or {@link Instructions} replaces the built-in prompt entirely, while + * {@link InstructionParts} (or nothing) fills the workflow's built-in template. + * + * @internal + */ +export function resolveWorkflowInstructions(options: { + instructions?: InstructionParts | Instructions | string; + /** @deprecated legacy `extraInstructions` option, ignored when `instructions` is provided. */ + extraInstructions?: string; + template: string; + defaultPersona: string; + kwargs?: Record; +}): string | Instructions { + const { instructions, extraInstructions = '', template, defaultPersona, kwargs = {} } = options; + + if (instructions !== undefined && extraInstructions) { + log().warn('`extraInstructions` will be ignored when `instructions` is provided'); + } + + // A full instruction string or Instructions replaces the built-in prompt entirely. + if (typeof instructions === 'string' || isInstructions(instructions)) { + return instructions; + } + + // No instructions or an `InstructionParts` override: fill the built-in template. + const parts: InstructionParts = instructions ?? { extra: extraInstructions }; + return Instructions.resolveTemplate(template, { + // Unset preserves the built-in default; an explicit empty string removes the section. + persona: parts.persona !== undefined ? parts.persona : defaultPersona, + extra: parts.extra !== undefined ? parts.extra : '', + ...kwargs, + }); +} + +export enum DtmfEvent { + ONE = '1', + TWO = '2', + THREE = '3', + FOUR = '4', + FIVE = '5', + SIX = '6', + SEVEN = '7', + EIGHT = '8', + NINE = '9', + ZERO = '0', + STAR = '*', + POUND = '#', + A = 'A', + B = 'B', + C = 'C', + D = 'D', +} + +export function dtmfEventToCode(event: DtmfEvent): number { + if (/^\d$/.test(event)) { + return Number(event); + } else if (event === DtmfEvent.STAR) { + return 10; + } else if (event === DtmfEvent.POUND) { + return 11; + } else if (['A', 'B', 'C', 'D'].includes(event)) { + // DTMF codes 12-15 are used for letters A-D + return event.charCodeAt(0) - 'A'.charCodeAt(0) + 12; + } + throw new Error(`Invalid DTMF event: ${event}`); +} + +export function formatDtmf(events: readonly DtmfEvent[]): string { + return events.join(' '); +} From 00340775406a1231611455e39110aa744bdc6eb7 Mon Sep 17 00:00:00 2001 From: Brian Yin Date: Fri, 10 Jul 2026 13:35:59 +0800 Subject: [PATCH 12/12] refactor(workflows): cleanup data-capture tasks (#2016) --- agents/src/workflows/address.ts | 78 ++++++------ agents/src/workflows/credit_card.ts | 164 +++++++++++--------------- agents/src/workflows/dob.ts | 52 ++++---- agents/src/workflows/email_address.ts | 54 ++++----- agents/src/workflows/name.ts | 92 +++++++-------- agents/src/workflows/phone_number.ts | 48 ++++---- agents/src/workflows/utils.ts | 25 ++-- 7 files changed, 240 insertions(+), 273 deletions(-) diff --git a/agents/src/workflows/address.ts b/agents/src/workflows/address.ts index 5440771c8..02510d501 100644 --- a/agents/src/workflows/address.ts +++ b/agents/src/workflows/address.ts @@ -25,12 +25,12 @@ export interface GetAddressTaskOptions { */ instructions?: InstructionParts | Instructions | string; chatCtx?: ChatContext; - turnDetection?: TurnDetectionMode | null; + turnDetection?: TurnDetectionMode; tools?: readonly ToolContextEntry[]; - stt?: STT | STTModelString | null; - vad?: VAD | null; - llm?: LLM | RealtimeModel | LLMModels | null; - tts?: TTS | TTSModelString | null; + stt?: STT | STTModelString; + vad?: VAD; + llm?: LLM | RealtimeModel | LLMModels; + tts?: TTS | TTSModelString; allowInterruptions?: boolean; /** * Whether to ask the user to confirm the captured address. Defaults to confirming on audio @@ -42,13 +42,11 @@ export interface GetAddressTaskOptions { * can't silently fill one from the chat context during `onEnter`. */ requireExplicitAsk?: boolean; - /** @deprecated use `instructions.extra` instead */ - extraInstructions?: string; } interface UpdateAddressArgs { - street_address: string; - unit_number: string; + streetAddress: string; + unitNumber: string; locality: string; country: string; } @@ -62,7 +60,7 @@ export function createGetAddressTask({ instructions, chatCtx, turnDetection, - tools, + tools = [], stt, vad, llm, @@ -70,7 +68,6 @@ export function createGetAddressTask({ allowInterruptions, requireConfirmation, requireExplicitAsk = false, - extraInstructions = '', }: GetAddressTaskOptions = {}): AgentTask { let currentAddress = ''; @@ -106,12 +103,12 @@ export function createGetAddressTask({ name: 'update_address', description: 'Update the address provided by the user.', parameters: z.object({ - street_address: z + streetAddress: z .string() .describe( 'Dependent on country, may include fields like house number, street name, block, or district', ), - unit_number: z + unitNumber: z .string() .describe( "The unit number, for example Floor 1 or Apartment 12. If there is no unit number, return ''", @@ -124,8 +121,10 @@ export function createGetAddressTask({ // With requireExplicitAsk, the model can't silent-fill from chatCtx during // onEnter — it must produce an asking utterance first. flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, - execute: async (args: UpdateAddressArgs, { ctx }) => { - const { street_address: streetAddress, unit_number: unitNumber, locality, country } = args; + execute: async ( + { streetAddress, unitNumber, locality, country }: UpdateAddressArgs, + { ctx }, + ) => { const addressFields = unitNumber.trim() ? [streetAddress, unitNumber, locality, country] : [streetAddress, locality, country]; @@ -136,7 +135,7 @@ export function createGetAddressTask({ if (!task.done) { task.complete({ address: currentAddress }); } - return undefined; + return; } const confirmTool = buildConfirmTool(address); @@ -171,28 +170,25 @@ export function createGetAddressTask({ id: 'get_address_task', instructions: resolveWorkflowInstructions({ instructions, - extraInstructions, template: INSTRUCTIONS_TEMPLATE, defaultPersona: PERSONA, - kwargs: { - _modality_specific: new Instructions('', { - audio: AUDIO_SPECIFIC, - text: TEXT_SPECIFIC, - }), - _confirmation: new Instructions('', { - // confirmation is enabled by default for audio, disabled by default for text - audio: requireConfirmation !== false ? CONFIRMATION_INSTRUCTION : '', - text: requireConfirmation === true ? CONFIRMATION_INSTRUCTION : '', - }), - }, + modalitySpecific: new Instructions('', { + audio: AUDIO_SPECIFIC, + text: TEXT_SPECIFIC, + }), + confirmation: new Instructions('', { + // confirmation is enabled by default for audio, disabled by default for text + audio: requireConfirmation !== false ? CONFIRMATION_INSTRUCTION : '', + text: requireConfirmation === true ? CONFIRMATION_INSTRUCTION : '', + }), }), chatCtx, - turnDetection: turnDetection ?? undefined, - tools: [...(tools ?? []), updateAddressTool, declineTool], - stt: stt ?? undefined, - vad: vad ?? undefined, - llm: llm ?? undefined, - tts: tts ?? undefined, + turnDetection, + tools: [...tools, updateAddressTool, declineTool], + stt, + vad, + llm, + tts, allowInterruptions, onEnter: async () => { task.session.generateReply({ instructions: 'Ask the user to provide their address.' }); @@ -228,11 +224,11 @@ const PERSONA = const AUDIO_SPECIFIC = `You will be handling addresses from any country. Expect that users will say address in different formats with fields filled like: -- 'street_address': '450 SOUTH MAIN ST', 'unit_number': 'FLOOR 2', 'locality': 'SALT LAKE CITY UT 84101', 'country': 'UNITED STATES', -- 'street_address': '123 MAPLE STREET', 'unit_number': 'APARTMENT 10', 'locality': 'OTTAWA ON K1A 0B1', 'country': 'CANADA', -- 'street_address': 'GUOMAO JIE 3 HAO, CHAOYANG QU', 'unit_number': 'GUOMAO DA SHA 18 LOU 101 SHI', 'locality': 'BEIJING SHI 100000', 'country': 'CHINA', -- 'street_address': '5 RUE DE L’ANCIENNE COMÉDIE', 'unit_number': 'APP C4', 'locality': '75006 PARIS', 'country': 'FRANCE', -- 'street_address': 'PLOT 10, NEHRU ROAD', 'unit_number': 'OFFICE 403, 4TH FLOOR', 'locality': 'VILE PARLE (E), MUMBAI MAHARASHTRA 400099', 'country': 'INDIA', +- 'streetAddress': '450 SOUTH MAIN ST', 'unitNumber': 'FLOOR 2', 'locality': 'SALT LAKE CITY UT 84101', 'country': 'UNITED STATES', +- 'streetAddress': '123 MAPLE STREET', 'unitNumber': 'APARTMENT 10', 'locality': 'OTTAWA ON K1A 0B1', 'country': 'CANADA', +- 'streetAddress': 'GUOMAO JIE 3 HAO, CHAOYANG QU', 'unitNumber': 'GUOMAO DA SHA 18 LOU 101 SHI', 'locality': 'BEIJING SHI 100000', 'country': 'CHINA', +- 'streetAddress': '5 RUE DE L’ANCIENNE COMÉDIE', 'unitNumber': 'APP C4', 'locality': '75006 PARIS', 'country': 'FRANCE', +- 'streetAddress': 'PLOT 10, NEHRU ROAD', 'unitNumber': 'OFFICE 403, 4TH FLOOR', 'locality': 'VILE PARLE (E), MUMBAI MAHARASHTRA 400099', 'country': 'INDIA', Normalize common spoken patterns silently: - Convert words like 'dash' and 'apostrophe' into symbols: \`-\`, \`'\`. - Convert spelled out numbers like 'six' and 'seven' into numerals: \`6\`, \`7\`. @@ -255,11 +251,11 @@ const CONFIRMATION_INSTRUCTION = `Call \`confirm_address\` after the user confir const INSTRUCTIONS_TEMPLATE = `{persona} -{_modality_specific} +{modalitySpecific} Call \`update_address\` at the first opportunity whenever you form a new hypothesis about the address. (before asking any questions or providing any answers.) Don't invent new addresses, stick strictly to what the user said. -{_confirmation} +{confirmation} If the address is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts in this order: street address, unit number if applicable, locality, and country. Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. diff --git a/agents/src/workflows/credit_card.ts b/agents/src/workflows/credit_card.ts index 1c54125d5..5e9b5c510 100644 --- a/agents/src/workflows/credit_card.ts +++ b/agents/src/workflows/credit_card.ts @@ -122,12 +122,12 @@ interface SubTaskOptions { requireConfirmation?: boolean; requireExplicitAsk?: boolean; extraInstructions?: string; - turnDetection?: TurnDetectionMode | null; + turnDetection?: TurnDetectionMode; tools?: readonly ToolContextEntry[]; - stt?: STT | STTModelString | null; - vad?: VAD | null; - llm?: LLM | RealtimeModel | LLMModels | null; - tts?: TTS | TTSModelString | null; + stt?: STT | STTModelString; + vad?: VAD; + llm?: LLM | RealtimeModel | LLMModels; + tts?: TTS | TTSModelString; allowInterruptions?: boolean; } @@ -143,13 +143,13 @@ function buildSubTaskInstructions( return new Instructions('', { audio: safeRender(baseInstructions, { - modality_specific: audioSpecific, - confirmation_instructions: requireConfirmation !== false ? confirmationInstructions : '', + modalitySpecific: audioSpecific, + confirmationInstructions: requireConfirmation !== false ? confirmationInstructions : '', }) + extraSuffix, text: safeRender(baseInstructions, { - modality_specific: textSpecific, - confirmation_instructions: requireConfirmation === true ? confirmationInstructions : '', + modalitySpecific: textSpecific, + confirmationInstructions: requireConfirmation === true ? confirmationInstructions : '', }) + extraSuffix, }); } @@ -160,7 +160,7 @@ function createGetCardNumberTask({ requireExplicitAsk = false, extraInstructions = '', turnDetection, - tools, + tools = [], stt, vad, llm, @@ -181,11 +181,9 @@ function createGetCardNumberTask({ name: 'confirm_card_number', description: 'Call after the user repeats their card number for confirmation.', parameters: z.object({ - repeated_card_number: z - .string() - .describe('The card number repeated by the user as a string'), + repeatedCardNumber: z.string().describe('The card number repeated by the user as a string'), }), - execute: async ({ repeated_card_number: repeatedCardNumber }) => { + execute: async ({ repeatedCardNumber }) => { repeatedCardNumber = repeatedCardNumber.replace(/\D/g, ''); if (repeatedCardNumber !== cardNumber) { task.session.generateReply({ @@ -213,21 +211,21 @@ function createGetCardNumberTask({ description: "Call to record the user's card number. Only call once the entire number has been given, do not call in increments.", parameters: z.object({ - card_number: z + cardNumber: z .string() .describe('The credit card number as a string with no dashes or spaces'), }), // With requireExplicitAsk, the model can't silent-fill from chatCtx during // onEnter — it must produce an asking utterance first. flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, - execute: async ({ card_number: rawCardNumber }: { card_number: string }, { ctx }) => { + execute: async ({ cardNumber: rawCardNumber }: { cardNumber: string }, { ctx }) => { const cardNumber = rawCardNumber.replace(/\D/g, ''); if (cardNumber.length < 13 || cardNumber.length > 19) { task.session.generateReply({ instructions: 'The length of the card number is invalid, ask the user to repeat their card number.', }); - return undefined; + return; } currentCardNumber = cardNumber; @@ -244,7 +242,7 @@ function createGetCardNumberTask({ task.complete({ issuer, cardNumber: currentCardNumber }); } } - return undefined; + return; } const confirmTool = buildConfirmTool(cardNumber); @@ -269,17 +267,12 @@ function createGetCardNumberTask({ extraInstructions, ), chatCtx, - turnDetection: turnDetection ?? undefined, - tools: [ - ...(tools ?? []), - updateCardNumberTool, - declineCardCaptureTool, - restartCardCollectionTool, - ], - stt: stt ?? undefined, - vad: vad ?? undefined, - llm: llm ?? undefined, - tts: tts ?? undefined, + turnDetection, + tools: [...tools, updateCardNumberTool, declineCardCaptureTool, restartCardCollectionTool], + stt, + vad, + llm, + tts, allowInterruptions, onEnter: async () => { await task.session.generateReply({ @@ -302,7 +295,7 @@ function createGetSecurityCodeTask({ requireExplicitAsk = false, extraInstructions = '', turnDetection, - tools, + tools = [], stt, vad, llm, @@ -323,9 +316,9 @@ function createGetSecurityCodeTask({ name: 'confirm_security_code', description: 'Call after the user repeats their security code for confirmation.', parameters: z.object({ - repeated_security_code: z.string().describe('The security code repeated by the user'), + repeatedSecurityCode: z.string().describe('The security code repeated by the user'), }), - execute: async ({ repeated_security_code: repeatedSecurityCode }) => { + execute: async ({ repeatedSecurityCode }) => { if (repeatedSecurityCode.trim() !== securityCode) { task.session.generateReply({ instructions: 'The repeated security code does not match, ask the user to try again.', @@ -343,21 +336,21 @@ function createGetSecurityCodeTask({ name: 'update_security_code', description: "Call to update the card's security code.", parameters: z.object({ - security_code: z + securityCode: z .string() .describe("The card's security code (3-4 digits, may have leading zeros)."), }), // With requireExplicitAsk, the model can't silent-fill from chatCtx during // onEnter — it must produce an asking utterance first. flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, - execute: async ({ security_code: securityCode }: { security_code: string }, { ctx }) => { + execute: async ({ securityCode }: { securityCode: string }, { ctx }) => { const stripped = securityCode.trim(); if (!/^\d+$/.test(stripped) || stripped.length < 3 || stripped.length > 4) { task.session.generateReply({ instructions: "The security code's length is invalid, ask the user to repeat or to provide a new card and start over.", }); - return undefined; + return; } currentSecurityCode = stripped; @@ -366,7 +359,7 @@ function createGetSecurityCodeTask({ if (!task.done) { task.complete({ securityCode: currentSecurityCode }); } - return undefined; + return; } const confirmTool = buildConfirmTool(stripped); @@ -392,17 +385,12 @@ function createGetSecurityCodeTask({ extraInstructions, ), chatCtx, - turnDetection: turnDetection ?? undefined, - tools: [ - ...(tools ?? []), - updateSecurityCodeTool, - declineCardCaptureTool, - restartCardCollectionTool, - ], - stt: stt ?? undefined, - vad: vad ?? undefined, - llm: llm ?? undefined, - tts: tts ?? undefined, + turnDetection, + tools: [...tools, updateSecurityCodeTool, declineCardCaptureTool, restartCardCollectionTool], + stt, + vad, + llm, + tts, allowInterruptions, onEnter: async () => { await task.session.generateReply({ @@ -423,7 +411,7 @@ function createGetExpirationDateTask({ requireExplicitAsk = false, extraInstructions = '', turnDetection, - tools, + tools = [], stt, vad, llm, @@ -455,19 +443,19 @@ function createGetExpirationDateTask({ name: 'confirm_expiration_date', description: 'Call after the user repeats their expiration date for confirmation.', parameters: z.object({ - repeated_expiration_month: z + repeatedExpirationMonth: z .number() .int() .describe('The expiration month repeated by the user'), - repeated_expiration_year: z + repeatedExpirationYear: z .number() .int() .describe('The expiration year repeated by the user'), }), - execute: async ({ repeated_expiration_month, repeated_expiration_year }) => { + execute: async ({ repeatedExpirationMonth, repeatedExpirationYear }) => { if ( - repeated_expiration_month !== expirationMonth || - repeated_expiration_year !== expirationYear + repeatedExpirationMonth !== expirationMonth || + repeatedExpirationYear !== expirationYear ) { task.session.generateReply({ instructions: 'The repeated expiration date does not match, ask the user to try again.', @@ -487,11 +475,11 @@ function createGetExpirationDateTask({ description: "Call to update the card's expiration date. Collect both the numerical month and year.", parameters: z.object({ - expiration_month: z + expirationMonth: z .number() .int() .describe("The numerical expiration month of the card, example: '04' for April"), - expiration_year: z + expirationYear: z .number() .int() .describe( @@ -502,10 +490,7 @@ function createGetExpirationDateTask({ // onEnter — it must produce an asking utterance first. flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, execute: async ( - { - expiration_month: expirationMonth, - expiration_year: expirationYear, - }: { expiration_month: number; expiration_year: number }, + { expirationMonth, expirationYear }: { expirationMonth: number; expirationYear: number }, { ctx }, ) => { if (expirationMonth < 1 || expirationMonth > 12) { @@ -513,21 +498,21 @@ function createGetExpirationDateTask({ instructions: 'The expiration month is invalid, ask the user to repeat the expiration month.', }); - return undefined; + return; } if (expirationYear < 0 || expirationYear > 99) { task.session.generateReply({ instructions: 'The expiration year is invalid, ask the user to repeat the expiration year.', }); - return undefined; + return; } if (isExpired(expirationMonth, expirationYear)) { task.session.generateReply({ instructions: 'The expiration date is in the past, the card is expired. Ask the user to provide another card.', }); - return undefined; + return; } currentExpirationDate = `${String(expirationMonth).padStart(2, '0')}/${String(expirationYear).padStart(2, '0')}`; @@ -536,7 +521,7 @@ function createGetExpirationDateTask({ if (!task.done) { task.complete({ date: currentExpirationDate }); } - return undefined; + return; } const confirmTool = buildConfirmTool(expirationMonth, expirationYear); @@ -562,17 +547,12 @@ function createGetExpirationDateTask({ extraInstructions, ), chatCtx, - turnDetection: turnDetection ?? undefined, - tools: [ - ...(tools ?? []), - updateExpirationDateTool, - declineCardCaptureTool, - restartCardCollectionTool, - ], - stt: stt ?? undefined, - vad: vad ?? undefined, - llm: llm ?? undefined, - tts: tts ?? undefined, + turnDetection, + tools: [...tools, updateExpirationDateTool, declineCardCaptureTool, restartCardCollectionTool], + stt, + vad, + llm, + tts, allowInterruptions, onEnter: async () => { await task.session.generateReply({ @@ -589,12 +569,12 @@ function createGetExpirationDateTask({ export interface GetCreditCardTaskOptions { chatCtx?: ChatContext; - turnDetection?: TurnDetectionMode | null; + turnDetection?: TurnDetectionMode; tools?: readonly ToolContextEntry[]; - stt?: STT | STTModelString | null; - vad?: VAD | null; - llm?: LLM | RealtimeModel | LLMModels | null; - tts?: TTS | TTSModelString | null; + stt?: STT | STTModelString; + vad?: VAD; + llm?: LLM | RealtimeModel | LLMModels; + tts?: TTS | TTSModelString; allowInterruptions?: boolean; /** * Whether to ask the user to confirm each captured card detail. Defaults to confirming on @@ -614,7 +594,7 @@ export interface GetCreditCardTaskOptions { export function createGetCreditCardTask({ chatCtx, turnDetection, - tools, + tools = [], stt, vad, llm, @@ -627,12 +607,12 @@ export function createGetCreditCardTask({ id: 'get_credit_card_task', instructions: '*none*', chatCtx, - turnDetection: turnDetection ?? undefined, - tools: tools ?? [], - stt: stt ?? undefined, - vad: vad ?? undefined, - llm: llm ?? undefined, - tts: tts ?? undefined, + turnDetection, + tools, + stt, + vad, + llm, + tts, allowInterruptions, onEnter: async () => { // Pass chatCtx into both the TaskGroup AND every sub-task. The @@ -797,12 +777,12 @@ export class GetCreditCardTask extends AgentTask { const CARD_NUMBER_BASE_INSTRUCTIONS = ` You are a single step in a broader process of collecting credit card information. You are solely responsible for collecting the credit card number. -{modality_specific} +{modalitySpecific} If the user refuses to provide a credit card number, call decline_card_capture(). If the user wishes to start over the credit card collection process, call restart_card_collection(). Avoid listing out questions with bullet points or numbers, use a natural conversational tone. Never repeat any sensitive information, such as the user's credit card number, back to the user. -{confirmation_instructions} +{confirmationInstructions} `; const CARD_NUMBER_AUDIO_SPECIFIC = ` @@ -818,12 +798,12 @@ Handle input as typed text. Users may type the number with or without spaces or const SECURITY_CODE_BASE_INSTRUCTIONS = ` You are a single step in a broader process of collecting credit card information. You are solely responsible for collecting the user's card's security code. -{modality_specific} +{modalitySpecific} If the user refuses to provide a code, call decline_card_capture(). If the user wishes to start over the card collection process, call restart_card_collection(). Avoid listing out questions with bullet points or numbers, use a natural conversational tone. Never repeat any sensitive information, such as the user's security code, back to the user. -{confirmation_instructions} +{confirmationInstructions} `; const SECURITY_CODE_AUDIO_SPECIFIC = ` @@ -839,12 +819,12 @@ Handle input as typed text. Users will type the security code directly. const EXPIRATION_DATE_BASE_INSTRUCTIONS = ` You are a single step in a broader process of collecting credit card information. You are solely responsible for collecting the user's card's expiration date. -{modality_specific} +{modalitySpecific} If the user refuses to provide a date, call decline_card_capture(). If the user wishes to start over the card collection process, call restart_card_collection(). Avoid listing out questions with bullet points or numbers, use a natural conversational tone. Never repeat any sensitive information, such as the user's expiration date, back to the user. -{confirmation_instructions} +{confirmationInstructions} `; const EXPIRATION_DATE_AUDIO_SPECIFIC = ` diff --git a/agents/src/workflows/dob.ts b/agents/src/workflows/dob.ts index 8b4a4a2f4..bb7c5a95c 100644 --- a/agents/src/workflows/dob.ts +++ b/agents/src/workflows/dob.ts @@ -37,12 +37,12 @@ export interface GetDOBTaskOptions { /** Also capture the (optional) time of birth. Defaults to false. */ includeTime?: boolean; chatCtx?: ChatContext; - turnDetection?: TurnDetectionMode | null; + turnDetection?: TurnDetectionMode; tools?: readonly ToolContextEntry[]; - stt?: STT | STTModelString | null; - vad?: VAD | null; - llm?: LLM | RealtimeModel | LLMModels | null; - tts?: TTS | TTSModelString | null; + stt?: STT | STTModelString; + vad?: VAD; + llm?: LLM | RealtimeModel | LLMModels; + tts?: TTS | TTSModelString; allowInterruptions?: boolean; /** * Whether to ask the user to confirm the captured date of birth. Defaults to confirming on @@ -106,7 +106,7 @@ export function createGetDOBTask({ includeTime = false, chatCtx, turnDetection, - tools, + tools = [], stt, vad, llm, @@ -236,7 +236,7 @@ export function createGetDOBTask({ if (!task.done) { task.complete(buildResult(currentDob)); } - return undefined; + return; } await injectConfirmTool(); @@ -274,7 +274,7 @@ export function createGetDOBTask({ if (!task.done) { task.complete(buildResult(currentDob)); } - return undefined; + return; } if (confirmationRequired(ctx)) { @@ -319,25 +319,25 @@ export function createGetDOBTask({ id: 'get_dob_task', instructions: new Instructions('', { audio: safeRender(BASE_INSTRUCTIONS, { - modality_specific: AUDIO_SPECIFIC, - time_instructions: timeInstructions, - confirmation_instructions: requireConfirmation !== false ? confirmationInstructions : '', - extra_instructions: extraInstructions, + modalitySpecific: AUDIO_SPECIFIC, + timeInstructions, + confirmationInstructions: requireConfirmation !== false ? confirmationInstructions : '', + extraInstructions, }), text: safeRender(BASE_INSTRUCTIONS, { - modality_specific: TEXT_SPECIFIC, - time_instructions: timeInstructions, - confirmation_instructions: requireConfirmation === true ? confirmationInstructions : '', - extra_instructions: extraInstructions, + modalitySpecific: TEXT_SPECIFIC, + timeInstructions, + confirmationInstructions: requireConfirmation === true ? confirmationInstructions : '', + extraInstructions, }), }), chatCtx, - turnDetection: turnDetection ?? undefined, - tools: [...(tools ?? []), updateDobTool, declineTool, ...(includeTime ? [updateTimeTool] : [])], - stt: stt ?? undefined, - vad: vad ?? undefined, - llm: llm ?? undefined, - tts: tts ?? undefined, + turnDetection, + tools: [...tools, updateDobTool, declineTool, ...(includeTime ? [updateTimeTool] : [])], + stt, + vad, + llm, + tts, allowInterruptions, onEnter: async () => { const prompt = includeTime @@ -372,16 +372,16 @@ export class GetDOBTask extends AgentTask { const BASE_INSTRUCTIONS = ` You are only a single step in a broader system, responsible solely for capturing a date of birth. -{modality_specific} -{time_instructions}Call \`update_dob\` at the first opportunity whenever you form a new hypothesis about the date of birth. (before asking any questions or providing any answers.) +{modalitySpecific} +{timeInstructions}Call \`update_dob\` at the first opportunity whenever you form a new hypothesis about the date of birth. (before asking any questions or providing any answers.) Don't invent dates, stick strictly to what the user said. -{confirmation_instructions} +{confirmationInstructions} When reading back dates, use a natural spoken format like 'January fifteenth, nineteen ninety'. If the date is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts: first the month, then the day, then the year. Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. Avoid verbosity by not sharing example dates or formats unless prompted to do so. Do not deviate from the goal of collecting the user's birthday. Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called.\ -{extra_instructions} +{extraInstructions} `; const AUDIO_SPECIFIC = ` diff --git a/agents/src/workflows/email_address.ts b/agents/src/workflows/email_address.ts index 2f094cc0f..60488222b 100644 --- a/agents/src/workflows/email_address.ts +++ b/agents/src/workflows/email_address.ts @@ -25,12 +25,12 @@ export interface GetEmailTaskOptions { */ instructions?: InstructionParts | Instructions | string; chatCtx?: ChatContext; - turnDetection?: TurnDetectionMode | null; + turnDetection?: TurnDetectionMode; tools?: readonly ToolContextEntry[]; - stt?: STT | STTModelString | null; - vad?: VAD | null; - llm?: LLM | RealtimeModel | LLMModels | null; - tts?: TTS | TTSModelString | null; + stt?: STT | STTModelString; + vad?: VAD; + llm?: LLM | RealtimeModel | LLMModels; + tts?: TTS | TTSModelString; allowInterruptions?: boolean; /** * Whether to ask the user to confirm the captured email address. Defaults to confirming on @@ -42,8 +42,6 @@ export interface GetEmailTaskOptions { * it can't silently fill one from the chat context during `onEnter`. */ requireExplicitAsk?: boolean; - /** @deprecated use `instructions.extra` instead */ - extraInstructions?: string; } /** @@ -55,7 +53,7 @@ export function createGetEmailTask({ instructions, chatCtx, turnDetection, - tools, + tools = [], stt, vad, llm, @@ -63,7 +61,6 @@ export function createGetEmailTask({ allowInterruptions, requireConfirmation, requireExplicitAsk = false, - extraInstructions = '', }: GetEmailTaskOptions = {}): AgentTask { let currentEmail = ''; @@ -116,7 +113,7 @@ export function createGetEmailTask({ if (!task.done) { task.complete({ emailAddress: currentEmail }); } - return undefined; // no need to continue the conversation + return; // no need to continue the conversation } const confirmTool = buildConfirmTool(email); @@ -151,28 +148,25 @@ export function createGetEmailTask({ id: 'get_email_task', instructions: resolveWorkflowInstructions({ instructions, - extraInstructions, template: INSTRUCTIONS_TEMPLATE, defaultPersona: PERSONA, - kwargs: { - _modality_specific: new Instructions('', { - audio: AUDIO_SPECIFIC, - text: TEXT_SPECIFIC, - }), - _confirmation: new Instructions('', { - // confirmation is enabled by default for audio, disabled by default for text - audio: requireConfirmation !== false ? CONFIRMATION_INSTRUCTION : '', - text: requireConfirmation === true ? CONFIRMATION_INSTRUCTION : '', - }), - }, + modalitySpecific: new Instructions('', { + audio: AUDIO_SPECIFIC, + text: TEXT_SPECIFIC, + }), + confirmation: new Instructions('', { + // confirmation is enabled by default for audio, disabled by default for text + audio: requireConfirmation !== false ? CONFIRMATION_INSTRUCTION : '', + text: requireConfirmation === true ? CONFIRMATION_INSTRUCTION : '', + }), }), chatCtx, - turnDetection: turnDetection ?? undefined, - tools: [...(tools ?? []), updateEmailTool, declineTool], - stt: stt ?? undefined, - vad: vad ?? undefined, - llm: llm ?? undefined, - tts: tts ?? undefined, + turnDetection, + tools: [...tools, updateEmailTool, declineTool], + stt, + vad, + llm, + tts, allowInterruptions, onEnter: async () => { task.session.generateReply({ instructions: 'Ask the user to provide an email address.' }); @@ -230,11 +224,11 @@ const CONFIRMATION_INSTRUCTION = `Call \`confirm_email_address\` after the user const INSTRUCTIONS_TEMPLATE = `{persona} -{_modality_specific} +{modalitySpecific} Call \`update_email_address\` at the first opportunity whenever you form a new hypothesis about the email. (before asking any questions or providing any answers.) Don't invent new email addresses, stick strictly to what the user said. -{_confirmation} +{confirmation} If the email is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts: first the part before the '@', then the domain—only if needed. Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. diff --git a/agents/src/workflows/name.ts b/agents/src/workflows/name.ts index 79ed95df0..0edfe56a9 100644 --- a/agents/src/workflows/name.ts +++ b/agents/src/workflows/name.ts @@ -27,8 +27,8 @@ export interface GetNameTaskOptions { /** Collect the user's middle name. Defaults to false. */ middleName?: boolean; /** - * Order in which to collect the name parts, using `{first_name}`, `{middle_name}` and - * `{last_name}` placeholders. Defaults to the enabled parts in first/middle/last order. + * Order in which to collect the name parts, using `{firstName}`, `{middleName}` and + * `{lastName}` placeholders. Defaults to the enabled parts in first/middle/last order. */ nameFormat?: string; /** Ask the user to verify the spelling of the captured name. Defaults to false. */ @@ -36,12 +36,12 @@ export interface GetNameTaskOptions { /** Extra instructions appended to the built-in prompt for domain-specific context. */ extraInstructions?: string; chatCtx?: ChatContext; - turnDetection?: TurnDetectionMode | null; + turnDetection?: TurnDetectionMode; tools?: readonly ToolContextEntry[]; - stt?: STT | STTModelString | null; - vad?: VAD | null; - llm?: LLM | RealtimeModel | LLMModels | null; - tts?: TTS | TTSModelString | null; + stt?: STT | STTModelString; + vad?: VAD; + llm?: LLM | RealtimeModel | LLMModels; + tts?: TTS | TTSModelString; allowInterruptions?: boolean; /** * Whether to ask the user to confirm the captured name. Defaults to confirming on audio @@ -56,9 +56,9 @@ export interface GetNameTaskOptions { } interface UpdateNameArgs { - first_name?: string | null; - middle_name?: string | null; - last_name?: string | null; + firstName?: string | null; + middleName?: string | null; + lastName?: string | null; } function cleanNameArg(value: string | null | undefined): string | undefined { @@ -87,12 +87,12 @@ function renderNameFormat( parts: { firstName: string; middleName: string; lastName: string }, ): string { const replacements: Record = { - first_name: parts.firstName, - middle_name: parts.middleName, - last_name: parts.lastName, + firstName: parts.firstName, + middleName: parts.middleName, + lastName: parts.lastName, }; return nameFormat - .replace(/\{(first_name|middle_name|last_name)\}/g, (_match, key: string) => { + .replace(/\{(firstName|middleName|lastName)\}/g, (_match, key: string) => { return replacements[key] ?? ''; }) .trim(); @@ -112,7 +112,7 @@ export function createGetNameTask({ extraInstructions = '', chatCtx, turnDetection, - tools, + tools = [], stt, vad, llm, @@ -130,9 +130,9 @@ export function createGetNameTask({ resolvedNameFormat = nameFormat; } else { const parts: string[] = []; - if (firstName) parts.push('{first_name}'); - if (middleName) parts.push('{middle_name}'); - if (lastName) parts.push('{last_name}'); + if (firstName) parts.push('{firstName}'); + if (middleName) parts.push('{middleName}'); + if (lastName) parts.push('{lastName}'); resolvedNameFormat = parts.join(' '); } @@ -188,21 +188,21 @@ export function createGetNameTask({ name: 'update_name', description: 'Update the name provided by the user.', parameters: z.object({ - first_name: z.string().nullable().optional().describe("The user's first name."), - middle_name: z + firstName: z.string().nullable().optional().describe("The user's first name."), + middleName: z .string() .nullable() .optional() .describe("The user's middle name, if collected."), - last_name: z.string().nullable().optional().describe("The user's last name, if collected."), + lastName: z.string().nullable().optional().describe("The user's last name, if collected."), }), // With requireExplicitAsk, the model can't silent-fill from chatCtx during // onEnter — it must produce an asking utterance first. flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, execute: async (args: UpdateNameArgs, { ctx }) => { - const cleanedFirst = cleanNameArg(args.first_name); - const cleanedMiddle = cleanNameArg(args.middle_name); - const cleanedLast = cleanNameArg(args.last_name); + const cleanedFirst = cleanNameArg(args.firstName); + const cleanedMiddle = cleanNameArg(args.middleName); + const cleanedLast = cleanNameArg(args.lastName); const errors: string[] = []; if (firstName && !cleanedFirst?.trim()) { @@ -249,7 +249,7 @@ export function createGetNameTask({ if (!task.done) { task.complete(buildResult()); } - return undefined; + return; } const confirmTool = buildConfirmTool({ @@ -296,27 +296,27 @@ export function createGetNameTask({ id: 'get_name_task', instructions: new Instructions('', { audio: safeRender(BASE_INSTRUCTIONS, { - name_format: resolvedNameFormat, - modality_specific: AUDIO_SPECIFIC, - spelling_instructions: spellingInstructions, - confirmation_instructions: requireConfirmation !== false ? confirmationInstructions : '', - extra_instructions: extraInstructions, + nameFormat: resolvedNameFormat, + modalitySpecific: AUDIO_SPECIFIC, + spellingInstructions, + confirmationInstructions: requireConfirmation !== false ? confirmationInstructions : '', + extraInstructions, }), text: safeRender(BASE_INSTRUCTIONS, { - name_format: resolvedNameFormat, - modality_specific: TEXT_SPECIFIC, - spelling_instructions: spellingInstructions, - confirmation_instructions: requireConfirmation === true ? confirmationInstructions : '', - extra_instructions: extraInstructions, + nameFormat: resolvedNameFormat, + modalitySpecific: TEXT_SPECIFIC, + spellingInstructions, + confirmationInstructions: requireConfirmation === true ? confirmationInstructions : '', + extraInstructions, }), }), chatCtx, - turnDetection: turnDetection ?? undefined, - tools: [...(tools ?? []), updateNameTool, declineTool], - stt: stt ?? undefined, - vad: vad ?? undefined, - llm: llm ?? undefined, - tts: tts ?? undefined, + turnDetection, + tools: [...tools, updateNameTool, declineTool], + stt, + vad, + llm, + tts, allowInterruptions, onEnter: async () => { task.session.generateReply({ @@ -360,16 +360,16 @@ export class GetNameTask extends AgentTask { const BASE_INSTRUCTIONS = ` You are only a single step in a broader system, responsible solely for capturing the user's name. -You need to naturally collect the name parts in this order: {name_format}. -{modality_specific} -{spelling_instructions}Call \`update_name\` at the first opportunity whenever you form a new hypothesis about the name. (before asking any questions or providing any answers.) +You need to naturally collect the name parts in this order: {nameFormat}. +{modalitySpecific} +{spellingInstructions}Call \`update_name\` at the first opportunity whenever you form a new hypothesis about the name. (before asking any questions or providing any answers.) Don't invent names, stick strictly to what the user said. -{confirmation_instructions} +{confirmationInstructions} If the name is unclear or it takes too much back-and-forth, prompt for each name part separately. Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. Avoid verbosity by not sharing example names or spellings unless prompted to do so. Do not deviate from the goal of collecting the user's name. Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called.\ -{extra_instructions} +{extraInstructions} `; const AUDIO_SPECIFIC = ` diff --git a/agents/src/workflows/phone_number.ts b/agents/src/workflows/phone_number.ts index 3ae7ccb17..306f3d87d 100644 --- a/agents/src/workflows/phone_number.ts +++ b/agents/src/workflows/phone_number.ts @@ -23,12 +23,12 @@ export interface GetPhoneNumberTaskOptions { /** Extra instructions appended to the built-in prompt for domain-specific context. */ extraInstructions?: string; chatCtx?: ChatContext; - turnDetection?: TurnDetectionMode | null; + turnDetection?: TurnDetectionMode; tools?: readonly ToolContextEntry[]; - stt?: STT | STTModelString | null; - vad?: VAD | null; - llm?: LLM | RealtimeModel | LLMModels | null; - tts?: TTS | TTSModelString | null; + stt?: STT | STTModelString; + vad?: VAD; + llm?: LLM | RealtimeModel | LLMModels; + tts?: TTS | TTSModelString; allowInterruptions?: boolean; /** * Whether to ask the user to confirm the captured phone number. Defaults to confirming on @@ -51,7 +51,7 @@ export function createGetPhoneNumberTask({ extraInstructions = '', chatCtx, turnDetection, - tools, + tools = [], stt, vad, llm, @@ -95,14 +95,14 @@ export function createGetPhoneNumberTask({ name: 'update_phone_number', description: 'Update the phone number provided by the user.', parameters: z.object({ - phone_number: z + phoneNumber: z .string() .describe('The phone number provided by the user, digits only with optional leading +'), }), // With requireExplicitAsk, the model can't silent-fill from chatCtx during // onEnter — it must produce an asking utterance first. flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, - execute: async ({ phone_number: phoneNumber }: { phone_number: string }, { ctx }) => { + execute: async ({ phoneNumber }: { phoneNumber: string }, { ctx }) => { const cleaned = phoneNumber.trim().replace(/[\s\-().]+/g, ''); if (!PHONE_REGEX.test(cleaned)) { @@ -115,7 +115,7 @@ export function createGetPhoneNumberTask({ if (!task.done) { task.complete({ phoneNumber: currentPhoneNumber }); } - return undefined; // no need to continue the conversation + return; // no need to continue the conversation } const confirmTool = buildConfirmTool(cleaned); @@ -150,23 +150,23 @@ export function createGetPhoneNumberTask({ id: 'get_phone_number_task', instructions: new Instructions('', { audio: safeRender(BASE_INSTRUCTIONS, { - modality_specific: AUDIO_SPECIFIC, - confirmation_instructions: requireConfirmation !== false ? confirmationInstructions : '', - extra_instructions: extraInstructions, + modalitySpecific: AUDIO_SPECIFIC, + confirmationInstructions: requireConfirmation !== false ? confirmationInstructions : '', + extraInstructions, }), text: safeRender(BASE_INSTRUCTIONS, { - modality_specific: TEXT_SPECIFIC, - confirmation_instructions: requireConfirmation === true ? confirmationInstructions : '', - extra_instructions: extraInstructions, + modalitySpecific: TEXT_SPECIFIC, + confirmationInstructions: requireConfirmation === true ? confirmationInstructions : '', + extraInstructions, }), }), chatCtx, - turnDetection: turnDetection ?? undefined, - tools: [...(tools ?? []), updatePhoneNumberTool, declineTool], - stt: stt ?? undefined, - vad: vad ?? undefined, - llm: llm ?? undefined, - tts: tts ?? undefined, + turnDetection, + tools: [...tools, updatePhoneNumberTool, declineTool], + stt, + vad, + llm, + tts, allowInterruptions, onEnter: async () => { task.session.generateReply({ instructions: 'Ask the user to provide their phone number.' }); @@ -198,16 +198,16 @@ export class GetPhoneNumberTask extends AgentTask { const BASE_INSTRUCTIONS = ` You are only a single step in a broader system, responsible solely for capturing a phone number. -{modality_specific} +{modalitySpecific} Call \`update_phone_number\` at the first opportunity whenever you form a new hypothesis about the phone number. (before asking any questions or providing any answers.) Don't invent phone numbers, stick strictly to what the user said. -{confirmation_instructions} +{confirmationInstructions} If the number is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts: first the area code, then the remaining digits. Never repeat the phone number back to the user as a single block of digits. Read it back in groups. Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. Avoid verbosity by not sharing example phone numbers or formats unless prompted to do so. Do not deviate from the goal of collecting the user's phone number. Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called.\ -{extra_instructions} +{extraInstructions} `; const AUDIO_SPECIFIC = ` diff --git a/agents/src/workflows/utils.ts b/agents/src/workflows/utils.ts index b0189939f..2d435d00a 100644 --- a/agents/src/workflows/utils.ts +++ b/agents/src/workflows/utils.ts @@ -2,7 +2,6 @@ // // SPDX-License-Identifier: Apache-2.0 import { Instructions, isInstructions } from '../llm/chat_context.js'; -import { log } from '../log.js'; /** * Customizable instruction sections for built-in workflow tasks. @@ -27,17 +26,14 @@ export interface InstructionParts { */ export function resolveWorkflowInstructions(options: { instructions?: InstructionParts | Instructions | string; - /** @deprecated legacy `extraInstructions` option, ignored when `instructions` is provided. */ - extraInstructions?: string; template: string; defaultPersona: string; - kwargs?: Record; + /** Fills the template's `{modalitySpecific}` slot with per-modality (audio/text) guidance. */ + modalitySpecific: Instructions; + /** Fills the template's `{confirmation}` slot with the per-modality confirmation instruction. */ + confirmation: Instructions; }): string | Instructions { - const { instructions, extraInstructions = '', template, defaultPersona, kwargs = {} } = options; - - if (instructions !== undefined && extraInstructions) { - log().warn('`extraInstructions` will be ignored when `instructions` is provided'); - } + const { instructions, template, defaultPersona, modalitySpecific, confirmation } = options; // A full instruction string or Instructions replaces the built-in prompt entirely. if (typeof instructions === 'string' || isInstructions(instructions)) { @@ -45,12 +41,13 @@ export function resolveWorkflowInstructions(options: { } // No instructions or an `InstructionParts` override: fill the built-in template. - const parts: InstructionParts = instructions ?? { extra: extraInstructions }; + // Unset preserves the built-in default; an explicit empty string removes the section. + const parts: InstructionParts = instructions ?? {}; return Instructions.resolveTemplate(template, { - // Unset preserves the built-in default; an explicit empty string removes the section. - persona: parts.persona !== undefined ? parts.persona : defaultPersona, - extra: parts.extra !== undefined ? parts.extra : '', - ...kwargs, + persona: parts.persona ?? defaultPersona, + extra: parts.extra ?? '', + modalitySpecific, + confirmation, }); }