From 9e1f52a594a28f8056028bfd6e77c9a4ccb304da Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 8 Jul 2026 08:26:32 +0000 Subject: [PATCH 1/2] feat(workflows): port built-in data-capture workflows from python Ports the built-in workflow tasks from python livekit-agents (livekit/agents beta/workflows) to the stable workflows namespace, following the warm_transfer.ts pattern (functional core + thin class wrapper): - GetEmailTask / createGetEmailTask (email_address.ts) - GetNameTask / createGetNameTask (name.ts) - GetPhoneNumberTask / createGetPhoneNumberTask (phone_number.ts) - GetDOBTask / createGetDOBTask (dob.ts) - GetAddressTask / createGetAddressTask (address.ts) - GetDtmfTask / createGetDtmfTask (dtmf_inputs.ts) - GetCreditCardTask / createGetCreditCardTask (credit_card.ts), composed of internal card number / expiration date / security code sub-tasks plus the cardholder name task via TaskGroup workflows/utils.ts gains the DtmfEvent enum with formatDtmf / dtmfEventToCode helpers and a shared resolveWorkflowInstructions that mirrors python's WorkflowInstructions.resolve on top of Instructions.resolveTemplate. TaskGroup.add now accepts factories returning typed AgentTask so typed sub-tasks compose without casts. Notable JS adaptations: time values are milliseconds (dtmfInputTimeout defaults to 4000ms), GetDOBResult uses structured DateOfBirth / TimeOfBirth objects instead of datetime.date/time, and DTMF input uses rtc-node's RoomEvent.DtmfReceived. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q3ZnFWr54AZysyyf4g5Dyd --- .changeset/port-builtin-workflows.md | 5 + agents/src/workflows/address.ts | 269 +++++++++ agents/src/workflows/credit_card.ts | 778 ++++++++++++++++++++++++++ agents/src/workflows/dob.ts | 406 ++++++++++++++ agents/src/workflows/dtmf_inputs.ts | 286 ++++++++++ agents/src/workflows/email_address.ts | 244 ++++++++ agents/src/workflows/index.ts | 48 +- agents/src/workflows/name.ts | 394 +++++++++++++ agents/src/workflows/phone_number.ts | 233 ++++++++ agents/src/workflows/task_group.ts | 6 +- agents/src/workflows/utils.ts | 76 ++- 11 files changed, 2741 insertions(+), 4 deletions(-) create mode 100644 .changeset/port-builtin-workflows.md create mode 100644 agents/src/workflows/address.ts create mode 100644 agents/src/workflows/credit_card.ts create mode 100644 agents/src/workflows/dob.ts create mode 100644 agents/src/workflows/dtmf_inputs.ts create mode 100644 agents/src/workflows/email_address.ts create mode 100644 agents/src/workflows/name.ts create mode 100644 agents/src/workflows/phone_number.ts diff --git a/.changeset/port-builtin-workflows.md b/.changeset/port-builtin-workflows.md new file mode 100644 index 000000000..3b1a64503 --- /dev/null +++ b/.changeset/port-builtin-workflows.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Port the built-in data-capture workflows from python livekit-agents (`beta/workflows`) to the stable `workflows` namespace: `GetEmailTask`, `GetNameTask`, `GetPhoneNumberTask`, `GetDOBTask`, `GetAddressTask`, `GetDtmfTask`, and `GetCreditCardTask` (with `create*Task` functional variants). Includes the `DtmfEvent` enum with `formatDtmf`/`dtmfEventToCode` helpers, and `TaskGroup.add` now accepts factories for typed `AgentTask` results. diff --git a/agents/src/workflows/address.ts b/agents/src/workflows/address.ts new file mode 100644 index 000000000..5440771c8 --- /dev/null +++ b/agents/src/workflows/address.ts @@ -0,0 +1,269 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; +import { type InstructionParts, resolveWorkflowInstructions } from './utils.js'; + +export interface GetAddressResult { + address: string; +} + +export interface GetAddressTaskOptions { + /** + * Instructions for the address capture prompt. Pass a full string or {@link Instructions} to + * replace the built-in prompt entirely, or {@link InstructionParts} to override individual + * sections (e.g. `persona`) while keeping the built-in template. + */ + instructions?: InstructionParts | Instructions | string; + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode | null; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString | null; + vad?: VAD | null; + llm?: LLM | RealtimeModel | LLMModels | null; + tts?: TTS | TTSModelString | null; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm the captured address. Defaults to confirming on audio + * input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** + * When true, the model must produce an asking utterance before recording an address — it + * can't silently fill one from the chat context during `onEnter`. + */ + requireExplicitAsk?: boolean; + /** @deprecated use `instructions.extra` instead */ + extraInstructions?: string; +} + +interface UpdateAddressArgs { + street_address: string; + unit_number: string; + locality: string; + country: string; +} + +/** + * Build an {@link AgentTask} that collects a postal address from the user. + * + * This is the functional core; {@link GetAddressTask} is a thin class wrapper over it. + */ +export function createGetAddressTask({ + instructions, + chatCtx, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + requireExplicitAsk = false, + extraInstructions = '', +}: GetAddressTaskOptions = {}): AgentTask { + let currentAddress = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + // confirm tool is only injected after update_address is called, + // preventing the LLM from hallucinating a confirmation without user input + const buildConfirmTool = (address: string) => + tool({ + name: 'confirm_address', + description: 'Call after the user confirms the address is correct.', + execute: async () => { + if (address !== currentAddress) { + task.session.generateReply({ + instructions: + 'The address has changed since confirmation was requested, ask the user to confirm the updated address.', + }); + return; + } + + if (!task.done) { + task.complete({ address }); + } + }, + }); + + const updateAddressTool = tool({ + name: 'update_address', + description: 'Update the address provided by the user.', + parameters: z.object({ + street_address: z + .string() + .describe( + 'Dependent on country, may include fields like house number, street name, block, or district', + ), + unit_number: z + .string() + .describe( + "The unit number, for example Floor 1 or Apartment 12. If there is no unit number, return ''", + ), + locality: z + .string() + .describe('Dependent on country, may include fields like city, zip code, or province'), + country: z.string().describe('The country the user lives in spelled out fully'), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async (args: UpdateAddressArgs, { ctx }) => { + const { street_address: streetAddress, unit_number: unitNumber, locality, country } = args; + const addressFields = unitNumber.trim() + ? [streetAddress, unitNumber, locality, country] + : [streetAddress, locality, country]; + const address = addressFields.join(' '); + currentAddress = address; + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete({ address: currentAddress }); + } + return undefined; + } + + const confirmTool = buildConfirmTool(address); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_address'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + `The address has been updated to ${address}\n` + + `Repeat the address field by field: ${JSON.stringify(addressFields)} if needed\n` + + `Prompt the user for confirmation, do not call \`confirm_address\` directly` + ); + }, + }); + + const declineTool = tool({ + name: 'decline_address_capture', + description: 'Handles the case when the user explicitly declines to provide an address.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide the address'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }) => { + if (!task.done) { + task.complete(new ToolError(`couldn't get the address: ${reason}`)); + } + }, + }); + + const task = AgentTask.create({ + id: 'get_address_task', + instructions: resolveWorkflowInstructions({ + instructions, + extraInstructions, + template: INSTRUCTIONS_TEMPLATE, + defaultPersona: PERSONA, + kwargs: { + _modality_specific: new Instructions('', { + audio: AUDIO_SPECIFIC, + text: TEXT_SPECIFIC, + }), + _confirmation: new Instructions('', { + // confirmation is enabled by default for audio, disabled by default for text + audio: requireConfirmation !== false ? CONFIRMATION_INSTRUCTION : '', + text: requireConfirmation === true ? CONFIRMATION_INSTRUCTION : '', + }), + }, + }), + chatCtx, + turnDetection: turnDetection ?? undefined, + tools: [...(tools ?? []), updateAddressTool, declineTool], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, + onEnter: async () => { + task.session.generateReply({ instructions: 'Ask the user to provide their address.' }); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetAddressTask}, preserving the + * `new GetAddressTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetAddressTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetAddressTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetAddressTask. + super({ instructions: '' }); + this.#task = createGetAddressTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +// instructions +const PERSONA = + 'You are only a single step in a broader system, responsible solely for capturing an address.'; + +const AUDIO_SPECIFIC = `You will be handling addresses from any country. +Expect that users will say address in different formats with fields filled like: +- 'street_address': '450 SOUTH MAIN ST', 'unit_number': 'FLOOR 2', 'locality': 'SALT LAKE CITY UT 84101', 'country': 'UNITED STATES', +- 'street_address': '123 MAPLE STREET', 'unit_number': 'APARTMENT 10', 'locality': 'OTTAWA ON K1A 0B1', 'country': 'CANADA', +- 'street_address': 'GUOMAO JIE 3 HAO, CHAOYANG QU', 'unit_number': 'GUOMAO DA SHA 18 LOU 101 SHI', 'locality': 'BEIJING SHI 100000', 'country': 'CHINA', +- 'street_address': '5 RUE DE L’ANCIENNE COMÉDIE', 'unit_number': 'APP C4', 'locality': '75006 PARIS', 'country': 'FRANCE', +- 'street_address': 'PLOT 10, NEHRU ROAD', 'unit_number': 'OFFICE 403, 4TH FLOOR', 'locality': 'VILE PARLE (E), MUMBAI MAHARASHTRA 400099', 'country': 'INDIA', +Normalize common spoken patterns silently: +- Convert words like 'dash' and 'apostrophe' into symbols: \`-\`, \`'\`. +- Convert spelled out numbers like 'six' and 'seven' into numerals: \`6\`, \`7\`. +- Recognize patterns where users speak their address field followed by spelling: e.g., 'guomao g u o m a o'. +- Filter out filler words or hesitations. +- Recognize when there may be accents on certain letters if explicitly said or common in the location specified. Be sure to verify the correct accents if existent. +Don't mention corrections. Treat inputs as possibly imperfect but fix them silently. +When reading a numerical ordinal suffix (st, nd, rd, th), the number must be verbally expanded into its full, correctly pronounced word form. +Do not read the number and the suffix letters separately. +Confirm postal codes by reading them out digit-by-digit as a sequence of single numbers. Do not read them as cardinal numbers. +For example, read 90210 as 'nine zero two one zero.' +Avoid using bullet points and parenthese in any responses. +Spell out the address letter-by-letter when applicable, such as street names and provinces, especially when the user spells it out initially.`; + +const TEXT_SPECIFIC = `You will be handling addresses from any country. +Expect users to type their address directly. +If the address looks almost correct but has minor issues (e.g. missing country or postal code), prompt for clarification.`; + +const CONFIRMATION_INSTRUCTION = `Call \`confirm_address\` after the user confirmed the address is correct.`; + +const INSTRUCTIONS_TEMPLATE = `{persona} + +{_modality_specific} + +Call \`update_address\` at the first opportunity whenever you form a new hypothesis about the address. (before asking any questions or providing any answers.) +Don't invent new addresses, stick strictly to what the user said. +{_confirmation} +If the address is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts in this order: street address, unit number if applicable, locality, and country. + +Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. +Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called. + +{extra} +`; diff --git a/agents/src/workflows/credit_card.ts b/agents/src/workflows/credit_card.ts new file mode 100644 index 000000000..a4b43b34a --- /dev/null +++ b/agents/src/workflows/credit_card.ts @@ -0,0 +1,778 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import { isToolError } from '../llm/tool_context.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import { safeRender } from '../utils.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; +import { type GetNameResult, createGetNameTask } from './name.js'; +import { TaskGroup } from './task_group.js'; + +const CARD_ISSUERS_LOOKUP: Record = { + '3': 'American Express', + '4': 'Visa', + '5': 'Mastercard', + '6': 'Discover', +}; + +export interface GetCreditCardResult { + cardholderName: string; + issuer: string; + cardNumber: string; + securityCode: string; + expirationDate: string; +} + +interface GetCardNumberResult { + issuer: string; + cardNumber: string; +} + +interface GetSecurityCodeResult { + securityCode: string; +} + +interface GetExpirationDateResult { + date: string; +} + +export class CardCaptureDeclinedError extends ToolError { + readonly reason: string; + + constructor(reason: string) { + super(`couldn't get the card details: ${reason}`); + this.reason = reason; + } +} + +export class CardCollectionRestartError extends ToolError { + readonly reason: string; + + constructor(reason: string) { + super(`starting over: ${reason}`); + this.reason = reason; + } +} + +const declineCardCaptureTool = tool({ + name: 'decline_card_capture', + description: + 'Handles the case when the user explicitly declines to provide a detail for their card information.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide card information'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }, { ctx }) => { + const task = ctx.session.currentAgent; + if (task instanceof AgentTask && !task.done) { + task.complete(new CardCaptureDeclinedError(reason)); + } + }, +}); + +const restartCardCollectionTool = tool({ + name: 'restart_card_collection', + description: + 'Handles the case when the user wishes to start over the card information collection process and validate a new card.', + parameters: z.object({ + reason: z.string().describe('A short explanation of why the user wishes to start over'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }, { ctx }) => { + const task = ctx.session.currentAgent; + if (task instanceof AgentTask && !task.done) { + task.complete(new CardCollectionRestartError(reason)); + } + }, +}); + +/** Validates a card number via the Luhn algorithm. */ +function validateCardNumber(cardNumber: string): boolean { + if (!cardNumber || !/^\d+$/.test(cardNumber)) { + return false; + } + let totalSum = 0; + + const reversedNumber = [...cardNumber].reverse(); + for (let index = 0; index < reversedNumber.length; index++) { + const digit = Number(reversedNumber[index]); + if (index % 2 === 1) { + const doubledDigit = digit * 2; + totalSum += doubledDigit > 9 ? doubledDigit - 9 : doubledDigit; + } else { + totalSum += digit; + } + } + + return totalSum % 10 === 0; +} + +interface SubTaskOptions { + chatCtx?: ChatContext; + requireConfirmation?: boolean; + requireExplicitAsk?: boolean; + extraInstructions?: string; +} + +function buildSubTaskInstructions( + baseInstructions: string, + audioSpecific: string, + textSpecific: string, + confirmationInstructions: string, + requireConfirmation: boolean | undefined, + extraInstructions: string, +): Instructions { + const extraSuffix = extraInstructions ? `\n${extraInstructions}` : ''; + return new Instructions('', { + audio: + safeRender(baseInstructions, { + modality_specific: audioSpecific, + confirmation_instructions: requireConfirmation !== false ? confirmationInstructions : '', + }) + extraSuffix, + text: + safeRender(baseInstructions, { + modality_specific: textSpecific, + confirmation_instructions: requireConfirmation === true ? confirmationInstructions : '', + }) + extraSuffix, + }); +} + +function createGetCardNumberTask({ + chatCtx, + requireConfirmation, + requireExplicitAsk = false, + extraInstructions = '', +}: SubTaskOptions = {}): AgentTask { + let currentCardNumber = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildConfirmTool = (cardNumber: string) => + tool({ + name: 'confirm_card_number', + description: 'Call after the user repeats their card number for confirmation.', + parameters: z.object({ + repeated_card_number: z + .string() + .describe('The card number repeated by the user as a string'), + }), + execute: async ({ repeated_card_number: repeatedCardNumber }) => { + repeatedCardNumber = repeatedCardNumber.replace(/\D/g, ''); + if (repeatedCardNumber !== cardNumber) { + task.session.generateReply({ + instructions: 'The repeated card number does not match, ask the user to try again.', + }); + return; + } + + if (!validateCardNumber(cardNumber)) { + task.session.generateReply({ + instructions: + 'The card number is not valid, ask the user if they made a mistake or to provide another card.', + }); + } else { + const issuer = CARD_ISSUERS_LOOKUP[cardNumber[0]!] ?? 'Other'; + if (!task.done) { + task.complete({ issuer, cardNumber }); + } + } + }, + }); + + const updateCardNumberTool = tool({ + name: 'update_card_number', + description: + "Call to record the user's card number. Only call once the entire number has been given, do not call in increments.", + parameters: z.object({ + card_number: z + .string() + .describe('The credit card number as a string with no dashes or spaces'), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ({ card_number: rawCardNumber }: { card_number: string }, { ctx }) => { + const cardNumber = rawCardNumber.replace(/\D/g, ''); + if (cardNumber.length < 13 || cardNumber.length > 19) { + task.session.generateReply({ + instructions: + 'The length of the card number is invalid, ask the user to repeat their card number.', + }); + return undefined; + } + + currentCardNumber = cardNumber; + + if (!confirmationRequired(ctx)) { + if (!validateCardNumber(currentCardNumber)) { + task.session.generateReply({ + instructions: + 'The card number is not valid, ask the user if they made a mistake or to provide another card.', + }); + } else { + const issuer = CARD_ISSUERS_LOOKUP[currentCardNumber[0]!] ?? 'Other'; + if (!task.done) { + task.complete({ issuer, cardNumber: currentCardNumber }); + } + } + return undefined; + } + + const confirmTool = buildConfirmTool(cardNumber); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_card_number'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + 'The card number has been updated.\n' + + 'Ask them to repeat the number, do not repeat the number back to them.\n' + ); + }, + }); + + const task = AgentTask.create({ + id: 'get_card_number_task', + instructions: buildSubTaskInstructions( + CARD_NUMBER_BASE_INSTRUCTIONS, + CARD_NUMBER_AUDIO_SPECIFIC, + CARD_NUMBER_TEXT_SPECIFIC, + 'Call `confirm_card_number` once the user has repeated their card number.', + requireConfirmation, + extraInstructions, + ), + chatCtx, + tools: [updateCardNumberTool, declineCardCaptureTool, restartCardCollectionTool], + onEnter: async () => { + await task.session.generateReply({ + instructions: + "Get the user's credit card number. First scan the conversation - if a " + + 'credit card number was already given (e.g. the user volunteered it ' + + 'before the task started), use it via update_card_number rather than ' + + 're-asking. Only ask fresh when no credit card number is in the ' + + 'conversation yet.', + }); + }, + }); + + return task; +} + +function createGetSecurityCodeTask({ + chatCtx, + requireConfirmation, + requireExplicitAsk = false, + extraInstructions = '', +}: SubTaskOptions = {}): AgentTask { + let currentSecurityCode = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildConfirmTool = (securityCode: string) => + tool({ + name: 'confirm_security_code', + description: 'Call after the user repeats their security code for confirmation.', + parameters: z.object({ + repeated_security_code: z.string().describe('The security code repeated by the user'), + }), + execute: async ({ repeated_security_code: repeatedSecurityCode }) => { + if (repeatedSecurityCode.trim() !== securityCode) { + task.session.generateReply({ + instructions: 'The repeated security code does not match, ask the user to try again.', + }); + return; + } + + if (!task.done) { + task.complete({ securityCode }); + } + }, + }); + + const updateSecurityCodeTool = tool({ + name: 'update_security_code', + description: "Call to update the card's security code.", + parameters: z.object({ + security_code: z + .string() + .describe("The card's security code (3-4 digits, may have leading zeros)."), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ({ security_code: securityCode }: { security_code: string }, { ctx }) => { + const stripped = securityCode.trim(); + if (!/^\d+$/.test(stripped) || stripped.length < 3 || stripped.length > 4) { + task.session.generateReply({ + instructions: + "The security code's length is invalid, ask the user to repeat or to provide a new card and start over.", + }); + return undefined; + } + + currentSecurityCode = stripped; + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete({ securityCode: currentSecurityCode }); + } + return undefined; + } + + const confirmTool = buildConfirmTool(stripped); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_security_code'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + 'The security code has been updated.\n' + + 'Do not repeat the security code back to the user, ask them to repeat themselves.\n' + + 'Call `confirm_security_code` once the user confirms, do not call it preemptively.\n' + ); + }, + }); + + const task = AgentTask.create({ + id: 'get_security_code_task', + instructions: buildSubTaskInstructions( + SECURITY_CODE_BASE_INSTRUCTIONS, + SECURITY_CODE_AUDIO_SPECIFIC, + SECURITY_CODE_TEXT_SPECIFIC, + 'Call `confirm_security_code` once the user has repeated their security code.', + requireConfirmation, + extraInstructions, + ), + chatCtx, + tools: [updateSecurityCodeTool, declineCardCaptureTool, restartCardCollectionTool], + onEnter: async () => { + await task.session.generateReply({ + instructions: + "Get the user's card security code. First scan the conversation - if a " + + 'code was already given, use it via update_security_code rather than ' + + 're-asking. Only ask fresh when no code is in the conversation yet.', + }); + }, + }); + + return task; +} + +function createGetExpirationDateTask({ + chatCtx, + requireConfirmation, + requireExplicitAsk = false, + extraInstructions = '', +}: SubTaskOptions = {}): AgentTask { + let currentExpirationDate = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const isExpired = (month: number, year: number): boolean => { + const today = new Date(); + const fullYear = 2000 + year; + return ( + fullYear < today.getFullYear() || + (fullYear === today.getFullYear() && month < today.getMonth() + 1) + ); + }; + + const buildConfirmTool = (expirationMonth: number, expirationYear: number) => { + const expirationDate = currentExpirationDate; + + return tool({ + name: 'confirm_expiration_date', + description: 'Call after the user repeats their expiration date for confirmation.', + parameters: z.object({ + repeated_expiration_month: z + .number() + .int() + .describe('The expiration month repeated by the user'), + repeated_expiration_year: z + .number() + .int() + .describe('The expiration year repeated by the user'), + }), + execute: async ({ repeated_expiration_month, repeated_expiration_year }) => { + if ( + repeated_expiration_month !== expirationMonth || + repeated_expiration_year !== expirationYear + ) { + task.session.generateReply({ + instructions: 'The repeated expiration date does not match, ask the user to try again.', + }); + return; + } + + if (!task.done) { + task.complete({ date: expirationDate }); + } + }, + }); + }; + + const updateExpirationDateTool = tool({ + name: 'update_expiration_date', + description: + "Call to update the card's expiration date. Collect both the numerical month and year.", + parameters: z.object({ + expiration_month: z + .number() + .int() + .describe("The numerical expiration month of the card, example: '04' for April"), + expiration_year: z + .number() + .int() + .describe( + "The numerical expiration year of the card shortened to the last two digits, for example, '35' for 2035", + ), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ( + { + expiration_month: expirationMonth, + expiration_year: expirationYear, + }: { expiration_month: number; expiration_year: number }, + { ctx }, + ) => { + if (expirationMonth < 1 || expirationMonth > 12) { + task.session.generateReply({ + instructions: + 'The expiration month is invalid, ask the user to repeat the expiration month.', + }); + return undefined; + } + if (expirationYear < 0 || expirationYear > 99) { + task.session.generateReply({ + instructions: + 'The expiration year is invalid, ask the user to repeat the expiration year.', + }); + return undefined; + } + if (isExpired(expirationMonth, expirationYear)) { + task.session.generateReply({ + instructions: + 'The expiration date is in the past, the card is expired. Ask the user to provide another card.', + }); + return undefined; + } + + currentExpirationDate = `${String(expirationMonth).padStart(2, '0')}/${String(expirationYear).padStart(2, '0')}`; + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete({ date: currentExpirationDate }); + } + return undefined; + } + + const confirmTool = buildConfirmTool(expirationMonth, expirationYear); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_expiration_date'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + 'The expiration date has been updated.\n' + + 'Do not repeat the expiration date back to the user, ask them to repeat themselves.\n' + + 'Call `confirm_expiration_date` once the user confirms, do not call it preemptively.\n' + ); + }, + }); + + const task = AgentTask.create({ + id: 'get_expiration_date_task', + instructions: buildSubTaskInstructions( + EXPIRATION_DATE_BASE_INSTRUCTIONS, + EXPIRATION_DATE_AUDIO_SPECIFIC, + EXPIRATION_DATE_TEXT_SPECIFIC, + 'Call `confirm_expiration_date` once the user has repeated their expiration date.', + requireConfirmation, + extraInstructions, + ), + chatCtx, + tools: [updateExpirationDateTool, declineCardCaptureTool, restartCardCollectionTool], + onEnter: async () => { + await task.session.generateReply({ + instructions: + "Get the user's card expiration date. First scan the conversation - if " + + 'an expiration date was already given, use it via update_expiration_date ' + + 'rather than re-asking. Only ask fresh when no date is in the conversation yet.', + }); + }, + }); + + return task; +} + +export interface GetCreditCardTaskOptions { + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode | null; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString | null; + vad?: VAD | null; + llm?: LLM | RealtimeModel | LLMModels | null; + tts?: TTS | TTSModelString | null; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm each captured card detail. Defaults to confirming on + * audio input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** Extra instructions appended to each sub-task's prompt for domain-specific context. */ + extraInstructions?: string; +} + +/** + * Build an {@link AgentTask} that collects the user's full credit card details (number, + * expiration date, security code, and cardholder name) via a {@link TaskGroup} of sub-tasks. + * + * This is the functional core; {@link GetCreditCardTask} is a thin class wrapper over it. + */ +export function createGetCreditCardTask({ + chatCtx, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + extraInstructions = '', +}: GetCreditCardTaskOptions = {}): AgentTask { + const task = AgentTask.create({ + id: 'get_credit_card_task', + instructions: '*none*', + chatCtx, + turnDetection: turnDetection ?? undefined, + tools: tools ?? [], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, + onEnter: async () => { + // Pass chatCtx into both the TaskGroup AND every sub-task. The + // TaskGroup overwrites each sub-task's chatCtx with its own (see + // TaskGroup.onEnter) - without seeding the TaskGroup, sub-tasks + // would run with empty context. + const ctx = task.chatCtx; + // Role hint for the cardholder sub-task. With IGNORE_ON_ENTER on + // update_name (via requireExplicitAsk=true), the model is + // structurally forced to ask before recording. The extra text + // just makes sure the *question* anchors to the card. + let cardholderExtra = + 'You are collecting the name on the credit card (the cardholder). ' + + 'When you ask the user to confirm a candidate name from earlier in ' + + 'the conversation, anchor the question to the card or cardholder ' + + "so the user knows which name you mean - not just 'is it [name]?' " + + 'in the abstract.'; + if (extraInstructions) { + cardholderExtra = `${extraInstructions}\n\n${cardholderExtra}`; + } + + while (!task.done) { + // Order: number first (most natural for the caller to give + // when asked for "card details"), then expiry and security + // code, then the cardholder name LAST. The name most often + // pre-fills from chatCtx (same as the booking name) so + // leaving it for the end avoids the failure mode where the + // caller's first response (typically the digits) gets crammed + // into update_name. + const taskGroup = new TaskGroup({ chatCtx: ctx }); + taskGroup.add( + () => + createGetCardNumberTask({ + chatCtx: ctx, + requireConfirmation, + extraInstructions, + }), + { + id: 'card_number_task', + description: "Collects the user's card number", + }, + ); + taskGroup.add( + () => + createGetExpirationDateTask({ + chatCtx: ctx, + requireConfirmation, + extraInstructions, + }), + { + id: 'expiration_date_task', + description: "Collects the card's expiration date", + }, + ); + taskGroup.add( + () => + createGetSecurityCodeTask({ + chatCtx: ctx, + requireConfirmation, + extraInstructions, + }), + { + id: 'security_code_task', + description: "Collects the card's security code", + }, + ); + taskGroup.add( + () => + createGetNameTask({ + lastName: true, + chatCtx: ctx, + extraInstructions: cardholderExtra, + requireConfirmation, + // The cardholder may differ from the caller or any guest + // mentioned earlier in chatCtx. Apply IGNORE_ON_ENTER on + // update_name so the model must produce an asking turn + // rather than silently filling from chatCtx. + requireExplicitAsk: true, + }), + { + id: 'cardholder_name_task', + description: "Collects the cardholder's full name", + }, + ); + + try { + const results = await taskGroup.run(); + const nameResult = results.taskResults['cardholder_name_task'] as GetNameResult; + const cardNumberResult = results.taskResults['card_number_task'] as GetCardNumberResult; + const securityCodeResult = results.taskResults[ + 'security_code_task' + ] as GetSecurityCodeResult; + const expirationDateResult = results.taskResults[ + 'expiration_date_task' + ] as GetExpirationDateResult; + + task.complete({ + cardholderName: `${nameResult.firstName} ${nameResult.lastName}`, + issuer: cardNumberResult.issuer, + cardNumber: cardNumberResult.cardNumber, + securityCode: securityCodeResult.securityCode, + expirationDate: expirationDateResult.date, + }); + } catch (e) { + if (e instanceof CardCollectionRestartError) { + continue; + } + if (e instanceof CardCaptureDeclinedError || isToolError(e)) { + if (!task.done) { + task.complete(e as ToolError); + } + return; + } + throw e; + } + } + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetCreditCardTask}, preserving the + * `new GetCreditCardTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetCreditCardTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetCreditCardTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetCreditCardTask. + super({ instructions: '' }); + this.#task = createGetCreditCardTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +const CARD_NUMBER_BASE_INSTRUCTIONS = ` +You are a single step in a broader process of collecting credit card information. +You are solely responsible for collecting the credit card number. +{modality_specific} +If the user refuses to provide a credit card number, call decline_card_capture(). +If the user wishes to start over the credit card collection process, call restart_card_collection(). +Avoid listing out questions with bullet points or numbers, use a natural conversational tone. +Never repeat any sensitive information, such as the user's credit card number, back to the user. +{confirmation_instructions} +`; + +const CARD_NUMBER_AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect users to read the card number digit by digit. +Normalize spoken digits silently: 'four' → 4, 'zero' / 'oh' → 0. +Filter out filler words or hesitations. +`; + +const CARD_NUMBER_TEXT_SPECIFIC = ` +Handle input as typed text. Users may type the number with or without spaces or dashes (e.g. '4152 6374 8901 2345'). +`; + +const SECURITY_CODE_BASE_INSTRUCTIONS = ` +You are a single step in a broader process of collecting credit card information. +You are solely responsible for collecting the user's card's security code. +{modality_specific} +If the user refuses to provide a code, call decline_card_capture(). +If the user wishes to start over the card collection process, call restart_card_collection(). +Avoid listing out questions with bullet points or numbers, use a natural conversational tone. +Never repeat any sensitive information, such as the user's security code, back to the user. +{confirmation_instructions} +`; + +const SECURITY_CODE_AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect users to read the security code digit by digit. +Normalize spoken digits silently: 'four' → 4, 'zero' / 'oh' → 0. +Filter out filler words or hesitations. +`; + +const SECURITY_CODE_TEXT_SPECIFIC = ` +Handle input as typed text. Users will type the security code directly. +`; + +const EXPIRATION_DATE_BASE_INSTRUCTIONS = ` +You are a single step in a broader process of collecting credit card information. +You are solely responsible for collecting the user's card's expiration date. +{modality_specific} +If the user refuses to provide a date, call decline_card_capture(). +If the user wishes to start over the card collection process, call restart_card_collection(). +Avoid listing out questions with bullet points or numbers, use a natural conversational tone. +Never repeat any sensitive information, such as the user's expiration date, back to the user. +{confirmation_instructions} +`; + +const EXPIRATION_DATE_AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect users to say the expiration date in formats like 'April twenty five', 'oh four twenty five', 'four slash twenty five', or 'April 2025'. +Normalize spoken months and digits silently. +Filter out filler words or hesitations. +`; + +const EXPIRATION_DATE_TEXT_SPECIFIC = ` +Handle input as typed text. Expect users to type the expiration date in formats like '04/25', '04/2025', or 'April 2025'. +`; diff --git a/agents/src/workflows/dob.ts b/agents/src/workflows/dob.ts new file mode 100644 index 000000000..8b4a4a2f4 --- /dev/null +++ b/agents/src/workflows/dob.ts @@ -0,0 +1,406 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import { safeRender } from '../utils.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; + +/** A calendar date without a timezone. `month` and `day` are 1-based. */ +export interface DateOfBirth { + year: number; + month: number; + day: number; +} + +/** A time of day. `hour` is 0-23 and `minute` is 0-59. */ +export interface TimeOfBirth { + hour: number; + minute: number; +} + +export interface GetDOBResult { + dateOfBirth: DateOfBirth; + timeOfBirth?: TimeOfBirth; +} + +export interface GetDOBTaskOptions { + /** Extra instructions appended to the built-in prompt for domain-specific context. */ + extraInstructions?: string; + /** Also capture the (optional) time of birth. Defaults to false. */ + includeTime?: boolean; + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode | null; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString | null; + vad?: VAD | null; + llm?: LLM | RealtimeModel | LLMModels | null; + tts?: TTS | TTSModelString | null; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm the captured date of birth. Defaults to confirming on + * audio input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** + * When true, the model must produce an asking utterance before recording a date of birth — + * it can't silently fill one from the chat context during `onEnter`. + */ + requireExplicitAsk?: boolean; +} + +const MONTH_NAMES = [ + 'January', + 'February', + 'March', + 'April', + 'May', + 'June', + 'July', + 'August', + 'September', + 'October', + 'November', + 'December', +] as const; + +function formatDate(dob: DateOfBirth): string { + return `${MONTH_NAMES[dob.month - 1]} ${String(dob.day).padStart(2, '0')}, ${dob.year}`; +} + +function formatTime(time: TimeOfBirth): string { + const hour12 = time.hour % 12 === 0 ? 12 : time.hour % 12; + const period = time.hour < 12 ? 'AM' : 'PM'; + return `${String(hour12).padStart(2, '0')}:${String(time.minute).padStart(2, '0')} ${period}`; +} + +function isSameDate(a: DateOfBirth | undefined, b: DateOfBirth | undefined): boolean { + if (a === undefined || b === undefined) { + return a === b; + } + return a.year === b.year && a.month === b.month && a.day === b.day; +} + +function isSameTime(a: TimeOfBirth | undefined, b: TimeOfBirth | undefined): boolean { + if (a === undefined || b === undefined) { + return a === b; + } + return a.hour === b.hour && a.minute === b.minute; +} + +/** + * Build an {@link AgentTask} that collects a date of birth (and optionally a time of birth) + * from the user. + * + * This is the functional core; {@link GetDOBTask} is a thin class wrapper over it. + */ +export function createGetDOBTask({ + extraInstructions = '', + includeTime = false, + chatCtx, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + requireExplicitAsk = false, +}: GetDOBTaskOptions = {}): AgentTask { + const timeInstructions = !includeTime + ? '' + : 'Also ask for and capture the time of birth if the user knows it. ' + + "The time is optional - if the user doesn't know it, proceed without it.\n"; + const confirmationInstructions = + 'Call `confirm_dob` after the user confirmed the date of birth is correct.'; + + let currentDob: DateOfBirth | undefined; + let currentTime: TimeOfBirth | undefined; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildResult = (dateOfBirth: DateOfBirth): GetDOBResult => ({ + dateOfBirth, + timeOfBirth: currentTime, + }); + + // confirm tool is only injected after update_dob/update_time is called, + // preventing the LLM from hallucinating a confirmation without user input + const buildConfirmTool = (capturedDob: DateOfBirth | undefined) => { + const capturedTime = currentTime; + + return tool({ + name: 'confirm_dob', + description: 'Call after the user confirms the date of birth is correct.', + execute: async () => { + if (!isSameDate(capturedDob, currentDob) || !isSameTime(capturedTime, currentTime)) { + task.session.generateReply({ + instructions: + 'The date of birth has changed since confirmation was requested, ask the user to confirm the updated date.', + }); + return; + } + + if (currentDob === undefined) { + task.session.generateReply({ + instructions: 'No date of birth was provided yet, ask the user to provide it.', + }); + return; + } + + if (!task.done) { + task.complete(buildResult(currentDob)); + } + }, + }); + }; + + const injectConfirmTool = async (): Promise => { + const confirmTool = buildConfirmTool(currentDob); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_dob'); + await task.updateTools([...currentTools, confirmTool]); + }; + + const updateDobTool = tool({ + name: 'update_dob', + description: + "Update the date of birth provided by the user. Given a spoken month and year (e.g., 'July 2030'), return its numerical representation (7/2030).", + parameters: z.object({ + year: z.number().int().describe('The birth year (e.g., 1990)'), + month: z.number().int().describe('The birth month (1-12)'), + day: z.number().int().describe('The birth day (1-31)'), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ( + { year, month, day }: { year: number; month: number; day: number }, + { ctx }, + ) => { + // Normalize two-digit years to the intended century, matching what the + // prompt already asks the model to do ("90" -> 1990, "05" -> 2005). A + // literal two-digit year is otherwise a valid date (e.g. 90 -> year 90 AD) + // that passes the future-date check and silently corrupts the result. + if (year >= 0 && year < 100) { + const currentYY = new Date().getFullYear() % 100; + year += year <= currentYY ? 2000 : 1900; + } + + if (month < 1 || month > 12 || day < 1 || day > 31) { + throw new ToolError(`Invalid date: year=${year} month=${month} day=${day}`); + } + const candidate = new Date(Date.UTC(year, month - 1, day)); + if ( + candidate.getUTCFullYear() !== year || + candidate.getUTCMonth() !== month - 1 || + candidate.getUTCDate() !== day + ) { + throw new ToolError(`Invalid date: year=${year} month=${month} day=${day}`); + } + + const dob: DateOfBirth = { year, month, day }; + const today = new Date(); + const todayParts: DateOfBirth = { + year: today.getFullYear(), + month: today.getMonth() + 1, + day: today.getDate(), + }; + const isFuture = + dob.year > todayParts.year || + (dob.year === todayParts.year && + (dob.month > todayParts.month || + (dob.month === todayParts.month && dob.day > todayParts.day))); + if (isFuture) { + throw new ToolError( + `Invalid date of birth: ${formatDate(dob)} is in the future. ` + + 'Date of birth cannot be a future date.', + ); + } + + currentDob = dob; + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete(buildResult(currentDob)); + } + return undefined; + } + + await injectConfirmTool(); + + let response = `The date of birth has been updated to ${formatDate(dob)}`; + + if (currentTime) { + response += ` at ${formatTime(currentTime)}`; + } + + response += + '\nRepeat the date back to the user in a natural spoken format.\n' + + 'Prompt the user for confirmation, do not call `confirm_dob` directly'; + + return response; + }, + }); + + const updateTimeTool = tool({ + name: 'update_time', + description: 'Update the time of birth provided by the user.', + parameters: z.object({ + hour: z.number().int().describe('The birth hour (0-23)'), + minute: z.number().int().describe('The birth minute (0-59)'), + }), + execute: async ({ hour, minute }: { hour: number; minute: number }, { ctx }) => { + if (hour < 0 || hour > 23 || minute < 0 || minute > 59) { + throw new ToolError(`Invalid time: hour=${hour} minute=${minute}`); + } + + const birthTime: TimeOfBirth = { hour, minute }; + currentTime = birthTime; + + if (!confirmationRequired(ctx) && currentDob !== undefined) { + if (!task.done) { + task.complete(buildResult(currentDob)); + } + return undefined; + } + + if (confirmationRequired(ctx)) { + await injectConfirmTool(); + } + + let response = `The time of birth has been updated to ${formatTime(birthTime)}`; + + if (currentDob) { + response = `The date and time of birth has been updated to ${formatDate(currentDob)} at ${formatTime(birthTime)}`; + } + + if (confirmationRequired(ctx)) { + response += + '\nRepeat the time back to the user in a natural spoken format.\n' + + 'Prompt the user for confirmation, do not call `confirm_dob` directly'; + } else { + response += '\nThe date of birth has not been provided yet, ask the user to provide it.'; + } + + return response; + }, + }); + + const declineTool = tool({ + name: 'decline_dob_capture', + description: 'Handles the case when the user explicitly declines to provide a date of birth.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide the date of birth'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }) => { + if (!task.done) { + task.complete(new ToolError(`couldn't get the date of birth: ${reason}`)); + } + }, + }); + + const task = AgentTask.create({ + id: 'get_dob_task', + instructions: new Instructions('', { + audio: safeRender(BASE_INSTRUCTIONS, { + modality_specific: AUDIO_SPECIFIC, + time_instructions: timeInstructions, + confirmation_instructions: requireConfirmation !== false ? confirmationInstructions : '', + extra_instructions: extraInstructions, + }), + text: safeRender(BASE_INSTRUCTIONS, { + modality_specific: TEXT_SPECIFIC, + time_instructions: timeInstructions, + confirmation_instructions: requireConfirmation === true ? confirmationInstructions : '', + extra_instructions: extraInstructions, + }), + }), + chatCtx, + turnDetection: turnDetection ?? undefined, + tools: [...(tools ?? []), updateDobTool, declineTool, ...(includeTime ? [updateTimeTool] : [])], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, + onEnter: async () => { + const prompt = includeTime + ? 'Ask the user to provide their date of birth and, if they know it, their time of birth.' + : 'Ask the user to provide their date of birth.'; + task.session.generateReply({ instructions: prompt }); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetDOBTask}, preserving the + * `new GetDOBTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetDOBTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetDOBTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetDOBTask. + super({ instructions: '' }); + this.#task = createGetDOBTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +const BASE_INSTRUCTIONS = ` +You are only a single step in a broader system, responsible solely for capturing a date of birth. +{modality_specific} +{time_instructions}Call \`update_dob\` at the first opportunity whenever you form a new hypothesis about the date of birth. (before asking any questions or providing any answers.) +Don't invent dates, stick strictly to what the user said. +{confirmation_instructions} +When reading back dates, use a natural spoken format like 'January fifteenth, nineteen ninety'. +If the date is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts: first the month, then the day, then the year. +Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. +Avoid verbosity by not sharing example dates or formats unless prompted to do so. Do not deviate from the goal of collecting the user's birthday. +Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called.\ +{extra_instructions} +`; + +const AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect that users will say dates aloud with formats like: +- 'January 15th 1990' +- 'the fifteenth of January nineteen ninety' +- '01 15 1990' or 'one fifteen ninety' +- 'Jan 15 90' +- '15th January 1990' +Normalize common spoken patterns silently: +- Convert spoken numbers and ordinals to their numeric form: 'fifteenth' → 15, 'ninety' → 1990. +- Recognize month names in various forms: 'Jan', 'January', etc. +- Handle two-digit years appropriately: '90' likely means 1990, '05' likely means 2005. +- Filter out filler words or hesitations. +Don't mention corrections. Treat inputs as possibly imperfect but fix them silently. +`; + +const TEXT_SPECIFIC = ` +Handle input as typed text. Expect users to type their date of birth directly. +Accept common date formats like 'MM/DD/YYYY', 'January 15, 1990', or '1990-01-15'. +Handle two-digit years appropriately: '90' likely means 1990, '05' likely means 2005. +`; diff --git a/agents/src/workflows/dtmf_inputs.ts b/agents/src/workflows/dtmf_inputs.ts new file mode 100644 index 000000000..8892c157c --- /dev/null +++ b/agents/src/workflows/dtmf_inputs.ts @@ -0,0 +1,286 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { RoomEvent } from '@livekit/rtc-node'; +import { z } from 'zod'; +import { getJobContext } from '../job.js'; +import type { ChatContext } from '../llm/index.js'; +import { ToolError, tool } from '../llm/index.js'; +import { log } from '../log.js'; +import { AgentTask } from '../voice/agent.js'; +import { + AgentSessionEventTypes, + type AgentStateChangedEvent, + type UserStateChangedEvent, +} from '../voice/events.js'; +import { DtmfEvent, formatDtmf } from './utils.js'; + +export interface GetDtmfResult { + userInput: string; +} + +export interface GetDtmfTaskOptions { + /** The number of digits to collect. */ + numDigits: number; + /** Whether to ask for confirmation when the agent has collected the full digits. */ + askForConfirmation?: boolean; + /** The per-digit timeout, in milliseconds. Defaults to 4000. */ + dtmfInputTimeout?: number; + /** The DTMF event that stops collecting inputs. Defaults to `DtmfEvent.POUND`. */ + dtmfStopEvent?: DtmfEvent; + /** The chat context to use. */ + chatCtx?: ChatContext; + /** Extra instructions to add to the task. */ + extraInstructions?: string; +} + +/** + * Debounced runner for a single async function: `schedule()` (re)starts a delay timer, + * `run()` fires immediately, `cancel()` clears any pending timer. Mirrors the Python + * `utils.aio.debounced` helper closely enough for this task's needs. + */ +class Debounced { + #timer?: ReturnType; + + constructor( + private readonly fn: () => Promise, + private readonly delay: number, + ) {} + + schedule(): void { + this.cancel(); + this.#timer = setTimeout(() => { + this.#timer = undefined; + this.#run(); + }, this.delay); + } + + run(): void { + this.cancel(); + this.#run(); + } + + cancel(): void { + if (this.#timer !== undefined) { + clearTimeout(this.#timer); + this.#timer = undefined; + } + } + + #run(): void { + void this.fn().catch((error) => { + log().error({ error }, 'error running debounced DTMF reply'); + }); + } +} + +const dtmfResultFromInputs = (inputs: readonly DtmfEvent[]): GetDtmfResult => ({ + userInput: formatDtmf(inputs), +}); + +/** + * Build an {@link AgentTask} that collects DTMF inputs from the user. + * + * The task completes with the collected digit string, or fails with a {@link ToolError} + * when the expected number of digits is not received in time. + * + * This is the functional core; {@link GetDtmfTask} is a thin class wrapper over it. + */ +export function createGetDtmfTask({ + numDigits, + askForConfirmation = false, + dtmfInputTimeout = 4000, + dtmfStopEvent = DtmfEvent.POUND, + chatCtx, + extraInstructions, +}: GetDtmfTaskOptions): AgentTask { + if (numDigits <= 0) { + throw new Error('numDigits must be greater than 0'); + } + + const logger = log().child({ component: 'dtmf-inputs' }); + + const currDtmfInputs: DtmfEvent[] = []; + let dtmfReplyRunning = false; + + const confirmInputsTool = tool({ + name: 'confirm_inputs', + description: + 'Finalize the collected digit inputs after explicit user confirmation.\n\n' + + 'Use this ONLY after the confirmation. You should confirm by verbally reading out the digits one by one and, once the ' + + 'user confirms they are correct, call this tool with the inputs.\n\n' + + 'Do not use this tool to capture the initial digits.', + parameters: z.object({ + inputs: z.array(z.nativeEnum(DtmfEvent)).describe('The digit inputs to finalize'), + }), + execute: async ({ inputs }: { inputs: DtmfEvent[] }) => { + task.complete(dtmfResultFromInputs(inputs)); + }, + }); + + const recordInputsTool = tool({ + name: 'record_inputs', + description: + 'Record the collected digit inputs without additional confirmation.\n\n' + + 'Call this tool as soon as a valid sequence of digits has been provided by the user (via DTMF or spoken).', + parameters: z.object({ + inputs: z.array(z.nativeEnum(DtmfEvent)).describe('The digit inputs to record'), + }), + execute: async ({ inputs }: { inputs: DtmfEvent[] }) => { + task.complete(dtmfResultFromInputs(inputs)); + }, + }); + + let instructions = + 'You are a single step in a broader system, responsible solely for gathering digits input from the user. ' + + 'You will either receive a sequence of digits through dtmf events tagged by , or ' + + 'user will directly say the digits to you. You should be able to handle both cases. '; + + if (askForConfirmation) { + instructions += + 'Once user has confirmed the digits (by verbally spoken or entered manually), call `confirm_inputs` with the inputs.'; + } else { + instructions += + 'If user provides the digits through voice and it is valid, call `record_inputs` with the inputs.'; + } + + if (extraInstructions !== undefined) { + instructions += `\n${extraInstructions}`; + } + + const generateDtmfReply = new Debounced(async () => { + dtmfReplyRunning = true; + + try { + task.session.interrupt(); + + const dtmfStr = formatDtmf(currDtmfInputs); + logger.debug(`Generating DTMF reply, current inputs: ${dtmfStr}`); + + // if input not fully received (i.e. timeout), fail the task + if (currDtmfInputs.length !== numDigits) { + const errorMsg = + `Digits input not fully received. ` + + `Expect ${numDigits} digits, got ${currDtmfInputs.length}`; + if (!task.done) { + task.complete(new ToolError(errorMsg)); + } + return; + } + + // if not asking for confirmation, return the DTMF inputs + if (!askForConfirmation) { + if (!task.done) { + task.complete(dtmfResultFromInputs(currDtmfInputs)); + } + return; + } + + const replyInstructions = + 'User has entered the following valid digits on the telephone keypad:\n' + + `${dtmfStr}\n` + + 'Please confirm it with the user by saying the digits one by one with space in between ' + + "(.e.g. 'one two three four five six seven eight nine ten'). " + + 'Once you are sure, call `confirm_inputs` with the inputs.'; + + await task.session.generateReply({ userInput: replyInstructions }); + } finally { + dtmfReplyRunning = false; + currDtmfInputs.length = 0; + } + }, dtmfInputTimeout); + + const onSipDtmfReceived = (_code: number, digit: string): void => { + if (dtmfReplyRunning) { + return; + } + + // immediately kick off the DTMF reply generation if matches the stop event + if (digit === dtmfStopEvent) { + generateDtmfReply.run(); + return; + } + + if (!(Object.values(DtmfEvent) as string[]).includes(digit)) { + logger.warn(`Ignoring invalid DTMF digit: ${digit}`); + return; + } + + currDtmfInputs.push(digit as DtmfEvent); + logger.info(`DTMF inputs: ${formatDtmf(currDtmfInputs)}`); + generateDtmfReply.schedule(); + }; + + const onUserStateChanged = (ev: UserStateChangedEvent): void => { + if (dtmfReplyRunning) { + return; + } + + if (ev.newState === 'speaking') { + // clear any pending DTMF reply generation + generateDtmfReply.cancel(); + } else if (currDtmfInputs.length !== 0) { + // resume any previously cancelled DTMF reply generation after user is back to non-speaking + generateDtmfReply.schedule(); + } + }; + + const onAgentStateChanged = (ev: AgentStateChangedEvent): void => { + if (dtmfReplyRunning) { + return; + } + + if (ev.newState === 'speaking' || ev.newState === 'thinking') { + // clear any pending DTMF reply generation + generateDtmfReply.cancel(); + } else if (currDtmfInputs.length !== 0) { + // resume any previously cancelled DTMF reply generation after agent is back to non-speaking + generateDtmfReply.schedule(); + } + }; + + const task = AgentTask.create({ + id: 'get_dtmf_task', + instructions, + chatCtx, + tools: askForConfirmation ? [confirmInputsTool] : [recordInputsTool], + onEnter: async () => { + const ctx = getJobContext(); + + ctx.room.on(RoomEvent.DtmfReceived, onSipDtmfReceived); + task.session.on(AgentSessionEventTypes.UserStateChanged, onUserStateChanged); + task.session.on(AgentSessionEventTypes.AgentStateChanged, onAgentStateChanged); + task.session.generateReply({ toolChoice: 'none' }); + }, + onExit: async () => { + const ctx = getJobContext(); + + ctx.room.off(RoomEvent.DtmfReceived, onSipDtmfReceived); + task.session.off(AgentSessionEventTypes.UserStateChanged, onUserStateChanged); + task.session.off(AgentSessionEventTypes.AgentStateChanged, onAgentStateChanged); + generateDtmfReply.cancel(); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetDtmfTask}, preserving the + * `new GetDtmfTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetDtmfTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetDtmfTaskOptions) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetDtmfTask. + super({ instructions: '' }); + this.#task = createGetDtmfTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} diff --git a/agents/src/workflows/email_address.ts b/agents/src/workflows/email_address.ts new file mode 100644 index 000000000..2f094cc0f --- /dev/null +++ b/agents/src/workflows/email_address.ts @@ -0,0 +1,244 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; +import { type InstructionParts, resolveWorkflowInstructions } from './utils.js'; + +export interface GetEmailResult { + emailAddress: string; +} + +export interface GetEmailTaskOptions { + /** + * Instructions for the email capture prompt. Pass a full string or {@link Instructions} to + * replace the built-in prompt entirely, or {@link InstructionParts} to override individual + * sections (e.g. `persona`) while keeping the built-in template. + */ + instructions?: InstructionParts | Instructions | string; + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode | null; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString | null; + vad?: VAD | null; + llm?: LLM | RealtimeModel | LLMModels | null; + tts?: TTS | TTSModelString | null; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm the captured email address. Defaults to confirming on + * audio input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** + * When true, the model must produce an asking utterance before recording an email address — + * it can't silently fill one from the chat context during `onEnter`. + */ + requireExplicitAsk?: boolean; + /** @deprecated use `instructions.extra` instead */ + extraInstructions?: string; +} + +/** + * Build an {@link AgentTask} that collects an email address from the user. + * + * This is the functional core; {@link GetEmailTask} is a thin class wrapper over it. + */ +export function createGetEmailTask({ + instructions, + chatCtx, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + requireExplicitAsk = false, + extraInstructions = '', +}: GetEmailTaskOptions = {}): AgentTask { + let currentEmail = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildConfirmTool = (email: string) => + tool({ + name: 'confirm_email_address', + description: 'Call after the user confirms the email address is correct.', + execute: async () => { + if (email !== currentEmail) { + task.session.generateReply({ + instructions: + 'The email has changed since confirmation was requested, ask the user to confirm the updated email.', + }); + return; + } + + if (!task.done) { + task.complete({ emailAddress: email }); + } + }, + }); + + const updateEmailTool = tool({ + name: 'update_email_address', + description: 'Update the email address provided by the user.', + parameters: z.object({ + email: z.string().describe('The email address provided by the user'), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ({ email }: { email: string }, { ctx }) => { + email = email.trim(); + + if (!EMAIL_REGEX.test(email)) { + throw new ToolError(`Invalid email address provided: ${email}`); + } + + currentEmail = email; + const separatedEmail = email.split('').join(' '); + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete({ emailAddress: currentEmail }); + } + return undefined; // no need to continue the conversation + } + + const confirmTool = buildConfirmTool(email); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_email_address'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + `The email has been updated to ${email}\n` + + `Repeat the email character by character: ${separatedEmail} if needed\n` + + `Prompt the user for confirmation, do not call \`confirm_email_address\` directly` + ); + }, + }); + + const declineTool = tool({ + name: 'decline_email_capture', + description: 'Handles the case when the user explicitly declines to provide an email address.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide the email address'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }) => { + if (!task.done) { + task.complete(new ToolError(`couldn't get the email address: ${reason}`)); + } + }, + }); + + const task = AgentTask.create({ + id: 'get_email_task', + instructions: resolveWorkflowInstructions({ + instructions, + extraInstructions, + template: INSTRUCTIONS_TEMPLATE, + defaultPersona: PERSONA, + kwargs: { + _modality_specific: new Instructions('', { + audio: AUDIO_SPECIFIC, + text: TEXT_SPECIFIC, + }), + _confirmation: new Instructions('', { + // confirmation is enabled by default for audio, disabled by default for text + audio: requireConfirmation !== false ? CONFIRMATION_INSTRUCTION : '', + text: requireConfirmation === true ? CONFIRMATION_INSTRUCTION : '', + }), + }, + }), + chatCtx, + turnDetection: turnDetection ?? undefined, + tools: [...(tools ?? []), updateEmailTool, declineTool], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, + onEnter: async () => { + task.session.generateReply({ instructions: 'Ask the user to provide an email address.' }); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetEmailTask}, preserving the + * `new GetEmailTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetEmailTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetEmailTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetEmailTask. + super({ instructions: '' }); + this.#task = createGetEmailTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +const EMAIL_REGEX = + /^[A-Za-z0-9][A-Za-z0-9._%+\-]*@(?:[A-Za-z0-9](?:[A-Za-z0-9\-]*[A-Za-z0-9])?\.)+[A-Za-z]{2,}$/; + +// instructions +const PERSONA = + 'You are only a single step in a broader system, responsible solely for capturing an email address.'; + +const AUDIO_SPECIFIC = `Handle input as noisy voice transcription. Expect that users will say emails aloud with formats like: +- 'john dot doe at gmail dot com' +- 'susan underscore smith at yahoo dot co dot uk' +- 'dave dash b at protonmail dot com' +- 'jane at example' (partial—prompt for the domain) +- 'theo t h e o at livekit dot io' (name followed by spelling) +Normalize common spoken patterns silently: +- Convert words like 'dot', 'underscore', 'dash', 'plus' into symbols: \`.\`, \`_\`, \`-\`, \`+\`. +- Convert 'at' to \`@\`. +- Recognize patterns where users speak their name or a word, followed by spelling: e.g., 'john j o h n'. +- Filter out filler words or hesitations. +- Assume some spelling if contextually obvious (e.g. 'mike b two two' → mikeb22). +Don't mention corrections. Treat inputs as possibly imperfect but fix them silently.`; + +const TEXT_SPECIFIC = `Handle input as typed text. Expect users to type their email address directly in standard format. +If the address looks almost correct but has minor typos (e.g. missing '@' or domain), prompt for clarification.`; + +const CONFIRMATION_INSTRUCTION = `Call \`confirm_email_address\` after the user confirmed the email address is correct.`; + +const INSTRUCTIONS_TEMPLATE = `{persona} + +{_modality_specific} + +Call \`update_email_address\` at the first opportunity whenever you form a new hypothesis about the email. (before asking any questions or providing any answers.) +Don't invent new email addresses, stick strictly to what the user said. +{_confirmation} +If the email is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts: first the part before the '@', then the domain—only if needed. + +Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. +Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called. + +{extra} +`; diff --git a/agents/src/workflows/index.ts b/agents/src/workflows/index.ts index 99c334d57..48cdca1d6 100644 --- a/agents/src/workflows/index.ts +++ b/agents/src/workflows/index.ts @@ -1,6 +1,52 @@ // SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +export { + GetAddressTask, + createGetAddressTask, + type GetAddressResult, + type GetAddressTaskOptions, +} from './address.js'; +export { + CardCaptureDeclinedError, + CardCollectionRestartError, + GetCreditCardTask, + createGetCreditCardTask, + type GetCreditCardResult, + type GetCreditCardTaskOptions, +} from './credit_card.js'; +export { + GetDOBTask, + createGetDOBTask, + type DateOfBirth, + type GetDOBResult, + type GetDOBTaskOptions, + type TimeOfBirth, +} from './dob.js'; +export { + GetDtmfTask, + createGetDtmfTask, + type GetDtmfResult, + type GetDtmfTaskOptions, +} from './dtmf_inputs.js'; +export { + GetEmailTask, + createGetEmailTask, + type GetEmailResult, + type GetEmailTaskOptions, +} from './email_address.js'; +export { + GetNameTask, + createGetNameTask, + type GetNameResult, + type GetNameTaskOptions, +} from './name.js'; +export { + GetPhoneNumberTask, + createGetPhoneNumberTask, + type GetPhoneNumberResult, + type GetPhoneNumberTaskOptions, +} from './phone_number.js'; export { TaskGroup, type TaskCompletedEvent, @@ -13,4 +59,4 @@ export { type WarmTransferResult, type WarmTransferTaskOptions, } from './warm_transfer.js'; -export type { InstructionParts } from './utils.js'; +export { DtmfEvent, dtmfEventToCode, formatDtmf, type InstructionParts } from './utils.js'; diff --git a/agents/src/workflows/name.ts b/agents/src/workflows/name.ts new file mode 100644 index 000000000..79ed95df0 --- /dev/null +++ b/agents/src/workflows/name.ts @@ -0,0 +1,394 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import { safeRender } from '../utils.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; + +export interface GetNameResult { + firstName?: string; + middleName?: string; + lastName?: string; +} + +export interface GetNameTaskOptions { + /** Collect the user's first name. Defaults to true. */ + firstName?: boolean; + /** Collect the user's last name. Defaults to false. */ + lastName?: boolean; + /** Collect the user's middle name. Defaults to false. */ + middleName?: boolean; + /** + * Order in which to collect the name parts, using `{first_name}`, `{middle_name}` and + * `{last_name}` placeholders. Defaults to the enabled parts in first/middle/last order. + */ + nameFormat?: string; + /** Ask the user to verify the spelling of the captured name. Defaults to false. */ + verifySpelling?: boolean; + /** Extra instructions appended to the built-in prompt for domain-specific context. */ + extraInstructions?: string; + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode | null; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString | null; + vad?: VAD | null; + llm?: LLM | RealtimeModel | LLMModels | null; + tts?: TTS | TTSModelString | null; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm the captured name. Defaults to confirming on audio + * input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** + * When true, the model must produce an asking utterance before recording a name — it can't + * silently fill one from the chat context during `onEnter`. + */ + requireExplicitAsk?: boolean; +} + +interface UpdateNameArgs { + first_name?: string | null; + middle_name?: string | null; + last_name?: string | null; +} + +function cleanNameArg(value: string | null | undefined): string | undefined { + // Some models (e.g. gemma) fill optional args with placeholder strings like + // "null"/"NULL" instead of omitting them, or wrap values in literal quotes. + // Normalize those to undefined/clean values so they hit the required-field + // validation below instead of being recorded as the user's name. + if (value == null) { + return undefined; + } + const cleaned = value + .trim() + .replace(/^['"]+/, '') + .replace(/['"]+$/, ''); + if ( + !cleaned || + ['null', 'none', 'nil', 'n/a', 'unknown', 'unspecified'].includes(cleaned.toLowerCase()) + ) { + return undefined; + } + return cleaned; +} + +function renderNameFormat( + nameFormat: string, + parts: { firstName: string; middleName: string; lastName: string }, +): string { + const replacements: Record = { + first_name: parts.firstName, + middle_name: parts.middleName, + last_name: parts.lastName, + }; + return nameFormat + .replace(/\{(first_name|middle_name|last_name)\}/g, (_match, key: string) => { + return replacements[key] ?? ''; + }) + .trim(); +} + +/** + * Build an {@link AgentTask} that collects the user's name. + * + * This is the functional core; {@link GetNameTask} is a thin class wrapper over it. + */ +export function createGetNameTask({ + firstName = true, + lastName = false, + middleName = false, + nameFormat, + verifySpelling = false, + extraInstructions = '', + chatCtx, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + requireExplicitAsk = false, +}: GetNameTaskOptions = {}): AgentTask { + if (!(firstName || middleName || lastName)) { + throw new Error('At least one of firstName, middleName, or lastName must be true'); + } + + let resolvedNameFormat: string; + if (nameFormat !== undefined) { + resolvedNameFormat = nameFormat; + } else { + const parts: string[] = []; + if (firstName) parts.push('{first_name}'); + if (middleName) parts.push('{middle_name}'); + if (lastName) parts.push('{last_name}'); + resolvedNameFormat = parts.join(' '); + } + + const spellingInstructions = !verifySpelling + ? '' + : 'After receiving the name, always verify the spelling by asking the user to confirm ' + + 'or spell out the name letter by letter. ' + + 'When confirming, spell out each name part letter by letter to the user. '; + const confirmationInstructions = + 'Call `confirm_name` after the user confirmed the name is correct.'; + + let currentFirstName = ''; + let currentMiddleName = ''; + let currentLastName = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildResult = (): GetNameResult => ({ + firstName: firstName ? currentFirstName : undefined, + middleName: middleName ? currentMiddleName : undefined, + lastName: lastName ? currentLastName : undefined, + }); + + const buildConfirmTool = (captured: { first: string; middle: string; last: string }) => + tool({ + name: 'confirm_name', + description: 'Call after the user confirms the name is correct.', + execute: async () => { + if ( + captured.first !== currentFirstName || + captured.middle !== currentMiddleName || + captured.last !== currentLastName + ) { + task.session.generateReply({ + instructions: + 'The name has changed since confirmation was requested, ask the user to confirm the updated name.', + }); + return; + } + + if (!task.done) { + task.complete(buildResult()); + } + }, + }); + + const updateNameTool = tool({ + name: 'update_name', + description: 'Update the name provided by the user.', + parameters: z.object({ + first_name: z.string().nullable().optional().describe("The user's first name."), + middle_name: z + .string() + .nullable() + .optional() + .describe("The user's middle name, if collected."), + last_name: z.string().nullable().optional().describe("The user's last name, if collected."), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async (args: UpdateNameArgs, { ctx }) => { + const cleanedFirst = cleanNameArg(args.first_name); + const cleanedMiddle = cleanNameArg(args.middle_name); + const cleanedLast = cleanNameArg(args.last_name); + + const errors: string[] = []; + if (firstName && !cleanedFirst?.trim()) { + errors.push('first name is required but was not provided'); + } + if (middleName && !cleanedMiddle?.trim()) { + errors.push('middle name is required but was not provided'); + } + if (lastName && !cleanedLast?.trim()) { + errors.push('last name is required but was not provided'); + } + + // A real name contains letters. Reject digit-only or punctuation-only + // values so a card number, ZIP code, phone number, etc. accidentally + // crammed into update_name fails fast instead of being recorded as + // the user's name. + for (const [label, value] of [ + ['first', cleanedFirst], + ['middle', cleanedMiddle], + ['last', cleanedLast], + ] as const) { + if (value && value.trim() && !/\p{L}/u.test(value)) { + errors.push( + `${label} name '${value}' contains no letters - that doesn't look like a name`, + ); + } + } + + if (errors.length > 0) { + throw new ToolError(`Incomplete name: ${errors.join('; ')}`); + } + + currentFirstName = cleanedFirst?.trim() ?? ''; + currentMiddleName = cleanedMiddle?.trim() ?? ''; + currentLastName = cleanedLast?.trim() ?? ''; + + const fullName = renderNameFormat(resolvedNameFormat, { + firstName: currentFirstName, + middleName: currentMiddleName, + lastName: currentLastName, + }); + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete(buildResult()); + } + return undefined; + } + + const confirmTool = buildConfirmTool({ + first: currentFirstName, + middle: currentMiddleName, + last: currentLastName, + }); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_name'); + await task.updateTools([...currentTools, confirmTool]); + + if (verifySpelling) { + return ( + `The name has been updated to ${fullName}\n` + + `Spell out the name letter by letter for verification: ${fullName}\n` + + `Prompt the user for confirmation, do not call \`confirm_name\` directly` + ); + } + + return ( + `The name has been updated to ${fullName}\n` + + `Repeat the name back to the user and prompt for confirmation, ` + + `do not call \`confirm_name\` directly` + ); + }, + }); + + const declineTool = tool({ + name: 'decline_name_capture', + description: 'Handles the case when the user explicitly declines to provide their name.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide their name'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }) => { + if (!task.done) { + task.complete(new ToolError(`couldn't get the name: ${reason}`)); + } + }, + }); + + const task = AgentTask.create({ + id: 'get_name_task', + instructions: new Instructions('', { + audio: safeRender(BASE_INSTRUCTIONS, { + name_format: resolvedNameFormat, + modality_specific: AUDIO_SPECIFIC, + spelling_instructions: spellingInstructions, + confirmation_instructions: requireConfirmation !== false ? confirmationInstructions : '', + extra_instructions: extraInstructions, + }), + text: safeRender(BASE_INSTRUCTIONS, { + name_format: resolvedNameFormat, + modality_specific: TEXT_SPECIFIC, + spelling_instructions: spellingInstructions, + confirmation_instructions: requireConfirmation === true ? confirmationInstructions : '', + extra_instructions: extraInstructions, + }), + }), + chatCtx, + turnDetection: turnDetection ?? undefined, + tools: [...(tools ?? []), updateNameTool, declineTool], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, + onEnter: async () => { + task.session.generateReply({ + instructions: + `Get the user's name (follow this order '${resolvedNameFormat}' but do not ` + + 'mention the format). First scan the conversation - if a name was already ' + + 'given earlier, ask a short confirmation question rather than asking from ' + + 'scratch. If context about what the name is FOR was provided (a role like ' + + "'cardholder', 'guest', 'emergency contact'), anchor your confirmation " + + "question to that role so the user knows which name you mean - don't ask " + + 'abstractly. When pointing at where an existing name came from, reference ' + + 'the source in the conversation (the earlier step, the booking they ' + + 'mentioned), not a presumption about how the name appears in the ' + + 'destination. Only ask fresh when the conversation has no name yet.', + }); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetNameTask}, preserving the + * `new GetNameTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetNameTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetNameTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetNameTask. + super({ instructions: '' }); + this.#task = createGetNameTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +const BASE_INSTRUCTIONS = ` +You are only a single step in a broader system, responsible solely for capturing the user's name. +You need to naturally collect the name parts in this order: {name_format}. +{modality_specific} +{spelling_instructions}Call \`update_name\` at the first opportunity whenever you form a new hypothesis about the name. (before asking any questions or providing any answers.) +Don't invent names, stick strictly to what the user said. +{confirmation_instructions} +If the name is unclear or it takes too much back-and-forth, prompt for each name part separately. +Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. +Avoid verbosity by not sharing example names or spellings unless prompted to do so. Do not deviate from the goal of collecting the user's name. +Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called.\ +{extra_instructions} +`; + +const AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect that users will say names aloud and may: +- Say their name followed by spelling: e.g., 'Michael m i c h a e l' +- Use phonetic alphabet: e.g., 'Mike as in Mike India Charlie Hotel Alpha Echo Lima' +- Have names with special characters or hyphens: e.g., 'Mary-Jane' or 'O'Brien' +- Have names from various cultural backgrounds with different pronunciation patterns +Normalize common spoken patterns silently: +- Convert 'dash' or 'hyphen' to \`-\`. +- Convert 'apostrophe' to \`'\`. +- Recognize when users spell out their name letter by letter. +- Filter out filler words or hesitations. +- Capitalize the first letter of each name part appropriately. +Don't mention corrections. Treat inputs as possibly imperfect but fix them silently. +`; + +const TEXT_SPECIFIC = ` +Handle input as typed text. Expect users to type their name directly. +Capitalize the first letter of each name part appropriately. +If the name contains special characters or hyphens (e.g., 'Mary-Jane' or 'O'Brien'), preserve them as typed. +`; diff --git a/agents/src/workflows/phone_number.ts b/agents/src/workflows/phone_number.ts new file mode 100644 index 000000000..3ae7ccb17 --- /dev/null +++ b/agents/src/workflows/phone_number.ts @@ -0,0 +1,233 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { z } from 'zod'; +import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; +import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { STT } from '../stt/index.js'; +import type { TTS } from '../tts/index.js'; +import { safeRender } from '../utils.js'; +import type { VAD } from '../vad.js'; +import { AgentTask } from '../voice/agent.js'; +import type { TurnDetectionMode } from '../voice/agent_session.js'; +import type { RunContext } from '../voice/run_context.js'; + +const PHONE_REGEX = /^\+?[1-9]\d{6,14}$/; + +export interface GetPhoneNumberResult { + phoneNumber: string; +} + +export interface GetPhoneNumberTaskOptions { + /** Extra instructions appended to the built-in prompt for domain-specific context. */ + extraInstructions?: string; + chatCtx?: ChatContext; + turnDetection?: TurnDetectionMode | null; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString | null; + vad?: VAD | null; + llm?: LLM | RealtimeModel | LLMModels | null; + tts?: TTS | TTSModelString | null; + allowInterruptions?: boolean; + /** + * Whether to ask the user to confirm the captured phone number. Defaults to confirming on + * audio input and skipping confirmation on text input. + */ + requireConfirmation?: boolean; + /** + * When true, the model must produce an asking utterance before recording a phone number — it + * can't silently fill one from the chat context during `onEnter`. + */ + requireExplicitAsk?: boolean; +} + +/** + * Build an {@link AgentTask} that collects a phone number from the user. + * + * This is the functional core; {@link GetPhoneNumberTask} is a thin class wrapper over it. + */ +export function createGetPhoneNumberTask({ + extraInstructions = '', + chatCtx, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + requireConfirmation, + requireExplicitAsk = false, +}: GetPhoneNumberTaskOptions = {}): AgentTask { + const confirmationInstructions = + 'Call `confirm_phone_number` after the user confirmed the phone number is correct.'; + + let currentPhoneNumber = ''; + + const confirmationRequired = (ctx: RunContext): boolean => { + if (requireConfirmation !== undefined) { + return requireConfirmation; + } + return ctx.speechHandle.inputDetails.modality === 'audio'; + }; + + const buildConfirmTool = (phoneNumber: string) => + tool({ + name: 'confirm_phone_number', + description: 'Call after the user confirms the phone number is correct.', + execute: async () => { + if (phoneNumber !== currentPhoneNumber) { + task.session.generateReply({ + instructions: + 'The phone number has changed since confirmation was requested, ask the user to confirm the updated number.', + }); + return; + } + + if (!task.done) { + task.complete({ phoneNumber }); + } + }, + }); + + const updatePhoneNumberTool = tool({ + name: 'update_phone_number', + description: 'Update the phone number provided by the user.', + parameters: z.object({ + phone_number: z + .string() + .describe('The phone number provided by the user, digits only with optional leading +'), + }), + // With requireExplicitAsk, the model can't silent-fill from chatCtx during + // onEnter — it must produce an asking utterance first. + flags: requireExplicitAsk ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, + execute: async ({ phone_number: phoneNumber }: { phone_number: string }, { ctx }) => { + const cleaned = phoneNumber.trim().replace(/[\s\-().]+/g, ''); + + if (!PHONE_REGEX.test(cleaned)) { + throw new ToolError(`Invalid phone number provided: ${phoneNumber}`); + } + + currentPhoneNumber = cleaned; + + if (!confirmationRequired(ctx)) { + if (!task.done) { + task.complete({ phoneNumber: currentPhoneNumber }); + } + return undefined; // no need to continue the conversation + } + + const confirmTool = buildConfirmTool(cleaned); + const currentTools = task.toolCtx.tools.filter((t) => t.id !== 'confirm_phone_number'); + await task.updateTools([...currentTools, confirmTool]); + + return ( + `The phone number has been updated to ${cleaned}\n` + + `Read the number back to the user in groups.\n` + + `Prompt the user for confirmation, do not call \`confirm_phone_number\` directly` + ); + }, + }); + + const declineTool = tool({ + name: 'decline_phone_number_capture', + description: 'Handles the case when the user explicitly declines to provide a phone number.', + parameters: z.object({ + reason: z + .string() + .describe('A short explanation of why the user declined to provide the phone number'), + }), + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async ({ reason }: { reason: string }) => { + if (!task.done) { + task.complete(new ToolError(`couldn't get the phone number: ${reason}`)); + } + }, + }); + + const task = AgentTask.create({ + id: 'get_phone_number_task', + instructions: new Instructions('', { + audio: safeRender(BASE_INSTRUCTIONS, { + modality_specific: AUDIO_SPECIFIC, + confirmation_instructions: requireConfirmation !== false ? confirmationInstructions : '', + extra_instructions: extraInstructions, + }), + text: safeRender(BASE_INSTRUCTIONS, { + modality_specific: TEXT_SPECIFIC, + confirmation_instructions: requireConfirmation === true ? confirmationInstructions : '', + extra_instructions: extraInstructions, + }), + }), + chatCtx, + turnDetection: turnDetection ?? undefined, + tools: [...(tools ?? []), updatePhoneNumberTool, declineTool], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, + onEnter: async () => { + task.session.generateReply({ instructions: 'Ask the user to provide their phone number.' }); + }, + }); + + return task; +} + +/** + * Class wrapper around {@link createGetPhoneNumberTask}, preserving the + * `new GetPhoneNumberTask(options).run()` API. It composes the functional task and + * delegates `run()` to it. + */ +export class GetPhoneNumberTask extends AgentTask { + readonly #task: AgentTask; + + constructor(options: GetPhoneNumberTaskOptions = {}) { + // The wrapper itself never runs as an agent; run() delegates to the + // composed task. Instructions are resolved inside createGetPhoneNumberTask. + super({ instructions: '' }); + this.#task = createGetPhoneNumberTask(options); + } + + override run(): Promise { + return this.#task.run(); + } +} + +const BASE_INSTRUCTIONS = ` +You are only a single step in a broader system, responsible solely for capturing a phone number. +{modality_specific} +Call \`update_phone_number\` at the first opportunity whenever you form a new hypothesis about the phone number. (before asking any questions or providing any answers.) +Don't invent phone numbers, stick strictly to what the user said. +{confirmation_instructions} +If the number is unclear or invalid, or it takes too much back-and-forth, prompt for it in parts: first the area code, then the remaining digits. +Never repeat the phone number back to the user as a single block of digits. Read it back in groups. +Ignore unrelated input and avoid going off-topic. Do not generate markdown, greetings, or unnecessary commentary. +Avoid verbosity by not sharing example phone numbers or formats unless prompted to do so. Do not deviate from the goal of collecting the user's phone number. +Always explicitly invoke a tool when applicable. Do not simulate tool usage, no real action is taken unless the tool is explicitly called.\ +{extra_instructions} +`; + +const AUDIO_SPECIFIC = ` +Handle input as noisy voice transcription. Expect that users will say phone numbers aloud with formats like: +- '555 123 4567' +- 'five five five, one two three, four five six seven' +- '+1 555 123 4567' +- 'area code 555, 123 4567' +- '555-123-4567' +Normalize common spoken patterns silently: +- Convert spoken digits to their numeric form: 'five' → 5, 'zero' → 0, 'oh' → 0. +- Remove filler words, pauses, and hesitations. +- Strip dashes, spaces, parentheses, and dots from the number. +- Recognize 'plus' at the start as the international prefix \`+\`. +- Recognize 'area code' as a prefix for the area code digits. +Don't mention corrections. Treat inputs as possibly imperfect but fix them silently. +`; + +const TEXT_SPECIFIC = ` +Handle input as typed text. Expect users to type their phone number directly. +Strip dashes, spaces, parentheses, and dots from the number. +If the number looks almost correct but has minor formatting issues, clean it up silently. +`; diff --git a/agents/src/workflows/task_group.ts b/agents/src/workflows/task_group.ts index 30a501987..a733c0240 100644 --- a/agents/src/workflows/task_group.ts +++ b/agents/src/workflows/task_group.ts @@ -8,7 +8,8 @@ import { asError } from '../utils.js'; import { AgentTask } from '../voice/agent.js'; interface FactoryInfo { - taskFactory: () => AgentTask; + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts tasks with any result type + taskFactory: () => AgentTask; id: string; description: string; } @@ -64,7 +65,8 @@ export class TaskGroup extends AgentTask { this._taskCompletedCallback = onTaskCompleted; } - add(task: () => AgentTask, { id, description }: { id: string; description: string }): this { + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts tasks with any result type + add(task: () => AgentTask, { id, description }: { id: string; description: string }): this { this._registeredFactories.set(id, { taskFactory: task, id, description }); return this; } diff --git a/agents/src/workflows/utils.ts b/agents/src/workflows/utils.ts index 25ba05f30..b0189939f 100644 --- a/agents/src/workflows/utils.ts +++ b/agents/src/workflows/utils.ts @@ -1,7 +1,8 @@ // SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import type { Instructions } from '../llm/index.js'; +import { Instructions, isInstructions } from '../llm/chat_context.js'; +import { log } from '../log.js'; /** * Customizable instruction sections for built-in workflow tasks. @@ -16,3 +17,76 @@ export interface InstructionParts { /** Extra instructions appended to the prompt for domain-specific context. */ extra?: Instructions | string; } + +/** + * Resolve workflow instructions the way the Python `WorkflowInstructions` class does: + * a full string or {@link Instructions} replaces the built-in prompt entirely, while + * {@link InstructionParts} (or nothing) fills the workflow's built-in template. + * + * @internal + */ +export function resolveWorkflowInstructions(options: { + instructions?: InstructionParts | Instructions | string; + /** @deprecated legacy `extraInstructions` option, ignored when `instructions` is provided. */ + extraInstructions?: string; + template: string; + defaultPersona: string; + kwargs?: Record; +}): string | Instructions { + const { instructions, extraInstructions = '', template, defaultPersona, kwargs = {} } = options; + + if (instructions !== undefined && extraInstructions) { + log().warn('`extraInstructions` will be ignored when `instructions` is provided'); + } + + // A full instruction string or Instructions replaces the built-in prompt entirely. + if (typeof instructions === 'string' || isInstructions(instructions)) { + return instructions; + } + + // No instructions or an `InstructionParts` override: fill the built-in template. + const parts: InstructionParts = instructions ?? { extra: extraInstructions }; + return Instructions.resolveTemplate(template, { + // Unset preserves the built-in default; an explicit empty string removes the section. + persona: parts.persona !== undefined ? parts.persona : defaultPersona, + extra: parts.extra !== undefined ? parts.extra : '', + ...kwargs, + }); +} + +export enum DtmfEvent { + ONE = '1', + TWO = '2', + THREE = '3', + FOUR = '4', + FIVE = '5', + SIX = '6', + SEVEN = '7', + EIGHT = '8', + NINE = '9', + ZERO = '0', + STAR = '*', + POUND = '#', + A = 'A', + B = 'B', + C = 'C', + D = 'D', +} + +export function dtmfEventToCode(event: DtmfEvent): number { + if (/^\d$/.test(event)) { + return Number(event); + } else if (event === DtmfEvent.STAR) { + return 10; + } else if (event === DtmfEvent.POUND) { + return 11; + } else if (['A', 'B', 'C', 'D'].includes(event)) { + // DTMF codes 12-15 are used for letters A-D + return event.charCodeAt(0) - 'A'.charCodeAt(0) + 12; + } + throw new Error(`Invalid DTMF event: ${event}`); +} + +export function formatDtmf(events: readonly DtmfEvent[]): string { + return events.join(' '); +} From 5af519acc4ce27f8f7d380b62095e8674fc7cc18 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 07:39:23 +0000 Subject: [PATCH 2/2] fix(workflows): address credit-card review findings - Skip TaskGroup chat-context summarization when the session runs a realtime model: TaskGroup.onEnter requires a standard LLM and would otherwise fail after all card details were collected. - Forward task-level overrides (llm, tts, stt, vad, turnDetection, tools, allowInterruptions) from createGetCreditCardTask into the card sub-tasks; previously they attached only to the outer placeholder task and sub-tasks silently ran with session defaults. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q3ZnFWr54AZysyyf4g5Dyd --- agents/src/workflows/credit_card.ts | 100 +++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 10 deletions(-) diff --git a/agents/src/workflows/credit_card.ts b/agents/src/workflows/credit_card.ts index a4b43b34a..1c54125d5 100644 --- a/agents/src/workflows/credit_card.ts +++ b/agents/src/workflows/credit_card.ts @@ -3,8 +3,8 @@ // SPDX-License-Identifier: Apache-2.0 import { z } from 'zod'; import type { LLMModels, STTModelString, TTSModelString } from '../inference/index.js'; -import type { ChatContext, LLM, RealtimeModel, ToolContextEntry } from '../llm/index.js'; -import { Instructions, ToolError, ToolFlag, tool } from '../llm/index.js'; +import type { ChatContext, RealtimeModel, ToolContextEntry } from '../llm/index.js'; +import { Instructions, LLM, ToolError, ToolFlag, tool } from '../llm/index.js'; import { isToolError } from '../llm/tool_context.js'; import type { STT } from '../stt/index.js'; import type { TTS } from '../tts/index.js'; @@ -122,6 +122,13 @@ interface SubTaskOptions { requireConfirmation?: boolean; requireExplicitAsk?: boolean; extraInstructions?: string; + turnDetection?: TurnDetectionMode | null; + tools?: readonly ToolContextEntry[]; + stt?: STT | STTModelString | null; + vad?: VAD | null; + llm?: LLM | RealtimeModel | LLMModels | null; + tts?: TTS | TTSModelString | null; + allowInterruptions?: boolean; } function buildSubTaskInstructions( @@ -152,6 +159,13 @@ function createGetCardNumberTask({ requireConfirmation, requireExplicitAsk = false, extraInstructions = '', + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, }: SubTaskOptions = {}): AgentTask { let currentCardNumber = ''; @@ -255,7 +269,18 @@ function createGetCardNumberTask({ extraInstructions, ), chatCtx, - tools: [updateCardNumberTool, declineCardCaptureTool, restartCardCollectionTool], + turnDetection: turnDetection ?? undefined, + tools: [ + ...(tools ?? []), + updateCardNumberTool, + declineCardCaptureTool, + restartCardCollectionTool, + ], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, onEnter: async () => { await task.session.generateReply({ instructions: @@ -276,6 +301,13 @@ function createGetSecurityCodeTask({ requireConfirmation, requireExplicitAsk = false, extraInstructions = '', + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, }: SubTaskOptions = {}): AgentTask { let currentSecurityCode = ''; @@ -360,7 +392,18 @@ function createGetSecurityCodeTask({ extraInstructions, ), chatCtx, - tools: [updateSecurityCodeTool, declineCardCaptureTool, restartCardCollectionTool], + turnDetection: turnDetection ?? undefined, + tools: [ + ...(tools ?? []), + updateSecurityCodeTool, + declineCardCaptureTool, + restartCardCollectionTool, + ], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, onEnter: async () => { await task.session.generateReply({ instructions: @@ -379,6 +422,13 @@ function createGetExpirationDateTask({ requireConfirmation, requireExplicitAsk = false, extraInstructions = '', + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, }: SubTaskOptions = {}): AgentTask { let currentExpirationDate = ''; @@ -512,7 +562,18 @@ function createGetExpirationDateTask({ extraInstructions, ), chatCtx, - tools: [updateExpirationDateTool, declineCardCaptureTool, restartCardCollectionTool], + turnDetection: turnDetection ?? undefined, + tools: [ + ...(tools ?? []), + updateExpirationDateTool, + declineCardCaptureTool, + restartCardCollectionTool, + ], + stt: stt ?? undefined, + vad: vad ?? undefined, + llm: llm ?? undefined, + tts: tts ?? undefined, + allowInterruptions, onEnter: async () => { await task.session.generateReply({ instructions: @@ -593,6 +654,19 @@ export function createGetCreditCardTask({ cardholderExtra = `${extraInstructions}\n\n${cardholderExtra}`; } + // Task-level model/tool overrides must reach the sub-tasks: the outer + // placeholder never converses, the sub-tasks do. + const subTaskOptions = { + requireConfirmation, + turnDetection, + tools, + stt, + vad, + llm, + tts, + allowInterruptions, + }; + while (!task.done) { // Order: number first (most natural for the caller to give // when asked for "card details"), then expiry and security @@ -601,12 +675,18 @@ export function createGetCreditCardTask({ // leaving it for the end avoids the failure mode where the // caller's first response (typically the digits) gets crammed // into update_name. - const taskGroup = new TaskGroup({ chatCtx: ctx }); + const taskGroup = new TaskGroup({ + chatCtx: ctx, + // TaskGroup summarization requires a standard LLM on the session; + // skip it on realtime-model sessions instead of failing after all + // card details were collected. + summarizeChatCtx: task.session.llm instanceof LLM, + }); taskGroup.add( () => createGetCardNumberTask({ + ...subTaskOptions, chatCtx: ctx, - requireConfirmation, extraInstructions, }), { @@ -617,8 +697,8 @@ export function createGetCreditCardTask({ taskGroup.add( () => createGetExpirationDateTask({ + ...subTaskOptions, chatCtx: ctx, - requireConfirmation, extraInstructions, }), { @@ -629,8 +709,8 @@ export function createGetCreditCardTask({ taskGroup.add( () => createGetSecurityCodeTask({ + ...subTaskOptions, chatCtx: ctx, - requireConfirmation, extraInstructions, }), { @@ -641,10 +721,10 @@ export function createGetCreditCardTask({ taskGroup.add( () => createGetNameTask({ + ...subTaskOptions, lastName: true, chatCtx: ctx, extraInstructions: cardholderExtra, - requireConfirmation, // The cardholder may differ from the caller or any guest // mentioned earlier in chatCtx. Apply IGNORE_ON_ENTER on // update_name so the model must produce an asking turn