feat: make LLM and TTS timeouts optional, improve Float32Array audio input handling, and refine human turn metric recording

Author: abdufelsayed

packages/talkio/src/agent/actors/llm.ts

@@ -54,10 +54,10 @@ export const llmActor = fromCallback<
   }
 
   // Set up timeout
-  const timeoutMs = config.timeout?.llmMs ?? 30000;
+  const timeoutMs = config.timeout?.llmMs;
   let timeoutId: ReturnType<typeof setTimeout> | null = null;
 
-  if (timeoutMs > 0) {
+  if (timeoutMs && timeoutMs > 0) {
     timeoutId = setTimeout(() => {
       if (!isAborted) {
         isAborted = true;

packages/talkio/src/agent/actors/tts.ts

@@ -52,10 +52,10 @@ export const ttsActor = fromCallback<
   }
 
   // Set up timeout
-  const timeoutMs = config.timeout?.ttsMs ?? 10000;
+  const timeoutMs = config.timeout?.ttsMs;
   let timeoutId: ReturnType<typeof setTimeout> | null = null;
 
-  if (timeoutMs > 0) {
+  if (timeoutMs && timeoutMs > 0) {
     timeoutId = setTimeout(() => {
       if (!isAborted) {
         isAborted = true;

packages/talkio/src/agent/create-agent.ts

@@ -22,6 +22,7 @@
 import { nanoid } from "nanoid";
 import { createActor } from "xstate";
 
+import { float32ToLinear16 } from "../audio/conversions";
 import type { AudioInput } from "../audio/preprocessor";
 import { normalizeFormat, type AudioFormat } from "../audio/types";
 import type { STTProvider, TTSProvider } from "../providers/types";
@@ -268,13 +269,23 @@ function isRunningState(value: string | Record<string, unknown>): boolean {
  * For Blob inputs, this will throw - use the async version or
  * convert the Blob to ArrayBuffer before calling sendAudio.
  */
-function toArrayBuffer(input: AudioInput): ArrayBuffer {
+function toArrayBuffer(input: AudioInput, encoding: AudioFormat["encoding"]): ArrayBuffer {
   if (input instanceof ArrayBuffer) {
     return input;
   }
 
-  if (input instanceof Float32Array || input instanceof Int16Array || input instanceof Uint8Array) {
-    // Create a copy to ensure we have a standalone ArrayBuffer
+  if (input instanceof Float32Array) {
+    if (encoding === "linear16") {
+      return float32ToLinear16(input);
+    }
+    if (encoding === "float32") {
+      const buffer = new ArrayBuffer(input.byteLength);
+      new Float32Array(buffer).set(input);
+      return buffer;
+    }
+  }
+
+  if (input instanceof Int16Array || input instanceof Uint8Array) {
     const buffer = new ArrayBuffer(input.byteLength);
     new Uint8Array(buffer).set(new Uint8Array(input.buffer, input.byteOffset, input.byteLength));
     return buffer;
@@ -465,7 +476,7 @@ export function createAgent<
     sendAudio(audio: AudioInput) {
       if (isStopped) return;
       // Convert input to ArrayBuffer synchronously
-      const arrayBuffer = toArrayBuffer(audio);
+      const arrayBuffer = toArrayBuffer(audio, inputFormat.encoding);
       actor.send({ type: "_audio:input", audio: arrayBuffer, timestamp: Date.now() });
     },
 

packages/talkio/src/agent/machine.ts

@@ -80,7 +80,7 @@ const agentMachineSetup = setup({
     },
 
     sttSpeechDurationMet: ({ context }) => {
-      if (!context.speechStartedAt) return false;
+      if (context.speechStartedAt === null) return false;
       const minDuration = context.config.interruption?.minDurationMs ?? 200;
       return Date.now() - context.speechStartedAt >= minDuration;
     },
@@ -390,11 +390,14 @@ const agentMachineSetup = setup({
       }),
     }),
     recordHumanTurnStart: assign({
-      metrics: ({ context }) => ({
-        ...context.metrics,
-        humanTurnStartTime: Date.now(),
-        totalTurns: context.metrics.totalTurns + 1,
-      }),
+      metrics: ({ context }) => {
+        if (context.metrics.humanTurnStartTime !== null) return context.metrics;
+        return {
+          ...context.metrics,
+          humanTurnStartTime: Date.now(),
+          totalTurns: context.metrics.totalTurns + 1,
+        };
+      },
     }),
 
     recordHumanTurnEnd: assign({
@@ -467,6 +470,17 @@ const agentMachineSetup = setup({
         };
       },
     }),
+    recordResponseLengthFromComplete: assign({
+      metrics: ({ context, event }) => {
+        if (event.type !== "_llm:complete") return context.metrics;
+        const responseLength = event.fullText.length;
+        if (responseLength <= context.metrics.currentResponseLength) return context.metrics;
+        return {
+          ...context.metrics,
+          currentResponseLength: responseLength,
+        };
+      },
+    }),
 
     recordFirstSentence: assign({
       metrics: ({ context }) => {
@@ -808,6 +822,7 @@ export const agentMachine = agentMachineSetup.createMachine({
                 actions: [
                   { type: "setHumanTurnStarted" },
                   { type: "emitHumanTurnStarted" },
+                  { type: "recordHumanTurnStart" },
                   { type: "debugLogEvent" },
                   { type: "emitHumanTurnTranscript" },
                   { type: "updatePartialTranscriptFromEvent" },
@@ -832,6 +847,7 @@ export const agentMachine = agentMachineSetup.createMachine({
                 actions: [
                   { type: "setHumanTurnStarted" },
                   { type: "emitHumanTurnStarted" },
+                  { type: "recordHumanTurnStart" },
                   { type: "debugLogEvent" },
                   { type: "emitHumanTurnTranscript" },
                   { type: "forwardToTurnDetector" },
@@ -860,6 +876,7 @@ export const agentMachine = agentMachineSetup.createMachine({
                 actions: [
                   { type: "setHumanTurnStarted" },
                   { type: "emitHumanTurnStarted" },
+                  { type: "recordHumanTurnStart" },
                   { type: "debugLogEvent" },
                   { type: "recordHumanTurnEnd" },
                   { type: "emitHumanTurnTranscript" },
@@ -920,6 +937,7 @@ export const agentMachine = agentMachineSetup.createMachine({
                 actions: [
                   { type: "setHumanTurnStarted" },
                   { type: "emitHumanTurnStarted" },
+                  { type: "recordHumanTurnStart" },
                   { type: "debugLogEvent" },
                   { type: "recordHumanTurnEnd" },
                   { type: "emitHumanTurnTranscript" },
@@ -1042,14 +1060,19 @@ export const agentMachine = agentMachineSetup.createMachine({
                 guard: "aiTurnHadNoAudio",
                 actions: [
                   { type: "clearLLMRef" },
+                  { type: "recordResponseLengthFromComplete" },
                   { type: "recordAITurnComplete" },
                   { type: "emitAITurnEnded" },
                   { type: "addAssistantMessageFromEvent" },
                   { type: "resetTurnMetrics" },
                 ],
               },
               {
-                actions: [{ type: "clearLLMRef" }, { type: "addAssistantMessageFromEvent" }],
+                actions: [
+                  { type: "clearLLMRef" },
+                  { type: "recordResponseLengthFromComplete" },
+                  { type: "addAssistantMessageFromEvent" },
+                ],
               },
             ],
           },
@@ -1177,7 +1200,11 @@ export const agentMachine = agentMachineSetup.createMachine({
       },
     },
     stopped: {
-      entry: [{ type: "abortSessionController" }, { type: "emitAgentStopped" }],
+      entry: [
+        { type: "abortSessionController" },
+        { type: "abortTurnController" },
+        { type: "emitAgentStopped" },
+      ],
       type: "final",
     },
   },

packages/talkio/src/types/config.ts

@@ -79,10 +79,10 @@ export interface InterruptionConfig {
  * ```
  */
 export interface TimeoutConfig {
-  /** Timeout for LLM generation in milliseconds. @default 30000 */
+  /** Timeout for LLM generation in milliseconds. Undefined disables the timeout. */
   llmMs?: number;
 
-  /** Timeout for TTS synthesis in milliseconds. @default 10000 */
+  /** Timeout for TTS synthesis in milliseconds. Undefined disables the timeout. */
   ttsMs?: number;
 }