feat(agents): add Toolset support to ToolContext and AgentActivity

.changeset/list-syntax-toolcontext.md

@@ -1,5 +1,5 @@
 ---
-"@livekit/agents": minor
+'@livekit/agents': minor
 ---
 
-**BREAKING**: `Agent({ tools })` and `agent.updateTools()` now accept a flat list `(FunctionTool | ProviderDefinedTool)[]` instead of a `Record<string, FunctionTool>` map, and `llm.tool({ ... })` requires a `name` field. `ToolContext` is now a Python-parity class with `functionTools` / `providerTools` / `toolsets` accessors, plus `flatten()`, `hasTool(name)`, `getFunctionTool(name)`, `updateTools()`, `copy()`, and `equals()`. To match the Python reference, registering two **different** function-tool instances under the same `name` now throws `duplicate function name: <name>` instead of silently overriding the earlier entry; passing the **same instance** twice is a no-op. `agent.toolCtx` returns a defensive copy so callers can no longer mutate the agent's internal state. `LLM.chat({ toolCtx })` accepts either a `ToolContext` instance or a raw `(FunctionTool | ProviderDefinedTool)[]` array (`ToolCtxInput`) and normalizes it internally, so callers don't have to construct a `ToolContext` themselves. Stateful `Toolset` containers are not part of this release — the `toolsets` accessor currently returns an empty list and `TODO`s in `tool_context.ts` mark every site where Python's Toolset support will plug in later.
+**BREAKING**: `Agent({ tools })` and `agent.updateTools()` now accept a flat list `(FunctionTool | ProviderDefinedTool | Toolset)[]` instead of a `Record<string, FunctionTool>` map, and `llm.tool({ ... })` requires a `name` field. `ToolContext` is now a Python-parity class with `functionTools` / `providerTools` / `toolsets` accessors, plus `flatten()`, `hasTool(name)`, `getFunctionTool(name)`, `updateTools()`, `copy()`, and `equals()`. To match the Python reference, registering two **different** function-tool instances under the same `name` now throws `duplicate function name: <name>` instead of silently overriding the earlier entry; passing the **same instance** twice is a no-op. `agent.toolCtx` returns a defensive copy so callers can no longer mutate the agent's internal state. `LLM.chat({ toolCtx })` accepts either a `ToolContext` instance or a raw `(FunctionTool | ProviderDefinedTool | Toolset)[]` array (`ToolCtxInput`) and normalizes it internally, so callers don't have to construct a `ToolContext` themselves.

.changeset/quick-meals-breathe.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents': patch
+---
+
+Adds base `Toolset` support: a stateful container for a group of tools with `setup()` / `aclose()` lifecycle hooks. Toolsets can be passed directly into `Agent({ tools: [...] })` alongside individual function tools; their tools are flattened into the agent's `ToolContext` and the runtime drives `setup()` on activity start, `aclose()` on close, and a setup/close diff when `agent.updateTools()` adds or removes Toolsets mid-session. Per-toolset `setup()` errors are logged but do not abort the activity. The `IGNORE_ON_ENTER` flag is also respected for function tools nested inside a Toolset. Every LLM and realtime plugin tool builder iterates `ToolContext.flatten()` so toolset-contributed tools are correctly advertised. Also exports `ToolCalledEvent` / `ToolCompletedEvent` payload types.

agents/src/llm/index.ts

@@ -10,12 +10,15 @@ export {
   ToolContext,
   ToolError,
   ToolFlag,
+  Toolset,
   toToolContext,
   type AgentHandoff,
   type FunctionTool,
   type ProviderDefinedTool,
   type Tool,
+  type ToolCalledEvent,
   type ToolChoice,
+  type ToolCompletedEvent,
   type ToolContextEntry,
   type ToolCtxInput,
   type ToolOptions,

agents/src/llm/tool_context.test.ts

@@ -5,7 +5,7 @@ import { describe, expect, it } from 'vitest';
 import { z } from 'zod';
 import * as z3 from 'zod/v3';
 import * as z4 from 'zod/v4';
-import { ToolContext, type ToolOptions, tool } from './tool_context.js';
+import { ToolContext, type ToolOptions, Toolset, tool } from './tool_context.js';
 import { createToolOptions, oaiParams } from './utils.js';
 
 describe('Tool Context', () => {
@@ -580,3 +580,78 @@ describe('ToolContext', () => {
     expect(ctx.flatten()).toEqual([b, a, provider]);
   });
 });
+
+describe('Toolset', () => {
+  const makeFn = (name: string) =>
+    tool({
+      name,
+      description: `${name} tool`,
+      execute: async () => name,
+    });
+
+  it('exposes its id and the tools it was constructed with', () => {
+    const a = makeFn('a');
+    const b = makeFn('b');
+    const ts = new Toolset({ id: 'set1', tools: [a, b] });
+
+    expect(ts.id).toBe('set1');
+    expect(ts.tools).toEqual([a, b]);
+  });
+
+  it('default setup and aclose are no-ops', async () => {
+    const ts = new Toolset({ id: 'noop', tools: [] });
+    await expect(ts.setup()).resolves.toBeUndefined();
+    await expect(ts.aclose()).resolves.toBeUndefined();
+  });
+
+  it('lets subclasses override lifecycle hooks', async () => {
+    const events: string[] = [];
+    class Recording extends Toolset {
+      override async setup(): Promise<void> {
+        events.push(`setup:${this.id}`);
+      }
+      override async aclose(): Promise<void> {
+        events.push(`close:${this.id}`);
+      }
+    }
+
+    const ts = new Recording({ id: 'rec', tools: [] });
+    await ts.setup();
+    await ts.aclose();
+    expect(events).toEqual(['setup:rec', 'close:rec']);
+  });
+
+  it('is flattened into a ToolContext: function tools merged, toolset tracked', () => {
+    const a = makeFn('a');
+    const b = makeFn('b');
+    const ts = new Toolset({ id: 'set', tools: [a, b] });
+    const direct = makeFn('direct');
+
+    const ctx = new ToolContext([direct, ts]);
+
+    expect(Object.keys(ctx.functionTools).sort()).toEqual(['a', 'b', 'direct']);
+    expect(ctx.toolsets).toEqual([ts]);
+  });
+
+  it('throws when a Toolset contributes a duplicate function name', () => {
+    // Mirrors Python's `add_tool`: a name collision between top-level and toolset-contributed
+    // tools is an error, not silent overwrite.
+    const a1 = makeFn('a');
+    const a2 = makeFn('a');
+    const ts = new Toolset({ id: 'collides', tools: [a2] });
+
+    expect(() => new ToolContext([a1, ts])).toThrow(/duplicate function name: a/);
+  });
+
+  it('equals() compares toolsets as identity sets, not by order', () => {
+    // Matches Python's `{id(ts) for ts in self._tool_sets}` semantics.
+    const ts1 = new Toolset({ id: 'one', tools: [] });
+    const ts2 = new Toolset({ id: 'two', tools: [] });
+
+    expect(new ToolContext([ts1, ts2]).equals(new ToolContext([ts2, ts1]))).toBe(true);
+
+    const ts3 = new Toolset({ id: 'three', tools: [] });
+    expect(new ToolContext([ts1, ts2]).equals(new ToolContext([ts1, ts3]))).toBe(false);
+    expect(new ToolContext([ts1]).equals(new ToolContext([ts1, ts2]))).toBe(false);
+  });
+});

agents/src/llm/tool_context.ts

@@ -196,6 +196,43 @@ export interface FunctionTool<
   [FUNCTION_TOOL_SYMBOL]: true;
 }
 
+export interface ToolCalledEvent<UserData = UnknownUserData> {
+  ctx: RunContext<UserData>;
+  arguments: Record<string, unknown>;
+}
+
+export interface ToolCompletedEvent<UserData = UnknownUserData> {
+  ctx: RunContext<UserData>;
+  output?: { type: 'output'; value: unknown } | { type: 'error'; value: Error };
+}
+
+/**
+ * A stateful collection of tools sharing a lifecycle. Tools registered through a `Toolset` are
+ * flattened into the surrounding `ToolContext`, while the `Toolset` itself is tracked so its
+ * `setup()` / `aclose()` hooks can be invoked by the agent runtime.
+ */
+export class Toolset {
+  readonly #id: string;
+  readonly #tools: Tool[];
+
+  constructor({ id, tools }: { id: string; tools: readonly Tool[] }) {
+    this.#id = id;
+    this.#tools = [...tools];
+  }
+
+  get id(): string {
+    return this.#id;
+  }
+
+  get tools(): readonly Tool[] {
+    return this.#tools;
+  }
+
+  async setup(): Promise<void> {}
+
+  async aclose(): Promise<void> {}
+}
+
 /**
  * Convenience input shape accepted by APIs that want to take a list of tools directly without
  * forcing callers to wrap them in `new ToolContext(...)`.
@@ -217,24 +254,18 @@ export function toToolContext<UserData = UnknownUserData>(
   return input instanceof ToolContext ? input : new ToolContext(input);
 }
 
-//TODO: toolset - accept stateful `Toolset` containers alongside `FunctionTool` /
 // eslint-disable-next-line @typescript-eslint/no-explicit-any -- ToolContext entries accept any function-tool parameter/result types
 export type ToolContextEntry<UserData = UnknownUserData> =
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  FunctionTool<any, UserData, any> | ProviderDefinedTool;
+  FunctionTool<any, UserData, any> | ProviderDefinedTool | Toolset;
 
 export class ToolContext<UserData = UnknownUserData> {
-  // TODO: toolset - widen entries to `FunctionTool | ProviderDefinedTool | Toolset` once Toolset
-  // lands so this stays heterogeneous like Python's `Sequence[Tool | Toolset]`.
   private _tools: ToolContextEntry<UserData>[] = [];
   // eslint-disable-next-line @typescript-eslint/no-explicit-any -- ToolContext stores generic function tools
   private _functionToolsMap: Map<string, FunctionTool<any, UserData, any>> = new Map();
   private _providerTools: ProviderDefinedTool[] = [];
-  // TODO: toolset - populate when Toolset support is supported.
-  // so the `toolsets` getter and `equals` toolset-identity check stay byte-compatible with the
-  private _toolSets: unknown[] = [];
+  private _toolsets: Toolset[] = [];
 
-  // TODO: toolset - widen `tools` to `Sequence<Tool | Toolset>` once Toolset lands.
   constructor(tools: readonly ToolContextEntry<UserData>[] = []) {
     this.updateTools(tools);
   }
@@ -254,13 +285,9 @@ export class ToolContext<UserData = UnknownUserData> {
     return this._providerTools;
   }
 
-  /**
-   * A copy of all tool sets in the tool context.
-   *
-   * TODO: toolset - wire up once Toolset is ported.
-   */
-  get toolsets(): unknown[] {
-    return this._toolSets;
+  /** A copy of all toolsets registered in the context. */
+  get toolsets(): readonly Toolset[] {
+    return [...this._toolsets];
   }
 
   /**
@@ -287,16 +314,22 @@ export class ToolContext<UserData = UnknownUserData> {
     return this._providerTools.some((tool) => tool.id === name);
   }
 
-  // TODO: toolset - widen `tools` to `Sequence<Tool | Toolset>` once Toolset lands.
   updateTools(tools: readonly ToolContextEntry<UserData>[]): void {
     this._tools = [...tools];
     this._functionToolsMap = new Map();
     this._providerTools = [];
-    this._toolSets = [];
+    this._toolsets = [];
 
-    // Mirrors Python's recursive `add_tool` (minus Toolset flattening, which is TODO).
     // eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any tool shape
     const addTool = (tool: any): void => {
+      if (tool instanceof Toolset) {
+        for (const inner of tool.tools) {
+          addTool(inner);
+        }
+        this._toolsets.push(tool);
+        return;
+      }
+
       if (isProviderDefinedTool(tool)) {
         this._providerTools.push(tool);
         return;
@@ -314,15 +347,9 @@ export class ToolContext<UserData = UnknownUserData> {
         return;
       }
 
-      // TODO: toolset - if (tool instanceof Toolset) { for (const t of tool.tools) addTool(t);
-      //                  this._toolSets.push(tool); return; }
-
       throw new Error(`unknown tool type: ${typeof tool}`);
     };
 
-    // TODO: toolset - Python also chains `find_function_tools(self)` here so subclasses can
-    // declare tools as class members. JS doesn't use that decorator pattern, so we only walk
-    // the explicit input list.
     for (const tool of tools) {
       addTool(tool);
     }
@@ -352,10 +379,15 @@ export class ToolContext<UserData = UnknownUserData> {
         return false;
       }
     }
-    // TODO: toolset - once Toolset lands, also compare `_toolSets` as identity sets per Python
-    //   self_tool_set_ids = {id(ts) for ts in self._tool_sets}
-    //   other_tool_set_ids = {id(ts) for ts in other._tool_sets}
-    //   if self_tool_set_ids != other_tool_set_ids: return False
+    if (this._toolsets.length !== other._toolsets.length) {
+      return false;
+    }
+    const otherToolsets = new Set(other._toolsets);
+    for (const ts of this._toolsets) {
+      if (!otherToolsets.has(ts)) {
+        return false;
+      }
+    }
     return true;
   }
 }

agents/src/voice/agent_activity.ts

@@ -31,14 +31,17 @@ import {
   type InputSpeechStartedEvent,
   type InputSpeechStoppedEvent,
   type InputTranscriptionCompleted,
+  isFunctionTool,
   LLM,
   RealtimeModel,
   type RealtimeModelError,
   type RealtimeSession,
+  type Tool,
   type ToolChoice,
   ToolContext,
   type ToolContextEntry,
   ToolFlag,
+  Toolset,
 } from '../llm/index.js';
 import type { LLMError } from '../llm/llm.js';
 import { isSameToolChoice } from '../llm/tool_context.js';
@@ -215,6 +218,7 @@ export class AgentActivity implements RecognitionHooks {
   private toolChoice: ToolChoice | null = null;
   private _preemptiveGeneration?: PreemptiveGeneration;
   private _preemptiveGenerationCount = 0;
+  private _toolsetsSetup = false;
   private interruptionDetector?: AdaptiveInterruptionDetector;
   private isInterruptionDetectionEnabled: boolean;
   private isInterruptionByAudioActivityEnabled: boolean;
@@ -421,6 +425,8 @@ export class AgentActivity implements RecognitionHooks {
 
     this.agent._agentActivity = this;
 
+    await this.setupToolsets();
+
     if (this.llm instanceof RealtimeModel) {
       const rtReused = reuseResources?.rtSession !== undefined;
 
@@ -767,13 +773,20 @@ export class AgentActivity implements RecognitionHooks {
   }
 
   async updateTools(tools: readonly ToolContextEntry<any>[]): Promise<void> {
-    const oldToolNames = new Set(Object.keys(this.agent._toolCtx.functionTools));
+    const oldToolCtx = this.agent._toolCtx;
+    const oldToolNames = new Set(Object.keys(oldToolCtx.functionTools));
+    const oldToolsets = oldToolCtx.toolsets;
     const newToolCtx = new ToolContext(tools);
     const newToolNames = new Set(Object.keys(newToolCtx.functionTools));
+    const newToolsets = newToolCtx.toolsets;
     const toolsAdded = [...newToolNames].filter((name) => !oldToolNames.has(name));
     const toolsRemoved = [...oldToolNames].filter((name) => !newToolNames.has(name));
+    const addedToolsets = newToolsets.filter((ts) => !oldToolsets.includes(ts));
+    const removedToolsets = oldToolsets.filter((ts) => !newToolsets.includes(ts));
 
+    await this.setupToolsetList(addedToolsets);
     this.agent._toolCtx = newToolCtx;
+    await this.closeToolsetList(removedToolsets);
 
     if (toolsAdded.length > 0 || toolsRemoved.length > 0) {
       const configUpdate = new AgentConfigUpdate({
@@ -1735,11 +1748,13 @@ export class AgentActivity implements RecognitionHooks {
 
       const tools: ToolContext = shouldFilterTools
         ? new ToolContext(
-            this.agent.toolCtx.tools.filter((t) => {
-              if (t.type === 'function') {
-                return !(t.flags & ToolFlag.IGNORE_ON_ENTER);
+            this.agent.toolCtx.tools.flatMap((t): ToolContextEntry[] => {
+              const keepFn = (fn: Tool): boolean =>
+                !isFunctionTool(fn) || !(fn.flags & ToolFlag.IGNORE_ON_ENTER);
+              if (t instanceof Toolset) {
+                return t.tools.filter(keepFn) as ToolContextEntry[];
               }
-              return true;
+              return keepFn(t) ? [t] : [];
             }),
           )
         : this.agent.toolCtx;
@@ -3728,9 +3743,42 @@ export class AgentActivity implements RecognitionHooks {
     this.realtimeSpans?.clear();
     await this.realtimeSession?.close();
     await this.audioRecognition?.close();
+    await this.closeToolsets();
     this.realtimeSession = undefined;
     this.audioRecognition = undefined;
   }
+
+  private async setupToolsets(): Promise<void> {
+    // Guard against resume() re-entering _startSession on an activity whose toolsets are
+    // already initialized.
+    if (this._toolsetsSetup) return;
+    this._toolsetsSetup = true;
+    await this.setupToolsetList(this.agent.toolCtx.toolsets);
+  }
+
+  private async closeToolsets(): Promise<void> {
+    if (!this._toolsetsSetup) return;
+    this._toolsetsSetup = false;
+    await this.closeToolsetList(this.agent.toolCtx.toolsets);
+  }
+
+  private async setupToolsetList(toolsets: readonly Toolset[]): Promise<void> {
+    const outputs = await Promise.allSettled(toolsets.map((ts) => ts.setup()));
+    for (const output of outputs) {
+      if (output.status === 'rejected') {
+        this.logger.error({ error: output.reason }, 'error setting up toolset');
+      }
+    }
+  }
+
+  private async closeToolsetList(toolsets: readonly Toolset[]): Promise<void> {
+    const outputs = await Promise.allSettled(toolsets.map((ts) => ts.aclose()));
+    for (const output of outputs) {
+      if (output.status === 'rejected') {
+        this.logger.error({ error: output.reason }, 'error closing toolset');
+      }
+    }
+  }
 }
 
 function toOaiToolChoice(toolChoice: ToolChoice | null): ToolChoice | undefined {