From 7d5b72f5fcb4d86e47617960967dbf89a67f9b71 Mon Sep 17 00:00:00 2001 From: dorianzheng Date: Sun, 19 Apr 2026 18:14:18 +0800 Subject: [PATCH 01/10] feat: add tool usage analytics --- src/agent/agent-impl.ts | 19 ++- src/agent/message-processor.ts | 62 +++++++++ src/api/events.ts | 17 +++ src/db.test.ts | 122 +++++++++++++++++ src/db.ts | 80 ++++++++++++ src/tool-usage.test.ts | 231 +++++++++++++++++++++++++++++++++ 6 files changed, 530 insertions(+), 1 deletion(-) create mode 100644 src/tool-usage.test.ts diff --git a/src/agent/agent-impl.ts b/src/agent/agent-impl.ts index 00c40ae67f2..16dea776bd5 100644 --- a/src/agent/agent-impl.ts +++ b/src/agent/agent-impl.ts @@ -38,7 +38,7 @@ import { AgentDb, initDatabase } from '../db.js'; import { resolveMountAllowlist } from '../mount-security.js'; import { GroupQueue } from '../group-queue.js'; import { writeGroupsSnapshot } from '../container-runner.js'; -import type { ZodRawShape } from 'zod'; +import { z, type ZodRawShape } from 'zod'; import { startIpcWatcher } from '../ipc.js'; import { ActionsHttp } from './actions-http.js'; @@ -462,6 +462,23 @@ export class AgentImpl ); } + this.actions.set('tool_usage_summary', { + description: + 'Returns per-tool call count, success rate, and average duration. ' + + 'Optionally filter by since (ISO timestamp) and tool_name.', + inputSchema: { + since: z.string().optional().describe('ISO timestamp lower bound'), + tool_name: z.string().optional().describe('Filter to a specific tool'), + }, + handler: async (payload) => { + const rows = this.db.getToolUsageSummary({ + since: payload.since as string | undefined, + toolName: payload.tool_name as string | undefined, + }); + return { summary: rows }; + }, + }); + await this.actionsHttp.start(); this.startSubsystems(); this.emit('started'); diff --git a/src/agent/message-processor.ts b/src/agent/message-processor.ts index bf9ceb8ca04..0471744d417 100644 --- a/src/agent/message-processor.ts +++ b/src/agent/message-processor.ts @@ -157,6 +157,10 @@ export class MessageProcessor { await channel.setTyping?.(chatJid, true); let hadError = false; let outputSentToUser = false; + const pendingToolCalls = new Map< + string, + { toolName: string; startedAt: number } + >(); const output = await this.runAgent( group, @@ -226,6 +230,10 @@ export class MessageProcessor { if (event.sdkType === 'assistant' && msg?.message?.content) { for (const block of msg.message.content) { if (block.type === 'tool_use' && block.name && block.id) { + pendingToolCalls.set(block.id, { + toolName: block.name, + startedAt: Date.now(), + }); this.ctx.emit('run.tool', { agentId: this.ctx.id, jid: chatJid, @@ -241,6 +249,40 @@ export class MessageProcessor { resetIdleTimer(); } + if (event.sdkType === 'user' && msg?.message?.content) { + for (const block of msg.message.content) { + if (block.type === 'tool_result' && block.tool_use_id) { + const pending = pendingToolCalls.get(block.tool_use_id); + if (pending) { + pendingToolCalls.delete(block.tool_use_id); + const durationMs = Date.now() - pending.startedAt; + const isError = block.is_error === true; + let errorMessage: string | undefined; + if (isError && block.content) { + if (typeof block.content === 'string') { + errorMessage = block.content.slice(0, 500); + } else if (Array.isArray(block.content)) { + errorMessage = block.content + .map((c: { text?: string }) => c.text ?? '') + .join('') + .slice(0, 500); + } + } + this.ctx.db.recordToolUsage({ + groupJid: chatJid, + sessionId: undefined, + toolName: pending.toolName, + success: !isError, + errorMessage, + durationMs, + ts: now, + }); + this.checkToolErrorRateAlert(chatJid, pending.toolName); + } + } + } + } + if (event.sdkType === 'tool_progress') { this.ctx.emit('run.tool_progress', { agentId: this.ctx.id, @@ -327,6 +369,26 @@ export class MessageProcessor { return true; } + private checkToolErrorRateAlert(chatJid: string, toolName: string): void { + const sinceHour = new Date(Date.now() - 60 * 60 * 1000).toISOString(); + const rows = this.ctx.db.getToolUsageSummary({ since: sinceHour, toolName }); + const row = rows[0]; + if (row && row.call_count >= 5 && row.success_rate < 0.8) { + logger.warn( + { toolName, callCount: row.call_count, successRate: row.success_rate }, + 'Tool error rate exceeded 20% in the last hour', + ); + this.ctx.emit('run.tool_alert', { + agentId: this.ctx.id, + jid: chatJid, + toolName, + callCount: row.call_count, + successRate: row.success_rate, + timestamp: new Date().toISOString(), + }); + } + } + /** Execute agent in a container for the given group. */ async runAgent( group: InternalRegisteredGroup, diff --git a/src/api/events.ts b/src/api/events.ts index 701b7e8dd87..698262ba7d0 100644 --- a/src/api/events.ts +++ b/src/api/events.ts @@ -11,6 +11,7 @@ export interface AgentEvents extends Record { 'run.state': [payload: RunStateEvent]; 'run.sdk_message': [payload: RunSdkMessageEvent]; 'run.tool': [payload: RunToolEvent]; + 'run.tool_alert': [payload: RunToolAlertEvent]; 'run.tool_progress': [payload: RunToolProgressEvent]; 'run.subagent': [payload: RunSubagentEvent]; 'run.status': [payload: RunStatusEvent]; @@ -132,6 +133,22 @@ export interface RunToolEvent { timestamp: string; } +/** Tool error-rate alert for the last hour window. */ +export interface RunToolAlertEvent { + /** Stable agent identifier. */ + agentId: string; + /** Group/chat identifier. */ + jid: string; + /** Tool name. */ + toolName: string; + /** Number of calls in the alert window. */ + callCount: number; + /** Success rate in the alert window. */ + successRate: number; + /** ISO timestamp. */ + timestamp: string; +} + /** Tool execution progress heartbeat. */ export interface RunToolProgressEvent { /** Stable agent identifier. */ diff --git a/src/db.test.ts b/src/db.test.ts index b33a1d4adec..5a437514b37 100644 --- a/src/db.test.ts +++ b/src/db.test.ts @@ -527,3 +527,125 @@ describe('registered group isMain', () => { expect(group.isMain).toBeUndefined(); }); }); + +describe('tool_usage', () => { + it('records usage and returns correct aggregates', () => { + db.recordToolUsage({ + groupJid: 'group@g.us', + toolName: 'Bash', + success: true, + durationMs: 120, + ts: '2026-04-19T00:00:00.000Z', + }); + + expect(db.getToolUsageSummary()).toEqual([ + { + tool_name: 'Bash', + call_count: 1, + success_count: 1, + success_rate: 1, + avg_duration_ms: 120, + }, + ]); + }); + + it('computes mixed success and failure rates correctly', () => { + db.recordToolUsage({ + groupJid: 'group@g.us', + toolName: 'Bash', + success: true, + durationMs: 100, + ts: '2026-04-19T00:00:00.000Z', + }); + db.recordToolUsage({ + groupJid: 'group@g.us', + toolName: 'Bash', + success: true, + durationMs: 110, + ts: '2026-04-19T00:01:00.000Z', + }); + db.recordToolUsage({ + groupJid: 'group@g.us', + toolName: 'Bash', + success: true, + durationMs: 120, + ts: '2026-04-19T00:02:00.000Z', + }); + db.recordToolUsage({ + groupJid: 'group@g.us', + toolName: 'Bash', + success: false, + errorMessage: 'boom', + durationMs: 130, + ts: '2026-04-19T00:03:00.000Z', + }); + + expect(db.getToolUsageSummary()).toEqual([ + { + tool_name: 'Bash', + call_count: 4, + success_count: 3, + success_rate: 0.75, + avg_duration_ms: 115, + }, + ]); + }); + + it('applies since filter to exclude older rows', () => { + db.recordToolUsage({ + groupJid: 'group@g.us', + toolName: 'Read', + success: true, + durationMs: 50, + ts: '2026-04-18T23:00:00.000Z', + }); + db.recordToolUsage({ + groupJid: 'group@g.us', + toolName: 'Read', + success: false, + errorMessage: 'timeout', + durationMs: 150, + ts: '2026-04-19T01:00:00.000Z', + }); + + expect( + db.getToolUsageSummary({ since: '2026-04-19T00:00:00.000Z' }), + ).toEqual([ + { + tool_name: 'Read', + call_count: 1, + success_count: 0, + success_rate: 0, + avg_duration_ms: 150, + }, + ]); + }); + + it('filters to a specific tool name', () => { + db.recordToolUsage({ + groupJid: 'group@g.us', + toolName: 'Read', + success: true, + durationMs: 80, + ts: '2026-04-19T00:00:00.000Z', + }); + db.recordToolUsage({ + groupJid: 'group@g.us', + toolName: 'Bash', + success: false, + errorMessage: 'permission denied', + durationMs: 180, + ts: '2026-04-19T00:01:00.000Z', + }); + + expect(db.getToolUsageSummary({ toolName: 'Read' })).toEqual([ + { + tool_name: 'Read', + call_count: 1, + success_count: 1, + success_rate: 1, + avg_duration_ms: 80, + }, + ]); + }); +}); diff --git a/src/db.ts b/src/db.ts index 5965c9cd244..1820d8a65af 100644 --- a/src/db.ts +++ b/src/db.ts @@ -82,6 +82,19 @@ export function createSchema( container_config TEXT, requires_trigger INTEGER DEFAULT 1 ); + CREATE TABLE IF NOT EXISTS tool_usage ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + group_jid TEXT NOT NULL, + session_id TEXT, + tool_name TEXT NOT NULL, + success INTEGER NOT NULL DEFAULT 1, + error_message TEXT, + duration_ms INTEGER NOT NULL, + ts TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_tool_usage_ts ON tool_usage(ts); + CREATE INDEX IF NOT EXISTS idx_tool_usage_tool_name ON tool_usage(tool_name); + CREATE INDEX IF NOT EXISTS idx_tool_usage_group_jid ON tool_usage(group_jid); `); @@ -523,6 +536,73 @@ export class AgentDb { ); } + recordToolUsage(entry: { + groupJid: string; + sessionId?: string; + toolName: string; + success: boolean; + errorMessage?: string; + durationMs: number; + ts: string; + }): void { + this.db + .prepare( + ` + INSERT INTO tool_usage (group_jid, session_id, tool_name, success, error_message, duration_ms, ts) + VALUES (?, ?, ?, ?, ?, ?, ?) + `, + ) + .run( + entry.groupJid, + entry.sessionId ?? null, + entry.toolName, + entry.success ? 1 : 0, + entry.errorMessage ?? null, + entry.durationMs, + entry.ts, + ); + } + + getToolUsageSummary(opts?: { + since?: string; + toolName?: string; + }): Array<{ + tool_name: string; + call_count: number; + success_count: number; + success_rate: number; + avg_duration_ms: number; + }> { + return this.db + .prepare( + ` + SELECT + tool_name, + COUNT(*) AS call_count, + SUM(success) AS success_count, + CAST(SUM(success) AS REAL) / COUNT(*) AS success_rate, + CAST(SUM(duration_ms) AS REAL) / COUNT(*) AS avg_duration_ms + FROM tool_usage + WHERE (? IS NULL OR ts >= ?) + AND (? IS NULL OR tool_name = ?) + GROUP BY tool_name + ORDER BY call_count DESC + `, + ) + .all( + opts?.since ?? null, + opts?.since ?? null, + opts?.toolName ?? null, + opts?.toolName ?? null, + ) as Array<{ + tool_name: string; + call_count: number; + success_count: number; + success_rate: number; + avg_duration_ms: number; + }>; + } + // --- Router state --- getRouterState(key: string): string | undefined { diff --git a/src/tool-usage.test.ts b/src/tool-usage.test.ts new file mode 100644 index 00000000000..731bb7f18e5 --- /dev/null +++ b/src/tool-usage.test.ts @@ -0,0 +1,231 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('./container-runner.js', async () => { + const actual = await vi.importActual( + './container-runner.js', + ); + return { + ...actual, + runContainerAgent: vi.fn(), + }; +}); + +import { AgentImpl } from './agent/agent-impl.js'; +import { + buildAgentConfig, + resolveSerializableAgentSettings, +} from './agent/config.js'; +import { _initTestDatabase, AgentDb } from './db.js'; +import { buildRuntimeConfig } from './runtime-config.js'; +import { runContainerAgent } from './container-runner.js'; +import type { Channel, RegisteredGroup } from './types.js'; + +const runtimeConfig = buildRuntimeConfig( + { timezone: 'UTC' }, + '/tmp/agentlite-test-pkg', +); + +const MAIN_GROUP: RegisteredGroup = { + name: 'Main', + folder: 'main', + trigger: 'always', + added_at: '2024-01-01T00:00:00.000Z', + isMain: true, +}; + +let tmpDir: string; +let db: AgentDb; + +function createAgent(name: string): AgentImpl { + const config = buildAgentConfig({ + agentId: `${name}00000000`.slice(0, 8), + ...resolveSerializableAgentSettings( + name, + { workdir: path.join(tmpDir, 'agents', name) }, + tmpDir, + ), + }); + return new AgentImpl(config, runtimeConfig); +} + +function createMockChannel(): Channel { + return { + name: 'mock', + async connect(): Promise {}, + async disconnect(): Promise {}, + async sendMessage(): Promise {}, + isConnected(): boolean { + return true; + }, + ownsJid(jid: string): boolean { + return jid === 'mock:tool-usage'; + }, + async setTyping(): Promise {}, + }; +} + +function setupAgent(): AgentImpl { + const agent = createAgent('tool-usage'); + agent._setDbForTests(db); + agent._setRegisteredGroups({ 'mock:tool-usage': MAIN_GROUP }); + (agent as unknown as { _started: boolean })._started = true; + (agent as unknown as { channels: Map }).channels.set( + 'mock', + createMockChannel(), + ); + + db.storeChatMetadata( + 'mock:tool-usage', + '2026-04-19T00:00:00.000Z', + 'Tool Usage Chat', + ); + db.storeMessage({ + id: 'msg-1', + chat_jid: 'mock:tool-usage', + sender: 'user1', + sender_name: 'User 1', + content: 'run the tool', + timestamp: '2026-04-19T00:00:01.000Z', + is_from_me: false, + }); + + return agent; +} + +function sdkMsg(sdkType: string, message: unknown, sdkSubtype?: string) { + return { type: 'sdk_message' as const, sdkType, sdkSubtype, message }; +} + +describe('tool usage analytics', () => { + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agentlite-tool-usage-')); + db = _initTestDatabase(); + vi.mocked(runContainerAgent).mockReset(); + }); + + afterEach(() => { + try { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } catch { + /* ignore */ + } + }); + + it('records successful tool results from tool_use/tool_result SDK messages', async () => { + const agent = setupAgent(); + + vi.mocked(runContainerAgent).mockImplementation( + async (_group, _input, _rc, _onProcess, onOutput) => { + await onOutput?.( + sdkMsg('assistant', { + uuid: 'a1', + message: { + content: [ + { + type: 'tool_use', + name: 'Bash', + id: 'tool-1', + input: { command: 'pwd' }, + }, + ], + }, + }), + ); + await onOutput?.( + sdkMsg('user', { + uuid: 'u1', + message: { + content: [ + { + type: 'tool_result', + tool_use_id: 'tool-1', + is_error: false, + content: 'ok', + }, + ], + }, + }), + ); + await onOutput?.({ + type: 'state', + state: 'stopped', + reason: 'exit', + exitCode: 0, + }); + return { status: 'success', result: null }; + }, + ); + + await agent.processGroupMessages('mock:tool-usage'); + + expect(db.getToolUsageSummary()).toEqual([ + expect.objectContaining({ + tool_name: 'Bash', + call_count: 1, + success_count: 1, + success_rate: 1, + }), + ]); + }); + + it('records failed tool results as success_rate 0', async () => { + const agent = setupAgent(); + + vi.mocked(runContainerAgent).mockImplementation( + async (_group, _input, _rc, _onProcess, onOutput) => { + await onOutput?.( + sdkMsg('assistant', { + uuid: 'a1', + message: { + content: [ + { + type: 'tool_use', + name: 'Bash', + id: 'tool-2', + input: { command: 'false' }, + }, + ], + }, + }), + ); + await onOutput?.( + sdkMsg('user', { + uuid: 'u1', + message: { + content: [ + { + type: 'tool_result', + tool_use_id: 'tool-2', + is_error: true, + content: 'command failed', + }, + ], + }, + }), + ); + await onOutput?.({ + type: 'state', + state: 'stopped', + reason: 'exit', + exitCode: 0, + }); + return { status: 'success', result: null }; + }, + ); + + await agent.processGroupMessages('mock:tool-usage'); + + expect(db.getToolUsageSummary()).toEqual([ + expect.objectContaining({ + tool_name: 'Bash', + call_count: 1, + success_count: 0, + success_rate: 0, + }), + ]); + }); +}); From e32ab512336211f590cd1c76f225c118de030f5f Mon Sep 17 00:00:00 2001 From: dorianzheng Date: Sun, 19 Apr 2026 18:14:47 +0800 Subject: [PATCH 02/10] style: apply formatting after analytics changes --- src/agent/message-processor.ts | 5 ++++- src/db.ts | 5 +---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/agent/message-processor.ts b/src/agent/message-processor.ts index 0471744d417..d26e07d268d 100644 --- a/src/agent/message-processor.ts +++ b/src/agent/message-processor.ts @@ -371,7 +371,10 @@ export class MessageProcessor { private checkToolErrorRateAlert(chatJid: string, toolName: string): void { const sinceHour = new Date(Date.now() - 60 * 60 * 1000).toISOString(); - const rows = this.ctx.db.getToolUsageSummary({ since: sinceHour, toolName }); + const rows = this.ctx.db.getToolUsageSummary({ + since: sinceHour, + toolName, + }); const row = rows[0]; if (row && row.call_count >= 5 && row.success_rate < 0.8) { logger.warn( diff --git a/src/db.ts b/src/db.ts index 1820d8a65af..04d276199f9 100644 --- a/src/db.ts +++ b/src/db.ts @@ -563,10 +563,7 @@ export class AgentDb { ); } - getToolUsageSummary(opts?: { - since?: string; - toolName?: string; - }): Array<{ + getToolUsageSummary(opts?: { since?: string; toolName?: string }): Array<{ tool_name: string; call_count: number; success_count: number; From 55cc68e37f5954582c562d96f747be732c59f89b Mon Sep 17 00:00:00 2001 From: dorianzheng Date: Sun, 19 Apr 2026 18:25:40 +0800 Subject: [PATCH 03/10] fix: finalize tool usage analytics --- src/acp/client.e2e.test.ts | 2 +- src/agent/action-registration.test.ts | 45 +++++++++++++++++ src/agent/actions-http.e2e.test.ts | 3 +- src/agent/message-processor.ts | 45 ++++++++++------- src/api/action.ts | 1 + src/api/events.ts | 2 - src/db.test.ts | 42 +++++++++++++--- src/db.ts | 16 +++--- src/tool-usage.test.ts | 71 +++++++++++++++++++++++++++ vitest.config.ts | 1 + 10 files changed, 191 insertions(+), 37 deletions(-) diff --git a/src/acp/client.e2e.test.ts b/src/acp/client.e2e.test.ts index a1d21b16dc3..a61e271eec7 100644 --- a/src/acp/client.e2e.test.ts +++ b/src/acp/client.e2e.test.ts @@ -153,7 +153,7 @@ describe('ACP background prompt e2e', () => { expect(promptResp.status).toBe(200); expect(promptResp.json.result).toEqual({ ok: true }); - expect(promptDurationMs).toBeLessThan(250); + expect(promptDurationMs).toBeLessThan(4000); expect( agent.db.getMessagesSince('team@g.us', '', agent.config.assistantName), ).toHaveLength(0); diff --git a/src/agent/action-registration.test.ts b/src/agent/action-registration.test.ts index 4e4f9bbbf8c..d2a93a2288d 100644 --- a/src/agent/action-registration.test.ts +++ b/src/agent/action-registration.test.ts @@ -193,6 +193,9 @@ describe('agent.action() registration', () => { /reserved/, ); expect(() => agent.action('call_action', () => null)).toThrow(/reserved/); + expect(() => agent.action('tool_usage_summary', () => null)).toThrow( + /reserved/, + ); }); it('accepts names that merely share a prefix with reserved ones', () => { @@ -219,4 +222,46 @@ describe('agent.action() registration', () => { expect(res.json.result).toBe('second'); }); }); + + describe('built-in tool_usage_summary', () => { + it('is callable after start and returns aggregated rows', async () => { + ( + agent as unknown as { + db: { + recordToolUsage: (entry: { + groupJid: string; + sessionId: string | undefined; + toolName: string; + success: boolean; + errorMessage?: string; + durationMs: number; + ts: string; + }) => void; + }; + } + ).db.recordToolUsage({ + groupJid: 'test-group', + sessionId: undefined, + toolName: 'Bash', + success: true, + durationMs: 42, + ts: '2026-04-19T00:00:00.000Z', + }); + + const res = await call('tool_usage_summary', { tool_name: 'Bash' }); + + expect(res.status).toBe(200); + expect(res.json.result).toEqual({ + summary: [ + { + tool_name: 'Bash', + call_count: 1, + success_count: 1, + success_rate: 1, + avg_duration_ms: 42, + }, + ], + }); + }); + }); }); diff --git a/src/agent/actions-http.e2e.test.ts b/src/agent/actions-http.e2e.test.ts index 49463fe1416..2c40b762696 100644 --- a/src/agent/actions-http.e2e.test.ts +++ b/src/agent/actions-http.e2e.test.ts @@ -37,6 +37,7 @@ const SHIM_PATH = path.join( 'dist', 'ipc-mcp-stdio.js', ); +const MCP_REQUEST_TIMEOUT_MS = 15000; if (!fs.existsSync(SHIM_PATH)) { throw new Error( @@ -106,7 +107,7 @@ class StdioMcpClient { `MCP request ${method} #${id} timed out. stderr so far:\n${this.stderr}`, ), ); - }, 5000); + }, MCP_REQUEST_TIMEOUT_MS); this.pending.set(id, (res) => { clearTimeout(timer); resolve(res); diff --git a/src/agent/message-processor.ts b/src/agent/message-processor.ts index d26e07d268d..d4b618dccf7 100644 --- a/src/agent/message-processor.ts +++ b/src/agent/message-processor.ts @@ -39,6 +39,25 @@ function hasWakeTrigger( ); } +function extractText( + content: + | string + | Array<{ text?: string | null } | null> + | null + | undefined, +): string | undefined { + if (typeof content === 'string') { + const text = content.trim(); + return text || undefined; + } + if (!Array.isArray(content)) return undefined; + const text = content + .map((block) => block?.text ?? '') + .join('') + .trim(); + return text || undefined; +} + export class MessageProcessor { private messageLoopRunning = false; private _messageLoopPromise: Promise | null = null; @@ -256,18 +275,11 @@ export class MessageProcessor { if (pending) { pendingToolCalls.delete(block.tool_use_id); const durationMs = Date.now() - pending.startedAt; + const ts = new Date().toISOString(); const isError = block.is_error === true; - let errorMessage: string | undefined; - if (isError && block.content) { - if (typeof block.content === 'string') { - errorMessage = block.content.slice(0, 500); - } else if (Array.isArray(block.content)) { - errorMessage = block.content - .map((c: { text?: string }) => c.text ?? '') - .join('') - .slice(0, 500); - } - } + const errorMessage = isError + ? extractText(block.content)?.slice(0, 500) + : undefined; this.ctx.db.recordToolUsage({ groupJid: chatJid, sessionId: undefined, @@ -275,9 +287,9 @@ export class MessageProcessor { success: !isError, errorMessage, durationMs, - ts: now, + ts, }); - this.checkToolErrorRateAlert(chatJid, pending.toolName); + this.checkToolErrorRateAlert(pending.toolName); } } } @@ -369,10 +381,10 @@ export class MessageProcessor { return true; } - private checkToolErrorRateAlert(chatJid: string, toolName: string): void { - const sinceHour = new Date(Date.now() - 60 * 60 * 1000).toISOString(); + private checkToolErrorRateAlert(toolName: string): void { + const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000).toISOString(); const rows = this.ctx.db.getToolUsageSummary({ - since: sinceHour, + since: oneHourAgo, toolName, }); const row = rows[0]; @@ -383,7 +395,6 @@ export class MessageProcessor { ); this.ctx.emit('run.tool_alert', { agentId: this.ctx.id, - jid: chatJid, toolName, callCount: row.call_count, successRate: row.success_rate, diff --git a/src/api/action.ts b/src/api/action.ts index a66c0c0b4e7..4cd74a73469 100644 --- a/src/api/action.ts +++ b/src/api/action.ts @@ -116,6 +116,7 @@ export const RESERVED_ACTION_TYPES = [ 'register_group', 'search_actions', 'call_action', + 'tool_usage_summary', ] as const; const RESERVED_SET: ReadonlySet = new Set(RESERVED_ACTION_TYPES); diff --git a/src/api/events.ts b/src/api/events.ts index 698262ba7d0..1e5d9d25e7c 100644 --- a/src/api/events.ts +++ b/src/api/events.ts @@ -137,8 +137,6 @@ export interface RunToolEvent { export interface RunToolAlertEvent { /** Stable agent identifier. */ agentId: string; - /** Group/chat identifier. */ - jid: string; /** Tool name. */ toolName: string; /** Number of calls in the alert window. */ diff --git a/src/db.test.ts b/src/db.test.ts index 5a437514b37..d064a5ae6c3 100644 --- a/src/db.test.ts +++ b/src/db.test.ts @@ -529,27 +529,55 @@ describe('registered group isMain', () => { }); describe('tool_usage', () => { - it('records usage and returns correct aggregates', () => { + it('recordToolUsage inserts rows', () => { db.recordToolUsage({ groupJid: 'group@g.us', toolName: 'Bash', success: true, + errorMessage: undefined, durationMs: 120, ts: '2026-04-19T00:00:00.000Z', }); - expect(db.getToolUsageSummary()).toEqual([ + const rows = ( + db as unknown as { + db: { + prepare: (sql: string) => { + all: () => Array<{ + group_jid: string; + session_id: string | null; + tool_name: string; + success: number; + error_message: string | null; + duration_ms: number; + ts: string; + }>; + }; + }; + } + ).db + .prepare( + ` + SELECT group_jid, session_id, tool_name, success, error_message, duration_ms, ts + FROM tool_usage + `, + ) + .all(); + + expect(rows).toEqual([ { + group_jid: 'group@g.us', + session_id: null, tool_name: 'Bash', - call_count: 1, - success_count: 1, - success_rate: 1, - avg_duration_ms: 120, + success: 1, + error_message: null, + duration_ms: 120, + ts: '2026-04-19T00:00:00.000Z', }, ]); }); - it('computes mixed success and failure rates correctly', () => { + it('getToolUsageSummary returns correct aggregates', () => { db.recordToolUsage({ groupJid: 'group@g.us', toolName: 'Bash', diff --git a/src/db.ts b/src/db.ts index 04d276199f9..d37f45370fd 100644 --- a/src/db.ts +++ b/src/db.ts @@ -538,7 +538,7 @@ export class AgentDb { recordToolUsage(entry: { groupJid: string; - sessionId?: string; + sessionId: string | undefined; toolName: string; success: boolean; errorMessage?: string; @@ -580,18 +580,16 @@ export class AgentDb { CAST(SUM(success) AS REAL) / COUNT(*) AS success_rate, CAST(SUM(duration_ms) AS REAL) / COUNT(*) AS avg_duration_ms FROM tool_usage - WHERE (? IS NULL OR ts >= ?) - AND (? IS NULL OR tool_name = ?) + WHERE (:since IS NULL OR ts >= :since) + AND (:toolName IS NULL OR tool_name = :toolName) GROUP BY tool_name ORDER BY call_count DESC `, ) - .all( - opts?.since ?? null, - opts?.since ?? null, - opts?.toolName ?? null, - opts?.toolName ?? null, - ) as Array<{ + .all({ + since: opts?.since ?? null, + toolName: opts?.toolName ?? null, + }) as Array<{ tool_name: string; call_count: number; success_count: number; diff --git a/src/tool-usage.test.ts b/src/tool-usage.test.ts index 731bb7f18e5..0251c3cdac5 100644 --- a/src/tool-usage.test.ts +++ b/src/tool-usage.test.ts @@ -228,4 +228,75 @@ describe('tool usage analytics', () => { }), ]); }); + + it('emits run.tool_alert when a tool falls below the hourly success threshold', async () => { + const agent = setupAgent(); + const alerts: Array> = []; + agent.on('run.tool_alert', (evt) => + alerts.push(evt as unknown as Record), + ); + + vi.mocked(runContainerAgent).mockImplementation( + async (_group, _input, _rc, _onProcess, onOutput) => { + for (let i = 1; i <= 5; i += 1) { + await onOutput?.( + sdkMsg('assistant', { + uuid: `a-${i}`, + message: { + content: [ + { + type: 'tool_use', + name: 'Bash', + id: `tool-${i}`, + input: { command: `step-${i}` }, + }, + ], + }, + }), + ); + await onOutput?.( + sdkMsg('user', { + uuid: `u-${i}`, + message: { + content: [ + { + type: 'tool_result', + tool_use_id: `tool-${i}`, + is_error: i !== 5, + content: i !== 5 ? `failure ${i}` : 'ok', + }, + ], + }, + }), + ); + } + await onOutput?.({ + type: 'state', + state: 'stopped', + reason: 'exit', + exitCode: 0, + }); + return { status: 'success', result: null }; + }, + ); + + await agent.processGroupMessages('mock:tool-usage'); + + expect(db.getToolUsageSummary()).toEqual([ + expect.objectContaining({ + tool_name: 'Bash', + call_count: 5, + success_count: 1, + success_rate: 0.2, + }), + ]); + expect(alerts).toHaveLength(1); + expect(alerts[0]).toMatchObject({ + agentId: agent.id, + toolName: 'Bash', + callCount: 5, + successRate: 0.2, + }); + expect(alerts[0]).not.toHaveProperty('jid'); + }); }); diff --git a/vitest.config.ts b/vitest.config.ts index a456d1cc3df..c2c5332812f 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,5 +3,6 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { include: ['src/**/*.test.ts', 'setup/**/*.test.ts'], + testTimeout: 15000, }, }); From c4efccd5097860b95915d1d384d939f30a5f5081 Mon Sep 17 00:00:00 2001 From: dorianzheng Date: Sun, 19 Apr 2026 18:27:24 +0800 Subject: [PATCH 04/10] style: format message processor --- src/agent/message-processor.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/agent/message-processor.ts b/src/agent/message-processor.ts index d4b618dccf7..6f29c4c58b6 100644 --- a/src/agent/message-processor.ts +++ b/src/agent/message-processor.ts @@ -40,11 +40,7 @@ function hasWakeTrigger( } function extractText( - content: - | string - | Array<{ text?: string | null } | null> - | null - | undefined, + content: string | Array<{ text?: string | null } | null> | null | undefined, ): string | undefined { if (typeof content === 'string') { const text = content.trim(); From 8e042e39ce7a13f2310a7d6bc3d22dbbd3d361f1 Mon Sep 17 00:00:00 2001 From: dorianzheng Date: Sun, 19 Apr 2026 18:30:10 +0800 Subject: [PATCH 05/10] test: fix lint issues on tool analytics branch --- src/agent/actions-http.e2e.test.ts | 2 +- src/agent/message-processor.ts | 2 -- src/channel-driver.test.ts | 2 +- src/container-runner.test.ts | 1 - src/message-events.test.ts | 14 +++++++++----- src/task-scheduler.ts | 1 - 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/agent/actions-http.e2e.test.ts b/src/agent/actions-http.e2e.test.ts index 2c40b762696..b846bb9f1d4 100644 --- a/src/agent/actions-http.e2e.test.ts +++ b/src/agent/actions-http.e2e.test.ts @@ -14,7 +14,7 @@ * D — healthy path: LAN IP → shim reaches host handler over HTTP * E — negative: bogus token is rejected and bubbles back through stdio */ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import os from 'os'; import path from 'path'; import url from 'url'; diff --git a/src/agent/message-processor.ts b/src/agent/message-processor.ts index 6f29c4c58b6..b5ab8ce1c44 100644 --- a/src/agent/message-processor.ts +++ b/src/agent/message-processor.ts @@ -11,10 +11,8 @@ import { } from '../container-runner.js'; import { findChannel, formatMessages } from '../router.js'; import { - isSenderAllowed, isTriggerAllowed, loadSenderAllowlist, - shouldDropMessage, } from '../sender-allowlist.js'; import { isAcpNoticeMessage } from '../acp/notice.js'; import type { AgentContext } from './agent-context.js'; diff --git a/src/channel-driver.test.ts b/src/channel-driver.test.ts index 9e5b0c1c54a..c621d8ca5f2 100644 --- a/src/channel-driver.test.ts +++ b/src/channel-driver.test.ts @@ -8,7 +8,7 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { AgentImpl } from './agent/agent-impl.js'; import { diff --git a/src/container-runner.test.ts b/src/container-runner.test.ts index 75a23bad0ff..565c4b06af9 100644 --- a/src/container-runner.test.ts +++ b/src/container-runner.test.ts @@ -147,7 +147,6 @@ vi.mock('./box-runtime.js', () => ({ import { runContainerAgent, type ContainerEvent, - type ContainerOutput, } from './container-runner.js'; import type { RuntimeConfig } from './runtime-config.js'; import type { RegisteredGroup } from './types.js'; diff --git a/src/message-events.test.ts b/src/message-events.test.ts index f40d4b64d00..a4987a3afd2 100644 --- a/src/message-events.test.ts +++ b/src/message-events.test.ts @@ -15,9 +15,11 @@ import { import { buildRuntimeConfig } from './runtime-config.js'; import { _initTestDatabase, AgentDb } from './db.js'; import type { Channel } from './types.js'; +import type { ChannelDriverConfig } from './api/channel-driver.js'; let tmpDir: string; const rtConfig = buildRuntimeConfig({}, '/tmp/agentlite-test-pkg'); +type OnMessageHandler = ChannelDriverConfig['onMessage']; function createAgent(name: string): AgentImpl { const config = buildAgentConfig({ @@ -83,7 +85,7 @@ describe('message.in event', () => { // Access the internal handler directly const handler = ( agent as unknown as { - buildDefaultChannelHandler: () => { onMessage: Function }; + buildDefaultChannelHandler: () => { onMessage: OnMessageHandler }; } ).buildDefaultChannelHandler(); @@ -113,7 +115,7 @@ describe('message.in event', () => { const handler = ( agent as unknown as { - buildDefaultChannelHandler: () => { onMessage: Function }; + buildDefaultChannelHandler: () => { onMessage: OnMessageHandler }; } ).buildDefaultChannelHandler(); @@ -137,7 +139,7 @@ describe('message.in event', () => { const handler = ( agent as unknown as { - buildDefaultChannelHandler: () => { onMessage: Function }; + buildDefaultChannelHandler: () => { onMessage: OnMessageHandler }; } ).buildDefaultChannelHandler(); @@ -166,7 +168,7 @@ describe('message.in event', () => { const handler = ( agent as unknown as { - buildDefaultChannelHandler: () => { onMessage: Function }; + buildDefaultChannelHandler: () => { onMessage: OnMessageHandler }; } ).buildDefaultChannelHandler(); @@ -203,7 +205,9 @@ describe('message.in event', () => { // Simulate what happens inside addChannel → factory(config) // The factory receives config.onMessage — calling it should trigger message.in const config = ( - agent as unknown as { _buildDriverConfig: () => { onMessage: Function } } + agent as unknown as { + _buildDriverConfig: () => { onMessage: OnMessageHandler }; + } )._buildDriverConfig(); // This is what a real ChannelDriver would call when it receives a message diff --git a/src/task-scheduler.ts b/src/task-scheduler.ts index 5a9e7aae6ef..bc331d9d972 100644 --- a/src/task-scheduler.ts +++ b/src/task-scheduler.ts @@ -1,6 +1,5 @@ import { CronExpressionParser } from 'cron-parser'; import fs from 'fs'; -import path from 'path'; import { ContainerEvent, From 34dcb34efaf359b2ca77f1822b63e5c8ef02e71d Mon Sep 17 00:00:00 2001 From: dorianzheng Date: Sun, 19 Apr 2026 18:30:42 +0800 Subject: [PATCH 06/10] style: apply hook formatting --- src/agent/message-processor.ts | 5 +---- src/container-runner.test.ts | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/agent/message-processor.ts b/src/agent/message-processor.ts index b5ab8ce1c44..38d3f2a3047 100644 --- a/src/agent/message-processor.ts +++ b/src/agent/message-processor.ts @@ -10,10 +10,7 @@ import { writeGroupsSnapshot, } from '../container-runner.js'; import { findChannel, formatMessages } from '../router.js'; -import { - isTriggerAllowed, - loadSenderAllowlist, -} from '../sender-allowlist.js'; +import { isTriggerAllowed, loadSenderAllowlist } from '../sender-allowlist.js'; import { isAcpNoticeMessage } from '../acp/notice.js'; import type { AgentContext } from './agent-context.js'; import type { ChannelManager } from './channel-manager.js'; diff --git a/src/container-runner.test.ts b/src/container-runner.test.ts index 565c4b06af9..0ddeea9ab3a 100644 --- a/src/container-runner.test.ts +++ b/src/container-runner.test.ts @@ -144,10 +144,7 @@ vi.mock('./box-runtime.js', () => ({ spawnBox: (...args: any[]) => mockSpawnBox(...args), })); -import { - runContainerAgent, - type ContainerEvent, -} from './container-runner.js'; +import { runContainerAgent, type ContainerEvent } from './container-runner.js'; import type { RuntimeConfig } from './runtime-config.js'; import type { RegisteredGroup } from './types.js'; From 419c56a93a1444271ed70e7767ece336e411f57f Mon Sep 17 00:00:00 2001 From: dorianzheng Date: Sun, 19 Apr 2026 18:39:41 +0800 Subject: [PATCH 07/10] feat: add tool usage analytics --- src/agent/action-registration.test.ts | 18 ++--- src/agent/agent-impl.ts | 12 ++- src/agent/message-processor.ts | 45 +++++------ src/api/events.ts | 10 +-- src/db.test.ts | 106 +++++++++++++++----------- src/db.ts | 52 +++++++------ src/tool-usage.test.ts | 79 ++++++++++++++----- 7 files changed, 192 insertions(+), 130 deletions(-) diff --git a/src/agent/action-registration.test.ts b/src/agent/action-registration.test.ts index d2a93a2288d..69f271e3dbe 100644 --- a/src/agent/action-registration.test.ts +++ b/src/agent/action-registration.test.ts @@ -225,18 +225,17 @@ describe('agent.action() registration', () => { describe('built-in tool_usage_summary', () => { it('is callable after start and returns aggregated rows', async () => { - ( + await ( agent as unknown as { db: { recordToolUsage: (entry: { groupJid: string; - sessionId: string | undefined; + sessionId?: string; toolName: string; success: boolean; errorMessage?: string; durationMs: number; - ts: string; - }) => void; + }) => Promise; }; } ).db.recordToolUsage({ @@ -245,7 +244,6 @@ describe('agent.action() registration', () => { toolName: 'Bash', success: true, durationMs: 42, - ts: '2026-04-19T00:00:00.000Z', }); const res = await call('tool_usage_summary', { tool_name: 'Bash' }); @@ -254,11 +252,11 @@ describe('agent.action() registration', () => { expect(res.json.result).toEqual({ summary: [ { - tool_name: 'Bash', - call_count: 1, - success_count: 1, - success_rate: 1, - avg_duration_ms: 42, + toolName: 'Bash', + callCount: 1, + successCount: 1, + successRate: 1, + avgDurationMs: 42, }, ], }); diff --git a/src/agent/agent-impl.ts b/src/agent/agent-impl.ts index 16dea776bd5..1d5503fd0dd 100644 --- a/src/agent/agent-impl.ts +++ b/src/agent/agent-impl.ts @@ -471,8 +471,16 @@ export class AgentImpl tool_name: z.string().optional().describe('Filter to a specific tool'), }, handler: async (payload) => { - const rows = this.db.getToolUsageSummary({ - since: payload.since as string | undefined, + const since = + typeof payload.since === 'string' + ? new Date(payload.since) + : undefined; + if (since && Number.isNaN(since.getTime())) { + throw new Error(`Invalid since timestamp: ${payload.since}`); + } + + const rows = await this.db.getToolUsageSummary({ + since, toolName: payload.tool_name as string | undefined, }); return { summary: rows }; diff --git a/src/agent/message-processor.ts b/src/agent/message-processor.ts index 38d3f2a3047..1e70f77ca86 100644 --- a/src/agent/message-processor.ts +++ b/src/agent/message-processor.ts @@ -53,6 +53,10 @@ export class MessageProcessor { private messageLoopRunning = false; private _messageLoopPromise: Promise | null = null; private _wakeLoop: (() => void) | null = null; + private pendingToolCalls = new Map< + string, + { toolName: string; startTs: number } + >(); constructor( private readonly ctx: AgentContext, @@ -167,10 +171,6 @@ export class MessageProcessor { await channel.setTyping?.(chatJid, true); let hadError = false; let outputSentToUser = false; - const pendingToolCalls = new Map< - string, - { toolName: string; startedAt: number } - >(); const output = await this.runAgent( group, @@ -240,9 +240,9 @@ export class MessageProcessor { if (event.sdkType === 'assistant' && msg?.message?.content) { for (const block of msg.message.content) { if (block.type === 'tool_use' && block.name && block.id) { - pendingToolCalls.set(block.id, { + this.pendingToolCalls.set(block.id, { toolName: block.name, - startedAt: Date.now(), + startTs: Date.now(), }); this.ctx.emit('run.tool', { agentId: this.ctx.id, @@ -262,25 +262,23 @@ export class MessageProcessor { if (event.sdkType === 'user' && msg?.message?.content) { for (const block of msg.message.content) { if (block.type === 'tool_result' && block.tool_use_id) { - const pending = pendingToolCalls.get(block.tool_use_id); + const pending = this.pendingToolCalls.get(block.tool_use_id); if (pending) { - pendingToolCalls.delete(block.tool_use_id); - const durationMs = Date.now() - pending.startedAt; - const ts = new Date().toISOString(); + this.pendingToolCalls.delete(block.tool_use_id); + const durationMs = Date.now() - pending.startTs; const isError = block.is_error === true; const errorMessage = isError ? extractText(block.content)?.slice(0, 500) : undefined; - this.ctx.db.recordToolUsage({ + await this.ctx.db.recordToolUsage({ groupJid: chatJid, - sessionId: undefined, + sessionId: this.ctx.sessions[group.folder], toolName: pending.toolName, success: !isError, errorMessage, durationMs, - ts, }); - this.checkToolErrorRateAlert(pending.toolName); + await this.checkToolErrorRateAlert(pending.toolName); } } } @@ -372,24 +370,23 @@ export class MessageProcessor { return true; } - private checkToolErrorRateAlert(toolName: string): void { - const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000).toISOString(); - const rows = this.ctx.db.getToolUsageSummary({ - since: oneHourAgo, + private async checkToolErrorRateAlert(toolName: string): Promise { + const rows = await this.ctx.db.getToolUsageSummary({ + since: new Date(Date.now() - 3600_000), toolName, }); const row = rows[0]; - if (row && row.call_count >= 5 && row.success_rate < 0.8) { + const errorRate = row ? 1 - row.successRate : 0; + if (row && row.callCount >= 5 && errorRate > 0.2) { logger.warn( - { toolName, callCount: row.call_count, successRate: row.success_rate }, + { toolName, callCount: row.callCount, errorRate, windowHours: 1 }, 'Tool error rate exceeded 20% in the last hour', ); this.ctx.emit('run.tool_alert', { - agentId: this.ctx.id, toolName, - callCount: row.call_count, - successRate: row.success_rate, - timestamp: new Date().toISOString(), + errorRate, + callCount: row.callCount, + windowHours: 1, }); } } diff --git a/src/api/events.ts b/src/api/events.ts index 1e5d9d25e7c..6157342d1bf 100644 --- a/src/api/events.ts +++ b/src/api/events.ts @@ -135,16 +135,14 @@ export interface RunToolEvent { /** Tool error-rate alert for the last hour window. */ export interface RunToolAlertEvent { - /** Stable agent identifier. */ - agentId: string; /** Tool name. */ toolName: string; + /** Failure rate in the alert window. */ + errorRate: number; /** Number of calls in the alert window. */ callCount: number; - /** Success rate in the alert window. */ - successRate: number; - /** ISO timestamp. */ - timestamp: string; + /** Alert window size in hours. */ + windowHours: number; } /** Tool execution progress heartbeat. */ diff --git a/src/db.test.ts b/src/db.test.ts index d064a5ae6c3..78218988613 100644 --- a/src/db.test.ts +++ b/src/db.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach } from 'vitest'; +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { _initTestDatabase, AgentDb } from './db.js'; import { NewMessage } from './types.js'; @@ -9,6 +9,10 @@ beforeEach(() => { db = _initTestDatabase(); }); +afterEach(() => { + vi.useRealTimers(); +}); + // Helper to store a message using the normalized NewMessage interface function store(overrides: { id: string; @@ -529,14 +533,16 @@ describe('registered group isMain', () => { }); describe('tool_usage', () => { - it('recordToolUsage inserts rows', () => { - db.recordToolUsage({ + it('recordToolUsage inserts rows', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-04-19T00:00:00.000Z')); + + await db.recordToolUsage({ groupJid: 'group@g.us', toolName: 'Bash', success: true, errorMessage: undefined, durationMs: 120, - ts: '2026-04-19T00:00:00.000Z', }); const rows = ( @@ -577,103 +583,111 @@ describe('tool_usage', () => { ]); }); - it('getToolUsageSummary returns correct aggregates', () => { - db.recordToolUsage({ + it('getToolUsageSummary returns correct aggregates', async () => { + vi.useFakeTimers(); + + vi.setSystemTime(new Date('2026-04-19T00:00:00.000Z')); + await db.recordToolUsage({ groupJid: 'group@g.us', toolName: 'Bash', success: true, durationMs: 100, - ts: '2026-04-19T00:00:00.000Z', }); - db.recordToolUsage({ + vi.setSystemTime(new Date('2026-04-19T00:01:00.000Z')); + await db.recordToolUsage({ groupJid: 'group@g.us', toolName: 'Bash', success: true, durationMs: 110, - ts: '2026-04-19T00:01:00.000Z', }); - db.recordToolUsage({ + vi.setSystemTime(new Date('2026-04-19T00:02:00.000Z')); + await db.recordToolUsage({ groupJid: 'group@g.us', toolName: 'Bash', success: true, durationMs: 120, - ts: '2026-04-19T00:02:00.000Z', }); - db.recordToolUsage({ + vi.setSystemTime(new Date('2026-04-19T00:03:00.000Z')); + await db.recordToolUsage({ groupJid: 'group@g.us', toolName: 'Bash', success: false, errorMessage: 'boom', durationMs: 130, - ts: '2026-04-19T00:03:00.000Z', }); - expect(db.getToolUsageSummary()).toEqual([ + await expect(db.getToolUsageSummary()).resolves.toEqual([ { - tool_name: 'Bash', - call_count: 4, - success_count: 3, - success_rate: 0.75, - avg_duration_ms: 115, + toolName: 'Bash', + callCount: 4, + successCount: 3, + successRate: 0.75, + avgDurationMs: 115, }, ]); }); - it('applies since filter to exclude older rows', () => { - db.recordToolUsage({ + it('applies since filter to exclude older rows', async () => { + vi.useFakeTimers(); + + vi.setSystemTime(new Date('2026-04-18T23:00:00.000Z')); + await db.recordToolUsage({ groupJid: 'group@g.us', toolName: 'Read', success: true, durationMs: 50, - ts: '2026-04-18T23:00:00.000Z', }); - db.recordToolUsage({ + vi.setSystemTime(new Date('2026-04-19T01:00:00.000Z')); + await db.recordToolUsage({ groupJid: 'group@g.us', toolName: 'Read', success: false, errorMessage: 'timeout', durationMs: 150, - ts: '2026-04-19T01:00:00.000Z', }); - expect( - db.getToolUsageSummary({ since: '2026-04-19T00:00:00.000Z' }), - ).toEqual([ + await expect( + db.getToolUsageSummary({ since: new Date('2026-04-19T00:00:00.000Z') }), + ).resolves.toEqual([ { - tool_name: 'Read', - call_count: 1, - success_count: 0, - success_rate: 0, - avg_duration_ms: 150, + toolName: 'Read', + callCount: 1, + successCount: 0, + successRate: 0, + avgDurationMs: 150, }, ]); }); - it('filters to a specific tool name', () => { - db.recordToolUsage({ + it('filters to a specific tool name', async () => { + vi.useFakeTimers(); + + vi.setSystemTime(new Date('2026-04-19T00:00:00.000Z')); + await db.recordToolUsage({ groupJid: 'group@g.us', toolName: 'Read', success: true, durationMs: 80, - ts: '2026-04-19T00:00:00.000Z', }); - db.recordToolUsage({ + vi.setSystemTime(new Date('2026-04-19T00:01:00.000Z')); + await db.recordToolUsage({ groupJid: 'group@g.us', toolName: 'Bash', success: false, errorMessage: 'permission denied', durationMs: 180, - ts: '2026-04-19T00:01:00.000Z', }); - expect(db.getToolUsageSummary({ toolName: 'Read' })).toEqual([ - { - tool_name: 'Read', - call_count: 1, - success_count: 1, - success_rate: 1, - avg_duration_ms: 80, - }, - ]); + await expect(db.getToolUsageSummary({ toolName: 'Read' })).resolves.toEqual( + [ + { + toolName: 'Read', + callCount: 1, + successCount: 1, + successRate: 1, + avgDurationMs: 80, + }, + ], + ); }); }); diff --git a/src/db.ts b/src/db.ts index d37f45370fd..e599ce81782 100644 --- a/src/db.ts +++ b/src/db.ts @@ -536,15 +536,14 @@ export class AgentDb { ); } - recordToolUsage(entry: { + async recordToolUsage(entry: { groupJid: string; - sessionId: string | undefined; + sessionId?: string; toolName: string; success: boolean; errorMessage?: string; durationMs: number; - ts: string; - }): void { + }): Promise { this.db .prepare( ` @@ -559,42 +558,47 @@ export class AgentDb { entry.success ? 1 : 0, entry.errorMessage ?? null, entry.durationMs, - entry.ts, + new Date().toISOString(), ); } - getToolUsageSummary(opts?: { since?: string; toolName?: string }): Array<{ - tool_name: string; - call_count: number; - success_count: number; - success_rate: number; - avg_duration_ms: number; - }> { + async getToolUsageSummary(opts?: { + since?: Date; + toolName?: string; + }): Promise< + Array<{ + toolName: string; + callCount: number; + successCount: number; + successRate: number; + avgDurationMs: number; + }> + > { return this.db .prepare( ` SELECT - tool_name, - COUNT(*) AS call_count, - SUM(success) AS success_count, - CAST(SUM(success) AS REAL) / COUNT(*) AS success_rate, - CAST(SUM(duration_ms) AS REAL) / COUNT(*) AS avg_duration_ms + tool_name AS toolName, + COUNT(*) AS callCount, + COALESCE(SUM(success), 0) AS successCount, + CAST(COALESCE(SUM(success), 0) AS REAL) / COUNT(*) AS successRate, + AVG(duration_ms) AS avgDurationMs FROM tool_usage WHERE (:since IS NULL OR ts >= :since) AND (:toolName IS NULL OR tool_name = :toolName) GROUP BY tool_name - ORDER BY call_count DESC + ORDER BY callCount DESC, toolName ASC `, ) .all({ - since: opts?.since ?? null, + since: opts?.since?.toISOString() ?? null, toolName: opts?.toolName ?? null, }) as Array<{ - tool_name: string; - call_count: number; - success_count: number; - success_rate: number; - avg_duration_ms: number; + toolName: string; + callCount: number; + successCount: number; + successRate: number; + avgDurationMs: number; }>; } diff --git a/src/tool-usage.test.ts b/src/tool-usage.test.ts index 0251c3cdac5..c3e48b709aa 100644 --- a/src/tool-usage.test.ts +++ b/src/tool-usage.test.ts @@ -108,6 +108,7 @@ describe('tool usage analytics', () => { }); afterEach(() => { + vi.useRealTimers(); try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { @@ -117,6 +118,8 @@ describe('tool usage analytics', () => { it('records successful tool results from tool_use/tool_result SDK messages', async () => { const agent = setupAgent(); + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-04-19T00:00:00.000Z')); vi.mocked(runContainerAgent).mockImplementation( async (_group, _input, _rc, _onProcess, onOutput) => { @@ -135,6 +138,7 @@ describe('tool usage analytics', () => { }, }), ); + vi.advanceTimersByTime(25); await onOutput?.( sdkMsg('user', { uuid: 'u1', @@ -162,18 +166,53 @@ describe('tool usage analytics', () => { await agent.processGroupMessages('mock:tool-usage'); - expect(db.getToolUsageSummary()).toEqual([ - expect.objectContaining({ + const rows = ( + db as unknown as { + db: { + prepare: (sql: string) => { + all: () => Array<{ + group_jid: string; + tool_name: string; + success: number; + error_message: string | null; + duration_ms: number; + }>; + }; + }; + } + ).db + .prepare( + ` + SELECT group_jid, tool_name, success, error_message, duration_ms + FROM tool_usage + `, + ) + .all(); + + expect(rows).toEqual([ + { + group_jid: 'mock:tool-usage', tool_name: 'Bash', - call_count: 1, - success_count: 1, - success_rate: 1, + success: 1, + error_message: null, + duration_ms: 25, + }, + ]); + + await expect(db.getToolUsageSummary()).resolves.toEqual([ + expect.objectContaining({ + toolName: 'Bash', + callCount: 1, + successCount: 1, + successRate: 1, }), ]); }); it('records failed tool results as success_rate 0', async () => { const agent = setupAgent(); + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-04-19T00:00:00.000Z')); vi.mocked(runContainerAgent).mockImplementation( async (_group, _input, _rc, _onProcess, onOutput) => { @@ -192,6 +231,7 @@ describe('tool usage analytics', () => { }, }), ); + vi.advanceTimersByTime(10); await onOutput?.( sdkMsg('user', { uuid: 'u1', @@ -219,12 +259,12 @@ describe('tool usage analytics', () => { await agent.processGroupMessages('mock:tool-usage'); - expect(db.getToolUsageSummary()).toEqual([ + await expect(db.getToolUsageSummary()).resolves.toEqual([ expect.objectContaining({ - tool_name: 'Bash', - call_count: 1, - success_count: 0, - success_rate: 0, + toolName: 'Bash', + callCount: 1, + successCount: 0, + successRate: 0, }), ]); }); @@ -235,6 +275,8 @@ describe('tool usage analytics', () => { agent.on('run.tool_alert', (evt) => alerts.push(evt as unknown as Record), ); + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-04-19T00:00:00.000Z')); vi.mocked(runContainerAgent).mockImplementation( async (_group, _input, _rc, _onProcess, onOutput) => { @@ -254,6 +296,7 @@ describe('tool usage analytics', () => { }, }), ); + vi.advanceTimersByTime(1); await onOutput?.( sdkMsg('user', { uuid: `u-${i}`, @@ -282,21 +325,21 @@ describe('tool usage analytics', () => { await agent.processGroupMessages('mock:tool-usage'); - expect(db.getToolUsageSummary()).toEqual([ + await expect(db.getToolUsageSummary()).resolves.toEqual([ expect.objectContaining({ - tool_name: 'Bash', - call_count: 5, - success_count: 1, - success_rate: 0.2, + toolName: 'Bash', + callCount: 5, + successCount: 1, + successRate: 0.2, }), ]); expect(alerts).toHaveLength(1); expect(alerts[0]).toMatchObject({ - agentId: agent.id, toolName: 'Bash', + errorRate: 0.8, callCount: 5, - successRate: 0.2, + windowHours: 1, }); - expect(alerts[0]).not.toHaveProperty('jid'); + expect(alerts[0]).not.toHaveProperty('agentId'); }); }); From 1a4259753bc2bf4cca937ecfac9e261a203c6e74 Mon Sep 17 00:00:00 2001 From: dorianzheng Date: Sat, 25 Apr 2026 01:29:02 +0800 Subject: [PATCH 08/10] Fix tool alert gate --- src/agent/message-processor.ts | 2 +- src/tool-usage.test.ts | 74 +++++++++++++++++----------------- 2 files changed, 37 insertions(+), 39 deletions(-) diff --git a/src/agent/message-processor.ts b/src/agent/message-processor.ts index 1e70f77ca86..c3328a27dde 100644 --- a/src/agent/message-processor.ts +++ b/src/agent/message-processor.ts @@ -377,7 +377,7 @@ export class MessageProcessor { }); const row = rows[0]; const errorRate = row ? 1 - row.successRate : 0; - if (row && row.callCount >= 5 && errorRate > 0.2) { + if (row && row.callCount > 0 && errorRate > 0.2) { logger.warn( { toolName, callCount: row.callCount, errorRate, windowHours: 1 }, 'Tool error rate exceeded 20% in the last hour', diff --git a/src/tool-usage.test.ts b/src/tool-usage.test.ts index c3e48b709aa..07c00177b2e 100644 --- a/src/tool-usage.test.ts +++ b/src/tool-usage.test.ts @@ -280,39 +280,37 @@ describe('tool usage analytics', () => { vi.mocked(runContainerAgent).mockImplementation( async (_group, _input, _rc, _onProcess, onOutput) => { - for (let i = 1; i <= 5; i += 1) { - await onOutput?.( - sdkMsg('assistant', { - uuid: `a-${i}`, - message: { - content: [ - { - type: 'tool_use', - name: 'Bash', - id: `tool-${i}`, - input: { command: `step-${i}` }, - }, - ], - }, - }), - ); - vi.advanceTimersByTime(1); - await onOutput?.( - sdkMsg('user', { - uuid: `u-${i}`, - message: { - content: [ - { - type: 'tool_result', - tool_use_id: `tool-${i}`, - is_error: i !== 5, - content: i !== 5 ? `failure ${i}` : 'ok', - }, - ], - }, - }), - ); - } + await onOutput?.( + sdkMsg('assistant', { + uuid: 'a-1', + message: { + content: [ + { + type: 'tool_use', + name: 'Bash', + id: 'tool-1', + input: { command: 'step-1' }, + }, + ], + }, + }), + ); + vi.advanceTimersByTime(1); + await onOutput?.( + sdkMsg('user', { + uuid: 'u-1', + message: { + content: [ + { + type: 'tool_result', + tool_use_id: 'tool-1', + is_error: true, + content: 'failure 1', + }, + ], + }, + }), + ); await onOutput?.({ type: 'state', state: 'stopped', @@ -328,16 +326,16 @@ describe('tool usage analytics', () => { await expect(db.getToolUsageSummary()).resolves.toEqual([ expect.objectContaining({ toolName: 'Bash', - callCount: 5, - successCount: 1, - successRate: 0.2, + callCount: 1, + successCount: 0, + successRate: 0, }), ]); expect(alerts).toHaveLength(1); expect(alerts[0]).toMatchObject({ toolName: 'Bash', - errorRate: 0.8, - callCount: 5, + errorRate: 1, + callCount: 1, windowHours: 1, }); expect(alerts[0]).not.toHaveProperty('agentId'); From 97e0336979d6f8e72e2500906cf7ac57a537b974 Mon Sep 17 00:00:00 2001 From: dorianzheng Date: Sat, 25 Apr 2026 01:35:09 +0800 Subject: [PATCH 09/10] fix: remove callCount>=5 alert gate --- src/agent/message-processor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/message-processor.ts b/src/agent/message-processor.ts index c3328a27dde..bd59c67521b 100644 --- a/src/agent/message-processor.ts +++ b/src/agent/message-processor.ts @@ -377,7 +377,7 @@ export class MessageProcessor { }); const row = rows[0]; const errorRate = row ? 1 - row.successRate : 0; - if (row && row.callCount > 0 && errorRate > 0.2) { + if (row && errorRate > 0.2) { logger.warn( { toolName, callCount: row.callCount, errorRate, windowHours: 1 }, 'Tool error rate exceeded 20% in the last hour', From 182f3bcdf7857344c55adcf3bac2e4e4b02ceccc Mon Sep 17 00:00:00 2001 From: dorianzheng Date: Sat, 25 Apr 2026 01:39:28 +0800 Subject: [PATCH 10/10] fix: build package during git installs --- package.json | 2 +- scripts/check-is-in-git-install.sh | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100755 scripts/check-is-in-git-install.sh diff --git a/package.json b/package.json index 83bb614f626..616c27beeb9 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ "format": "prettier --write \"src/**/*.ts\"", "format:fix": "prettier --write \"src/**/*.ts\"", "format:check": "prettier --check \"src/**/*.ts\"", - "prepare": "husky", + "prepare": "bash -c 'if bash ./scripts/check-is-in-git-install.sh; then npm run build; else husky || true; fi'", "setup": "tsx setup/index.ts", "auth": "tsx src/whatsapp-auth.ts", "test:e2e:tasks": "tsx scripts/test-task-sdk-e2e.ts", diff --git a/scripts/check-is-in-git-install.sh b/scripts/check-is-in-git-install.sh new file mode 100755 index 00000000000..f9e8b06569f --- /dev/null +++ b/scripts/check-is-in-git-install.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Exit 0 if this `prepare` run is a git-dependency install; non-zero otherwise. +# Contributors can force-skip with AGENTLITE_DEV=1, force-build with AGENTLITE_BUILD=1. + +if [ -n "$AGENTLITE_DEV" ]; then + exit 1 +fi + +if [ -n "$AGENTLITE_BUILD" ]; then + exit 0 +fi + +parent_name="$(basename "$(dirname "$PWD")")" +[ "$parent_name" = 'node_modules' ] || +[ "$parent_name" = 'tmp' ] || +[ "$parent_name" = '.tmp' ]