diff --git a/packages/core/src/llm-core/agent/executor.ts b/packages/core/src/llm-core/agent/executor.ts index c0d38bcc4..0d4130c48 100644 --- a/packages/core/src/llm-core/agent/executor.ts +++ b/packages/core/src/llm-core/agent/executor.ts @@ -11,6 +11,7 @@ import type { AgentExecutorInput, AgentExecutorOutput } from './legacy-executor' export { coerceToAgentObservation, LegacyAgentExecutor, + observationToMessageContent, runAgent, toToolInputErrorObservation, type AgentExecutorInput, diff --git a/packages/core/src/llm-core/agent/legacy-executor.ts b/packages/core/src/llm-core/agent/legacy-executor.ts index e65042d9d..fa90cf425 100644 --- a/packages/core/src/llm-core/agent/legacy-executor.ts +++ b/packages/core/src/llm-core/agent/legacy-executor.ts @@ -479,6 +479,10 @@ function isAgentObservation(value: unknown): value is AgentObservation { return true } + if (isDirectToolOutput(value)) { + return true + } + if (!Array.isArray(value)) { return false } @@ -490,6 +494,10 @@ export function coerceToAgentObservation( observation: unknown, toolName?: string ): AgentObservation { + if (isDirectToolOutput(observation)) { + return observation + } + if (isAgentObservation(observation)) { if ( Array.isArray(observation) && @@ -591,3 +599,7 @@ function toParsingErrorAction( log: text } } + +export function observationToMessageContent(observation: AgentObservation) { + return isDirectToolOutput(observation) ? '' : observation +} diff --git a/packages/core/src/llm-core/agent/openai/index.ts b/packages/core/src/llm-core/agent/openai/index.ts index b7366f3cc..8952b4d6e 100644 --- a/packages/core/src/llm-core/agent/openai/index.ts +++ b/packages/core/src/llm-core/agent/openai/index.ts @@ -30,6 +30,7 @@ import { } from './output_parser' import { BaseChatPromptTemplate } from '@langchain/core/prompts' import { getMessageContent } from 'koishi-plugin-chatluna/utils/string' +import { observationToMessageContent } from '../legacy-executor' /** * Checks if the given action is a FunctionsAgentAction. @@ -55,16 +56,22 @@ function _convertAgentStepToMessages( ) { if (isToolsAgentAction(action) && action.toolCallId !== undefined) { const log = action.messageLog as BaseMessage[] + const content = observationToMessageContent(observation) if ( - observation.length < 1 || - observation == null || - observation === 'null' + content === observation && + (content.length < 1 || content === 'null') ) { - observation = `The tool ${action.tool} returned no output. Try again or stop the tool call, tell the user failed to execute the tool.` + return log.concat( + new ToolMessage({ + content: `The tool ${action.tool} returned no output. Try again or stop the tool call, tell the user failed to execute the tool.`, + name: action.tool, + tool_call_id: action.toolCallId + }) + ) } return log.concat( new ToolMessage({ - content: observation, + content, name: action.tool, tool_call_id: action.toolCallId }) diff --git a/packages/core/src/llm-core/agent/sub-agent.ts b/packages/core/src/llm-core/agent/sub-agent.ts index 36c777d32..15d49e8a5 100644 --- a/packages/core/src/llm-core/agent/sub-agent.ts +++ b/packages/core/src/llm-core/agent/sub-agent.ts @@ -12,6 +12,7 @@ import { z } from 'zod' import type { ChatLunaToolRunnable } from '../platform/types' import { getMessageContent } from 'koishi-plugin-chatluna/utils/string' import type { ChatLunaAgent } from './agent' +import { observationToMessageContent } from './legacy-executor' import { MessageQueue } from './types' import type { AgentEvent, AgentStep, SubagentContext, ToolMask } from './types' @@ -868,7 +869,7 @@ function createAgentToolMessages(steps: AgentStep[]): BaseMessage[] { ...steps.map( (step) => new ToolMessage({ - content: step.observation, + content: observationToMessageContent(step.observation), tool_call_id: step.action.toolCallId, name: step.action.tool }) diff --git a/packages/core/src/llm-core/agent/types.ts b/packages/core/src/llm-core/agent/types.ts index 59119623a..d16e8a653 100644 --- a/packages/core/src/llm-core/agent/types.ts +++ b/packages/core/src/llm-core/agent/types.ts @@ -10,6 +10,7 @@ import type { MessageContentFileUrl, MessageContentVideo } from 'koishi-plugin-chatluna/utils/langchain' +import type { DirectToolOutput } from '@langchain/core/messages/tool' export interface ChatCompletionMessageToolCall { /** @@ -193,7 +194,14 @@ export type AgentObservationComplexContent = | MessageContentAudio | MessageContentVideo -export type AgentObservation = AgentObservationComplexContent[] | string +export type AgentDirectToolObservation = DirectToolOutput & { + replyEmitted?: boolean +} + +export type AgentObservation = + | AgentObservationComplexContent[] + | AgentDirectToolObservation + | string export interface ToolMask { mode: 'all' | 'allow' | 'deny' diff --git a/packages/core/src/llm-core/memory/message/database_history.ts b/packages/core/src/llm-core/memory/message/database_history.ts index 51a60af0c..6450cfb90 100644 --- a/packages/core/src/llm-core/memory/message/database_history.ts +++ b/packages/core/src/llm-core/memory/message/database_history.ts @@ -15,6 +15,7 @@ import { gzipEncode } from 'koishi-plugin-chatluna/utils/string' import { randomUUID } from 'crypto' +import { observationToMessageContent } from '../../agent/legacy-executor' import type { AgentStep } from '../../agent/types' import type { MessageRecord } from '../../../services/conversation_types' @@ -71,7 +72,7 @@ function createAgentToolMessages(steps: AgentStep[]): BaseMessage[] { ...steps.map( (step) => new ToolMessage({ - content: step.observation, + content: observationToMessageContent(step.observation), tool_call_id: step.action.toolCallId, name: step.action.tool }) diff --git a/packages/extension-agent/src/sub-agent/session.ts b/packages/extension-agent/src/sub-agent/session.ts index b0f242092..3d9c40fe3 100644 --- a/packages/extension-agent/src/sub-agent/session.ts +++ b/packages/extension-agent/src/sub-agent/session.ts @@ -6,7 +6,10 @@ import { HumanMessage, ToolMessage } from '@langchain/core/messages' -import type { AgentStep } from 'koishi-plugin-chatluna/llm-core/agent' +import { + type AgentStep, + observationToMessageContent +} from 'koishi-plugin-chatluna/llm-core/agent' import { getMessageContent } from 'koishi-plugin-chatluna/utils/string' import type { SubAgentInfo, SubAgentRunInfo } from '../types' @@ -219,7 +222,7 @@ function createAgentToolMessages(steps: AgentStep[]): BaseMessage[] { ...steps.map( (step) => new ToolMessage({ - content: step.observation, + content: observationToMessageContent(step.observation), tool_call_id: step.action.toolCallId, name: step.action.tool })