From fb9e9376f8c0a25e7d5192066309bc9a301e2a98 Mon Sep 17 00:00:00 2001 From: mariobrajkovski Date: Sat, 4 Jul 2026 15:58:21 +0200 Subject: [PATCH] fix(agents): malformed tool-call JSON no longer kills the rollout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A completion truncated at the output-token limit mid-arguments (e.g. Qwen emitting '{"command": "' and stopping) raised JSONDecodeError from the bare json.loads in get_response, which escaped the agent loop and errored the whole trace with 'Unterminated string starting at: line 1 column 13'. Parse failures now produce the tool call with a sentinel argument (MALFORMED_TOOL_ARGS_KEY); the dispatcher returns an isError tool result for that call id without executing anything, so the provider contract holds and the model can re-issue the call. The recorded assistant message replays "{}" instead of the raw malformed string — some backends (Qwen/vLLM) re-parse arguments when templating history and 500 on the raw string (verified live). Applied to both the chat.completions and Responses parse sites. Verified end-to-end against Qwen/Qwen3.5-397B-A17B via the HUD gateway with forced wire truncation: no crash, replay accepted, call re-issued with complete JSON. --- hud/agents/openai/agent.py | 7 +-- hud/agents/openai_compatible/agent.py | 36 +++++++++----- .../tests/test_openai_compatible_agent.py | 47 +++++++++++++++++++ hud/agents/tool_agent.py | 31 ++++++++++++ 4 files changed, 105 insertions(+), 16 deletions(-) diff --git a/hud/agents/openai/agent.py b/hud/agents/openai/agent.py index 6b504b2fa..597388d71 100644 --- a/hud/agents/openai/agent.py +++ b/hud/agents/openai/agent.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import logging from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal, cast @@ -24,7 +23,7 @@ ) from openai.types.shared_params.reasoning import Reasoning # noqa: TC002 -from hud.agents.tool_agent import RunState, ToolAgent +from hud.agents.tool_agent import RunState, ToolAgent, parse_tool_arguments from hud.agents.types import AgentStep, Citation, OpenAIConfig, Usage from hud.settings import settings from hud.types import MCPToolCall, MCPToolResult @@ -273,7 +272,9 @@ async def get_response( tool_calls.append( MCPToolCall( name=item.name or "", - arguments=json.loads(item.arguments), + arguments=parse_tool_arguments( + item.arguments, tool_name=item.name or "" + ), id=item.call_id, ) ) diff --git a/hud/agents/openai_compatible/agent.py b/hud/agents/openai_compatible/agent.py index 9887a0861..c8af23755 100644 --- a/hud/agents/openai_compatible/agent.py +++ b/hud/agents/openai_compatible/agent.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import logging from dataclasses import dataclass from typing import TYPE_CHECKING, Any, cast @@ -10,7 +9,12 @@ from openai import AsyncOpenAI from openai.types.chat import ChatCompletion, ChatCompletionMessageParam -from hud.agents.tool_agent import RunState, ToolAgent +from hud.agents.tool_agent import ( + MALFORMED_TOOL_ARGS_KEY, + RunState, + ToolAgent, + parse_tool_arguments, +) from hud.agents.types import AgentStep, OpenAIChatConfig, Sample, Usage from hud.settings import settings from hud.types import MCPToolCall, MCPToolResult @@ -170,6 +174,14 @@ async def get_response( choice = response.choices[0] message = choice.message function_calls = [tc for tc in message.tool_calls or [] if tc.type == "function"] + tool_calls: list[MCPToolCall] = [ + MCPToolCall( + id=tc.id, + name=tc.function.name, + arguments=parse_tool_arguments(tc.function.arguments, tool_name=tc.function.name), + ) + for tc in function_calls + ] assistant_message = message.model_dump(exclude_none=True) reasoning_content = getattr(message, "reasoning_content", None) @@ -187,10 +199,17 @@ async def get_response( "type": "function", "function": { "name": tc.function.name, - "arguments": tc.function.arguments, + # Replaying a malformed arguments string 500s some backends (Qwen/vLLM + # re-parse it when templating history); record "{}" instead — the raw + # string rides in the sentinel and the trace. + "arguments": ( + "{}" + if MALFORMED_TOOL_ARGS_KEY in (call.arguments or {}) + else tc.function.arguments + ), }, } - for tc in function_calls + for tc, call in zip(function_calls, tool_calls, strict=True) ] messages.append(cast("ChatCompletionMessageParam", assistant_message)) @@ -219,15 +238,6 @@ async def get_response( chat_state.continuation_token_ids = None chat_state.continuation_message_count = None - tool_calls: list[MCPToolCall] = [] - for tc in function_calls: - provider_name = tc.function.name - raw_args = json.loads(tc.function.arguments or "{}") - arguments = cast("dict[str, Any]", raw_args) if isinstance(raw_args, dict) else {} - tool_calls.append( - MCPToolCall(id=tc.id, name=provider_name, arguments=arguments), - ) - usage: Usage | None = None if response.usage is not None: details = response.usage.prompt_tokens_details diff --git a/hud/agents/tests/test_openai_compatible_agent.py b/hud/agents/tests/test_openai_compatible_agent.py index 3f7f1d122..f3a49d8e9 100644 --- a/hud/agents/tests/test_openai_compatible_agent.py +++ b/hud/agents/tests/test_openai_compatible_agent.py @@ -6,6 +6,8 @@ from types import SimpleNamespace from typing import Any, cast +import mcp.types as mcp_types + from hud.agents.openai_compatible.agent import OpenAIChatAgent, OpenAIChatRunState from hud.agents.types import OpenAIChatConfig @@ -81,3 +83,48 @@ async def test_get_response_error_path() -> None: result = await agent.get_response(_state(agent)) assert result.done is True assert result.error is not None and "boom" in result.error + + +async def test_get_response_malformed_tool_args_do_not_crash() -> None: + """Truncated arguments must yield a dispatchable sentinel call, not a rollout-killing + JSONDecodeError (live Qwen failure: 'Unterminated string starting at line 1 column 13').""" + from hud.agents.tool_agent import MALFORMED_TOOL_ARGS_KEY + + tc = SimpleNamespace( + type="function", + id="c1", + function=SimpleNamespace(name="bash", arguments='{"command": "'), + ) + agent = _agent(_response("", [tc])) + state = _state(agent) + result = await agent.get_response(state) + assert result.error is None + assert [c.name for c in result.tool_calls] == ["bash"] + assert result.tool_calls[0].id == "c1" # provider still gets a result + assert MALFORMED_TOOL_ARGS_KEY in (result.tool_calls[0].arguments or {}) + assert result.done is False # the loop continues + # the RECORDED assistant message must replay "{}" — a raw malformed arguments string 500s + # some backends (Qwen/vLLM re-parse it when templating history) + recorded = state.messages[-1] + assert recorded["tool_calls"][0]["function"]["arguments"] == "{}" + + +async def test_dispatch_returns_error_result_for_malformed_args() -> None: + from hud.agents.tool_agent import MALFORMED_TOOL_ARGS_KEY + from hud.types import MCPToolCall + + agent = _agent(_response("", [])) + state = _state(agent) + state.tools = {} + call = MCPToolCall( + id="c1", + name="bash", + arguments={MALFORMED_TOOL_ARGS_KEY: "Unterminated string starting at: line 1 column 13"}, + ) + result = await agent._dispatch_call(call, state) + assert result.isError is True + content = result.content[0] + assert isinstance(content, mcp_types.TextContent) + text = content.text + assert "not valid JSON" in text and "Re-issue" in text + assert "Unterminated string" in text diff --git a/hud/agents/tool_agent.py b/hud/agents/tool_agent.py index 9f5448ade..3c4f46c63 100644 --- a/hud/agents/tool_agent.py +++ b/hud/agents/tool_agent.py @@ -18,6 +18,7 @@ def _format_result(self, call, result) -> BetaMessageParam | None: ... from __future__ import annotations import asyncio +import json import logging from abc import abstractmethod from dataclasses import dataclass, field @@ -40,6 +41,20 @@ def _format_result(self, call, result) -> BetaMessageParam | None: ... logger = logging.getLogger(__name__) +# Tool calls whose arguments never parsed as JSON (usually a completion truncated at the token +# limit): _dispatch_call answers the call id with an error result so the model can re-issue. +MALFORMED_TOOL_ARGS_KEY = "__hud_malformed_tool_arguments__" + + +def parse_tool_arguments(raw: str | None, *, tool_name: str) -> dict[str, Any]: + try: + parsed = json.loads(raw or "{}") + except json.JSONDecodeError as e: + logger.warning("Malformed tool-call arguments for %s: %s", tool_name, e) + return {MALFORMED_TOOL_ARGS_KEY: f"{e} in arguments: {(raw or '')[:200]!r}"} + return cast("dict[str, Any]", parsed) if isinstance(parsed, dict) else {} + + MessageT = TypeVar("MessageT") ConfigT = TypeVar("ConfigT", bound="AgentConfig") @@ -242,6 +257,22 @@ async def _dispatch_call( call: MCPToolCall, state: RunState[MessageT], ) -> MCPToolResult: + args_maybe = call.arguments if isinstance(call.arguments, dict) else {} + if MALFORMED_TOOL_ARGS_KEY in args_maybe: + return MCPToolResult( + content=[ + mcp_types.TextContent( + type="text", + text=( + f"tool error: arguments for {call.name!r} were not valid JSON " + f"(response likely truncated at the output-token limit): " + f"{args_maybe[MALFORMED_TOOL_ARGS_KEY]}. " + f"Re-issue the tool call with complete JSON arguments." + ), + ) + ], + isError=True, + ) tool = state.tools.get(call.name) if tool is None: return MCPToolResult(