done

Author: Dylan Huang

eval_protocol/pytest/default_pydantic_ai_rollout_processor.py

@@ -4,6 +4,7 @@
 from collections.abc import Callable
 import logging
 import time
+from pydantic_ai.toolsets import FunctionToolset
 from pydantic_ai.usage import UsageLimits
 from typing_extensions import override
 from eval_protocol.models import EvaluationRow, Message
@@ -53,18 +54,19 @@ async def process_row(row: EvaluationRow) -> EvaluationRow:
             start_time = time.perf_counter()
 
             tools = []
-            # for _, tool in agent._function_tools.items():
-            #     tool_dict = {
-            #         "type": "function",
-            #         "function": {
-            #             "name": tool.name,
-            #             "parameters": tool.function_schema.json_schema,
-            #         },
-            #     }
-            #     if tool.description:
-            #         tool_dict["function"]["description"] = tool.description
-
-            #     tools.append(tool_dict)
+            for toolset in agent.toolsets:
+                if isinstance(toolset, FunctionToolset):
+                    for _, tool in toolset.tools.items():
+                        tool_dict = {
+                            "type": "function",
+                            "function": {
+                                "name": tool.name,
+                                "parameters": tool.function_schema.json_schema,
+                            },
+                        }
+                        if tool.description:
+                            tool_dict["function"]["description"] = tool.description
+                        tools.append(tool_dict)
             row.tools = tools
 
             model_messages = [self.convert_ep_message_to_pyd_message(m, row) for m in row.messages]

tests/chinook/pydantic/test_pydantic_chinook.py

@@ -44,6 +44,23 @@ async def test_simple_query(row: EvaluationRow) -> EvaluationRow:
     """
     Super simple query for the Chinook database
     """
+    expected_tools = [
+        {
+            "type": "function",
+            "function": {
+                "name": "execute_sql",
+                "parameters": {
+                    "additionalProperties": False,
+                    "properties": {"query": {"type": "string"}},
+                    "required": ["query"],
+                    "type": "object",
+                },
+            },
+        }
+    ]
+    assert hasattr(row, "tools"), "Row missing 'tools' attribute"
+    assert row.tools == expected_tools, f"Tools validation failed. Expected: {expected_tools}, Got: {row.tools}"
+
     last_assistant_message = row.last_assistant_message()
     if last_assistant_message is None:
         row.evaluation_result = EvaluateResult(