type fix round 6

Author: Benny Chen

eval_protocol/benchmarks/test_aime25.py

@@ -1,6 +1,12 @@
 from typing import Any, Dict, List, Optional
 
-from eval_protocol.models import EvaluateResult, EvaluationRow, Message, MetricResult
+from eval_protocol.models import (
+    EvaluateResult,
+    EvaluationRow,
+    Message,
+    MetricResult,
+    ChatCompletionContentPartTextParam,
+)
 from eval_protocol.pytest.default_single_turn_rollout_process import (
     SingleTurnRolloutProcessor,
 )
@@ -11,6 +17,14 @@
 )
 
 
+def _coerce_content_to_str(
+    content: str | list[ChatCompletionContentPartTextParam] | None,
+) -> str:
+    if isinstance(content, list):
+        return "".join([getattr(p, "text", str(p)) for p in content])
+    return str(content or "")
+
+
 def _extract_boxed_text(text: str) -> str:
     import re
 
@@ -80,9 +94,10 @@ def aime2025_dataset_adapter(rows: List[Dict[str, Any]]) -> List[EvaluationRow]:
 )
 def test_aime25_pointwise(row: EvaluationRow) -> EvaluationRow:
     assistant_msgs = [m for m in row.messages if m.role == "assistant"]
-    content = assistant_msgs[-1].content if assistant_msgs else ""
+    raw_content = assistant_msgs[-1].content if assistant_msgs else ""
+    content_str = _coerce_content_to_str(raw_content)
 
-    extracted_text = _extract_boxed_text(content or "")
+    extracted_text = _extract_boxed_text(content_str)
     extracted_int = _normalize_to_int_or_none(extracted_text)
     gt_int = _normalize_to_int_or_none(row.ground_truth or "")
 

eval_protocol/benchmarks/test_gpqa.py

@@ -5,7 +5,13 @@
 
 import requests
 
-from eval_protocol.models import EvaluateResult, EvaluationRow, Message, MetricResult
+from eval_protocol.models import (
+    EvaluateResult,
+    EvaluationRow,
+    Message,
+    MetricResult,
+    ChatCompletionContentPartTextParam,
+)
 from eval_protocol.pytest.default_single_turn_rollout_process import (
     SingleTurnRolloutProcessor,
 )
@@ -47,6 +53,14 @@ def _load_gpqa_messages_from_csv() -> list[list[list[Message]]]:
     return [messages_list]
 
 
+def _coerce_content_to_str(
+    content: str | list[ChatCompletionContentPartTextParam] | None,
+) -> str:
+    if isinstance(content, list):
+        return "".join([getattr(p, "text", str(p)) for p in content])
+    return str(content or "")
+
+
 def _extract_abcd_letter(text: str) -> str | None:
     if not text:
         return None
@@ -58,9 +72,12 @@ def _extract_abcd_letter(text: str) -> str | None:
 
 
 def _strip_gt_messages(msgs: list[Message]) -> list[Message]:
-    # assert that all the messages just have a plain .content string field
-    assert all(isinstance(m.content, str) for m in msgs), "Messages must have a plain .content string field"
-    return [m for m in msgs if not (m.role == "system" and (m.content or "").startswith("__GT__:"))]
+    result: list[Message] = []
+    for m in msgs:
+        content_str = _coerce_content_to_str(m.content)
+        if not (m.role == "system" and content_str.startswith("__GT__:")):
+            result.append(m)
+    return result
 
 
 class GPQAStripGTRolloutProcessor(RolloutProcessor):
@@ -75,15 +92,23 @@ def __call__(self, rows: list[EvaluationRow], config: RolloutProcessorConfig) ->
         processed: list[EvaluationRow] = []
 
         for r in rows:
-            gt_tokens = [
-                m.content for m in r.messages if m.role == "system" and (m.content or "").startswith("__GT__:")
-            ]
+            gt_tokens: list[str] = []
+            for m in r.messages:
+                if m.role == "system":
+                    content_str = _coerce_content_to_str(m.content)
+                    if content_str.startswith("__GT__:"):
+                        gt_tokens.append(content_str)
             if gt_tokens:
                 gt_val = gt_tokens[-1].split(":", 1)[1].strip()
                 r.ground_truth = gt_val
-                r.messages = [
-                    m for m in r.messages if not (m.role == "system" and (m.content or "").startswith("__GT__:"))
-                ]
+                filtered: list[Message] = []
+                for m in r.messages:
+                    if m.role == "system":
+                        content_str = _coerce_content_to_str(m.content)
+                        if content_str.startswith("__GT__:"):
+                            continue
+                    filtered.append(m)
+                r.messages = filtered
             processed.append(r)
 
         # Delegate to SingleTurnRolloutProcessor
@@ -103,9 +128,10 @@ def __call__(self, rows: list[EvaluationRow], config: RolloutProcessorConfig) ->
 )
 def test_gpqa_pointwise(row: EvaluationRow) -> EvaluationRow:
     assistant_msgs = [m for m in row.messages if m.role == "assistant"]
-    content = assistant_msgs[-1].content if assistant_msgs else ""
+    raw_content = assistant_msgs[-1].content if assistant_msgs else ""
+    content_str = _coerce_content_to_str(raw_content)
 
-    pred = _extract_abcd_letter(content or "")
+    pred = _extract_abcd_letter(content_str)
     # GPQA diamond CSV constructs options so that the correct answer is always A
     gt = "A"
 

eval_protocol/models.py

@@ -271,6 +271,7 @@ class MetricResult(BaseModel):
     is_score_valid: bool = True
     score: float = Field(..., ge=0.0, le=1.0)
     reason: str
+    data: Dict[str, Any] = Field(default_factory=dict, description="Optional extra metric data for debugging.")
 
     def __getitem__(self, key: str) -> Any:
         if key in self.__fields__:  # Changed to __fields__ for Pydantic v1 compatibility

eval_protocol/pytest/default_agent_rollout_processor.py

@@ -6,13 +6,13 @@
 
 from mcp.types import CallToolResult, TextContent
 from openai import NOT_GIVEN, NotGiven
-from openai.types.chat import ChatCompletionContentPartTextParam
+from openai.types.chat import ChatCompletionContentPartTextParam as OpenAIChatContentPart
 from openai.types.chat.chat_completion_message_param import ChatCompletionMessageParam
 
 from eval_protocol.dataset_logger.dataset_logger import DatasetLogger
 from eval_protocol.mcp.execution.policy import LiteLLMPolicy
 from eval_protocol.mcp.mcp_multi_client import MCPMultiClient
-from eval_protocol.models import EvaluationRow, Message
+from eval_protocol.models import EvaluationRow, Message, ChatCompletionContentPartTextParam
 from eval_protocol.pytest.rollout_processor import RolloutProcessor
 from eval_protocol.pytest.types import Dataset, RolloutProcessorConfig
 from pydantic import BaseModel
@@ -215,6 +215,7 @@ def _format_tool_message_content(
         """
         if len(content) == 1 and isinstance(content[0], TextContent):
             return content[0].text
+        # Build our SDK's ChatCompletionContentPartTextParam instances, not OpenAI types
         return [ChatCompletionContentPartTextParam(text=c.text, type="text") for c in content]
 
 

eval_protocol/pytest/default_langchain_rollout_processor.py

@@ -43,7 +43,11 @@ def __init__(self, content: str):
             if row.messages:
                 last_user = [m for m in row.messages if m.role == "user"]
                 if last_user:
-                    lm_messages.append(HumanMessage(content=last_user[-1].content or ""))
+                    content = last_user[-1].content or ""
+                    if isinstance(content, list):
+                        # Flatten our SDK content parts into a single string for LangChain
+                        content = "".join([getattr(p, "text", str(p)) for p in content])
+                    lm_messages.append(HumanMessage(content=str(content)))
             if not lm_messages:
                 lm_messages = [HumanMessage(content="")]  # minimal