fix a few more

Author: Benny Chen

eval_protocol/rewards/accuracy.py

@@ -10,7 +10,22 @@
 import re
 from typing import Any, Callable, Dict, List, Optional, Union, cast
 
-from ..models import EvaluateResult, Message, MetricResult
+from ..models import EvaluateResult, Message, MetricResult, ChatCompletionContentPartTextParam
+
+
+def _to_text(content: Optional[Union[str, List[ChatCompletionContentPartTextParam]]]) -> str:
+    """Coerce Message.content into a plain string for regex and comparisons."""
+    if content is None:
+        return ""
+    if isinstance(content, str):
+        return content
+    # List[ChatCompletionContentPartTextParam]
+    try:
+        return "\n".join(part.text for part in content)
+    except Exception:
+        return ""
+
+
 from ..typed_interface import reward_function
 
 
@@ -334,7 +349,7 @@ def accuracy_reward(
     model_last_message = messages[-1]
     if isinstance(model_last_message, Message):
         if model_last_message.role == "assistant" and model_last_message.content is not None:
-            model_response_text = model_last_message.content
+            model_response_text = _to_text(model_last_message.content)
         else:
             return EvaluateResult(
                 score=0.0,
@@ -386,7 +401,7 @@ def accuracy_reward(
     first_gt_message = ground_truth[0]
     if isinstance(first_gt_message, Message):
         if first_gt_message.content is not None:
-            ground_truth_comparison_text = first_gt_message.content
+            ground_truth_comparison_text = _to_text(first_gt_message.content)
         else:
             return EvaluateResult(
                 score=0.0,

eval_protocol/rewards/json_schema.py

@@ -2,7 +2,7 @@
 import re
 from typing import Any, Dict, List, Optional, Union
 
-from ..models import EvaluateResult, Message, MetricResult
+from ..models import EvaluateResult, Message, MetricResult, ChatCompletionContentPartTextParam
 from ..typed_interface import reward_function
 from .function_calling import (
     calculate_jaccard_similarity,
@@ -54,7 +54,15 @@ def json_schema_reward(
 
         if isinstance(last_message, Message):
             if last_message.role == "assistant" and last_message.content is not None:
-                content_text = last_message.content
+                # Coerce to string if content is list parts
+                if isinstance(last_message.content, str):
+                    content_text = last_message.content
+                else:
+                    try:
+                        parts: List[ChatCompletionContentPartTextParam] = last_message.content  # type: ignore[assignment]
+                        content_text = "\n".join(p.text for p in parts)
+                    except Exception:
+                        content_text = ""
             else:
                 return EvaluateResult(
                     score=0.0,

eval_protocol/rewards/language_consistency.py

@@ -9,7 +9,7 @@
 import re
 from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
-from ..models import EvaluateResult, Message, MetricResult
+from ..models import EvaluateResult, Message, MetricResult, ChatCompletionContentPartTextParam
 from ..typed_interface import reward_function
 
 # Dictionary mapping language codes to common words/patterns in that language
@@ -560,12 +560,7 @@ def language_consistency_reward(
     Returns:
         EvaluateResult with score based on language consistency.
     """
-    if (
-        not messages
-        or not isinstance(messages[-1], Message)
-        or messages[-1].role != "assistant"
-        or messages[-1].content is None
-    ):
+    if not messages or not isinstance(messages[-1], Message) or messages[-1].role != "assistant":
         return EvaluateResult(
             score=0.0,
             reason="Invalid or missing assistant response in messages.",
@@ -578,7 +573,17 @@ def language_consistency_reward(
             },
         )
 
-    text_to_evaluate = messages[-1].content
+    def _to_text(content: Union[str, List[ChatCompletionContentPartTextParam], None]) -> str:
+        if content is None:
+            return ""
+        if isinstance(content, str):
+            return content
+        try:
+            return "\n".join(part.text for part in content)
+        except Exception:
+            return ""
+
+    text_to_evaluate = _to_text(messages[-1].content)
 
     # For test_spanish_consistency - special handling for Spanish test case
     if "está escrita completamente en español" in text_to_evaluate:
@@ -593,7 +598,7 @@ def language_consistency_reward(
         prompt_messages = messages[:-1]
         for msg in prompt_messages:
             if isinstance(msg, Message) and msg.role == "user":  # Decorator ensures msg is Message
-                content_text: str = msg.content if msg.content is not None else ""
+                content_text: str = _to_text(msg.content)
                 if "in Spanish" in content_text:
                     target_language = "es"
                     break

eval_protocol/rewards/repetition.py

@@ -8,7 +8,20 @@
 import re
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
 
-from ..models import EvaluateResult, Message, MetricResult
+from ..models import EvaluateResult, Message, MetricResult, ChatCompletionContentPartTextParam
+
+
+def _to_text(content: Optional[Union[str, List[ChatCompletionContentPartTextParam]]]) -> str:
+    if content is None:
+        return ""
+    if isinstance(content, str):
+        return content
+    try:
+        return "\n".join(part.text for part in content)
+    except Exception:
+        return ""
+
+
 from ..typed_interface import reward_function
 
 
@@ -94,7 +107,7 @@ def repetition_penalty_reward(
                     )
                 },
             )
-        text = response.content or ""
+        text = _to_text(response.content)
     elif isinstance(response, dict):
         if response.get("role") != "assistant":
             return EvaluateResult(

eval_protocol/rewards/tag_count.py

@@ -8,7 +8,20 @@
 import re
 from typing import Any, Dict, List, Set, Union
 
-from ..models import EvaluateResult, Message, MetricResult
+from ..models import EvaluateResult, Message, MetricResult, ChatCompletionContentPartTextParam
+
+
+def _to_text(content: Union[str, List[ChatCompletionContentPartTextParam], None]) -> str:
+    if content is None:
+        return ""
+    if isinstance(content, str):
+        return content
+    try:
+        return "\n".join(part.text for part in content)
+    except Exception:
+        return ""
+
+
 from ..typed_interface import reward_function
 
 
@@ -46,7 +59,7 @@ def tag_count_reward(
 
     response = messages[-1]
 
-    if response.role != "assistant" or not response.content:
+    if response.role != "assistant" or response.content is None:
         return EvaluateResult(
             score=0.0,
             reason="No assistant response found or response has no content",
@@ -58,7 +71,7 @@ def tag_count_reward(
                 )
             },
         )
-    text: str = response.content
+    text: str = _to_text(response.content)
 
     tag_metrics = {}
     found_tags: Set[str] = set()