Merge branch 'main' into derekx/test-log-level

Author: xzrderek

eval_protocol/mcp/mcp_multi_client.py

@@ -70,6 +70,38 @@ def _validate_environment_variables(self, server_name: str, required_env: List[s
                 f"Please set these variables in your environment or .env file."
             )
 
+    def _process_headers(self, headers: Dict[str, str]) -> Dict[str, str]:
+        """Process headers by substituting environment variables.
+
+        Supports environment variable substitution in the format:
+        - ${ENV_VAR} or $ENV_VAR for environment variables
+        - Raw strings are passed through unchanged
+
+        Example:
+            {"Authorization": "Bearer ${API_KEY}"}
+            -> {"Authorization": "Bearer abc123"} (if API_KEY=abc123)
+        """
+        import re
+
+        processed_headers = {}
+        for key, value in headers.items():
+            # Match ${VAR} or $VAR patterns
+            def replace_env_var(match):
+                var_name = match.group(1) or match.group(2)
+                env_value = os.environ.get(var_name)
+                if env_value is None:
+                    raise ValueError(
+                        f"Environment variable '{var_name}' referenced in header '{key}' "
+                        f"is not set. Please set it in your environment or .env file."
+                    )
+                return env_value
+
+            # Replace ${VAR} or $VAR with environment variable value
+            processed_value = re.sub(r"\$\{([^}]+)\}|\$([A-Za-z_][A-Za-z0-9_]*)", replace_env_var, value)
+            processed_headers[key] = processed_value
+
+        return processed_headers
+
     async def connect_to_servers(self):
         """Connect to all configured MCP servers"""
         if not self.config.mcpServers:
@@ -111,8 +143,17 @@ async def _connect_to_server(
             if not url:
                 raise ValueError(f"Server '{server_name}' must have a 'url' specified")
 
-            # Connect using streamable HTTP client - manage resources manually
-            http_transport = await self.exit_stack.enter_async_context(streamablehttp_client(url))
+            # Build headers only from the authorization field (Responses-style)
+            processed_headers: Dict[str, str] = {}
+            auth_token = getattr(server_config, "authorization", None)
+            if auth_token:
+                # Support env substitution in the authorization value as well
+                processed_headers = self._process_headers({"Authorization": auth_token})
+
+            # Connect using streamable HTTP client with auth headers
+            http_transport = await self.exit_stack.enter_async_context(
+                streamablehttp_client(url, headers=processed_headers)
+            )
             read_stream, write_stream, get_session_id = http_transport
             session = await self.exit_stack.enter_async_context(ClientSession(read_stream, write_stream))
         else:

eval_protocol/models.py

@@ -1119,6 +1119,7 @@ class MCPConfigurationServerUrl(BaseModel):
     """Represents a Remote MCP configuration server."""
 
     url: str  # url to the MCP server
+    authorization: Optional[str] = None
 
 
 class MCPMultiClientConfiguration(BaseModel):

eval_protocol/pytest/default_agent_rollout_processor.py

@@ -133,8 +133,11 @@ async def call_agent(self) -> Optional[Union[str, List[ChatCompletionContentPart
 
     async def _call_model(self, messages: list[Message], tools: Optional[List[dict[str, Any]]]) -> Message:
         # Convert Message models to plain dicts for LLM call
+        # Filter out fields that are not supported by OpenAI/LiteLLM APIs (e.g., weight, control_plane_step, reasoning_content)
         messages_payload: List[Dict[str, Any]] = [
-            message.model_dump() if hasattr(message, "model_dump") else message  # type: ignore[misc]
+            message.dump_mdoel_for_chat_completion_request()
+            if hasattr(message, "dump_mdoel_for_chat_completion_request")
+            else (message.model_dump() if hasattr(message, "model_dump") else message)  # type: ignore[misc]
             for message in messages
         ]
         # Normalize tool definitions into OpenAI-compatible dicts

eval_protocol/pytest/default_single_turn_rollout_process.py

@@ -48,7 +48,9 @@ async def process_row(row: EvaluationRow) -> EvaluationRow:
                 while messages_for_request and messages_for_request[-1].role == "assistant":
                     messages_for_request.pop()
 
-            messages_payload = [message.model_dump() for message in messages_for_request]
+            # Filter out fields that are not supported by OpenAI/LiteLLM APIs (e.g., weight, control_plane_step, reasoning_content)
+            # Use the Message class method that excludes unsupported fields
+            messages_payload = [message.dump_mdoel_for_chat_completion_request() for message in messages_for_request]
 
             request_params = {"messages": messages_payload, **config.completion_params}
             # Ensure caching is disabled only for this request (review feedback)

eval_protocol/pytest/tracing_utils.py

@@ -101,11 +101,14 @@ def build_init_request(
     completion_params_base_url: Optional[str] = completion_params_dict.get("base_url")
 
     # Strip non-OpenAI fields from messages
-    allowed_message_fields = {"role", "content", "tool_calls", "tool_call_id", "name"}
+    # Use dump_mdoel_for_chat_completion_request() to automatically exclude unsupported fields (weight, control_plane_step, reasoning_content)
     clean_messages = []
     for m in row.messages:
         md: Dict[str, Any]
-        if hasattr(m, "model_dump"):
+        if hasattr(m, "dump_mdoel_for_chat_completion_request"):
+            # Use the Message method that automatically filters unsupported fields
+            md = m.dump_mdoel_for_chat_completion_request()
+        elif hasattr(m, "model_dump"):
             md = m.model_dump()
         elif isinstance(m, dict):
             md = m
@@ -118,6 +121,8 @@ def build_init_request(
                 "tool_call_id": getattr(m, "tool_call_id", None),
                 "name": getattr(m, "name", None),
             }
+        # Additional filtering to ensure only allowed fields are kept (already handled by dump_mdoel_for_chat_completion_request for Message objects)
+        allowed_message_fields = {"role", "content", "tool_calls", "tool_call_id", "name"}
         clean_messages.append({k: v for k, v in md.items() if k in allowed_message_fields and v is not None})
 
     # Build final model base URL with tracing metadata

tests/test_message_field_filtering.py

@@ -0,0 +1,64 @@
+"""
+Test to verify that message fields are properly filtered before sending to API.
+
+This test verifies that unsupported fields like 'weight', 'control_plane_step',
+and 'reasoning_content' are excluded from messages when preparing API requests.
+"""
+
+from eval_protocol.models import Message
+
+
+def test_dump_model_excludes_unsupported_fields():
+    """Test that dump_mdoel_for_chat_completion_request excludes unsupported fields."""
+    # Create a message with all possible fields including unsupported ones
+    message = Message(
+        role="user",
+        content="Hello",
+        weight=0,
+        control_plane_step={"step": 1},
+        reasoning_content="Some reasoning",
+        name="test_user",
+    )
+
+    # Get the filtered dictionary
+    filtered = message.dump_mdoel_for_chat_completion_request()
+
+    # Verify unsupported fields are excluded
+    assert "weight" not in filtered, "weight field should be excluded"
+    assert "control_plane_step" not in filtered, "control_plane_step field should be excluded"
+    assert "reasoning_content" not in filtered, "reasoning_content field should be excluded"
+
+    # Verify supported fields are included
+    assert "role" in filtered, "role field should be included"
+    assert "content" in filtered, "content field should be included"
+    assert filtered["role"] == "user"
+    assert filtered["content"] == "Hello"
+
+    # Verify name is included (it's a supported field for tool calls)
+    assert "name" in filtered
+    assert filtered["name"] == "test_user"
+
+
+def test_dump_model_with_only_supported_fields():
+    """Test that supported fields are preserved."""
+    message = Message(
+        role="assistant",
+        content="I can help you",
+        tool_calls=None,
+        tool_call_id=None,
+    )
+
+    filtered = message.dump_mdoel_for_chat_completion_request()
+
+    # Should only contain supported fields
+    assert filtered["role"] == "assistant"
+    assert filtered["content"] == "I can help you"
+
+    # Should not contain unsupported fields even if None
+    assert "weight" not in filtered
+
+
+if __name__ == "__main__":
+    import pytest
+
+    pytest.main([__file__, "-v"])