update

Author: xzrderek

eval_protocol/pytest/default_agent_rollout_processor.py

@@ -254,6 +254,7 @@ async def process_row(row: EvaluationRow) -> EvaluationRow:
 
             # Normalize Fireworks model names for LiteLLM routing
             completion_params = normalize_fireworks_model_for_litellm(row.input_metadata.completion_params) or {}
+            row.input_metadata.completion_params = completion_params
             agent = Agent(
                 model=completion_params["model"],
                 row=row,

eval_protocol/pytest/default_klavis_sandbox_rollout_processor.py

@@ -110,6 +110,7 @@ async def process_row(row: EvaluationRow) -> EvaluationRow:
                 )
                 # Normalize Fireworks model names for LiteLLM routing
                 completion_params = normalize_fireworks_model_for_litellm(row.input_metadata.completion_params) or {}
+                row.input_metadata.completion_params = completion_params
                 agent = Agent(
                     model=completion_params["model"],
                     row=row,

eval_protocol/pytest/default_mcp_gym_rollout_processor.py

@@ -283,6 +283,9 @@ def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) ->
 
         # Normalize Fireworks model names for LiteLLM routing
         completion_params = normalize_fireworks_model_for_litellm(config.completion_params) or {}
+        # Update all rows with normalized completion_params
+        for row in rows:
+            row.input_metadata.completion_params = completion_params
         model_id = str(completion_params.get("model") or "gpt-4o-mini")
         temperature = completion_params.get("temperature", 0.0)
         max_tokens = completion_params.get("max_tokens", 4096)

eval_protocol/pytest/default_single_turn_rollout_process.py

@@ -80,6 +80,7 @@ async def process_row(row: EvaluationRow) -> EvaluationRow:
 
             # Normalize Fireworks model names for LiteLLM routing
             completion_params = normalize_fireworks_model_for_litellm(config.completion_params) or {}
+            row.input_metadata.completion_params = completion_params
             request_params = {"messages": messages_payload, **completion_params}
             # Ensure caching is disabled only for this request (review feedback)
             request_params["cache"] = {"no-cache": True}

eval_protocol/pytest/github_action_rollout_processor.py

@@ -11,6 +11,7 @@
 from .rollout_processor import RolloutProcessor
 from .types import RolloutProcessorConfig
 from .tracing_utils import default_fireworks_output_data_loader, build_init_request, update_row_with_remote_trace
+from .utils import normalize_fireworks_model_for_litellm
 
 
 class GithubActionRolloutProcessor(RolloutProcessor):
@@ -80,8 +81,16 @@ async def _process_row(row: EvaluationRow) -> EvaluationRow:
             if row.input_metadata.row_id is None:
                 raise ValueError("Row ID is required in GithubActionRolloutProcessor")
 
+            # Normalize Fireworks model names for LiteLLM routing
+            config.completion_params = (
+                normalize_fireworks_model_for_litellm(config.completion_params) or config.completion_params
+            )
+
             init_request = build_init_request(row, config, self.model_base_url)
 
+            # Update row with normalized completion_params for downstream access
+            row.input_metadata.completion_params = init_request.completion_params
+
             def _dispatch_workflow():
                 url = f"https://api.github.com/repos/{self.owner}/{self.repo}/actions/workflows/{self.workflow_id}/dispatches"
 

eval_protocol/pytest/openenv_rollout_processor.py

@@ -180,6 +180,7 @@ async def process_row(row: EvaluationRow) -> EvaluationRow:
             try:
                 # Normalize Fireworks model names for LiteLLM routing
                 completion_params = normalize_fireworks_model_for_litellm(config.completion_params) or {}
+                row.input_metadata.completion_params = completion_params
                 # Get model config
                 raw_model = completion_params.get("model", "gpt-4o-mini")
                 model = raw_model

eval_protocol/pytest/remote_rollout_processor.py

@@ -11,6 +11,7 @@
 from .rollout_processor import RolloutProcessor
 from .types import RolloutProcessorConfig
 from .tracing_utils import default_fireworks_output_data_loader, build_init_request, update_row_with_remote_trace
+from .utils import normalize_fireworks_model_for_litellm
 import logging
 
 import os
@@ -87,8 +88,16 @@ async def _process_row(row: EvaluationRow) -> EvaluationRow:
             if row.input_metadata.row_id is None:
                 raise ValueError("Row ID is required in RemoteRolloutProcessor")
 
+            # Normalize Fireworks model names for LiteLLM routing
+            config.completion_params = (
+                normalize_fireworks_model_for_litellm(config.completion_params) or config.completion_params
+            )
+
             init_payload = build_init_request(row, config, model_base_url)
 
+            # Update row with normalized completion_params for downstream access
+            row.input_metadata.completion_params = init_payload.completion_params
+
             # Fire-and-poll
             init_url = f"{remote_base_url}/init"
 

eval_protocol/pytest/tracing_utils.py

@@ -12,7 +12,6 @@
 from eval_protocol.utils.evaluation_row_utils import filter_longest_conversation
 from eval_protocol.types.remote_rollout_processor import DataLoaderConfig, RolloutMetadata, InitRequest
 from eval_protocol.pytest.types import RolloutProcessorConfig
-from eval_protocol.pytest.utils import normalize_fireworks_model_for_litellm
 
 
 def default_fireworks_output_data_loader(config: DataLoaderConfig) -> DynamicDataLoader:
@@ -96,12 +95,6 @@ def build_init_request(
         if isinstance(row_cp, dict):
             completion_params_dict.update(row_cp)
 
-    # Normalize Fireworks model names for LiteLLM routing
-    completion_params_dict = normalize_fireworks_model_for_litellm(completion_params_dict) or {}
-
-    # Update row's completion_params with normalized value
-    row.input_metadata.completion_params = completion_params_dict
-
     # Validate model is present
     if not completion_params_dict.get("model"):
         raise ValueError("Model must be provided in completion_params")

eval_protocol/pytest/utils.py

@@ -1,7 +1,6 @@
 """Utility functions for model name handling."""
 
 import re
-from typing import Optional
 
 from eval_protocol.models import CompletionParams