linterrors

Author: Shrey Modi

eval_protocol/pytest/github_action_rollout_processor.py

@@ -97,6 +97,7 @@ def _dispatch_workflow():
                         "model": model,
                         "metadata": init_request.metadata.model_dump_json(),
                         "model_base_url": init_request.model_base_url,
+                        "completion_params": json.dumps(init_request.completion_params),
                     },
                 }
                 r = requests.post(url, json=payload, headers=self._headers(), timeout=30)

eval_protocol/pytest/tracing_utils.py

@@ -82,21 +82,21 @@ def build_init_request(
 
     # Build completion_params from row and config
     completion_params_dict: Dict[str, Any] = {}
-    
+
     # Start with config-level completion_params
     if config.completion_params and isinstance(config.completion_params, dict):
         completion_params_dict.update(config.completion_params)
-    
+
     # Override with row-specific completion_params
     if row.input_metadata and row.input_metadata.completion_params:
         row_cp = row.input_metadata.completion_params
         if isinstance(row_cp, dict):
             completion_params_dict.update(row_cp)
-    
+
     # Validate model is present
     if not completion_params_dict.get("model"):
         raise ValueError("Model must be provided in completion_params")
-    
+
     # Extract base_url from completion_params
     completion_params_base_url: Optional[str] = completion_params_dict.get("base_url")
 

eval_protocol/types/remote_rollout_processor.py

@@ -46,7 +46,7 @@ class InitRequest(BaseModel):
 
     completion_params: Dict[str, Any] = Field(
         default_factory=dict,
-        description="Completion parameters including model and optional model_kwargs, temperature, etc."
+        description="Completion parameters including model and optional model_kwargs, temperature, etc.",
     )
     elastic_search_config: Optional[ElasticsearchConfig] = None
     messages: Optional[List[Message]] = None

tests/github_actions/rollout_worker.py

@@ -58,13 +58,20 @@ def main():
 
     try:
         completion_kwargs = {"model": args.model, "messages": messages}
-
-        if completion_params.get("model_kwargs"):
-            completion_kwargs.update(completion_params["model_kwargs"])
+        # Parse and apply completion_params if provided
+        if args.completion_params:
+            try:
+                cp = json.loads(args.completion_params)
+                if cp.get("model_kwargs"):
+                    completion_kwargs.update(cp["model_kwargs"])
+                    print(f"   Applied model_kwargs: {cp.get('model_kwargs')}")
+            except Exception as e:
+                print(f"⚠️  Failed to parse completion_params: {e}")
 
         client = OpenAI(base_url=args.model_base_url, api_key=os.environ.get("FIREWORKS_API_KEY"))
 
         print("📡 Calling OpenAI completion...")
+        print(f"   Completion kwargs: {completion_kwargs}")
         completion = client.chat.completions.create(**completion_kwargs)
 
         print(f"✅ Rollout {rollout_id} completed successfully")

tests/github_actions/test_github_actions_rollout.py

@@ -54,7 +54,10 @@ def rows() -> List[EvaluationRow]:
 
 
 @pytest.mark.skipif(os.environ.get("CI") == "true", reason="Only run this test locally (skipped in CI)")
-@pytest.mark.parametrize("completion_params", [{"model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b"}])
+@pytest.mark.parametrize(
+    "completion_params",
+    [{"model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b", "model_kwargs": {"temperature": 0.5}}],
+)
 @evaluation_test(
     data_loaders=DynamicDataLoader(
         generators=[rows],

tests/remote_server/remote_server.py

@@ -35,7 +35,7 @@ def _worker():
         try:
             if not req.messages:
                 raise ValueError("messages is required")
-            
+
             model = req.completion_params.get("model")
             if not model:
                 raise ValueError("model is required in completion_params")
@@ -44,7 +44,7 @@ def _worker():
                 "model": model,
                 "messages": req.messages,
             }
-            
+
             # Apply model_kwargs if present
             if req.completion_params.get("model_kwargs"):
                 model_kwargs = req.completion_params["model_kwargs"]
@@ -55,7 +55,7 @@ def _worker():
                 completion_kwargs["tools"] = req.tools
 
             logger.info(f"Final completion_kwargs: {completion_kwargs}")
-            
+
             client = OpenAI(base_url=req.model_base_url, api_key=os.environ.get("FIREWORKS_API_KEY"))
 
             logger.info(f"Sending completion request to model {model}")

tests/remote_server/remote_server_multi_turn.py

@@ -31,7 +31,7 @@ def _worker():
         try:
             if not req.messages:
                 raise ValueError("messages is required")
-            
+
             model = req.completion_params.get("model")
             if not model:
                 raise ValueError("model is required in completion_params")
@@ -52,13 +52,12 @@ def _worker():
                 "What else can you share about this topic?",
             ]
 
-            
             # First completion (turns 1-2: initial user message + assistant response)
             logger.info(f"Turn 1-2: Sending initial completion request to model {model}")
             completion = client.chat.completions.create(
                 model=model,
                 messages=conversation_history,  # type: ignore,
-                **completion_kwargs
+                **completion_kwargs,
             )
             assistant_message = completion.choices[0].message
             assistant_content = assistant_message.content or ""

tests/remote_server/test_remote_fireworks.py

@@ -58,9 +58,10 @@ def rows() -> List[EvaluationRow]:
 
 
 @pytest.mark.skipif(os.environ.get("CI") == "true", reason="Only run this test locally (skipped in CI)")
-@pytest.mark.parametrize("completion_params", [{"model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b", 
-"model_kwargs": {"temperature": 0.5}
-}])
+@pytest.mark.parametrize(
+    "completion_params",
+    [{"model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b", "model_kwargs": {"temperature": 0.5}}],
+)
 @evaluation_test(
     data_loaders=DynamicDataLoader(
         generators=[rows],
@@ -84,6 +85,8 @@ async def test_remote_rollout_and_fetch_fireworks(row: EvaluationRow) -> Evaluat
     assert row.execution_metadata.rollout_id in ROLLOUT_IDS, (
         f"Row rollout_id {row.execution_metadata.rollout_id} should be in tracked rollout_ids: {ROLLOUT_IDS}"
     )
-    assert row.input_metadata.completion_params["model_kwargs"] == {"temperature": 0.5}, "Row should have correct model_kwargs"
-    
+    assert row.input_metadata.completion_params["model_kwargs"] == {"temperature": 0.5}, (
+        "Row should have correct model_kwargs"
+    )
+
     return row