update test

Author: xzrderek

.github/workflows/rollout.yml

@@ -13,14 +13,6 @@ on:
         description: 'JSON serialized metadata object'
         required: true
         type: string
-      messages:
-        description: 'JSON serialized messages array'
-        required: true
-        type: string
-      tools:
-        description: 'JSON serialized tools array'
-        required: false
-        type: string
       model_base_url:
         description: 'Base URL for the model API'
         required: true
@@ -51,6 +43,4 @@ jobs:
           python tests/github_actions/rollout_worker.py \
             --model "${{ inputs.model }}" \
             --metadata '${{ inputs.metadata }}' \
-            --messages '${{ inputs.messages }}' \
-            --tools '${{ inputs.tools }}' \
             --model-base-url "${{ inputs.model_base_url }}"

eval_protocol/models.py

@@ -598,7 +598,9 @@ class EvaluationRow(BaseModel):
     model_config = ConfigDict(extra="allow")
 
     # Core OpenAI ChatCompletion compatible conversation data
-    messages: List[Message] = Field(description="List of messages in the conversation. Also known as a trajectory.")
+    messages: List[Message] = Field(
+        default_factory=list, description="List of messages in the conversation. Also known as a trajectory."
+    )
 
     # Tool and function call information
     tools: Optional[List[Dict[str, Any]]] = Field(

eval_protocol/pytest/github_action_rollout_processor.py

@@ -162,8 +162,6 @@ def _dispatch_workflow():
                     "inputs": {
                         "model": init_request.model,
                         "metadata": init_request.metadata.model_dump_json(),
-                        "messages": json.dumps(init_request.messages),
-                        "tools": json.dumps(init_request.tools),
                         "model_base_url": init_request.model_base_url,
                     },
                 }

tests/github_actions/rollout_worker.py

@@ -11,7 +11,6 @@
 import os
 
 from openai import OpenAI
-from eval_protocol.types.remote_rollout_processor import InitRequest
 
 
 def main():
@@ -20,44 +19,42 @@ def main():
     # Required arguments from workflow inputs
     parser.add_argument("--model", required=True, help="Model to use")
     parser.add_argument("--metadata", required=True, help="JSON serialized metadata object")
-    parser.add_argument("--messages", required=True, help="JSON serialized messages array")
-    parser.add_argument("--tools", required=False, help="JSON serialized tools array")
     parser.add_argument("--model-base-url", required=True, help="Base URL for the model API")
 
     args = parser.parse_args()
 
-    # Parse the JSON inputs
+    # Parse the metadata
     try:
         metadata = json.loads(args.metadata)
-        messages = json.loads(args.messages)
-        tools = json.loads(args.tools) if args.tools else None
     except Exception as e:
-        print(f"❌ Failed to parse JSON inputs: {e}")
+        print(f"❌ Failed to parse metadata: {e}")
         exit(1)
 
     rollout_id = metadata["rollout_id"]
+    row_id = metadata["row_id"]
+
     print(f"🚀 Starting rollout {rollout_id}")
     print(f"   Model: {args.model}")
-    print(f"   Messages: {len(messages)} messages")
+    print(f"   Row ID: {row_id}")
+
+    dataset = [  # In this example, worker has access to the dataset and we use index to associate rows.
+        "What is the capital of France?",
+        "What is the capital of Germany?",
+        "What is the capital of Italy?",
+    ]
 
-    # Perform the rollout
-    conversation = messages.copy()
+    user_content = dataset[int(row_id)]
+    messages = [{"role": "user", "content": user_content}]
+
+    print(f"   Messages: {len(messages)} messages")
 
     try:
         completion_kwargs = {"model": args.model, "messages": messages}
-        if tools:
-            completion_kwargs["tools"] = tools
 
         client = OpenAI(base_url=args.model_base_url, api_key=os.environ.get("FIREWORKS_API_KEY"))
 
         print("📡 Calling OpenAI completion...")
         completion = client.chat.completions.create(**completion_kwargs)
-        print("✅ Received response")
-
-        # Add assistant response to conversation
-        if completion.choices and completion.choices[0].message:
-            assistant_message = completion.choices[0].message.model_dump()
-            conversation.append(assistant_message)
 
         print(f"✅ Rollout {rollout_id} completed successfully")
 

tests/github_actions/test_github_actions_rollout.py

@@ -18,9 +18,12 @@
 import pytest
 
 from eval_protocol.data_loader.dynamic_data_loader import DynamicDataLoader
-from eval_protocol.models import EvaluationRow, Message
+from eval_protocol.models import EvaluationRow, InputMetadata
 from eval_protocol.pytest import evaluation_test
 from eval_protocol.pytest.github_action_rollout_processor import GithubActionRolloutProcessor
+from eval_protocol.types.remote_rollout_processor import DataLoaderConfig
+from eval_protocol.adapters.fireworks_tracing import FireworksTracingAdapter
+from eval_protocol.quickstart.utils import filter_longest_conversation
 
 ROLLOUT_IDS = set()
 
@@ -35,9 +38,26 @@ def check_rollout_coverage():
     assert len(ROLLOUT_IDS) == 3, f"Expected to see 3 rollout_ids, but only saw {ROLLOUT_IDS}"
 
 
+def fetch_fireworks_traces(config: DataLoaderConfig) -> List[EvaluationRow]:
+    global ROLLOUT_IDS  # Track all rollout_ids we've seen
+    ROLLOUT_IDS.add(config.rollout_id)
+
+    base_url = config.model_base_url or "https://tracing.fireworks.ai"
+    adapter = FireworksTracingAdapter(base_url=base_url)
+    return adapter.get_evaluation_rows(tags=[f"rollout_id:{config.rollout_id}"], max_retries=5)
+
+
+def fireworks_output_data_loader(config: DataLoaderConfig) -> DynamicDataLoader:
+    return DynamicDataLoader(
+        generators=[lambda: fetch_fireworks_traces(config)], preprocess_fn=filter_longest_conversation
+    )
+
+
 def rows() -> List[EvaluationRow]:
-    row = EvaluationRow(messages=[Message(role="user", content="What is the capital of France?")])
-    return [row, row, row]
+    return [
+        EvaluationRow(input_metadata=InputMetadata(row_id=str(i)))
+        for i in range(3)  # In this example we use index to associate rows.
+    ]
 
 
 @pytest.mark.skipif(os.environ.get("CI") == "true", reason="Only run this test locally (skipped in CI)")
@@ -52,6 +72,7 @@ def rows() -> List[EvaluationRow]:
         workflow_id="rollout.yml",  # or you can use numeric ID like "12345678"
         ref=os.getenv("GITHUB_REF", "main"),
         timeout_seconds=300,
+        output_data_loader=fireworks_output_data_loader,
     ),
 )
 async def test_github_actions_rollout_direct_artifacts(row: EvaluationRow) -> EvaluationRow: