revert

Author: Dylan Huang

tests/chinook/test_pydantic_chinook.py

@@ -82,15 +82,6 @@ class Response(BaseModel):
     return row
 
 
-"""
-invocation ids:
-
- - lead-low-war-kind-business (worked)
- - miss-wonder-early-friend-side (failed)
- - 
-"""
-
-
 @pytest.mark.asyncio
 @evaluation_test(
     input_rows=collect_dataset(),
@@ -103,18 +94,9 @@ class Response(BaseModel):
                 }
             }
         },
-        {
-            "model": {
-                "orchestrator_agent_model": {
-                    "model": "accounts/fireworks/models/deepseek-v3p1",
-                    "provider": "fireworks",
-                }
-            }
-        },
     ],
     rollout_processor=PydanticAgentRolloutProcessor(),
     rollout_processor_kwargs={"agent": setup_agent},
-    num_runs=3,
     mode="pointwise",
 )
 async def test_complex_queries(row: EvaluationRow) -> EvaluationRow:
@@ -154,7 +136,7 @@ class Response(BaseModel):
             system_prompt=LLM_JUDGE_PROMPT,
             output_type=Response,
             model=model,
-            result_retries=5,
+            result_retries=3,
         )
         result = await comparison_agent.run(
             f"Expected answer: {row.ground_truth}\nResponse: {last_assistant_message.content}"