try

Author: xzrderek

eval_protocol/pytest/default_single_turn_rollout_process.py

@@ -5,20 +5,8 @@
 from typing import List
 
 from litellm import acompletion
-import litellm
 from typing import Dict
 
-# Fix LiteLLM event loop binding issues by disabling the logging worker
-try:
-    # Disable LiteLLM's async logging worker to prevent event loop binding issues
-    import litellm.litellm_core_utils.logging_worker
-
-    # Monkey patch to disable the worker entirely
-    original_start_worker = litellm.litellm_core_utils.logging_worker.start_worker
-    litellm.litellm_core_utils.logging_worker.start_worker = lambda *args, **kwargs: None
-except Exception:
-    pass  # Best effort
-
 from eval_protocol.dataset_logger import default_logger
 from eval_protocol.models import EvaluationRow, Message
 from openai.types import CompletionUsage
@@ -33,7 +21,6 @@ class SingleTurnRolloutProcessor(RolloutProcessor):
 
     def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) -> List[asyncio.Task[EvaluationRow]]:
         """Generate single turn rollout tasks and return them for external handling."""
-
         # Do not modify global LiteLLM cache. Disable caching per-request instead.
 
         async def process_row(row: EvaluationRow) -> EvaluationRow:

eval_protocol/quickstart/llm_judge.py

@@ -38,7 +38,7 @@
         # {
         #     "model": "fireworks_ai/accounts/fireworks/models/qwen3-235b-a22b-instruct-2507",
         # },
-        {"model": "gpt-4.1"},
+        # {"model": "gpt-4.1"},
         {
             "max_tokens": 131000,
             "extra_body": {"reasoning_effort": "medium"},
@@ -49,6 +49,11 @@
             "extra_body": {"reasoning_effort": "low"},
             "model": "fireworks_ai/accounts/fireworks/models/gpt-oss-20b",
         },
+        {
+            "max_tokens": 131000,
+            "extra_body": {"reasoning_effort": "low"},
+            "model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b",
+        },
     ],
     rollout_processor=SingleTurnRolloutProcessor(),
     # preprocess_fn=split_multi_turn_rows,