fixing mcp gym rollout processor

Author: xzrderek

eval_protocol/mcp_servers/frozen_lake/server.py

@@ -14,8 +14,13 @@
 import sys
 from pathlib import Path
 
-# Add root directory to path so we can import eval_protocol
-sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+# Add current directory first for local imports (frozen_lake_mcp)
+sys.path.insert(0, str(Path(__file__).parent))
+
+# Add eval_protocol parent to path, but use append to avoid priority conflicts
+parent_dir = str(Path(__file__).parent.parent.parent)
+if parent_dir not in sys.path:
+    sys.path.append(parent_dir)
 
 from frozen_lake_mcp import FrozenLakeMcp
 

eval_protocol/pytest/default_mcp_gym_rollout_processor.py

@@ -25,7 +25,7 @@ class MCPServerManager:
     def __init__(self, server_script: str, port: int = 8000, **kwargs):
         self.server_script = server_script
         self.port = port
-        self.domain = str(kwargs.get("domain", "airline"))
+        self.domain = kwargs.get("domain", None)
         self.process: Optional[subprocess.Popen] = None
         self.base_dir = Path(".").resolve()
         self._log_file = None
@@ -59,11 +59,14 @@ def start(self) -> None:
         env = os.environ.copy()
         env["PORT"] = str(self.port)
 
-        # Start server process (no domain argument needed for tau2_mcp server)
-        cmd = ["python", self.server_script, "--port", str(self.port), "--domain", self.domain]
+        # Build command, add --domain only if provided (e.g. tau2 needs it, frozen_lake doesn't)
+        cmd = ["python", self.server_script, "--port", str(self.port)]
+        if self.domain:
+            cmd.extend(["--domain", self.domain])
 
         # Setup log file with cleanup
-        log_file_path = os.path.join(self.base_dir, f"server_output_{self.domain}_{self.port}.log")
+        domain_part = self.domain if self.domain else "server"
+        log_file_path = os.path.join(self.base_dir, f"server_output_{domain_part}_{self.port}.log")
         if os.path.exists(log_file_path):
             os.remove(log_file_path)
 

tests/pytest/test_tau_bench_airline.py

@@ -73,6 +73,7 @@ def tau_bench_airline_to_evaluation_row(data: List[Dict[str, Any]]) -> List[Eval
         }
     ],
     rollout_processor=MCPGymRolloutProcessor(),
+    rollout_processor_kwargs={"domain": "airline"},
     passed_threshold={"success": 0.4, "standard_error": 0.02},
     num_runs=8,
     mode="pointwise",