Merge branch 'main' into rollout-logs-integration-part-2

Author: Dylan Huang

eval_protocol/__init__.py

@@ -23,15 +23,6 @@
     test_mcp,
 )
 from .data_loader import DynamicDataLoader, InlineDataLoader
-
-# Try to import FireworksPolicy if available
-try:
-    from .mcp_env import FireworksPolicy
-
-    _FIREWORKS_AVAILABLE = True
-except (ImportError, AttributeError):
-    _FIREWORKS_AVAILABLE = False
-# Import submodules to make them available via eval_protocol.rewards, etc.
 from . import mcp, rewards
 from .models import EvaluateResult, Message, MetricResult, EvaluationRow, InputMetadata, Status
 from .playback_policy import PlaybackPolicyBase
@@ -42,6 +33,13 @@
 from .pytest import evaluation_test, SingleTurnRolloutProcessor, RemoteRolloutProcessor
 from .pytest.parameterize import DefaultParameterIdGenerator
 
+from .types.remote_rollout_processor import (
+    InitRequest,
+    RolloutMetadata,
+    StatusResponse,
+    create_langfuse_config_tags,
+)
+
 try:
     from .adapters import OpenAIResponsesAdapter
 except ImportError:
@@ -62,14 +60,6 @@
 except ImportError:
     LangSmithAdapter = None
 
-# Remote server types
-from .types.remote_rollout_processor import (
-    InitRequest,
-    RolloutMetadata,
-    StatusResponse,
-    create_langfuse_config_tags,
-)
-
 warnings.filterwarnings("default", category=DeprecationWarning, module="eval_protocol")
 
 __all__ = [

eval_protocol/adapters/huggingface.py

@@ -13,12 +13,9 @@
 logger = logging.getLogger(__name__)
 
 try:
-    from datasets import Dataset, DatasetDict, load_dataset
-
-    DATASETS_AVAILABLE = True
+    from datasets import Dataset, DatasetDict, load_dataset  # pyright: ignore[reportAttributeAccessIssue]
 except ImportError:
-    DATASETS_AVAILABLE = False
-    logger.warning("HuggingFace datasets not installed. Install with: pip install 'eval-protocol[huggingface]'")
+    raise ImportError("HuggingFace datasets not installed. Install with: pip install 'eval-protocol[huggingface]'")
 
 # Type alias for transformation function
 TransformFunction = Callable[[Dict[str, Any]], Dict[str, Any]]
@@ -80,11 +77,6 @@ def __init__(
             revision: Optional dataset revision/commit hash
             **load_dataset_kwargs: Additional arguments to pass to load_dataset
         """
-        if not DATASETS_AVAILABLE:
-            raise ImportError(
-                "HuggingFace datasets not installed. Install with: pip install 'eval-protocol[huggingface]'"
-            )
-
         self.dataset_id = dataset_id
         self.transform_fn = transform_fn
         self.config_name = config_name

eval_protocol/cli.py

@@ -28,6 +28,7 @@
 from .cli_commands.logs import logs_command
 from .cli_commands.preview import preview_command
 from .cli_commands.run_eval_cmd import hydra_cli_entry_point
+from .cli_commands.upload import upload_command
 
 
 def parse_args(args=None):
@@ -291,6 +292,44 @@ def parse_args(args=None):
     logs_parser = subparsers.add_parser("logs", help="Serve logs with file watching and real-time updates")
     logs_parser.add_argument("--port", type=int, default=8000, help="Port to bind to (default: 8000)")
 
+    # Upload command
+    upload_parser = subparsers.add_parser(
+        "upload",
+        help="Scan for evaluation tests, select, and upload as Fireworks evaluators",
+    )
+    upload_parser.add_argument(
+        "--path",
+        default=".",
+        help="Path to search for evaluation tests (default: current directory)",
+    )
+    upload_parser.add_argument(
+        "--entry",
+        help="Entrypoint of evaluation test to upload (module:function or path::function). For multiple, separate by commas.",
+    )
+    upload_parser.add_argument(
+        "--id",
+        help="Evaluator ID to use (if multiple selections, a numeric suffix is appended)",
+    )
+    upload_parser.add_argument(
+        "--display-name",
+        help="Display name for evaluator (defaults to ID)",
+    )
+    upload_parser.add_argument(
+        "--description",
+        help="Description for evaluator",
+    )
+    upload_parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Overwrite existing evaluator with the same ID",
+    )
+    upload_parser.add_argument(
+        "--yes",
+        "-y",
+        action="store_true",
+        help="Non-interactive: upload all discovered evaluation tests",
+    )
+
     # Run command (for Hydra-based evaluations)
     # This subparser intentionally defines no arguments itself.
     # All arguments after 'run' will be passed to Hydra by parse_known_args.
@@ -346,6 +385,8 @@ def main():
         return agent_eval_command(args)
     elif args.command == "logs":
         return logs_command(args)
+    elif args.command == "upload":
+        return upload_command(args)
     elif args.command == "run":
         # For the 'run' command, Hydra takes over argument parsing.