Add support for uploading secrets from .env file in CLI

- Introduced `--env-file` argument in the CLI for specifying the path to the .env file containing secrets.
- Refactored the upload process to include a new function `upload_secrets_to_fireworks` that handles the uploading of secrets.
- Updated the `upload_command` to optionally skip secret uploads if already handled.
- Enhanced the evaluator upload process to ensure secrets are uploaded before ensuring evaluator existence.

Author: Dylan Huang

eval_protocol/cli.py

@@ -156,6 +156,11 @@ def _configure_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParse
         metavar="",
         help="Extra flags to pass to 'docker run' when validating evaluator (quoted string, e.g. \"--env-file .env --memory=8g\")",
     )
+    rft_parser.add_argument(
+        "--env-file",
+        metavar="",
+        help="Path to .env file containing secrets to upload to Fireworks (default: .env in project root)",
+    )
 
     # The flags below are Eval Protocol CLI workflow controls (not part of the Fireworks SDK `create()` signature),
     # so they can’t be auto-generated via signature introspection and must be maintained here.

eval_protocol/cli_commands/create_rft.py

@@ -19,7 +19,7 @@
     materialize_dataset_via_builder,
 )
 from ..models import EvaluationRow
-from .upload import upload_command
+from .upload import upload_command, upload_secrets_to_fireworks
 from .utils import (
     _build_entry_point,
     _build_trimmed_dataset_id,
@@ -569,6 +569,8 @@ def _upload_and_ensure_evaluator(
     api_key: str,
     api_base: str,
     force: bool,
+    env_file: Optional[str] = None,
+    non_interactive: bool = False,
 ) -> bool:
     """Ensure the evaluator exists and is ACTIVE, uploading it if needed."""
     # Optional short-circuit: if evaluator already exists and not forcing, skip upload path
@@ -600,6 +602,13 @@ def _upload_and_ensure_evaluator(
         except requests.exceptions.RequestException:
             pass
 
+    # Upload secrets before ensuring evaluator exists
+    upload_secrets_to_fireworks(
+        root=project_root,
+        env_file=env_file,
+        non_interactive=non_interactive,
+    )
+
     # Ensure evaluator exists by invoking the upload flow programmatically
     try:
         tests = _discover_tests(project_root)
@@ -625,13 +634,13 @@ def _upload_and_ensure_evaluator(
             description=None,
             force=force,  # Pass through the --force flag
             yes=True,
-            env_file=None,  # Add the new env_file parameter
+            env_file=None,  # Secrets already handled above via upload_secrets_to_fireworks
         )
 
         if force:
             print(f"🔄 Force flag enabled - will overwrite existing evaluator '{evaluator_id}'")
 
-        rc = upload_command(upload_args)
+        rc = upload_command(upload_args, skip_secrets=True)  # Secrets already handled above
         if rc == 0:
             print(f"✓ Uploaded/ensured evaluator: {evaluator_id}")
 
@@ -811,13 +820,16 @@ def create_rft_command(args) -> int:
         return 1
 
     # 5) Ensure evaluator exists and is ACTIVE (upload + poll if needed)
+    env_file: Optional[str] = getattr(args, "env_file", None)
     if not _upload_and_ensure_evaluator(
         project_root=project_root,
         evaluator_id=evaluator_id,
         evaluator_resource_name=evaluator_resource_name,
         api_key=api_key,
         api_base=api_base,
         force=force,
+        env_file=env_file,
+        non_interactive=non_interactive,
     ):
         return 1
 

eval_protocol/cli_commands/upload.py

@@ -270,29 +270,24 @@ def _prompt_select_secrets_fallback(
         return {}
 
 
-def upload_command(args: argparse.Namespace) -> int:
-    root = os.path.abspath(getattr(args, "path", "."))
-    entries_arg = getattr(args, "entry", None)
-    non_interactive: bool = bool(getattr(args, "yes", False))
-    if entries_arg:
-        entries = [e.strip() for e in re.split(r"[,\s]+", entries_arg) if e.strip()]
-        selected_specs: list[tuple[str, str]] = []
-        for e in entries:
-            qualname, resolved_path = _resolve_entry_to_qual_and_source(e, root)
-            selected_specs.append((qualname, resolved_path))
-    else:
-        selected_tests: list[DiscoveredTest] | None = _discover_and_select_tests(root, non_interactive=non_interactive)
-        if not selected_tests:
-            return 1
-        selected_specs = [(t.qualname, t.file_path) for t in selected_tests]
+def upload_secrets_to_fireworks(
+    root: str,
+    env_file: str | None = None,
+    non_interactive: bool = False,
+) -> None:
+    """
+    Upload secrets from .env file and environment to Fireworks.
 
-    base_id = getattr(args, "id", None)
-    display_name = getattr(args, "display_name", None)
-    description = getattr(args, "description", None)
-    force = bool(getattr(args, "force", False))
-    env_file = getattr(args, "env_file", None)
+    This function:
+    1. Loads secrets from the specified .env file (or default .env in root)
+    2. Prompts user to select which secrets to upload (unless non_interactive)
+    3. Creates/updates the selected secrets on Fireworks
 
-    # Load secrets from .env file and ensure they're available on Fireworks
+    Args:
+        root: The project root directory
+        env_file: Optional path to a .env file. If None, uses {root}/.env
+        non_interactive: If True, skip interactive prompts and upload all secrets
+    """
     try:
         fw_account_id = _ensure_account_id()
 
@@ -350,6 +345,37 @@ def upload_command(args: argparse.Namespace) -> int:
     except Exception as e:
         print(f"Warning: Skipped Fireworks secret registration due to error: {e}")
 
+
+def upload_command(args: argparse.Namespace, skip_secrets: bool = False) -> int:
+    root = os.path.abspath(getattr(args, "path", "."))
+    entries_arg = getattr(args, "entry", None)
+    non_interactive: bool = bool(getattr(args, "yes", False))
+    if entries_arg:
+        entries = [e.strip() for e in re.split(r"[,\s]+", entries_arg) if e.strip()]
+        selected_specs: list[tuple[str, str]] = []
+        for e in entries:
+            qualname, resolved_path = _resolve_entry_to_qual_and_source(e, root)
+            selected_specs.append((qualname, resolved_path))
+    else:
+        selected_tests: list[DiscoveredTest] | None = _discover_and_select_tests(root, non_interactive=non_interactive)
+        if not selected_tests:
+            return 1
+        selected_specs = [(t.qualname, t.file_path) for t in selected_tests]
+
+    base_id = getattr(args, "id", None)
+    display_name = getattr(args, "display_name", None)
+    description = getattr(args, "description", None)
+    force = bool(getattr(args, "force", False))
+    env_file = getattr(args, "env_file", None)
+
+    # Upload secrets from .env file and environment to Fireworks
+    if not skip_secrets:
+        upload_secrets_to_fireworks(
+            root=root,
+            env_file=env_file,
+            non_interactive=non_interactive,
+        )
+
     exit_code = 0
     for i, (qualname, source_file_path) in enumerate(selected_specs):
         # Generate a short default ID from just the test function name