check FIREWORKS_API_KEY in upload flow (#255)

* check FIREWORKS_API_KEY in upload flow

* normalize resource id

Author: benjibc

eval_protocol/cli_commands/upload.py

@@ -12,6 +12,8 @@
 from typing import Any, Callable, Iterable, Optional
 
 import pytest
+from eval_protocol.auth import get_fireworks_account_id, get_fireworks_api_key
+from eval_protocol.platform_api import create_or_update_fireworks_secret
 
 from eval_protocol.evaluation import create_evaluation
 
@@ -343,28 +345,45 @@ def _prompt_select_interactive(tests: list[DiscoveredTest]) -> list[DiscoveredTe
             else:
                 return []
 
-        # Create choices with nice formatting
-        choices = []
-        for idx, test in enumerate(tests, 1):
-            choice_text = _format_test_choice(test, idx)
-            choices.append({"name": choice_text, "value": idx - 1, "checked": False})
+        # Enter-only selection UX with optional multi-select via repeat
+        remaining_indices = list(range(len(tests)))
+        selected_indices: list[int] = []
 
         print("\n")
-        print("💡 Tip: Use ↑/↓ arrows to navigate, SPACE to select/deselect, ENTER when done")
-        print("        You can select multiple tests!\n")
-        selected_indices = questionary.checkbox(
-            "Select evaluation tests to upload:",
-            choices=choices,
-            style=custom_style,
-        ).ask()
-
-        if selected_indices is None:  # User pressed Ctrl+C
-            print("\nUpload cancelled.")
-            return []
+        print("Tip: Use ↑/↓ arrows to navigate and press ENTER to select.")
+        print("     After selecting one, you can choose to add more.\n")
+
+        while remaining_indices:
+            # Build choices from remaining
+            choices = []
+            for idx, test_idx in enumerate(remaining_indices, 1):
+                t = tests[test_idx]
+                choice_text = _format_test_choice(t, idx)
+                choices.append({"name": choice_text, "value": test_idx})
+
+            selected = questionary.select(
+                "Select an evaluation test to upload:", choices=choices, style=custom_style
+            ).ask()
+
+            if selected is None:  # Ctrl+C
+                print("\nUpload cancelled.")
+                return []
+
+            if isinstance(selected, int):
+                selected_indices.append(selected)
+                # Remove from remaining
+                if selected in remaining_indices:
+                    remaining_indices.remove(selected)
+
+                # Ask whether to add another (ENTER to finish)
+                add_more = questionary.confirm("Add another?", default=False, style=custom_style).ask()
+                if not add_more:
+                    break
+            else:
+                break
 
         if not selected_indices:
             print("\n⚠️  No tests were selected.")
-            print("   Remember: Use SPACE bar to select tests, then press ENTER to confirm.")
             return []
 
         print(f"\n✓ Selected {len(selected_indices)} test(s)")
@@ -474,6 +493,28 @@ def upload_command(args: argparse.Namespace) -> int:
     description = getattr(args, "description", None)
     force = bool(getattr(args, "force", False))
 
+    # Ensure FIREWORKS_API_KEY is available to the remote by storing it as a Fireworks secret
+    try:
+        fw_account_id = get_fireworks_account_id()
+        fw_api_key_value = get_fireworks_api_key()
+        if fw_account_id and fw_api_key_value:
+            print("Ensuring FIREWORKS_API_KEY is registered as a secret on Fireworks for rollout...")
+            if create_or_update_fireworks_secret(
+                account_id=fw_account_id,
+                key_name="FIREWORKS_API_KEY",
+                secret_value=fw_api_key_value,
+            ):
+                print("✓ FIREWORKS_API_KEY secret created/updated on Fireworks.")
+            else:
+                print("Warning: Failed to create/update FIREWORKS_API_KEY secret on Fireworks.")
+        else:
+            if not fw_account_id:
+                print("Warning: FIREWORKS_ACCOUNT_ID not found; cannot register FIREWORKS_API_KEY secret.")
+            if not fw_api_key_value:
+                print("Warning: FIREWORKS_API_KEY not found locally; cannot register secret.")
+    except Exception as e:
+        print(f"Warning: Skipped Fireworks secret registration due to error: {e}")
+
     exit_code = 0
     for i, (code, file_name, qualname, source_file_path) in enumerate(selected_specs):
         # Use ts_mode to upload evaluator

eval_protocol/platform_api.py

@@ -53,6 +53,16 @@ def __str__(self) -> str:
         return f"{super().__str__()} (Status: {self.status_code}, Response: {self.response_text or 'N/A'})"
 
 
+def _normalize_secret_resource_id(key_name: str) -> str:
+    """
+    Normalize a secret's resource ID for Fireworks paths:
+    - Lowercase
+    - Replace underscores with hyphens
+    - Leave other characters as-is (server enforces allowed set)
+    """
+    return key_name.lower().replace("_", "-")
+
+
 def create_or_update_fireworks_secret(
     account_id: str,
     key_name: str,  # This is the identifier for the secret, e.g., "my-eval-api-key"
@@ -95,8 +105,9 @@ def create_or_update_fireworks_secret(
     # Let's assume for POST, we send 'keyName' and 'value'.
     # For PATCH, the path contains {secret_id} which is the key_name. The body is also gatewaySecret.
 
-    # Check if secret exists using GET (path uses secret_id which is our key_name)
-    get_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{key_name}"
+    # Check if secret exists using GET (path uses normalized resource id)
+    resource_id = _normalize_secret_resource_id(key_name)
+    get_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
     secret_exists = False
     try:
         response = requests.get(get_url, headers=headers, timeout=10)
@@ -120,7 +131,7 @@ def create_or_update_fireworks_secret(
 
     if secret_exists:
         # Update existing secret (PATCH)
-        patch_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{key_name}"
+        patch_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
         # Body for PATCH requires 'keyName' and 'value'.
         # Transform key_name for payload: uppercase and underscores
         payload_key_name = key_name.upper().replace("-", "_")
@@ -148,16 +159,14 @@ def create_or_update_fireworks_secret(
     else:
         # Create new secret (POST)
         post_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets"
-        # Body for POST is gatewaySecret. 'name' field in payload is tricky.
+        # Body for POST is gatewaySecret. 'name' field in payload is the resource path.
         # Let's assume for POST, the 'name' in payload can be omitted or is the key_name.
         # The API should ideally use 'keyName' from URL or a specific 'secretId' in payload for creation if 'name' is server-assigned.
         # Given the Swagger, 'name' is required in gatewaySecret.
         # Let's try with 'name' being the 'key_name' for the payload, as the full path is not known yet.
         # This might need adjustment based on actual API behavior.
         # Construct the full 'name' path for the POST payload as per Swagger's title for 'name'
-        full_resource_name_for_payload = (
-            f"accounts/{resolved_account_id}/secrets/{key_name}"  # Path uses lowercase-hyphenated key_name
-        )
+        full_resource_name_for_payload = f"accounts/{resolved_account_id}/secrets/{resource_id}"
 
         # Transform key_name for payload "keyName" field: uppercase and underscores
         payload_key_name = key_name.upper().replace("-", "_")
@@ -209,7 +218,8 @@ def get_fireworks_secret(
         return None
 
     headers = {"Authorization": f"Bearer {resolved_api_key}"}
-    url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{key_name}"
+    resource_id = _normalize_secret_resource_id(key_name)
+    url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
 
     try:
         response = requests.get(url, headers=headers, timeout=10)
@@ -245,7 +255,8 @@ def delete_fireworks_secret(
         return False
 
     headers = {"Authorization": f"Bearer {resolved_api_key}"}
-    url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{key_name}"
+    resource_id = _normalize_secret_resource_id(key_name)
+    url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
 
     try:
         response = requests.delete(url, headers=headers, timeout=30)