From 7582e7d22a76fadca514767e48ecf673f2ead4ff Mon Sep 17 00:00:00 2001 From: benjibc Date: Tue, 7 Oct 2025 18:47:41 +0000 Subject: [PATCH 1/2] check FIREWORKS_API_KEY in upload flow --- eval_protocol/cli_commands/upload.py | 75 +++++++++++++++++++++------- 1 file changed, 58 insertions(+), 17 deletions(-) diff --git a/eval_protocol/cli_commands/upload.py b/eval_protocol/cli_commands/upload.py index 44ff6607..93a7ee0b 100644 --- a/eval_protocol/cli_commands/upload.py +++ b/eval_protocol/cli_commands/upload.py @@ -12,6 +12,8 @@ from typing import Any, Callable, Iterable, Optional import pytest +from eval_protocol.auth import get_fireworks_account_id, get_fireworks_api_key +from eval_protocol.platform_api import create_or_update_fireworks_secret from eval_protocol.evaluation import create_evaluation @@ -343,28 +345,45 @@ def _prompt_select_interactive(tests: list[DiscoveredTest]) -> list[DiscoveredTe else: return [] - # Create choices with nice formatting - choices = [] - for idx, test in enumerate(tests, 1): - choice_text = _format_test_choice(test, idx) - choices.append({"name": choice_text, "value": idx - 1, "checked": False}) + # Enter-only selection UX with optional multi-select via repeat + remaining_indices = list(range(len(tests))) + selected_indices: list[int] = [] print("\n") - print("šŸ’” Tip: Use ā†‘/ā†“ arrows to navigate, SPACE to select/deselect, ENTER when done") - print(" You can select multiple tests!\n") - selected_indices = questionary.checkbox( - "Select evaluation tests to upload:", - choices=choices, - style=custom_style, - ).ask() - - if selected_indices is None: # User pressed Ctrl+C - print("\nUpload cancelled.") - return [] + print("Tip: Use ā†‘/ā†“ arrows to navigate and press ENTER to select.") + print(" After selecting one, you can choose to add more.\n") + + while remaining_indices: + # Build choices from remaining + choices = [] + for idx, test_idx in enumerate(remaining_indices, 1): + t = tests[test_idx] + choice_text = _format_test_choice(t, idx) + choices.append({"name": choice_text, "value": test_idx}) + + selected = questionary.select( + "Select an evaluation test to upload:", choices=choices, style=custom_style + ).ask() + + if selected is None: # Ctrl+C + print("\nUpload cancelled.") + return [] + + if isinstance(selected, int): + selected_indices.append(selected) + # Remove from remaining + if selected in remaining_indices: + remaining_indices.remove(selected) + + # Ask whether to add another (ENTER to finish) + add_more = questionary.confirm("Add another?", default=False, style=custom_style).ask() + if not add_more: + break + else: + break if not selected_indices: print("\nāš ļø No tests were selected.") - print(" Remember: Use SPACE bar to select tests, then press ENTER to confirm.") return [] print(f"\nāœ“ Selected {len(selected_indices)} test(s)") @@ -474,6 +493,28 @@ def upload_command(args: argparse.Namespace) -> int: description = getattr(args, "description", None) force = bool(getattr(args, "force", False)) + # Ensure FIREWORKS_API_KEY is available to the remote by storing it as a Fireworks secret + try: + fw_account_id = get_fireworks_account_id() + fw_api_key_value = get_fireworks_api_key() + if fw_account_id and fw_api_key_value: + print("Ensuring FIREWORKS_API_KEY is registered as a secret on Fireworks for rollout...") + if create_or_update_fireworks_secret( + account_id=fw_account_id, + key_name="FIREWORKS_API_KEY", + secret_value=fw_api_key_value, + ): + print("āœ“ FIREWORKS_API_KEY secret created/updated on Fireworks.") + else: + print("Warning: Failed to create/update FIREWORKS_API_KEY secret on Fireworks.") + else: + if not fw_account_id: + print("Warning: FIREWORKS_ACCOUNT_ID not found; cannot register FIREWORKS_API_KEY secret.") + if not fw_api_key_value: + print("Warning: FIREWORKS_API_KEY not found locally; cannot register secret.") + except Exception as e: + print(f"Warning: Skipped Fireworks secret registration due to error: {e}") + exit_code = 0 for i, (code, file_name, qualname, source_file_path) in enumerate(selected_specs): # Use ts_mode to upload evaluator From 078d04688bf0f7a40bedfccce3f6cc484892240d Mon Sep 17 00:00:00 2001 From: benjibc Date: Tue, 7 Oct 2025 22:31:58 +0000 Subject: [PATCH 2/2] normalize resource id --- eval_protocol/platform_api.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/eval_protocol/platform_api.py b/eval_protocol/platform_api.py index c5c4d62e..5158d8e0 100644 --- a/eval_protocol/platform_api.py +++ b/eval_protocol/platform_api.py @@ -53,6 +53,16 @@ def __str__(self) -> str: return f"{super().__str__()} (Status: {self.status_code}, Response: {self.response_text or 'N/A'})" +def _normalize_secret_resource_id(key_name: str) -> str: + """ + Normalize a secret's resource ID for Fireworks paths: + - Lowercase + - Replace underscores with hyphens + - Leave other characters as-is (server enforces allowed set) + """ + return key_name.lower().replace("_", "-") + + def create_or_update_fireworks_secret( account_id: str, key_name: str, # This is the identifier for the secret, e.g., "my-eval-api-key" @@ -95,8 +105,9 @@ def create_or_update_fireworks_secret( # Let's assume for POST, we send 'keyName' and 'value'. # For PATCH, the path contains {secret_id} which is the key_name. The body is also gatewaySecret. - # Check if secret exists using GET (path uses secret_id which is our key_name) - get_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{key_name}" + # Check if secret exists using GET (path uses normalized resource id) + resource_id = _normalize_secret_resource_id(key_name) + get_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}" secret_exists = False try: response = requests.get(get_url, headers=headers, timeout=10) @@ -120,7 +131,7 @@ def create_or_update_fireworks_secret( if secret_exists: # Update existing secret (PATCH) - patch_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{key_name}" + patch_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}" # Body for PATCH requires 'keyName' and 'value'. # Transform key_name for payload: uppercase and underscores payload_key_name = key_name.upper().replace("-", "_") @@ -148,16 +159,14 @@ def create_or_update_fireworks_secret( else: # Create new secret (POST) post_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets" - # Body for POST is gatewaySecret. 'name' field in payload is tricky. + # Body for POST is gatewaySecret. 'name' field in payload is the resource path. # Let's assume for POST, the 'name' in payload can be omitted or is the key_name. # The API should ideally use 'keyName' from URL or a specific 'secretId' in payload for creation if 'name' is server-assigned. # Given the Swagger, 'name' is required in gatewaySecret. # Let's try with 'name' being the 'key_name' for the payload, as the full path is not known yet. # This might need adjustment based on actual API behavior. # Construct the full 'name' path for the POST payload as per Swagger's title for 'name' - full_resource_name_for_payload = ( - f"accounts/{resolved_account_id}/secrets/{key_name}" # Path uses lowercase-hyphenated key_name - ) + full_resource_name_for_payload = f"accounts/{resolved_account_id}/secrets/{resource_id}" # Transform key_name for payload "keyName" field: uppercase and underscores payload_key_name = key_name.upper().replace("-", "_") @@ -209,7 +218,8 @@ def get_fireworks_secret( return None headers = {"Authorization": f"Bearer {resolved_api_key}"} - url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{key_name}" + resource_id = _normalize_secret_resource_id(key_name) + url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}" try: response = requests.get(url, headers=headers, timeout=10) @@ -245,7 +255,8 @@ def delete_fireworks_secret( return False headers = {"Authorization": f"Bearer {resolved_api_key}"} - url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{key_name}" + resource_id = _normalize_secret_resource_id(key_name) + url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}" try: response = requests.delete(url, headers=headers, timeout=30)