diff --git a/eval_protocol/adapters/fireworks_tracing.py b/eval_protocol/adapters/fireworks_tracing.py index 233371cc..218f9d1d 100644 --- a/eval_protocol/adapters/fireworks_tracing.py +++ b/eval_protocol/adapters/fireworks_tracing.py @@ -273,7 +273,12 @@ def search_logs(self, tags: List[str], limit: int = 100, hours_back: int = 24) - if not tags: raise ValueError("At least one tag is required to fetch logs") - headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"} + from ..common_utils import get_user_agent + + headers = { + "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}", + "User-Agent": get_user_agent(), + } params: Dict[str, Any] = {"tags": tags, "limit": limit, "hours_back": hours_back, "program": "eval_protocol"} # Try /logs first, fall back to /v1/logs if not found @@ -398,7 +403,12 @@ def get_evaluation_rows( else: url = f"{self.base_url}/v1/traces/pointwise" - headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"} + from ..common_utils import get_user_agent + + headers = { + "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}", + "User-Agent": get_user_agent(), + } result = None try: diff --git a/eval_protocol/auth.py b/eval_protocol/auth.py index 6b0845bb..3002bd4a 100644 --- a/eval_protocol/auth.py +++ b/eval_protocol/auth.py @@ -242,9 +242,16 @@ def verify_api_key_and_get_account_id( if not resolved_key: return None resolved_base = api_base or get_fireworks_api_base() + + from .common_utils import get_user_agent + url = f"{resolved_base.rstrip('/')}/verifyApiKey" - headers = {"Authorization": f"Bearer {resolved_key}"} + headers = { + "Authorization": f"Bearer {resolved_key}", + "User-Agent": get_user_agent(), + } resp = requests.get(url, headers=headers, timeout=10) + if resp.status_code != 200: logger.debug("verifyApiKey returned status %s", resp.status_code) return None diff --git a/eval_protocol/common_utils.py b/eval_protocol/common_utils.py index 9b9032ab..3bca887f 100644 --- a/eval_protocol/common_utils.py +++ b/eval_protocol/common_utils.py @@ -5,6 +5,23 @@ import requests +def get_user_agent() -> str: + """ + Returns the user-agent string for eval-protocol CLI requests. + + Format: eval-protocol-cli/{version} + + Returns: + User-agent string identifying the eval-protocol CLI and version. + """ + try: + from . import __version__ + + return f"eval-protocol/{__version__}" + except Exception: + return "eval-protocol/unknown" + + def load_jsonl(file_path: str) -> List[Dict[str, Any]]: """ Reads a JSONL file where each line is a valid JSON object and returns a list of these objects. diff --git a/eval_protocol/evaluation.py b/eval_protocol/evaluation.py index 72459828..7d7ef3da 100644 --- a/eval_protocol/evaluation.py +++ b/eval_protocol/evaluation.py @@ -20,6 +20,7 @@ get_fireworks_api_key, verify_api_key_and_get_account_id, ) +from eval_protocol.common_utils import get_user_agent from eval_protocol.typed_interface import EvaluationMode from eval_protocol.get_pep440_version import get_pep440_version @@ -405,6 +406,7 @@ def preview(self, sample_file, max_samples=5): headers = { "Authorization": f"Bearer {auth_token}", "Content-Type": "application/json", + "User-Agent": get_user_agent(), } logger.info(f"Previewing evaluator using API endpoint: {url} with account: {account_id}") logger.debug(f"Preview API Request URL: {url}") @@ -748,6 +750,7 @@ def create(self, evaluator_id, display_name=None, description=None, force=False) headers = { "Authorization": f"Bearer {auth_token}", "Content-Type": "application/json", + "User-Agent": get_user_agent(), } self._ensure_requirements_present(os.getcwd()) diff --git a/eval_protocol/fireworks_rft.py b/eval_protocol/fireworks_rft.py index 6bd2e62e..05b49291 100644 --- a/eval_protocol/fireworks_rft.py +++ b/eval_protocol/fireworks_rft.py @@ -11,6 +11,7 @@ import requests from .auth import get_fireworks_account_id, get_fireworks_api_base, get_fireworks_api_key +from .common_utils import get_user_agent def _map_api_host_to_app_host(api_base: str) -> str: @@ -157,12 +158,17 @@ def create_dataset_from_jsonl( display_name: Optional[str], jsonl_path: str, ) -> Tuple[str, Dict[str, Any]]: - headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"} + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "User-Agent": get_user_agent(), + } # Count examples quickly example_count = 0 with open(jsonl_path, "r", encoding="utf-8") as f: for _ in f: example_count += 1 + dataset_url = f"{api_base.rstrip('/')}/v1/accounts/{account_id}/datasets" payload = { "dataset": { @@ -181,7 +187,10 @@ def create_dataset_from_jsonl( upload_url = f"{api_base.rstrip('/')}/v1/accounts/{account_id}/datasets/{dataset_id}:upload" with open(jsonl_path, "rb") as f: files = {"file": f} - up_headers = {"Authorization": f"Bearer {api_key}"} + up_headers = { + "Authorization": f"Bearer {api_key}", + "User-Agent": get_user_agent(), + } up_resp = requests.post(upload_url, files=files, headers=up_headers, timeout=600) if up_resp.status_code not in (200, 201): raise RuntimeError(f"Dataset upload failed: {up_resp.status_code} {up_resp.text}") @@ -195,7 +204,12 @@ def create_reinforcement_fine_tuning_job( body: Dict[str, Any], ) -> Dict[str, Any]: url = f"{api_base.rstrip('/')}/v1/accounts/{account_id}/reinforcementFineTuningJobs" - headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json", "Accept": "application/json"} + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "Accept": "application/json", + "User-Agent": get_user_agent(), + } resp = requests.post(url, json=body, headers=headers, timeout=60) if resp.status_code not in (200, 201): raise RuntimeError(f"RFT job creation failed: {resp.status_code} {resp.text}") diff --git a/eval_protocol/generation/clients.py b/eval_protocol/generation/clients.py index 873f587e..7ac80272 100644 --- a/eval_protocol/generation/clients.py +++ b/eval_protocol/generation/clients.py @@ -13,6 +13,8 @@ from omegaconf import DictConfig from pydantic import BaseModel # Added for new models +from ..common_utils import get_user_agent + logger = logging.getLogger(__name__) @@ -101,6 +103,7 @@ async def generate( "Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json", "Accept": "application/json", + "User-Agent": get_user_agent(), } debug_payload_log = json.loads(json.dumps(payload)) diff --git a/eval_protocol/platform_api.py b/eval_protocol/platform_api.py index 5158d8e0..81754e13 100644 --- a/eval_protocol/platform_api.py +++ b/eval_protocol/platform_api.py @@ -11,6 +11,7 @@ get_fireworks_api_base, get_fireworks_api_key, ) +from eval_protocol.common_utils import get_user_agent logger = logging.getLogger(__name__) @@ -95,6 +96,7 @@ def create_or_update_fireworks_secret( headers = { "Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json", + "User-Agent": get_user_agent(), } # The secret_id for GET/PATCH/DELETE operations is the key_name. @@ -107,10 +109,10 @@ def create_or_update_fireworks_secret( # Check if secret exists using GET (path uses normalized resource id) resource_id = _normalize_secret_resource_id(key_name) - get_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}" secret_exists = False try: - response = requests.get(get_url, headers=headers, timeout=10) + url = f"{resolved_api_base}/v1/accounts/{resolved_account_id}/secrets/{resource_id}" + response = requests.get(url, headers=headers, timeout=10) if response.status_code == 200: secret_exists = True logger.info(f"Secret '{key_name}' already exists. Will attempt to update.") @@ -131,7 +133,6 @@ def create_or_update_fireworks_secret( if secret_exists: # Update existing secret (PATCH) - patch_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}" # Body for PATCH requires 'keyName' and 'value'. # Transform key_name for payload: uppercase and underscores payload_key_name = key_name.upper().replace("-", "_") @@ -146,7 +147,8 @@ def create_or_update_fireworks_secret( payload = {"keyName": payload_key_name, "value": secret_value} try: logger.debug(f"PATCH payload for '{key_name}': {payload}") - response = requests.patch(patch_url, headers=headers, json=payload, timeout=30) + url = f"{resolved_api_base}/v1/accounts/{resolved_account_id}/secrets/{resource_id}" + response = requests.patch(url, json=payload, headers=headers, timeout=30) response.raise_for_status() logger.info(f"Successfully updated secret '{key_name}' on Fireworks platform.") return True @@ -158,7 +160,6 @@ def create_or_update_fireworks_secret( return False else: # Create new secret (POST) - post_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets" # Body for POST is gatewaySecret. 'name' field in payload is the resource path. # Let's assume for POST, the 'name' in payload can be omitted or is the key_name. # The API should ideally use 'keyName' from URL or a specific 'secretId' in payload for creation if 'name' is server-assigned. @@ -183,7 +184,8 @@ def create_or_update_fireworks_secret( } try: logger.debug(f"POST payload for '{key_name}': {payload}") - response = requests.post(post_url, headers=headers, json=payload, timeout=30) + url = f"{resolved_api_base}/v1/accounts/{resolved_account_id}/secrets" + response = requests.post(url, json=payload, headers=headers, timeout=30) response.raise_for_status() logger.info( f"Successfully created secret '{key_name}' on Fireworks platform. Full name: {response.json().get('name')}" @@ -217,11 +219,14 @@ def get_fireworks_secret( logger.error("Missing Fireworks API key, base URL, or account ID for getting secret.") return None - headers = {"Authorization": f"Bearer {resolved_api_key}"} + headers = { + "Authorization": f"Bearer {resolved_api_key}", + "User-Agent": get_user_agent(), + } resource_id = _normalize_secret_resource_id(key_name) - url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}" try: + url = f"{resolved_api_base}/v1/accounts/{resolved_account_id}/secrets/{resource_id}" response = requests.get(url, headers=headers, timeout=10) if response.status_code == 200: logger.info(f"Successfully retrieved secret '{key_name}'.") @@ -254,11 +259,14 @@ def delete_fireworks_secret( logger.error("Missing Fireworks API key, base URL, or account ID for deleting secret.") return False - headers = {"Authorization": f"Bearer {resolved_api_key}"} + headers = { + "Authorization": f"Bearer {resolved_api_key}", + "User-Agent": get_user_agent(), + } resource_id = _normalize_secret_resource_id(key_name) - url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}" try: + url = f"{resolved_api_base}/v1/accounts/{resolved_account_id}/secrets/{resource_id}" response = requests.delete(url, headers=headers, timeout=30) if response.status_code == 200 or response.status_code == 204: # 204 No Content is also success for DELETE logger.info(f"Successfully deleted secret '{key_name}'.") diff --git a/eval_protocol/pytest/handle_persist_flow.py b/eval_protocol/pytest/handle_persist_flow.py index e2f2a93d..07a627f3 100644 --- a/eval_protocol/pytest/handle_persist_flow.py +++ b/eval_protocol/pytest/handle_persist_flow.py @@ -7,9 +7,11 @@ import re from typing import Any +from eval_protocol.common_utils import get_user_agent from eval_protocol.directory_utils import find_eval_protocol_dir from eval_protocol.models import EvaluationRow from eval_protocol.pytest.store_experiment_link import store_experiment_link + import requests @@ -127,10 +129,14 @@ def get_auth_value(key: str) -> str | None: ) continue - headers = {"Authorization": f"Bearer {fireworks_api_key}", "Content-Type": "application/json"} + api_base = "https://api.fireworks.ai" + headers = { + "Authorization": f"Bearer {fireworks_api_key}", + "Content-Type": "application/json", + "User-Agent": get_user_agent(), + } # Make dataset first - dataset_url = f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/datasets" dataset_payload = { # pyright: ignore[reportUnknownVariableType] "dataset": { @@ -142,6 +148,7 @@ def get_auth_value(key: str) -> str | None: "datasetId": dataset_name, } + dataset_url = f"{api_base}/v1/accounts/{fireworks_account_id}/datasets" dataset_response = requests.post(dataset_url, json=dataset_payload, headers=headers) # pyright: ignore[reportUnknownArgumentType] # Skip if dataset creation failed @@ -157,13 +164,13 @@ def get_auth_value(key: str) -> str | None: dataset_id = dataset_data.get("datasetId", dataset_name) # pyright: ignore[reportAny] # Upload the JSONL file content - upload_url = ( - f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/datasets/{dataset_id}:upload" - ) - upload_headers = {"Authorization": f"Bearer {fireworks_api_key}"} - + upload_url = f"{api_base}/v1/accounts/{fireworks_account_id}/datasets/{dataset_id}:upload" with open(exp_file, "rb") as f: files = {"file": f} + upload_headers = { + "Authorization": f"Bearer {fireworks_api_key}", + "User-Agent": get_user_agent(), + } upload_response = requests.post(upload_url, files=files, headers=upload_headers) # Skip if upload failed @@ -176,7 +183,6 @@ def get_auth_value(key: str) -> str | None: continue # Create evaluation job (optional - don't skip experiment if this fails) - eval_job_url = f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/evaluationJobs" # Truncate job ID to fit 63 character limit job_id_base = f"{dataset_name}-job" if len(job_id_base) > 63: @@ -194,6 +200,7 @@ def get_auth_value(key: str) -> str | None: }, } + eval_job_url = f"{api_base}/v1/accounts/{fireworks_account_id}/evaluationJobs" eval_response = requests.post(eval_job_url, json=eval_job_payload, headers=headers) if eval_response.status_code in [200, 201]: