Add eval-protocol user-agent to fireworks api requests (#311)

* Add User-Agent header to Fireworks API requests

Co-authored-by: dhuang <dhuang@fireworks.ai>

* Remove unused User-Agent header from RewardFunction

Co-authored-by: dhuang <dhuang@fireworks.ai>

* Remove unnecessary User-Agent header from Fireworks adapter

Co-authored-by: dhuang <dhuang@fireworks.ai>

* Refactor: Use FireworksAPIClient for all API requests

This change centralizes API request logic into a new FireworksAPIClient class, simplifying
and standardizing how the Fireworks API is interacted with across the project.
It removes redundant request setup code and ensures consistent headers are sent.

Co-authored-by: dhuang <dhuang@fireworks.ai>

* fix lint errors

* fix lint errors

* Update User-Agent format in get_user_agent function to use 'eval-protocol' prefix

* added tests

* update

* reject absolute within path arg

* revert

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>

Author: Dylan Huang

eval_protocol/adapters/fireworks_tracing.py

@@ -273,7 +273,12 @@ def search_logs(self, tags: List[str], limit: int = 100, hours_back: int = 24) -
         if not tags:
             raise ValueError("At least one tag is required to fetch logs")
 
-        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
+        from ..common_utils import get_user_agent
+
+        headers = {
+            "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}",
+            "User-Agent": get_user_agent(),
+        }
         params: Dict[str, Any] = {"tags": tags, "limit": limit, "hours_back": hours_back, "program": "eval_protocol"}
 
         # Try /logs first, fall back to /v1/logs if not found
@@ -398,7 +403,12 @@ def get_evaluation_rows(
         else:
             url = f"{self.base_url}/v1/traces/pointwise"
 
-        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
+        from ..common_utils import get_user_agent
+
+        headers = {
+            "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}",
+            "User-Agent": get_user_agent(),
+        }
 
         result = None
         try:

eval_protocol/auth.py

@@ -242,9 +242,16 @@ def verify_api_key_and_get_account_id(
         if not resolved_key:
             return None
         resolved_base = api_base or get_fireworks_api_base()
+
+        from .common_utils import get_user_agent
+
         url = f"{resolved_base.rstrip('/')}/verifyApiKey"
-        headers = {"Authorization": f"Bearer {resolved_key}"}
+        headers = {
+            "Authorization": f"Bearer {resolved_key}",
+            "User-Agent": get_user_agent(),
+        }
         resp = requests.get(url, headers=headers, timeout=10)
+
         if resp.status_code != 200:
             logger.debug("verifyApiKey returned status %s", resp.status_code)
             return None

eval_protocol/common_utils.py

@@ -5,6 +5,23 @@
 import requests
 
 
+def get_user_agent() -> str:
+    """
+    Returns the user-agent string for eval-protocol CLI requests.
+
+    Format: eval-protocol-cli/{version}
+
+    Returns:
+        User-agent string identifying the eval-protocol CLI and version.
+    """
+    try:
+        from . import __version__
+
+        return f"eval-protocol/{__version__}"
+    except Exception:
+        return "eval-protocol/unknown"
+
+
 def load_jsonl(file_path: str) -> List[Dict[str, Any]]:
     """
     Reads a JSONL file where each line is a valid JSON object and returns a list of these objects.

eval_protocol/evaluation.py

@@ -20,6 +20,7 @@
     get_fireworks_api_key,
     verify_api_key_and_get_account_id,
 )
+from eval_protocol.common_utils import get_user_agent
 from eval_protocol.typed_interface import EvaluationMode
 
 from eval_protocol.get_pep440_version import get_pep440_version
@@ -405,6 +406,7 @@ def preview(self, sample_file, max_samples=5):
         headers = {
             "Authorization": f"Bearer {auth_token}",
             "Content-Type": "application/json",
+            "User-Agent": get_user_agent(),
         }
         logger.info(f"Previewing evaluator using API endpoint: {url} with account: {account_id}")
         logger.debug(f"Preview API Request URL: {url}")
@@ -748,6 +750,7 @@ def create(self, evaluator_id, display_name=None, description=None, force=False)
         headers = {
             "Authorization": f"Bearer {auth_token}",
             "Content-Type": "application/json",
+            "User-Agent": get_user_agent(),
         }
 
         self._ensure_requirements_present(os.getcwd())

eval_protocol/fireworks_rft.py

@@ -11,6 +11,7 @@
 import requests
 
 from .auth import get_fireworks_account_id, get_fireworks_api_base, get_fireworks_api_key
+from .common_utils import get_user_agent
 
 
 def _map_api_host_to_app_host(api_base: str) -> str:
@@ -157,12 +158,17 @@ def create_dataset_from_jsonl(
     display_name: Optional[str],
     jsonl_path: str,
 ) -> Tuple[str, Dict[str, Any]]:
-    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+        "User-Agent": get_user_agent(),
+    }
     # Count examples quickly
     example_count = 0
     with open(jsonl_path, "r", encoding="utf-8") as f:
         for _ in f:
             example_count += 1
+
     dataset_url = f"{api_base.rstrip('/')}/v1/accounts/{account_id}/datasets"
     payload = {
         "dataset": {
@@ -181,7 +187,10 @@ def create_dataset_from_jsonl(
     upload_url = f"{api_base.rstrip('/')}/v1/accounts/{account_id}/datasets/{dataset_id}:upload"
     with open(jsonl_path, "rb") as f:
         files = {"file": f}
-        up_headers = {"Authorization": f"Bearer {api_key}"}
+        up_headers = {
+            "Authorization": f"Bearer {api_key}",
+            "User-Agent": get_user_agent(),
+        }
         up_resp = requests.post(upload_url, files=files, headers=up_headers, timeout=600)
     if up_resp.status_code not in (200, 201):
         raise RuntimeError(f"Dataset upload failed: {up_resp.status_code} {up_resp.text}")
@@ -195,7 +204,12 @@ def create_reinforcement_fine_tuning_job(
     body: Dict[str, Any],
 ) -> Dict[str, Any]:
     url = f"{api_base.rstrip('/')}/v1/accounts/{account_id}/reinforcementFineTuningJobs"
-    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json", "Accept": "application/json"}
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        "User-Agent": get_user_agent(),
+    }
     resp = requests.post(url, json=body, headers=headers, timeout=60)
     if resp.status_code not in (200, 201):
         raise RuntimeError(f"RFT job creation failed: {resp.status_code} {resp.text}")

eval_protocol/generation/clients.py

@@ -13,6 +13,8 @@
 from omegaconf import DictConfig
 from pydantic import BaseModel  # Added for new models
 
+from ..common_utils import get_user_agent
+
 logger = logging.getLogger(__name__)
 
 
@@ -101,6 +103,7 @@ async def generate(
             "Authorization": f"Bearer {self.api_key}",
             "Content-Type": "application/json",
             "Accept": "application/json",
+            "User-Agent": get_user_agent(),
         }
 
         debug_payload_log = json.loads(json.dumps(payload))

eval_protocol/platform_api.py

@@ -11,6 +11,7 @@
     get_fireworks_api_base,
     get_fireworks_api_key,
 )
+from eval_protocol.common_utils import get_user_agent
 
 logger = logging.getLogger(__name__)
 
@@ -95,6 +96,7 @@ def create_or_update_fireworks_secret(
     headers = {
         "Authorization": f"Bearer {resolved_api_key}",
         "Content-Type": "application/json",
+        "User-Agent": get_user_agent(),
     }
 
     # The secret_id for GET/PATCH/DELETE operations is the key_name.
@@ -107,10 +109,10 @@ def create_or_update_fireworks_secret(
 
     # Check if secret exists using GET (path uses normalized resource id)
     resource_id = _normalize_secret_resource_id(key_name)
-    get_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
     secret_exists = False
     try:
-        response = requests.get(get_url, headers=headers, timeout=10)
+        url = f"{resolved_api_base}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
+        response = requests.get(url, headers=headers, timeout=10)
         if response.status_code == 200:
             secret_exists = True
             logger.info(f"Secret '{key_name}' already exists. Will attempt to update.")
@@ -131,7 +133,6 @@ def create_or_update_fireworks_secret(
 
     if secret_exists:
         # Update existing secret (PATCH)
-        patch_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
         # Body for PATCH requires 'keyName' and 'value'.
         # Transform key_name for payload: uppercase and underscores
         payload_key_name = key_name.upper().replace("-", "_")
@@ -146,7 +147,8 @@ def create_or_update_fireworks_secret(
         payload = {"keyName": payload_key_name, "value": secret_value}
         try:
             logger.debug(f"PATCH payload for '{key_name}': {payload}")
-            response = requests.patch(patch_url, headers=headers, json=payload, timeout=30)
+            url = f"{resolved_api_base}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
+            response = requests.patch(url, json=payload, headers=headers, timeout=30)
             response.raise_for_status()
             logger.info(f"Successfully updated secret '{key_name}' on Fireworks platform.")
             return True
@@ -158,7 +160,6 @@ def create_or_update_fireworks_secret(
             return False
     else:
         # Create new secret (POST)
-        post_url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets"
         # Body for POST is gatewaySecret. 'name' field in payload is the resource path.
         # Let's assume for POST, the 'name' in payload can be omitted or is the key_name.
         # The API should ideally use 'keyName' from URL or a specific 'secretId' in payload for creation if 'name' is server-assigned.
@@ -183,7 +184,8 @@ def create_or_update_fireworks_secret(
         }
         try:
             logger.debug(f"POST payload for '{key_name}': {payload}")
-            response = requests.post(post_url, headers=headers, json=payload, timeout=30)
+            url = f"{resolved_api_base}/v1/accounts/{resolved_account_id}/secrets"
+            response = requests.post(url, json=payload, headers=headers, timeout=30)
             response.raise_for_status()
             logger.info(
                 f"Successfully created secret '{key_name}' on Fireworks platform. Full name: {response.json().get('name')}"
@@ -217,11 +219,14 @@ def get_fireworks_secret(
         logger.error("Missing Fireworks API key, base URL, or account ID for getting secret.")
         return None
 
-    headers = {"Authorization": f"Bearer {resolved_api_key}"}
+    headers = {
+        "Authorization": f"Bearer {resolved_api_key}",
+        "User-Agent": get_user_agent(),
+    }
     resource_id = _normalize_secret_resource_id(key_name)
-    url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
 
     try:
+        url = f"{resolved_api_base}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
         response = requests.get(url, headers=headers, timeout=10)
         if response.status_code == 200:
             logger.info(f"Successfully retrieved secret '{key_name}'.")
@@ -254,11 +259,14 @@ def delete_fireworks_secret(
         logger.error("Missing Fireworks API key, base URL, or account ID for deleting secret.")
         return False
 
-    headers = {"Authorization": f"Bearer {resolved_api_key}"}
+    headers = {
+        "Authorization": f"Bearer {resolved_api_key}",
+        "User-Agent": get_user_agent(),
+    }
     resource_id = _normalize_secret_resource_id(key_name)
-    url = f"{resolved_api_base.rstrip('/')}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
 
     try:
+        url = f"{resolved_api_base}/v1/accounts/{resolved_account_id}/secrets/{resource_id}"
         response = requests.delete(url, headers=headers, timeout=30)
         if response.status_code == 200 or response.status_code == 204:  # 204 No Content is also success for DELETE
             logger.info(f"Successfully deleted secret '{key_name}'.")

eval_protocol/pytest/handle_persist_flow.py

@@ -7,9 +7,11 @@
 import re
 from typing import Any
 
+from eval_protocol.common_utils import get_user_agent
 from eval_protocol.directory_utils import find_eval_protocol_dir
 from eval_protocol.models import EvaluationRow
 from eval_protocol.pytest.store_experiment_link import store_experiment_link
+
 import requests
 
 
@@ -127,10 +129,14 @@ def get_auth_value(key: str) -> str | None:
                         )
                         continue
 
-                    headers = {"Authorization": f"Bearer {fireworks_api_key}", "Content-Type": "application/json"}
+                    api_base = "https://api.fireworks.ai"
+                    headers = {
+                        "Authorization": f"Bearer {fireworks_api_key}",
+                        "Content-Type": "application/json",
+                        "User-Agent": get_user_agent(),
+                    }
 
                     # Make dataset first
-                    dataset_url = f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/datasets"
 
                     dataset_payload = {  # pyright: ignore[reportUnknownVariableType]
                         "dataset": {
@@ -142,6 +148,7 @@ def get_auth_value(key: str) -> str | None:
                         "datasetId": dataset_name,
                     }
 
+                    dataset_url = f"{api_base}/v1/accounts/{fireworks_account_id}/datasets"
                     dataset_response = requests.post(dataset_url, json=dataset_payload, headers=headers)  # pyright: ignore[reportUnknownArgumentType]
 
                     # Skip if dataset creation failed
@@ -157,13 +164,13 @@ def get_auth_value(key: str) -> str | None:
                     dataset_id = dataset_data.get("datasetId", dataset_name)  # pyright: ignore[reportAny]
 
                     # Upload the JSONL file content
-                    upload_url = (
-                        f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/datasets/{dataset_id}:upload"
-                    )
-                    upload_headers = {"Authorization": f"Bearer {fireworks_api_key}"}
-
+                    upload_url = f"{api_base}/v1/accounts/{fireworks_account_id}/datasets/{dataset_id}:upload"
                     with open(exp_file, "rb") as f:
                         files = {"file": f}
+                        upload_headers = {
+                            "Authorization": f"Bearer {fireworks_api_key}",
+                            "User-Agent": get_user_agent(),
+                        }
                         upload_response = requests.post(upload_url, files=files, headers=upload_headers)
 
                     # Skip if upload failed
@@ -176,7 +183,6 @@ def get_auth_value(key: str) -> str | None:
                         continue
 
                     # Create evaluation job (optional - don't skip experiment if this fails)
-                    eval_job_url = f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/evaluationJobs"
                     # Truncate job ID to fit 63 character limit
                     job_id_base = f"{dataset_name}-job"
                     if len(job_id_base) > 63:
@@ -194,6 +200,7 @@ def get_auth_value(key: str) -> str | None:
                         },
                     }
 
+                    eval_job_url = f"{api_base}/v1/accounts/{fireworks_account_id}/evaluationJobs"
                     eval_response = requests.post(eval_job_url, json=eval_job_payload, headers=headers)
 
                     if eval_response.status_code in [200, 201]: