Implement Evaluator Versions

.gitignore

@@ -243,3 +243,5 @@ package.json
 tau2-bench
 *.err
 eval-protocol
+
+.vscode/launch.json

eval_protocol/auth.py

@@ -3,9 +3,30 @@
 from typing import Optional
 
 import requests
+from dotenv import find_dotenv, load_dotenv
 
 logger = logging.getLogger(__name__)
 
+# --- Load .env files ---
+# Attempt to load .env.dev first, then .env as a fallback.
+# This happens when the module is imported.
+# We use override=False (default) so that existing environment variables
+# (e.g., set in the shell) are NOT overridden by .env files.
+_ENV_DEV_PATH = find_dotenv(filename=".env.dev", raise_error_if_not_found=False, usecwd=True)
+if _ENV_DEV_PATH:
+    load_dotenv(dotenv_path=_ENV_DEV_PATH, override=False)
+    logger.debug(f"eval_protocol.auth: Loaded environment variables from: {_ENV_DEV_PATH}")
+else:
+    _ENV_PATH = find_dotenv(filename=".env", raise_error_if_not_found=False, usecwd=True)
+    if _ENV_PATH:
+        load_dotenv(dotenv_path=_ENV_PATH, override=False)
+        logger.debug(f"eval_protocol.auth: Loaded environment variables from: {_ENV_PATH}")
+    else:
+        logger.debug(
+            "eval_protocol.auth: No .env.dev or .env file found. Relying on shell/existing environment variables."
+        )
+# --- End .env loading ---
+
 
 def get_fireworks_api_key() -> Optional[str]:
     """
@@ -73,6 +94,8 @@ def verify_api_key_and_get_account_id(
     Args:
         api_key: Optional explicit API key. When None, resolves via get_fireworks_api_key().
         api_base: Optional explicit API base. When None, resolves via get_fireworks_api_base().
+            If api_base is api.fireworks.ai, it is used directly. Otherwise, defaults to
+            dev.api.fireworks.ai for the verification call.
 
     Returns:
         The resolved account id if verification succeeds and the header is present; otherwise None.
@@ -81,7 +104,12 @@ def verify_api_key_and_get_account_id(
         resolved_key = api_key or get_fireworks_api_key()
         if not resolved_key:
             return None
-        resolved_base = api_base or get_fireworks_api_base()
+        provided_base = api_base or get_fireworks_api_base()
+        # Use api.fireworks.ai if explicitly provided, otherwise fall back to dev
+        if "api.fireworks.ai" in provided_base:
+            resolved_base = provided_base
+        else:
+            resolved_base = "https://dev.api.fireworks.ai"
 
         from .common_utils import get_user_agent
 

eval_protocol/cli.py

@@ -8,10 +8,9 @@
 import sys
 from pathlib import Path
 
-from fireworks import Fireworks
-
 from .cli_commands.common import setup_logging
 from .cli_commands.utils import add_args_from_callable_signature
+from .fireworks_client import create_fireworks_client
 
 logger = logging.getLogger(__name__)
 
@@ -88,7 +87,7 @@ def _configure_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParse
     )
 
     # Auto-generate flags from SDK Fireworks().evaluators.create() signature
-    create_evaluator_fn = Fireworks().evaluators.create
+    create_evaluator_fn = create_fireworks_client().evaluators.create
 
     upload_skip_fields = {
         "__top_level__": {
@@ -198,7 +197,7 @@ def _configure_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParse
         "loss_config.method": "RL loss method for underlying trainers. One of {grpo,dapo}.",
     }
 
-    create_rft_job_fn = Fireworks().reinforcement_fine_tuning_jobs.create
+    create_rft_job_fn = create_fireworks_client().reinforcement_fine_tuning_jobs.create
 
     add_args_from_callable_signature(
         rft_parser,

eval_protocol/cli_commands/create_rft.py

@@ -12,6 +12,7 @@
 from pydantic import ValidationError
 
 from ..auth import get_fireworks_api_base, get_fireworks_api_key
+from ..fireworks_client import create_fireworks_client
 from ..common_utils import get_user_agent, load_jsonl
 from ..fireworks_rft import (
     create_dataset_from_jsonl,
@@ -35,8 +36,6 @@
 )
 from .local_test import run_evaluator_test
 
-from fireworks import Fireworks
-
 
 def _extract_dataset_adapter(
     test_file_path: str, test_func_name: str
@@ -672,7 +671,7 @@ def _create_rft_job(
 ) -> int:
     """Build and submit the RFT job request (via Fireworks SDK)."""
 
-    signature = inspect.signature(Fireworks().reinforcement_fine_tuning_jobs.create)
+    signature = inspect.signature(create_fireworks_client().reinforcement_fine_tuning_jobs.create)
 
     # Build top-level SDK kwargs
     sdk_kwargs: Dict[str, Any] = {
@@ -711,7 +710,7 @@ def _create_rft_job(
         return 0
 
     try:
-        fw: Fireworks = Fireworks(api_key=api_key, base_url=api_base)
+        fw: Fireworks = create_fireworks_client(api_key=api_key, base_url=api_base)
         job: ReinforcementFineTuningJob = fw.reinforcement_fine_tuning_jobs.create(account_id=account_id, **sdk_kwargs)
         job_name = job.name
         print(f"\n✅ Created Reinforcement Fine-tuning Job: {job_name}")

eval_protocol/evaluation.py

@@ -4,14 +4,15 @@
 from typing import List, Optional
 
 import fireworks
+from fireworks.types import EvaluatorVersionParam
 import requests
-from fireworks import Fireworks
 
 from eval_protocol.auth import (
     get_fireworks_account_id,
     get_fireworks_api_key,
     verify_api_key_and_get_account_id,
 )
+from eval_protocol.fireworks_client import create_fireworks_client
 from eval_protocol.get_pep440_version import get_pep440_version
 
 logger = logging.getLogger(__name__)
@@ -164,7 +165,11 @@
             logger.error("Authentication error: API credentials appear to be invalid or incomplete.")
             raise ValueError("Invalid or missing API credentials.")
 
-        client = Fireworks(api_key=auth_token, base_url=self.api_base, account_id=account_id)
+        client = create_fireworks_client(
+            api_key=auth_token,
+            base_url=self.api_base,
+            account_id=account_id,
+        )
 
         self.display_name = display_name or evaluator_id
         self.description = description or f"Evaluator created from {evaluator_id}"
@@ -230,6 +235,25 @@
                     f"Cannot proceed with code upload. Response: {result}"
                 )
 
+            evaluator_version_param: EvaluatorVersionParam = {}
+            if "commit_hash" in evaluator_params:
+                evaluator_version_param["commit_hash"] = evaluator_params["commit_hash"]
+            if "entry_point" in evaluator_params:
+                evaluator_version_param["entry_point"] = evaluator_params["entry_point"]
+            if "requirements" in evaluator_params:
+                evaluator_version_param["requirements"] = evaluator_params["requirements"]
+
+            evaluator_version = client.evaluator_versions.create(
+                evaluator_id=evaluator_id,
+                evaluator_version=evaluator_version_param,
+            )
+            evaluator_version_id = evaluator_version.name.split("/")[-1] if evaluator_version.name else None
+            if not evaluator_version_id:
+                raise ValueError(
+                    "Create evaluator version response missing 'name' field. "
+                    f"Cannot proceed with code upload. Response: {evaluator_version}"
+                )
+
             try:
                 # Create tar.gz of current directory
                 cwd = os.getcwd()
@@ -241,7 +265,8 @@
 
                 # Call GetEvaluatorUploadEndpoint using SDK
                 logger.info(f"Requesting upload endpoint for {tar_filename}")
-                upload_response = client.evaluators.get_upload_endpoint(
+                upload_response = client.evaluator_versions.get_upload_endpoint(
+                    version_id=evaluator_version_id,
                     evaluator_id=evaluator_id,
                     filename_to_size={tar_filename: str(tar_size)},
                 )
@@ -322,9 +347,9 @@
                             raise
 
                 # Step 3: Validate upload using SDK
-                client.evaluators.validate_upload(
+                client.evaluator_versions.validate_upload(
+                    version_id=evaluator_version_id,
                     evaluator_id=evaluator_id,
-                    body={},
                 )
                 logger.info("Upload validated successfully")
 

eval_protocol/fireworks_client.py

@@ -0,0 +1,132 @@
+"""
+Consolidated Fireworks client factory.
+
+This module provides a single point of instantiation for the Fireworks SDK client,
+ensuring consistent handling of environment variables and configuration across the
+eval_protocol codebase.
+
+Environment variables:
+    FIREWORKS_API_KEY: API key for authentication (required)
+    FIREWORKS_ACCOUNT_ID: Account ID (optional, can be derived from API key)
+    FIREWORKS_API_BASE: Base URL for the API (default: https://api.fireworks.ai)
+    FIREWORKS_EXTRA_HEADERS: JSON-encoded extra headers to include in requests
+        Example: '{"X-Custom-Header": "value", "X-Another": "another-value"}'
+"""
+
+import json
+import logging
+import os
+from typing import Mapping, Optional
+
+from fireworks import Fireworks
+
+from eval_protocol.auth import (
+    get_fireworks_account_id,
+    get_fireworks_api_base,
+    get_fireworks_api_key,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def get_fireworks_extra_headers() -> Optional[Mapping[str, str]]:
+    """
+    Retrieves extra headers from the FIREWORKS_EXTRA_HEADERS environment variable.
+
+    The value should be a JSON-encoded object mapping header names to values.
+    Example: '{"X-Custom-Header": "value"}'
+
+    Returns:
+            A mapping of header names to values if set and valid, otherwise None.
+    """
+    extra_headers_str = os.environ.get("FIREWORKS_EXTRA_HEADERS")
+    if not extra_headers_str or not extra_headers_str.strip():
+        return None
+
+    try:
+        headers = json.loads(extra_headers_str)
+        if not isinstance(headers, dict):
+            logger.warning(
+                "FIREWORKS_EXTRA_HEADERS must be a JSON object, got %s. Ignoring.",
+                type(headers).__name__,
+            )
+            return None
+        # Validate all keys and values are strings
+        for k, v in headers.items():
+            if not isinstance(k, str) or not isinstance(v, str):
+                logger.warning(
+                    "FIREWORKS_EXTRA_HEADERS contains non-string key or value: %s=%s. Ignoring all extra headers.",
+                    k,
+                    v,
+                )
+                return None
+        logger.debug("Using FIREWORKS_EXTRA_HEADERS: %s", list(headers.keys()))
+        return headers
+    except json.JSONDecodeError as e:
+        logger.warning("Failed to parse FIREWORKS_EXTRA_HEADERS as JSON: %s. Ignoring.", e)
+        return None
+
+
+def create_fireworks_client(
+    *,
+    api_key: Optional[str] = None,
+    account_id: Optional[str] = None,
+    base_url: Optional[str] = None,
+    extra_headers: Optional[Mapping[str, str]] = None,
+) -> Fireworks:
+    """
+    Create a Fireworks client with consistent configuration.
+
+    This factory function centralizes the logic for creating Fireworks clients,
+    ensuring that environment variables are handled consistently across the codebase.
+
+    Resolution order for each parameter:
+            1. Explicit argument passed to this function
+            2. Environment variable (via auth module helpers)
+            3. SDK defaults (for base_url only)
+
+    Args:
+            api_key: Fireworks API key. If not provided, resolves from FIREWORKS_API_KEY.
+            account_id: Fireworks account ID. If not provided, resolves from FIREWORKS_ACCOUNT_ID
+                    or derives from the API key via the verifyApiKey endpoint.
+            base_url: Base URL for the Fireworks API. If not provided, resolves from
+                    FIREWORKS_API_BASE or defaults to https://api.fireworks.ai.
+            extra_headers: Additional headers to include in all requests. If not provided,
+                    resolves from FIREWORKS_EXTRA_HEADERS environment variable (JSON-encoded).
+
+    Returns:
+            A configured Fireworks client instance.
+
+    Raises:
+            fireworks.FireworksError: If api_key is not provided and FIREWORKS_API_KEY
+                    environment variable is not set.
+    """
+    # Resolve parameters from environment if not explicitly provided
+    resolved_api_key = api_key or get_fireworks_api_key()
+    resolved_account_id = account_id or get_fireworks_account_id()
+    resolved_base_url = base_url or get_fireworks_api_base()
+
+    # Merge extra headers: env var headers first, then explicit headers override
+    env_extra_headers = get_fireworks_extra_headers()
+    merged_headers: Optional[Mapping[str, str]] = None
+    if env_extra_headers or extra_headers:
+        merged = {}
+        if env_extra_headers:
+            merged.update(env_extra_headers)
+        if extra_headers:
+            merged.update(extra_headers)
+        merged_headers = merged if merged else None
+
+    logger.debug(
+        "Creating Fireworks client: base_url=%s, account_id=%s, extra_headers=%s",
+        resolved_base_url,
+        resolved_account_id,
+        list(merged_headers.keys()) if merged_headers else None,
+    )
+
+    return Fireworks(
+        api_key=resolved_api_key,
+        account_id=resolved_account_id,
+        base_url=resolved_base_url,
+        default_headers=merged_headers,
+    )