Implement Evaluator Versions

.gitignore

@@ -243,3 +243,5 @@ package.json
 tau2-bench
 *.err
 eval-protocol
+
+.vscode/launch.json

eval_protocol/adapters/fireworks_tracing.py

@@ -253,17 +253,24 @@ def __init__(
         project_id: Optional[str] = None,
         base_url: str = "https://tracing.fireworks.ai",
         timeout: int = 300,
+        api_key: Optional[str] = None,
     ):
         """Initialize the Fireworks Tracing adapter.
 
         Args:
             project_id: Optional project ID. If not provided, uses the default project configured on the server.
             base_url: The base URL of the tracing proxy (default: https://tracing.fireworks.ai)
             timeout: Request timeout in seconds (default: 300)
+            api_key: Optional API key. If not provided, falls back to FIREWORKS_API_KEY environment variable.
         """
         self.project_id = project_id
         self.base_url = base_url.rstrip("/")
         self.timeout = timeout
+        self._api_key = api_key
+
+    def _get_api_key(self) -> Optional[str]:
+        """Get the API key, preferring instance-level key over environment variable."""
+        return self._api_key or os.environ.get("FIREWORKS_API_KEY")
 
     def search_logs(self, tags: List[str], limit: int = 100, hours_back: int = 24) -> List[Dict[str, Any]]:
         """Fetch logs from Fireworks tracing gateway /logs endpoint.
@@ -276,7 +283,7 @@ def search_logs(self, tags: List[str], limit: int = 100, hours_back: int = 24) -
         from ..common_utils import get_user_agent
 
         headers = {
-            "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}",
+            "Authorization": f"Bearer {self._get_api_key()}",
             "User-Agent": get_user_agent(),
         }
         params: Dict[str, Any] = {"tags": tags, "limit": limit, "hours_back": hours_back, "program": "eval_protocol"}
@@ -407,7 +414,7 @@ def get_evaluation_rows(
         from ..common_utils import get_user_agent
 
         headers = {
-            "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}",
+            "Authorization": f"Bearer {self._get_api_key()}",
             "User-Agent": get_user_agent(),
         }
 

eval_protocol/auth.py

@@ -3,9 +3,30 @@
 from typing import Optional
 
 import requests
+from dotenv import find_dotenv, load_dotenv
 
 logger = logging.getLogger(__name__)
 
+# --- Load .env files ---
+# Attempt to load .env.dev first, then .env as a fallback.
+# This happens when the module is imported.
+# We use override=False (default) so that existing environment variables
+# (e.g., set in the shell) are NOT overridden by .env files.
+_ENV_DEV_PATH = find_dotenv(filename=".env.dev", raise_error_if_not_found=False, usecwd=True)
+if _ENV_DEV_PATH:
+    load_dotenv(dotenv_path=_ENV_DEV_PATH, override=False)
+    logger.debug(f"eval_protocol.auth: Loaded environment variables from: {_ENV_DEV_PATH}")
+else:
+    _ENV_PATH = find_dotenv(filename=".env", raise_error_if_not_found=False, usecwd=True)
+    if _ENV_PATH:
+        load_dotenv(dotenv_path=_ENV_PATH, override=False)
+        logger.debug(f"eval_protocol.auth: Loaded environment variables from: {_ENV_PATH}")
+    else:
+        logger.debug(
+            "eval_protocol.auth: No .env.dev or .env file found. Relying on shell/existing environment variables."
+        )
+# --- End .env loading ---
+
 
 def get_fireworks_api_key() -> Optional[str]:
     """
@@ -73,6 +94,8 @@ def verify_api_key_and_get_account_id(
     Args:
         api_key: Optional explicit API key. When None, resolves via get_fireworks_api_key().
         api_base: Optional explicit API base. When None, resolves via get_fireworks_api_base().
+            If api_base is api.fireworks.ai, it is used directly. Otherwise, defaults to
+            dev.api.fireworks.ai for the verification call.
 
     Returns:
         The resolved account id if verification succeeds and the header is present; otherwise None.
@@ -81,7 +104,12 @@ def verify_api_key_and_get_account_id(
         resolved_key = api_key or get_fireworks_api_key()
         if not resolved_key:
             return None
-        resolved_base = api_base or get_fireworks_api_base()
+        provided_base = api_base or get_fireworks_api_base()
+        # Use api.fireworks.ai if explicitly provided, otherwise fall back to dev
+        if "api.fireworks.ai" in provided_base:
+            resolved_base = provided_base
+        else:
+            resolved_base = "https://dev.api.fireworks.ai"
 
         from .common_utils import get_user_agent
 

eval_protocol/cli.py

@@ -8,10 +8,9 @@
 import sys
 from pathlib import Path
 
-from fireworks import Fireworks
-
 from .cli_commands.common import setup_logging
 from .cli_commands.utils import add_args_from_callable_signature
+from .fireworks_client import create_fireworks_client
 
 logger = logging.getLogger(__name__)
 
@@ -81,14 +80,9 @@ def _configure_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParse
         "--env-file",
         help="Path to .env file containing secrets to upload (default: .env in current directory)",
     )
-    upload_parser.add_argument(
-        "--force",
-        action="store_true",
-        help="Overwrite existing evaluator with the same ID",
-    )
 
     # Auto-generate flags from SDK Fireworks().evaluators.create() signature
-    create_evaluator_fn = Fireworks().evaluators.create
+    create_evaluator_fn = create_fireworks_client().evaluators.create
 
     upload_skip_fields = {
         "__top_level__": {
@@ -137,7 +131,6 @@ def _configure_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParse
 
     rft_parser.add_argument("--yes", "-y", action="store_true", help="Non-interactive mode")
     rft_parser.add_argument("--dry-run", action="store_true", help="Print planned SDK call without sending")
-    rft_parser.add_argument("--force", action="store_true", help="Overwrite existing evaluator with the same ID")
     rft_parser.add_argument("--skip-validation", action="store_true", help="Skip local dataset/evaluator validation")
     rft_parser.add_argument(
         "--ignore-docker",
@@ -198,7 +191,7 @@ def _configure_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParse
         "loss_config.method": "RL loss method for underlying trainers. One of {grpo,dapo}.",
     }
 
-    create_rft_job_fn = Fireworks().reinforcement_fine_tuning_jobs.create
+    create_rft_job_fn = create_fireworks_client().reinforcement_fine_tuning_jobs.create
 
     add_args_from_callable_signature(
         rft_parser,

eval_protocol/cli_commands/create_rft.py

@@ -12,6 +12,7 @@
 from pydantic import ValidationError
 
 from ..auth import get_fireworks_api_base, get_fireworks_api_key
+from ..fireworks_client import create_fireworks_client
 from ..common_utils import get_user_agent, load_jsonl
 from ..fireworks_rft import (
     create_dataset_from_jsonl,
@@ -35,8 +36,6 @@
 )
 from .local_test import run_evaluator_test
 
-from fireworks import Fireworks
-
 
 def _extract_dataset_adapter(
     test_file_path: str, test_func_name: str
@@ -568,37 +567,35 @@ def _upload_and_ensure_evaluator(
     evaluator_resource_name: str,
     api_key: str,
     api_base: str,
-    force: bool,
 ) -> bool:
     """Ensure the evaluator exists and is ACTIVE, uploading it if needed."""
-    # Optional short-circuit: if evaluator already exists and not forcing, skip upload path
-    if not force:
-        try:
-            headers = {
-                "Authorization": f"Bearer {api_key}",
-                "Content-Type": "application/json",
-                "User-Agent": get_user_agent(),
-            }
-            resp = requests.get(f"{api_base}/v1/{evaluator_resource_name}", headers=headers, timeout=10)
-            if resp.ok:
-                state = resp.json().get("state", "STATE_UNSPECIFIED")
-                print(f"✓ Evaluator exists (state: {state}). Skipping upload (use --force to overwrite).")
-                # Poll for ACTIVE before proceeding
-                print(f"Waiting for evaluator '{evaluator_id}' to become ACTIVE...")
-                if not _poll_evaluator_status(
-                    evaluator_resource_name=evaluator_resource_name,
-                    api_key=api_key,
-                    api_base=api_base,
-                    timeout_minutes=10,
-                ):
-                    dashboard_url = _build_evaluator_dashboard_url(evaluator_id)
-                    print("\n❌ Evaluator is not ready within the timeout period.")
-                    print(f"📊 Please check the evaluator status at: {dashboard_url}")
-                    print("   Wait for it to become ACTIVE, then run 'eval-protocol create rft' again.")
-                    return False
-                return True
-        except requests.exceptions.RequestException:
-            pass
+    # Check if evaluator already exists
+    try:
+        headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+            "User-Agent": get_user_agent(),
+        }
+        resp = requests.get(f"{api_base}/v1/{evaluator_resource_name}", headers=headers, timeout=10)
+        if resp.ok:
+            state = resp.json().get("state", "STATE_UNSPECIFIED")
+            print(f"✓ Evaluator exists (state: {state}). Skipping upload.")
+            # Poll for ACTIVE before proceeding
+            print(f"Waiting for evaluator '{evaluator_id}' to become ACTIVE...")
+            if not _poll_evaluator_status(
+                evaluator_resource_name=evaluator_resource_name,
+                api_key=api_key,
+                api_base=api_base,
+                timeout_minutes=10,
+            ):
+                dashboard_url = _build_evaluator_dashboard_url(evaluator_id)
+                print("\n❌ Evaluator is not ready within the timeout period.")
+                print(f"📊 Please check the evaluator status at: {dashboard_url}")
+                print("   Wait for it to become ACTIVE, then run 'eval-protocol create rft' again.")
+                return False
+            return True
+    except requests.exceptions.RequestException:
+        pass
 
     # Ensure evaluator exists by invoking the upload flow programmatically
     try:
@@ -623,14 +620,10 @@ def _upload_and_ensure_evaluator(
             id=evaluator_id,
             display_name=None,
             description=None,
-            force=force,  # Pass through the --force flag
             yes=True,
-            env_file=None,  # Add the new env_file parameter
+            env_file=None,
         )
 
-        if force:
-            print(f"🔄 Force flag enabled - will overwrite existing evaluator '{evaluator_id}'")
-
         rc = upload_command(upload_args)
         if rc == 0:
             print(f"✓ Uploaded/ensured evaluator: {evaluator_id}")
@@ -672,7 +665,7 @@ def _create_rft_job(
 ) -> int:
     """Build and submit the RFT job request (via Fireworks SDK)."""
 
-    signature = inspect.signature(Fireworks().reinforcement_fine_tuning_jobs.create)
+    signature = inspect.signature(create_fireworks_client().reinforcement_fine_tuning_jobs.create)
 
     # Build top-level SDK kwargs
     sdk_kwargs: Dict[str, Any] = {
@@ -711,7 +704,7 @@ def _create_rft_job(
         return 0
 
     try:
-        fw: Fireworks = Fireworks(api_key=api_key, base_url=api_base)
+        fw: Fireworks = create_fireworks_client(api_key=api_key, base_url=api_base)
         job: ReinforcementFineTuningJob = fw.reinforcement_fine_tuning_jobs.create(account_id=account_id, **sdk_kwargs)
         job_name = job.name
         print(f"\n✅ Created Reinforcement Fine-tuning Job: {job_name}")
@@ -739,7 +732,6 @@ def create_rft_command(args) -> int:
     evaluator_arg: Optional[str] = getattr(args, "evaluator", None)
     non_interactive: bool = bool(getattr(args, "yes", False))
     dry_run: bool = bool(getattr(args, "dry_run", False))
-    force: bool = bool(getattr(args, "force", False))
     skip_validation: bool = bool(getattr(args, "skip_validation", False))
     ignore_docker: bool = bool(getattr(args, "ignore_docker", False))
     docker_build_extra: str = getattr(args, "docker_build_extra", "") or ""
@@ -817,7 +809,6 @@ def create_rft_command(args) -> int:
         evaluator_resource_name=evaluator_resource_name,
         api_key=api_key,
         api_base=api_base,
-        force=force,
     ):
         return 1
 

eval_protocol/cli_commands/upload.py

@@ -289,7 +289,6 @@ def upload_command(args: argparse.Namespace) -> int:
     base_id = getattr(args, "id", None)
     display_name = getattr(args, "display_name", None)
     description = getattr(args, "description", None)
-    force = bool(getattr(args, "force", False))
     env_file = getattr(args, "env_file", None)
 
     # Load secrets from .env file and ensure they're available on Fireworks
@@ -382,7 +381,6 @@ def upload_command(args: argparse.Namespace) -> int:
                 evaluator_id=evaluator_id,
                 display_name=display_name or evaluator_id,
                 description=description or f"Evaluator for {qualname}",
-                force=force,
                 entry_point=entry_point,
             )
             name = result.get("name", evaluator_id) if isinstance(result, dict) else evaluator_id