udpate

Author: xzrderek

eval_protocol/cli.py

@@ -429,7 +429,7 @@ def parse_args(args=None):
     rft_parser.add_argument("--wandb-run-id")
     rft_parser.add_argument("--wandb-api-key")
     # Misc
-    rft_parser.add_argument("--rft-job-id", help="Specify an explicit RFT job id")
+    rft_parser.add_argument("--job-id", help="Specify an explicit RFT job id")
     rft_parser.add_argument("--yes", "-y", action="store_true", help="Non-interactive mode")
     rft_parser.add_argument("--dry-run", action="store_true", help="Print planned REST calls without sending")
     rft_parser.add_argument("--force", action="store_true", help="Overwrite existing evaluator with the same ID")

eval_protocol/cli_commands/create_rft.py

@@ -344,8 +344,7 @@ def _poll_evaluator_status(
 
 
 def create_rft_command(args) -> int:
-    # Accept both new and legacy names for backwards-compat
-    evaluator_id: Optional[str] = getattr(args, "evaluator", None) or getattr(args, "evaluator_id", None)
+    evaluator_id: Optional[str] = getattr(args, "evaluator", None)
     non_interactive: bool = bool(getattr(args, "yes", False))
     dry_run: bool = bool(getattr(args, "dry_run", False))
     force: bool = bool(getattr(args, "force", False))
@@ -374,19 +373,19 @@ def create_rft_command(args) -> int:
             print("No evaluation tests found.")
             print("\nHint: Make sure your tests use the @evaluation_test decorator.")
             return 1
-        # Always interactive selection here (no implicit quiet unless --evaluator-id was provided)
+        # Always interactive selection here
         try:
             selected_tests = _prompt_select(tests, non_interactive=non_interactive)
         except Exception:
-            print("Error: Failed to open selector UI. Please pass --evaluator-id or --entry explicitly.")
+            print("Error: Failed to open selector UI. Please pass --evaluator or --entry explicitly.")
             return 1
         if not selected_tests:
             print("No tests selected.")
             return 1
         if len(selected_tests) != 1:
             if non_interactive and len(selected_tests) > 1:
                 print("Error: Multiple evaluation tests found in --yes (non-interactive) mode.")
-                print("       Please pass --evaluator-id or --entry to disambiguate.")
+                print("       Please pass --evaluator or --entry to disambiguate.")
                 try:
                     # Offer candidate evaluator ids for convenience
                     tests = _discover_tests(project_root)
@@ -476,10 +475,10 @@ def create_rft_command(args) -> int:
             # If still unresolved and multiple tests exist, fail fast to avoid uploading unintended evaluators
             if selected_entry is None and len(tests) > 1:
                 print(
-                    f"Error: Multiple evaluation tests found, and the selected evaluator_id {evaluator_id} does not match any discovered test.\n"
-                    "       Please re-run specifying the evaluator id.\n"
+                    f"Error: Multiple evaluation tests found, and the selected evaluator {evaluator_id} does not match any discovered test.\n"
+                    "       Please re-run specifying the evaluator.\n"
                     "       Hints:\n"
-                    "         - eval-protocol create rft --evaluator-id <existing-evaluator-id>\n"
+                    "         - eval-protocol create rft --evaluator <existing-evaluator-id>\n"
                 )
                 return 1
 
@@ -529,8 +528,7 @@ def create_rft_command(args) -> int:
             print(f"Warning: Failed to upload evaluator automatically: {e}")
 
     # Determine dataset id and materialization path
-    # Accept both new and legacy names for dataset
-    dataset_id = getattr(args, "dataset", None) or getattr(args, "dataset_id", None)
+    dataset_id = getattr(args, "dataset", None)
     dataset_jsonl = getattr(args, "dataset_jsonl", None)
     dataset_display_name = getattr(args, "dataset_display_name", None)
     dataset_builder = getattr(args, "dataset_builder", None)  # accepted but unused in simplified flow
@@ -585,7 +583,7 @@ def create_rft_command(args) -> int:
                             print(f"Warning: dataset builder failed: {e}")
         if not dataset_jsonl:
             print(
-                "Error: Could not determine dataset. Provide --dataset-id or --dataset-jsonl, or ensure a JSONL-based data loader or input_dataset is used in your single discovered test."
+                "Error: Could not determine dataset. Provide --dataset or --dataset-jsonl, or ensure a JSONL-based data loader or input_dataset is used in your single discovered test."
             )
             return 1
 
@@ -640,7 +638,7 @@ def create_rft_command(args) -> int:
         ("learningRate", "learning_rate"),
         ("maxContextLength", "max_context_length"),
         ("loraRank", "lora_rank"),
-        ("gradAccumulationSteps", "gradient_accumulation_steps"),
+        ("gradientAccumulationSteps", "gradient_accumulation_steps"),
         ("learningRateWarmupSteps", "learning_rate_warmup_steps"),
         ("acceleratorCount", "accelerator_count"),
         ("region", "region"),
@@ -655,13 +653,24 @@ def create_rft_command(args) -> int:
         ("topP", "top_p"),
         ("topK", "top_k"),
         ("maxTokens", "max_output_tokens"),
-        ("responseCandidatesCount", "response_candidates_count"),
+        ("n", "response_candidates_count"),
     ]:
         val = getattr(args, arg_name, None)
         if val is not None:
             inference_params[key] = val
     if getattr(args, "extra_body", None):
-        inference_params["extraBody"] = args.extra_body
+        extra = getattr(args, "extra_body")
+        if isinstance(extra, (dict, list)):
+            try:
+                inference_params["extraBody"] = json.dumps(extra, ensure_ascii=False)
+            except (TypeError, ValueError) as e:
+                print(f"Error: --extra-body dict/list must be JSON-serializable: {e}")
+                return 1
+        elif isinstance(extra, str):
+            inference_params["extraBody"] = extra
+        else:
+            print("Error: --extra-body must be a JSON string or a JSON-serializable dict/list.")
+            return 1
 
     wandb_config: Optional[Dict[str, Any]] = None
     if getattr(args, "wandb_enabled", False):
@@ -688,7 +697,7 @@ def create_rft_command(args) -> int:
         "outputStats": None,
         "outputMetrics": None,
         "mcpServer": getattr(args, "mcp_server", None),
-        "jobId": getattr(args, "rft_job_id", None),
+        "jobId": getattr(args, "job_id", None),
     }
     # Debug: print minimal summary
     print(f"Prepared RFT job for evaluator '{evaluator_id}' using dataset '{dataset_id}'")

eval_protocol/fireworks_rft.py

@@ -8,6 +8,7 @@
 import uuid
 from pathlib import Path
 from typing import Any, Callable, Dict, Iterable, Optional, Tuple
+from urllib.parse import urlencode
 
 import requests
 
@@ -186,6 +187,14 @@ def create_reinforcement_fine_tuning_job(
     body: Dict[str, Any],
 ) -> Dict[str, Any]:
     url = f"{api_base.rstrip('/')}/v1/accounts/{account_id}/reinforcementFineTuningJobs"
+    # Move optional jobId from body to query parameter if provided
+    job_id = body.get("jobId")
+    if isinstance(job_id, str):
+        job_id = job_id.strip()
+    if job_id:
+        # Remove from body and append as query param
+        body.pop("jobId", None)
+        url = f"{url}?{urlencode({'reinforcementFineTuningJobId': job_id})}"
     headers = {
         "Authorization": f"Bearer {api_key}",
         "Content-Type": "application/json",

tests/test_cli_create_rft_infer.py

@@ -105,7 +105,7 @@ def _fake_create_job(account_id, api_key, api_base, body):
         wandb_run_id="run123",
         wandb_api_key="key123",
         # Unused in body but accepted by parser
-        rft_job_id=None,
+        job_id=None,
         display_name=None,
     )
 
@@ -139,7 +139,7 @@ def _fake_create_job(account_id, api_key, api_base, body):
     assert abs(ip["topP"] - 0.95) < 1e-12
     assert ip["topK"] == 50
     assert ip["maxTokens"] == 4096
-    assert ip["responseCandidatesCount"] == 6
+    assert ip["n"] == 6
     assert ip["extraBody"] == '{"foo":"bar"}'
 
     # W&B mapping
@@ -866,14 +866,23 @@ def raise_for_status(self):
 
     monkeypatch.setattr(cr.requests, "get", lambda *a, **k: _Resp())
 
-    # Capture body
-    captured = {"body": None}
+    # Capture URL and JSON via fireworks layer
+    import eval_protocol.fireworks_rft as fr
 
-    def _fake_create_job(account_id, api_key, api_base, body):
-        captured["body"] = body
-        return {"name": f"accounts/{account_id}/reinforcementFineTuningJobs/xyz"}
+    captured = {"url": None, "json": None}
 
-    monkeypatch.setattr(cr, "create_reinforcement_fine_tuning_job", _fake_create_job)
+    class _RespPost:
+        status_code = 200
+
+        def json(self):
+            return {"name": "accounts/pyroworks-dev/reinforcementFineTuningJobs/xyz"}
+
+    def _fake_post(url, json=None, headers=None, timeout=None):
+        captured["url"] = url
+        captured["json"] = json
+        return _RespPost()
+
+    monkeypatch.setattr(fr.requests, "post", _fake_post)
 
     # Build args via CLI parser to validate flag names
     from eval_protocol.cli import parse_args
@@ -905,15 +914,17 @@ def _fake_create_job(account_id, api_key, api_base, body):
         "0.00003",
         "--lora-rank",
         "16",
+        "--job-id",
+        "custom-job-123",
         "--yes",
     ]
     args, _ = parse_args(argv)
 
     # Execute command
     rc = cr.create_rft_command(args)
     assert rc == 0
-    assert captured["body"] is not None
-    body = captured["body"]
+    assert captured["json"] is not None
+    body = captured["json"]
 
     # Evaluator and dataset resources
     assert body["evaluator"] == "accounts/pyroworks-dev/evaluators/test-livesvgbench-test-svg-combined-evaluation1"
@@ -931,8 +942,11 @@ def _fake_create_job(account_id, api_key, api_base, body):
 
     # Inference params mapping
     ip = body["inferenceParameters"]
-    assert ip["responseCandidatesCount"] == 4
+    assert ip["n"] == 4
     assert ip["maxTokens"] == 32768
 
     # Other top-level
     assert body["chunkSize"] == 50
+    # Job id sent as query param
+    assert captured["url"] is not None and "reinforcementFineTuningJobId=custom-job-123" in captured["url"]
+    assert "jobId" not in body