fix bug

Author: xzrderek

eval_protocol/cli_commands/create_rft.py

@@ -20,7 +20,7 @@
     create_dataset_from_jsonl,
     create_reinforcement_fine_tuning_job,
 )
-from .upload import _discover_tests, _normalize_evaluator_id, _resolve_entry_to_qual_and_source
+from .upload import _discover_tests, _normalize_evaluator_id, _prompt_select
 
 
 def _ensure_account_id() -> Optional[str]:
@@ -331,7 +331,6 @@ def create_rft_command(args) -> int:
 
     # Resolve evaluator id/entry if omitted (reuse upload's selector flow)
     project_root = os.getcwd()
-    preselected_entry: Optional[str] = None
     if not evaluator_id:
         print("Scanning for evaluation tests...")
         tests = _discover_tests(project_root)
@@ -341,9 +340,7 @@ def create_rft_command(args) -> int:
             return 1
         # Always interactive selection here (no implicit quiet unless --evaluator-id was provided)
         try:
-            from .upload import _prompt_select  # reuse the same selector UX as 'upload'
-
-            selected_tests = _prompt_select(tests, non_interactive=False)
+            selected_tests = _prompt_select(tests, non_interactive=non_interactive)
         except Exception:
             print("Error: Failed to open selector UI. Please pass --evaluator-id or --entry explicitly.")
             return 1
@@ -355,12 +352,6 @@ def create_rft_command(args) -> int:
             return 1
         chosen = selected_tests[0]
         func_name = chosen.qualname.split(".")[-1]
-        abs_path = os.path.abspath(chosen.file_path)
-        try:
-            rel = os.path.relpath(abs_path, project_root)
-        except Exception:
-            rel = abs_path
-        preselected_entry = f"{rel}::{func_name}"
         source_file_name = os.path.splitext(os.path.basename(chosen.file_path))[0]
         evaluator_id = _normalize_evaluator_id(f"{source_file_name}-{func_name}")
     # Resolve evaluator resource name to fully-qualified format required by API

tests/test_cli_create_rft_infer.py

@@ -86,7 +86,7 @@ def _fake_create_dataset_from_jsonl(account_id, api_key, api_base, dataset_id, d
 
     # Assert dataset id derived from selected test: metric-test_single
     assert captured["dataset_id"] is not None
-    assert captured["dataset_id"].startswith("metric-test-single-dataset-")
+    assert captured["dataset_id"].startswith("test-single-test-single-dataset-")
 
 
 def test_create_rft_passes_matching_evaluator_id_and_entry_with_multiple_tests(tmp_path, monkeypatch):
@@ -184,3 +184,205 @@ def _fake_create_dataset_from_jsonl(account_id, api_key, api_base, dataset_id, d
         + "-dataset-"
     )
     assert captured["dataset_id"].startswith(expected_prefix)
+
+
+def test_create_rft_interactive_selector_single_test(tmp_path, monkeypatch):
+    # Setup project
+    project = tmp_path / "proj"
+    project.mkdir()
+    monkeypatch.chdir(project)
+
+    # Single discovered test
+    test_file = project / "metric" / "test_one.py"
+    test_file.parent.mkdir(parents=True, exist_ok=True)
+    test_file.write_text("# one", encoding="utf-8")
+    single_disc = SimpleNamespace(qualname="metric.test_one", file_path=str(test_file))
+    monkeypatch.setattr(cr, "_discover_tests", lambda cwd: [single_disc])
+
+    # Environment
+    monkeypatch.setenv("FIREWORKS_API_KEY", "fw_dummy")
+    monkeypatch.setenv("FIREWORKS_ACCOUNT_ID", "acct123")
+    monkeypatch.setenv("FIREWORKS_API_BASE", "https://api.fireworks.ai")
+
+    # Stub selector to return the single test; stub upload and polling
+    import eval_protocol.cli_commands.upload as upload_mod
+
+    monkeypatch.setattr(upload_mod, "_prompt_select", lambda tests, non_interactive=False: tests[:1])
+    captured = {"id": None, "entry": None, "dataset_id": None}
+
+    def _fake_upload(ns):
+        captured["id"] = getattr(ns, "id", None)
+        captured["entry"] = getattr(ns, "entry", None)
+        return 0
+
+    monkeypatch.setattr(upload_mod, "upload_command", _fake_upload)
+    monkeypatch.setattr(cr, "_poll_evaluator_status", lambda **kwargs: True)
+
+    # Provide dataset jsonl
+    ds_path = project / "metric" / "dataset.jsonl"
+    ds_path.write_text('{"input":"x"}\n', encoding="utf-8")
+    monkeypatch.setattr(
+        cr,
+        "create_dataset_from_jsonl",
+        lambda account_id, api_key, api_base, dataset_id, display_name, jsonl_path: (
+            dataset_id,
+            {"name": f"accounts/{account_id}/datasets/{dataset_id}"},
+        ),
+    )
+    monkeypatch.setattr(cr, "create_reinforcement_fine_tuning_job", lambda *a, **k: {"name": "jobs/123"})
+
+    # Run without evaluator_id; use --yes so selector returns tests directly (no UI)
+    import argparse
+
+    args = argparse.Namespace(
+        evaluator_id=None,
+        yes=True,
+        dry_run=False,
+        force=False,
+        env_file=None,
+        dataset_id=None,
+        dataset_jsonl=str(ds_path),
+        dataset_display_name=None,
+        dataset_builder=None,
+        base_model=None,
+        warm_start_from="accounts/acct123/models/ft-abc123",
+        output_model=None,
+        n=None,
+        max_tokens=None,
+        learning_rate=None,
+        batch_size=None,
+        epochs=None,
+        lora_rank=None,
+        max_context_length=None,
+        chunk_size=None,
+        eval_auto_carveout=None,
+    )
+
+    rc = cr.create_rft_command(args)
+    assert rc == 0
+    assert captured["id"] is not None
+    assert captured["entry"] is not None and captured["entry"].endswith("test_one.py::test_one")
+
+
+def test_create_rft_quiet_existing_evaluator_skips_upload(tmp_path, monkeypatch):
+    project = tmp_path / "proj"
+    project.mkdir()
+    monkeypatch.chdir(project)
+
+    # Env
+    monkeypatch.setenv("FIREWORKS_API_KEY", "fw_dummy")
+    monkeypatch.setenv("FIREWORKS_ACCOUNT_ID", "acct123")
+    monkeypatch.setenv("FIREWORKS_API_BASE", "https://api.fireworks.ai")
+
+    # Mock evaluator exists and is ACTIVE
+    class _Resp:
+        ok = True
+
+        def json(self):
+            return {"state": "ACTIVE"}
+
+        def raise_for_status(self):
+            return None
+
+    monkeypatch.setattr(cr.requests, "get", lambda *a, **k: _Resp())
+
+    # Provide dataset via --dataset-jsonl so no test discovery needed
+    ds_path = project / "dataset.jsonl"
+    ds_path.write_text('{"input":"x"}\n', encoding="utf-8")
+    monkeypatch.setattr(
+        cr,
+        "create_dataset_from_jsonl",
+        lambda account_id, api_key, api_base, dataset_id, display_name, jsonl_path: (
+            dataset_id,
+            {"name": f"accounts/{account_id}/datasets/{dataset_id}"},
+        ),
+    )
+    monkeypatch.setattr(cr, "create_reinforcement_fine_tuning_job", lambda *a, **k: {"name": "jobs/123"})
+
+    import argparse
+
+    args = argparse.Namespace(
+        evaluator_id="some-eval",
+        yes=True,
+        dry_run=False,
+        force=False,
+        env_file=None,
+        dataset_id=None,
+        dataset_jsonl=str(ds_path),
+        dataset_display_name=None,
+        dataset_builder=None,
+        base_model=None,
+        warm_start_from="accounts/acct123/models/ft-abc123",
+        output_model=None,
+        n=None,
+        max_tokens=None,
+        learning_rate=None,
+        batch_size=None,
+        epochs=None,
+        lora_rank=None,
+        max_context_length=None,
+        chunk_size=None,
+        eval_auto_carveout=None,
+    )
+
+    rc = cr.create_rft_command(args)
+    assert rc == 0
+
+
+def test_create_rft_quiet_new_evaluator_ambiguous_without_entry_errors(tmp_path, monkeypatch):
+    project = tmp_path / "proj"
+    project.mkdir()
+    monkeypatch.chdir(project)
+
+    # Env
+    monkeypatch.setenv("FIREWORKS_API_KEY", "fw_dummy")
+    monkeypatch.setenv("FIREWORKS_ACCOUNT_ID", "acct123")
+    monkeypatch.setenv("FIREWORKS_API_BASE", "https://api.fireworks.ai")
+
+    # Evaluator does not exist (force path into upload section)
+    def _raise(*a, **k):
+        raise requests.exceptions.RequestException("nope")
+
+    import requests
+
+    monkeypatch.setattr(cr.requests, "get", _raise)
+
+    # Two discovered tests (ambiguous)
+    f1 = project / "a.py"
+    f2 = project / "b.py"
+    f1.write_text("# a", encoding="utf-8")
+    f2.write_text("# b", encoding="utf-8")
+    d1 = SimpleNamespace(qualname="a.test_one", file_path=str(f1))
+    d2 = SimpleNamespace(qualname="b.test_two", file_path=str(f2))
+    monkeypatch.setattr(cr, "_discover_tests", lambda cwd: [d1, d2])
+
+    import argparse
+
+    args = argparse.Namespace(
+        evaluator_id="some-eval",
+        yes=True,
+        dry_run=False,
+        force=False,
+        env_file=None,
+        dataset_id=None,
+        dataset_jsonl=str(project / "dataset.jsonl"),
+        dataset_display_name=None,
+        dataset_builder=None,
+        base_model=None,
+        warm_start_from="accounts/acct123/models/ft-abc123",
+        output_model=None,
+        n=None,
+        max_tokens=None,
+        learning_rate=None,
+        batch_size=None,
+        epochs=None,
+        lora_rank=None,
+        max_context_length=None,
+        chunk_size=None,
+        eval_auto_carveout=None,
+    )
+    # create the dataset file so we don't fail earlier
+    (project / "dataset.jsonl").write_text('{"input":"x"}\n', encoding="utf-8")
+
+    rc = cr.create_rft_command(args)
+    assert rc == 1