Skip to content

Commit fb12028

Browse files
xzrderekxzrderek
committed
try skipping if possible
1 parent e6cbe86 commit fb12028

File tree

1 file changed

+110
-62
lines changed

1 file changed

+110
-62
lines changed

‎eval_protocol/cli_commands/create_rft.py‎

Lines changed: 110 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,8 @@ def _build_trimmed_dataset_id(evaluator_id: str) -> str:
321321
if not base:
322322
base = "dataset"
323323
# Ensure first char is a letter
324+
if not base:
325+
base = "dataset"
324326
if not base[0].isalpha():
325327
base = f"eval-{base}"
326328
if len(base) > max_base_len:
@@ -449,76 +451,122 @@ def create_rft_command(args) -> int:
449451
# Resolve evaluator resource name to fully-qualified format required by API
450452
evaluator_resource_name = f"accounts/{account_id}/evaluators/{evaluator_id}"
451453

454+
# Optional short-circuit: if evaluator already exists and not forcing, skip upload path
455+
skip_upload = False
456+
if not force:
457+
try:
458+
headers = {
459+
"Authorization": f"Bearer {api_key}",
460+
"Content-Type": "application/json",
461+
"User-Agent": get_user_agent(),
462+
}
463+
resp = requests.get(f"{api_base}/v1/{evaluator_resource_name}", headers=headers, timeout=10)
464+
if resp.ok:
465+
state = resp.json().get("state", "STATE_UNSPECIFIED")
466+
print(f"✓ Evaluator exists (state: {state}). Skipping upload (use --force to overwrite).")
467+
# Poll for ACTIVE before proceeding
468+
print(f"Waiting for evaluator '{evaluator_id}' to become ACTIVE...")
469+
if not _poll_evaluator_status(
470+
evaluator_resource_name=evaluator_resource_name,
471+
api_key=api_key,
472+
api_base=api_base,
473+
timeout_minutes=10,
474+
):
475+
app_base = _map_api_host_to_app_host(api_base)
476+
evaluator_slug = _extract_terminal_segment(evaluator_id)
477+
dashboard_url = f"{app_base}/dashboard/evaluators/{evaluator_slug}"
478+
print("\n❌ Evaluator is not ready within the timeout period.")
479+
print(f"📊 Please check the evaluator status at: {dashboard_url}")
480+
print(" Wait for it to become ACTIVE, then run 'eval-protocol create rft' again.")
481+
return 1
482+
_save_last_evaluator(project_root, evaluator_id)
483+
skip_upload = True
484+
except requests.exceptions.RequestException:
485+
pass
486+
452487
# Ensure evaluator exists by invoking the upload flow programmatically
453-
try:
454-
from .upload import upload_command
488+
if not skip_upload:
489+
try:
490+
from .upload import upload_command
455491

456-
tests = _discover_tests(project_root)
457-
selected_entry: Optional[str] = None
458-
if len(tests) == 1:
459-
func_name = tests[0].qualname.split(".")[-1]
460-
abs_path = os.path.abspath(tests[0].file_path)
461-
try:
462-
rel = os.path.relpath(abs_path, project_root)
463-
except Exception:
464-
rel = abs_path
465-
selected_entry = f"{rel}::{func_name}"
466-
else:
467-
# Try to match evaluator_id to a discovered test's normalized ID
468-
for t in tests:
469-
func_name = t.qualname.split(".")[-1]
470-
source_file_name = os.path.splitext(os.path.basename(t.file_path))[0]
471-
candidate = _normalize_evaluator_id(f"{source_file_name}-{func_name}")
472-
if candidate == evaluator_id:
473-
abs_path = os.path.abspath(t.file_path)
474-
try:
475-
rel = os.path.relpath(abs_path, project_root)
476-
except Exception:
477-
rel = abs_path
478-
selected_entry = f"{rel}::{func_name}"
479-
break
480-
481-
upload_args = argparse.Namespace(
482-
path=project_root,
483-
entry=selected_entry,
484-
id=evaluator_id,
485-
display_name=None,
486-
description=None,
487-
force=force, # Pass through the --force flag
488-
yes=True,
489-
env_file=None, # Add the new env_file parameter
490-
)
492+
tests = _discover_tests(project_root)
493+
selected_entry: Optional[str] = None
494+
if len(tests) == 1:
495+
func_name = tests[0].qualname.split(".")[-1]
496+
abs_path = os.path.abspath(tests[0].file_path)
497+
try:
498+
rel = os.path.relpath(abs_path, project_root)
499+
except Exception:
500+
rel = abs_path
501+
selected_entry = f"{rel}::{func_name}"
502+
else:
503+
# Try to match evaluator_id to a discovered test's normalized ID
504+
for t in tests:
505+
func_name = t.qualname.split(".")[-1]
506+
source_file_name = os.path.splitext(os.path.basename(t.file_path))[0]
507+
candidate = _normalize_evaluator_id(f"{source_file_name}-{func_name}")
508+
if candidate == evaluator_id:
509+
abs_path = os.path.abspath(t.file_path)
510+
try:
511+
rel = os.path.relpath(abs_path, project_root)
512+
except Exception:
513+
rel = abs_path
514+
selected_entry = f"{rel}::{func_name}"
515+
break
516+
# If still unresolved and multiple tests exist, fail fast to avoid uploading unintended evaluators
517+
if selected_entry is None:
518+
print(
519+
f"Error: Multiple evaluation tests found, and the selected evaluator_id {evaluator_id} does not match any discovered test.\n"
520+
" Please re-run specifying the evaluator id.\n"
521+
" Hints:\n"
522+
" - eval-protocol create rft --evaluator-id <existing-evaluator-id>\n"
523+
)
524+
return 1
525+
526+
upload_args = argparse.Namespace(
527+
path=project_root,
528+
entry=selected_entry,
529+
id=evaluator_id,
530+
display_name=None,
531+
description=None,
532+
force=force, # Pass through the --force flag
533+
yes=True,
534+
env_file=None, # Add the new env_file parameter
535+
)
491536

492-
if force:
493-
print(f"🔄 Force flag enabled - will overwrite existing evaluator '{evaluator_id}'")
537+
if force:
538+
print(f"🔄 Force flag enabled - will overwrite existing evaluator '{evaluator_id}'")
494539

495-
rc = upload_command(upload_args)
496-
if rc == 0:
497-
print(f"✓ Uploaded/ensured evaluator: {evaluator_id}")
540+
rc = upload_command(upload_args)
541+
if rc == 0:
542+
print(f"✓ Uploaded/ensured evaluator: {evaluator_id}")
498543

499-
# Poll for evaluator status
500-
print(f"Waiting for evaluator '{evaluator_id}' to become ACTIVE...")
501-
is_active = _poll_evaluator_status(
502-
evaluator_resource_name=evaluator_resource_name, api_key=api_key, api_base=api_base, timeout_minutes=10
503-
)
544+
# Poll for evaluator status
545+
print(f"Waiting for evaluator '{evaluator_id}' to become ACTIVE...")
546+
is_active = _poll_evaluator_status(
547+
evaluator_resource_name=evaluator_resource_name,
548+
api_key=api_key,
549+
api_base=api_base,
550+
timeout_minutes=10,
551+
)
504552

505-
if not is_active:
506-
# Print helpful message with dashboard link
507-
app_base = _map_api_host_to_app_host(api_base)
508-
evaluator_slug = _extract_terminal_segment(evaluator_id)
509-
dashboard_url = f"{app_base}/dashboard/evaluators/{evaluator_slug}"
553+
if not is_active:
554+
# Print helpful message with dashboard link
555+
app_base = _map_api_host_to_app_host(api_base)
556+
evaluator_slug = _extract_terminal_segment(evaluator_id)
557+
dashboard_url = f"{app_base}/dashboard/evaluators/{evaluator_slug}"
510558

511-
print("\n❌ Evaluator is not ready within the timeout period.")
512-
print(f"📊 Please check the evaluator status at: {dashboard_url}")
513-
print(" Wait for it to become ACTIVE, then run 'eval-protocol create rft' again.")
514-
return 1
559+
print("\n❌ Evaluator is not ready within the timeout period.")
560+
print(f"📊 Please check the evaluator status at: {dashboard_url}")
561+
print(" Wait for it to become ACTIVE, then run 'eval-protocol create rft' again.")
562+
return 1
563+
else:
564+
# Only persist last-used evaluator after successful ensure + ACTIVE
565+
_save_last_evaluator(project_root, evaluator_id)
515566
else:
516-
# Only persist last-used evaluator after successful ensure + ACTIVE
517-
_save_last_evaluator(project_root, evaluator_id)
518-
else:
519-
print("Warning: Evaluator upload did not complete successfully; proceeding to RFT creation.")
520-
except Exception as e:
521-
print(f"Warning: Failed to upload evaluator automatically: {e}")
567+
print("Warning: Evaluator upload did not complete successfully; proceeding to RFT creation.")
568+
except Exception as e:
569+
print(f"Warning: Failed to upload evaluator automatically: {e}")
522570

523571
# Determine dataset id and materialization path
524572
dataset_id = getattr(args, "dataset_id", None)

0 commit comments

Comments
 (0)