Skip to content

Commit 2030e3d

Browse files
xzrderekxzrderek
authored
More granular API or account ID missing (#133)
1 parent 58dd9d2 commit 2030e3d

File tree

1 file changed

+101
-88
lines changed

1 file changed

+101
-88
lines changed

‎eval_protocol/pytest/evaluation_test.py‎

Lines changed: 101 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -331,99 +331,112 @@ def get_auth_value(key):
331331
fireworks_api_key = get_auth_value("FIREWORKS_API_KEY")
332332
fireworks_account_id = get_auth_value("FIREWORKS_ACCOUNT_ID")
333333

334-
if fireworks_api_key and fireworks_account_id:
335-
headers = {"Authorization": f"Bearer {fireworks_api_key}", "Content-Type": "application/json"}
336-
337-
# Make dataset first
338-
dataset_url = f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/datasets"
339-
340-
dataset_payload = {
341-
"dataset": {
342-
"displayName": dataset_name,
343-
"evalProtocol": {},
344-
"format": "FORMAT_UNSPECIFIED",
345-
"exampleCount": f"{len(exp_rows)}",
346-
},
347-
"datasetId": dataset_name,
348-
}
349-
350-
dataset_response = requests.post(dataset_url, json=dataset_payload, headers=headers)
351-
352-
# Skip if dataset creation failed
353-
if dataset_response.status_code not in [200, 201]:
354-
_store_experiment_link(
355-
experiment_id,
356-
f"Dataset creation failed: {dataset_response.status_code} {dataset_response.text}",
357-
"failure",
358-
)
359-
continue
334+
if not (fireworks_api_key and fireworks_account_id):
335+
_store_experiment_link(
336+
experiment_id,
337+
"No Fireworks API key AND account ID found",
338+
"failure",
339+
)
340+
continue
341+
elif not fireworks_api_key:
342+
_store_experiment_link(
343+
experiment_id,
344+
"No Fireworks API key found",
345+
"failure",
346+
)
347+
continue
348+
elif not fireworks_account_id:
349+
_store_experiment_link(
350+
experiment_id,
351+
"No Fireworks account ID found",
352+
"failure",
353+
)
354+
continue
355+
356+
headers = {"Authorization": f"Bearer {fireworks_api_key}", "Content-Type": "application/json"}
357+
358+
# Make dataset first
359+
dataset_url = f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/datasets"
360+
361+
dataset_payload = {
362+
"dataset": {
363+
"displayName": dataset_name,
364+
"evalProtocol": {},
365+
"format": "FORMAT_UNSPECIFIED",
366+
"exampleCount": f"{len(exp_rows)}",
367+
},
368+
"datasetId": dataset_name,
369+
}
370+
371+
dataset_response = requests.post(dataset_url, json=dataset_payload, headers=headers)
372+
373+
# Skip if dataset creation failed
374+
if dataset_response.status_code not in [200, 201]:
375+
_store_experiment_link(
376+
experiment_id,
377+
f"Dataset creation failed: {dataset_response.status_code} {dataset_response.text}",
378+
"failure",
379+
)
380+
continue
360381

361-
dataset_data = dataset_response.json()
362-
dataset_id = dataset_data.get("datasetId", dataset_name)
382+
dataset_data = dataset_response.json()
383+
dataset_id = dataset_data.get("datasetId", dataset_name)
363384

364-
# Upload the JSONL file content
365-
upload_url = (
366-
f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/datasets/{dataset_id}:upload"
385+
# Upload the JSONL file content
386+
upload_url = (
387+
f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/datasets/{dataset_id}:upload"
388+
)
389+
upload_headers = {"Authorization": f"Bearer {fireworks_api_key}"}
390+
391+
with open(exp_file, "rb") as f:
392+
files = {"file": f}
393+
upload_response = requests.post(upload_url, files=files, headers=upload_headers)
394+
395+
# Skip if upload failed
396+
if upload_response.status_code not in [200, 201]:
397+
_store_experiment_link(
398+
experiment_id,
399+
f"File upload failed: {upload_response.status_code} {upload_response.text}",
400+
"failure",
367401
)
368-
upload_headers = {"Authorization": f"Bearer {fireworks_api_key}"}
369-
370-
with open(exp_file, "rb") as f:
371-
files = {"file": f}
372-
upload_response = requests.post(upload_url, files=files, headers=upload_headers)
373-
374-
# Skip if upload failed
375-
if upload_response.status_code not in [200, 201]:
376-
_store_experiment_link(
377-
experiment_id,
378-
f"File upload failed: {upload_response.status_code} {upload_response.text}",
379-
"failure",
380-
)
381-
continue
382-
383-
# Create evaluation job (optional - don't skip experiment if this fails)
384-
eval_job_url = f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/evaluationJobs"
385-
# Truncate job ID to fit 63 character limit
386-
job_id_base = f"{dataset_name}-job"
387-
if len(job_id_base) > 63:
388-
# Keep the "-job" suffix and truncate the dataset_name part
389-
max_dataset_name_len = 63 - 4 # 4 = len("-job")
390-
truncated_dataset_name = dataset_name[:max_dataset_name_len]
391-
job_id_base = f"{truncated_dataset_name}-job"
392-
393-
eval_job_payload = {
394-
"evaluationJobId": job_id_base,
395-
"evaluationJob": {
396-
"evaluator": f"accounts/{fireworks_account_id}/evaluators/dummy",
397-
"inputDataset": f"accounts/{fireworks_account_id}/datasets/dummy",
398-
"outputDataset": f"accounts/{fireworks_account_id}/datasets/{dataset_id}",
399-
},
400-
}
401-
402-
eval_response = requests.post(eval_job_url, json=eval_job_payload, headers=headers)
403-
404-
if eval_response.status_code in [200, 201]:
405-
eval_job_data = eval_response.json()
406-
job_id = eval_job_data.get("evaluationJobId", job_id_base)
407-
408-
_store_experiment_link(
409-
experiment_id,
410-
f"https://app.fireworks.ai/dashboard/evaluation-jobs/{job_id}",
411-
"success",
412-
)
413-
else:
414-
_store_experiment_link(
415-
experiment_id,
416-
f"Job creation failed: {eval_response.status_code} {eval_response.text}",
417-
"failure",
418-
)
402+
continue
419403

404+
# Create evaluation job (optional - don't skip experiment if this fails)
405+
eval_job_url = f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/evaluationJobs"
406+
# Truncate job ID to fit 63 character limit
407+
job_id_base = f"{dataset_name}-job"
408+
if len(job_id_base) > 63:
409+
# Keep the "-job" suffix and truncate the dataset_name part
410+
max_dataset_name_len = 63 - 4 # 4 = len("-job")
411+
truncated_dataset_name = dataset_name[:max_dataset_name_len]
412+
job_id_base = f"{truncated_dataset_name}-job"
413+
414+
eval_job_payload = {
415+
"evaluationJobId": job_id_base,
416+
"evaluationJob": {
417+
"evaluator": f"accounts/{fireworks_account_id}/evaluators/dummy",
418+
"inputDataset": f"accounts/{fireworks_account_id}/datasets/dummy",
419+
"outputDataset": f"accounts/{fireworks_account_id}/datasets/{dataset_id}",
420+
},
421+
}
422+
423+
eval_response = requests.post(eval_job_url, json=eval_job_payload, headers=headers)
424+
425+
if eval_response.status_code in [200, 201]:
426+
eval_job_data = eval_response.json()
427+
job_id = eval_job_data.get("evaluationJobId", job_id_base)
428+
429+
_store_experiment_link(
430+
experiment_id,
431+
f"https://app.fireworks.ai/dashboard/evaluation-jobs/{job_id}",
432+
"success",
433+
)
420434
else:
421-
# Store failure for missing credentials for all experiments
422-
for experiment_id, exp_rows in experiments.items():
423-
if experiment_id and exp_rows:
424-
_store_experiment_link(
425-
experiment_id, "No Fireworks API key or account ID found", "failure"
426-
)
435+
_store_experiment_link(
436+
experiment_id,
437+
f"Job creation failed: {eval_response.status_code} {eval_response.text}",
438+
"failure",
439+
)
427440

428441
except Exception as e:
429442
# Do not fail evaluation if experiment JSONL writing fails

0 commit comments

Comments
 (0)