Skip to content

Commit a766d40

Browse files
author
Shrey Modi
committed
failure modes
1 parent aeb8863 commit a766d40

File tree

2 files changed

+119
-97
lines changed

2 files changed

+119
-97
lines changed

‎eval_protocol/cli_commands/upload.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,14 @@ def _extract_param_info_from_marks(obj: Any) -> tuple[bool, int, list[str]]:
9797
(has_parametrize, param_count, param_ids)
9898
"""
9999
marks = getattr(obj, "pytestmark", [])
100+
101+
# Handle pytest proxy objects (APIRemovedInV1Proxy) - same as _is_eval_protocol_test
102+
if not isinstance(marks, (list, tuple)):
103+
try:
104+
marks = list(marks) if marks else []
105+
except (TypeError, AttributeError):
106+
marks = []
107+
100108
has_parametrize = False
101109
total_combinations = 0
102110
all_param_ids: list[str] = []

‎eval_protocol/evaluation.py‎

Lines changed: 111 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -741,108 +741,122 @@ def create(self, evaluator_id, display_name=None, description=None, force=False)
741741
# Upload code as tar.gz to GCS
742742
evaluator_name = result.get("name") # e.g., "accounts/pyroworks/evaluators/test-123"
743743

744-
if evaluator_name:
745-
try:
746-
# Create tar.gz of current directory
747-
cwd = os.getcwd()
748-
dir_name = os.path.basename(cwd)
749-
tar_filename = f"{dir_name}.tar.gz"
750-
tar_path = os.path.join(cwd, tar_filename)
751-
752-
tar_size = self._create_tar_gz_with_ignores(tar_path, cwd)
753-
754-
# Call GetEvaluatorUploadEndpoint
755-
756-
upload_endpoint_url = f"{self.api_base}/v1/{evaluator_name}:getUploadEndpoint"
757-
upload_payload = {"name": evaluator_name, "filename_to_size": {tar_filename: tar_size}}
758-
759-
logger.info(f"Requesting upload endpoint for {tar_filename}")
760-
upload_response = requests.post(upload_endpoint_url, json=upload_payload, headers=headers)
761-
762-
upload_response.raise_for_status()
763-
764-
signed_urls = upload_response.json().get("filenameToSignedUrls", {})
765-
signed_url = signed_urls.get(tar_filename)
766-
767-
if signed_url:
768-
logger.info(f"Uploading {tar_filename} to GCS...")
769-
770-
file_size = os.path.getsize(tar_path)
771-
772-
# Retry configuration
773-
max_retries = 3
774-
retry_delay = 2 # seconds
775-
776-
for attempt in range(max_retries):
777-
try:
778-
with open(tar_path, "rb") as f:
779-
# Create request exactly like Golang
780-
req = requests.Request(
781-
"PUT",
782-
signed_url,
783-
data=f,
784-
headers={
785-
"Content-Type": "application/octet-stream",
786-
"X-Goog-Content-Length-Range": f"{file_size},{file_size}",
787-
},
788-
)
789-
prepared = req.prepare()
790-
791-
# Don't let requests add extra headers
792-
session = requests.Session()
793-
gcs_response = session.send(prepared, timeout=600)
794-
gcs_response.raise_for_status()
795-
796-
logger.info(f"Successfully uploaded {tar_filename}")
797-
break # Success, exit retry loop
798-
799-
except (requests.exceptions.RequestException, IOError) as e:
800-
if attempt < max_retries - 1:
801-
# Check if it's a retryable error
802-
is_retryable = False
803-
if isinstance(e, requests.exceptions.RequestException):
804-
if hasattr(e, "response") and e.response is not None:
805-
# Retry on 5xx errors or 408 (timeout)
806-
is_retryable = (
807-
e.response.status_code >= 500 or e.response.status_code == 408
808-
)
809-
else:
810-
# Network errors (no response) are retryable
811-
is_retryable = True
812-
else:
813-
# IOError is retryable
814-
is_retryable = True
815-
816-
if is_retryable:
817-
wait_time = retry_delay * (2**attempt) # Exponential backoff
818-
logger.warning(
819-
f"Upload attempt {attempt + 1}/{max_retries} failed: {e}. "
820-
f"Retrying in {wait_time}s..."
821-
)
822-
time.sleep(wait_time)
823-
else:
824-
# Non-retryable error, raise immediately
825-
raise
744+
if not evaluator_name:
745+
raise ValueError(
746+
"Create evaluator response missing 'name' field. "
747+
f"Cannot proceed with code upload. Response: {result}"
748+
)
749+
750+
try:
751+
# Create tar.gz of current directory
752+
cwd = os.getcwd()
753+
dir_name = os.path.basename(cwd)
754+
tar_filename = f"{dir_name}.tar.gz"
755+
tar_path = os.path.join(cwd, tar_filename)
756+
757+
tar_size = self._create_tar_gz_with_ignores(tar_path, cwd)
758+
759+
# Call GetEvaluatorUploadEndpoint
760+
upload_endpoint_url = f"{self.api_base}/v1/{evaluator_name}:getUploadEndpoint"
761+
upload_payload = {"name": evaluator_name, "filename_to_size": {tar_filename: tar_size}}
762+
763+
logger.info(f"Requesting upload endpoint for {tar_filename}")
764+
upload_response = requests.post(upload_endpoint_url, json=upload_payload, headers=headers)
765+
upload_response.raise_for_status()
766+
767+
# Check for signed URLs
768+
upload_response_data = upload_response.json()
769+
signed_urls = upload_response_data.get("filenameToSignedUrls", {})
770+
771+
if not signed_urls:
772+
raise ValueError(f"GetUploadEndpoint returned no signed URLs. Response: {upload_response_data}")
773+
774+
signed_url = signed_urls.get(tar_filename)
775+
776+
if not signed_url:
777+
raise ValueError(
778+
f"No signed URL received for {tar_filename}. Available files: {list(signed_urls.keys())}"
779+
)
780+
781+
# Upload to GCS
782+
logger.info(f"Uploading {tar_filename} to GCS...")
783+
784+
file_size = os.path.getsize(tar_path)
785+
786+
# Retry configuration
787+
max_retries = 3
788+
retry_delay = 2 # seconds
789+
790+
for attempt in range(max_retries):
791+
try:
792+
with open(tar_path, "rb") as f:
793+
# Create request exactly like Golang
794+
req = requests.Request(
795+
"PUT",
796+
signed_url,
797+
data=f,
798+
headers={
799+
"Content-Type": "application/octet-stream",
800+
"X-Goog-Content-Length-Range": f"{file_size},{file_size}",
801+
},
802+
)
803+
prepared = req.prepare()
804+
805+
# Don't let requests add extra headers
806+
session = requests.Session()
807+
gcs_response = session.send(prepared, timeout=600)
808+
gcs_response.raise_for_status()
809+
810+
logger.info(f"Successfully uploaded {tar_filename}")
811+
break # Success, exit retry loop
812+
813+
except (requests.exceptions.RequestException, IOError) as e:
814+
if attempt < max_retries - 1:
815+
# Check if it's a retryable error
816+
is_retryable = False
817+
if isinstance(e, requests.exceptions.RequestException):
818+
if hasattr(e, "response") and e.response is not None:
819+
# Retry on 5xx errors or 408 (timeout)
820+
is_retryable = e.response.status_code >= 500 or e.response.status_code == 408
826821
else:
827-
# Last attempt failed
828-
logger.error(f"Upload failed after {max_retries} attempts")
829-
raise
822+
# Network errors (no response) are retryable
823+
is_retryable = True
824+
else:
825+
# IOError is retryable
826+
is_retryable = True
827+
828+
if is_retryable:
829+
wait_time = retry_delay * (2**attempt) # Exponential backoff
830+
logger.warning(
831+
f"Upload attempt {attempt + 1}/{max_retries} failed: {e}. "
832+
f"Retrying in {wait_time}s..."
833+
)
834+
time.sleep(wait_time)
835+
else:
836+
# Non-retryable error, raise immediately
837+
raise
838+
else:
839+
# Last attempt failed
840+
logger.error(f"Upload failed after {max_retries} attempts")
841+
raise
842+
843+
# Step 3: Validate upload
844+
validate_url = f"{self.api_base}/v1/{evaluator_name}:validateUpload"
845+
validate_payload = {"name": evaluator_name}
846+
validate_response = requests.post(validate_url, json=validate_payload, headers=headers)
847+
validate_response.raise_for_status()
830848

831-
# Step 3: Validate upload
832-
validate_url = f"{self.api_base}/v1/{evaluator_name}:validateUpload"
833-
validate_payload = {"name": evaluator_name}
834-
validate_response = requests.post(validate_url, json=validate_payload, headers=headers)
835-
validate_response.raise_for_status()
849+
validate_data = validate_response.json()
836850

837-
logger.info("Upload validated successfully")
851+
logger.info("Upload validated successfully")
838852

839-
# Clean up tar file
840-
if os.path.exists(tar_path):
841-
os.remove(tar_path)
853+
# Clean up tar file
854+
if os.path.exists(tar_path):
855+
os.remove(tar_path)
842856

843-
except Exception as upload_error:
844-
logger.warning(f"Code upload failed (evaluator created but code not uploaded): {upload_error}")
845-
# Don't fail - evaluator is created, just code upload failed
857+
except Exception as upload_error:
858+
logger.warning(f"Code upload failed (evaluator created but code not uploaded): {upload_error}")
859+
# Don't fail - evaluator is created, just code upload failed
846860

847861
return result # Return after attempting upload
848862
except Exception as e:

0 commit comments

Comments
 (0)