Skip to content

updated retry and timeout in lws #1359

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lkolluru05 wants to merge 3 commits into
base: main
Choose a base branch
from
+34 −4
Open

updated retry and timeout in lws #1359

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
dd7e072
updated retry and timeout in lws
lkolluru05 Jan 15, 2026
6112b28
imports on top
lkolluru05 Jan 15, 2026
cebc5ed
logging added
lkolluru05 Jan 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 34 additions & 4 deletions axlearn/cloud/gcp/k8s_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
from axlearn.cloud.gcp.utils import custom_leaderworkerset_kwargs
from axlearn.common.config import REQUIRED, Required, config_class
from axlearn.common.utils import Nested
from kubernetes.client.exceptions import ApiException
import time


class Service(FlagConfigurable):
Expand Down Expand Up @@ -159,10 +161,38 @@ def _build_service(self) -> Nested[Any]:
lws_name = self.name.split("-service")[0]
custom_api = k8s.client.CustomObjectsApi()

# Fetch the CR object
lws = custom_api.get_namespaced_custom_object(
group=group, version=version, namespace=namespace, plural=plural, name=lws_name
)
max_tries = 5
retry_delay = 20

for attempt in range(max_tries):
try:
lws = custom_api.get_namespaced_custom_object(
group=group,
version=version,
namespace=namespace,
plural=plural,
name=lws_name
)
logging.info("Successfully retrieved %s", lws_name)
break
except ApiException as e:
# Check if it's a 404 error
if e.status == 404:
logging.info("Attempt %s: Resource %s not found yet.",str(attempt + 1),lws_name)

if attempt < max_tries - 1:
logging.info("Waiting %s seconds...",str(retry_delay))
time.sleep(retry_delay)
else:
logging.info("Max retries reached. Resource was never found.")
raise
else:
logging.info("An unexpected Kubernetes API error occurred")
raise

except Exception as e:
logging.info("An unexpected error occurred: %s",str(e))
raise

ports_map_list = []
for i in range(len(self.ports)):
Expand Down
Loading