From 72b61fd91bd7415e3f7e30d06b6b0de5f7fd1995 Mon Sep 17 00:00:00 2001 From: spalen0 Date: Sat, 4 Apr 2026 11:25:06 +0200 Subject: [PATCH 1/3] fix(aave): add retry logic to Graph API query in proposals The Graph gateway can return transient 504 timeouts which crashes the entire GH Actions workflow. Use existing request_with_retry from utils/http to retry with exponential backoff, and return gracefully on failure instead of raising. Co-Authored-By: Claude Opus 4.6 (1M context) --- aave/proposals.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/aave/proposals.py b/aave/proposals.py index f42a60e0..8856d4a5 100644 --- a/aave/proposals.py +++ b/aave/proposals.py @@ -4,6 +4,7 @@ import requests from utils.cache import get_last_queued_id_from_file, write_last_queued_id_to_file +from utils.http import request_with_retry from utils.logging import get_logger from utils.telegram import send_telegram_message @@ -11,17 +12,33 @@ logger = get_logger(PROTOCOL) -def run_query(query, variables): +def run_query(query: str, variables: dict) -> dict | None: + """Run a GraphQL query against The Graph API with retry logic. + + Args: + query: The GraphQL query string. + variables: Variables for the GraphQL query. + + Returns: + Parsed JSON response dict, or None on failure. + """ api_key = os.getenv("GRAPH_API_KEY") subgraph_id = "A7QMszgomC9cnnfpAcqZVLr2DffvkGNfimD8iUSMiurK" url = f"https://gateway-arbitrum.network.thegraph.com/api/{api_key}/subgraphs/id/{subgraph_id}" - headers = {"Content-Type": "application/json"} request_body = {"query": query, "variables": variables} - response = requests.post(url, json=request_body, headers=headers) - if response.status_code == 200: - return response.json() - else: - raise Exception(f"Query failed with status code {response.status_code}: {response.text}") + + try: + response = request_with_retry("post", url, json=request_body) + except requests.RequestException as e: + logger.error("Graph API query failed after retries: %s", e) + return None + + data = response.json() + if "errors" in data: + logger.error("GraphQL error in response: %s", data["errors"]) + return None + + return data def fetch_queued_proposals(last_reported_id: int): @@ -46,9 +63,7 @@ def fetch_queued_proposals(last_reported_id: int): # get only last 10 proposals variables = {"lastId": last_reported_id} response = run_query(query, variables) - if "errors" in response: - # NOTE: not raising error because the graph is not reliable. We have Tenderly alerts for this also - logger.error("Error fetching proposals: %s", response["errors"]) + if response is None: return [] proposals = response["data"]["proposals"] From 27b20460598aefc4297c266aea9ffe0e9a243c9e Mon Sep 17 00:00:00 2001 From: spalen0 Date: Sat, 4 Apr 2026 11:26:24 +0200 Subject: [PATCH 2/3] fix(silo): add retry logic to Graph API query in positions check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same issue as aave/proposals — direct requests.post() with no error handling will crash the workflow on transient Graph API failures. Co-Authored-By: Claude Opus 4.6 (1M context) --- silo/main.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/silo/main.py b/silo/main.py index ac99a4b5..ad17f565 100644 --- a/silo/main.py +++ b/silo/main.py @@ -3,6 +3,7 @@ import requests from dotenv import load_dotenv +from utils.http import request_with_retry from utils.logging import get_logger from utils.telegram import send_telegram_message @@ -68,12 +69,20 @@ def check_positions(): "operationName": "QueryPositions", } - response = requests.post( - f"https://gateway-arbitrum.network.thegraph.com/api/{api_key}/subgraphs/id/2ufoztRpybsgogPVW6j9NTn1JmBWFYPKbP7pAabizADU", - json=json_data, - ) + try: + response = request_with_retry( + "post", + f"https://gateway-arbitrum.network.thegraph.com/api/{api_key}/subgraphs/id/2ufoztRpybsgogPVW6j9NTn1JmBWFYPKbP7pAabizADU", + json=json_data, + ) + except requests.RequestException as e: + logger.error("Graph API query failed after retries: %s", e) + return response_data = response.json() + if "errors" in response_data: + logger.error("GraphQL error in response: %s", response_data["errors"]) + return # Check if there are any positions returned positions = response_data["data"]["siloPositions"] From 70cbe57fcac7cf04620ec9faf0b7d115a227a73e Mon Sep 17 00:00:00 2001 From: spalen0 Date: Sat, 4 Apr 2026 11:28:13 +0200 Subject: [PATCH 3/3] fix(aave,silo): send silent telegram message on Graph API failure Send a low-priority (silent) telegram notification when Graph API queries fail after all retries, so the team is aware without being disrupted. Co-Authored-By: Claude Opus 4.6 (1M context) --- aave/proposals.py | 2 ++ silo/main.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/aave/proposals.py b/aave/proposals.py index 8856d4a5..4311a840 100644 --- a/aave/proposals.py +++ b/aave/proposals.py @@ -31,11 +31,13 @@ def run_query(query: str, variables: dict) -> dict | None: response = request_with_retry("post", url, json=request_body) except requests.RequestException as e: logger.error("Graph API query failed after retries: %s", e) + send_telegram_message(f"Graph API query failed after retries: {e}", PROTOCOL, True) return None data = response.json() if "errors" in data: logger.error("GraphQL error in response: %s", data["errors"]) + send_telegram_message(f"GraphQL error in response: {data['errors']}", PROTOCOL, True) return None return data diff --git a/silo/main.py b/silo/main.py index ad17f565..1627da4b 100644 --- a/silo/main.py +++ b/silo/main.py @@ -77,11 +77,13 @@ def check_positions(): ) except requests.RequestException as e: logger.error("Graph API query failed after retries: %s", e) + send_telegram_message(f"Graph API query failed after retries: {e}", PROTOCOL, True) return response_data = response.json() if "errors" in response_data: logger.error("GraphQL error in response: %s", response_data["errors"]) + send_telegram_message(f"GraphQL error in response: {response_data['errors']}", PROTOCOL, True) return # Check if there are any positions returned