fix(aave): add retry logic to Graph API query in proposals

aave/proposals.py

@@ -4,24 +4,43 @@
 import requests
 
 from utils.cache import get_last_queued_id_from_file, write_last_queued_id_to_file
+from utils.http import request_with_retry
 from utils.logging import get_logger
 from utils.telegram import send_telegram_message
 
 PROTOCOL = "aave"
 logger = get_logger(PROTOCOL)
 
 
-def run_query(query, variables):
+def run_query(query: str, variables: dict) -> dict | None:
+    """Run a GraphQL query against The Graph API with retry logic.
+
+    Args:
+        query: The GraphQL query string.
+        variables: Variables for the GraphQL query.
+
+    Returns:
+        Parsed JSON response dict, or None on failure.
+    """
     api_key = os.getenv("GRAPH_API_KEY")
     subgraph_id = "A7QMszgomC9cnnfpAcqZVLr2DffvkGNfimD8iUSMiurK"
     url = f"https://gateway-arbitrum.network.thegraph.com/api/{api_key}/subgraphs/id/{subgraph_id}"
-    headers = {"Content-Type": "application/json"}
     request_body = {"query": query, "variables": variables}
-    response = requests.post(url, json=request_body, headers=headers)
-    if response.status_code == 200:
-        return response.json()
-    else:
-        raise Exception(f"Query failed with status code {response.status_code}: {response.text}")
+
+    try:
+        response = request_with_retry("post", url, json=request_body)
+    except requests.RequestException as e:
+        logger.error("Graph API query failed after retries: %s", e)
+        send_telegram_message(f"Graph API query failed after retries: {e}", PROTOCOL, True)
+        return None
+
+    data = response.json()
+    if "errors" in data:
+        logger.error("GraphQL error in response: %s", data["errors"])
+        send_telegram_message(f"GraphQL error in response: {data['errors']}", PROTOCOL, True)
+        return None
+
+    return data
 
 
 def fetch_queued_proposals(last_reported_id: int):
@@ -46,9 +65,7 @@ def fetch_queued_proposals(last_reported_id: int):
     # get only last 10 proposals
     variables = {"lastId": last_reported_id}
     response = run_query(query, variables)
-    if "errors" in response:
-        # NOTE: not raising error because the graph is not reliable. We have Tenderly alerts for this also
-        logger.error("Error fetching proposals: %s", response["errors"])
+    if response is None:
         return []
 
     proposals = response["data"]["proposals"]

silo/main.py

@@ -3,6 +3,7 @@
 import requests
 from dotenv import load_dotenv
 
+from utils.http import request_with_retry
 from utils.logging import get_logger
 from utils.telegram import send_telegram_message
 
@@ -68,12 +69,22 @@ def check_positions():
             "operationName": "QueryPositions",
         }
 
-        response = requests.post(
-            f"https://gateway-arbitrum.network.thegraph.com/api/{api_key}/subgraphs/id/2ufoztRpybsgogPVW6j9NTn1JmBWFYPKbP7pAabizADU",
-            json=json_data,
-        )
+        try:
+            response = request_with_retry(
+                "post",
+                f"https://gateway-arbitrum.network.thegraph.com/api/{api_key}/subgraphs/id/2ufoztRpybsgogPVW6j9NTn1JmBWFYPKbP7pAabizADU",
+                json=json_data,
+            )
+        except requests.RequestException as e:
+            logger.error("Graph API query failed after retries: %s", e)
+            send_telegram_message(f"Graph API query failed after retries: {e}", PROTOCOL, True)
+            return
 
         response_data = response.json()
+        if "errors" in response_data:
+            logger.error("GraphQL error in response: %s", response_data["errors"])
+            send_telegram_message(f"GraphQL error in response: {response_data['errors']}", PROTOCOL, True)
+            return
 
         # Check if there are any positions returned
         positions = response_data["data"]["siloPositions"]