gh-151535: Walk awaited_by graph iteratively to avoid C stack overflow

The depth-bounded recursion still overflowed the debugger's C stack on
platforms with a small default thread stack (Windows uses 1 MiB): every
level keeps a SIZEOF_TASK_OBJ buffer alive in process_task_awaited_by, so
the MAX_TASK_AWAITED_BY_DEPTH limit of 1000 was only reached after several
MiB of stack had already been consumed, and the process aborted with a
stack overflow before the limit could fire.

Walk the awaited_by graph with an explicit, heap-allocated work-stack
instead of mutual recursion, so the C stack depth stays constant no matter
how deep the graph is. The depth limit is retained as a cycle guard for
corrupted or concurrently-mutated remote memory.

Also make the regression test deterministic under load: signal readiness
from the leaf task itself, immediately before it busy-spins, so the
observer always inspects while the full chain is built and rooted at a
running task. The previous handshake was sent before the leaf started
running and could race, letting the observer see a shallow graph.

Author: tonghuaroot

Lib/test/test_external_inspection.py

@@ -1191,6 +1191,11 @@ async def main():
                 async def leaf():
                     while not started.is_set():
                         await asyncio.sleep(0)
+                    # The whole awaited_by chain is built and this is now the
+                    # running task at the bottom of it. Signal here, then
+                    # busy-spin, so the observer inspects while the chain is
+                    # fully present and rooted at a running task.
+                    sock.sendall(b"ready")
                     end = time.time() + 10_000
                     while time.time() < end:
                         pass
@@ -1209,7 +1214,6 @@ async def waiter(child):
                 for _ in range(5):
                     await asyncio.sleep(0)
 
-                sock.sendall(b"ready")
                 started.set()
                 try:
                     await leaf_t

Modules/_remote_debugging/_remote_debugging.h

@@ -147,7 +147,7 @@ typedef enum _WIN32_THREADSTATE {
 #define MAX_STACK_CHUNK_SIZE (16 * 1024 * 1024)  /* 16 MB max for stack chunks */
 #define MAX_LONG_DIGITS 64  /* Allows values up to ~2^1920 */
 #define MAX_SET_TABLE_SIZE (1 << 20)  /* 1 million entries max for set iteration */
-#define MAX_TASK_AWAITED_BY_DEPTH 1000  /* Bound recursion over the awaited_by graph */
+#define MAX_TASK_AWAITED_BY_DEPTH 1000  /* Bound the awaited_by graph walk so a cycle terminates */
 
 #ifndef MAX
 #define MAX(a, b) ((a) > (b) ? (a) : (b))

Modules/_remote_debugging/asyncio.c

@@ -513,19 +513,24 @@ parse_task(
  * TASK AWAITED_BY PROCESSING
  * ============================================================================ */
 
-// Forward declaration for mutual recursion
-static int process_waiter_task(RemoteUnwinderObject *unwinder, uintptr_t key_addr, void *context);
-
-// Carries the recursion depth so a cyclic or corrupted remote awaited_by graph
-// cannot drive unbounded C recursion and overflow the debugger's stack.
+// The awaited_by graph is walked with an explicit, heap-allocated work-stack
+// rather than C recursion. A deeply nested -- or, in a corrupted or
+// concurrently-mutated remote process, cyclic -- chain would otherwise drive
+// unbounded recursion and overflow the debugger's own C stack: each level holds
+// a SIZEOF_TASK_OBJ buffer (see process_task_awaited_by), so even a few hundred
+// levels can exhaust a 1 MB stack. MAX_TASK_AWAITED_BY_DEPTH bounds the walk so
+// a cycle terminates.
 typedef struct {
-    PyObject *result;
+    uintptr_t addr;
     int depth;
-} waiter_context_t;
+} awaited_by_entry_t;
 
-static int process_task_and_waiters_impl(
-    RemoteUnwinderObject *unwinder, uintptr_t task_addr, PyObject *result,
-    int depth);
+typedef struct {
+    awaited_by_entry_t *items;
+    Py_ssize_t size;
+    Py_ssize_t capacity;
+    int current_depth;
+} awaited_by_stack_t;
 
 // Processor function for parsing tasks in sets
 static int
@@ -670,49 +675,88 @@ process_single_task_node(
 }
 
 static int
-process_task_and_waiters_impl(
+awaited_by_stack_push(
     RemoteUnwinderObject *unwinder,
-    uintptr_t task_addr,
-    PyObject *result,
+    awaited_by_stack_t *stack,
+    uintptr_t addr,
     int depth
 ) {
-    // First, add this task to the result
-    if (process_single_task_node(unwinder, task_addr, NULL, result) < 0) {
-        return -1;
+    if (stack->size >= stack->capacity) {
+        Py_ssize_t new_capacity = stack->capacity ? stack->capacity * 2 : 16;
+        awaited_by_entry_t *new_items = PyMem_Realloc(
+            stack->items, (size_t)new_capacity * sizeof(awaited_by_entry_t));
+        if (new_items == NULL) {
+            PyErr_NoMemory();
+            set_exception_cause(unwinder, PyExc_MemoryError,
+                                "Failed to grow awaited_by work-stack");
+            return -1;
+        }
+        stack->items = new_items;
+        stack->capacity = new_capacity;
     }
-
-    // Now find all tasks that are waiting for this task and process them
-    waiter_context_t ctx = {result, depth};
-    return process_task_awaited_by(unwinder, task_addr, process_waiter_task, &ctx);
-}
-
-int
-process_task_and_waiters(
-    RemoteUnwinderObject *unwinder,
-    uintptr_t task_addr,
-    PyObject *result
-) {
-    return process_task_and_waiters_impl(unwinder, task_addr, result, 0);
+    stack->items[stack->size].addr = addr;
+    stack->items[stack->size].depth = depth;
+    stack->size++;
+    return 0;
 }
 
-// Processor function for task waiters
+// set_entry_processor_func: enqueue a task waiting on the one currently being
+// expanded, one level deeper. The depth bound makes a cyclic or corrupted
+// awaited_by graph terminate instead of looping forever.
 static int
-process_waiter_task(
+push_awaited_by_waiter(
     RemoteUnwinderObject *unwinder,
     uintptr_t key_addr,
     void *context
 ) {
-    waiter_context_t *ctx = (waiter_context_t *)context;
-    if (ctx->depth >= MAX_TASK_AWAITED_BY_DEPTH) {
+    awaited_by_stack_t *stack = (awaited_by_stack_t *)context;
+    if (stack->current_depth >= MAX_TASK_AWAITED_BY_DEPTH) {
         PyErr_SetString(PyExc_RuntimeError,
                         "Task awaited_by chain is too deep or cyclic "
                         "(corrupted remote memory)");
         set_exception_cause(unwinder, PyExc_RuntimeError,
-                            "Task awaited_by recursion limit exceeded");
+                            "Task awaited_by depth limit exceeded");
         return -1;
     }
-    return process_task_and_waiters_impl(unwinder, key_addr, ctx->result,
-                                         ctx->depth + 1);
+    return awaited_by_stack_push(unwinder, stack, key_addr,
+                                 stack->current_depth + 1);
+}
+
+// Drain the work-stack: append each task node to result, then enqueue the
+// tasks waiting on it. Depth-first, with no C recursion over the graph.
+static int
+drain_awaited_by_stack(
+    RemoteUnwinderObject *unwinder,
+    PyObject *result,
+    awaited_by_stack_t *stack
+) {
+    while (stack->size > 0) {
+        awaited_by_entry_t entry = stack->items[--stack->size];
+        if (process_single_task_node(unwinder, entry.addr, NULL, result) < 0) {
+            return -1;
+        }
+        stack->current_depth = entry.depth;
+        if (process_task_awaited_by(unwinder, entry.addr,
+                                    push_awaited_by_waiter, stack) < 0) {
+            return -1;
+        }
+    }
+    return 0;
+}
+
+int
+process_task_and_waiters(
+    RemoteUnwinderObject *unwinder,
+    uintptr_t task_addr,
+    PyObject *result
+) {
+    awaited_by_stack_t stack = {0};
+    int result_code = -1;
+    if (awaited_by_stack_push(unwinder, &stack, task_addr, 0) == 0) {
+        result_code = drain_awaited_by_stack(unwinder, result, &stack);
+    }
+    PyMem_Free(stack.items);
+    return result_code;
 }
 
 /* ============================================================================
@@ -1008,9 +1052,17 @@ process_running_task_chain(
         return -1;
     }
 
-    // Now find all tasks that are waiting for this task and process them
-    waiter_context_t ctx = {result, 0};
-    if (process_task_awaited_by(unwinder, running_task_addr, process_waiter_task, &ctx) < 0) {
+    // Now find all tasks that are waiting for this task and process them with
+    // the same iterative, heap-stacked walk as process_task_and_waiters (the
+    // running task itself is already recorded via the frame chain above).
+    awaited_by_stack_t stack = {0};
+    int waiters_code = process_task_awaited_by(unwinder, running_task_addr,
+                                               push_awaited_by_waiter, &stack);
+    if (waiters_code == 0) {
+        waiters_code = drain_awaited_by_stack(unwinder, result, &stack);
+    }
+    PyMem_Free(stack.items);
+    if (waiters_code < 0) {
         return -1;
     }