GH-148726: Add heap_size to generational GC (#149195)

Author: sergey-miryanov

Include/internal/pycore_gc.h

@@ -223,12 +223,14 @@ static inline void _PyObject_GC_TRACK(
                           "object is in generation which is garbage collected",
                           filename, lineno, __func__);
 
-    PyGC_Head *generation0 = _PyInterpreterState_GET()->gc.generation0;
+    struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc;
+    PyGC_Head *generation0 = gcstate->generation0;
     PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev);
     _PyGCHead_SET_NEXT(last, gc);
     _PyGCHead_SET_PREV(gc, last);
     _PyGCHead_SET_NEXT(gc, generation0);
     generation0->_gc_prev = (uintptr_t)gc;
+    gcstate->heap_size++;
 #endif
 }
 
@@ -263,6 +265,8 @@ static inline void _PyObject_GC_UNTRACK(
     _PyGCHead_SET_PREV(next, prev);
     gc->_gc_next = 0;
     gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED;
+    struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc;
+    gcstate->heap_size--;
 #endif
 }
 

Include/internal/pycore_interp_structs.h

@@ -191,6 +191,8 @@ struct gc_generation_stats {
     Py_ssize_t candidates;
     // Total duration of the collection in seconds:
     double duration;
+    /* heap_size on the start of the collection */
+    Py_ssize_t heap_size;
 };
 
 #ifdef Py_GIL_DISABLED
@@ -226,7 +228,6 @@ struct _gc_runtime_state {
     /* linked lists of container objects */
 #ifndef Py_GIL_DISABLED
     struct gc_generation generations[NUM_GENERATIONS];
-    PyGC_Head *generation0;
 #else
     struct gc_generation young;
     struct gc_generation old[2];
@@ -244,6 +245,9 @@ struct _gc_runtime_state {
     /* a list of callbacks to be invoked when collection is performed */
     PyObject *callbacks;
 
+    /* The number of live objects. */
+    Py_ssize_t heap_size;
+
     /* This is the number of objects that survived the last full
        collection. It approximates the number of long lived objects
        tracked by the GC.
@@ -269,6 +273,8 @@ struct _gc_runtime_state {
 
     /* Mutex held for gc_should_collect_mem_usage(). */
     PyMutex mutex;
+#else
+    PyGC_Head *generation0;
 #endif
 };
 
@@ -278,7 +284,8 @@ struct _gc_runtime_state {
         { .threshold = 2000, }, \
         { .threshold = 10, }, \
         { .threshold = 10, }, \
-    },
+    }, \
+    .heap_size = 0,
 #else
 #define GC_GENERATION_INIT \
     .young = { .threshold = 2000, }, \

Lib/test/test_gc.py

@@ -1288,6 +1288,15 @@ def test_tuple_untrack_counts(self):
         # Use n // 2 just in case some other objects were collected.
         self.assertTrue(new_count - count > (n // 2))
 
+    @requires_gil_enabled('need generational GC')
+    @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi")
+    def test_heap_size(self):
+        count = _testinternalcapi.get_tracked_heap_size()
+        l = []
+        self.assertEqual(count + 1, _testinternalcapi.get_tracked_heap_size())
+        del l
+        self.assertEqual(count, _testinternalcapi.get_tracked_heap_size())
+
 
 class GCCallbackTests(unittest.TestCase):
     def setUp(self):

Lib/test/test_gc_stats.py

@@ -22,7 +22,7 @@
 
 GC_STATS_FIELDS = (
     "gen", "iid", "ts_start", "ts_stop", "collections", "collected",
-    "uncollectable", "candidates", "duration")
+    "uncollectable", "candidates", "heap_size", "duration")
 
 
 def get_interpreter_identifiers(gc_stats) -> tuple[int,...]:

Modules/_remote_debugging/clinic/module.c.h

@@ -53,6 +53,7 @@ read_gc_stats(struct gc_stats *stats, int64_t iid, PyObject *result,
             SET_FIELD(PyLong_FromSsize_t, items->collected);
             SET_FIELD(PyLong_FromSsize_t, items->uncollectable);
             SET_FIELD(PyLong_FromSsize_t, items->candidates);
+            SET_FIELD(PyLong_FromSsize_t, items->heap_size);
 
             SET_FIELD(PyFloat_FromDouble, items->duration);
 

Modules/_remote_debugging/gc_stats.c

@@ -143,6 +143,7 @@ static PyStructSequence_Field GCStatsInfo_fields[] = {
     {"collected", "Total number of collected objects"},
     {"uncollectable", "Total number of uncollectable objects"},
     {"candidates", "Total objects considered and traversed"},
+    {"heap_size", "Number of live objects"},
     {"duration", "Total collection time, in seconds"},
     {NULL}
 };
@@ -151,7 +152,7 @@ PyStructSequence_Desc GCStatsInfo_desc = {
     "_remote_debugging.GCStatsInfo",
     "Information about a garbage collector stats sample",
     GCStatsInfo_fields,
-    9
+    10
 };
 
 /* ============================================================================
@@ -1225,6 +1226,7 @@ Returns a list of GCStatsInfo objects with GC statistics data.
         - collected: Total number of collected objects.
         - uncollectable: Total number of uncollectable objects.
         - candidates: Total objects considered and traversed.
+        - heap_size: number of live objects.
         - duration: Total collection time, in seconds.
 
 Raises:
@@ -1235,7 +1237,7 @@ Returns a list of GCStatsInfo objects with GC statistics data.
 static PyObject *
 _remote_debugging_GCMonitor_get_gc_stats_impl(GCMonitorObject *self,
                                               int all_interpreters)
-/*[clinic end generated code: output=f73f365725224f7a input=09e647719c65f9e4]*/
+/*[clinic end generated code: output=f73f365725224f7a input=12f7c1a288cf2741]*/
 {
     RemoteDebuggingState *st = RemoteDebugging_GetStateFromType(Py_TYPE(self));
     return get_gc_stats(&self->offsets, all_interpreters, st->GCStatsInfo_Type);

Modules/_remote_debugging/module.c

@@ -2731,8 +2731,7 @@ has_deferred_refcount(PyObject *self, PyObject *op)
 static PyObject *
 get_tracked_heap_size(PyObject *self, PyObject *Py_UNUSED(ignored))
 {
-    // Generational GC doesn't track heap_size, return -1.
-    return PyLong_FromInt64(-1);
+    return PyLong_FromInt64(PyInterpreterState_Get()->gc.heap_size);
 }
 
 static PyObject *

Modules/_testinternalcapi.c

@@ -1405,13 +1405,13 @@ add_stats(GCState *gcstate, int gen, struct gc_generation_stats *stats)
     memcpy(cur_stats, prev_stats, sizeof(struct gc_generation_stats));
 
     cur_stats->ts_start = stats->ts_start;
-
     cur_stats->collections += 1;
     cur_stats->collected += stats->collected;
     cur_stats->uncollectable += stats->uncollectable;
     cur_stats->candidates += stats->candidates;
 
     cur_stats->duration += stats->duration;
+    cur_stats->heap_size = stats->heap_size;
     /* Publish ts_stop last so remote readers do not select a partially
        updated stats record as the latest collection. */
     cur_stats->ts_stop = stats->ts_stop;
@@ -1471,6 +1471,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason)
         invoke_gc_callback(tstate, "start", generation, &stats);
     }
 
+    stats.heap_size = gcstate->heap_size;
     // ignore error: don't interrupt the GC if reading the clock fails
     (void)PyTime_PerfCounterRaw(&stats.ts_start);
     if (gcstate->debug & _PyGC_DEBUG_STATS) {
@@ -2097,6 +2098,8 @@ PyObject_GC_Del(void *op)
     PyGC_Head *g = AS_GC(op);
     if (_PyObject_GC_IS_TRACKED(op)) {
         gc_list_remove(g);
+        GCState *gcstate = get_gc_state();
+        gcstate->heap_size--;
 #ifdef Py_DEBUG
         PyObject *exc = PyErr_GetRaisedException();
         if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0,