gh-145685: per-type method cache implementation

Doc/c-api/type.rst

@@ -37,6 +37,10 @@ Type Objects
 
    Clear the internal lookup cache. Return the current version tag.
 
+   .. versionchanged:: 3.16
+      This function is now a no-op as the type cache is now implemented
+      per-type. It still returns the current version tag.
+
 .. c:function:: unsigned long PyType_GetFlags(PyTypeObject* type)
 
    Return the :c:member:`~PyTypeObject.tp_flags` member of *type*. This function is primarily

Doc/whatsnew/3.16.rst

@@ -325,7 +325,8 @@ New features
 Porting to Python 3.16
 ----------------------
 
-* TODO
+* :c:func:`PyType_ClearCache` is now a no-op as the type cache is now
+  implemented per-type. It still returns the current version tag.
 
 Deprecated C APIs
 -----------------

Include/cpython/object.h

@@ -246,6 +246,8 @@ struct _typeobject {
       * This function must escape to any code that can result in
       * the GC being run, such as Py_DECREF.  */
     _Py_iteritemfunc _tp_iteritem;
+
+    void *_tp_cache;
 };
 
 #define _Py_ATTR_CACHE_UNUSED (30000)  // (see tp_versions_used)

Include/internal/pycore_interp_structs.h

@@ -554,23 +554,6 @@ struct _types_runtime_state {
 };
 
 
-// Type attribute lookup cache: speed up attribute and method lookups,
-// see _PyType_Lookup().
-struct type_cache_entry {
-    unsigned int version;  // initialized from type->tp_version_tag
-#ifdef Py_GIL_DISABLED
-   _PySeqLock sequence;
-#endif
-    PyObject *name;        // reference to exactly a str or None
-    PyObject *value;       // borrowed reference or NULL
-};
-
-#define MCACHE_SIZE_EXP 12
-
-struct type_cache {
-    struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP];
-};
-
 typedef struct {
     PyTypeObject *type;
     int isbuiltin;
@@ -585,6 +568,10 @@ typedef struct {
        are also some diagnostic uses for the list of weakrefs,
        so we still keep it. */
     PyObject *tp_weaklist;
+    /* Per-interpreter attribute lookup cache (struct type_cache *).
+       For static builtin types the cache must be per-interpreter
+       because tp_dict and the values it stores are per-interpreter. */
+    void *_tp_cache;
 } managed_static_type_state;
 
 #define TYPE_VERSION_CACHE_SIZE (1<<12)  /* Must be a power of 2 */
@@ -595,8 +582,6 @@ struct types_state {
        where all those lower numbers are used for core static types. */
     unsigned int next_version_tag;
 
-    struct type_cache type_cache;
-
     /* Every static builtin type is initialized for each interpreter
        during its own initialization, including for the main interpreter
        during global runtime initialization.  This is done by calling

Include/internal/pycore_object.h

@@ -285,8 +285,6 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) {
     return ((type->tp_flags) & feature) != 0;
 }
 
-extern void _PyType_InitCache(PyInterpreterState *interp);
-
 extern PyStatus _PyObject_InitState(PyInterpreterState *interp);
 extern void _PyObject_FiniState(PyInterpreterState *interp);
 extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj);

Include/internal/pycore_typecache.h

@@ -0,0 +1,47 @@
+#ifndef PY_INTERNAL_TYPECACHE_H
+#define PY_INTERNAL_TYPECACHE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+#include "pycore_stackref.h"
+
+
+#define _Py_TYPECACHE_MINSIZE (1 << 3)
+#define _Py_TYPECACHE_MAXSIZE (1 << 16)
+
+struct type_cache_entry {
+    PyObject *name;     // name of the attribute or method, interned string, NULL if the entry is empty
+    PyObject *value;    // borrowed reference or NULL
+};
+
+// Per-type attribute lookup cache: speed up attribute and method lookups,
+// see _PyTypeCache_Lookup().
+struct type_cache {
+    uint32_t mask;                                              // mask for indexing into hashtable, i.e. size of hashtable is mask + 1
+    uint32_t available;                                         // number of available entries in hashtable
+    uint32_t used;                                              // number of used entries in hashtable
+    unsigned int version_tag;                                   // initialized from type->tp_version_tag
+    struct type_cache_entry hashtable[_Py_TYPECACHE_MINSIZE];   // hashtable entries
+};
+
+struct _PyTypeCacheLookupResult {
+    _PyStackRef value;      // value is a stack reference to the cached attribute or method, or NULL if not found
+    int cache_hit;          // 1 if the cache entry is valid and matches the type's version tag, 0 otherwise
+    unsigned int version_tag;   // version tag of the type when the value was cached
+};
+
+
+extern void _PyTypeCache_InitType(PyTypeObject *type);
+extern void _PyTypeCache_Insert(PyTypeObject *type, PyObject *name, PyObject *value);
+PyAPI_FUNC(struct _PyTypeCacheLookupResult) _PyTypeCache_Lookup(PyTypeObject *type, PyObject *name);
+PyAPI_FUNC(void) _PyTypeCache_Invalidate(PyTypeObject *type);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* PY_INTERNAL_TYPECACHE_H */

Include/internal/pycore_typeobject.h

@@ -41,7 +41,6 @@ extern PyStatus _PyTypes_InitTypes(PyInterpreterState *);
 extern void _PyTypes_FiniTypes(PyInterpreterState *);
 extern void _PyTypes_FiniExtTypes(PyInterpreterState *interp);
 extern void _PyTypes_Fini(PyInterpreterState *);
-extern void _PyTypes_AfterFork(void);
 extern void _PyTypes_FiniCachedDescriptors(PyInterpreterState *);
 
 static inline PyObject **

Lib/test/test_free_threading/test_type.py

@@ -4,8 +4,10 @@
 from concurrent.futures import ThreadPoolExecutor
 from threading import Thread
 from unittest import TestCase
+import sys
+from test.support import import_helper, threading_helper
 
-from test.support import threading_helper
+_testinternalcapi = import_helper.import_module("_testinternalcapi")
 
 
 
@@ -84,6 +86,24 @@ def reader_func():
 
         self.run_one(writer_func, reader_func)
 
+    def test_attr_cache_mortal(self):
+        class C:
+            x = object()
+
+        class D(C):
+            pass
+
+        def writer_func():
+            for _ in range(3000):
+                C.x = object()
+
+        def reader_func():
+            for _ in range(3000):
+                C.x
+                D.x
+
+        self.run_one(writer_func, reader_func)
+
     def test___class___modification(self):
         loops = 200
 
@@ -160,6 +180,51 @@ def reader():
 
         self.run_one(writer, reader)
 
+    def test_per_type_cache_concurrent_reads(self):
+        class C:
+            pass
+
+        names = [sys.intern(f"attr_{i}") for i in range(
+            _testinternalcapi._Py_TYPECACHE_MINSIZE * 4)]
+        for name in names:
+            setattr(C, name, name)
+        # Prime the cache.
+        for name in names:
+            getattr(C, name)
+
+        lookup = _testinternalcapi.type_cache_lookup
+
+        def reader():
+            for _ in range(500):
+                for name in names:
+                    hit, value, _ = lookup(C, name)
+                    self.assertEqual(hit, 1, name)
+                    self.assertEqual(value, name)
+
+        threading_helper.run_concurrently(reader, nthreads=NTHREADS)
+
+    def test_per_type_cache_concurrent_invalidate(self):
+        class C:
+            x = "value"
+
+        # Prime the cache.
+        C.x
+        hit, value, version = _testinternalcapi.type_cache_lookup(C, "x")
+        self.assertEqual(hit, 1)
+        self.assertIs(value, "value")
+        self.assertGreater(version, 0)
+
+        def reader():
+            for _ in range(10_000):
+                self.assertIs(C.x, "value")
+
+        def invalidator():
+            for _ in range(10_000):
+                _testinternalcapi.type_cache_invalidate(C)
+
+        workers = [invalidator] + [reader] * (NTHREADS - 1)
+        threading_helper.run_concurrently(workers)
+
     def run_one(self, writer_func, reader_func):
         barrier = threading.Barrier(NTHREADS)
 

Lib/test/test_sys.py

@@ -1788,7 +1788,7 @@ def delx(self): del self.__x
         check((1,2,3), vsize('') + self.P + 3*self.P)
         # type
         # static type: PyTypeObject
-        fmt = 'P2nPI13Pl4Pn9Pn12PI2Pc'
+        fmt = 'P2nPI13Pl4Pn9Pn12PI2PcP'
         s = vsize(fmt)
         check(int, s)
         typeid = 'n' if support.Py_GIL_DISABLED else ''