From 4956d2be9d5e555f2cf64faed9ef39e6a797c360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81rni=20M=C3=A1r=20J=C3=B3nsson?= Date: Mon, 11 May 2026 11:54:09 +0000 Subject: [PATCH 1/8] gh-149663: fix typo in `unittest` docs (#149670) `hastattr` -> `hasattr` --- Doc/library/unittest.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index c54f3e2792c388..ff619f97923325 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1262,10 +1262,10 @@ Test cases | :meth:`assertNotEndsWith(a, b) | ``not a.endswith(b)`` | 3.14 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertHasAttr(a, b) | ``hastattr(a, b)`` | 3.14 | + | :meth:`assertHasAttr(a, b) | ``hasattr(a, b)`` | 3.14 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertNotHasAttr(a, b) | ``not hastattr(a, b)`` | 3.14 | + | :meth:`assertNotHasAttr(a, b) | ``not hasattr(a, b)`` | 3.14 | | ` | | | +---------------------------------------+--------------------------------+--------------+ From ef877318a0dd522389e03cfb943b1af857964598 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Mon, 11 May 2026 16:05:50 +0300 Subject: [PATCH 2/8] gh-149402: Don't assume single-character type codes (struct/array/ctypes) (#149483) In the struct docs, section "Format Characters" was renamed to "Type Codes". Co-authored-by: Victor Stinner --- Doc/library/array.rst | 4 ++-- Doc/library/ctypes.rst | 5 ++--- Doc/library/struct.rst | 35 ++++++++++++++++++----------------- Objects/memoryobject.c | 4 ++-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Doc/library/array.rst b/Doc/library/array.rst index ca7c055285aa82..4b3ffbbfe80406 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -12,7 +12,7 @@ This module defines an object type which can compactly represent an array of basic values: characters, integers, floating-point numbers, complex numbers. Arrays are mutable :term:`sequence` types and behave very much like lists, except that the type of objects stored in them is constrained. The type is specified at object creation time by using a -:dfn:`type code`, which is a single character. The following type codes are +:dfn:`type code`. The following type codes are defined: +-----------+--------------------+-------------------+-----------------------+-------+ @@ -91,7 +91,7 @@ Notes: .. seealso:: The :ref:`ctypes ` and - :ref:`struct ` modules, + :ref:`struct ` modules, as well as third-party modules like `numpy `__, use similar -- but slightly different -- type codes. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 438afa04c6630d..4330be922013f3 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -2572,8 +2572,7 @@ Fundamental data types .. attribute:: _type_ - Class attribute that contains an internal type code, as a - single-character string. + Class attribute that contains an internal type code, as a string. See :ref:`ctypes-fundamental-data-types` for a summary. Types marked \* in the summary may be (or always are) aliases of a @@ -2587,7 +2586,7 @@ Fundamental data types .. seealso:: - The :mod:`array` and :ref:`struct ` modules, + The :mod:`array` and :ref:`struct ` modules, as well as third-party modules like `numpy `__, use similar -- but slightly different -- type codes. diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 05662c6c2d898d..ec872fddee05c7 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -111,7 +111,7 @@ Format Strings -------------- Format strings describe the data layout when -packing and unpacking data. They are built up from :ref:`format characters`, +packing and unpacking data. They are built up from :ref:`type codes `, which specify the type of data being packed/unpacked. In addition, special characters control the :ref:`byte order, size and alignment`. Each format string consists of an optional prefix character which @@ -183,8 +183,8 @@ Use :data:`sys.byteorder` to check the endianness of your system. Native size and alignment are determined using the C compiler's ``sizeof`` expression. This is always combined with native byte order. -Standard size depends only on the format character; see the table in -the :ref:`format-characters` section. +Standard size depends only on the type code; see the table in +the :ref:`type-codes` section. Note the difference between ``'@'`` and ``'='``: both use native byte order, but the size and alignment of the latter is standardized. @@ -208,12 +208,13 @@ Notes: count of zero. See :ref:`struct-examples`. +.. _type-codes: .. _format-characters: -Format Characters -^^^^^^^^^^^^^^^^^ +Type Codes +^^^^^^^^^^ -Format characters have the following meaning; the conversion between C and +Type codes (or format codes) have the following meaning; the conversion between C and Python values should be obvious given their types. The 'Standard size' column refers to the size of the packed value in bytes when using standard size; that is, when the format string starts with one of ``'<'``, ``'>'``, ``'!'`` or @@ -324,7 +325,7 @@ Notes: format used by the platform. (5) - The ``'P'`` format character is only available for the native byte ordering + The ``'P'`` type code is only available for the native byte ordering (selected as the default or with the ``'@'`` byte order character). The byte order character ``'='`` chooses to use little- or big-endian ordering based on the host system. The struct module does not interpret this as native @@ -346,22 +347,22 @@ Notes: When packing, ``'x'`` inserts one NUL byte. (8) - The ``'p'`` format character encodes a "Pascal string", meaning a short + The ``'p'`` type code encodes a "Pascal string", meaning a short variable-length string stored in a *fixed number of bytes*, given by the count. The first byte stored is the length of the string, or 255, whichever is smaller. The bytes of the string follow. If the byte string passed in to :func:`pack` is too long (longer than the count minus 1), only the leading ``count-1`` bytes of the string are stored. If the byte string is shorter than ``count-1``, it is padded with null bytes so that exactly count bytes in all - are used. Note that for :func:`unpack`, the ``'p'`` format character consumes + are used. Note that for :func:`unpack`, the ``'p'`` type code consumes ``count`` bytes, but that the :class:`!bytes` object returned can never contain more than 255 bytes. When packing, arguments of types :class:`bytes` and :class:`bytearray` are accepted. (9) - For the ``'s'`` format character, the count is interpreted as the length of the - byte string, not a repeat count like for the other format characters; for example, + For the ``'s'`` type code, the count is interpreted as the length of the + byte string, not a repeat count like for the other type codes; for example, ``'10s'`` means a single 10-byte string mapping to or from a single Python byte string, while ``'10c'`` means 10 separate one byte character elements (e.g., ``cccccccccc``) mapping @@ -376,7 +377,7 @@ Notes: are accepted. (10) - For the ``'F'`` and ``'D'`` format characters, the packed representation uses + For the ``'F'`` and ``'D'`` type codes, the packed representation uses the IEEE 754 binary32 and binary64 format for components of the complex number, regardless of the floating-point format used by the platform. Note that complex types (``F``/``Zf`` and ``D``/``Zd``) are available unconditionally, @@ -385,7 +386,7 @@ Notes: two-element C array containing, respectively, the real and imaginary parts. -A format character may be preceded by an integral repeat count. For example, +A type code may be preceded by an integral repeat count. For example, the format string ``'4h'`` means exactly the same as ``'hhhh'``. Whitespace characters between formats are ignored; a count and its format must @@ -402,7 +403,7 @@ then :exc:`struct.error` is raised. .. index:: single: ? (question mark); in struct format strings -For the ``'?'`` format character, the return value is either :const:`True` or +For the ``'?'`` type code, the return value is either :const:`True` or :const:`False`. When packing, the truth value of the argument object is used. Either 0 or 1 in the native or standard bool representation will be packed, and any non-zero value will be ``True`` when unpacking. @@ -457,7 +458,7 @@ the result in a named tuple:: >>> Student._make(unpack('<10sHHb', record)) Student(name=b'raymond ', serialnum=4658, school=264, gradelevel=8) -The ordering of format characters may have an impact on size in native +The ordering of type codes may have an impact on size in native mode since padding is implicit. In standard mode, the user is responsible for inserting any desired padding. Note in @@ -515,7 +516,7 @@ When constructing format strings which mimic native layouts, the compiler and machine architecture determine byte ordering and padding. In such cases, the ``@`` format character should be used to specify native byte ordering and data sizes. Internal pad bytes are normally inserted -automatically. It is possible that a zero-repeat format code will be +automatically. It is possible that a zero-repeat type code will be needed at the end of a format string to round up to the correct byte boundary for proper alignment of consecutive chunks of data. @@ -534,7 +535,7 @@ code solves that problem:: >>> calcsize('@llh0l') 24 -The ``'x'`` format code can be used to specify the repeat, but for +The ``'x'`` type code can be used to specify the repeat, but for native formats it is better to use a zero-repeat format like ``'0l'``. By default, native byte ordering and alignment is used, but it is diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 900db864621a84..c0fd0b8b2f0f53 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1806,7 +1806,7 @@ pylong_as_zu(PyObject *item) dest = x; \ } while (0) -/* Unpack a single item. 'fmt' can be any native format character in struct +/* Unpack a single item. 'fmt' can be any native format in struct module syntax. This function is very sensitive to small changes. With this layout gcc automatically generates a fast jump table. */ static inline PyObject * @@ -1926,7 +1926,7 @@ unpack_single(PyMemoryViewObject *self, const char *ptr, const char *fmt) memcpy(ptr, (char *)&x, sizeof x); \ } while (0) -/* Pack a single item. 'fmt' can be any native format character in +/* Pack a single item. 'fmt' can be any native format in struct module syntax. */ static int pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt) From 56171da3417bc14fded2f42033d72f63e1bf7cd9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 11 May 2026 06:08:12 -0700 Subject: [PATCH 3/8] gh-144957: Fix lazy imports + module __getattr__ (GH-149624) --- Lib/test/test_lazy_import/__init__.py | 31 +++++++++++++++++++ .../data/module_with_getattr.py | 4 +++ .../test_lazy_import/data/pkg/__init__.py | 5 +++ ...-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst | 2 ++ Objects/moduleobject.c | 19 ++++++++++++ 5 files changed, 61 insertions(+) create mode 100644 Lib/test/test_lazy_import/data/module_with_getattr.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index 1d1d2e00bd733f..ea534a8ee5b981 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -88,6 +88,26 @@ def test_basic_used(self): import test.test_lazy_import.data.basic_used self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + @support.requires_subprocess() + def test_from_import_with_module_getattr(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + lazy from test.test_lazy_import.data.module_with_getattr import dynamic_attr + assert dynamic_attr == "from_getattr" + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_imported_module_getattr(self): + """Lazy from import should not shadow an imported module's __getattr__.""" + code = textwrap.dedent(""" + import test.test_lazy_import.data.module_with_getattr as mod + lazy from test.test_lazy_import.data.module_with_getattr import dynamic_attr + assert dynamic_attr == "from_getattr" + assert mod.dynamic_attr == "from_getattr" + """) + assert_python_ok("-c", code) + class GlobalLazyImportModeTests(unittest.TestCase): """Tests for sys.set_lazy_imports() global mode control.""" @@ -385,6 +405,17 @@ def test_lazy_import_pkg_cross_import(self): self.assertEqual(type(g["x"]), int) self.assertEqual(type(g["b"]), types.LazyImportType) + @support.requires_subprocess() + def test_package_from_import_with_module_getattr(self): + """Lazy from import should respect a package's __getattr__.""" + code = textwrap.dedent(""" + import test.test_lazy_import.data.pkg as pkg + lazy from test.test_lazy_import.data.pkg import dynamic_attr + assert dynamic_attr == "from_getattr" + assert pkg.dynamic_attr == "from_getattr" + """) + assert_python_ok("-c", code) + class DunderLazyImportTests(unittest.TestCase): """Tests for __lazy_import__ builtin function.""" diff --git a/Lib/test/test_lazy_import/data/module_with_getattr.py b/Lib/test/test_lazy_import/data/module_with_getattr.py new file mode 100644 index 00000000000000..2ac01a90d76e62 --- /dev/null +++ b/Lib/test/test_lazy_import/data/module_with_getattr.py @@ -0,0 +1,4 @@ +def __getattr__(name): + if name == "dynamic_attr": + return "from_getattr" + raise AttributeError(name) diff --git a/Lib/test/test_lazy_import/data/pkg/__init__.py b/Lib/test/test_lazy_import/data/pkg/__init__.py index 2d76abaa89f893..e526aab94131b8 100644 --- a/Lib/test/test_lazy_import/data/pkg/__init__.py +++ b/Lib/test/test_lazy_import/data/pkg/__init__.py @@ -1 +1,6 @@ x = 42 + +def __getattr__(name): + if name == "dynamic_attr": + return "from_getattr" + raise AttributeError(name) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst new file mode 100644 index 00000000000000..3063f1a3c0e6d3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst @@ -0,0 +1,2 @@ +Fix lazy ``from`` imports of module attributes provided by module-level +``__getattr__``. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index b7d2e5ffde4fe7..f7b83c1d111cde 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1307,6 +1307,25 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) attr = _PyObject_GenericGetAttrWithDict((PyObject *)m, name, NULL, suppress); if (attr) { if (PyLazyImport_CheckExact(attr)) { + // gh-144957: Module __getattr__ should get a chance to provide + // the attribute before resolving a lazy import placeholder. + if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { + Py_DECREF(attr); + return NULL; + } + if (getattr) { + PyObject *result = PyObject_CallOneArg(getattr, name); + Py_DECREF(getattr); + if (result != NULL) { + Py_DECREF(attr); + return result; + } + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + Py_DECREF(attr); + return NULL; + } + PyErr_Clear(); + } PyObject *new_value = _PyImport_LoadLazyImportTstate( PyThreadState_GET(), attr); if (new_value == NULL) { From 2d3dec0fbda9460cfc3d2ee786969f7d0c7530ea Mon Sep 17 00:00:00 2001 From: Guo Ci Date: Mon, 11 May 2026 10:21:03 -0400 Subject: [PATCH 4/8] gh-140924: In locale module, add missing names to __all__ (GH-140925) --- Lib/locale.py | 33 ++++++++++++++++--- Lib/test/test_locale.py | 9 +++++ ...-11-02-22-24-13.gh-issue-140924.NQVbR_.rst | 3 ++ 3 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-02-22-24-13.gh-issue-140924.NQVbR_.rst diff --git a/Lib/locale.py b/Lib/locale.py index 4ff6f8c0f0a775..25efff5b856854 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -17,17 +17,14 @@ from builtins import str as _builtin_str import functools -# Try importing the _locale module. -# -# If this fails, fall back on a basic 'C' locale emulation. - # Yuck: LC_MESSAGES is non-standard: can't tell whether it exists before # trying the import. So __all__ is also fiddled at the end of the file. __all__ = ["getlocale", "getdefaultlocale", "getpreferredencoding", "Error", "setlocale", "localeconv", "strcoll", "strxfrm", "str", "atof", "atoi", "format_string", "currency", "normalize", "LC_CTYPE", "LC_COLLATE", "LC_TIME", "LC_MONETARY", - "LC_NUMERIC", "LC_ALL", "CHAR_MAX", "getencoding"] + "LC_NUMERIC", "LC_ALL", "CHAR_MAX", "getencoding", "delocalize", + "localize"] def _strcoll(a,b): """ strcoll(string,string) -> int. @@ -41,6 +38,9 @@ def _strxfrm(s): """ return s +# Try importing the _locale module. +# +# If this fails, fall back on a basic 'C' locale emulation. try: from _locale import * @@ -91,6 +91,29 @@ def setlocale(category, value=None): raise Error('_locale emulation only supports "C" locale') return 'C' +else: + _conditional_constants_names = ["ABDAY_1", "ABDAY_2", "ABDAY_3", + "ABDAY_4", "ABDAY_5", "ABDAY_6", + "ABDAY_7", "ABMON_1", "ABMON_10", + "ABMON_11", "ABMON_12", "ABMON_2", + "ABMON_3", "ABMON_4", "ABMON_5", + "ABMON_6", "ABMON_7", "ABMON_8", + "ABMON_9", "ALT_DIGITS", "CODESET", + "CRNCYSTR", "DAY_1", "DAY_2", "DAY_3", + "DAY_4", "DAY_5", "DAY_6", + "DAY_7", "D_FMT", "D_T_FMT", + "ERA", "ERA_D_FMT", "ERA_D_T_FMT", + "ERA_T_FMT", "MON_1", "MON_10", + "MON_11", "MON_12", "MON_2", "MON_3", + "MON_4", "MON_5", "MON_6", "MON_7", + "MON_8", "MON_9", "NOEXPR", + "RADIXCHAR", "THOUSEP", "T_FMT", + "T_FMT_AMPM", "YESEXPR", "AM_STR", + "PM_STR"] + # The constants defined in _locale are platform-dependent, + # so we only include those that are available on the current platform. + __all__.extend(vars().keys() & _conditional_constants_names) + # These may or may not exist in _locale, so be sure to set them. if 'strxfrm' not in globals(): strxfrm = _strxfrm diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index a06c600cf56689..8057c35f540841 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -9,6 +9,15 @@ import sys import codecs + +class MiscTestCase(unittest.TestCase): + maxDiff = None + def test__all__(self): + extra = ["localeconv", "strcoll", "strxfrm", "getencoding", + "Error"] + not_exported = ["locale_encoding_alias", "locale_alias", "windows_locale"] + support.check__all__(self, locale, extra=extra, not_exported=not_exported) + class LazyImportTest(unittest.TestCase): @cpython_only def test_lazy_import(self): diff --git a/Misc/NEWS.d/next/Library/2025-11-02-22-24-13.gh-issue-140924.NQVbR_.rst b/Misc/NEWS.d/next/Library/2025-11-02-22-24-13.gh-issue-140924.NQVbR_.rst new file mode 100644 index 00000000000000..91c30c95469f08 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-02-22-24-13.gh-issue-140924.NQVbR_.rst @@ -0,0 +1,3 @@ +Add :func:`locale.localize`, :func:`locale.delocalize` +and platform-specific locale constants +from the :mod:`!_locale` module to ``locale.__all__``. From 6a26b78c470ee69d3ac095899fdc85004b260145 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Mon, 11 May 2026 18:03:57 +0300 Subject: [PATCH 5/8] gh-149634: Fix removed docs from `TarFile.tarfile` to `TarInfo.tarfile` (#149680) --- Doc/deprecations/pending-removal-in-3.16.rst | 2 +- Doc/whatsnew/3.13.rst | 2 +- .../next/Library/2026-05-10-14-10-00.gh-issue-149634.iT5cwC.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index a64212e38e61cb..50450658d31440 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -101,5 +101,5 @@ Pending removal in Python 3.16 * :mod:`tarfile`: - * The undocumented and unused :attr:`!TarFile.tarfile` attribute + * The undocumented and unused :attr:`!TarInfo.tarfile` attribute has been deprecated since Python 3.13. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index f4489bfa1b74e8..de5a37042a9203 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1965,7 +1965,7 @@ New Deprecations * :mod:`tarfile`: - * Deprecate the undocumented and unused :attr:`!TarFile.tarfile` attribute, + * Deprecate the undocumented and unused :attr:`!TarInfo.tarfile` attribute, to be removed in Python 3.16. (Contributed in :gh:`115256`.) diff --git a/Misc/NEWS.d/next/Library/2026-05-10-14-10-00.gh-issue-149634.iT5cwC.rst b/Misc/NEWS.d/next/Library/2026-05-10-14-10-00.gh-issue-149634.iT5cwC.rst index 620b66f754f5b5..73e8d73c36891a 100644 --- a/Misc/NEWS.d/next/Library/2026-05-10-14-10-00.gh-issue-149634.iT5cwC.rst +++ b/Misc/NEWS.d/next/Library/2026-05-10-14-10-00.gh-issue-149634.iT5cwC.rst @@ -1 +1 @@ -Remove deprecated and unused :attr:`!tarfile.Tarfile.tarfile` attribute. +Remove deprecated and unused :attr:`!tarfile.TarInfo.tarfile` attribute. From 374f9d3f5e70d2204d88ab123f29825d71537de2 Mon Sep 17 00:00:00 2001 From: Manoj K M Date: Mon, 11 May 2026 20:57:16 +0530 Subject: [PATCH 6/8] Fix incorrect sentence in stable.rst (GH-149684) --- Doc/c-api/stable.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 0ff066680b8c73..13e5d5c96135c0 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -114,7 +114,7 @@ versions of Python. All functions in Stable ABI are present as functions in Python's shared library, not solely as macros. -This makes them usable are usable from languages that don't use the C +This makes them usable in languages that don't use the C preprocessor, including Python's :py:mod:`ctypes`. From fadd9bc14e43041c84bb8d06824990264fe1434a Mon Sep 17 00:00:00 2001 From: David Ellis Date: Mon, 11 May 2026 16:28:23 +0100 Subject: [PATCH 7/8] gh-149614 - Restore deepcopiability of argparse.ArgumentParser instances (#149617) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Savannah Ostrowski --- Lib/argparse.py | 2 + Lib/test/test_argparse.py | 55 +++++++++++++++++++ ...-05-09-21-02-08.gh-issue-149614.U4snj3.rst | 1 + 3 files changed, 58 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-05-09-21-02-08.gh-issue-149614.U4snj3.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 6d21823e652429..29e6ebb9634261 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -163,6 +163,8 @@ class _ColorlessTheme: def __getattr__(self, name): # _colorize's no_color themes are just all empty strings # by directly using empty strings the import is avoided + if name.startswith("_"): + raise AttributeError(name) return "" _colorless_theme = _ColorlessTheme() diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 4ea5b6f53a0426..1dc3f538f4ad8b 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -140,6 +140,48 @@ def test_parse_args(self): ) +class TestArgumentParserCopiable(unittest.TestCase): + def _get_parser(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo', type=int, default=42) + parser.add_argument('bar', nargs='?', default='baz') + return parser + + @force_not_colorized + def test_copiable(self): + import copy + parser = self._get_parser() + parser2 = copy.copy(parser) + ns = parser2.parse_args(['--foo', '123', 'quux']) + self.assertEqual(ns.foo, 123) + self.assertEqual(ns.bar, 'quux') + ns2 = parser2.parse_args([]) + self.assertEqual(ns2.foo, 42) + self.assertEqual(ns2.bar, 'baz') + + # Test shallow copy also gets new arguments + parser.add_argument("--extra") + ns3 = parser2.parse_args(["--extra", "bar"]) + self.assertEqual(ns3.extra, "bar") + + @force_not_colorized + def test_deepcopiable(self): + import copy + parser = self._get_parser() + parser2 = copy.deepcopy(parser) + ns = parser2.parse_args(['--foo', '123', 'quux']) + self.assertEqual(ns.foo, 123) + self.assertEqual(ns.bar, 'quux') + ns2 = parser2.parse_args([]) + self.assertEqual(ns2.foo, 42) + self.assertEqual(ns2.bar, 'baz') + + # Test deep copy does not get new arguments + parser.add_argument("--extra") + with self.assertRaises(argparse.ArgumentError): + parser2.parse_args(["--extra", "bar"]) + + class TestArgumentParserPickleable(unittest.TestCase): @force_not_colorized @@ -7863,12 +7905,25 @@ def fake_can_colorize(*, file=None): def test_fake_color_theme_matches_real(self): from argparse import _colorless_theme + + # Check the attributes match those of the 'real' theme _colorize_nocolor = _colorize.get_theme(force_no_color=True).argparse for k in _colorize_nocolor: self.assertEqual( getattr(_colorless_theme, k), getattr(_colorize_nocolor, k) ) + def test_fake_color_theme_raises(self): + from argparse import _colorless_theme + + # Make sure the _colorless_theme doesn't return empty strings + # for magic methods or private attributes + with self.assertRaises(AttributeError): + _colorless_theme.__unknown_dunder__ + + with self.assertRaises(AttributeError): + _colorless_theme._private_attribute + class TestModule(unittest.TestCase): def test_deprecated__version__(self): diff --git a/Misc/NEWS.d/next/Library/2026-05-09-21-02-08.gh-issue-149614.U4snj3.rst b/Misc/NEWS.d/next/Library/2026-05-09-21-02-08.gh-issue-149614.U4snj3.rst new file mode 100644 index 00000000000000..5169c6c203fc1b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-09-21-02-08.gh-issue-149614.U4snj3.rst @@ -0,0 +1 @@ +Fix a regression that broke the ability to deepcopy :class:`argparse.ArgumentParser` instances. From 8a4895985f42282504d83b9bd0c77b129f95a5d5 Mon Sep 17 00:00:00 2001 From: Alper Date: Mon, 11 May 2026 08:39:55 -0700 Subject: [PATCH 8/8] gh-145235: Make dict watcher API thread-safe for free-threaded builds (gh-145233) In free-threaded builds, concurrent calls to PyDict_AddWatcher, PyDict_ClearWatcher, PyDict_Watch, and PyDict_Unwatch can race on the shared callback array and the per-dict watcher tags. This change adds a mutex to serialize watcher registration and removal, atomic operations for tag updates, and atomic acquire/release synchronization for callback dispatch in _PyDict_SendEvent. --- Include/internal/pycore_dict_state.h | 2 + .../internal/pycore_pyatomic_ft_wrappers.h | 2 + .../test_free_threading/test_dict_watcher.py | 89 +++++++++++++++++++ ...-02-25-13-37-10.gh-issue-145235.-1ySNR.rst | 3 + Modules/_testcapi/watchers.c | 26 +++--- Objects/dictobject.c | 42 ++++++--- Python/optimizer_analysis.c | 20 +++-- Python/pystate.c | 1 + Tools/c-analyzer/cpython/ignored.tsv | 1 + 9 files changed, 159 insertions(+), 27 deletions(-) create mode 100644 Lib/test/test_free_threading/test_dict_watcher.py create mode 100644 Misc/NEWS.d/next/C_API/2026-02-25-13-37-10.gh-issue-145235.-1ySNR.rst diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 11932b8d1e1ab6..bb6fe262597559 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -13,6 +13,8 @@ extern "C" { struct _Py_dict_state { uint32_t next_keys_version; + PyMutex watcher_mutex; // Protects the watchers array (free-threaded builds) + _PyOnceFlag watcher_setup_once; // One-time optimizer watcher setup PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index fafdd728a8229a..d8ec306a0dae3f 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -138,6 +138,7 @@ extern "C" { #define FT_ATOMIC_ADD_SSIZE(value, new_value) \ (void)_Py_atomic_add_ssize(&value, new_value) #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) PyMutex_LockFlags(lock, flags) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) #else @@ -201,6 +202,7 @@ extern "C" { #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value) #define FT_MUTEX_LOCK(lock) do {} while (0) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) do {} while (0) #define FT_MUTEX_UNLOCK(lock) do {} while (0) #endif diff --git a/Lib/test/test_free_threading/test_dict_watcher.py b/Lib/test/test_free_threading/test_dict_watcher.py new file mode 100644 index 00000000000000..6a6843f9344f64 --- /dev/null +++ b/Lib/test/test_free_threading/test_dict_watcher.py @@ -0,0 +1,89 @@ +import unittest + +from test.support import import_helper, threading_helper + +_testcapi = import_helper.import_module("_testcapi") + +ITERS = 100 +NTHREADS = 20 + + +@threading_helper.requires_working_threading() +class TestDictWatcherThreadSafety(unittest.TestCase): + # Watcher kinds from _testcapi + EVENTS = 0 # appends dict events as strings to global event list + + def test_concurrent_add_clear_watchers(self): + """Race AddWatcher and ClearWatcher from multiple threads. + + Uses more threads than available watcher slots (5 user slots out + of DICT_MAX_WATCHERS=8). + """ + results = [] + + def worker(): + for _ in range(ITERS): + try: + wid = _testcapi.add_dict_watcher(self.EVENTS) + except RuntimeError: + continue # All slots taken + self.assertGreaterEqual(wid, 0) + results.append(wid) + _testcapi.clear_dict_watcher(wid) + + threading_helper.run_concurrently(worker, NTHREADS) + + # Verify at least some watchers were successfully added + self.assertGreater(len(results), 0) + + def test_concurrent_watch_unwatch(self): + """Race Watch and Unwatch on the same dict from multiple threads.""" + wid = _testcapi.add_dict_watcher(self.EVENTS) + dicts = [{} for _ in range(10)] + + def worker(): + for _ in range(ITERS): + for d in dicts: + _testcapi.watch_dict(wid, d) + for d in dicts: + _testcapi.unwatch_dict(wid, d) + + try: + threading_helper.run_concurrently(worker, NTHREADS) + + # Verify watching still works after concurrent watch/unwatch + _testcapi.watch_dict(wid, dicts[0]) + dicts[0]["key"] = "value" + events = _testcapi.get_dict_watcher_events() + self.assertIn("new:key:value", events) + finally: + _testcapi.clear_dict_watcher(wid) + + def test_concurrent_modify_watched_dict(self): + """Race dict mutations (triggering callbacks) with watch/unwatch.""" + wid = _testcapi.add_dict_watcher(self.EVENTS) + d = {} + _testcapi.watch_dict(wid, d) + + def mutator(): + for i in range(ITERS): + d[f"key_{i}"] = i + d.pop(f"key_{i}", None) + + def toggler(): + for i in range(ITERS): + _testcapi.watch_dict(wid, d) + d[f"toggler_{i}"] = i + _testcapi.unwatch_dict(wid, d) + + workers = [mutator, toggler] * (NTHREADS // 2) + try: + threading_helper.run_concurrently(workers) + events = _testcapi.get_dict_watcher_events() + self.assertGreater(len(events), 0) + finally: + _testcapi.clear_dict_watcher(wid) + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/C_API/2026-02-25-13-37-10.gh-issue-145235.-1ySNR.rst b/Misc/NEWS.d/next/C_API/2026-02-25-13-37-10.gh-issue-145235.-1ySNR.rst new file mode 100644 index 00000000000000..98a8c268735726 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-02-25-13-37-10.gh-issue-145235.-1ySNR.rst @@ -0,0 +1,3 @@ +Made :c:func:`PyDict_AddWatcher`, :c:func:`PyDict_ClearWatcher`, +:c:func:`PyDict_Watch`, and :c:func:`PyDict_Unwatch` thread-safe on the +:term:`free threaded ` build. diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index e0abf6b1845d8e..71cdc54009017a 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -9,6 +9,7 @@ #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_interp_structs.h" // CODE_MAX_WATCHERS #include "pycore_context.h" // CONTEXT_MAX_WATCHERS +#include "pycore_lock.h" // _PyOnceFlag /*[clinic input] module _testcapi @@ -18,6 +19,14 @@ module _testcapi // Test dict watching static PyObject *g_dict_watch_events = NULL; static int g_dict_watchers_installed = 0; +static _PyOnceFlag g_dict_watch_once = {0}; + +static int +_init_dict_watch_events(void *arg) +{ + g_dict_watch_events = PyList_New(0); + return g_dict_watch_events ? 0 : -1; +} static int dict_watch_callback(PyDict_WatchEvent event, @@ -106,13 +115,10 @@ add_dict_watcher(PyObject *self, PyObject *kind) if (watcher_id < 0) { return NULL; } - if (!g_dict_watchers_installed) { - assert(!g_dict_watch_events); - if (!(g_dict_watch_events = PyList_New(0))) { - return NULL; - } + if (_PyOnceFlag_CallOnce(&g_dict_watch_once, _init_dict_watch_events, NULL) < 0) { + return NULL; } - g_dict_watchers_installed++; + _Py_atomic_add_int(&g_dict_watchers_installed, 1); return PyLong_FromLong(watcher_id); } @@ -122,10 +128,8 @@ clear_dict_watcher(PyObject *self, PyObject *watcher_id) if (PyDict_ClearWatcher(PyLong_AsLong(watcher_id))) { return NULL; } - g_dict_watchers_installed--; - if (!g_dict_watchers_installed) { - assert(g_dict_watch_events); - Py_CLEAR(g_dict_watch_events); + if (_Py_atomic_add_int(&g_dict_watchers_installed, -1) == 1) { + PyList_Clear(g_dict_watch_events); } Py_RETURN_NONE; } @@ -164,7 +168,7 @@ _testcapi_unwatch_dict_impl(PyObject *module, int watcher_id, PyObject *dict) static PyObject * get_dict_watcher_events(PyObject *self, PyObject *Py_UNUSED(args)) { - if (!g_dict_watch_events) { + if (_Py_atomic_load_int(&g_dict_watchers_installed) <= 0) { PyErr_SetString(PyExc_RuntimeError, "no watchers active"); return NULL; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 42bc63acd9049c..09135e031e6fc7 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -8015,13 +8015,19 @@ validate_watcher_id(PyInterpreterState *interp, int watcher_id) PyErr_Format(PyExc_ValueError, "Invalid dict watcher ID %d", watcher_id); return -1; } - if (!interp->dict_state.watchers[watcher_id]) { + PyDict_WatchCallback cb = FT_ATOMIC_LOAD_PTR_RELAXED( + interp->dict_state.watchers[watcher_id]); + if (cb == NULL) { PyErr_Format(PyExc_ValueError, "No dict watcher set for ID %d", watcher_id); return -1; } return 0; } +// In free-threaded builds, Add/Clear serialize on watcher_mutex and publish +// callbacks with release stores. SendEvent reads them lock-free using +// acquire loads. + int PyDict_Watch(int watcher_id, PyObject* dict) { @@ -8033,7 +8039,8 @@ PyDict_Watch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - FT_ATOMIC_OR_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, (1LL << watcher_id)); + FT_ATOMIC_OR_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, + 1ULL << watcher_id); return 0; } @@ -8048,36 +8055,48 @@ PyDict_Unwatch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - FT_ATOMIC_AND_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, ~(1LL << watcher_id)); + FT_ATOMIC_AND_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, + ~(1ULL << watcher_id)); return 0; } int PyDict_AddWatcher(PyDict_WatchCallback callback) { + int watcher_id = -1; PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_MUTEX_LOCK_FLAGS(&interp->dict_state.watcher_mutex, + _Py_LOCK_DONT_DETACH); /* Some watchers are reserved for CPython, start at the first available one */ for (int i = FIRST_AVAILABLE_WATCHER; i < DICT_MAX_WATCHERS; i++) { if (!interp->dict_state.watchers[i]) { - interp->dict_state.watchers[i] = callback; - return i; + FT_ATOMIC_STORE_PTR_RELEASE(interp->dict_state.watchers[i], callback); + watcher_id = i; + goto done; } } - PyErr_SetString(PyExc_RuntimeError, "no more dict watcher IDs available"); - return -1; +done: + FT_MUTEX_UNLOCK(&interp->dict_state.watcher_mutex); + return watcher_id; } int PyDict_ClearWatcher(int watcher_id) { + int res = 0; PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_MUTEX_LOCK_FLAGS(&interp->dict_state.watcher_mutex, + _Py_LOCK_DONT_DETACH); if (validate_watcher_id(interp, watcher_id)) { - return -1; + res = -1; + goto done; } - interp->dict_state.watchers[watcher_id] = NULL; - return 0; + FT_ATOMIC_STORE_PTR_RELEASE(interp->dict_state.watchers[watcher_id], NULL); +done: + FT_MUTEX_UNLOCK(&interp->dict_state.watcher_mutex); + return res; } static const char * @@ -8102,7 +8121,8 @@ _PyDict_SendEvent(int watcher_bits, PyInterpreterState *interp = _PyInterpreterState_GET(); for (int i = 0; i < DICT_MAX_WATCHERS; i++) { if (watcher_bits & 1) { - PyDict_WatchCallback cb = interp->dict_state.watchers[i]; + PyDict_WatchCallback cb = FT_ATOMIC_LOAD_PTR_ACQUIRE( + interp->dict_state.watchers[i]); if (cb && (cb(event, (PyObject*)mp, key, value) < 0)) { // We don't want to resurrect the dict by potentially having an // unraisablehook keep a reference to it, so we don't pass the diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 1dc3a248f45f0c..e726dc0e6fd111 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -18,6 +18,7 @@ #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_uop_metadata.h" #include "pycore_long.h" @@ -127,7 +128,7 @@ static void increment_mutations(PyObject* dict) { assert(PyDict_CheckExact(dict)); PyDictObject *d = (PyDictObject *)dict; - FT_ATOMIC_ADD_UINT64(d->_ma_watcher_tag, (1 << DICT_MAX_WATCHERS)); + FT_ATOMIC_ADD_UINT64(d->_ma_watcher_tag, 1ULL << DICT_MAX_WATCHERS); } /* The first two dict watcher IDs are reserved for CPython, @@ -156,6 +157,17 @@ type_watcher_callback(PyTypeObject* type) return 0; } +static int +_setup_optimizer_watchers(void *Py_UNUSED(arg)) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_ATOMIC_STORE_PTR_RELEASE( + interp->dict_state.watchers[GLOBALS_WATCHER_ID], + globals_watcher_callback); + interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; + return 0; +} + static void watch_type(PyTypeObject *type, _PyBloomFilter *filter) { @@ -580,10 +592,8 @@ optimize_uops( // Make sure that watchers are set up PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { - interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; - interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; - } + _PyOnceFlag_CallOnce(&interp->dict_state.watcher_setup_once, + _setup_optimizer_watchers, NULL); _Py_uop_abstractcontext_init(ctx, dependencies); _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, (PyCodeObject *)func->func_code, NULL, 0); diff --git a/Python/pystate.c b/Python/pystate.c index bf2616a49148a7..ff712019affbf9 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -320,6 +320,7 @@ _Py_COMP_DIAG_POP &(runtime)->allocators.mutex, \ &(runtime)->_main_interpreter.types.mutex, \ &(runtime)->_main_interpreter.code_state.mutex, \ + &(runtime)->_main_interpreter.dict_state.watcher_mutex, \ } static void diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 7af64ed017ba73..ddfb93a424c018 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -467,6 +467,7 @@ Modules/_testcapi/object.c - MyObject_dealloc_called - Modules/_testcapi/object.c - MyType - Modules/_testcapi/structmember.c - test_structmembersType_OldAPI - Modules/_testcapi/watchers.c - g_dict_watch_events - +Modules/_testcapi/watchers.c - g_dict_watch_once - Modules/_testcapi/watchers.c - g_dict_watchers_installed - Modules/_testcapi/watchers.c - g_type_modified_events - Modules/_testcapi/watchers.c - g_type_watchers_installed -