Merge branch '3.14' into backport-5915a1f-3.14

Author: AbduazizZiyodov

Doc/library/pyexpat.rst

@@ -238,16 +238,71 @@ XMLParser Objects
    .. versionadded:: 3.13
 
 
-:class:`!xmlparser` objects have the following methods to mitigate some
-common XML vulnerabilities.
+:class:`!xmlparser` objects have the following methods to tune protections
+against some common XML vulnerabilities.
+
+.. method:: xmlparser.SetBillionLaughsAttackProtectionActivationThreshold(threshold, /)
+
+   Sets the number of output bytes needed to activate protection against
+   `billion laughs`_ attacks.
+
+   The number of output bytes includes amplification from entity expansion
+   and reading DTD files.
+
+   Parser objects usually have a protection activation threshold of 8 MiB,
+   but the actual default value depends on the underlying Expat library.
+
+   An :exc:`ExpatError` is raised if this method is called on a
+   |xml-non-root-parser| parser.
+   The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset`
+   should not be used as they may have no special meaning.
+
+   .. note::
+
+      Activation thresholds below 4 MiB are known to break support for DITA 1.3
+      payload and are hence not recommended.
+
+   .. versionadded:: next
+
+.. method:: xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification(max_factor, /)
+
+   Sets the maximum tolerated amplification factor for protection against
+   `billion laughs`_ attacks.
+
+   The amplification factor is calculated as ``(direct + indirect) / direct``
+   while parsing, where ``direct`` is the number of bytes read from
+   the primary document in parsing and ``indirect`` is the number of
+   bytes added by expanding entities and reading of external DTD files.
+
+   The *max_factor* value must be a non-NaN :class:`float` value greater than
+   or equal to 1.0. Peak amplifications of factor 15,000 for the entire payload
+   and of factor 30,000 in the middle of parsing have been observed with small
+   benign files in practice. In particular, the activation threshold should be
+   carefully chosen to avoid false positives.
+
+   Parser objects usually have a maximum amplification factor of 100,
+   but the actual default value depends on the underlying Expat library.
+
+   An :exc:`ExpatError` is raised if this method is called on a
+   |xml-non-root-parser| parser or if *max_factor* is outside the valid range.
+   The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset`
+   should not be used as they may have no special meaning.
+
+   .. note::
+
+      The maximum amplification factor is only considered if the threshold
+      that can be adjusted by :meth:`.SetBillionLaughsAttackProtectionActivationThreshold`
+      is exceeded.
+
+   .. versionadded:: next
 
 .. method:: xmlparser.SetAllocTrackerActivationThreshold(threshold, /)
 
    Sets the number of allocated bytes of dynamic memory needed to activate
    protection against disproportionate use of RAM.
 
-   By default, parser objects have an allocation activation threshold of 64 MiB,
-   or equivalently 67,108,864 bytes.
+   Parser objects usually have an allocation activation threshold of 64 MiB,
+   but the actual default value depends on the underlying Expat library.
 
    An :exc:`ExpatError` is raised if this method is called on a
    |xml-non-root-parser| parser.
@@ -271,7 +326,8 @@ common XML vulnerabilities.
    near the start of parsing even with benign files in practice. In particular,
    the activation threshold should be carefully chosen to avoid false positives.
 
-   By default, parser objects have a maximum amplification factor of 100.0.
+   Parser objects usually have a maximum amplification factor of 100,
+   but the actual default value depends on the underlying Expat library.
 
    An :exc:`ExpatError` is raised if this method is called on a
    |xml-non-root-parser| parser or if *max_factor* is outside the valid range.
@@ -1019,4 +1075,6 @@ The ``errors`` module has the following attributes:
    not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl
    and https://www.iana.org/assignments/character-sets/character-sets.xhtml.
 
+
+.. _billion laughs: https://en.wikipedia.org/wiki/Billion_laughs_attack
 .. |xml-non-root-parser| replace:: :ref:`non-root <xmlparser-non-root>`

Doc/library/shutil.rst

@@ -388,10 +388,14 @@ Directory and files operations
    If *dst* already exists but is not a directory, it may be overwritten
    depending on :func:`os.rename` semantics.
 
-   If the destination is on the current filesystem, then :func:`os.rename` is
-   used. Otherwise, *src* is copied to the destination using *copy_function*
-   and then removed.  In case of symlinks, a new symlink pointing to the target
-   of *src* will be created as the destination and *src* will be removed.
+   :func:`os.rename` is preferably used internally when *src* and the destination are on
+   the same filesystem. In case :func:`os.rename` fails due to :exc:`OSError`
+   (e.g. the user has write permission to the destination file but not to its parent
+   directory), this method falls back to using *copy_function*, in which case
+   *src* is copied to the destination using *copy_function* and then removed.
+
+   In case of symlinks, a new symlink pointing to the target of *src* will be
+   created in or as the destination, and *src* will be removed.
 
    If *copy_function* is given, it must be a callable that takes two arguments,
    *src* and the destination, and will be used to copy *src* to the destination

Include/pyexpat.h

@@ -57,6 +57,11 @@ struct PyExpat_CAPI
         XML_Parser parser, unsigned long long activationThresholdBytes);
     XML_Bool (*SetAllocTrackerMaximumAmplification)(
         XML_Parser parser, float maxAmplificationFactor);
+    /* might be NULL for expat < 2.4.0 */
+    XML_Bool (*SetBillionLaughsAttackProtectionActivationThreshold)(
+        XML_Parser parser, unsigned long long activationThresholdBytes);
+    XML_Bool (*SetBillionLaughsAttackProtectionMaximumAmplification)(
+        XML_Parser parser, float maxAmplificationFactor);
     /* always add new stuff to the end! */
 };
 

Lib/shutil.py

@@ -885,10 +885,14 @@ def move(src, dst, copy_function=copy2):
     If dst already exists but is not a directory, it may be overwritten
     depending on os.rename() semantics.
 
-    If the destination is on our current filesystem, then rename() is used.
-    Otherwise, src is copied to the destination and then removed. Symlinks are
-    recreated under the new name if os.rename() fails because of cross
-    filesystem renames.
+    os.rename() is preferably used if the source and destination are on the
+    same filesystem. In case os.rename() fails due to OSError (e.g. the user
+    has write permission to *dst* file but not to its parent directory),
+    this method falls back to using *copy_function* silently.
+    Symlinks are also recreated under the new name if os.rename() fails
+    because of cross filesystem renames.
+
+    It's recommended to use os.rename() if atomic move is strictly required.
 
     The optional `copy_function` argument is a callable that will be used
     to copy the source or it will be delegated to `copytree`.

Lib/test/datetimetester.py

@@ -7313,6 +7313,36 @@ def func():
         self.assertEqual(out, b"a" * 8)
         self.assertEqual(err, b"")
 
+    @support.cpython_only
+    @support.subTests(("setup", "call"), [
+        ("obj = _datetime.timedelta", "obj(seconds=2)"),
+        ("obj = _datetime.timedelta(seconds=2)", "obj.total_seconds()"),
+        ("obj = _datetime.date(2026, 6, 7)", "obj.isocalendar()"),
+    ])
+    def test_static_datetime_types_outlive_collected_module(self, setup, call):
+        # gh-151039: This code used to crash
+        script = f"""if True:
+            import sys, gc
+            import _datetime
+
+            {setup}                          # static C type, survives the module
+            del sys.modules['_datetime']
+            del _datetime
+            sys.modules['_datetime'] = None  # block re-import
+            gc.collect()                     # module object is collected
+
+            try:
+                {call}                       # used to be a segmentation fault
+            except ImportError:
+                pass
+            else:
+                raise AssertionError("ImportError not raised")
+        """
+        rc, out, err = script_helper.assert_python_ok("-c", script)
+        self.assertEqual(rc, 0)
+        self.assertEqual(out, b'')
+        self.assertEqual(err, b'')
+
 
 def load_tests(loader, standard_tests, pattern):
     standard_tests.addTest(ZoneInfoCompleteTest())

Lib/test/test_capi/test_weakref.py

@@ -0,0 +1,136 @@
+import weakref
+import unittest
+from test.support import import_helper
+
+_testcapi = import_helper.import_module('_testcapi')
+_testlimitedcapi = import_helper.import_module('_testlimitedcapi')
+NULL = None
+
+class Object:
+    pass
+
+class Ref(weakref.ReferenceType):
+    pass
+
+
+class CAPIWeakrefTest(unittest.TestCase):
+    def test_pyweakref_check(self):
+        # Test PyWeakref_Check()
+        check = _testlimitedcapi.pyweakref_check
+        obj = Object()
+        self.assertEqual(check(obj), 0)
+        self.assertEqual(check(weakref.ref(obj)), 1)
+        self.assertEqual(check(Ref(obj)), 1)
+        self.assertEqual(check(weakref.proxy(obj)), 1)
+
+        # CRASHES check(NULL)
+
+    def test_pyweakref_checkref(self):
+        # Test PyWeakref_CheckRef()
+        checkref = _testlimitedcapi.pyweakref_checkref
+        obj = Object()
+        self.assertEqual(checkref(obj), 0)
+        self.assertEqual(checkref(weakref.ref(obj)), 1)
+        self.assertEqual(checkref(Ref(obj)), 1)
+        self.assertEqual(checkref(weakref.proxy(obj)), 0)
+
+        # CRASHES checkref(NULL)
+
+    def test_pyweakref_checkrefexact(self):
+        # Test PyWeakref_CheckRefExact()
+        checkrefexact = _testlimitedcapi.pyweakref_checkrefexact
+        obj = Object()
+        self.assertEqual(checkrefexact(obj), 0)
+        self.assertEqual(checkrefexact(weakref.ref(obj)), 1)
+        self.assertEqual(checkrefexact(Ref(obj)), 0)
+        self.assertEqual(checkrefexact(weakref.proxy(obj)), 0)
+
+        # CRASHES checkrefexact(NULL)
+
+    def test_pyweakref_checkproxy(self):
+        # Test PyWeakref_CheckProxy()
+        checkproxy = _testlimitedcapi.pyweakref_checkproxy
+        obj = Object()
+        self.assertEqual(checkproxy(obj), 0)
+        self.assertEqual(checkproxy(weakref.ref(obj)), 0)
+        self.assertEqual(checkproxy(Ref(obj)), 0)
+        self.assertEqual(checkproxy(weakref.proxy(obj)), 1)
+
+        # CRASHES checkproxy(NULL)
+
+    def test_pyweakref_getref(self):
+        # Test PyWeakref_GetRef()
+        getref = _testcapi.pyweakref_getref
+        obj = Object()
+        wr = weakref.ref(obj)
+        wp = weakref.proxy(obj)
+        self.assertEqual(getref(wr), (1, obj))
+        self.assertEqual(getref(wp), (1, obj))
+        del obj
+        self.assertEqual(getref(wr), 0)
+        self.assertEqual(getref(wp), 0)
+
+        self.assertRaises(TypeError, getref, 42)
+        self.assertRaises(SystemError, getref, NULL)
+
+    def test_pyweakref_isdead(self):
+        # Test PyWeakref_IsDead()
+        isdead = _testcapi.pyweakref_isdead
+        obj = Object()
+        wr = weakref.ref(obj)
+        wp = weakref.proxy(obj)
+        self.assertEqual(isdead(wr), 0)
+        self.assertEqual(isdead(wp), 0)
+        del obj
+        self.assertEqual(isdead(wr), 1)
+        self.assertEqual(isdead(wp), 1)
+
+        self.assertRaises(TypeError, isdead, 42)
+        self.assertRaises(SystemError, isdead, NULL)
+
+    def test_pyweakref_newref(self):
+        # Test PyWeakref_NewRef()
+        newref = _testlimitedcapi.pyweakref_newref
+        obj = Object()
+        wr = newref(obj)
+        self.assertIs(type(wr), weakref.ReferenceType)
+        # PyWeakref_NewRef() handles None callback as NULL callback
+        wr = newref(obj, None)
+        self.assertIs(type(wr), weakref.ReferenceType)
+        log = []
+        wr = newref(obj, log.append)
+        self.assertIs(type(wr), weakref.ReferenceType)
+        self.assertEqual(log, [])
+        del obj
+        self.assertEqual(log, [wr])
+
+        self.assertRaises(TypeError, newref, [])
+        # CRASHES newref(NULL)
+
+    def test_pyweakref_newproxy(self):
+        # Test PyWeakref_NewProxy()
+        newproxy = _testlimitedcapi.pyweakref_newproxy
+        obj = Object()
+        wp = newproxy(obj)
+        self.assertIs(type(wp), weakref.ProxyType)
+        # PyWeakref_NewProxy() handles None callback as NULL callback
+        wp = newproxy(obj, None)
+        self.assertIs(type(wp), weakref.ProxyType)
+        log = []
+        wp = newproxy(obj, log.append)
+        self.assertIs(type(wp), weakref.ProxyType)
+        self.assertEqual(log, [])
+        del obj
+        self.assertEqual(log, [wp])
+
+        def func():
+            pass
+        wp = newproxy(func)
+        self.assertIs(type(wp), weakref.CallableProxyType)
+
+        self.assertRaises(TypeError, newproxy, [])
+        # CRASHES newproxy(NULL)
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_pyexpat.py

@@ -1183,6 +1183,64 @@ def test_set_maximum_amplification__fail_for_subparser(self):
         self.assert_root_parser_failure(setter, 123.45)
 
 
+@unittest.skipIf(expat.version_info < (2, 4, 0), "requires Expat >= 2.4.0")
+class ExpansionProtectionTest(AttackProtectionTestBase, unittest.TestCase):
+
+    def assert_rejected(self, func, /, *args, **kwargs):
+        """Check that func(*args, **kwargs) hits the allocation limit."""
+        msg = (
+            r"limit on input amplification factor \(from DTD and entities\) "
+            r"breached: line \d+, column \d+"
+        )
+        self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs)
+
+    def set_activation_threshold(self, parser, threshold):
+        return parser.SetBillionLaughsAttackProtectionActivationThreshold(threshold)
+
+    def set_maximum_amplification(self, parser, max_factor):
+        return parser.SetBillionLaughsAttackProtectionMaximumAmplification(max_factor)
+
+    def test_set_activation_threshold__threshold_reached(self):
+        parser = expat.ParserCreate()
+        # Choose a threshold expected to be always reached.
+        self.set_activation_threshold(parser, 3)
+        # Check that the threshold is reached by choosing a small factor
+        # and a payload whose peak amplification factor exceeds it.
+        self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
+        payload = self.exponential_expansion_payload(ncols=10, nrows=4)
+        self.assert_rejected(parser.Parse, payload, True)
+
+    def test_set_activation_threshold__threshold_not_reached(self):
+        parser = expat.ParserCreate()
+        # Choose a threshold expected to be never reached.
+        self.set_activation_threshold(parser, pow(10, 5))
+        # Check that the threshold is reached by choosing a small factor
+        # and a payload whose peak amplification factor exceeds it.
+        self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
+        payload = self.exponential_expansion_payload(ncols=10, nrows=4)
+        self.assertIsNotNone(parser.Parse(payload, True))
+
+    def test_set_maximum_amplification__amplification_exceeded(self):
+        parser = expat.ParserCreate()
+        # Unconditionally enable maximum activation factor.
+        self.set_activation_threshold(parser, 0)
+        # Choose a max amplification factor expected to always be exceeded.
+        self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
+        # Craft a payload for which the peak amplification factor is > 1.0.
+        payload = self.exponential_expansion_payload(ncols=1, nrows=2)
+        self.assert_rejected(parser.Parse, payload, True)
+
+    def test_set_maximum_amplification__amplification_not_exceeded(self):
+        parser = expat.ParserCreate()
+        # Unconditionally enable maximum activation factor.
+        self.set_activation_threshold(parser, 0)
+        # Choose a max amplification factor expected to never be exceeded.
+        self.assertIsNone(self.set_maximum_amplification(parser, 1e4))
+        # Craft a payload for which the peak amplification factor is < 1e4.
+        payload = self.exponential_expansion_payload(ncols=1, nrows=2)
+        self.assertIsNotNone(parser.Parse(payload, True))
+
+
 @unittest.skipIf(not hasattr(expat.XMLParserType,
                              "SetAllocTrackerMaximumAmplification"),
                  "requires Python compiled with Expat >= 2.7.2")

Mac/BuildScript/build-installer.py

@@ -378,9 +378,9 @@ def library_recipes():
               install=f"make && ranlib libsqlite3.a && make install DESTDIR={shellQuote(os.path.join(WORKDIR, 'libraries'))}",
           ),
           dict(
-              name="libmpdec 4.0.0",
-              url="https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-4.0.0.tar.gz",
-              checksum="942445c3245b22730fd41a67a7c5c231d11cb1b9936b9c0f76334fb7d0b4468c",
+              name="libmpdec 4.0.1",
+              url="https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-4.0.1.tar.gz",
+              checksum="96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8",
               configure_pre=[
                   "--disable-cxx",
                   "MACHINE=universal",