From a02cf19deed353d1e0e7564468f10aced61c12e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 15 Jul 2025 14:03:21 +0200 Subject: [PATCH 1/8] gh-72570: mention the incompatibility of XOFs with HMAC (#136676) --- Doc/library/hmac.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index d6692033b2d4c3..57076c38086c79 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -12,6 +12,9 @@ -------------- This module implements the HMAC algorithm as described by :rfc:`2104`. +The interface allows to use any hash function with a *fixed* digest size. +In particular, extendable output functions such as SHAKE-128 or SHAKE-256 +cannot be used with HMAC. .. function:: new(key, msg=None, digestmod) From 624bf52c83abcb1f948f9059e29729fa94d38086 Mon Sep 17 00:00:00 2001 From: Maciej Olko Date: Tue, 15 Jul 2025 14:26:24 +0200 Subject: [PATCH 2/8] gh-136155: Docs: check for EPUB fatal errors in CI (#134074) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- .github/workflows/reusable-docs.yml | 14 ++++++++++- Doc/conf.py | 1 + Doc/tools/check-epub.py | 24 +++++++++++++++++++ ...-07-01-23-00-58.gh-issue-136155.4siQQO.rst | 1 + 4 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 Doc/tools/check-epub.py create mode 100644 Misc/NEWS.d/next/Documentation/2025-07-01-23-00-58.gh-issue-136155.4siQQO.rst diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 657e0a6bf662f7..7b9dc4818577eb 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -66,7 +66,7 @@ jobs: run: | set -Eeuo pipefail # Build docs with the nit-picky option; write warnings to file - make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --fail-on-warning --warning-file sphinx-warnings.txt" html + make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --warning-file sphinx-warnings.txt" html - name: 'Check warnings' if: github.event_name == 'pull_request' run: | @@ -75,6 +75,18 @@ jobs: --fail-if-regression \ --fail-if-improved \ --fail-if-new-news-nit + - name: 'Build EPUB documentation' + continue-on-error: true + run: | + set -Eeuo pipefail + make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet" epub + pip install epubcheck + epubcheck Doc/build/epub/Python.epub &> Doc/epubcheck.txt + - name: 'Check for fatal errors in EPUB' + if: github.event_name == 'pull_request' + continue-on-error: true # until gh-136155 is fixed + run: | + python Doc/tools/check-epub.py # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: diff --git a/Doc/conf.py b/Doc/conf.py index c1ed94d7b46ec2..1c1f36e5bc0737 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -448,6 +448,7 @@ epub_author = 'Python Documentation Authors' epub_publisher = 'Python Software Foundation' +epub_exclude_files = ('index.xhtml', 'download.xhtml') # index pages are not valid xhtml # https://github.com/sphinx-doc/sphinx/issues/12359 diff --git a/Doc/tools/check-epub.py b/Doc/tools/check-epub.py new file mode 100644 index 00000000000000..693dc239c8ad58 --- /dev/null +++ b/Doc/tools/check-epub.py @@ -0,0 +1,24 @@ +import sys +from pathlib import Path + + +def main() -> int: + wrong_directory_msg = "Must run this script from the repo root" + if not Path("Doc").exists() or not Path("Doc").is_dir(): + raise RuntimeError(wrong_directory_msg) + + with Path("Doc/epubcheck.txt").open(encoding="UTF-8") as f: + messages = [message.split(" - ") for message in f.read().splitlines()] + + fatal_errors = [message for message in messages if message[0] == "FATAL"] + + if fatal_errors: + print("\nError: must not contain fatal errors:\n") + for error in fatal_errors: + print(" - ".join(error)) + + return len(fatal_errors) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Misc/NEWS.d/next/Documentation/2025-07-01-23-00-58.gh-issue-136155.4siQQO.rst b/Misc/NEWS.d/next/Documentation/2025-07-01-23-00-58.gh-issue-136155.4siQQO.rst new file mode 100644 index 00000000000000..70f54936c80f55 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2025-07-01-23-00-58.gh-issue-136155.4siQQO.rst @@ -0,0 +1 @@ +We are now checking for fatal errors in EPUB builds in CI. From 7e10a103dfe52feb0ef3d541e08abc2640838101 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Tue, 15 Jul 2025 15:49:11 +0300 Subject: [PATCH 3/8] gh-136682: Remove incorrect statement that `os.path.samestat` accepts file-like objects (#136683) --- Doc/library/os.path.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 1c1cf07a655ae7..abb0131d7d058e 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -508,9 +508,6 @@ the :mod:`glob` module.) .. versionchanged:: 3.4 Added Windows support. - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - .. function:: split(path) From a8f42e6e884e7d63d5d63a817bc490f3bbbdba17 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Tue, 15 Jul 2025 19:15:11 +0530 Subject: [PATCH 4/8] gh-111968: remove redundant fetching of interpreter state in `dict` implementation (#136673) --- Objects/dictobject.c | 104 ++++++++++++++++++------------------------- 1 file changed, 44 insertions(+), 60 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index be62ae5eefd00d..0ed52ac5e87b6e 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -217,7 +217,7 @@ set_values(PyDictObject *mp, PyDictValues *values) #define LOAD_KEYS_NENTRIES(keys) _Py_atomic_load_ssize_relaxed(&keys->dk_nentries) #define INCREF_KEYS_FT(dk) dictkeys_incref(dk) -#define DECREF_KEYS_FT(dk, shared) dictkeys_decref(_PyInterpreterState_GET(), dk, shared) +#define DECREF_KEYS_FT(dk, shared) dictkeys_decref(dk, shared) static inline void split_keys_entry_added(PyDictKeysObject *keys) { @@ -380,8 +380,7 @@ equally good collision statistics, needed less code & used less memory. */ -static int dictresize(PyInterpreterState *interp, PyDictObject *mp, - uint8_t log_newsize, int unicode); +static int dictresize(PyDictObject *mp, uint8_t log_newsize, int unicode); static PyObject* dict_iter(PyObject *dict); @@ -444,7 +443,7 @@ dictkeys_incref(PyDictKeysObject *dk) } static inline void -dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) +dictkeys_decref(PyDictKeysObject *dk, bool use_qsbr) { if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) < 0) { assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_DICT_IMMORTAL_INITIAL_REFCNT); @@ -753,7 +752,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) static PyDictKeysObject* -new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) +new_keys_object(uint8_t log2_size, bool unicode) { Py_ssize_t usable; int log2_bytes; @@ -867,8 +866,7 @@ free_values(PyDictValues *values, bool use_qsbr) /* Consumes a reference to the keys object */ static PyObject * -new_dict(PyInterpreterState *interp, - PyDictKeysObject *keys, PyDictValues *values, +new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) { assert(keys != NULL); @@ -876,7 +874,7 @@ new_dict(PyInterpreterState *interp, if (mp == NULL) { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { - dictkeys_decref(interp, keys, false); + dictkeys_decref(keys, false); if (free_values_on_failure) { free_values(values, false); } @@ -894,7 +892,7 @@ new_dict(PyInterpreterState *interp, } static PyObject * -new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) +new_dict_with_shared_keys(PyDictKeysObject *keys) { size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); @@ -905,7 +903,7 @@ new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } - return new_dict(interp, keys, values, 0, 1); + return new_dict(keys, values, 0, 1); } @@ -971,9 +969,8 @@ clone_combined_dict_keys(PyDictObject *orig) PyObject * PyDict_New(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); /* We don't incref Py_EMPTY_KEYS here because it is immortal. */ - return new_dict(interp, Py_EMPTY_KEYS, NULL, 0, 0); + return new_dict(Py_EMPTY_KEYS, NULL, 0, 0); } /* Search index of hash table from offset of entry table */ @@ -1714,9 +1711,9 @@ find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) } static int -insertion_resize(PyInterpreterState *interp, PyDictObject *mp, int unicode) +insertion_resize(PyDictObject *mp, int unicode) { - return dictresize(interp, mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); + return dictresize(mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); } static inline int @@ -1725,7 +1722,7 @@ insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, { if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ - if (insertion_resize(interp, mp, 1) < 0) { + if (insertion_resize(mp, 1) < 0) { return -1; } } @@ -1823,7 +1820,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, ASSERT_DICT_LOCKED(mp); if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { - if (insertion_resize(interp, mp, 0) < 0) + if (insertion_resize(mp, 0) < 0) goto Fail; assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); } @@ -1838,7 +1835,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, } /* No space in shared keys. Resize and continue below. */ - if (insertion_resize(interp, mp, 1) < 0) { + if (insertion_resize(mp, 1) < 0) { goto Fail; } } @@ -1893,8 +1890,7 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, ASSERT_DICT_LOCKED(mp); int unicode = PyUnicode_CheckExact(key); - PyDictKeysObject *newkeys = new_keys_object( - interp, PyDict_LOG_MINSIZE, unicode); + PyDictKeysObject *newkeys = new_keys_object(PyDict_LOG_MINSIZE, unicode); if (newkeys == NULL) { Py_DECREF(key); Py_DECREF(value); @@ -1989,7 +1985,7 @@ This function supports: - Generic -> Generic */ static int -dictresize(PyInterpreterState *interp, PyDictObject *mp, +dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) { PyDictKeysObject *oldkeys, *newkeys; @@ -2017,7 +2013,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, */ /* Allocate a new table. */ - newkeys = new_keys_object(interp, log2_newsize, unicode); + newkeys = new_keys_object(log2_newsize, unicode); if (newkeys == NULL) { return -1; } @@ -2060,7 +2056,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, } UNLOCK_KEYS(oldkeys); set_keys(mp, newkeys); - dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + dictkeys_decref(oldkeys, IS_DICT_SHARED(mp)); set_values(mp, NULL); if (oldvalues->embedded) { assert(oldvalues->embedded == 1); @@ -2141,7 +2137,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, } static PyObject * -dict_new_presized(PyInterpreterState *interp, Py_ssize_t minused, bool unicode) +dict_new_presized(Py_ssize_t minused, bool unicode) { const uint8_t log2_max_presize = 17; const Py_ssize_t max_presize = ((Py_ssize_t)1) << log2_max_presize; @@ -2162,17 +2158,16 @@ dict_new_presized(PyInterpreterState *interp, Py_ssize_t minused, bool unicode) log2_newsize = estimate_log2_keysize(minused); } - new_keys = new_keys_object(interp, log2_newsize, unicode); + new_keys = new_keys_object(log2_newsize, unicode); if (new_keys == NULL) return NULL; - return new_dict(interp, new_keys, NULL, 0, 0); + return new_dict(new_keys, NULL, 0, 0); } PyObject * _PyDict_NewPresized(Py_ssize_t minused) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - return dict_new_presized(interp, minused, false); + return dict_new_presized(minused, false); } PyObject * @@ -2182,7 +2177,6 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, { bool unicode = true; PyObject *const *ks = keys; - PyInterpreterState *interp = _PyInterpreterState_GET(); for (Py_ssize_t i = 0; i < length; i++) { if (!PyUnicode_CheckExact(*ks)) { @@ -2192,7 +2186,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, ks += keys_offset; } - PyObject *dict = dict_new_presized(interp, length, unicode); + PyObject *dict = dict_new_presized(length, unicode); if (dict == NULL) { return NULL; } @@ -2895,7 +2889,7 @@ clear_lock_held(PyObject *op) if (oldvalues == NULL) { set_keys(mp, Py_EMPTY_KEYS); assert(oldkeys->dk_refcnt == 1); - dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + dictkeys_decref(oldkeys, IS_DICT_SHARED(mp)); } else { n = oldkeys->dk_nentries; @@ -2909,7 +2903,7 @@ clear_lock_held(PyObject *op) set_values(mp, NULL); set_keys(mp, Py_EMPTY_KEYS); free_values(oldvalues, IS_DICT_SHARED(mp)); - dictkeys_decref(interp, oldkeys, false); + dictkeys_decref(oldkeys, false); } } ASSERT_CONSISTENT(mp); @@ -3161,7 +3155,7 @@ dict_dict_fromkeys(PyInterpreterState *interp, PyDictObject *mp, uint8_t new_size = Py_MAX( estimate_log2_keysize(PyDict_GET_SIZE(iterable)), DK_LOG_SIZE(mp->ma_keys)); - if (dictresize(interp, mp, new_size, unicode)) { + if (dictresize(mp, new_size, unicode)) { Py_DECREF(mp); return NULL; } @@ -3186,7 +3180,7 @@ dict_set_fromkeys(PyInterpreterState *interp, PyDictObject *mp, uint8_t new_size = Py_MAX( estimate_log2_keysize(PySet_GET_SIZE(iterable)), DK_LOG_SIZE(mp->ma_keys)); - if (dictresize(interp, mp, new_size, 0)) { + if (dictresize(mp, new_size, 0)) { Py_DECREF(mp); return NULL; } @@ -3298,11 +3292,11 @@ dict_dealloc(PyObject *self) } free_values(values, false); } - dictkeys_decref(interp, keys, false); + dictkeys_decref(keys, false); } else if (keys != NULL) { assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); - dictkeys_decref(interp, keys, false); + dictkeys_decref(keys, false); } if (Py_IS_TYPE(mp, &PyDict_Type)) { _Py_FREELIST_FREE(dicts, mp, Py_TYPE(mp)->tp_free); @@ -3832,7 +3826,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe return -1; ensure_shared_on_resize(mp); - dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); + dictkeys_decref(mp->ma_keys, IS_DICT_SHARED(mp)); set_keys(mp, keys); STORE_USED(mp, other->ma_used); ASSERT_CONSISTENT(mp); @@ -3851,8 +3845,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe */ if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { int unicode = DK_IS_UNICODE(other->ma_keys); - if (dictresize(interp, mp, - estimate_log2_keysize(mp->ma_used + other->ma_used), + if (dictresize(mp, estimate_log2_keysize(mp->ma_used + other->ma_used), unicode)) { return -1; } @@ -4117,7 +4110,7 @@ copy_lock_held(PyObject *o) if (keys == NULL) { return NULL; } - PyDictObject *new = (PyDictObject *)new_dict(interp, keys, NULL, 0, 0); + PyDictObject *new = (PyDictObject *)new_dict(keys, NULL, 0, 0); if (new == NULL) { /* In case of an error, `new_dict()` takes care of cleaning up `keys`. */ @@ -4362,7 +4355,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } if (!PyUnicode_CheckExact(key) && DK_IS_UNICODE(mp->ma_keys)) { - if (insertion_resize(interp, mp, 0) < 0) { + if (insertion_resize(mp, 0) < 0) { if (result) { *result = NULL; } @@ -4386,7 +4379,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } /* No space in shared keys. Resize and continue below. */ - if (insertion_resize(interp, mp, 1) < 0) { + if (insertion_resize(mp, 1) < 0) { goto error; } } @@ -4555,7 +4548,7 @@ dict_popitem_impl(PyDictObject *self) } /* Convert split table to combined table */ if (_PyDict_HasSplitTable(self)) { - if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1) < 0) { + if (dictresize(self, DK_LOG_SIZE(self->ma_keys), 1) < 0) { Py_DECREF(res); return NULL; } @@ -6725,10 +6718,7 @@ dictvalues_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) PyDictKeysObject * _PyDict_NewKeysForClass(PyHeapTypeObject *cls) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - - PyDictKeysObject *keys = new_keys_object( - interp, NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); + PyDictKeysObject *keys = new_keys_object(NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); if (keys == NULL) { PyErr_Clear(); } @@ -6792,8 +6782,7 @@ _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) } static PyDictObject * -make_dict_from_instance_attributes(PyInterpreterState *interp, - PyDictKeysObject *keys, PyDictValues *values) +make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) { dictkeys_incref(keys); Py_ssize_t used = 0; @@ -6804,7 +6793,7 @@ make_dict_from_instance_attributes(PyInterpreterState *interp, used += 1; } } - PyDictObject *res = (PyDictObject *)new_dict(interp, keys, values, used, 0); + PyDictObject *res = (PyDictObject *)new_dict(keys, values, used, 0); return res; } @@ -6818,9 +6807,8 @@ _PyObject_MaterializeManagedDict_LockHeld(PyObject *obj) PyDictValues *values = _PyObject_InlineValues(obj); PyDictObject *dict; if (values->valid) { - PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); - dict = make_dict_from_instance_attributes(interp, keys, values); + dict = make_dict_from_instance_attributes(keys, values); } else { dict = (PyDictObject *)PyDict_New(); @@ -6916,7 +6904,7 @@ store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, if (dict == NULL) { // Make the dict but don't publish it in the object // so that no one else will see it. - dict = make_dict_from_instance_attributes(PyInterpreterState_Get(), keys, values); + dict = make_dict_from_instance_attributes(keys, values); if (dict == NULL || _PyDict_SetItem_LockHeld(dict, name, value) < 0) { Py_XDECREF(dict); @@ -7449,11 +7437,10 @@ PyObject_ClearManagedDict(PyObject *obj) "clearing an object managed dict"); /* Clear the dict */ Py_BEGIN_CRITICAL_SECTION(dict); - PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *oldkeys = dict->ma_keys; set_keys(dict, Py_EMPTY_KEYS); dict->ma_values = NULL; - dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(dict)); + dictkeys_decref(oldkeys, IS_DICT_SHARED(dict)); STORE_USED(dict, 0); clear_inline_values(_PyObject_InlineValues(obj)); Py_END_CRITICAL_SECTION(); @@ -7490,8 +7477,7 @@ ensure_managed_dict(PyObject *obj) goto done; } #endif - dict = (PyDictObject *)new_dict_with_shared_keys(_PyInterpreterState_GET(), - CACHED_KEYS(tp)); + dict = (PyDictObject *)new_dict_with_shared_keys(CACHED_KEYS(tp)); FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, (PyDictObject *)dict); @@ -7520,9 +7506,8 @@ ensure_nonmanaged_dict(PyObject *obj, PyObject **dictptr) #endif PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { - PyInterpreterState *interp = _PyInterpreterState_GET(); assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)); - dict = new_dict_with_shared_keys(interp, cached); + dict = new_dict_with_shared_keys(cached); } else { dict = PyDict_New(); @@ -7578,8 +7563,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, void _PyDictKeys_DecRef(PyDictKeysObject *keys) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - dictkeys_decref(interp, keys, false); + dictkeys_decref(keys, false); } static inline uint32_t From be02e68158aee4d70f15baa1d8329df2c35a57f2 Mon Sep 17 00:00:00 2001 From: Richard Si Date: Tue, 15 Jul 2025 10:25:07 -0400 Subject: [PATCH 5/8] gh-72327: Suggest using system terminal for pip install in PyREPL (#136328) Users new to Python packaging often try to use pip from the REPL only to be met with a confusing SyntaxError. If this happens, guide the user to use a system terminal instead to invoke pip. Closes #72327 --------- Co-authored-by: Tom Viner Co-authored-by: Brian Schubert Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Alyssa Coghlan --- Lib/_pyrepl/console.py | 15 ++++++++++++++- Lib/test/test_pyrepl/test_pyrepl.py | 11 +++++++++++ Misc/ACKS | 2 ++ .../2025-07-07-16-46-55.gh-issue-72327.wLvRuj.rst | 2 ++ 4 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-07-16-46-55.gh-issue-72327.wLvRuj.rst diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index 8956fb1242e52a..e0535d50396316 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -27,6 +27,7 @@ import linecache from dataclasses import dataclass, field import os.path +import re import sys @@ -195,7 +196,19 @@ def runsource(self, source, filename="", symbol="single"): ast.PyCF_ONLY_AST, incomplete_input=False, ) - except (SyntaxError, OverflowError, ValueError): + except SyntaxError as e: + # If it looks like pip install was entered (a common beginner + # mistake), provide a hint to use the system command prompt. + if re.match(r"^\s*(pip3?|py(thon3?)? -m pip) install.*", source): + e.add_note( + "The Python package manager (pip) can only be used" + " outside of the Python REPL.\n" + "Try the 'pip' command in a separate terminal or" + " command prompt." + ) + self.showsyntaxerror(filename, source=source) + return False + except (OverflowError, ValueError): self.showsyntaxerror(filename, source=source) return False if tree.body: diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 98bae7dd703fd9..657a971f8769df 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1757,3 +1757,14 @@ def test_showrefcount(self): output, _ = self.run_repl("1\n1+2\nexit()\n", cmdline_args=['-Xshowrefcount'], env=env) matches = re.findall(r'\[-?\d+ refs, \d+ blocks\]', output) self.assertEqual(len(matches), 3) + + def test_detect_pip_usage_in_repl(self): + for pip_cmd in ("pip", "pip3", "python -m pip", "python3 -m pip"): + with self.subTest(pip_cmd=pip_cmd): + output, exit_code = self.run_repl([f"{pip_cmd} install sampleproject", "exit"]) + self.assertIn("SyntaxError", output) + hint = ( + "The Python package manager (pip) can only be used" + " outside of the Python REPL" + ) + self.assertIn(hint, output) diff --git a/Misc/ACKS b/Misc/ACKS index 3814509aea030a..fabd79b9f74210 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1744,6 +1744,7 @@ Joel Shprentz Yue Shuaijie Jaysinh Shukla Terrel Shumway +Richard Si Eric Siegerman Reilly Tucker Siemens Paul Sijben @@ -1988,6 +1989,7 @@ Olivier Vielpeau Kannan Vijayan Kurt Vile Norman Vine +Tom Viner Pauli Virtanen Frank Visser Long Vo diff --git a/Misc/NEWS.d/next/Library/2025-07-07-16-46-55.gh-issue-72327.wLvRuj.rst b/Misc/NEWS.d/next/Library/2025-07-07-16-46-55.gh-issue-72327.wLvRuj.rst new file mode 100644 index 00000000000000..f305abb655a6f6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-07-16-46-55.gh-issue-72327.wLvRuj.rst @@ -0,0 +1,2 @@ +Suggest using the system command prompt when ``pip install`` is typed into +the REPL. Patch by Tom Viner, Richard Si, and Brian Schubert. From 2500eb96b260b05387d4ab1063fcfafebf37f1a4 Mon Sep 17 00:00:00 2001 From: andrewreds Date: Wed, 16 Jul 2025 01:26:16 +1000 Subject: [PATCH 6/8] gh-135909: Assert incoming `refcnt != 0` for the free threaded GC (GH-136009) This helps catch double deallocation bugs and is similar to the assertion in the GIL-enabled build. The call to `validate_refcounts` is moved up to start of the GC because `queue_untracked_obj_decref()` creates it own zero reference count garbage. --- Python/gc_free_threading.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index d46598b23b3b2f..0b0ddf227e4952 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1073,6 +1073,14 @@ validate_refcounts(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } + // This assert mirrors the one in Python/gc.c:update_refs(). There must be + // no tracked objects with a reference count of 0 when the cyclic + // collector starts. If there is, then the collector will double dealloc + // the object. The likely cause for hitting this is a faulty .tp_dealloc. + // Also see the comment in `update_refs()`. + _PyObject_ASSERT_WITH_MSG(op, Py_REFCNT(op) > 0, + "tracked objects must have a reference count > 0"); + _PyObject_ASSERT_WITH_MSG(op, !gc_is_unreachable(op), "object should not be marked as unreachable yet"); @@ -1422,13 +1430,6 @@ static int deduce_unreachable_heap(PyInterpreterState *interp, struct collection_state *state) { - -#ifdef GC_DEBUG - // Check that all objects are marked as unreachable and that the computed - // reference count difference (stored in `ob_tid`) is non-negative. - gc_visit_heaps(interp, &validate_refcounts, &state->base); -#endif - // Identify objects that are directly reachable from outside the GC heap // by computing the difference between the refcount and the number of // incoming references. @@ -2158,6 +2159,13 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, state->gcstate->old[i-1].count = 0; } +#ifdef GC_DEBUG + // Before we start, check that the heap is in a good condition. There must + // be no objects with a zero reference count. And `ob_tid` must only have a + // thread if the refcount is unmerged. + gc_visit_heaps(interp, &validate_refcounts, &state->base); +#endif + _Py_FOR_EACH_TSTATE_BEGIN(interp, p) { _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)p; From 7689407fa4406ab79d7e9e02363f50be4ec35b5e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 15 Jul 2025 18:52:51 +0300 Subject: [PATCH 7/8] Fix index entry and anchor for module.__test__ (GH-136674) It was "doctest.module attribute". Now it is "module attribute". --- Doc/library/doctest.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index fb43cf918b84dd..5a2c6bdd27c386 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -311,9 +311,13 @@ Which Docstrings Are Examined? The module docstring, and all function, class and method docstrings are searched. Objects imported into the module are not searched. +.. currentmodule:: None + .. attribute:: module.__test__ :no-typesetting: +.. currentmodule:: doctest + In addition, there are cases when you want tests to be part of a module but not part of the help text, which requires that the tests not be included in the docstring. Doctest looks for a module-level variable called ``__test__`` and uses it to locate other From cb59eaefeda5ff44ac0c742bff2b8afc023be313 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 15 Jul 2025 19:42:02 +0300 Subject: [PATCH 8/8] Fix the doctest.testmod() docstring (GH-136675) __test__ = None is not supported since Python 2.4. --- Lib/doctest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/doctest.py b/Lib/doctest.py index c8c95ecbb273b2..e77823f64b67e4 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1997,8 +1997,8 @@ def testmod(m=None, name=None, globs=None, verbose=None, from module m (or the current module if m is not supplied), starting with m.__doc__. - Also test examples reachable from dict m.__test__ if it exists and is - not None. m.__test__ maps names to functions, classes and strings; + Also test examples reachable from dict m.__test__ if it exists. + m.__test__ maps names to functions, classes and strings; function and class docstrings are tested even if the name is private; strings are tested directly, as if they were docstrings.