From 8ed5a2b56cc6a8635e586c641b0b837669f6677b Mon Sep 17 00:00:00 2001 From: Albert Zeyer Date: Sun, 7 Sep 2025 09:18:28 +0200 Subject: [PATCH 1/4] gh-118981: multiprocessing.popen_spawn_posix, fix potential hang (gh-118982) fix potential hang. It can happen that the child crashes right in the beginning for whatever reason. In this case, the parent will hang when writing into the pipe, because the child fd is not closed yet. The normal pattern is to close the child fds right after the child proc is forked/executed/spawned, so when the child dies, then also the pipes will be closed, and there will be no hang (the parent gets SIGPIPE instead). --- Lib/multiprocessing/popen_spawn_posix.py | 4 ++++ .../Library/2024-05-13-09-50-31.gh-issue-118981.zgOQPv.rst | 2 ++ 2 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-05-13-09-50-31.gh-issue-118981.zgOQPv.rst diff --git a/Lib/multiprocessing/popen_spawn_posix.py b/Lib/multiprocessing/popen_spawn_posix.py index 24b8634523e5f2..cccd659ae77637 100644 --- a/Lib/multiprocessing/popen_spawn_posix.py +++ b/Lib/multiprocessing/popen_spawn_posix.py @@ -57,6 +57,10 @@ def _launch(self, process_obj): self._fds.extend([child_r, child_w]) self.pid = util.spawnv_passfds(spawn.get_executable(), cmd, self._fds) + os.close(child_r) + child_r = None + os.close(child_w) + child_w = None self.sentinel = parent_r with open(parent_w, 'wb', closefd=False) as f: f.write(fp.getbuffer()) diff --git a/Misc/NEWS.d/next/Library/2024-05-13-09-50-31.gh-issue-118981.zgOQPv.rst b/Misc/NEWS.d/next/Library/2024-05-13-09-50-31.gh-issue-118981.zgOQPv.rst new file mode 100644 index 00000000000000..72b9f6c9e4eb99 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-13-09-50-31.gh-issue-118981.zgOQPv.rst @@ -0,0 +1,2 @@ +Fix potential hang in ``multiprocessing.popen_spawn_posix`` that can happen +when the child proc dies early by closing the child fds right away. From 0912b3a6dbd226bea79eb8d70d5a06230804d4cb Mon Sep 17 00:00:00 2001 From: Duane Griffin Date: Sun, 7 Sep 2025 19:49:54 +1200 Subject: [PATCH 2/4] gh-126631: gh-137996: fix pre-loading of `__main__` (GH-135295) gh-126631: gh-137996: fix pre-loading of `__main__` The `main_path` parameter was renamed `init_main_from_name`, update the forkserver code accordingly. This was leading to slower startup times when people were trying to preload the main module. --------- Co-authored-by: Gregory P. Smith --- Lib/multiprocessing/forkserver.py | 9 +++++---- Lib/test/_test_multiprocessing.py | 12 ++++++++++++ Lib/test/mp_preload_main.py | 14 ++++++++++++++ .../2025-06-10-21-00-48.gh-issue-126631.eITVJd.rst | 2 ++ 4 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 Lib/test/mp_preload_main.py create mode 100644 Misc/NEWS.d/next/Library/2025-06-10-21-00-48.gh-issue-126631.eITVJd.rst diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index c91891ff162c2d..cc8947c5e04fb1 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -145,12 +145,13 @@ def ensure_running(self): cmd = ('from multiprocessing.forkserver import main; ' + 'main(%d, %d, %r, **%r)') + main_kws = {} if self._preload_modules: - desired_keys = {'main_path', 'sys_path'} data = spawn.get_preparation_data('ignore') - main_kws = {x: y for x, y in data.items() if x in desired_keys} - else: - main_kws = {} + if 'sys_path' in data: + main_kws['sys_path'] = data['sys_path'] + if 'init_main_from_path' in data: + main_kws['main_path'] = data['init_main_from_path'] with socket.socket(socket.AF_UNIX) as listener: address = connection.arbitrary_address('AF_UNIX') diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index a1c30bcfd51747..d9e572961152b3 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -7028,6 +7028,18 @@ def child(): self.assertEqual(q.get_nowait(), "done") close_queue(q) + def test_preload_main(self): + # gh-126631: Check that __main__ can be pre-loaded + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver specific test") + + name = os.path.join(os.path.dirname(__file__), 'mp_preload_main.py') + _, out, err = test.support.script_helper.assert_python_ok(name) + self.assertEqual(err, b'') + + # The trailing empty string comes from split() on output ending with \n + out = out.decode().split("\n") + self.assertEqual(out, ['__main__', '__mp_main__', 'f', 'f', '']) # # Mixins diff --git a/Lib/test/mp_preload_main.py b/Lib/test/mp_preload_main.py new file mode 100644 index 00000000000000..acb342822ecc89 --- /dev/null +++ b/Lib/test/mp_preload_main.py @@ -0,0 +1,14 @@ +import multiprocessing + +print(f"{__name__}") + +def f(): + print("f") + +if __name__ == "__main__": + ctx = multiprocessing.get_context("forkserver") + ctx.set_forkserver_preload(['__main__']) + for _ in range(2): + p = ctx.Process(target=f) + p.start() + p.join() diff --git a/Misc/NEWS.d/next/Library/2025-06-10-21-00-48.gh-issue-126631.eITVJd.rst b/Misc/NEWS.d/next/Library/2025-06-10-21-00-48.gh-issue-126631.eITVJd.rst new file mode 100644 index 00000000000000..195253b1ec1e39 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-10-21-00-48.gh-issue-126631.eITVJd.rst @@ -0,0 +1,2 @@ +Fix :mod:`multiprocessing` ``forkserver`` bug which prevented ``__main__`` +from being preloaded. From 4ed046c48178118d935c879b97fdccac92c16a94 Mon Sep 17 00:00:00 2001 From: Emma Smith Date: Sun, 7 Sep 2025 01:11:19 -0700 Subject: [PATCH 3/4] Add Emma Smith to CODEOWNERS (#138588) Added myself as a code owner for build system files, WASM platform files, and compression.zstd files. --- .github/CODEOWNERS | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7d26f8b526c3a0..0928f051d5148a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -80,9 +80,10 @@ Tools/patchcheck/ @AA-Turner # ---------------------------------------------------------------------------- # Autotools -configure* @erlend-aasland @corona10 @AA-Turner -Makefile.pre.in @erlend-aasland @AA-Turner -Modules/Setup* @erlend-aasland @AA-Turner +configure* @erlend-aasland @corona10 @AA-Turner @emmatyping +Makefile.pre.in @erlend-aasland @AA-Turner @emmatyping +Modules/Setup* @erlend-aasland @AA-Turner @emmatyping +Modules/makesetup @emmatyping Tools/build/regen-configure.sh @AA-Turner @@ -157,16 +158,16 @@ Lib/_osx_support.py @python/macos-team Lib/test/test__osx_support.py @python/macos-team # WebAssembly -Tools/wasm/README.md @brettcannon @freakboy3742 +Tools/wasm/README.md @brettcannon @freakboy3742 @emmatyping # WebAssembly (Emscripten) -Tools/wasm/config.site-wasm32-emscripten @freakboy3742 -Tools/wasm/emscripten @freakboy3742 +Tools/wasm/config.site-wasm32-emscripten @freakboy3742 @emmatyping +Tools/wasm/emscripten @freakboy3742 @emmatyping # WebAssembly (WASI) -Tools/wasm/wasi-env @brettcannon -Tools/wasm/wasi.py @brettcannon -Tools/wasm/wasi @brettcannon +Tools/wasm/wasi-env @brettcannon @emmatyping +Tools/wasm/wasi.py @brettcannon @emmatyping +Tools/wasm/wasi @brettcannon @emmatyping # Windows PC/ @python/windows-team @@ -603,9 +604,9 @@ Lib/test/test_zipfile/_path/ @jaraco Lib/zipfile/_path/ @jaraco # Zstandard -Lib/compression/zstd/ @AA-Turner -Lib/test/test_zstd.py @AA-Turner -Modules/_zstd/ @AA-Turner +Lib/compression/zstd/ @AA-Turner @emmatyping +Lib/test/test_zstd.py @AA-Turner @emmatyping +Modules/_zstd/ @AA-Turner @emmatyping # ---------------------------------------------------------------------------- From d7b9ea5cab984497526fcc0b988e4eb5988c1e88 Mon Sep 17 00:00:00 2001 From: dbXD320 Date: Sun, 7 Sep 2025 14:23:22 +0530 Subject: [PATCH 4/4] gh-138584: Increase test coverage for `collections.UserList` (#138590) Some common tests in `test.list_tests.CommonTest` explicitly tested `list` instead of testing the underlying list-like type defined in `type2test`. --------- Co-authored-by: Devansh Baghla --- Lib/test/list_tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index 68d6bad2094268..e76f79c274e744 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -32,13 +32,13 @@ def test_init(self): self.assertEqual(a, b) def test_getitem_error(self): - a = [] + a = self.type2test([]) msg = "list indices must be integers or slices" with self.assertRaisesRegex(TypeError, msg): a['a'] def test_setitem_error(self): - a = [] + a = self.type2test([]) msg = "list indices must be integers or slices" with self.assertRaisesRegex(TypeError, msg): a['a'] = "python" @@ -561,7 +561,7 @@ def test_constructor_exception_handling(self): class F(object): def __iter__(self): raise KeyboardInterrupt - self.assertRaises(KeyboardInterrupt, list, F()) + self.assertRaises(KeyboardInterrupt, self.type2test, F()) def test_exhausted_iterator(self): a = self.type2test([1, 2, 3])