diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 69d900091a3bd1f..62325250bd368ee 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -68,7 +68,7 @@ jobs: - true - false llvm: - - 20 + - 21 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -138,7 +138,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: @@ -166,7 +166,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: @@ -193,7 +193,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: diff --git a/Apple/testbed/__main__.py b/Apple/testbed/__main__.py index 42eb60a4c8dc02e..49974cb142853ca 100644 --- a/Apple/testbed/__main__.py +++ b/Apple/testbed/__main__.py @@ -2,6 +2,7 @@ import json import os import re +import shlex import shutil import subprocess import sys @@ -252,7 +253,7 @@ def update_test_plan(testbed_path, platform, args): test_plan = json.load(f) test_plan["defaultOptions"]["commandLineArgumentEntries"] = [ - {"argument": arg} for arg in args + {"argument": shlex.quote(arg)} for arg in args ] with test_plan_path.open("w", encoding="utf-8") as f: diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 916c616dfee589a..3b07b5fbed5959b 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -100,18 +100,12 @@ the same library that the Python runtime is using. Otherwise, Python may not handle script file with LF line ending correctly. -.. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) - - This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, - leaving *flags* set to ``NULL``. - - -.. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) +.. c:function:: int PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) Read and execute a single statement from a file associated with an interactive device according to the *flags* argument. The user will be - prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the - :term:`filesystem encoding and error handler`. + prompted using ``sys.ps1`` and ``sys.ps2``. *filename* must be a Python + :class:`str` object. Returns ``0`` when the input was executed successfully, ``-1`` if there was an exception, or an error code @@ -120,6 +114,19 @@ the same library that the Python runtime is using. :file:`Python.h`, so must be included specifically if needed.) +.. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) + + This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, + leaving *flags* set to ``NULL``. + + +.. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) + + Similar to :c:func:`PyRun_InteractiveOneObject`, but *filename* is a + :c:expr:`const char*`, which is decoded from the + :term:`filesystem encoding and error handler`. + + .. c:function:: int PyRun_InteractiveLoop(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_InteractiveLoopFlags` below, diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 49a3e370a4c018e..342c1a00193959f 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -183,13 +183,13 @@ protocols. Two of the simplest are :mod:`urllib.request` for retrieving data from URLs and :mod:`smtplib` for sending mail:: >>> from urllib.request import urlopen - >>> with urlopen('http://worldtimeapi.org/api/timezone/etc/UTC.txt') as response: + >>> with urlopen('https://docs.python.org/3/') as response: ... for line in response: ... line = line.decode() # Convert bytes to a str - ... if line.startswith('datetime'): + ... if 'updated' in line: ... print(line.rstrip()) # Remove trailing newline ... - datetime: 2022-01-01T01:36:47.689215+00:00 + Last updated on Nov 11, 2025 (20:11 UTC). >>> import smtplib >>> server = smtplib.SMTP('localhost') diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 4e94f62d35eba29..e65556fb28f92d6 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -2660,6 +2660,25 @@ def f(): f() + def test_interpreter_finalization_with_generator_alive(self): + script_helper.assert_python_ok("-c", textwrap.dedent(""" + import sys + t = tuple(range(%d)) + def simple_for(): + for x in t: + x + + def gen(): + try: + yield + except: + simple_for() + + sys.settrace(lambda *args: None) + simple_for() + g = gen() + next(g) + """ % _testinternalcapi.SPECIALIZATION_THRESHOLD)) def global_identity(x): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst new file mode 100644 index 000000000000000..0a615ed131127f9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst @@ -0,0 +1 @@ +Update JIT compilation to use LLVM 21 at build time. diff --git a/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst b/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst new file mode 100644 index 000000000000000..8436cd2415dbd63 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst @@ -0,0 +1,2 @@ +Updated Tcl threading configuration in :mod:`_tkinter` to assume that +threads are always available in Tcl 9 and later. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst b/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst new file mode 100644 index 000000000000000..073c070413f7e06 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst @@ -0,0 +1 @@ +The iOS testbed now correctly handles test arguments that contain spaces. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index c0ed8977d8fd6fa..8cea7b59fe730ef 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -575,8 +575,12 @@ Tkapp_New(const char *screenName, const char *className, v->interp = Tcl_CreateInterp(); v->wantobjects = wantobjects; +#if TCL_MAJOR_VERSION >= 9 + v->threaded = 1; +#else v->threaded = Tcl_GetVar2Ex(v->interp, "tcl_platform", "threaded", TCL_GLOBAL_ONLY) != NULL; +#endif v->thread_id = Tcl_GetCurrentThread(); v->dispatching = 0; v->trace = NULL; diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 115203cecc8e484..9d02e2121cc6233 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -82,7 +82,7 @@ if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.18 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 -if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-20.1.8.0 +if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-21.1.4.0 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( @@ -92,7 +92,7 @@ for %%b in (%binaries%) do ( git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b" ) else ( echo.Fetching %%b... - if "%%b"=="llvm-20.1.8.0" ( + if "%%b"=="llvm-21.1.4.0" ( %PYTHON% -E "%PCBUILD%\get_external.py" --release --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b ) else ( %PYTHON% -E "%PCBUILD%\get_external.py" --binary --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b diff --git a/Python/optimizer.c b/Python/optimizer.c index f44f8a9614b8467..3b7e2dafab85bbd 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -118,7 +118,13 @@ _PyOptimizer_Optimize( { _PyStackRef *stack_pointer = frame->stackpointer; PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(interp->jit); + if (!interp->jit) { + // gh-140936: It is possible that interp->jit will become false during + // interpreter finalization. However, the specialized JUMP_BACKWARD_JIT + // instruction may still be present. In this case, we should + // return immediately without optimization. + return 0; + } assert(!interp->compiling); #ifndef Py_GIL_DISABLED interp->compiling = true; diff --git a/Tools/jit/README.md b/Tools/jit/README.md index d83b09aab59f8cd..c70c0c47d94ad22 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,32 +9,32 @@ Python 3.11 or newer is required to build the JIT. The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 20 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. It's easy to install all of the required tools: ### Linux -Install LLVM 20 on Ubuntu/Debian: +Install LLVM 21 on Ubuntu/Debian: ```sh wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -sudo ./llvm.sh 20 +sudo ./llvm.sh 21 ``` -Install LLVM 20 on Fedora Linux 40 or newer: +Install LLVM 21 on Fedora Linux 40 or newer: ```sh -sudo dnf install 'clang(major) = 20' 'llvm(major) = 20' +sudo dnf install 'clang(major) = 21' 'llvm(major) = 21' ``` ### macOS -Install LLVM 20 with [Homebrew](https://brew.sh): +Install LLVM 21 with [Homebrew](https://brew.sh): ```sh -brew install llvm@20 +brew install llvm@21 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. @@ -43,18 +43,18 @@ Homebrew won't add any of the tools to your `$PATH`. That's okay; the build scri LLVM is downloaded automatically (along with other external binary dependencies) by `PCbuild\build.bat`. -Otherwise, you can install LLVM 20 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=20), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Otherwise, you can install LLVM 21 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=21), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** Alternatively, you can use [chocolatey](https://chocolatey.org): ```sh -choco install llvm --version=20.1.8 +choco install llvm --version=21.1.0 ``` ### Dev Containers If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no -need to install LLVM as the Fedora 42 base image includes LLVM 20 out of the box. +need to install LLVM as the Fedora 43 base image includes LLVM 21 out of the box. ## Building @@ -66,6 +66,9 @@ Otherwise, just configure and build as you normally would. Cross-compiling "just The JIT can also be enabled or disabled using the `PYTHON_JIT` environment variable, even on builds where it is enabled or disabled by default. More details about configuring CPython with the JIT and optional values for `--enable-experimental-jit` can be found [here](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit). +## Miscellaneous +If you're looking for information on how to update the JIT build dependencies, see [JIT Build Infrastructure](jit_infra.md). + [^pep-744]: [PEP 744](https://peps.python.org/pep-0744/) [^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time. diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index f1b0ad3f5dbc432..0b9cb5192f1b75d 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -11,8 +11,8 @@ import _targets -_LLVM_VERSION = "20" -_EXTERNALS_LLVM_TAG = "llvm-20.1.8.0" +_LLVM_VERSION = "21" +_EXTERNALS_LLVM_TAG = "llvm-21.1.4.0" _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") diff --git a/Tools/jit/jit_infra.md b/Tools/jit/jit_infra.md new file mode 100644 index 000000000000000..1a954755611d19c --- /dev/null +++ b/Tools/jit/jit_infra.md @@ -0,0 +1,28 @@ +# JIT Build Infrastructure + +This document includes details about the intricacies of the JIT build infrastructure. + +## Updating LLVM + +When we update LLVM, we need to also update the LLVM release artifact for Windows builds. This is because Windows builds automatically pull prebuilt LLVM binaries in our pipelines (e.g. notice that `.github/workflows/jit.yml` does not explicitly download LLVM or build it from source). + +To update the LLVM release artifact for Windows builds, follow these steps: +1. Go to the [LLVM releases page](https://github.com/llvm/llvm-project/releases). +1. Download x86_64 Windows artifact for the desired LLVM version (e.g. `clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz`). +1. Extract and repackage the tarball with the correct directory structure. For example: + ```bash + tar -xf clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz + mv clang+llvm-21.1.4-x86_64-pc-windows-msvc llvm-21.1.4.0 + tar -cf - llvm-21.1.4.0 | pv | xz > llvm-21.1.4.0.tar.xz + ``` + The tarball must contain a top-level directory named `llvm-{version}.0/`. +1. Go to [cpython-bin-deps](https://github.com/python/cpython-bin-deps). +1. Create a new release with the updated LLVM artifact. + - Create a new tag to match the LLVM version (e.g. `llvm-21.1.4.0`). + - Specify the release title (e.g. `LLVM 21.1.4 for x86_64 Windows`). + - Upload the asset (you can leave all other fields the same). + +### Other notes +- You must make sure that the name of the artifact matches exactly what is expected in `Tools/jit/_llvm.py` and `PCbuild/get_externals.py`. +- We don't need multiple release artifacts for each architecture because LLVM can cross-compile for different architectures on Windows; x86_64 is sufficient. +- You must have permissions to create releases in the `cpython-bin-deps` repository. If you don't have permissions, you should contact one of the organization admins. \ No newline at end of file