From 1e6d78690ef2c35115957a9945241834df69595a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 7 Jun 2026 15:33:55 +0000 Subject: [PATCH 1/4] Support PYO3_BASE_PYTHON to make builds caching-friendly When a project is built by a PEP 517 frontend with build isolation, the frontend creates a randomly-named temporary virtualenv and the build tool points PYO3_PYTHON inside it. Because pyo3-build-config registers rerun-if-env-changed=PYO3_PYTHON, the ephemeral path defeats cargo's build cache and forces recompilation of the pyo3 crates on every build, even when the underlying interpreter is identical. Build tools can now additionally export PYO3_BASE_PYTHON pointing at a stable interpreter path (e.g. sys._base_executable). When set, it takes precedence over PYO3_PYTHON, and PYO3_PYTHON is not read at all - so changes to the ephemeral path no longer trigger rebuilds. Fixes #6113 https://claude.ai/code/session_01JhPTZXLDhuxTSXCZNJuuAA --- guide/src/building-and-distribution.md | 3 +++ newsfragments/6113.added.md | 1 + pyo3-build-config/src/impl_.rs | 20 +++++++++++++++----- 3 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 newsfragments/6113.added.md diff --git a/guide/src/building-and-distribution.md b/guide/src/building-and-distribution.md index 31156a71c43..97403e8fa54 100644 --- a/guide/src/building-and-distribution.md +++ b/guide/src/building-and-distribution.md @@ -21,6 +21,8 @@ By default it will attempt to use the following in order: You can override the Python interpreter by setting the `PYO3_PYTHON` environment variable, e.g. `PYO3_PYTHON=python3.8`, `PYO3_PYTHON=/usr/bin/python3.9`, or even a PyPy interpreter `PYO3_PYTHON=pypy3`. +Build tools may additionally set the `PYO3_BASE_PYTHON` environment variable, which takes precedence over `PYO3_PYTHON`. This is intended to point at a stable interpreter path (e.g. `sys._base_executable`) when `PYO3_PYTHON` points inside an ephemeral virtual environment (as created by PEP 517 build frontends with build isolation). When `PYO3_BASE_PYTHON` is set, changes to `PYO3_PYTHON` do not trigger rebuilds, which keeps compilation caches warm across builds in freshly-created (and randomly-named) temporary environments. + Once the Python interpreter is located, `pyo3-build-config` executes it to query the information in the `sysconfig` module which is needed to configure the rest of the compilation. To validate the configuration which PyO3 will use, you can run a compilation with the environment variable `PYO3_PRINT_CONFIG=1` set. @@ -57,6 +59,7 @@ Caused by: cargo:rerun-if-env-changed=PYO3_CROSS_PYTHON_IMPLEMENTATION cargo:rerun-if-env-changed=PYO3_NO_PYTHON cargo:rerun-if-env-changed=PYO3_ENVIRONMENT_SIGNATURE + cargo:rerun-if-env-changed=PYO3_BASE_PYTHON cargo:rerun-if-env-changed=PYO3_PYTHON cargo:rerun-if-env-changed=VIRTUAL_ENV cargo:rerun-if-env-changed=CONDA_PREFIX diff --git a/newsfragments/6113.added.md b/newsfragments/6113.added.md new file mode 100644 index 00000000000..74ddb4d3831 --- /dev/null +++ b/newsfragments/6113.added.md @@ -0,0 +1 @@ +Support the `PYO3_BASE_PYTHON` environment variable in `pyo3-build-config`, which takes precedence over `PYO3_PYTHON`. Build tools can set it to a stable interpreter path (e.g. `sys._base_executable`) so that ephemeral virtualenv paths in `PYO3_PYTHON` (as used by PEP 517 build frontends with build isolation) do not trigger unnecessary rebuilds. diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 572ef4665bb..18ecd155a51 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -2489,16 +2489,26 @@ fn get_env_interpreter() -> Option { /// Attempts to locate a python interpreter. /// /// Locations are checked in the order listed: -/// 1. If `PYO3_PYTHON` is set, this interpreter is used. -/// 2. If in a virtualenv, that environment's interpreter is used. -/// 3. `python`, if this is functional a Python 3.x interpreter -/// 4. `python3`, as above +/// 1. If `PYO3_BASE_PYTHON` is set, this interpreter is used. Build tools (such as maturin) may +/// set this to a stable interpreter path outside of any temporary virtual environment (e.g. +/// `sys._base_executable`), so that rebuilds are not triggered by ephemeral virtualenv paths +/// changing between otherwise identical builds. +/// 2. If `PYO3_PYTHON` is set, this interpreter is used. +/// 3. If in a virtualenv, that environment's interpreter is used. +/// 4. `python`, if this is functional a Python 3.x interpreter +/// 5. `python3`, as above pub fn find_interpreter() -> Result { // Trigger rebuilds when `PYO3_ENVIRONMENT_SIGNATURE` env var value changes // See https://github.com/PyO3/pyo3/issues/2724 println!("cargo:rerun-if-env-changed=PYO3_ENVIRONMENT_SIGNATURE"); - if let Some(exe) = env_var("PYO3_PYTHON") { + // Note that `PYO3_PYTHON` is deliberately not read (and so no rebuild is triggered when it + // changes) when `PYO3_BASE_PYTHON` is set; allowing builds to stay cached when only the + // (ephemeral) `PYO3_PYTHON` path changes is the purpose of `PYO3_BASE_PYTHON`. + // See https://github.com/PyO3/pyo3/issues/6113 + if let Some(exe) = env_var("PYO3_BASE_PYTHON") { + Ok(exe.into()) + } else if let Some(exe) = env_var("PYO3_PYTHON") { Ok(exe.into()) } else if let Some(env_interpreter) = get_env_interpreter() { Ok(env_interpreter) From 4015967b813d2eba189ab253208fb7cdb707fe8d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 7 Jun 2026 15:42:12 +0000 Subject: [PATCH 2/4] Fix CI: split guide paragraph per markdown lint, rename newsfragment to PR number - rumdl requires one sentence per line in the guide - the changelog check requires the newsfragment to be named after the PR number (6114), not the issue number https://claude.ai/code/session_01JhPTZXLDhuxTSXCZNJuuAA --- guide/src/building-and-distribution.md | 4 +++- newsfragments/{6113.added.md => 6114.added.md} | 0 2 files changed, 3 insertions(+), 1 deletion(-) rename newsfragments/{6113.added.md => 6114.added.md} (100%) diff --git a/guide/src/building-and-distribution.md b/guide/src/building-and-distribution.md index 97403e8fa54..f7004f189ca 100644 --- a/guide/src/building-and-distribution.md +++ b/guide/src/building-and-distribution.md @@ -21,7 +21,9 @@ By default it will attempt to use the following in order: You can override the Python interpreter by setting the `PYO3_PYTHON` environment variable, e.g. `PYO3_PYTHON=python3.8`, `PYO3_PYTHON=/usr/bin/python3.9`, or even a PyPy interpreter `PYO3_PYTHON=pypy3`. -Build tools may additionally set the `PYO3_BASE_PYTHON` environment variable, which takes precedence over `PYO3_PYTHON`. This is intended to point at a stable interpreter path (e.g. `sys._base_executable`) when `PYO3_PYTHON` points inside an ephemeral virtual environment (as created by PEP 517 build frontends with build isolation). When `PYO3_BASE_PYTHON` is set, changes to `PYO3_PYTHON` do not trigger rebuilds, which keeps compilation caches warm across builds in freshly-created (and randomly-named) temporary environments. +Build tools may additionally set the `PYO3_BASE_PYTHON` environment variable, which takes precedence over `PYO3_PYTHON`. +This is intended to point at a stable interpreter path (e.g. `sys._base_executable`) when `PYO3_PYTHON` points inside an ephemeral virtual environment (as created by PEP 517 build frontends with build isolation). +When `PYO3_BASE_PYTHON` is set, changes to `PYO3_PYTHON` do not trigger rebuilds, which keeps compilation caches warm across builds in freshly-created (and randomly-named) temporary environments. Once the Python interpreter is located, `pyo3-build-config` executes it to query the information in the `sysconfig` module which is needed to configure the rest of the compilation. diff --git a/newsfragments/6113.added.md b/newsfragments/6114.added.md similarity index 100% rename from newsfragments/6113.added.md rename to newsfragments/6114.added.md From bad2223e53e37462aeac5b2d275ce01aea70e580 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 8 Jun 2026 16:18:08 +0000 Subject: [PATCH 3/4] Add test coverage for explicit interpreter selection Extract the PYO3_BASE_PYTHON / PYO3_PYTHON resolution out of find_interpreter into a pure `explicit_interpreter` helper and unit-test all three branches: PYO3_BASE_PYTHON taking precedence (and PYO3_PYTHON not being read at all in that case), falling back to PYO3_PYTHON, and neither being set. Addresses review feedback from @ngoldbaum requesting coverage of the newly-added branches. https://claude.ai/code/session_01JhPTZXLDhuxTSXCZNJuuAA --- pyo3-build-config/src/impl_.rs | 51 ++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 18ecd155a51..2d4fab06775 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -2502,14 +2502,9 @@ pub fn find_interpreter() -> Result { // See https://github.com/PyO3/pyo3/issues/2724 println!("cargo:rerun-if-env-changed=PYO3_ENVIRONMENT_SIGNATURE"); - // Note that `PYO3_PYTHON` is deliberately not read (and so no rebuild is triggered when it - // changes) when `PYO3_BASE_PYTHON` is set; allowing builds to stay cached when only the - // (ephemeral) `PYO3_PYTHON` path changes is the purpose of `PYO3_BASE_PYTHON`. - // See https://github.com/PyO3/pyo3/issues/6113 - if let Some(exe) = env_var("PYO3_BASE_PYTHON") { - Ok(exe.into()) - } else if let Some(exe) = env_var("PYO3_PYTHON") { - Ok(exe.into()) + if let Some(exe) = explicit_interpreter(env_var("PYO3_BASE_PYTHON"), || env_var("PYO3_PYTHON")) + { + Ok(exe) } else if let Some(env_interpreter) = get_env_interpreter() { Ok(env_interpreter) } else { @@ -2531,6 +2526,25 @@ pub fn find_interpreter() -> Result { } } +/// Selects the interpreter explicitly requested via the `PYO3_BASE_PYTHON` and `PYO3_PYTHON` +/// environment variables (whose values are passed as `base_python` and `python` respectively), +/// returning `None` if neither is set. +/// +/// `PYO3_BASE_PYTHON` takes precedence. Crucially, `python` is only evaluated when `base_python` +/// is `None`: this ensures `PYO3_PYTHON` is not read (and so does not register a +/// `rerun-if-env-changed`) when `PYO3_BASE_PYTHON` is set, which is what allows builds to stay +/// cached when only the (ephemeral) `PYO3_PYTHON` path changes. +/// See https://github.com/PyO3/pyo3/issues/6113 +fn explicit_interpreter( + base_python: Option, + python: impl FnOnce() -> Option, +) -> Option { + match base_python { + Some(exe) => Some(exe.into()), + None => python().map(PathBuf::from), + } +} + /// Locates and extracts the build host Python interpreter configuration. /// /// Lowers the configured Python version to `abi3_version` or `abi3t_version` if required. @@ -4119,6 +4133,27 @@ mod tests { READ_ENV_VARS.with(|vars| assert!(vars.borrow().contains(&"PYO3_CONFIG_FILE".to_string()))); } + #[test] + fn test_explicit_interpreter() { + // `PYO3_BASE_PYTHON` is used when set, and `PYO3_PYTHON` is not consulted at all (the + // closure must not run) so that changes to it do not trigger rebuilds. + assert_eq!( + explicit_interpreter(Some(OsString::from("/base/python")), || panic!( + "PYO3_PYTHON must not be read when PYO3_BASE_PYTHON is set" + )), + Some(PathBuf::from("/base/python")), + ); + + // Falls back to `PYO3_PYTHON` when `PYO3_BASE_PYTHON` is absent. + assert_eq!( + explicit_interpreter(None, || Some(OsString::from("/venv/python"))), + Some(PathBuf::from("/venv/python")), + ); + + // Neither set: no explicit interpreter, so detection falls through to virtualenv / PATH. + assert_eq!(explicit_interpreter(None, || None), None); + } + #[test] fn test_default_lib_name_for_target() { let cpython = PythonImplementation::CPython; From 6f852ae8d90c91fbf1ad5788ffc19ccd880759ed Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 8 Jun 2026 16:25:54 +0000 Subject: [PATCH 4/4] Add nox test covering PYO3_BASE_PYTHON / PYO3_PYTHON discovery ngoldbaum asked for tests covering the branches added to find_interpreter. Those branches only run inside the build script, which the normal test suite exercises via the active virtualenv, so they were never hit. Add a test-interpreter-discovery nox session that drives a build with PYO3_PRINT_CONFIG=1 under controlled PYO3_BASE_PYTHON / PYO3_PYTHON values and asserts the resulting config. It covers all three cases: PYO3_BASE_PYTHON selecting the interpreter, taking precedence over a bogus PYO3_PYTHON (proving PYO3_PYTHON is not executed), and the PYO3_PYTHON fallback. Run it in the test-version-limits CI job, which already reports coverage with --include-build-script. This reverts the earlier find_interpreter refactor, which is no longer needed now that the branches are covered via the build script. https://claude.ai/code/session_01JhPTZXLDhuxTSXCZNJuuAA --- .github/workflows/ci.yml | 1 + noxfile.py | 48 ++++++++++++++++++++++++++++++++ pyo3-build-config/src/impl_.rs | 51 ++++++---------------------------- 3 files changed, 57 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c45571dc02..0d478bfb023 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -620,6 +620,7 @@ jobs: save-if: ${{ needs.resolve.outputs.save-cache }} - uses: ./.github/actions/prepare-coverage - run: uvx nox -s test-version-limits + - run: uvx nox -s test-interpreter-discovery - uses: ./.github/actions/report-coverage with: name: ${{ github.job }} diff --git a/noxfile.py b/noxfile.py index e68ccd41472..540d37de1a0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1236,6 +1236,54 @@ def ffi_check(session: nox.Session): _check_raw_dylib_macro(session) +@nox.session(name="test-interpreter-discovery") +def test_interpreter_discovery(session: nox.Session): + """Check that PYO3_BASE_PYTHON and PYO3_PYTHON select the interpreter as expected. + + These build-script code paths are otherwise not exercised by the normal test + suite (which discovers the interpreter via the active virtualenv). + """ + + def print_config(**interpreter_env: str) -> str: + env = os.environ.copy() + # Take full control of interpreter discovery by clearing anything in the + # ambient environment that would otherwise select an interpreter. + for var in ("PYO3_BASE_PYTHON", "PYO3_PYTHON", "VIRTUAL_ENV", "CONDA_PREFIX"): + env.pop(var, None) + env.update(interpreter_env) + # Halt the build once the interpreter has been located and queried, and + # print the resulting configuration to stderr. + env["PYO3_PRINT_CONFIG"] = "1" + with tempfile.TemporaryFile() as stderr: + _run_cargo( + session, + "check", + "--package=pyo3-ffi", + env=env, + stderr=stderr, + expect_error=True, # PYO3_PRINT_CONFIG always halts the build + ) + stderr.seek(0) + return stderr.read().decode() + + interpreter = sys.executable + bogus = os.path.join(os.path.sep, "pyo3", "does", "not", "exist") + + # `PYO3_BASE_PYTHON` is used to locate the interpreter when set. + config = print_config(PYO3_BASE_PYTHON=interpreter) + assert "version=" in config, config + + # `PYO3_BASE_PYTHON` takes precedence over `PYO3_PYTHON`; the latter is not even + # read, so a bogus (non-runnable) value for it must not break the build. + config = print_config(PYO3_BASE_PYTHON=interpreter, PYO3_PYTHON=bogus) + assert "version=" in config, config + assert "does/not/exist" not in config, config + + # `PYO3_PYTHON` is used to locate the interpreter when `PYO3_BASE_PYTHON` is unset. + config = print_config(PYO3_PYTHON=interpreter) + assert "version=" in config, config + + @nox.session(name="test-version-limits") def test_version_limits(session: nox.Session): env = os.environ.copy() diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 2d4fab06775..18ecd155a51 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -2502,9 +2502,14 @@ pub fn find_interpreter() -> Result { // See https://github.com/PyO3/pyo3/issues/2724 println!("cargo:rerun-if-env-changed=PYO3_ENVIRONMENT_SIGNATURE"); - if let Some(exe) = explicit_interpreter(env_var("PYO3_BASE_PYTHON"), || env_var("PYO3_PYTHON")) - { - Ok(exe) + // Note that `PYO3_PYTHON` is deliberately not read (and so no rebuild is triggered when it + // changes) when `PYO3_BASE_PYTHON` is set; allowing builds to stay cached when only the + // (ephemeral) `PYO3_PYTHON` path changes is the purpose of `PYO3_BASE_PYTHON`. + // See https://github.com/PyO3/pyo3/issues/6113 + if let Some(exe) = env_var("PYO3_BASE_PYTHON") { + Ok(exe.into()) + } else if let Some(exe) = env_var("PYO3_PYTHON") { + Ok(exe.into()) } else if let Some(env_interpreter) = get_env_interpreter() { Ok(env_interpreter) } else { @@ -2526,25 +2531,6 @@ pub fn find_interpreter() -> Result { } } -/// Selects the interpreter explicitly requested via the `PYO3_BASE_PYTHON` and `PYO3_PYTHON` -/// environment variables (whose values are passed as `base_python` and `python` respectively), -/// returning `None` if neither is set. -/// -/// `PYO3_BASE_PYTHON` takes precedence. Crucially, `python` is only evaluated when `base_python` -/// is `None`: this ensures `PYO3_PYTHON` is not read (and so does not register a -/// `rerun-if-env-changed`) when `PYO3_BASE_PYTHON` is set, which is what allows builds to stay -/// cached when only the (ephemeral) `PYO3_PYTHON` path changes. -/// See https://github.com/PyO3/pyo3/issues/6113 -fn explicit_interpreter( - base_python: Option, - python: impl FnOnce() -> Option, -) -> Option { - match base_python { - Some(exe) => Some(exe.into()), - None => python().map(PathBuf::from), - } -} - /// Locates and extracts the build host Python interpreter configuration. /// /// Lowers the configured Python version to `abi3_version` or `abi3t_version` if required. @@ -4133,27 +4119,6 @@ mod tests { READ_ENV_VARS.with(|vars| assert!(vars.borrow().contains(&"PYO3_CONFIG_FILE".to_string()))); } - #[test] - fn test_explicit_interpreter() { - // `PYO3_BASE_PYTHON` is used when set, and `PYO3_PYTHON` is not consulted at all (the - // closure must not run) so that changes to it do not trigger rebuilds. - assert_eq!( - explicit_interpreter(Some(OsString::from("/base/python")), || panic!( - "PYO3_PYTHON must not be read when PYO3_BASE_PYTHON is set" - )), - Some(PathBuf::from("/base/python")), - ); - - // Falls back to `PYO3_PYTHON` when `PYO3_BASE_PYTHON` is absent. - assert_eq!( - explicit_interpreter(None, || Some(OsString::from("/venv/python"))), - Some(PathBuf::from("/venv/python")), - ); - - // Neither set: no explicit interpreter, so detection falls through to virtualenv / PATH. - assert_eq!(explicit_interpreter(None, || None), None); - } - #[test] fn test_default_lib_name_for_target() { let cpython = PythonImplementation::CPython;