Support PYO3_BASE_PYTHON to make builds caching-friendly

.github/workflows/ci.yml

@@ -620,6 +620,7 @@ jobs:
           save-if: ${{ needs.resolve.outputs.save-cache }}
       - uses: ./.github/actions/prepare-coverage
       - run: uvx nox -s test-version-limits
+      - run: uvx nox -s test-interpreter-discovery
       - uses: ./.github/actions/report-coverage
         with:
           name: ${{ github.job }}

guide/src/building-and-distribution.md

@@ -21,6 +21,10 @@ By default it will attempt to use the following in order:
 
 You can override the Python interpreter by setting the `PYO3_PYTHON` environment variable, e.g. `PYO3_PYTHON=python3.8`, `PYO3_PYTHON=/usr/bin/python3.9`, or even a PyPy interpreter `PYO3_PYTHON=pypy3`.
 
+Build tools may additionally set the `PYO3_BASE_PYTHON` environment variable, which takes precedence over `PYO3_PYTHON`.
+This is intended to point at a stable interpreter path (e.g. `sys._base_executable`) when `PYO3_PYTHON` points inside an ephemeral virtual environment (as created by PEP 517 build frontends with build isolation).
+When `PYO3_BASE_PYTHON` is set, changes to `PYO3_PYTHON` do not trigger rebuilds, which keeps compilation caches warm across builds in freshly-created (and randomly-named) temporary environments.
+
 Once the Python interpreter is located, `pyo3-build-config` executes it to query the information in the `sysconfig` module which is needed to configure the rest of the compilation.
 
 To validate the configuration which PyO3 will use, you can run a compilation with the environment variable `PYO3_PRINT_CONFIG=1` set.
@@ -57,6 +61,7 @@ Caused by:
   cargo:rerun-if-env-changed=PYO3_CROSS_PYTHON_IMPLEMENTATION
   cargo:rerun-if-env-changed=PYO3_NO_PYTHON
   cargo:rerun-if-env-changed=PYO3_ENVIRONMENT_SIGNATURE
+  cargo:rerun-if-env-changed=PYO3_BASE_PYTHON
   cargo:rerun-if-env-changed=PYO3_PYTHON
   cargo:rerun-if-env-changed=VIRTUAL_ENV
   cargo:rerun-if-env-changed=CONDA_PREFIX

newsfragments/6114.added.md

@@ -0,0 +1 @@
+Support the `PYO3_BASE_PYTHON` environment variable in `pyo3-build-config`, which takes precedence over `PYO3_PYTHON`. Build tools can set it to a stable interpreter path (e.g. `sys._base_executable`) so that ephemeral virtualenv paths in `PYO3_PYTHON` (as used by PEP 517 build frontends with build isolation) do not trigger unnecessary rebuilds.

noxfile.py

@@ -1236,6 +1236,54 @@ def ffi_check(session: nox.Session):
     _check_raw_dylib_macro(session)
 
 
+@nox.session(name="test-interpreter-discovery")
+def test_interpreter_discovery(session: nox.Session):
+    """Check that PYO3_BASE_PYTHON and PYO3_PYTHON select the interpreter as expected.
+
+    These build-script code paths are otherwise not exercised by the normal test
+    suite (which discovers the interpreter via the active virtualenv).
+    """
+
+    def print_config(**interpreter_env: str) -> str:
+        env = os.environ.copy()
+        # Take full control of interpreter discovery by clearing anything in the
+        # ambient environment that would otherwise select an interpreter.
+        for var in ("PYO3_BASE_PYTHON", "PYO3_PYTHON", "VIRTUAL_ENV", "CONDA_PREFIX"):
+            env.pop(var, None)
+        env.update(interpreter_env)
+        # Halt the build once the interpreter has been located and queried, and
+        # print the resulting configuration to stderr.
+        env["PYO3_PRINT_CONFIG"] = "1"
+        with tempfile.TemporaryFile() as stderr:
+            _run_cargo(
+                session,
+                "check",
+                "--package=pyo3-ffi",
+                env=env,
+                stderr=stderr,
+                expect_error=True,  # PYO3_PRINT_CONFIG always halts the build
+            )
+            stderr.seek(0)
+            return stderr.read().decode()
+
+    interpreter = sys.executable
+    bogus = os.path.join(os.path.sep, "pyo3", "does", "not", "exist")
+
+    # `PYO3_BASE_PYTHON` is used to locate the interpreter when set.
+    config = print_config(PYO3_BASE_PYTHON=interpreter)
+    assert "version=" in config, config
+
+    # `PYO3_BASE_PYTHON` takes precedence over `PYO3_PYTHON`; the latter is not even
+    # read, so a bogus (non-runnable) value for it must not break the build.
+    config = print_config(PYO3_BASE_PYTHON=interpreter, PYO3_PYTHON=bogus)
+    assert "version=" in config, config
+    assert "does/not/exist" not in config, config
+
+    # `PYO3_PYTHON` is used to locate the interpreter when `PYO3_BASE_PYTHON` is unset.
+    config = print_config(PYO3_PYTHON=interpreter)
+    assert "version=" in config, config
+
+
 @nox.session(name="test-version-limits")
 def test_version_limits(session: nox.Session):
     env = os.environ.copy()

pyo3-build-config/src/impl_.rs

@@ -2489,16 +2489,26 @@ fn get_env_interpreter() -> Option<PathBuf> {
 /// Attempts to locate a python interpreter.
 ///
 /// Locations are checked in the order listed:
-///   1. If `PYO3_PYTHON` is set, this interpreter is used.
-///   2. If in a virtualenv, that environment's interpreter is used.
-///   3. `python`, if this is functional a Python 3.x interpreter
-///   4. `python3`, as above
+///   1. If `PYO3_BASE_PYTHON` is set, this interpreter is used. Build tools (such as maturin) may
+///      set this to a stable interpreter path outside of any temporary virtual environment (e.g.
+///      `sys._base_executable`), so that rebuilds are not triggered by ephemeral virtualenv paths
+///      changing between otherwise identical builds.
+///   2. If `PYO3_PYTHON` is set, this interpreter is used.
+///   3. If in a virtualenv, that environment's interpreter is used.
+///   4. `python`, if this is functional a Python 3.x interpreter
+///   5. `python3`, as above
 pub fn find_interpreter() -> Result<PathBuf> {
     // Trigger rebuilds when `PYO3_ENVIRONMENT_SIGNATURE` env var value changes
     // See https://github.com/PyO3/pyo3/issues/2724
     println!("cargo:rerun-if-env-changed=PYO3_ENVIRONMENT_SIGNATURE");
 
-    if let Some(exe) = env_var("PYO3_PYTHON") {
+    // Note that `PYO3_PYTHON` is deliberately not read (and so no rebuild is triggered when it
+    // changes) when `PYO3_BASE_PYTHON` is set; allowing builds to stay cached when only the
+    // (ephemeral) `PYO3_PYTHON` path changes is the purpose of `PYO3_BASE_PYTHON`.
+    // See https://github.com/PyO3/pyo3/issues/6113
+    if let Some(exe) = env_var("PYO3_BASE_PYTHON") {
+        Ok(exe.into())
+    } else if let Some(exe) = env_var("PYO3_PYTHON") {
         Ok(exe.into())
     } else if let Some(env_interpreter) = get_env_interpreter() {
         Ok(env_interpreter)