diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c45571dc02..0d478bfb023 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -620,6 +620,7 @@ jobs: save-if: ${{ needs.resolve.outputs.save-cache }} - uses: ./.github/actions/prepare-coverage - run: uvx nox -s test-version-limits + - run: uvx nox -s test-interpreter-discovery - uses: ./.github/actions/report-coverage with: name: ${{ github.job }} diff --git a/guide/src/building-and-distribution.md b/guide/src/building-and-distribution.md index 31156a71c43..f7004f189ca 100644 --- a/guide/src/building-and-distribution.md +++ b/guide/src/building-and-distribution.md @@ -21,6 +21,10 @@ By default it will attempt to use the following in order: You can override the Python interpreter by setting the `PYO3_PYTHON` environment variable, e.g. `PYO3_PYTHON=python3.8`, `PYO3_PYTHON=/usr/bin/python3.9`, or even a PyPy interpreter `PYO3_PYTHON=pypy3`. +Build tools may additionally set the `PYO3_BASE_PYTHON` environment variable, which takes precedence over `PYO3_PYTHON`. +This is intended to point at a stable interpreter path (e.g. `sys._base_executable`) when `PYO3_PYTHON` points inside an ephemeral virtual environment (as created by PEP 517 build frontends with build isolation). +When `PYO3_BASE_PYTHON` is set, changes to `PYO3_PYTHON` do not trigger rebuilds, which keeps compilation caches warm across builds in freshly-created (and randomly-named) temporary environments. + Once the Python interpreter is located, `pyo3-build-config` executes it to query the information in the `sysconfig` module which is needed to configure the rest of the compilation. To validate the configuration which PyO3 will use, you can run a compilation with the environment variable `PYO3_PRINT_CONFIG=1` set. @@ -57,6 +61,7 @@ Caused by: cargo:rerun-if-env-changed=PYO3_CROSS_PYTHON_IMPLEMENTATION cargo:rerun-if-env-changed=PYO3_NO_PYTHON cargo:rerun-if-env-changed=PYO3_ENVIRONMENT_SIGNATURE + cargo:rerun-if-env-changed=PYO3_BASE_PYTHON cargo:rerun-if-env-changed=PYO3_PYTHON cargo:rerun-if-env-changed=VIRTUAL_ENV cargo:rerun-if-env-changed=CONDA_PREFIX diff --git a/newsfragments/6114.added.md b/newsfragments/6114.added.md new file mode 100644 index 00000000000..74ddb4d3831 --- /dev/null +++ b/newsfragments/6114.added.md @@ -0,0 +1 @@ +Support the `PYO3_BASE_PYTHON` environment variable in `pyo3-build-config`, which takes precedence over `PYO3_PYTHON`. Build tools can set it to a stable interpreter path (e.g. `sys._base_executable`) so that ephemeral virtualenv paths in `PYO3_PYTHON` (as used by PEP 517 build frontends with build isolation) do not trigger unnecessary rebuilds. diff --git a/noxfile.py b/noxfile.py index e68ccd41472..540d37de1a0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1236,6 +1236,54 @@ def ffi_check(session: nox.Session): _check_raw_dylib_macro(session) +@nox.session(name="test-interpreter-discovery") +def test_interpreter_discovery(session: nox.Session): + """Check that PYO3_BASE_PYTHON and PYO3_PYTHON select the interpreter as expected. + + These build-script code paths are otherwise not exercised by the normal test + suite (which discovers the interpreter via the active virtualenv). + """ + + def print_config(**interpreter_env: str) -> str: + env = os.environ.copy() + # Take full control of interpreter discovery by clearing anything in the + # ambient environment that would otherwise select an interpreter. + for var in ("PYO3_BASE_PYTHON", "PYO3_PYTHON", "VIRTUAL_ENV", "CONDA_PREFIX"): + env.pop(var, None) + env.update(interpreter_env) + # Halt the build once the interpreter has been located and queried, and + # print the resulting configuration to stderr. + env["PYO3_PRINT_CONFIG"] = "1" + with tempfile.TemporaryFile() as stderr: + _run_cargo( + session, + "check", + "--package=pyo3-ffi", + env=env, + stderr=stderr, + expect_error=True, # PYO3_PRINT_CONFIG always halts the build + ) + stderr.seek(0) + return stderr.read().decode() + + interpreter = sys.executable + bogus = os.path.join(os.path.sep, "pyo3", "does", "not", "exist") + + # `PYO3_BASE_PYTHON` is used to locate the interpreter when set. + config = print_config(PYO3_BASE_PYTHON=interpreter) + assert "version=" in config, config + + # `PYO3_BASE_PYTHON` takes precedence over `PYO3_PYTHON`; the latter is not even + # read, so a bogus (non-runnable) value for it must not break the build. + config = print_config(PYO3_BASE_PYTHON=interpreter, PYO3_PYTHON=bogus) + assert "version=" in config, config + assert "does/not/exist" not in config, config + + # `PYO3_PYTHON` is used to locate the interpreter when `PYO3_BASE_PYTHON` is unset. + config = print_config(PYO3_PYTHON=interpreter) + assert "version=" in config, config + + @nox.session(name="test-version-limits") def test_version_limits(session: nox.Session): env = os.environ.copy() diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 572ef4665bb..18ecd155a51 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -2489,16 +2489,26 @@ fn get_env_interpreter() -> Option { /// Attempts to locate a python interpreter. /// /// Locations are checked in the order listed: -/// 1. If `PYO3_PYTHON` is set, this interpreter is used. -/// 2. If in a virtualenv, that environment's interpreter is used. -/// 3. `python`, if this is functional a Python 3.x interpreter -/// 4. `python3`, as above +/// 1. If `PYO3_BASE_PYTHON` is set, this interpreter is used. Build tools (such as maturin) may +/// set this to a stable interpreter path outside of any temporary virtual environment (e.g. +/// `sys._base_executable`), so that rebuilds are not triggered by ephemeral virtualenv paths +/// changing between otherwise identical builds. +/// 2. If `PYO3_PYTHON` is set, this interpreter is used. +/// 3. If in a virtualenv, that environment's interpreter is used. +/// 4. `python`, if this is functional a Python 3.x interpreter +/// 5. `python3`, as above pub fn find_interpreter() -> Result { // Trigger rebuilds when `PYO3_ENVIRONMENT_SIGNATURE` env var value changes // See https://github.com/PyO3/pyo3/issues/2724 println!("cargo:rerun-if-env-changed=PYO3_ENVIRONMENT_SIGNATURE"); - if let Some(exe) = env_var("PYO3_PYTHON") { + // Note that `PYO3_PYTHON` is deliberately not read (and so no rebuild is triggered when it + // changes) when `PYO3_BASE_PYTHON` is set; allowing builds to stay cached when only the + // (ephemeral) `PYO3_PYTHON` path changes is the purpose of `PYO3_BASE_PYTHON`. + // See https://github.com/PyO3/pyo3/issues/6113 + if let Some(exe) = env_var("PYO3_BASE_PYTHON") { + Ok(exe.into()) + } else if let Some(exe) = env_var("PYO3_PYTHON") { Ok(exe.into()) } else if let Some(env_interpreter) = get_env_interpreter() { Ok(env_interpreter)