From d174f5a98d7481820e11abb413743a950624ddc2 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Tue, 27 May 2025 16:41:22 -0700 Subject: [PATCH 01/25] find_nvidia_headers.py initial version (untested). --- .../_path_finder/find_nvidia_headers.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 cuda_bindings/cuda/bindings/_path_finder/find_nvidia_headers.py diff --git a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_headers.py b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_headers.py new file mode 100644 index 0000000000..bf56097f2f --- /dev/null +++ b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_headers.py @@ -0,0 +1,38 @@ +# Copyright 2025 NVIDIA Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +import functools +import glob +import os + +from cuda.bindings._path_finder.find_sub_dirs import find_sub_dirs_all_sitepackages +from cuda.bindings._path_finder.supported_libs import IS_WINDOWS + + +@functools.cache +def find_nvidia_header_directory(libname: str) -> str: + assert libname == "nvshmem" + assert not IS_WINDOWS + + # Installed from a wheel + nvidia_sub_dirs = ("nvidia", "nvshmem", "include") + for hdr_dir in find_sub_dirs_all_sitepackages(nvidia_sub_dirs): + nvshmem_h_path = os.path.join(hdr_dir, "nvshmem.h") + if os.path.isfile(nvshmem_h_path): + return hdr_dir + + conda_prefix = os.environ.get("CONDA_PREFIX") + if conda_prefix and os.path.isdir(conda_prefix): + hdr_dir = os.path.join(conda_prefix, "include") + if os.path.isdir(hdr_dir): + nvshmem_h_path = os.path.join(hdr_dir, "nvshmem.h") + if os.path.isfile(nvshmem_h_path): + return hdr_dir + + for hdr_dir in sorted(glob.glob("/usr/include/nvshmem_*")): + if os.path.isdir(hdr_dir): + nvshmem_h_path = os.path.join(hdr_dir, "nvshmem.h") + if os.path.isfile(nvshmem_h_path): + return hdr_dir + + return None From 5ed86c56b0743f4aee8a9d0077c56e307c0b59d2 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 28 May 2025 10:09:11 -0700 Subject: [PATCH 02/25] Add tests/test_path_finder_find_headers.py, with hard-coded paths. --- .../tests/test_path_finder_find_headers.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 cuda_bindings/tests/test_path_finder_find_headers.py diff --git a/cuda_bindings/tests/test_path_finder_find_headers.py b/cuda_bindings/tests/test_path_finder_find_headers.py new file mode 100644 index 0000000000..5d9b095144 --- /dev/null +++ b/cuda_bindings/tests/test_path_finder_find_headers.py @@ -0,0 +1,24 @@ +# Copyright 2025 NVIDIA Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +import pytest + +from cuda.bindings._path_finder import find_nvidia_headers + + +def test_find_nvidia_header_directory(info_summary_append): + with pytest.raises(AssertionError): + find_nvidia_headers.find_nvidia_header_directory("unknown-libname") + + hdr_dir = find_nvidia_headers.find_nvidia_header_directory("nvshmem") + assert hdr_dir in [ + # pip install nvidia-nvshmem-cu12 + "/home/rgrossekunst/forked/cuda-python/venvs/scratch/lib/python3.12/site-packages/nvidia/nvshmem/include", + # conda create -y -n nvshmem python=3.12 + # conda activate nvshmem + # conda install -y conda-forge::libnvshmem3 conda-forge::libnvshmem-dev + "/home/rgrossekunst/miniforge3/envs/nvshmem/include", + # sudo apt install libnvshmem3-cuda-12 libnvshmem3-dev-cuda-12 + "/usr/include/nvshmem_12", + ] + info_summary_append(f"{hdr_dir=!r}") From a98de70227eda8ac42f9d1e62ab1c5e59ccf9b2e Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 28 May 2025 10:29:14 -0700 Subject: [PATCH 03/25] Better error message: UNKNOWN libname='unknown-libname' --- .../cuda/bindings/_path_finder/find_nvidia_headers.py | 4 +++- cuda_bindings/tests/test_path_finder_find_headers.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_headers.py b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_headers.py index bf56097f2f..4632354507 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_headers.py +++ b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_headers.py @@ -11,7 +11,9 @@ @functools.cache def find_nvidia_header_directory(libname: str) -> str: - assert libname == "nvshmem" + if libname != "nvshmem": + raise RuntimeError(f"UNKNOWN {libname=}") + assert not IS_WINDOWS # Installed from a wheel diff --git a/cuda_bindings/tests/test_path_finder_find_headers.py b/cuda_bindings/tests/test_path_finder_find_headers.py index 5d9b095144..587cfd0a98 100644 --- a/cuda_bindings/tests/test_path_finder_find_headers.py +++ b/cuda_bindings/tests/test_path_finder_find_headers.py @@ -7,7 +7,7 @@ def test_find_nvidia_header_directory(info_summary_append): - with pytest.raises(AssertionError): + with pytest.raises(RuntimeError, match="^UNKNOWN libname='unknown-libname'$"): find_nvidia_headers.find_nvidia_header_directory("unknown-libname") hdr_dir = find_nvidia_headers.find_nvidia_header_directory("nvshmem") From 6a93fe7b9f3ccb538fa26bba07e5808db97c03d4 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 28 May 2025 10:40:04 -0700 Subject: [PATCH 04/25] if libname == "nvshmem" and IS_WINDOWS: return None --- .../cuda/bindings/_path_finder/find_nvidia_headers.py | 4 +++- cuda_bindings/tests/test_path_finder_find_headers.py | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_headers.py b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_headers.py index 4632354507..befd4ca819 100644 --- a/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_headers.py +++ b/cuda_bindings/cuda/bindings/_path_finder/find_nvidia_headers.py @@ -14,7 +14,9 @@ def find_nvidia_header_directory(libname: str) -> str: if libname != "nvshmem": raise RuntimeError(f"UNKNOWN {libname=}") - assert not IS_WINDOWS + if libname == "nvshmem" and IS_WINDOWS: + # nvshmem has no Windows support. + return None # Installed from a wheel nvidia_sub_dirs = ("nvidia", "nvshmem", "include") diff --git a/cuda_bindings/tests/test_path_finder_find_headers.py b/cuda_bindings/tests/test_path_finder_find_headers.py index 587cfd0a98..d760817890 100644 --- a/cuda_bindings/tests/test_path_finder_find_headers.py +++ b/cuda_bindings/tests/test_path_finder_find_headers.py @@ -11,14 +11,20 @@ def test_find_nvidia_header_directory(info_summary_append): find_nvidia_headers.find_nvidia_header_directory("unknown-libname") hdr_dir = find_nvidia_headers.find_nvidia_header_directory("nvshmem") + # TODO: Find ways to test more meaningfully, and how to avoid HARD-WIRED PATHS in particular. assert hdr_dir in [ # pip install nvidia-nvshmem-cu12 "/home/rgrossekunst/forked/cuda-python/venvs/scratch/lib/python3.12/site-packages/nvidia/nvshmem/include", + # # conda create -y -n nvshmem python=3.12 # conda activate nvshmem # conda install -y conda-forge::libnvshmem3 conda-forge::libnvshmem-dev "/home/rgrossekunst/miniforge3/envs/nvshmem/include", + # # sudo apt install libnvshmem3-cuda-12 libnvshmem3-dev-cuda-12 "/usr/include/nvshmem_12", + # + # nvshmem not available + None, ] info_summary_append(f"{hdr_dir=!r}") From c1e93856e10888682d2f65b642ba4641abf2d0f1 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 11 Aug 2025 22:15:27 -0700 Subject: [PATCH 05/25] =?UTF-8?q?Move=20find=5Fnvidia=5Fheaders.py=20?= =?UTF-8?q?=E2=86=92=20=5Fheaders/find=5Fnvidia=5Fheaders.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cuda_pathfinder/cuda/pathfinder/__init__.py | 1 + .../cuda/pathfinder/{ => _headers}/find_nvidia_headers.py | 0 cuda_pathfinder/tests/test_find_nvidia_headers.py | 6 +++--- 3 files changed, 4 insertions(+), 3 deletions(-) rename cuda_pathfinder/cuda/pathfinder/{ => _headers}/find_nvidia_headers.py (100%) diff --git a/cuda_pathfinder/cuda/pathfinder/__init__.py b/cuda_pathfinder/cuda/pathfinder/__init__.py index 6bdeb151ed..06cdff2f7a 100644 --- a/cuda_pathfinder/cuda/pathfinder/__init__.py +++ b/cuda_pathfinder/cuda/pathfinder/__init__.py @@ -7,4 +7,5 @@ from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import ( SUPPORTED_LIBNAMES as SUPPORTED_NVIDIA_LIBNAMES, # noqa: F401 ) +from cuda.pathfinder._headers.find_nvidia_headers import find_nvidia_header_directory as find_nvidia_header_directory from cuda.pathfinder._version import __version__ as __version__ diff --git a/cuda_pathfinder/cuda/pathfinder/find_nvidia_headers.py b/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py similarity index 100% rename from cuda_pathfinder/cuda/pathfinder/find_nvidia_headers.py rename to cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py diff --git a/cuda_pathfinder/tests/test_find_nvidia_headers.py b/cuda_pathfinder/tests/test_find_nvidia_headers.py index ea6a03a4d2..399ccc1046 100644 --- a/cuda_pathfinder/tests/test_find_nvidia_headers.py +++ b/cuda_pathfinder/tests/test_find_nvidia_headers.py @@ -3,14 +3,14 @@ import pytest -from cuda.pathfinder import find_nvidia_headers +from cuda.pathfinder import find_nvidia_header_directory def test_find_nvidia_header_directory(info_summary_append): with pytest.raises(RuntimeError, match=r"^UNKNOWN libname='unknown-libname'$"): - find_nvidia_headers.find_nvidia_header_directory("unknown-libname") + find_nvidia_header_directory("unknown-libname") - hdr_dir = find_nvidia_headers.find_nvidia_header_directory("nvshmem") + hdr_dir = find_nvidia_header_directory("nvshmem") # TODO: Find ways to test more meaningfully, and how to avoid HARD-WIRED PATHS in particular. assert hdr_dir in [ # pip install nvidia-nvshmem-cu12 From c3464b31ae273f6f7e80577be444f62b06f9ebe3 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Mon, 11 Aug 2025 23:20:02 -0700 Subject: [PATCH 06/25] test_find_nvidia_headers.py: removed hard-wired paths, comments with more complete commands for setting up manual testing. --- .../tests/test_find_nvidia_headers.py | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/cuda_pathfinder/tests/test_find_nvidia_headers.py b/cuda_pathfinder/tests/test_find_nvidia_headers.py index 399ccc1046..5977f7735b 100644 --- a/cuda_pathfinder/tests/test_find_nvidia_headers.py +++ b/cuda_pathfinder/tests/test_find_nvidia_headers.py @@ -11,20 +11,19 @@ def test_find_nvidia_header_directory(info_summary_append): find_nvidia_header_directory("unknown-libname") hdr_dir = find_nvidia_header_directory("nvshmem") - # TODO: Find ways to test more meaningfully, and how to avoid HARD-WIRED PATHS in particular. - assert hdr_dir in [ - # pip install nvidia-nvshmem-cu12 - "/home/rgrossekunst/forked/cuda-python/venvs/scratch/lib/python3.12/site-packages/nvidia/nvshmem/include", - # - # conda create -y -n nvshmem python=3.12 - # conda activate nvshmem - # conda install -y conda-forge::libnvshmem3 conda-forge::libnvshmem-dev - "/home/rgrossekunst/miniforge3/envs/nvshmem/include", - # - # sudo apt install libnvshmem3-cuda-12 libnvshmem3-dev-cuda-12 - "/usr/include/nvshmem_12", - # - # nvshmem not available - None, - ] + # TODO: Find ways to test more meaningfully. + + # pip install nvidia-nvshmem-cu12 + # pip install nvidia-nvshmem-cu13 + + # conda create -y -n nvshmem python=3.12 + # conda activate nvshmem + # conda install -y conda-forge::libnvshmem3 conda-forge::libnvshmem-dev + + # wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb + # sudo dpkg -i cuda-keyring_1.1-1_all.deb + # sudo apt update + # sudo apt install libnvshmem3-cuda-12 libnvshmem3-dev-cuda-12 + # sudo apt install libnvshmem3-cuda-13 libnvshmem3-dev-cuda-13 + info_summary_append(f"{hdr_dir=!r}") From 26f94d13582368d41c5f6f6ed0214239528fe42d Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 22 Aug 2025 12:15:38 -0700 Subject: [PATCH 07/25] Make _find_nvidia_header_directory private for now. --- cuda_pathfinder/cuda/pathfinder/__init__.py | 4 +++- cuda_pathfinder/tests/test_find_nvidia_headers.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cuda_pathfinder/cuda/pathfinder/__init__.py b/cuda_pathfinder/cuda/pathfinder/__init__.py index 06cdff2f7a..53f2527c71 100644 --- a/cuda_pathfinder/cuda/pathfinder/__init__.py +++ b/cuda_pathfinder/cuda/pathfinder/__init__.py @@ -7,5 +7,7 @@ from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import ( SUPPORTED_LIBNAMES as SUPPORTED_NVIDIA_LIBNAMES, # noqa: F401 ) -from cuda.pathfinder._headers.find_nvidia_headers import find_nvidia_header_directory as find_nvidia_header_directory +from cuda.pathfinder._headers.find_nvidia_headers import ( + find_nvidia_header_directory as _find_nvidia_header_directory, # noqa: F401 +) from cuda.pathfinder._version import __version__ as __version__ diff --git a/cuda_pathfinder/tests/test_find_nvidia_headers.py b/cuda_pathfinder/tests/test_find_nvidia_headers.py index 5977f7735b..b2b12618e0 100644 --- a/cuda_pathfinder/tests/test_find_nvidia_headers.py +++ b/cuda_pathfinder/tests/test_find_nvidia_headers.py @@ -3,7 +3,7 @@ import pytest -from cuda.pathfinder import find_nvidia_header_directory +from cuda.pathfinder import _find_nvidia_header_directory as find_nvidia_header_directory def test_find_nvidia_header_directory(info_summary_append): From 6adddb010139aee7575c161477427f71b3c20049 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 22 Aug 2025 12:19:30 -0700 Subject: [PATCH 08/25] test_find_nvidia_headers.py: Move comments with installation commands up. --- .../tests/test_find_nvidia_headers.py | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/cuda_pathfinder/tests/test_find_nvidia_headers.py b/cuda_pathfinder/tests/test_find_nvidia_headers.py index b2b12618e0..9d20bcbda6 100644 --- a/cuda_pathfinder/tests/test_find_nvidia_headers.py +++ b/cuda_pathfinder/tests/test_find_nvidia_headers.py @@ -1,6 +1,21 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# Currently these installations are only manually tested: + +# pip install nvidia-nvshmem-cu12 +# pip install nvidia-nvshmem-cu13 + +# conda create -y -n nvshmem python=3.12 +# conda activate nvshmem +# conda install -y conda-forge::libnvshmem3 conda-forge::libnvshmem-dev + +# wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb +# sudo dpkg -i cuda-keyring_1.1-1_all.deb +# sudo apt update +# sudo apt install libnvshmem3-cuda-12 libnvshmem3-dev-cuda-12 +# sudo apt install libnvshmem3-cuda-13 libnvshmem3-dev-cuda-13 + import pytest from cuda.pathfinder import _find_nvidia_header_directory as find_nvidia_header_directory @@ -13,17 +28,4 @@ def test_find_nvidia_header_directory(info_summary_append): hdr_dir = find_nvidia_header_directory("nvshmem") # TODO: Find ways to test more meaningfully. - # pip install nvidia-nvshmem-cu12 - # pip install nvidia-nvshmem-cu13 - - # conda create -y -n nvshmem python=3.12 - # conda activate nvshmem - # conda install -y conda-forge::libnvshmem3 conda-forge::libnvshmem-dev - - # wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb - # sudo dpkg -i cuda-keyring_1.1-1_all.deb - # sudo apt update - # sudo apt install libnvshmem3-cuda-12 libnvshmem3-dev-cuda-12 - # sudo apt install libnvshmem3-cuda-13 libnvshmem3-dev-cuda-13 - info_summary_append(f"{hdr_dir=!r}") From 363b64915cef75a2ec8cb51c3961fe41b6f8830a Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 22 Aug 2025 12:36:06 -0700 Subject: [PATCH 09/25] Add `have_nvidia_nvshmem_package()` function to enable `assert hdr_dir is not None` --- .../tests/test_find_nvidia_headers.py | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/cuda_pathfinder/tests/test_find_nvidia_headers.py b/cuda_pathfinder/tests/test_find_nvidia_headers.py index 9d20bcbda6..fe154e171d 100644 --- a/cuda_pathfinder/tests/test_find_nvidia_headers.py +++ b/cuda_pathfinder/tests/test_find_nvidia_headers.py @@ -3,9 +3,6 @@ # Currently these installations are only manually tested: -# pip install nvidia-nvshmem-cu12 -# pip install nvidia-nvshmem-cu13 - # conda create -y -n nvshmem python=3.12 # conda activate nvshmem # conda install -y conda-forge::libnvshmem3 conda-forge::libnvshmem-dev @@ -16,16 +13,30 @@ # sudo apt install libnvshmem3-cuda-12 libnvshmem3-dev-cuda-12 # sudo apt install libnvshmem3-cuda-13 libnvshmem3-dev-cuda-13 +import functools +import importlib.metadata +import re + import pytest from cuda.pathfinder import _find_nvidia_header_directory as find_nvidia_header_directory -def test_find_nvidia_header_directory(info_summary_append): +@functools.cache +def have_nvidia_nvshmem_package() -> bool: + pattern = re.compile(r"^nvidia-nvshmem-.*$") + return any( + pattern.match(dist.metadata["Name"]) for dist in importlib.metadata.distributions() if "Name" in dist.metadata + ) + + +def test_unknown_libname(): with pytest.raises(RuntimeError, match=r"^UNKNOWN libname='unknown-libname'$"): find_nvidia_header_directory("unknown-libname") - hdr_dir = find_nvidia_header_directory("nvshmem") - # TODO: Find ways to test more meaningfully. +def test_find_libname_nvshmem(info_summary_append): + hdr_dir = find_nvidia_header_directory("nvshmem") info_summary_append(f"{hdr_dir=!r}") + if have_nvidia_nvshmem_package(): + assert hdr_dir is not None From d3f97e46b4205184c8f76757e60ba79d64d51b7d Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 22 Aug 2025 13:20:20 -0700 Subject: [PATCH 10/25] Add nvidia-nvshmem-cu12,13 in pyproject.toml --- cuda_pathfinder/pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cuda_pathfinder/pyproject.toml b/cuda_pathfinder/pyproject.toml index 6545c4e51e..80df9c3f6b 100644 --- a/cuda_pathfinder/pyproject.toml +++ b/cuda_pathfinder/pyproject.toml @@ -17,10 +17,12 @@ test = [ nvidia_wheels_cu12 = [ "cuda-toolkit[nvcc,cublas,nvrtc,cudart,cufft,curand,cusolver,cusparse,npp,nvfatbin,nvjitlink,nvjpeg]==12.*", "cuda-toolkit[cufile]==12.*; sys_platform != 'win32'", + "nvidia-nvshmem-cu12; sys_platform != 'win32'", ] nvidia_wheels_cu13 = [ "cuda-toolkit[nvcc,cublas,nvrtc,cudart,cufft,curand,cusolver,cusparse,npp,nvfatbin,nvjitlink,nvjpeg,nvvm]==13.*", "cuda-toolkit[cufile]==13.*; sys_platform != 'win32'", + "nvidia-nvshmem-cu13; sys_platform != 'win32'", ] [project.urls] From 6f4d762b32ab7934e7d3478f20284df8d77ace1b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 22 Aug 2025 13:45:59 -0700 Subject: [PATCH 11/25] assert site-packages or dist-packages --- cuda_pathfinder/tests/test_find_nvidia_headers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cuda_pathfinder/tests/test_find_nvidia_headers.py b/cuda_pathfinder/tests/test_find_nvidia_headers.py index fe154e171d..880a3133f4 100644 --- a/cuda_pathfinder/tests/test_find_nvidia_headers.py +++ b/cuda_pathfinder/tests/test_find_nvidia_headers.py @@ -15,6 +15,7 @@ import functools import importlib.metadata +import os import re import pytest @@ -40,3 +41,11 @@ def test_find_libname_nvshmem(info_summary_append): info_summary_append(f"{hdr_dir=!r}") if have_nvidia_nvshmem_package(): assert hdr_dir is not None + hdr_dir_parts = hdr_dir.split(os.path.sep) + assert any( + sub_dir in hdr_dir_parts + for sub_dir in ( + "site-packages", # pip install + "dist-packages", # apt install + ) + ), hdr_dir From dfa3384102f820526521e990649248578730dff0 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 22 Aug 2025 13:57:47 -0700 Subject: [PATCH 12/25] Add CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS --- cuda_pathfinder/tests/test_find_nvidia_headers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cuda_pathfinder/tests/test_find_nvidia_headers.py b/cuda_pathfinder/tests/test_find_nvidia_headers.py index 880a3133f4..2b4599facd 100644 --- a/cuda_pathfinder/tests/test_find_nvidia_headers.py +++ b/cuda_pathfinder/tests/test_find_nvidia_headers.py @@ -22,6 +22,9 @@ from cuda.pathfinder import _find_nvidia_header_directory as find_nvidia_header_directory +STRICTNESS = os.environ.get("CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS", "see_what_works") +assert STRICTNESS in ("see_what_works", "all_must_work") + @functools.cache def have_nvidia_nvshmem_package() -> bool: @@ -39,7 +42,7 @@ def test_unknown_libname(): def test_find_libname_nvshmem(info_summary_append): hdr_dir = find_nvidia_header_directory("nvshmem") info_summary_append(f"{hdr_dir=!r}") - if have_nvidia_nvshmem_package(): + if STRICTNESS == "all_must_work" or have_nvidia_nvshmem_package(): assert hdr_dir is not None hdr_dir_parts = hdr_dir.split(os.path.sep) assert any( From 673c38c5fed81332bea0cc3f001096c9a9f3139b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 22 Aug 2025 14:07:05 -0700 Subject: [PATCH 13/25] Transfer `ci/`, `.github/` changes from PR #864 --- .github/workflows/test-wheel-linux.yml | 9 ++++----- .github/workflows/test-wheel-windows.yml | 8 +++----- ci/tools/env-vars | 1 + 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index b775670a9b..1ffeec0b99 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -321,16 +321,15 @@ jobs: pip install $(ls cuda_python*.whl)[all] fi - - name: Install cuda.pathfinder nvidia_wheels_cu13 - if: startsWith(matrix.CUDA_VER, '13.') + - name: Install cuda.pathfinder extra wheels for testing run: | + set -euo pipefail pushd cuda_pathfinder - pip install -v .[nvidia_wheels_cu13] - pip freeze + pip install -v ".[nvidia_wheels_cu${TEST_CUDA_MAJOR}]" + pip list popd - name: Run cuda.pathfinder tests with all_must_work - if: startsWith(matrix.CUDA_VER, '13.') env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work run: run-tests pathfinder diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index 18ddbcb45a..ec4443035d 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -288,17 +288,15 @@ jobs: pip install "$((Get-ChildItem -Filter cuda_python*.whl).FullName)[all]" } - - name: Install cuda.pathfinder nvidia_wheels_cu13 - if: startsWith(matrix.CUDA_VER, '13.') + - name: Install cuda.pathfinder extra wheels for testing shell: bash --noprofile --norc -xeuo pipefail {0} run: | pushd cuda_pathfinder - pip install -v .[nvidia_wheels_cu13] - pip freeze + pip install -v ".[nvidia_wheels_cu${TEST_CUDA_MAJOR}]" + pip list popd - name: Run cuda.pathfinder tests with all_must_work - if: startsWith(matrix.CUDA_VER, '13.') env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work shell: bash --noprofile --norc -xeuo pipefail {0} diff --git a/ci/tools/env-vars b/ci/tools/env-vars index 3dcb81a4c0..19126cd133 100755 --- a/ci/tools/env-vars +++ b/ci/tools/env-vars @@ -69,6 +69,7 @@ elif [[ "${1}" == "test" ]]; then echo "SETUP_SANITIZER=${SETUP_SANITIZER}" >> $GITHUB_ENV echo "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $GITHUB_ENV echo "SKIP_CYTHON_TEST=${SKIP_CYTHON_TEST}" >> $GITHUB_ENV + echo "TEST_CUDA_MAJOR=${TEST_CUDA_MAJOR}" >> $GITHUB_ENV fi echo "CUDA_BINDINGS_ARTIFACT_BASENAME=${CUDA_BINDINGS_ARTIFACT_BASENAME}" >> $GITHUB_ENV From 80cece39fd030187b9f7f9918251224689f32c75 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 22 Aug 2025 14:19:57 -0700 Subject: [PATCH 14/25] Add CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS in `ci/`, `.github/` --- .github/workflows/test-wheel-linux.yml | 2 ++ .github/workflows/test-wheel-windows.yml | 2 ++ ci/tools/run-tests | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 1ffeec0b99..c70dcc273f 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -298,6 +298,7 @@ jobs: - name: Run cuda.pathfinder tests with see_what_works env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: see_what_works + CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: see_what_works run: run-tests pathfinder - name: Run cuda.bindings tests @@ -332,4 +333,5 @@ jobs: - name: Run cuda.pathfinder tests with all_must_work env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work + CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: all_must_work run: run-tests pathfinder diff --git a/.github/workflows/test-wheel-windows.yml b/.github/workflows/test-wheel-windows.yml index ec4443035d..b0a05badee 100644 --- a/.github/workflows/test-wheel-windows.yml +++ b/.github/workflows/test-wheel-windows.yml @@ -262,6 +262,7 @@ jobs: - name: Run cuda.pathfinder tests with see_what_works env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: see_what_works + CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: see_what_works shell: bash --noprofile --norc -xeuo pipefail {0} run: run-tests pathfinder @@ -299,5 +300,6 @@ jobs: - name: Run cuda.pathfinder tests with all_must_work env: CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS: all_must_work + CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS: all_must_work shell: bash --noprofile --norc -xeuo pipefail {0} run: run-tests pathfinder diff --git a/ci/tools/run-tests b/ci/tools/run-tests index ad1dbd6a9e..22d6bd07c6 100755 --- a/ci/tools/run-tests +++ b/ci/tools/run-tests @@ -31,7 +31,9 @@ popd if [[ "${test_module}" == "pathfinder" ]]; then pushd ./cuda_pathfinder - echo "Running pathfinder tests with ${CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS}" + echo "Running pathfinder tests with " \ + "LD:${CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS} " \ + "FH:${CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS}" pwd pytest -ra -s -v tests/ popd From 7c3029286b9fa61b40a911ac3eba56ef5674cf9f Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 22 Aug 2025 14:33:23 -0700 Subject: [PATCH 15/25] reverse=True in sorting of "/usr/include/nvshmem_*" (find newest first) --- cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py b/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py index a63fb51a50..fcf0d10adf 100644 --- a/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py +++ b/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py @@ -35,7 +35,7 @@ def find_nvidia_header_directory(libname: str) -> Optional[str]: if os.path.isfile(nvshmem_h_path): return hdr_dir - for hdr_dir in sorted(glob.glob("/usr/include/nvshmem_*")): + for hdr_dir in sorted(glob.glob("/usr/include/nvshmem_*"), reverse=True): if os.path.isdir(hdr_dir): nvshmem_h_path = os.path.join(hdr_dir, "nvshmem.h") if os.path.isfile(nvshmem_h_path): From a3004194f65cbfaf6eb021f41ecb8bd4e0eea245 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 22 Aug 2025 14:39:21 -0700 Subject: [PATCH 16/25] Fix: assert site-packages or dist-packages only if have_nvidia_nvshmem_package() --- .../tests/test_find_nvidia_headers.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/cuda_pathfinder/tests/test_find_nvidia_headers.py b/cuda_pathfinder/tests/test_find_nvidia_headers.py index 2b4599facd..27799eb26f 100644 --- a/cuda_pathfinder/tests/test_find_nvidia_headers.py +++ b/cuda_pathfinder/tests/test_find_nvidia_headers.py @@ -44,11 +44,12 @@ def test_find_libname_nvshmem(info_summary_append): info_summary_append(f"{hdr_dir=!r}") if STRICTNESS == "all_must_work" or have_nvidia_nvshmem_package(): assert hdr_dir is not None - hdr_dir_parts = hdr_dir.split(os.path.sep) - assert any( - sub_dir in hdr_dir_parts - for sub_dir in ( - "site-packages", # pip install - "dist-packages", # apt install - ) - ), hdr_dir + if have_nvidia_nvshmem_package(): + hdr_dir_parts = hdr_dir.split(os.path.sep) + assert any( + sub_dir in hdr_dir_parts + for sub_dir in ( + "site-packages", # pip install + "dist-packages", # apt install + ) + ), hdr_dir From 3ae15e06c2191360caab82cf3ac9d345cc094d85 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 22 Aug 2025 15:27:43 -0700 Subject: [PATCH 17/25] pytest.skip("nvshmem has no Windows support.") --- cuda_pathfinder/tests/test_find_nvidia_headers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cuda_pathfinder/tests/test_find_nvidia_headers.py b/cuda_pathfinder/tests/test_find_nvidia_headers.py index 27799eb26f..ad26510911 100644 --- a/cuda_pathfinder/tests/test_find_nvidia_headers.py +++ b/cuda_pathfinder/tests/test_find_nvidia_headers.py @@ -21,6 +21,7 @@ import pytest from cuda.pathfinder import _find_nvidia_header_directory as find_nvidia_header_directory +from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import IS_WINDOWS STRICTNESS = os.environ.get("CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS", "see_what_works") assert STRICTNESS in ("see_what_works", "all_must_work") @@ -42,6 +43,9 @@ def test_unknown_libname(): def test_find_libname_nvshmem(info_summary_append): hdr_dir = find_nvidia_header_directory("nvshmem") info_summary_append(f"{hdr_dir=!r}") + if IS_WINDOWS: + assert hdr_dir is None + pytest.skip("nvshmem has no Windows support.") if STRICTNESS == "all_must_work" or have_nvidia_nvshmem_package(): assert hdr_dir is not None if have_nvidia_nvshmem_package(): From eb2e78a4825619ea4d703709dbe25b9239af4ab8 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 3 Sep 2025 16:22:59 -0700 Subject: [PATCH 18/25] Add new cuda/pathfinder/_utils/conda_env.py and use from find_nvidia_headers.py --- .../_headers/find_nvidia_headers.py | 7 +- .../cuda/pathfinder/_utils/conda_env.py | 41 +++++++ cuda_pathfinder/tests/test_utils_conda_env.py | 105 ++++++++++++++++++ 3 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 cuda_pathfinder/cuda/pathfinder/_utils/conda_env.py create mode 100644 cuda_pathfinder/tests/test_utils_conda_env.py diff --git a/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py b/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py index fcf0d10adf..8821976ddf 100644 --- a/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py +++ b/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py @@ -7,6 +7,7 @@ from typing import Optional from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import IS_WINDOWS +from cuda.pathfinder._utils.conda_env import get_conda_prefix from cuda.pathfinder._utils.find_sub_dirs import find_sub_dirs_all_sitepackages @@ -27,9 +28,9 @@ def find_nvidia_header_directory(libname: str) -> Optional[str]: if os.path.isfile(nvshmem_h_path): return hdr_dir - conda_prefix = os.environ.get("CONDA_PREFIX") - if conda_prefix and os.path.isdir(conda_prefix): - hdr_dir = os.path.join(conda_prefix, "include") + conda_prefix = get_conda_prefix() + if conda_prefix and os.path.isdir(conda_prefix.path): + hdr_dir = os.path.join(conda_prefix.path, "include") if os.path.isdir(hdr_dir): nvshmem_h_path = os.path.join(hdr_dir, "nvshmem.h") if os.path.isfile(nvshmem_h_path): diff --git a/cuda_pathfinder/cuda/pathfinder/_utils/conda_env.py b/cuda_pathfinder/cuda/pathfinder/_utils/conda_env.py new file mode 100644 index 0000000000..15221660e1 --- /dev/null +++ b/cuda_pathfinder/cuda/pathfinder/_utils/conda_env.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import functools +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Literal, Optional + +# https://docs.conda.io/projects/conda-build/en/stable/user-guide/environment-variables.html + +BUILD_STATES = ("RENDER", "BUILD", "TEST") + + +@dataclass(frozen=True) +class CondaPrefix: + env_state: Literal["RENDER", "BUILD", "TEST", "activated"] + path: Path + + +@functools.cache +def get_conda_prefix() -> Optional[CondaPrefix]: + """ + Return the effective conda prefix. + - RENDER, BUILD, TEST: inside conda-build (host prefix at $PREFIX) + - activated: user-activated env ($CONDA_PREFIX) + - None: neither detected + """ + state = os.getenv("CONDA_BUILD_STATE") + if state: + if state in BUILD_STATES: + p = os.getenv("PREFIX") + if p: + return CondaPrefix(state, Path(p)) # type: ignore[arg-type] + return None + + cp = os.getenv("CONDA_PREFIX") + if cp: + return CondaPrefix("activated", Path(cp)) + + return None diff --git a/cuda_pathfinder/tests/test_utils_conda_env.py b/cuda_pathfinder/tests/test_utils_conda_env.py new file mode 100644 index 0000000000..9e7a1b44e7 --- /dev/null +++ b/cuda_pathfinder/tests/test_utils_conda_env.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from cuda.pathfinder._utils.conda_env import ( + BUILD_STATES, + CondaPrefix, + get_conda_prefix, +) + +# Auto-clean environment & cache before every test ----------------------------- + + +@pytest.fixture(autouse=True) +def _clean_env_and_cache(monkeypatch): + # Remove any possibly inherited variables from the test runner environment + for k in ("CONDA_BUILD_STATE", "PREFIX", "CONDA_PREFIX"): + monkeypatch.delenv(k, raising=False) + # Clear the cached result between tests + get_conda_prefix.cache_clear() + return + # (No teardown needed; monkeypatch auto-reverts) + + +# Tests ----------------------------------------------------------------------- + + +def test_returns_none_when_no_relevant_env_vars(): + assert get_conda_prefix() is None + + +@pytest.mark.parametrize("state", BUILD_STATES) +def test_build_state_returns_prefix_when_present(state, monkeypatch, tmp_path): + monkeypatch.setenv("CONDA_BUILD_STATE", state) + monkeypatch.setenv("PREFIX", str(tmp_path)) + res = get_conda_prefix() + assert isinstance(res, CondaPrefix) + assert res.env_state == state + assert res.path == tmp_path + + +@pytest.mark.parametrize("state", BUILD_STATES) +def test_build_state_requires_prefix_otherwise_none(state, monkeypatch): + monkeypatch.setenv("CONDA_BUILD_STATE", state) + # No PREFIX set + assert get_conda_prefix() is None + + +@pytest.mark.parametrize("state", BUILD_STATES) +def test_build_state_with_empty_prefix_returns_none(state, monkeypatch): + monkeypatch.setenv("CONDA_BUILD_STATE", state) + monkeypatch.setenv("PREFIX", "") + assert get_conda_prefix() is None + + +def test_activated_env_returns_conda_prefix(monkeypatch, tmp_path): + monkeypatch.setenv("CONDA_PREFIX", str(tmp_path)) + res = get_conda_prefix() + assert isinstance(res, CondaPrefix) + assert res.env_state == "activated" + assert res.path == tmp_path + + +def test_activated_env_ignores_empty_conda_prefix(monkeypatch): + monkeypatch.setenv("CONDA_PREFIX", "") + assert get_conda_prefix() is None + + +def test_build_state_wins_over_activated_when_valid(monkeypatch, tmp_path): + build_p = tmp_path / "host" + user_p = tmp_path / "user" + monkeypatch.setenv("CONDA_BUILD_STATE", "TEST") + monkeypatch.setenv("PREFIX", str(build_p)) + monkeypatch.setenv("CONDA_PREFIX", str(user_p)) + res = get_conda_prefix() + assert res + assert res.env_state == "TEST" + assert res.path == build_p + + +def test_unknown_build_state_returns_none_even_if_conda_prefix_set(monkeypatch, tmp_path): + # Any non-empty CONDA_BUILD_STATE that is not recognized -> None + monkeypatch.setenv("CONDA_BUILD_STATE", "SOMETHING_ELSE") + monkeypatch.setenv("CONDA_PREFIX", str(tmp_path)) + assert get_conda_prefix() is None + + +def test_empty_build_state_treated_as_absent_and_falls_back_to_activated(monkeypatch, tmp_path): + # Empty string is falsy -> treated like "not set" -> activated path + monkeypatch.setenv("CONDA_BUILD_STATE", "") + monkeypatch.setenv("CONDA_PREFIX", str(tmp_path)) + res = get_conda_prefix() + assert res + assert res.env_state == "activated" + assert res.path == tmp_path + + +def test_have_cache(monkeypatch, tmp_path): + monkeypatch.setenv("CONDA_PREFIX", str(tmp_path)) + res = get_conda_prefix() + assert res + assert res.path == tmp_path + res2 = get_conda_prefix() + assert res2 is res From c90c393c1971d75dbdd924f9ade52c1886b82427 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Wed, 3 Sep 2025 17:25:45 -0700 Subject: [PATCH 19/25] Add new cuda/pathfinder/_utils/env_vars_for_include.py and use from find_nvidia_headers.py --- .../_headers/find_nvidia_headers.py | 9 +- .../pathfinder/_utils/env_vars_for_include.py | 25 ++++ .../tests/test_utils_env_vars_for_include.py | 107 ++++++++++++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 cuda_pathfinder/cuda/pathfinder/_utils/env_vars_for_include.py create mode 100644 cuda_pathfinder/tests/test_utils_env_vars_for_include.py diff --git a/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py b/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py index 8821976ddf..add1a1a3d5 100644 --- a/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py +++ b/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py @@ -4,10 +4,11 @@ import functools import glob import os -from typing import Optional +from typing import Optional, cast from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import IS_WINDOWS from cuda.pathfinder._utils.conda_env import get_conda_prefix +from cuda.pathfinder._utils.env_vars_for_include import iter_env_vars_for_include_dirs from cuda.pathfinder._utils.find_sub_dirs import find_sub_dirs_all_sitepackages @@ -42,4 +43,10 @@ def find_nvidia_header_directory(libname: str) -> Optional[str]: if os.path.isfile(nvshmem_h_path): return hdr_dir + for hdr_dir in iter_env_vars_for_include_dirs(): + if os.path.isdir(hdr_dir): + nvshmem_h_path = os.path.join(hdr_dir, "nvshmem.h") + if os.path.isfile(nvshmem_h_path): + return cast(str, hdr_dir) # help mypy + return None diff --git a/cuda_pathfinder/cuda/pathfinder/_utils/env_vars_for_include.py b/cuda_pathfinder/cuda/pathfinder/_utils/env_vars_for_include.py new file mode 100644 index 0000000000..f419af7f2d --- /dev/null +++ b/cuda_pathfinder/cuda/pathfinder/_utils/env_vars_for_include.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +from collections.abc import Iterable + +IS_WINDOWS = sys.platform == "win32" + +# GCC/Clang-style include vars +GCC_VNAMES = ("CPATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH") + +# MSVC: INCLUDE is the canonical header search variable +MSVC_GCC_VNAMES = ("INCLUDE",) + +VNAMES: tuple[str, ...] = MSVC_GCC_VNAMES + GCC_VNAMES if IS_WINDOWS else GCC_VNAMES + + +def iter_env_vars_for_include_dirs() -> Iterable[str]: + for vname in VNAMES: + v = os.getenv(vname) + if v: + for d in v.split(os.pathsep): + if d: + yield d diff --git a/cuda_pathfinder/tests/test_utils_env_vars_for_include.py b/cuda_pathfinder/tests/test_utils_env_vars_for_include.py new file mode 100644 index 0000000000..5b184ffbed --- /dev/null +++ b/cuda_pathfinder/tests/test_utils_env_vars_for_include.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os + +import pytest + +from cuda.pathfinder._utils.env_vars_for_include import ( + VNAMES, + iter_env_vars_for_include_dirs, +) + +# --- Fixtures ---------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + # Ensure a clean slate for all variables the helper may read + for k in set(VNAMES) | {"CPATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "INCLUDE"}: + monkeypatch.delenv(k, raising=False) + return + + +def _join(*parts: str) -> str: + return os.pathsep.join(parts) + + +# --- Tests ------------------------------------------------------------------- + + +def test_no_relevant_env_vars_yields_nothing(): + assert list(iter_env_vars_for_include_dirs()) == [] + + +def test_cpath_splits_on_pathsep(monkeypatch): + # Includes an empty element which should be ignored + monkeypatch.setenv("CPATH", _join("/a/include", "", "/b/include")) + out = list(iter_env_vars_for_include_dirs()) + assert out == ["/a/include", "/b/include"] + + +def test_c_include_path_splits_on_pathsep(monkeypatch): + monkeypatch.setenv("C_INCLUDE_PATH", _join("/x", "/y")) + out = list(iter_env_vars_for_include_dirs()) + # Order depends on VNAMES; on all platforms C_INCLUDE_PATH is included + # but may come after CPATH (and INCLUDE on Windows). + # Since only C_INCLUDE_PATH is set here, we get exactly those two. + assert out == ["/x", "/y"] + + +def test_duplicates_are_not_deduplicated(monkeypatch): + # Same directory appears across different vars; should be yielded twice + monkeypatch.setenv("CPATH", _join("/shared", "/only-in-cpath")) + monkeypatch.setenv("C_INCLUDE_PATH", _join("/shared", "/only-in-c-include")) + out = list(iter_env_vars_for_include_dirs()) + expected = [] + # Build the expected list in VNAMES order so the test is platform-agnostic + env_values = { + "CPATH": ["/shared", "/only-in-cpath"], + "C_INCLUDE_PATH": ["/shared", "/only-in-c-include"], + "INCLUDE": [], # may or may not be consulted, but it's unset here + } + for var in VNAMES: + expected.extend(env_values.get(var, [])) + assert out == expected + + +def test_order_follows_vnames(monkeypatch): + # Put distinctive values in each variable to verify overall ordering + mapping = { + "INCLUDE": ["W1", "W2"], # only used on Windows + "CPATH": ["P1", "P2"], + "C_INCLUDE_PATH": ["C1", "C2"], + "CPLUS_INCLUDE_PATH": ["CP1", "CP2"], + } + for var, vals in mapping.items(): + # Only set those that are actually referenced on this platform + if var in VNAMES: + monkeypatch.setenv(var, _join(*vals)) + + out = list(iter_env_vars_for_include_dirs()) + + expected = [] + for var in VNAMES: + expected.extend(mapping.get(var, [])) + assert out == expected + + +def test_ignore_wholly_empty_values(monkeypatch): + # Variable is set but contains only separators / empties + monkeypatch.setenv("CPATH", _join("", "")) # effectively empty + assert list(iter_env_vars_for_include_dirs()) == [] + + +def test_windows_include_behavior(monkeypatch): + # This test is platform-agnostic by keying off VNAMES: + # - On Windows, INCLUDE is honored and should appear in output first. + # - On non-Windows, INCLUDE is ignored entirely. + monkeypatch.setenv("INCLUDE", _join("W:/inc1", "W:/inc2")) + out = list(iter_env_vars_for_include_dirs()) + + if "INCLUDE" in VNAMES: + assert out[:2] == ["W:/inc1", "W:/inc2"] + else: + # Non-Windows platforms should ignore INCLUDE + assert "W:/inc1" not in out + assert "W:/inc2" not in out From cee717ac487b9dc0fd121dfb2ea1fa66e973869e Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 5 Sep 2025 10:05:01 -0700 Subject: [PATCH 20/25] Revert "Add new cuda/pathfinder/_utils/env_vars_for_include.py and use from find_nvidia_headers.py" This reverts commit c90c393c1971d75dbdd924f9ade52c1886b82427. --- .../_headers/find_nvidia_headers.py | 9 +- .../pathfinder/_utils/env_vars_for_include.py | 25 ---- .../tests/test_utils_env_vars_for_include.py | 107 ------------------ 3 files changed, 1 insertion(+), 140 deletions(-) delete mode 100644 cuda_pathfinder/cuda/pathfinder/_utils/env_vars_for_include.py delete mode 100644 cuda_pathfinder/tests/test_utils_env_vars_for_include.py diff --git a/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py b/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py index add1a1a3d5..8821976ddf 100644 --- a/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py +++ b/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py @@ -4,11 +4,10 @@ import functools import glob import os -from typing import Optional, cast +from typing import Optional from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import IS_WINDOWS from cuda.pathfinder._utils.conda_env import get_conda_prefix -from cuda.pathfinder._utils.env_vars_for_include import iter_env_vars_for_include_dirs from cuda.pathfinder._utils.find_sub_dirs import find_sub_dirs_all_sitepackages @@ -43,10 +42,4 @@ def find_nvidia_header_directory(libname: str) -> Optional[str]: if os.path.isfile(nvshmem_h_path): return hdr_dir - for hdr_dir in iter_env_vars_for_include_dirs(): - if os.path.isdir(hdr_dir): - nvshmem_h_path = os.path.join(hdr_dir, "nvshmem.h") - if os.path.isfile(nvshmem_h_path): - return cast(str, hdr_dir) # help mypy - return None diff --git a/cuda_pathfinder/cuda/pathfinder/_utils/env_vars_for_include.py b/cuda_pathfinder/cuda/pathfinder/_utils/env_vars_for_include.py deleted file mode 100644 index f419af7f2d..0000000000 --- a/cuda_pathfinder/cuda/pathfinder/_utils/env_vars_for_include.py +++ /dev/null @@ -1,25 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import os -import sys -from collections.abc import Iterable - -IS_WINDOWS = sys.platform == "win32" - -# GCC/Clang-style include vars -GCC_VNAMES = ("CPATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH") - -# MSVC: INCLUDE is the canonical header search variable -MSVC_GCC_VNAMES = ("INCLUDE",) - -VNAMES: tuple[str, ...] = MSVC_GCC_VNAMES + GCC_VNAMES if IS_WINDOWS else GCC_VNAMES - - -def iter_env_vars_for_include_dirs() -> Iterable[str]: - for vname in VNAMES: - v = os.getenv(vname) - if v: - for d in v.split(os.pathsep): - if d: - yield d diff --git a/cuda_pathfinder/tests/test_utils_env_vars_for_include.py b/cuda_pathfinder/tests/test_utils_env_vars_for_include.py deleted file mode 100644 index 5b184ffbed..0000000000 --- a/cuda_pathfinder/tests/test_utils_env_vars_for_include.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import os - -import pytest - -from cuda.pathfinder._utils.env_vars_for_include import ( - VNAMES, - iter_env_vars_for_include_dirs, -) - -# --- Fixtures ---------------------------------------------------------------- - - -@pytest.fixture(autouse=True) -def _clean_env(monkeypatch): - # Ensure a clean slate for all variables the helper may read - for k in set(VNAMES) | {"CPATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "INCLUDE"}: - monkeypatch.delenv(k, raising=False) - return - - -def _join(*parts: str) -> str: - return os.pathsep.join(parts) - - -# --- Tests ------------------------------------------------------------------- - - -def test_no_relevant_env_vars_yields_nothing(): - assert list(iter_env_vars_for_include_dirs()) == [] - - -def test_cpath_splits_on_pathsep(monkeypatch): - # Includes an empty element which should be ignored - monkeypatch.setenv("CPATH", _join("/a/include", "", "/b/include")) - out = list(iter_env_vars_for_include_dirs()) - assert out == ["/a/include", "/b/include"] - - -def test_c_include_path_splits_on_pathsep(monkeypatch): - monkeypatch.setenv("C_INCLUDE_PATH", _join("/x", "/y")) - out = list(iter_env_vars_for_include_dirs()) - # Order depends on VNAMES; on all platforms C_INCLUDE_PATH is included - # but may come after CPATH (and INCLUDE on Windows). - # Since only C_INCLUDE_PATH is set here, we get exactly those two. - assert out == ["/x", "/y"] - - -def test_duplicates_are_not_deduplicated(monkeypatch): - # Same directory appears across different vars; should be yielded twice - monkeypatch.setenv("CPATH", _join("/shared", "/only-in-cpath")) - monkeypatch.setenv("C_INCLUDE_PATH", _join("/shared", "/only-in-c-include")) - out = list(iter_env_vars_for_include_dirs()) - expected = [] - # Build the expected list in VNAMES order so the test is platform-agnostic - env_values = { - "CPATH": ["/shared", "/only-in-cpath"], - "C_INCLUDE_PATH": ["/shared", "/only-in-c-include"], - "INCLUDE": [], # may or may not be consulted, but it's unset here - } - for var in VNAMES: - expected.extend(env_values.get(var, [])) - assert out == expected - - -def test_order_follows_vnames(monkeypatch): - # Put distinctive values in each variable to verify overall ordering - mapping = { - "INCLUDE": ["W1", "W2"], # only used on Windows - "CPATH": ["P1", "P2"], - "C_INCLUDE_PATH": ["C1", "C2"], - "CPLUS_INCLUDE_PATH": ["CP1", "CP2"], - } - for var, vals in mapping.items(): - # Only set those that are actually referenced on this platform - if var in VNAMES: - monkeypatch.setenv(var, _join(*vals)) - - out = list(iter_env_vars_for_include_dirs()) - - expected = [] - for var in VNAMES: - expected.extend(mapping.get(var, [])) - assert out == expected - - -def test_ignore_wholly_empty_values(monkeypatch): - # Variable is set but contains only separators / empties - monkeypatch.setenv("CPATH", _join("", "")) # effectively empty - assert list(iter_env_vars_for_include_dirs()) == [] - - -def test_windows_include_behavior(monkeypatch): - # This test is platform-agnostic by keying off VNAMES: - # - On Windows, INCLUDE is honored and should appear in output first. - # - On non-Windows, INCLUDE is ignored entirely. - monkeypatch.setenv("INCLUDE", _join("W:/inc1", "W:/inc2")) - out = list(iter_env_vars_for_include_dirs()) - - if "INCLUDE" in VNAMES: - assert out[:2] == ["W:/inc1", "W:/inc2"] - else: - # Non-Windows platforms should ignore INCLUDE - assert "W:/inc1" not in out - assert "W:/inc2" not in out From a50da30990108cc24345f5b06789a17bd2214b58 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 5 Sep 2025 10:05:24 -0700 Subject: [PATCH 21/25] Revert "Add new cuda/pathfinder/_utils/conda_env.py and use from find_nvidia_headers.py" This reverts commit eb2e78a4825619ea4d703709dbe25b9239af4ab8. --- .../_headers/find_nvidia_headers.py | 7 +- .../cuda/pathfinder/_utils/conda_env.py | 41 ------- cuda_pathfinder/tests/test_utils_conda_env.py | 105 ------------------ 3 files changed, 3 insertions(+), 150 deletions(-) delete mode 100644 cuda_pathfinder/cuda/pathfinder/_utils/conda_env.py delete mode 100644 cuda_pathfinder/tests/test_utils_conda_env.py diff --git a/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py b/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py index 8821976ddf..fcf0d10adf 100644 --- a/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py +++ b/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py @@ -7,7 +7,6 @@ from typing import Optional from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import IS_WINDOWS -from cuda.pathfinder._utils.conda_env import get_conda_prefix from cuda.pathfinder._utils.find_sub_dirs import find_sub_dirs_all_sitepackages @@ -28,9 +27,9 @@ def find_nvidia_header_directory(libname: str) -> Optional[str]: if os.path.isfile(nvshmem_h_path): return hdr_dir - conda_prefix = get_conda_prefix() - if conda_prefix and os.path.isdir(conda_prefix.path): - hdr_dir = os.path.join(conda_prefix.path, "include") + conda_prefix = os.environ.get("CONDA_PREFIX") + if conda_prefix and os.path.isdir(conda_prefix): + hdr_dir = os.path.join(conda_prefix, "include") if os.path.isdir(hdr_dir): nvshmem_h_path = os.path.join(hdr_dir, "nvshmem.h") if os.path.isfile(nvshmem_h_path): diff --git a/cuda_pathfinder/cuda/pathfinder/_utils/conda_env.py b/cuda_pathfinder/cuda/pathfinder/_utils/conda_env.py deleted file mode 100644 index 15221660e1..0000000000 --- a/cuda_pathfinder/cuda/pathfinder/_utils/conda_env.py +++ /dev/null @@ -1,41 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import functools -import os -from dataclasses import dataclass -from pathlib import Path -from typing import Literal, Optional - -# https://docs.conda.io/projects/conda-build/en/stable/user-guide/environment-variables.html - -BUILD_STATES = ("RENDER", "BUILD", "TEST") - - -@dataclass(frozen=True) -class CondaPrefix: - env_state: Literal["RENDER", "BUILD", "TEST", "activated"] - path: Path - - -@functools.cache -def get_conda_prefix() -> Optional[CondaPrefix]: - """ - Return the effective conda prefix. - - RENDER, BUILD, TEST: inside conda-build (host prefix at $PREFIX) - - activated: user-activated env ($CONDA_PREFIX) - - None: neither detected - """ - state = os.getenv("CONDA_BUILD_STATE") - if state: - if state in BUILD_STATES: - p = os.getenv("PREFIX") - if p: - return CondaPrefix(state, Path(p)) # type: ignore[arg-type] - return None - - cp = os.getenv("CONDA_PREFIX") - if cp: - return CondaPrefix("activated", Path(cp)) - - return None diff --git a/cuda_pathfinder/tests/test_utils_conda_env.py b/cuda_pathfinder/tests/test_utils_conda_env.py deleted file mode 100644 index 9e7a1b44e7..0000000000 --- a/cuda_pathfinder/tests/test_utils_conda_env.py +++ /dev/null @@ -1,105 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import pytest - -from cuda.pathfinder._utils.conda_env import ( - BUILD_STATES, - CondaPrefix, - get_conda_prefix, -) - -# Auto-clean environment & cache before every test ----------------------------- - - -@pytest.fixture(autouse=True) -def _clean_env_and_cache(monkeypatch): - # Remove any possibly inherited variables from the test runner environment - for k in ("CONDA_BUILD_STATE", "PREFIX", "CONDA_PREFIX"): - monkeypatch.delenv(k, raising=False) - # Clear the cached result between tests - get_conda_prefix.cache_clear() - return - # (No teardown needed; monkeypatch auto-reverts) - - -# Tests ----------------------------------------------------------------------- - - -def test_returns_none_when_no_relevant_env_vars(): - assert get_conda_prefix() is None - - -@pytest.mark.parametrize("state", BUILD_STATES) -def test_build_state_returns_prefix_when_present(state, monkeypatch, tmp_path): - monkeypatch.setenv("CONDA_BUILD_STATE", state) - monkeypatch.setenv("PREFIX", str(tmp_path)) - res = get_conda_prefix() - assert isinstance(res, CondaPrefix) - assert res.env_state == state - assert res.path == tmp_path - - -@pytest.mark.parametrize("state", BUILD_STATES) -def test_build_state_requires_prefix_otherwise_none(state, monkeypatch): - monkeypatch.setenv("CONDA_BUILD_STATE", state) - # No PREFIX set - assert get_conda_prefix() is None - - -@pytest.mark.parametrize("state", BUILD_STATES) -def test_build_state_with_empty_prefix_returns_none(state, monkeypatch): - monkeypatch.setenv("CONDA_BUILD_STATE", state) - monkeypatch.setenv("PREFIX", "") - assert get_conda_prefix() is None - - -def test_activated_env_returns_conda_prefix(monkeypatch, tmp_path): - monkeypatch.setenv("CONDA_PREFIX", str(tmp_path)) - res = get_conda_prefix() - assert isinstance(res, CondaPrefix) - assert res.env_state == "activated" - assert res.path == tmp_path - - -def test_activated_env_ignores_empty_conda_prefix(monkeypatch): - monkeypatch.setenv("CONDA_PREFIX", "") - assert get_conda_prefix() is None - - -def test_build_state_wins_over_activated_when_valid(monkeypatch, tmp_path): - build_p = tmp_path / "host" - user_p = tmp_path / "user" - monkeypatch.setenv("CONDA_BUILD_STATE", "TEST") - monkeypatch.setenv("PREFIX", str(build_p)) - monkeypatch.setenv("CONDA_PREFIX", str(user_p)) - res = get_conda_prefix() - assert res - assert res.env_state == "TEST" - assert res.path == build_p - - -def test_unknown_build_state_returns_none_even_if_conda_prefix_set(monkeypatch, tmp_path): - # Any non-empty CONDA_BUILD_STATE that is not recognized -> None - monkeypatch.setenv("CONDA_BUILD_STATE", "SOMETHING_ELSE") - monkeypatch.setenv("CONDA_PREFIX", str(tmp_path)) - assert get_conda_prefix() is None - - -def test_empty_build_state_treated_as_absent_and_falls_back_to_activated(monkeypatch, tmp_path): - # Empty string is falsy -> treated like "not set" -> activated path - monkeypatch.setenv("CONDA_BUILD_STATE", "") - monkeypatch.setenv("CONDA_PREFIX", str(tmp_path)) - res = get_conda_prefix() - assert res - assert res.env_state == "activated" - assert res.path == tmp_path - - -def test_have_cache(monkeypatch, tmp_path): - monkeypatch.setenv("CONDA_PREFIX", str(tmp_path)) - res = get_conda_prefix() - assert res - assert res.path == tmp_path - res2 = get_conda_prefix() - assert res2 is res From 7cfcbfef3da5c39e4f3e9c345077c62925bb587e Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 5 Sep 2025 10:08:14 -0700 Subject: [PATCH 22/25] Bump pathfinder version to 1.2.2 and add release/1.2.2-notes.rst --- cuda_pathfinder/cuda/pathfinder/_version.py | 2 +- cuda_pathfinder/docs/nv-versions.json | 4 ++++ cuda_pathfinder/docs/source/release.rst | 1 + .../docs/source/release/1.2.2-notes.rst | 19 +++++++++++++++++++ 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 cuda_pathfinder/docs/source/release/1.2.2-notes.rst diff --git a/cuda_pathfinder/cuda/pathfinder/_version.py b/cuda_pathfinder/cuda/pathfinder/_version.py index 62f65a8732..70aa6255c8 100644 --- a/cuda_pathfinder/cuda/pathfinder/_version.py +++ b/cuda_pathfinder/cuda/pathfinder/_version.py @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -__version__ = "1.2.2a0" +__version__ = "1.2.2" diff --git a/cuda_pathfinder/docs/nv-versions.json b/cuda_pathfinder/docs/nv-versions.json index a8f26a1ae1..eb5b96a0a1 100644 --- a/cuda_pathfinder/docs/nv-versions.json +++ b/cuda_pathfinder/docs/nv-versions.json @@ -3,6 +3,10 @@ "version": "latest", "url": "https://nvidia.github.io/cuda-python/cuda-pathfinder/latest/" }, + { + "version": "1.2.2", + "url": "https://nvidia.github.io/cuda-python/cuda-pathfinder/1.2.2/" + }, { "version": "1.2.1", "url": "https://nvidia.github.io/cuda-python/cuda-pathfinder/1.2.1/" diff --git a/cuda_pathfinder/docs/source/release.rst b/cuda_pathfinder/docs/source/release.rst index 56b4be8141..b7c0ff6e19 100644 --- a/cuda_pathfinder/docs/source/release.rst +++ b/cuda_pathfinder/docs/source/release.rst @@ -7,6 +7,7 @@ Release Notes .. toctree:: :maxdepth: 3 + 1.2.2 1.2.1 1.2.0 1.1.0 diff --git a/cuda_pathfinder/docs/source/release/1.2.2-notes.rst b/cuda_pathfinder/docs/source/release/1.2.2-notes.rst new file mode 100644 index 0000000000..0a483081e5 --- /dev/null +++ b/cuda_pathfinder/docs/source/release/1.2.2-notes.rst @@ -0,0 +1,19 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +.. module:: cuda.pathfinder + +``cuda-pathfinder`` 1.2.2 Release notes +======================================= + +Released on Sep 8, 2025 + + +Highlights +---------- + +* Support nccl library (`PR #945 `_) + +* Add experimental ``cuda.pathfinder._find_nvidia_headers`` API, + currently limited to supporting ``nvshmem`` + (`PR #661 `_) From 510f4701a32d3218b5ce2dfad71496e67afafbc9 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 5 Sep 2025 10:44:01 -0700 Subject: [PATCH 23/25] Remove os.path.isdir() tests that are not strictly needed. --- .../pathfinder/_headers/find_nvidia_headers.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py b/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py index fcf0d10adf..cc2c8654cb 100644 --- a/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py +++ b/cuda_pathfinder/cuda/pathfinder/_headers/find_nvidia_headers.py @@ -30,15 +30,13 @@ def find_nvidia_header_directory(libname: str) -> Optional[str]: conda_prefix = os.environ.get("CONDA_PREFIX") if conda_prefix and os.path.isdir(conda_prefix): hdr_dir = os.path.join(conda_prefix, "include") - if os.path.isdir(hdr_dir): - nvshmem_h_path = os.path.join(hdr_dir, "nvshmem.h") - if os.path.isfile(nvshmem_h_path): - return hdr_dir + nvshmem_h_path = os.path.join(hdr_dir, "nvshmem.h") + if os.path.isfile(nvshmem_h_path): + return hdr_dir for hdr_dir in sorted(glob.glob("/usr/include/nvshmem_*"), reverse=True): - if os.path.isdir(hdr_dir): - nvshmem_h_path = os.path.join(hdr_dir, "nvshmem.h") - if os.path.isfile(nvshmem_h_path): - return hdr_dir + nvshmem_h_path = os.path.join(hdr_dir, "nvshmem.h") + if os.path.isfile(nvshmem_h_path): + return hdr_dir return None From 2426260f6413de3a68d68b94076ccd3de39d4fba Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 5 Sep 2025 11:28:17 -0700 Subject: [PATCH 24/25] test_find_nvidia_headers.py: remove check for `dist-packages` because a .deb that installs into dist-packages does not exist. --- cuda_pathfinder/tests/test_find_nvidia_headers.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cuda_pathfinder/tests/test_find_nvidia_headers.py b/cuda_pathfinder/tests/test_find_nvidia_headers.py index ad26510911..2a0a3d0386 100644 --- a/cuda_pathfinder/tests/test_find_nvidia_headers.py +++ b/cuda_pathfinder/tests/test_find_nvidia_headers.py @@ -50,10 +50,4 @@ def test_find_libname_nvshmem(info_summary_append): assert hdr_dir is not None if have_nvidia_nvshmem_package(): hdr_dir_parts = hdr_dir.split(os.path.sep) - assert any( - sub_dir in hdr_dir_parts - for sub_dir in ( - "site-packages", # pip install - "dist-packages", # apt install - ) - ), hdr_dir + assert "site-packages" in hdr_dir_parts From b74a84c41a28d04e7a83e34530ae5d2eaad764ae Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 5 Sep 2025 11:38:29 -0700 Subject: [PATCH 25/25] Additional testing --- cuda_pathfinder/tests/test_find_nvidia_headers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cuda_pathfinder/tests/test_find_nvidia_headers.py b/cuda_pathfinder/tests/test_find_nvidia_headers.py index 2a0a3d0386..2d432b0f21 100644 --- a/cuda_pathfinder/tests/test_find_nvidia_headers.py +++ b/cuda_pathfinder/tests/test_find_nvidia_headers.py @@ -46,8 +46,15 @@ def test_find_libname_nvshmem(info_summary_append): if IS_WINDOWS: assert hdr_dir is None pytest.skip("nvshmem has no Windows support.") + if hdr_dir: + assert os.path.isdir(hdr_dir) + assert os.path.isfile(os.path.join(hdr_dir, "nvshmem.h")) if STRICTNESS == "all_must_work" or have_nvidia_nvshmem_package(): assert hdr_dir is not None if have_nvidia_nvshmem_package(): hdr_dir_parts = hdr_dir.split(os.path.sep) assert "site-packages" in hdr_dir_parts + elif conda_prefix := os.getenv("CONDA_PREFIX"): + assert hdr_dir.startswith(conda_prefix) + else: + assert hdr_dir.startswith("/usr/include/nvshmem_")