diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87d27d7..e439cde 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,8 +1,10 @@ -name: Run tests +name: Run Tests on: workflow_dispatch: pull_request: + schedule: + - cron: "14 3 15 * *" # Runs at 03:14 UTC on the 15th of every month push: branches: - main @@ -27,7 +29,7 @@ jobs: path: vcpkg_installed key: vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg.json') }} - - name: Set compile target + - name: Export vcpkg host triplet for compilation shell: bash run: | if [[ "$RUNNER_OS" == "Windows" ]]; then @@ -76,7 +78,7 @@ jobs: export CMAKE_PREFIX_PATH=${{ github.workspace }}/pybind11/pybind11/share/cmake export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) - uv pip install -v -e . + uv pip install -v -e ".[dev]" py.test @@ -92,6 +94,6 @@ jobs: $env:CMAKE_PREFIX_PATH="${{ github.workspace }}\pybind11\pybind11\share\cmake" $env:CMAKE_BUILD_PARALLEL_LEVEL = [Environment]::ProcessorCount - uv pip install -v -e . + uv pip install -v -e ".[dev]" py.test diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 929e237..0e192a2 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -1,8 +1,10 @@ -name: Build package wheels +name: Build Python Wheels on: workflow_dispatch: pull_request: + schedule: + - cron: "14 3 15 * *" # Runs at 03:14 UTC on the 15th of every month push: branches: - main @@ -19,7 +21,7 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Cache dependencies. # TODO - this may not work on linux b/c os is different than container? + - name: Cache dependencies id: cache-vcpkg-deps uses: actions/cache@v4 with: @@ -33,24 +35,16 @@ jobs: path: vcpkg ref: c9c17dcea3016bc241df0422e82b8aea212dcb93 - - name: Install libraries with vcpkg.json (macOS/Linux) + - name: Export vcpkg host triplet for compilation shell: bash run: | - case "${RUNNER_OS}" in - "Windows") - VCPKG_HOST_TRIPLET="x64-windows-static-md" - ;; - "Linux") - VCPKG_HOST_TRIPLET="x64-linux" - ;; - "macOS") - VCPKG_HOST_TRIPLET="arm64-osx" - ;; - *) - echo "Unsupported RUNNER_OS: ${RUNNER_OS}" - exit 1 - ;; - esac + if [[ "$RUNNER_OS" == "Windows" ]]; then + VCPKG_HOST_TRIPLET=x64-windows-static-md + elif [[ "$RUNNER_OS" == "Linux" ]]; then + VCPKG_HOST_TRIPLET=x64-linux + elif [[ "$RUNNER_OS" == "macOS" ]]; then + VCPKG_HOST_TRIPLET=arm64-osx + fi echo VCPKG_HOST_TRIPLET="$VCPKG_HOST_TRIPLET" >> $GITHUB_ENV echo $VCPKG_HOST_TRIPLET @@ -58,7 +52,10 @@ jobs: env: CIBW_BUILD: "cp313-*" CIBW_SKIP: "*musllinux* *win32*" + CIBW_TEST_COMMAND: pytest {project}/tests + CIBW_TEST_EXTRAS: dev + CIBW_BEFORE_BUILD: rm -rf {project}/build CIBW_BEFORE_ALL_WINDOWS: bash tools\cibw_before.sh CIBW_ENVIRONMENT_WINDOWS: "CMAKE_PREFIX_PATH=D:/a/pycpp/pycpp/pybind11/pybind11/share/cmake" diff --git a/CMakeLists.txt b/CMakeLists.txt index 34d1481..36d0b71 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,11 @@ cmake_minimum_required(VERSION 3.4...3.18) + +# required to find dependencies +if(NOT DEFINED ENV{VCPKG_HOST_TRIPLET}) + message(FATAL_ERROR "Environment variable 'VCPKG_HOST_TRIPLET' is not set") +endif() + project(cppcore) set(VCPKG_HOST_TRIPLET $ENV{VCPKG_HOST_TRIPLET}) @@ -27,9 +33,14 @@ find_library(NLOPT_LIB nlopt PATHS "${NLOPT_DIR_LIBRARY_DIR}") pybind11_add_module(cppcore src/cpp/main.cpp) -include_directories(${CMAKE_SOURCE_DIR}/src/cpp ${EIGEN_PATH} ${GSL_INCLUDE_DIR} ${NLOPT_DIR_INCLUDE_DIR}) -target_include_directories(cppcore PRIVATE ${GSL_INCLUDE_DIR} ${NLOPT_DIR_INCLUDE_DIR}) -target_link_libraries(cppcore PRIVATE ${GSL_LIB} ${GSLCBLAS_LIB} ${NLOPT_LIB}) +target_include_directories(cppcore PRIVATE ${CMAKE_SOURCE_DIR}/src/cpp) +target_include_directories(cppcore PRIVATE ${EIGEN_PATH}) +target_include_directories(cppcore PRIVATE ${GSL_INCLUDE_DIR}) +target_include_directories(cppcore PRIVATE ${NLOPT_DIR_INCLUDE_DIR}) + +target_link_libraries(cppcore PRIVATE ${NLOPT_LIB}) +target_link_libraries(cppcore PRIVATE ${GSL_LIB}) +target_link_libraries(cppcore PRIVATE ${GSLCBLAS_LIB}) target_compile_definitions( cppcore diff --git a/README.md b/README.md index 0f004f1..665c5bc 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,29 @@ -# cppcore for pybind11 +# A Python Package with a C++ extension -MacOS installation: +Explorations in how to use Python, C++, and bindings between the two systems. Build compiled code in multiple environments via GitHub Actions. Key technologies used include [pybind11](https://pybind11.readthedocs.io/en/stable/), [vcpkg](https://github.com/microsoft/vcpkg), and [cibuildwheel](https://cibuildwheel.readthedocs.io/en/stable/). + +The project does the following: + +- Builds a simple C++ package +- Includes third party C++ packages installed via the vcpkg package manager +- Builds a python interface to the C++ package +- Has some tests to ensure the C++ package works correctly from python +- Automatically builds the packages on Mac, Linux, and Windows + +## Quickstart + +For this particular example, we install 3 scientific C++ packages: as dependencies for our library: + +* [Eigen3](https://vcpkg.io/en/package/eigen3.html) (headers only template library) +* [GNU Scientific Library](https://vcpkg.io/en/package/gsl.html) +* [NLopt](https://vcpkg.io/en/package/nlopt.html) + +Though you could customize this template as needed to install other packages from vcpkg. + +### MacOS/Linux: ```bash +# check out a single commit from a git repository mkdir vcpkg cd vcpkg git init @@ -11,14 +32,82 @@ git fetch --depth 1 origin c9c17dcea3016bc241df0422e82b8aea212dcb93 git checkout FETCH_HEAD cd .. -export VCPKG_HOST_TRIPLET=arm64-osx-dynamic +# build static dependencies; use the appropriate triplet (e.g., x64-linux, arm64-osx) +export VCPKG_HOST_TRIPLET="arm64-osx" ./vcpkg/bootstrap-vcpkg.sh ./vcpkg/vcpkg install --host-triplet=$VCPKG_HOST_TRIPLET - +# install pybind11 at the root so for our build phase uv pip install pybind11==3.0.0 --target=./pybind11 -export VCPKG_HOST_TRIPLET=arm64-osx-dynamic -export CMAKE_PREFIX_PATH=/Users/andyshapiro/dev/pycpp/pybind11/pybind11/share/cmake + +# set environment variables for building python extension +export CMAKE_PREFIX_PATH="$(readlink -f ./pybind11/pybind11/share/cmake)" export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) -uv pip install -e . && pytest + +echo "$VCPKG_HOST_TRIPLET" +echo "$CMAKE_PREFIX_PATH" +echo "$CMAKE_BUILD_PARALLEL_LEVEL" + +# create a new python virtual environment +uv venv --python=3.13 +source .venv/bin/activate + +# compile and install the package +uv pip install -v -e ".[dev]" + +# generate typing stubs for C++ file +stubgen -p demo.cppcore -o src + +# test +pytest ``` + +### Windows: + +On Windows, install Visual Studio (2019 or 2022) and the C++ build packages (this is not related to Visual Studio Code). All commands below must be run within a Developer PowerShell for VS environment in order to use the appropriate compiler toolchain. + +We'll build static dependencies ([x64-windows-static-md](https://learn.microsoft.com/en-us/vcpkg/users/platforms/windows)) for C++ to make them easier to add to our custom Python module. + +```ps1 +# check out a single commit from a git repository +mkdir vcpkg +cd vcpkg +git init +git remote add origin git@github.com:microsoft/vcpkg.git +git fetch --depth 1 origin c9c17dcea3016bc241df0422e82b8aea212dcb93 +git checkout FETCH_HEAD +cd .. + +# build static dependencies +$env:VCPKG_HOST_TRIPLET="x64-windows-static-md" +.\vcpkg\bootstrap-vcpkg.bat +.\vcpkg\vcpkg install --host-triplet="$env:VCPKG_HOST_TRIPLET" + +# install pybind11 at the root so for our build phase +uv pip install pybind11==3.0.0 --target=./pybind11 + +# set environment variables for building python extension +$env:CMAKE_PREFIX_PATH=Resolve-Path "./pybind11/pybind11/share/cmake" | Select-Object -ExpandProperty Path +$env:CMAKE_BUILD_PARALLEL_LEVEL=[Environment]::ProcessorCount + +echo "$env:VCPKG_HOST_TRIPLET" +echo "$env:CMAKE_PREFIX_PATH" +echo "$env:CMAKE_BUILD_PARALLEL_LEVEL" + +# create a new python virtual environment +uv venv --python=3.13 +.venv/Scripts/activate + +# compile and install the package +uv pip install -v -e ".[dev]" + +# generate typing stubs for C++ file +stubgen -p demo.cppcore -o src + +# test +pytest +``` + +## Distributable Python Wheels + +See the GitHub Actions. diff --git a/pyproject.toml b/pyproject.toml index 5249c89..85e3a0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,12 @@ authors = [ requires-python = ">=3.13" dependencies = [ "numpy", - "pytest", +] + +[project.optional-dependencies] +dev = [ + "pytest~=8.4.1", + "mypy~=1.17.1", ] [tool.setuptools.dynamic] @@ -20,18 +25,11 @@ include = ["demo*"] [build-system] requires = [ - "setuptools", - "pybind11" + "setuptools", + "pybind11~=3.0.0" ] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] testpaths = "tests" python_files = ["test_*.py"] - -[tool.cibuildwheel] -test-command = "pytest {project}/tests" -test-extras = ["test"] -test-skip = ["*universal2:arm64"] -# Setuptools bug causes collision between pypy and cpython artifacts -before-build = "rm -rf {project}/build" diff --git a/src/demo/cppcore.pyi b/src/demo/cppcore.pyi new file mode 100644 index 0000000..0dfa17d --- /dev/null +++ b/src/demo/cppcore.pyi @@ -0,0 +1,16 @@ +import collections.abc +import numpy +import numpy.typing +import typing + +def add(arg0: typing.SupportsInt, arg1: typing.SupportsInt) -> int: ... +def eigen_matmul( + arg0: typing.Annotated[numpy.typing.ArrayLike, numpy.float64], + arg1: typing.Annotated[numpy.typing.ArrayLike, numpy.float64], +) -> numpy.typing.NDArray[numpy.float64]: ... +def gsl_bessel(arg0: typing.SupportsFloat) -> float: ... +def nlopt_optimize( + lower_bounds: collections.abc.Sequence[typing.SupportsFloat], + upper_bounds: collections.abc.Sequence[typing.SupportsFloat], +) -> list[float]: ... +def subtract(arg0: typing.SupportsInt, arg1: typing.SupportsInt) -> int: ...