From 9550f91b0b056fa0f459ac0dc503db5624ea53cd Mon Sep 17 00:00:00 2001 From: Mugunthan Selvanayagam Date: Tue, 17 Feb 2026 10:52:53 +0530 Subject: [PATCH] Add support for building confluent-kafka-python on Windows ARM64 --- requirements/requirements-examples.txt | 2 +- requirements/requirements-rules.txt | 2 +- setup.py | 29 ++++++++++ tools/windows-build.bat | 64 +++++++++++++++------- tools/windows-install-librdkafka-arm64.bat | 47 ++++++++++++++++ 5 files changed, 121 insertions(+), 23 deletions(-) create mode 100644 tools/windows-install-librdkafka-arm64.bat diff --git a/requirements/requirements-examples.txt b/requirements/requirements-examples.txt index 657aef915..84c33d7ec 100644 --- a/requirements/requirements-examples.txt +++ b/requirements/requirements-examples.txt @@ -33,4 +33,4 @@ hvac jsonata-python # Dependency of cel-python. Use version 6 due to https://github.com/yaml/pyyaml/issues/601 pyyaml>=6.0.0 -tink +tink; sys_platform != "win32" or platform_machine != "ARM64" diff --git a/requirements/requirements-rules.txt b/requirements/requirements-rules.txt index 51ba6ac1a..237b798db 100644 --- a/requirements/requirements-rules.txt +++ b/requirements/requirements-rules.txt @@ -12,4 +12,4 @@ hvac jsonata-python # Dependency of cel-python. Use version 6 due to https://github.com/yaml/pyyaml/issues/601 pyyaml>=6.0.0 -tink +tink; sys_platform != "win32" or platform_machine != "ARM64" diff --git a/setup.py b/setup.py index db46e1036..9fc230886 100755 --- a/setup.py +++ b/setup.py @@ -2,6 +2,7 @@ import os import platform +import shutil from setuptools import Extension, setup @@ -16,9 +17,37 @@ else: librdkafka_libname = 'rdkafka' +# Use LIBRDKAFKA_DIR environment variable on Windows ARM64 +include_dirs = [] +library_dirs = [] + +librdkafka_dir = os.environ.get('LIBRDKAFKA_DIR') +if platform.system() == 'Windows' and librdkafka_dir: + include_dirs = [os.path.join(librdkafka_dir, 'include', 'librdkafka'), + os.path.join(librdkafka_dir, 'include')] + library_dirs = [os.path.join(librdkafka_dir, 'lib')] + + # Auto-detect library name[CMake uses 'rdkafka', NuGet uses 'librdkafka'] + lib_dir = library_dirs[0] + if os.path.exists(os.path.join(lib_dir, 'rdkafka.lib')): + librdkafka_libname = 'rdkafka' + + # Copy DLLs to package directory for bundling + dll_dir = os.path.join(librdkafka_dir, 'bin') + if os.path.exists(dll_dir): + for dll_file in os.listdir(dll_dir): + if dll_file.endswith('.dll'): + src = os.path.join(dll_dir, dll_file) + dst = os.path.join(mod_dir, dll_file) + if not os.path.exists(dst) or os.path.getmtime(src) > os.path.getmtime(dst): + print(f"Copying {dll_file} to package directory") + shutil.copy2(src, dst) + module = Extension( 'confluent_kafka.cimpl', libraries=[librdkafka_libname], + include_dirs=include_dirs, + library_dirs=library_dirs, sources=[ os.path.join(ext_dir, 'confluent_kafka.c'), os.path.join(ext_dir, 'Producer.c'), diff --git a/tools/windows-build.bat b/tools/windows-build.bat index eb5075634..7c62baf0f 100644 --- a/tools/windows-build.bat +++ b/tools/windows-build.bat @@ -8,14 +8,23 @@ set PATH=%PATH%;c:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\bin set rem Download and install librdkafka from NuGet. -call tools\windows-install-librdkafka.bat %LIBRDKAFKA_NUGET_VERSION% dest || exit /b 1 +rem Check if LIBRDKAFKA_DIR is already set for Windows ARM64 +if not defined LIBRDKAFKA_DIR ( + rem Download and install librdkafka from NuGet. + call tools\windows-install-librdkafka.bat %LIBRDKAFKA_NUGET_VERSION% dest || exit /b 1 +) else ( + call tools\windows-install-librdkafka-arm64.bat %LIBRDKAFKA_VERSION% %LIBRDKAFKA_DIR% || exit /b 1 +) pip install -r requirements\requirements-tests-install.txt || exit /b 1 pip install cibuildwheel==3.2.1 || exit /b 1 rem Build wheels (without tests) -cibuildwheel --platform windows --output-dir wheelhouse || exit /b 1 - +if defined LIBRDKAFKA_DIR ( + cibuildwheel --platform windows --archs ARM64 --output-dir wheelhouse || exit /b 1 +) else ( + cibuildwheel --platform windows --output-dir wheelhouse || exit /b 1 +) dir wheelhouse rem cibuildwheel installs the generated packages, but they're not ready yet, @@ -23,28 +32,41 @@ rem so remove them. rem FIXME: this only covers python27 (default) pip uninstall -y confluent_kafka[dev] - -rem Copy the librdkafka DLLs to a path structure that is identical to cimpl.pyd's location -md stage\x86\confluent_kafka -copy dest\librdkafka.redist.%LIBRDKAFKA_VERSION%\runtimes\win-x86\native\*.dll stage\x86\confluent_kafka\ || exit /b 1 - -md stage\x64\confluent_kafka -copy dest\librdkafka.redist.%LIBRDKAFKA_VERSION%\runtimes\win-x64\native\*.dll stage\x64\confluent_kafka\ || exit /b 1 - -rem For each wheel, add the corresponding x86 or x64 dlls to the wheel zip file -cd stage\x86 -for %%W in (..\..\wheelhouse\*win32.whl) do ( - 7z a -r %%~W confluent_kafka\*.dll || exit /b 1 - unzip -l %%~W +rem Only copy x86/x64 DLLs if building from NuGet +for %%A in (x86 x64 arm64) do ( + md stage\%%A\confluent_kafka 2>nul +) +if not defined LIBRDKAFKA_DIR ( + copy dest\librdkafka.redist.%LIBRDKAFKA_VERSION%\runtimes\win-x86\native\*.dll stage\x86\confluent_kafka\ || exit /b 1 + copy dest\librdkafka.redist.%LIBRDKAFKA_VERSION%\runtimes\win-x64\native\*.dll stage\x64\confluent_kafka\ || exit /b 1 ) -cd ..\x64 -for %%W in (..\..\wheelhouse\*amd64.whl) do ( - 7z a -r %%~W confluent_kafka\*.dll || exit /b 1 - unzip -l %%~W +rem Handle Windows ARM64 if LIBRDKAFKA_DIR is defined +if defined LIBRDKAFKA_DIR ( + copy %LIBRDKAFKA_DIR%\bin\*.dll stage\arm64\confluent_kafka\ || exit /b 1 ) -cd ..\.. +rem Only process x86/x64 wheels if not ARM64 build +if not defined LIBRDKAFKA_DIR ( + cd stage\x86 + for %%W in (..\..\wheelhouse\*win32.whl) do ( + 7z a -r %%~W confluent_kafka\*.dll || exit /b 1 + unzip -l %%~W + ) + cd ..\x64 + for %%W in (..\..\wheelhouse\*amd64.whl) do ( + 7z a -r %%~W confluent_kafka\*.dll || exit /b 1 + unzip -l %%~W + ) + cd ..\.. +) else ( + cd stage\arm64 + for %%W in (..\..\wheelhouse\*arm64.whl) do ( + 7z a -r %%~W confluent_kafka\*.dll || exit /b 1 + unzip -l %%~W + ) + cd ..\.. +) rem Basic testing for %%W in (wheelhouse\confluent_kafka-*cp%PYTHON_SHORTVER%*win*%PYTHON_ARCH%.whl) do ( diff --git a/tools/windows-install-librdkafka-arm64.bat b/tools/windows-install-librdkafka-arm64.bat new file mode 100644 index 000000000..dd9d7ede7 --- /dev/null +++ b/tools/windows-install-librdkafka-arm64.bat @@ -0,0 +1,47 @@ +@echo off +setlocal enabledelayedexpansion + +set VERSION=v2.13.0 +set dest=C:\librdkafka-ARM64 + +if not "%~1"=="" set VERSION=%~1 +if not "%~2"=="" set dest=%~2 + +if exist "%dest%" ( + set LIBRDKAFKA_DIR=%dest% + exit /b 0 +) + +set TEMP_DIR=%TEMP%\librdkafka-build-%RANDOM% +mkdir "%TEMP_DIR%" 2>nul + +cd /d "%TEMP_DIR%" + +git clone --depth 1 --branch %VERSION% https://github.com/confluentinc/librdkafka.git +if errorlevel 1 goto :error + +cd librdkafka +mkdir build-arm64 +cd build-arm64 + +cmake .. -G "Visual Studio 17 2022" -A ARM64 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=%dest% -DRDKAFKA_BUILD_STATIC=OFF -DRDKAFKA_BUILD_EXAMPLES=OFF -DRDKAFKA_BUILD_TESTS=OFF -DWITH_SSL=OFF -DWITH_ZLIB=OFF -DWITH_ZSTD=OFF -DWITH_SASL=OFF -DENABLE_LZ4_EXT=OFF +if errorlevel 1 goto :error + +cmake --build . --config Release --parallel +if errorlevel 1 goto :error + +cmake --install . --config Release +if errorlevel 1 goto :error + +set LIBRDKAFKA_DIR=%dest% +goto :cleanup + +:error +cd /d %TEMP% +if exist "%TEMP_DIR%" rd /s /q "%TEMP_DIR%" 2>nul +exit /b 1 + +:cleanup +cd /d %TEMP% +if exist "%TEMP_DIR%" rd /s /q "%TEMP_DIR%" 2>nul +exit /b 0