Add support for building confluent-kafka-python on Windows ARM64

Author: MugundanMCW

requirements/requirements-examples.txt

@@ -33,4 +33,4 @@ hvac
 jsonata-python
 # Dependency of cel-python.  Use version 6 due to https://github.com/yaml/pyyaml/issues/601
 pyyaml>=6.0.0
-tink
+tink; sys_platform != "win32" or platform_machine != "ARM64"

requirements/requirements-rules.txt

@@ -12,4 +12,4 @@ hvac
 jsonata-python
 # Dependency of cel-python.  Use version 6 due to https://github.com/yaml/pyyaml/issues/601
 pyyaml>=6.0.0
-tink
+tink; sys_platform != "win32" or platform_machine != "ARM64"

setup.py

@@ -2,6 +2,7 @@
 
 import os
 import platform
+import shutil
 
 from setuptools import Extension, setup
 
@@ -16,9 +17,37 @@
 else:
     librdkafka_libname = 'rdkafka'
 
+# Use LIBRDKAFKA_DIR environment variable on Windows ARM64
+include_dirs = []
+library_dirs = []
+
+librdkafka_dir = os.environ.get('LIBRDKAFKA_DIR')
+if platform.system() == 'Windows' and librdkafka_dir:
+    include_dirs = [os.path.join(librdkafka_dir, 'include', 'librdkafka'),
+                    os.path.join(librdkafka_dir, 'include')]
+    library_dirs = [os.path.join(librdkafka_dir, 'lib')]
+    
+    # Auto-detect library name[CMake uses 'rdkafka', NuGet uses 'librdkafka']
+    lib_dir = library_dirs[0]
+    if os.path.exists(os.path.join(lib_dir, 'rdkafka.lib')):
+        librdkafka_libname = 'rdkafka'
+    
+    # Copy DLLs to package directory for bundling
+    dll_dir = os.path.join(librdkafka_dir, 'bin')
+    if os.path.exists(dll_dir):
+        for dll_file in os.listdir(dll_dir):
+            if dll_file.endswith('.dll'):
+                src = os.path.join(dll_dir, dll_file)
+                dst = os.path.join(mod_dir, dll_file)
+                if not os.path.exists(dst) or os.path.getmtime(src) > os.path.getmtime(dst):
+                    print(f"Copying {dll_file} to package directory")
+                    shutil.copy2(src, dst)
+
 module = Extension(
     'confluent_kafka.cimpl',
     libraries=[librdkafka_libname],
+    include_dirs=include_dirs,
+    library_dirs=library_dirs,
     sources=[
         os.path.join(ext_dir, 'confluent_kafka.c'),
         os.path.join(ext_dir, 'Producer.c'),

tools/windows-build.bat

@@ -8,43 +8,65 @@ set PATH=%PATH%;c:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\bin
 set
 
 rem Download and install librdkafka from NuGet.
-call tools\windows-install-librdkafka.bat %LIBRDKAFKA_NUGET_VERSION% dest || exit /b 1
+rem Check if LIBRDKAFKA_DIR is already set for Windows ARM64
+if not defined LIBRDKAFKA_DIR (
+    rem Download and install librdkafka from NuGet.
+    call tools\windows-install-librdkafka.bat %LIBRDKAFKA_NUGET_VERSION% dest || exit /b 1
+) else (
+    call tools\windows-install-librdkafka-arm64.bat %LIBRDKAFKA_VERSION% %LIBRDKAFKA_DIR% || exit /b 1
+)
 
 pip install -r requirements\requirements-tests-install.txt || exit /b 1
 pip install cibuildwheel==3.2.1 || exit /b 1
 
 rem Build wheels (without tests)
-cibuildwheel --platform windows --output-dir wheelhouse || exit /b 1
-
+if defined LIBRDKAFKA_DIR (
+    cibuildwheel --platform windows --archs ARM64 --output-dir wheelhouse || exit /b 1
+) else (
+    cibuildwheel --platform windows --output-dir wheelhouse || exit /b 1
+)
 dir wheelhouse
 
 rem cibuildwheel installs the generated packages, but they're not ready yet,
 rem so remove them.
 rem FIXME: this only covers python27 (default)
 pip uninstall -y confluent_kafka[dev]
 
-
-rem Copy the librdkafka DLLs to a path structure that is identical to cimpl.pyd's location
-md stage\x86\confluent_kafka
-copy dest\librdkafka.redist.%LIBRDKAFKA_VERSION%\runtimes\win-x86\native\*.dll stage\x86\confluent_kafka\ || exit /b 1
-
-md stage\x64\confluent_kafka
-copy dest\librdkafka.redist.%LIBRDKAFKA_VERSION%\runtimes\win-x64\native\*.dll stage\x64\confluent_kafka\ || exit /b 1
-
-rem For each wheel, add the corresponding x86 or x64 dlls to the wheel zip file
-cd stage\x86
-for %%W in (..\..\wheelhouse\*win32.whl) do (
-    7z a -r %%~W confluent_kafka\*.dll || exit /b 1
-    unzip -l %%~W
+rem Only copy x86/x64 DLLs if building from NuGet
+for %%A in (x86 x64 arm64) do (
+    md stage\%%A\confluent_kafka 2>nul
+)
+if not defined LIBRDKAFKA_DIR (
+    copy dest\librdkafka.redist.%LIBRDKAFKA_VERSION%\runtimes\win-x86\native\*.dll stage\x86\confluent_kafka\ || exit /b 1
+    copy dest\librdkafka.redist.%LIBRDKAFKA_VERSION%\runtimes\win-x64\native\*.dll stage\x64\confluent_kafka\ || exit /b 1
 )
 
-cd ..\x64
-for %%W in (..\..\wheelhouse\*amd64.whl) do (
-    7z a -r %%~W confluent_kafka\*.dll || exit /b 1
-    unzip -l %%~W
+rem Handle Windows ARM64 if LIBRDKAFKA_DIR is defined
+if defined LIBRDKAFKA_DIR (
+    copy %LIBRDKAFKA_DIR%\bin\*.dll stage\arm64\confluent_kafka\ || exit /b 1
 )
 
-cd ..\..
+rem Only process x86/x64 wheels if not ARM64 build
+if not defined LIBRDKAFKA_DIR (
+    cd stage\x86
+    for %%W in (..\..\wheelhouse\*win32.whl) do (
+        7z a -r %%~W confluent_kafka\*.dll || exit /b 1
+        unzip -l %%~W
+    )
+    cd ..\x64
+    for %%W in (..\..\wheelhouse\*amd64.whl) do (
+        7z a -r %%~W confluent_kafka\*.dll || exit /b 1
+        unzip -l %%~W
+    )
+    cd ..\..
+) else (
+    cd stage\arm64
+    for %%W in (..\..\wheelhouse\*arm64.whl) do (
+        7z a -r %%~W confluent_kafka\*.dll || exit /b 1
+        unzip -l %%~W
+    )
+    cd ..\..
+)
 
 rem Basic testing
 for %%W in (wheelhouse\confluent_kafka-*cp%PYTHON_SHORTVER%*win*%PYTHON_ARCH%.whl) do (

tools/windows-install-librdkafka-arm64.bat

@@ -0,0 +1,47 @@
+@echo off
+setlocal enabledelayedexpansion
+
+set VERSION=v2.13.0
+set dest=C:\librdkafka-ARM64
+
+if not "%~1"=="" set VERSION=%~1
+if not "%~2"=="" set dest=%~2
+
+if exist "%dest%" (
+    set LIBRDKAFKA_DIR=%dest%
+    exit /b 0
+)
+
+set TEMP_DIR=%TEMP%\librdkafka-build-%RANDOM%
+mkdir "%TEMP_DIR%" 2>nul
+
+cd /d "%TEMP_DIR%"
+
+git clone --depth 1 --branch %VERSION% https://github.com/confluentinc/librdkafka.git
+if errorlevel 1 goto :error
+
+cd librdkafka
+mkdir build-arm64
+cd build-arm64
+
+cmake .. -G "Visual Studio 17 2022" -A ARM64 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=%dest% -DRDKAFKA_BUILD_STATIC=OFF -DRDKAFKA_BUILD_EXAMPLES=OFF -DRDKAFKA_BUILD_TESTS=OFF -DWITH_SSL=OFF -DWITH_ZLIB=OFF -DWITH_ZSTD=OFF -DWITH_SASL=OFF -DENABLE_LZ4_EXT=OFF
+if errorlevel 1 goto :error
+
+cmake --build . --config Release --parallel
+if errorlevel 1 goto :error
+
+cmake --install . --config Release
+if errorlevel 1 goto :error
+
+set LIBRDKAFKA_DIR=%dest%
+goto :cleanup
+
+:error
+cd /d %TEMP%
+if exist "%TEMP_DIR%" rd /s /q "%TEMP_DIR%" 2>nul
+exit /b 1
+
+:cleanup
+cd /d %TEMP%
+if exist "%TEMP_DIR%" rd /s /q "%TEMP_DIR%" 2>nul
+exit /b 0