From 6a4a99dedb15ea8fa7a16df7676d27c907b71fda Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 00:57:03 +0530 Subject: [PATCH 01/24] TEST: PR-Validation --- eng/pipelines/pr-validation-pipeline.yml | 693 ++++++++++++++++++++++- 1 file changed, 663 insertions(+), 30 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 14e07885..5185060c 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -6,44 +6,677 @@ trigger: include: - main +pr: + branches: + include: + - main + jobs: -- job: CodeQLAnalysis - displayName: 'CodeQL Security Analysis' +# ===== PYTHON 3.14 TESTING JOBS ===== + +- job: BuildWindowsWheels_Python314 pool: - vmImage: 'ubuntu-latest' - + vmImage: 'windows-latest' + displayName: 'Build Windows Python 3.14 -' + strategy: + matrix: + py314_x64: + pythonVersion: '3.14' + shortPyVer: '314' + architecture: 'x64' + targetArch: 'x64' + py314_arm64: + pythonVersion: '3.14' + shortPyVer: '314' + architecture: 'x64' + targetArch: 'arm64' + steps: - - script: | - sudo apt-get update - sudo apt-get install -y build-essential cmake curl git python3 python3-pip python3-dev python3-venv unixodbc-dev - displayName: 'Install build dependencies for CodeQL' + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(pythonVersion)' + architecture: '$(architecture)' + addToPath: true + githubToken: $(GITHUB_TOKEN) + displayName: 'Use Python $(pythonVersion) ($(architecture))' + + - script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install cmake pybind11 + displayName: 'Install dependencies' + + - powershell: | + sqllocaldb create MSSQLLocalDB + sqllocaldb start MSSQLLocalDB + displayName: 'Start LocalDB instance' + + - powershell: | + sqlcmd -S "(localdb)\MSSQLLocalDB" -Q "CREATE DATABASE TestDB" + sqlcmd -S "(localdb)\MSSQLLocalDB" -Q "CREATE LOGIN testuser WITH PASSWORD = '$(DB_PASSWORD)'" + sqlcmd -S "(localdb)\MSSQLLocalDB" -d TestDB -Q "CREATE USER testuser FOR LOGIN testuser" + sqlcmd -S "(localdb)\MSSQLLocalDB" -d TestDB -Q "ALTER ROLE db_owner ADD MEMBER testuser" + displayName: 'Setup database and user' + env: + DB_PASSWORD: $(DB_PASSWORD) + + - task: DownloadPipelineArtifact@2 + condition: eq(variables['targetArch'], 'arm64') + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: 2162 + buildVersionToDownload: 'latest' + artifactName: 'mssql-python-arm64-libs' + targetPath: '$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64' + displayName: 'Download ARM64 Python libs' + + - script: | + echo "Python Version: $(pythonVersion)" + echo "Short Tag: $(shortPyVer)" + echo "Architecture: Host=$(architecture), Target=$(targetArch)" + + cd "$(Build.SourcesDirectory)\mssql_python\pybind" + + if "$(targetArch)"=="arm64" ( + echo Using arm64-specific Python library... + set CUSTOM_PYTHON_LIB_DIR=$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64 + ) - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.13' - addToPath: true - displayName: 'Use Python 3.13 for CodeQL' + call build.bat $(targetArch) + call keep_single_arch.bat $(targetArch) - - script: | - python -m pip install --upgrade pip - pip install -r requirements.txt - displayName: 'Install Python dependencies for CodeQL' + cd ..\.. + displayName: 'Build PYD for $(targetArch)' + continueOnError: false - - task: CodeQL3000Init@0 - inputs: - Enabled: true - displayName: 'Initialize CodeQL' + - powershell: | + Write-Host "Running pytests to validate bindings" + if ("$(targetArch)" -eq "arm64") { + Write-Host "Skipping pytests on Windows ARM64" + } else { + python -m pytest -v + } + displayName: 'Run pytests' + env: + DB_CONNECTION_STRING: 'Server=(localdb)\MSSQLLocalDB;Database=TestDB;Uid=testuser;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)\mssql_python\pybind\build\$(targetArch)\py$(shortPyVer)\Release' + Contents: 'ddbc_bindings.cp$(shortPyVer)-*.pyd' + TargetFolder: '$(Build.ArtifactStagingDirectory)\ddbc-bindings\windows' + displayName: 'Place PYD file into artifacts directory' + + - script: | + python -m pip install --upgrade pip + pip install wheel setuptools + set ARCHITECTURE=$(targetArch) + python setup.py bdist_wheel + displayName: 'Build wheel package for Python $(pythonVersion) ($(targetArch))' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)\dist' + Contents: '*.whl' + TargetFolder: '$(Build.ArtifactStagingDirectory)\dist' + displayName: 'Collect wheel package' + + - task: PublishBuildArtifacts@1 + condition: succeededOrFailed() + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\ddbc-bindings' + ArtifactName: 'mssql-python-ddbc-bindings-py314' + publishLocation: 'Container' + displayName: 'Publish PYDs as artifacts' + + - task: PublishBuildArtifacts@1 + condition: eq(variables['targetArch'], 'arm64') + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64' + ArtifactName: 'mssql-python-arm64-libs-py314' + publishLocation: 'Container' + displayName: 'Publish arm64 libs as artifacts' + + - task: PublishBuildArtifacts@1 + condition: succeededOrFailed() + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\dist' + ArtifactName: 'mssql-python-wheels-dist-py314' + publishLocation: 'Container' + displayName: 'Publish wheels as artifacts' + +- job: BuildMacOSWheels_Python314 + pool: + vmImage: 'macos-latest' + displayName: 'Build macOS Python 3.14 - Universal2' - # Build the C++ extension for CodeQL analysis - - script: | - cd mssql_python/pybind - chmod +x build.sh - ./build.sh - displayName: 'Build C++ extension for CodeQL analysis' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.14' + addToPath: true + githubToken: $(GITHUB_TOKEN) + displayName: 'Use Python 3.14 (Universal2)' + + - script: | + brew update + brew uninstall cmake --ignore-dependencies || echo "CMake not installed or already removed" + brew install cmake + displayName: 'Install CMake' + + - script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install cmake pybind11 + displayName: 'Install dependencies' + + - script: | + echo "Python Version: 3.14" + echo "Building Universal2 Binary" + cd "$(Build.SourcesDirectory)/mssql_python/pybind" + ./build.sh + displayName: 'Build .so file' + continueOnError: false + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)/mssql_python' + Contents: '*.so' + TargetFolder: '$(Build.ArtifactStagingDirectory)/ddbc-bindings/macOS' + displayName: 'Place .so file into artifacts directory' + + - script: | + brew update + brew install docker colima + + colima start --cpu 3 --memory 10 --disk 30 --vm-type=vz || \ + colima start --cpu 3 --memory 10 --disk 30 --vm-type=qemu + + sleep 30 + docker context use colima >/dev/null || true + docker version + docker ps + displayName: 'Install and start Colima-based Docker' + timeoutInMinutes: 15 + + - script: | + docker pull mcr.microsoft.com/mssql/server:2022-latest + docker run \ + --name sqlserver \ + -e ACCEPT_EULA=Y \ + -e MSSQL_SA_PASSWORD="${DB_PASSWORD}" \ + -p 1433:1433 \ + -d mcr.microsoft.com/mssql/server:2022-latest + + for i in {1..30}; do + docker exec sqlserver \ + /opt/mssql-tools18/bin/sqlcmd \ + -S localhost \ + -U SA \ + -P "$DB_PASSWORD" \ + -C -Q "SELECT 1" && break + sleep 2 + done + displayName: 'Pull & start SQL Server (Docker)' + env: + DB_PASSWORD: $(DB_PASSWORD) + + - script: | + python -m pytest -v + displayName: 'Run Pytest to validate bindings' + env: + DB_CONNECTION_STRING: 'Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' + + - script: | + python -m pip install --upgrade pip + pip install wheel setuptools + python setup.py bdist_wheel + displayName: 'Build 3.14 universal2 whl' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)/dist' + Contents: '*.whl' + TargetFolder: '$(Build.ArtifactStagingDirectory)/dist' + displayName: 'Collect wheel package' + + - task: PublishBuildArtifacts@1 + condition: succeededOrFailed() + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/ddbc-bindings' + ArtifactName: 'mssql-python-ddbc-bindings-py314' + publishLocation: 'Container' + displayName: 'Publish .so files as artifacts' + + - task: PublishBuildArtifacts@1 + condition: succeededOrFailed() + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/dist' + ArtifactName: 'mssql-python-wheels-dist-py314' + publishLocation: 'Container' + displayName: 'Publish wheels as artifacts' + +- job: BuildLinuxWheels_Python314 + displayName: 'Build Linux Python 3.14 -' + pool: { vmImage: 'ubuntu-latest' } + timeoutInMinutes: 120 - - task: CodeQL3000Finalize@0 - condition: always() - displayName: 'Finalize CodeQL' + strategy: + matrix: + manylinux_x86_64: + LINUX_TAG: 'manylinux' + ARCH: 'x86_64' + DOCKER_PLATFORM: 'linux/amd64' + IMAGE: 'quay.io/pypa/manylinux_2_28_x86_64' + manylinux_aarch64: + LINUX_TAG: 'manylinux' + ARCH: 'aarch64' + DOCKER_PLATFORM: 'linux/arm64' + IMAGE: 'quay.io/pypa/manylinux_2_28_aarch64' + musllinux_x86_64: + LINUX_TAG: 'musllinux' + ARCH: 'x86_64' + DOCKER_PLATFORM: 'linux/amd64' + IMAGE: 'quay.io/pypa/musllinux_1_2_x86_64' + musllinux_aarch64: + LINUX_TAG: 'musllinux' + ARCH: 'aarch64' + DOCKER_PLATFORM: 'linux/arm64' + IMAGE: 'quay.io/pypa/musllinux_1_2_aarch64' + + steps: + - checkout: self + fetchDepth: 0 + + - script: | + sudo docker run --rm --privileged tonistiigi/binfmt --install all + displayName: 'Enable QEMU (for aarch64)' + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/dist $(Build.ArtifactStagingDirectory)/ddbc-bindings + mkdir -p $(Build.ArtifactStagingDirectory)/dist + mkdir -p $(Build.ArtifactStagingDirectory)/ddbc-bindings/$(LINUX_TAG)-$(ARCH) + displayName: 'Prepare artifact directories' + + - script: | + docker run -d --name build-$(LINUX_TAG)-$(ARCH) \ + --platform $(DOCKER_PLATFORM) \ + -v $(Build.SourcesDirectory):/workspace \ + -w /workspace \ + $(IMAGE) \ + tail -f /dev/null + displayName: 'Start $(LINUX_TAG) $(ARCH) container' + + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "manylinux" ]]; then + docker exec build-$(LINUX_TAG)-$(ARCH) bash -lc ' + set -euxo pipefail + if command -v dnf >/dev/null 2>&1; then + dnf -y update || true + dnf -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache || true + elif command -v yum >/dev/null 2>&1; then + yum -y update || true + yum -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache || true + else + echo "No dnf/yum found in manylinux image" >&2 + fi + + echo "---- tool versions ----" + gcc --version || true + cmake --version || true + ' + else + docker exec build-$(LINUX_TAG)-$(ARCH) sh -lc ' + set -euxo pipefail + apk update || true + apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache || true + + echo "---- tool versions ----" + gcc --version || true + cmake --version || true + ' + fi + displayName: 'Install system build dependencies' + + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "manylinux" ]]; then SHELL_EXE=bash; else SHELL_EXE=sh; fi + + docker exec build-$(LINUX_TAG)-$(ARCH) $SHELL_EXE -lc 'mkdir -p /workspace/dist' + + # Build only for Python 3.14 (cp314) + PYBIN=cp314 + echo "=== Building for $PYBIN on $(LINUX_TAG)/$(ARCH) ===" + if [[ "$(LINUX_TAG)" == "manylinux" ]]; then + docker exec build-$(LINUX_TAG)-$(ARCH) bash -lc " + set -euxo pipefail; + PY=/opt/python/${PYBIN}-${PYBIN}/bin/python; + test -x \$PY || { echo 'Python \$PY missing'; exit 0; } + ln -sf \$PY /usr/local/bin/python; + python -m pip install -U pip setuptools wheel pybind11; + echo 'python:' \$(python -V); which python; + cd /workspace/mssql_python/pybind; + bash build.sh; + + cd /workspace; + python setup.py bdist_wheel; + " + else + docker exec build-$(LINUX_TAG)-$(ARCH) sh -lc " + set -euxo pipefail; + PY=/opt/python/${PYBIN}-${PYBIN}/bin/python; + test -x \$PY || { echo 'Python \$PY missing'; exit 0; } + ln -sf \$PY /usr/local/bin/python; + python -m pip install -U pip setuptools wheel pybind11; + echo 'python:' \$(python -V); which python; + cd /workspace/mssql_python/pybind; + bash build.sh; + + cd /workspace; + python setup.py bdist_wheel; + " + fi + displayName: 'Run build.sh and build wheels for Python 3.14' + + - script: | + set -euxo pipefail + docker cp build-$(LINUX_TAG)-$(ARCH):/workspace/dist/. "$(Build.ArtifactStagingDirectory)/dist/" || echo "No wheels to copy" + + mkdir -p "$(Build.ArtifactStagingDirectory)/ddbc-bindings/$(LINUX_TAG)-$(ARCH)" + + docker exec build-$(LINUX_TAG)-$(ARCH) $([[ "$(LINUX_TAG)" == "manylinux" ]] && echo bash -lc || echo sh -lc) ' + set -euxo pipefail; + echo "Listing package dirs for sanity:"; + ls -la /workspace/mssql_python || true; + ls -la /workspace/mssql_python/pybind || true; + + OUT="/tmp/ddbc-out-$(LINUX_TAG)-$(ARCH)"; + rm -rf "$OUT"; mkdir -p "$OUT"; + + find /workspace/mssql_python -maxdepth 1 -type f -name "*.so" -exec cp -v {} "$OUT"/ \; || true + + echo "Top-level .so collected in $OUT:"; + ls -la "$OUT" || true + ' + + docker cp "build-$(LINUX_TAG)-$(ARCH):/tmp/ddbc-out-$(LINUX_TAG)-$(ARCH)/." \ + "$(Build.ArtifactStagingDirectory)/ddbc-bindings/$(LINUX_TAG)-$(ARCH)/" \ + || echo "No top-level .so files to copy" + + find "$(Build.ArtifactStagingDirectory)/ddbc-bindings/$(LINUX_TAG)-$(ARCH)" -maxdepth 1 -type f ! -name "*.so" -delete || true + displayName: 'Copy wheels and .so back to host' + + - script: | + docker stop build-$(LINUX_TAG)-$(ARCH) || true + docker rm build-$(LINUX_TAG)-$(ARCH) || true + displayName: 'Clean up container' + condition: always() + + - task: PublishBuildArtifacts@1 + condition: succeededOrFailed() + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/dist' + ArtifactName: 'mssql-python-wheels-dist-py314' + publishLocation: 'Container' + displayName: 'Publish wheels as artifacts' + + - task: PublishBuildArtifacts@1 + condition: succeededOrFailed() + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/ddbc-bindings' + ArtifactName: 'mssql-python-ddbc-bindings-py314' + publishLocation: 'Container' + displayName: 'Publish .so files as artifacts' + +- job: TestWheelsOnLinux_Python314 + displayName: 'Pytests on Linux Python 3.14 -' + dependsOn: BuildLinuxWheels_Python314 + condition: succeeded('BuildLinuxWheels_Python314') + pool: { vmImage: 'ubuntu-latest' } + timeoutInMinutes: 60 + + strategy: + matrix: + debian12: + BASE_IMAGE: 'debian:12-slim' + ARCH: 'x86_64' + DOCKER_PLATFORM: 'linux/amd64' + rhel_ubi9: + BASE_IMAGE: 'registry.access.redhat.com/ubi9/ubi:latest' + ARCH: 'x86_64' + DOCKER_PLATFORM: 'linux/amd64' + alpine320: + BASE_IMAGE: 'alpine:3.20' + ARCH: 'x86_64' + DOCKER_PLATFORM: 'linux/amd64' + debian12_arm64: + BASE_IMAGE: 'debian:12-slim' + ARCH: 'arm64' + DOCKER_PLATFORM: 'linux/arm64' + rhel_ubi9_arm64: + BASE_IMAGE: 'registry.access.redhat.com/ubi9/ubi:latest' + ARCH: 'arm64' + DOCKER_PLATFORM: 'linux/arm64' + alpine320_arm64: + BASE_IMAGE: 'alpine:3.20' + ARCH: 'arm64' + DOCKER_PLATFORM: 'linux/arm64' + + steps: + - checkout: self + + - task: DownloadBuildArtifacts@0 + inputs: + buildType: 'current' + downloadType: 'single' + artifactName: 'mssql-python-wheels-dist-py314' + downloadPath: '$(System.ArtifactsDirectory)' + displayName: 'Download wheel artifacts from current build' + + - script: | + set -euxo pipefail + WHEEL_DIR="$(System.ArtifactsDirectory)/mssql-python-wheels-dist-py314" + if [ ! -d "$WHEEL_DIR" ] || [ -z "$(ls -A $WHEEL_DIR/*.whl 2>/dev/null)" ]; then + echo "ERROR: No wheel files found in $WHEEL_DIR" + echo "Contents of artifacts directory:" + find "$(System.ArtifactsDirectory)" -type f -name "*.whl" || echo "No .whl files found anywhere" + exit 1 + fi + echo "Found wheel files:" + ls -la "$WHEEL_DIR"/*.whl + displayName: 'Verify wheel artifacts exist' + + - script: | + set -euxo pipefail + docker run -d --name sqlserver \ + --network bridge \ + -e ACCEPT_EULA=Y \ + -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ + -p 1433:1433 \ + mcr.microsoft.com/mssql/server:2022-latest + + echo "Waiting for SQL Server to start..." + for i in {1..30}; do + if docker exec sqlserver /opt/mssql-tools18/bin/sqlcmd \ + -S localhost -U SA -P "$(DB_PASSWORD)" -C -Q "SELECT 1" >/dev/null 2>&1; then + echo "SQL Server is ready!" + break + fi + echo "Attempt $i/30: SQL Server not ready yet..." + sleep 3 + done + + docker exec sqlserver /opt/mssql-tools18/bin/sqlcmd \ + -S localhost -U SA -P "$(DB_PASSWORD)" -C \ + -Q "CREATE DATABASE TestDB" + displayName: 'Start SQL Server and create test database' + env: + DB_PASSWORD: $(DB_PASSWORD) + + - script: | + set -euxo pipefail + + if [[ "$(ARCH)" == "arm64" ]] || [[ "$(ARCH)" == "aarch64" ]]; then + sudo docker run --rm --privileged tonistiigi/binfmt --install all + fi + + for i in {1..3}; do + if docker run -d --name test-$(ARCH) \ + --platform $(DOCKER_PLATFORM) \ + --network bridge \ + -v $(System.ArtifactsDirectory):/artifacts:ro \ + $(BASE_IMAGE) \ + tail -f /dev/null; then + echo "Container started successfully on attempt $i" + break + else + echo "Failed to start container on attempt $i, retrying..." + docker rm test-$(ARCH) 2>/dev/null || true + sleep 5 + fi + done + + if ! docker ps | grep -q test-$(ARCH); then + echo "ERROR: Container test-$(ARCH) is not running" + docker logs test-$(ARCH) || true + exit 1 + fi + + if [[ "$(BASE_IMAGE)" == alpine* ]]; then + echo "Setting up Alpine Linux..." + docker exec test-$(ARCH) sh -c " + apk update && apk add --no-cache python3 py3-pip python3-dev unixodbc-dev curl libtool libltdl krb5-libs + python3 -m venv /venv + /venv/bin/pip install pytest + " + PY_CMD="/venv/bin/python" + elif [[ "$(BASE_IMAGE)" == *ubi* ]] || [[ "$(BASE_IMAGE)" == *rocky* ]] || [[ "$(BASE_IMAGE)" == *alma* ]]; then + echo "Setting up RHEL-based system..." + docker exec test-$(ARCH) bash -c " + set -euo pipefail + echo 'Installing Python on UBI/RHEL...' + if command -v dnf >/dev/null; then + dnf clean all + rm -rf /var/cache/dnf + dnf -y makecache + + dnf list --showduplicates python3.11 python3.12 || true + + if dnf -y install python3.12 python3.12-pip unixODBC-devel; then + PY=python3.12 + echo 'Installed Python 3.12' + elif dnf -y install python3.11 python3.11-pip unixODBC-devel; then + PY=python3.11 + echo 'Installed Python 3.11' + else + dnf -y install python3 python3-pip unixODBC-devel + PY=python3 + echo 'Falling back to default Python' + fi + + \$PY -m venv /venv + /venv/bin/python -m pip install -U 'pip>=25' pytest + /venv/bin/python --version + /venv/bin/pip --version + else + echo 'ERROR: dnf not found' + exit 1 + fi + " + PY_CMD="/venv/bin/python" + else + echo "Setting up Debian/Ubuntu..." + docker exec test-$(ARCH) bash -c " + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y python3 python3-pip python3-venv python3-full unixodbc-dev curl + python3 -m venv /venv + /venv/bin/pip install pytest + " + PY_CMD="/venv/bin/python" + fi + + if [[ "$(BASE_IMAGE)" == alpine* ]]; then + SHELL_CMD="sh -c" + WHEEL_PATTERN="*musllinux*$(ARCH)*.whl" + else + SHELL_CMD="bash -c" + WHEEL_PATTERN="*manylinux*$(ARCH)*.whl" + fi + + docker exec test-$(ARCH) $SHELL_CMD " + mkdir -p /test_whl + cd /test_whl + + echo 'Available wheels:' + ls -la /artifacts/mssql-python-wheels-dist-py314/*.whl + echo 'Installing package (letting pip auto-select in isolated environment):' + $PY_CMD -m pip install mssql_python --find-links /artifacts/mssql-python-wheels-dist-py314 --no-index --no-deps + + echo 'Installed package location:' + $PY_CMD -c 'import mssql_python; print(\"Package location:\", mssql_python.__file__)' + + $PY_CMD -c 'import mssql_python; print(\"Package imported successfully\")' + " + + displayName: 'Test wheel installation on $(BASE_IMAGE)' + env: + DB_CONNECTION_STRING: 'Server=localhost;Database=TestDB;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' + + - script: | + set -euxo pipefail + + echo "Copying source code to container for pytest..." + docker cp $(Build.SourcesDirectory)/. test-$(ARCH):/workspace/ + + if [[ "$(BASE_IMAGE)" == alpine* ]]; then + SHELL_CMD="sh -c" + PY_CMD="/venv/bin/python" + else + SHELL_CMD="bash -c" + PY_CMD="/venv/bin/python" + fi + + docker exec test-$(ARCH) $SHELL_CMD " + cd /workspace + + echo 'Running pytest suite with installed wheel...' + echo 'Current directory:' \$(pwd) + echo 'Python version:' + $PY_CMD --version + + echo 'Package import verification:' + $PY_CMD -c 'import mssql_python; print(\"Testing installed wheel from:\", mssql_python.__file__)' + + if [ -f requirements.txt ]; then + echo 'Installing test requirements...' + $PY_CMD -m pip install -r requirements.txt || echo 'Failed to install some requirements' + fi + + $PY_CMD -m pip install pytest || echo 'pytest installation failed' + + echo 'Available test files:' + find tests/ -name 'test_*.py' 2>/dev/null || echo 'No test files found in tests/' + + if [ -d tests/ ]; then + echo 'Starting pytest...' + $PY_CMD -m pytest -v || echo 'Some tests failed - this may be expected in containerized environment' + else + echo 'No tests directory found, skipping pytest' + fi + " + displayName: 'Run pytest suite on $(BASE_IMAGE) $(ARCH)' + env: + DB_CONNECTION_STRING: 'Server=localhost;Database=TestDB;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' + continueOnError: true + + - script: | + docker stop test-$(ARCH) sqlserver || true + docker rm test-$(ARCH) sqlserver || true + displayName: 'Cleanup containers' + condition: always() - job: PytestOnWindows displayName: 'Windows x64' From 872f1c689d89e347ac2aba8077a5eea9f61ed0c5 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 01:02:17 +0530 Subject: [PATCH 02/24] Testing --- eng/pipelines/pr-validation-pipeline.yml | 1897 +--------------------- 1 file changed, 1 insertion(+), 1896 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 5185060c..ab817d27 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -12,7 +12,7 @@ pr: - main jobs: -# ===== PYTHON 3.14 TESTING JOBS ===== +# ===== PYTHON 3.14 TESTING ONLY ===== - job: BuildWindowsWheels_Python314 pool: @@ -677,1898 +677,3 @@ jobs: docker rm test-$(ARCH) sqlserver || true displayName: 'Cleanup containers' condition: always() - -- job: PytestOnWindows - displayName: 'Windows x64' - pool: - vmImage: 'windows-latest' - - strategy: - matrix: - LocalDB: - sqlVersion: 'LocalDB' - pythonVersion: '3.13' - SQLServer2022: - sqlVersion: 'SQL2022' - pythonVersion: '3.13' - LocalDB_Python314: - sqlVersion: 'LocalDB' - pythonVersion: '3.14' - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(pythonVersion)' - addToPath: true - githubToken: $(GITHUB_TOKEN) - displayName: 'Use Python $(pythonVersion)' - - - script: | - python -m pip install --upgrade pip - pip install -r requirements.txt - displayName: 'Install dependencies' - - # Start LocalDB instance (for LocalDB matrix) - - powershell: | - sqllocaldb create MSSQLLocalDB - sqllocaldb start MSSQLLocalDB - displayName: 'Start LocalDB instance' - condition: eq(variables['sqlVersion'], 'LocalDB') - - # Create database and user for LocalDB - - powershell: | - sqlcmd -S "(localdb)\MSSQLLocalDB" -Q "CREATE DATABASE TestDB" - sqlcmd -S "(localdb)\MSSQLLocalDB" -Q "CREATE LOGIN testuser WITH PASSWORD = '$(DB_PASSWORD)'" - sqlcmd -S "(localdb)\MSSQLLocalDB" -d TestDB -Q "CREATE USER testuser FOR LOGIN testuser" - sqlcmd -S "(localdb)\MSSQLLocalDB" -d TestDB -Q "ALTER ROLE db_owner ADD MEMBER testuser" - displayName: 'Setup database and user for LocalDB' - condition: eq(variables['sqlVersion'], 'LocalDB') - env: - DB_PASSWORD: $(DB_PASSWORD) - - # Install SQL Server 2022 (for SQL2022 matrix) - - powershell: | - Write-Host "Downloading SQL Server 2022 Express..." - # Download SQL Server 2022 Express installer - $ProgressPreference = 'SilentlyContinue' - Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?linkid=2216019" -OutFile "SQL2022-SSEI-Expr.exe" - - Write-Host "Installing SQL Server 2022 Express..." - # Install SQL Server 2022 Express with basic features - Start-Process -FilePath "SQL2022-SSEI-Expr.exe" -ArgumentList "/Action=Download","/MediaPath=$env:TEMP","/MediaType=Core","/Quiet" -Wait - - # Find the downloaded setup file - $setupFile = Get-ChildItem -Path $env:TEMP -Filter "SQLEXPR_x64_ENU.exe" -Recurse | Select-Object -First 1 - - if ($setupFile) { - Write-Host "Extracting SQL Server setup files..." - Start-Process -FilePath $setupFile.FullName -ArgumentList "/x:$env:TEMP\SQLSetup","/u" -Wait - - Write-Host "Running SQL Server setup..." - Start-Process -FilePath "$env:TEMP\SQLSetup\setup.exe" -ArgumentList "/Q","/ACTION=Install","/FEATURES=SQLEngine","/INSTANCENAME=MSSQLSERVER","/SQLSVCACCOUNT=`"NT AUTHORITY\SYSTEM`"","/SQLSYSADMINACCOUNTS=`"BUILTIN\Administrators`"","/TCPENABLED=1","/SECURITYMODE=SQL","/SAPWD=$(DB_PASSWORD)","/IACCEPTSQLSERVERLICENSETERMS" -Wait - } else { - Write-Error "Failed to download SQL Server setup file" - exit 1 - } - - Write-Host "SQL Server 2022 installation completed" - displayName: 'Install SQL Server 2022 Express' - condition: eq(variables['sqlVersion'], 'SQL2022') - env: - DB_PASSWORD: $(DB_PASSWORD) - - # Create database for SQL Server 2022 - - powershell: | - # Wait for SQL Server to start - $maxAttempts = 30 - $attempt = 0 - $connected = $false - - Write-Host "Waiting for SQL Server 2022 to start..." - while (-not $connected -and $attempt -lt $maxAttempts) { - try { - sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -Q "SELECT 1" -C - $connected = $true - Write-Host "SQL Server is ready!" - } catch { - $attempt++ - Write-Host "Waiting... ($attempt/$maxAttempts)" - Start-Sleep -Seconds 2 - } - } - - if (-not $connected) { - Write-Error "Failed to connect to SQL Server after $maxAttempts attempts" - exit 1 - } - - # Create database and user - sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -Q "CREATE DATABASE TestDB" -C - sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -Q "CREATE LOGIN testuser WITH PASSWORD = '$(DB_PASSWORD)'" -C - sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -d TestDB -Q "CREATE USER testuser FOR LOGIN testuser" -C - sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -d TestDB -Q "ALTER ROLE db_owner ADD MEMBER testuser" -C - displayName: 'Setup database and user for SQL Server 2022' - condition: eq(variables['sqlVersion'], 'SQL2022') - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - cd mssql_python\pybind - build.bat x64 - displayName: 'Build .pyd file' - - # Run tests for LocalDB - - script: | - python -m pytest -v --junitxml=test-results-localdb.xml --cov=. --cov-report=xml:coverage-localdb.xml --capture=tee-sys --cache-clear - displayName: 'Run tests with coverage on LocalDB' - condition: eq(variables['sqlVersion'], 'LocalDB') - env: - DB_CONNECTION_STRING: 'Server=(localdb)\MSSQLLocalDB;Database=TestDB;Uid=testuser;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' - - # Run tests for SQL Server 2022 - - script: | - python -m pytest -v --junitxml=test-results-sql2022.xml --cov=. --cov-report=xml:coverage-sql2022.xml --capture=tee-sys --cache-clear - displayName: 'Run tests with coverage on SQL Server 2022' - condition: eq(variables['sqlVersion'], 'SQL2022') - env: - DB_CONNECTION_STRING: 'Server=localhost;Database=TestDB;Uid=testuser;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' - - # Download and restore AdventureWorks2022 database for benchmarking - - powershell: | - Write-Host "Downloading AdventureWorks2022.bak..." - $ProgressPreference = 'SilentlyContinue' - Invoke-WebRequest -Uri "https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorks2022.bak" -OutFile "$env:TEMP\AdventureWorks2022.bak" - - Write-Host "Restoring AdventureWorks2022 database..." - # Get the default data and log paths - $dataPath = sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -Q "SET NOCOUNT ON; SELECT SERVERPROPERTY('InstanceDefaultDataPath') AS DataPath" -h -1 -C | Out-String - $logPath = sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -Q "SET NOCOUNT ON; SELECT SERVERPROPERTY('InstanceDefaultLogPath') AS LogPath" -h -1 -C | Out-String - - $dataPath = $dataPath.Trim() - $logPath = $logPath.Trim() - - Write-Host "Data path: $dataPath" - Write-Host "Log path: $logPath" - - # Restore the database - sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -C -Q @" - RESTORE DATABASE AdventureWorks2022 - FROM DISK = '$env:TEMP\AdventureWorks2022.bak' - WITH - MOVE 'AdventureWorks2022' TO '${dataPath}AdventureWorks2022.mdf', - MOVE 'AdventureWorks2022_log' TO '${logPath}AdventureWorks2022_log.ldf', - REPLACE - "@ - - if ($LASTEXITCODE -eq 0) { - Write-Host "AdventureWorks2022 database restored successfully" - } else { - Write-Error "Failed to restore AdventureWorks2022 database" - exit 1 - } - displayName: 'Download and restore AdventureWorks2022 database' - condition: eq(variables['sqlVersion'], 'SQL2022') - env: - DB_PASSWORD: $(DB_PASSWORD) - - # Run performance benchmarks on SQL Server 2022 - - powershell: | - Write-Host "Checking and installing ODBC Driver 18 for SQL Server..." - - # Check if ODBC Driver 18 is registered in Windows registry - $odbcDriverKey = "HKLM:\SOFTWARE\ODBC\ODBCINST.INI\ODBC Driver 18 for SQL Server" - $driverExists = Test-Path $odbcDriverKey - - if ($driverExists) { - Write-Host "✓ ODBC Driver 18 for SQL Server is already installed and registered" - $driverPath = (Get-ItemProperty -Path $odbcDriverKey -Name "Driver" -ErrorAction SilentlyContinue).Driver - if ($driverPath) { - Write-Host " Driver location: $driverPath" - } - } else { - Write-Host "ODBC Driver 18 for SQL Server not found, installing..." - - # Download ODBC Driver 18.5.2.1 (x64) from official Microsoft link - $ProgressPreference = 'SilentlyContinue' - $installerUrl = "https://go.microsoft.com/fwlink/?linkid=2335671" - $installerPath = "$env:TEMP\msodbcsql_18.5.2.1_x64.msi" - - Write-Host "Downloading ODBC Driver 18 (x64) from Microsoft..." - Write-Host " URL: $installerUrl" - try { - Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing - Write-Host "✓ Download completed: $installerPath" - } catch { - Write-Error "Failed to download ODBC driver: $_" - exit 1 - } - - Write-Host "Installing ODBC Driver 18..." - $installArgs = @( - "/i" - "`"$installerPath`"" - "/quiet" - "/qn" - "/norestart" - "IACCEPTMSODBCSQLLICENSETERMS=YES" - "/l*v" - "`"$env:TEMP\odbc_install.log`"" - ) - - $installCmd = "msiexec.exe $($installArgs -join ' ')" - Write-Host " Command: $installCmd" - - $process = Start-Process msiexec.exe -ArgumentList $installArgs -Wait -PassThru -NoNewWindow - - if ($process.ExitCode -eq 0) { - Write-Host "✓ ODBC Driver 18 installation completed successfully" - } elseif ($process.ExitCode -eq 3010) { - Write-Host "✓ ODBC Driver 18 installed (reboot recommended but not required)" - } else { - Write-Error "ODBC Driver 18 installation failed with exit code: $($process.ExitCode)" - Write-Host "Check installation log: $env:TEMP\odbc_install.log" - Get-Content "$env:TEMP\odbc_install.log" -Tail 50 -ErrorAction SilentlyContinue - exit 1 - } - - # Wait for registry update - Start-Sleep -Seconds 2 - - # Clean up installer - Remove-Item $installerPath -ErrorAction SilentlyContinue - } - - # Final verification using registry - Write-Host "`nVerifying ODBC Driver 18 installation..." - $verifyKey = Test-Path "HKLM:\SOFTWARE\ODBC\ODBCINST.INI\ODBC Driver 18 for SQL Server" - - if ($verifyKey) { - $driverInfo = Get-ItemProperty -Path "HKLM:\SOFTWARE\ODBC\ODBCINST.INI\ODBC Driver 18 for SQL Server" -ErrorAction SilentlyContinue - Write-Host "✓ SUCCESS: ODBC Driver 18 for SQL Server is registered" - Write-Host " Driver: $($driverInfo.Driver)" - Write-Host " Setup: $($driverInfo.Setup)" - } else { - Write-Error "ODBC Driver 18 for SQL Server is not registered in ODBC" - Write-Host "`nListing all installed ODBC drivers from registry:" - Get-ChildItem "HKLM:\SOFTWARE\ODBC\ODBCINST.INI" -ErrorAction SilentlyContinue | ForEach-Object { Write-Host " - $($_.PSChildName)" } - exit 1 - } - - Write-Host "`nInstalling pyodbc..." - pip install pyodbc - - Write-Host "`nRunning performance benchmarks..." - python benchmarks/perf-benchmarking.py - displayName: 'Run performance benchmarks on SQL Server 2022' - condition: eq(variables['sqlVersion'], 'SQL2022') - continueOnError: true - env: - DB_CONNECTION_STRING: 'Server=localhost;Database=AdventureWorks2022;Uid=sa;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' - - - task: CopyFiles@2 - inputs: - SourceFolder: 'mssql_python' - Contents: 'ddbc_bindings.cp*-amd64.pyd' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - displayName: 'Copy pyd file to staging' - - - task: CopyFiles@2 - inputs: - SourceFolder: 'mssql_python' - Contents: 'ddbc_bindings.cp*-amd64.pdb' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - displayName: 'Copy pdb file to staging' - - - task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: 'ddbc_bindings' - publishLocation: 'Container' - displayName: 'Publish build artifacts' - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-results-*.xml' - testRunTitle: 'Publish test results for Windows $(sqlVersion)' - - # - task: PublishCodeCoverageResults@1 - # inputs: - # codeCoverageTool: 'Cobertura' - # summaryFileLocation: 'coverage.xml' - # displayName: 'Publish code coverage results' - -- job: PytestOnMacOS - displayName: 'macOS x86_64' - pool: - vmImage: 'macos-latest' - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.13' - addToPath: true - displayName: 'Use Python 3.13 on macOS' - - - script: | - brew update - # Uninstall existing CMake to avoid tap conflicts - brew uninstall cmake --ignore-dependencies || echo "CMake not installed or already removed" - # Install CMake from homebrew/core - brew install cmake - displayName: 'Install CMake' - - - script: | - brew update - brew install docker colima - - # Start Colima with extra resources - colima start --cpu 4 --memory 8 --disk 50 - - # Optional: set Docker context (usually automatic) - docker context use colima >/dev/null || true - - # Confirm Docker is operational - docker version - docker ps - displayName: 'Install and start Colima-based Docker' - - - script: | - # Pull and run SQL Server container - docker pull mcr.microsoft.com/mssql/server:2022-latest - docker run \ - --name sqlserver \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="${DB_PASSWORD}" \ - -p 1433:1433 \ - -d mcr.microsoft.com/mssql/server:2022-latest - - # Starting SQL Server container… - for i in {1..30}; do - docker exec sqlserver \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$DB_PASSWORD" \ - -C -Q "SELECT 1" && break - sleep 2 - done - displayName: 'Pull & start SQL Server (Docker)' - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - python -m pip install --upgrade pip - pip install -r requirements.txt - displayName: 'Install Python dependencies' - - - script: | - cd mssql_python/pybind - ./build.sh - displayName: 'Build pybind bindings (.so)' - - - script: | - echo "Build successful, running tests now" - python -m pytest -v --junitxml=test-results.xml --cov=. --cov-report=xml --capture=tee-sys --cache-clear - displayName: 'Run pytest with coverage' - env: - DB_CONNECTION_STRING: 'Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' - DB_PASSWORD: $(DB_PASSWORD) - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Publish pytest results on macOS' - -- job: PytestOnLinux - displayName: 'Linux x86_64' - pool: - vmImage: 'ubuntu-latest' - - strategy: - matrix: - Ubuntu: - dockerImage: 'ubuntu:22.04' - distroName: 'Ubuntu' - sqlServerImage: 'mcr.microsoft.com/mssql/server:2022-latest' - useAzureSQL: 'false' - Ubuntu_SQL2025: - dockerImage: 'ubuntu:22.04' - distroName: 'Ubuntu-SQL2025' - sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest' - useAzureSQL: 'false' - ${{ if ne(variables['AZURE_CONNECTION_STRING'], '') }}: - Ubuntu_AzureSQL: - dockerImage: 'ubuntu:22.04' - distroName: 'Ubuntu-AzureSQL' - sqlServerImage: '' - useAzureSQL: 'true' - Debian: - dockerImage: 'debian:12' - distroName: 'Debian' - sqlServerImage: 'mcr.microsoft.com/mssql/server:2022-latest' - useAzureSQL: 'false' - - steps: - - script: | - # Create a Docker container for testing - docker run -d --name test-container-$(distroName) \ - -v $(Build.SourcesDirectory):/workspace \ - -w /workspace \ - --network bridge \ - $(dockerImage) \ - tail -f /dev/null - displayName: 'Create $(distroName) container' - - - script: | - # Start SQL Server container - docker run -d --name sqlserver-$(distroName) \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ - -p 1433:1433 \ - $(sqlServerImage) - - # Wait for SQL Server to be ready - echo "Waiting for SQL Server to start..." - for i in {1..60}; do - if docker exec sqlserver-$(distroName) \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - break - fi - echo "Waiting... ($i/60)" - sleep 2 - done - - # Create test database - docker exec sqlserver-$(distroName) \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "CREATE DATABASE TestDB" - displayName: 'Start SQL Server container for $(distroName)' - condition: eq(variables['useAzureSQL'], 'false') - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - # Install dependencies in the container - if [ "$(distroName)" = "Ubuntu" ]; then - docker exec test-container-$(distroName) bash -c " - export DEBIAN_FRONTEND=noninteractive - export TZ=UTC - ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone - apt-get update && - apt-get install -y python3 python3-pip python3-venv python3-full cmake curl wget gnupg software-properties-common build-essential python3-dev pybind11-dev - " - else - # Debian - docker exec test-container-$(distroName) bash -c " - export DEBIAN_FRONTEND=noninteractive - export TZ=UTC - ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone - apt-get update && - apt-get install -y python3 python3-pip python3-venv python3-full cmake curl wget gnupg software-properties-common build-essential python3-dev pybind11-dev - " - fi - displayName: 'Install basic dependencies in $(distroName) container' - - - script: | - # Install ODBC driver in the container - docker exec test-container-$(distroName) bash -c " - export DEBIAN_FRONTEND=noninteractive - - # Download the package to configure the Microsoft repo - if [ '$(distroName)' = 'Ubuntu' ]; then - curl -sSL -O https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb - else - # Debian 12 - curl -sSL -O https://packages.microsoft.com/config/debian/12/packages-microsoft-prod.deb - fi - - # Install the package - dpkg -i packages-microsoft-prod.deb || true - rm packages-microsoft-prod.deb - - # Update package list - apt-get update - - # Install the driver - ACCEPT_EULA=Y apt-get install -y msodbcsql18 - # optional: for bcp and sqlcmd - ACCEPT_EULA=Y apt-get install -y mssql-tools18 - # optional: for unixODBC development headers - apt-get install -y unixodbc-dev - " - displayName: 'Install ODBC Driver in $(distroName) container' - - - script: | - # Install Python dependencies in the container using virtual environment - docker exec test-container-$(distroName) bash -c " - # Create a virtual environment - python3 -m venv /opt/venv - source /opt/venv/bin/activate - - # Install dependencies in the virtual environment - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - - # Make the virtual environment globally available - echo 'source /opt/venv/bin/activate' >> ~/.bashrc - " - displayName: 'Install Python dependencies in $(distroName) container' - - - script: | - # Build pybind bindings in the container - docker exec test-container-$(distroName) bash -c " - source /opt/venv/bin/activate - cd mssql_python/pybind - chmod +x build.sh - ./build.sh - " - displayName: 'Build pybind bindings (.so) in $(distroName) container' - - - script: | - # Uninstall ODBC Driver before running tests - docker exec test-container-$(distroName) bash -c " - export DEBIAN_FRONTEND=noninteractive - apt-get remove --purge -y msodbcsql18 mssql-tools18 unixodbc-dev - rm -f /usr/bin/sqlcmd - rm -f /usr/bin/bcp - rm -rf /opt/microsoft/msodbcsql - rm -f /lib/x86_64-linux-gnu/libodbcinst.so.2 - odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true - echo 'Uninstalled ODBC Driver and cleaned up libraries' - echo 'Verifying x86_64 debian_ubuntu driver library signatures:' - ldd mssql_python/libs/linux/debian_ubuntu/x86_64/lib/libmsodbcsql-18.5.so.1.1 - " - displayName: 'Uninstall ODBC Driver before running tests in $(distroName) container' - - - script: | - # Run tests in the container - if [ "$(useAzureSQL)" = "true" ]; then - # Azure SQL Database testing - echo "Testing against Azure SQL Database" - - docker exec \ - -e DB_CONNECTION_STRING="$(AZURE_CONNECTION_STRING)" \ - test-container-$(distroName) bash -c " - source /opt/venv/bin/activate - echo 'Build successful, running tests now on $(distroName) with Azure SQL' - echo 'Using Azure SQL connection string' - python -m pytest -v --junitxml=test-results-$(distroName).xml --cov=. --cov-report=xml:coverage-$(distroName).xml --capture=tee-sys --cache-clear - " - else - # Local SQL Server testing - SQLSERVER_IP=$(docker inspect sqlserver-$(distroName) --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}') - echo "SQL Server IP: $SQLSERVER_IP" - - docker exec \ - -e DB_CONNECTION_STRING="Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes" \ - -e DB_PASSWORD="$(DB_PASSWORD)" \ - test-container-$(distroName) bash -c " - source /opt/venv/bin/activate - echo 'Build successful, running tests now on $(distroName)' - echo 'Using connection string: Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=***;TrustServerCertificate=yes' - python -m pytest -v --junitxml=test-results-$(distroName).xml --cov=. --cov-report=xml:coverage-$(distroName).xml --capture=tee-sys --cache-clear - " - fi - displayName: 'Run pytest with coverage in $(distroName) container' - condition: or(eq(variables['useAzureSQL'], 'false'), and(eq(variables['useAzureSQL'], 'true'), ne(variables['AZURE_CONNECTION_STRING'], ''))) - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - # Download and restore AdventureWorks2022 database for benchmarking on Ubuntu only - if [ "$(distroName)" = "Ubuntu" ] && [ "$(useAzureSQL)" = "false" ]; then - echo "Downloading AdventureWorks2022.bak..." - wget -q https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorks2022.bak -O /tmp/AdventureWorks2022.bak - - echo "Copying backup file into SQL Server container..." - docker cp /tmp/AdventureWorks2022.bak sqlserver-$(distroName):/tmp/AdventureWorks2022.bak - - echo "Restoring AdventureWorks2022 database..." - docker exec sqlserver-$(distroName) /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C \ - -Q "RESTORE DATABASE AdventureWorks2022 FROM DISK = '/tmp/AdventureWorks2022.bak' WITH MOVE 'AdventureWorks2022' TO '/var/opt/mssql/data/AdventureWorks2022.mdf', MOVE 'AdventureWorks2022_log' TO '/var/opt/mssql/data/AdventureWorks2022_log.ldf', REPLACE" - - if [ $? -eq 0 ]; then - echo "AdventureWorks2022 database restored successfully" - else - echo "Failed to restore AdventureWorks2022 database" - fi - - # Clean up (ignore errors if files are locked) - rm -f /tmp/AdventureWorks2022.bak || true - docker exec sqlserver-$(distroName) rm -f /tmp/AdventureWorks2022.bak || true - fi - displayName: 'Download and restore AdventureWorks2022 database in $(distroName)' - condition: and(eq(variables['distroName'], 'Ubuntu'), eq(variables['useAzureSQL'], 'false')) - continueOnError: true - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - # Run performance benchmarks on Ubuntu with SQL Server 2022 only - if [ "$(distroName)" = "Ubuntu" ] && [ "$(useAzureSQL)" = "false" ]; then - SQLSERVER_IP=$(docker inspect sqlserver-$(distroName) --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}') - echo "Running performance benchmarks on Ubuntu with SQL Server IP: $SQLSERVER_IP" - - docker exec \ - -e DB_CONNECTION_STRING="Server=$SQLSERVER_IP;Database=AdventureWorks2022;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes" \ - test-container-$(distroName) bash -c " - source /opt/venv/bin/activate - - echo 'Reinstalling ODBC Driver for benchmarking...' - export DEBIAN_FRONTEND=noninteractive - - # Remove duplicate repository sources if they exist - rm -f /etc/apt/sources.list.d/microsoft-prod.list - - # Add Microsoft repository - curl -sSL https://packages.microsoft.com/keys/microsoft.asc | apt-key add - - curl -sSL https://packages.microsoft.com/config/ubuntu/22.04/prod.list > /etc/apt/sources.list.d/mssql-release.list - - # Update package lists - apt-get update -qq - - # Install unixodbc and its dependencies first (provides libodbcinst.so.2 needed by msodbcsql18) - echo 'Installing unixODBC dependencies...' - apt-get install -y --no-install-recommends unixodbc unixodbc-dev libodbc1 odbcinst odbcinst1debian2 - - # Verify libodbcinst.so.2 is available - ldconfig - ls -la /usr/lib/x86_64-linux-gnu/libodbcinst.so.2 || echo 'Warning: libodbcinst.so.2 not found' - - # Install ODBC Driver 18 - echo 'Installing msodbcsql18...' - ACCEPT_EULA=Y apt-get install -y msodbcsql18 - - # Verify ODBC driver installation - odbcinst -q -d -n 'ODBC Driver 18 for SQL Server' || echo 'Warning: ODBC Driver 18 not registered' - - echo 'Installing pyodbc for benchmarking...' - pip install pyodbc - echo 'Running performance benchmarks on $(distroName)' - python benchmarks/perf-benchmarking.py || echo 'Performance benchmark failed or database not available' - " - else - echo "Skipping performance benchmarks on $(distroName) (only runs on Ubuntu with local SQL Server)" - fi - displayName: 'Run performance benchmarks in $(distroName) container' - condition: and(eq(variables['distroName'], 'Ubuntu'), eq(variables['useAzureSQL'], 'false')) - continueOnError: true - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - # Copy test results from container to host - docker cp test-container-$(distroName):/workspace/test-results-$(distroName).xml $(Build.SourcesDirectory)/ - docker cp test-container-$(distroName):/workspace/coverage-$(distroName).xml $(Build.SourcesDirectory)/ - displayName: 'Copy test results from $(distroName) container' - condition: always() - - - script: | - # Clean up containers - docker stop test-container-$(distroName) || true - docker rm test-container-$(distroName) || true - if [ "$(useAzureSQL)" = "false" ]; then - docker stop sqlserver-$(distroName) || true - docker rm sqlserver-$(distroName) || true - fi - displayName: 'Clean up $(distroName) containers' - condition: always() - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-results-$(distroName).xml' - testRunTitle: 'Publish pytest results on $(distroName)' - -- job: PytestOnLinux_ARM64 - displayName: 'Linux ARM64' - pool: - vmImage: 'ubuntu-latest' - - strategy: - matrix: - Ubuntu_ARM64: - dockerImage: 'ubuntu:22.04' - distroName: 'Ubuntu' - archName: 'arm64' - Debian_ARM64: - dockerImage: 'debian:12' - distroName: 'Debian' - archName: 'arm64' - - steps: - - script: | - # Set up Docker buildx for multi-architecture support - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker buildx create --name multiarch --driver docker-container --use - docker buildx inspect --bootstrap - displayName: 'Setup Docker buildx for ARM64 emulation' - - - script: | - # Create a Docker container for testing on ARM64 - docker run -d --name test-container-$(distroName)-$(archName) \ - --platform linux/arm64 \ - -v $(Build.SourcesDirectory):/workspace \ - -w /workspace \ - --network bridge \ - $(dockerImage) \ - tail -f /dev/null - displayName: 'Create $(distroName) ARM64 container' - - - script: | - # Start SQL Server container (x86_64 - SQL Server doesn't support ARM64) - docker run -d --name sqlserver-$(distroName)-$(archName) \ - --platform linux/amd64 \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ - -p 1433:1433 \ - mcr.microsoft.com/mssql/server:2022-latest - - # Wait for SQL Server to be ready - echo "Waiting for SQL Server to start..." - for i in {1..60}; do - if docker exec sqlserver-$(distroName)-$(archName) \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - break - fi - echo "Waiting... ($i/60)" - sleep 2 - done - - # Create test database - docker exec sqlserver-$(distroName)-$(archName) \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "CREATE DATABASE TestDB" - displayName: 'Start SQL Server container for $(distroName) ARM64' - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - # Install dependencies in the ARM64 container - if [ "$(distroName)" = "Ubuntu" ]; then - docker exec test-container-$(distroName)-$(archName) bash -c " - export DEBIAN_FRONTEND=noninteractive - export TZ=UTC - ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone - apt-get update && - apt-get install -y python3 python3-pip python3-venv python3-full cmake curl wget gnupg software-properties-common build-essential python3-dev pybind11-dev - # Verify architecture - uname -m - dpkg --print-architecture - " - else - # Debian ARM64 - docker exec test-container-$(distroName)-$(archName) bash -c " - export DEBIAN_FRONTEND=noninteractive - export TZ=UTC - ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone - apt-get update && - apt-get install -y python3 python3-pip python3-venv python3-full cmake curl wget gnupg software-properties-common build-essential python3-dev pybind11-dev - # Verify architecture - uname -m - dpkg --print-architecture - " - fi - displayName: 'Install basic dependencies in $(distroName) ARM64 container' - - - script: | - # Install ODBC driver in the ARM64 container - docker exec test-container-$(distroName)-$(archName) bash -c " - export DEBIAN_FRONTEND=noninteractive - - # Download the package to configure the Microsoft repo - if [ '$(distroName)' = 'Ubuntu' ]; then - curl -sSL -O https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb - else - # Debian 12 - curl -sSL -O https://packages.microsoft.com/config/debian/12/packages-microsoft-prod.deb - fi - - # Install the package - dpkg -i packages-microsoft-prod.deb || true - rm packages-microsoft-prod.deb - - # Update package list - apt-get update - - # Install the driver (ARM64 version) - ACCEPT_EULA=Y apt-get install -y msodbcsql18 - # optional: for bcp and sqlcmd - ACCEPT_EULA=Y apt-get install -y mssql-tools18 - # optional: for unixODBC development headers - apt-get install -y unixodbc-dev - " - displayName: 'Install ODBC Driver in $(distroName) ARM64 container' - - - script: | - # Install Python dependencies in the ARM64 container using virtual environment - docker exec test-container-$(distroName)-$(archName) bash -c " - # Create a virtual environment - python3 -m venv /opt/venv - source /opt/venv/bin/activate - - # Install dependencies in the virtual environment - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - - # Make the virtual environment globally available - echo 'source /opt/venv/bin/activate' >> ~/.bashrc - " - displayName: 'Install Python dependencies in $(distroName) ARM64 container' - - - script: | - # Build pybind bindings in the ARM64 container - docker exec test-container-$(distroName)-$(archName) bash -c " - source /opt/venv/bin/activate - cd mssql_python/pybind - chmod +x build.sh - ./build.sh - " - displayName: 'Build pybind bindings (.so) in $(distroName) ARM64 container' - - - script: | - # Uninstall ODBC Driver before running tests - docker exec test-container-$(distroName)-$(archName) bash -c " - export DEBIAN_FRONTEND=noninteractive - apt-get remove --purge -y msodbcsql18 mssql-tools18 unixodbc-dev - rm -f /usr/bin/sqlcmd - rm -f /usr/bin/bcp - rm -rf /opt/microsoft/msodbcsql - rm -f /lib/aarch64-linux-gnu/libodbcinst.so.2 - odbcinst -u -d -n 'ODBC Driver 11 for SQL Server' || true - echo 'Uninstalled ODBC Driver and cleaned up libraries' - echo 'Verifying arm64 debian_ubuntu driver library signatures:' - ldd mssql_python/libs/linux/debian_ubuntu/arm64/lib/libmsodbcsql-18.5.so.1.1 - " - displayName: 'Uninstall ODBC Driver before running tests in $(distroName) ARM64 container' - - - script: | - # Run tests in the ARM64 container - # Get SQL Server container IP - SQLSERVER_IP=$(docker inspect sqlserver-$(distroName)-$(archName) --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}') - echo "SQL Server IP: $SQLSERVER_IP" - - docker exec \ - -e DB_CONNECTION_STRING="Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes" \ - -e DB_PASSWORD="$(DB_PASSWORD)" \ - test-container-$(distroName)-$(archName) bash -c " - source /opt/venv/bin/activate - echo 'Build successful, running tests now on $(distroName) ARM64' - echo 'Architecture:' \$(uname -m) - echo 'Using connection string: Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=***;TrustServerCertificate=yes' - python main.py - python -m pytest -v --junitxml=test-results-$(distroName)-$(archName).xml --cov=. --cov-report=xml:coverage-$(distroName)-$(archName).xml --capture=tee-sys --cache-clear - " - displayName: 'Run pytest with coverage in $(distroName) ARM64 container' - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - # Copy test results from container to host - docker cp test-container-$(distroName)-$(archName):/workspace/test-results-$(distroName)-$(archName).xml $(Build.SourcesDirectory)/ - docker cp test-container-$(distroName)-$(archName):/workspace/coverage-$(distroName)-$(archName).xml $(Build.SourcesDirectory)/ - displayName: 'Copy test results from $(distroName) ARM64 container' - condition: always() - - - script: | - # Clean up containers - docker stop test-container-$(distroName)-$(archName) || true - docker rm test-container-$(distroName)-$(archName) || true - docker stop sqlserver-$(distroName)-$(archName) || true - docker rm sqlserver-$(distroName)-$(archName) || true - displayName: 'Clean up $(distroName) ARM64 containers' - condition: always() - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-results-$(distroName)-$(archName).xml' - testRunTitle: 'Publish pytest results on $(distroName) ARM64' - -- job: PytestOnLinux_RHEL9 - displayName: 'Linux RedHat x86_64' - pool: - vmImage: 'ubuntu-latest' - - steps: - - script: | - # Create a Docker container for testing - docker run -d --name test-container-rhel9 \ - -v $(Build.SourcesDirectory):/workspace \ - -w /workspace \ - --network bridge \ - redhat/ubi9:latest \ - tail -f /dev/null - displayName: 'Create RHEL 9 container' - - - script: | - # Start SQL Server container - docker run -d --name sqlserver-rhel9 \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ - -p 1433:1433 \ - mcr.microsoft.com/mssql/server:2022-latest - - # Wait for SQL Server to be ready - echo "Waiting for SQL Server to start..." - for i in {1..60}; do - if docker exec sqlserver-rhel9 \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - break - fi - echo "Waiting... ($i/60)" - sleep 2 - done - - # Create test database - docker exec sqlserver-rhel9 \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "CREATE DATABASE TestDB" - displayName: 'Start SQL Server container for RHEL 9' - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - # Install dependencies in the RHEL 9 container - docker exec test-container-rhel9 bash -c " - # Enable CodeReady Builder repository for additional packages - dnf update -y - dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm - subscription-manager repos --enable codeready-builder-for-rhel-9-$(arch)-rpms || dnf config-manager --set-enabled ubi-9-codeready-builder - - # Install Python 3.9 (available in RHEL 9 UBI) and development tools - dnf install -y python3 python3-pip python3-devel cmake curl wget gnupg2 glibc-devel kernel-headers - dnf install -y python3-libs python3-debug - dnf install -y gcc gcc-c++ make binutils - dnf install -y cmake - # If that doesn't work, try installing from different repositories - if ! which gcc; then - echo 'Trying alternative gcc installation...' - dnf --enablerepo=ubi-9-codeready-builder install -y gcc gcc-c++ - fi - # Verify installation - python3 --version - which gcc && which g++ - gcc --version - g++ --version - " - displayName: 'Install basic dependencies in RHEL 9 container' - - - script: | - # Verify compiler installation and set environment for RHEL 9 - docker exec test-container-rhel9 bash -c " - # Verify compilers are available - which gcc || echo 'GCC not found' - which g++ || echo 'G++ not found' - gcc --version || echo 'GCC version check failed' - g++ --version || echo 'G++ version check failed' - - # Set compiler environment variables - export CC=/usr/bin/gcc - export CXX=/usr/bin/g++ - echo 'CC set to:' \$CC - echo 'CXX set to:' \$CXX - - # Create a wrapper script to preserve environment - cat > /workspace/setup_env.sh << 'EOF' - #!/bin/bash - export CC=/usr/bin/gcc - export CXX=/usr/bin/g++ - export PATH=/usr/bin:\$PATH - exec \"\$@\" - EOF - chmod +x /workspace/setup_env.sh - " - displayName: 'Verify and configure compilers in RHEL 9 container' - - - script: | - # Install ODBC driver in the RHEL 9 container - docker exec test-container-rhel9 bash -c " - # Add Microsoft repository for RHEL 9 - curl -sSL -o /etc/yum.repos.d/msprod.repo https://packages.microsoft.com/config/rhel/9/prod.repo - - # Install the driver - ACCEPT_EULA=Y dnf install -y msodbcsql18 - # optional: for bcp and sqlcmd - ACCEPT_EULA=Y dnf install -y mssql-tools18 - # optional: for unixODBC development headers - dnf install -y unixODBC-devel - " - displayName: 'Install ODBC Driver in RHEL 9 container' - - - script: | - # Install Python dependencies in the container using virtual environment - docker exec test-container-rhel9 bash -c " - # Create a virtual environment with Python 3.9 - python3 -m venv myvenv - source myvenv/bin/activate - - # Install dependencies in the virtual environment - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - - # Make the virtual environment globally available - echo 'source myvenv/bin/activate' >> ~/.bashrc - " - displayName: 'Install Python dependencies in RHEL 9 container' - - - script: | - # Build pybind bindings in the container - docker exec test-container-rhel9 bash -c " - source myvenv/bin/activate - ls /usr/include/python3.9 - # Set compiler environment variables - export CC=/usr/bin/gcc - export CXX=/usr/bin/g++ - - cd mssql_python/pybind - chmod +x build.sh - ./build.sh - " - displayName: 'Build pybind bindings (.so) in RHEL 9 container' - - - script: | - # Uninstall ODBC Driver before running tests - docker exec test-container-rhel9 bash -c " - dnf remove -y msodbcsql18 mssql-tools18 unixODBC-devel - rm -f /usr/bin/sqlcmd - rm -f /usr/bin/bcp - rm -rf /opt/microsoft/msodbcsql - rm -f /lib64/libodbcinst.so.2 - odbcinst -u -d -n 'ODBC Driver 11 for SQL Server' || true - echo 'Uninstalled ODBC Driver and cleaned up libraries' - echo 'Verifying x86_64 rhel driver library signatures:' - ldd mssql_python/libs/linux/rhel/x86_64/lib/libmsodbcsql-18.5.so.1.1 - " - displayName: 'Uninstall ODBC Driver before running tests in RHEL 9 container' - - - script: | - # Run tests in the container - # Get SQL Server container IP - SQLSERVER_IP=$(docker inspect sqlserver-rhel9 --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}') - echo "SQL Server IP: $SQLSERVER_IP" - - docker exec \ - -e DB_CONNECTION_STRING="Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes" \ - -e DB_PASSWORD="$(DB_PASSWORD)" \ - test-container-rhel9 bash -c " - source myvenv/bin/activate - echo 'Build successful, running tests now on RHEL 9' - echo 'Using connection string: Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=***;TrustServerCertificate=yes' - python main.py - python -m pytest -v --junitxml=test-results-rhel9.xml --cov=. --cov-report=xml:coverage-rhel9.xml --capture=tee-sys --cache-clear - " - displayName: 'Run pytest with coverage in RHEL 9 container' - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - # Copy test results from container to host - docker cp test-container-rhel9:/workspace/test-results-rhel9.xml $(Build.SourcesDirectory)/ - docker cp test-container-rhel9:/workspace/coverage-rhel9.xml $(Build.SourcesDirectory)/ - displayName: 'Copy test results from RHEL 9 container' - condition: always() - - - script: | - # Clean up containers - docker stop test-container-rhel9 || true - docker rm test-container-rhel9 || true - docker stop sqlserver-rhel9 || true - docker rm sqlserver-rhel9 || true - displayName: 'Clean up RHEL 9 containers' - condition: always() - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-results-rhel9.xml' - testRunTitle: 'Publish pytest results on RHEL 9' - -- job: PytestOnLinux_RHEL9_ARM64 - displayName: 'Linux RedHat ARM64' - pool: - vmImage: 'ubuntu-latest' - - steps: - - script: | - # Set up Docker buildx for multi-architecture support - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker buildx create --name multiarch --driver docker-container --use - docker buildx inspect --bootstrap - displayName: 'Setup Docker buildx for ARM64 emulation' - - - script: | - # Create a Docker container for testing on ARM64 - docker run -d --name test-container-rhel9-arm64 \ - --platform linux/arm64 \ - -v $(Build.SourcesDirectory):/workspace \ - -w /workspace \ - --network bridge \ - redhat/ubi9:latest \ - tail -f /dev/null - displayName: 'Create RHEL 9 ARM64 container' - - - script: | - # Start SQL Server container (x86_64 - SQL Server doesn't support ARM64) - docker run -d --name sqlserver-rhel9-arm64 \ - --platform linux/amd64 \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ - -p 1433:1433 \ - mcr.microsoft.com/mssql/server:2022-latest - - # Wait for SQL Server to be ready - echo "Waiting for SQL Server to start..." - for i in {1..60}; do - if docker exec sqlserver-rhel9-arm64 \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - break - fi - echo "Waiting... ($i/60)" - sleep 2 - done - - # Create test database - docker exec sqlserver-rhel9-arm64 \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "CREATE DATABASE TestDB" - displayName: 'Start SQL Server container for RHEL 9 ARM64' - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - # Install dependencies in the RHEL 9 ARM64 container - docker exec test-container-rhel9-arm64 bash -c " - # Enable CodeReady Builder repository for additional packages - dnf update -y - dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm - subscription-manager repos --enable codeready-builder-for-rhel-9-$(arch)-rpms || dnf config-manager --set-enabled ubi-9-codeready-builder - - # Install Python 3.9 (available in RHEL 9 UBI) and development tools - dnf install -y python3 python3-pip python3-devel cmake curl wget gnupg2 glibc-devel kernel-headers - dnf install -y python3-libs python3-debug - dnf install -y gcc gcc-c++ make binutils - dnf install -y cmake - # If that doesn't work, try installing from different repositories - if ! which gcc; then - echo 'Trying alternative gcc installation...' - dnf --enablerepo=ubi-9-codeready-builder install -y gcc gcc-c++ - fi - # Verify installation and architecture - python3 --version - which gcc && which g++ - gcc --version - g++ --version - uname -m - echo 'Architecture:' \$(uname -m) - " - displayName: 'Install basic dependencies in RHEL 9 ARM64 container' - - - script: | - # Verify compiler installation and set environment for RHEL 9 ARM64 - docker exec test-container-rhel9-arm64 bash -c " - # Verify compilers are available - which gcc || echo 'GCC not found' - which g++ || echo 'G++ not found' - gcc --version || echo 'GCC version check failed' - g++ --version || echo 'G++ version check failed' - - # Set compiler environment variables - export CC=/usr/bin/gcc - export CXX=/usr/bin/g++ - echo 'CC set to:' \$CC - echo 'CXX set to:' \$CXX - echo 'Running on architecture:' \$(uname -m) - - # Create a wrapper script to preserve environment - cat > /workspace/setup_env.sh << 'EOF' - #!/bin/bash - export CC=/usr/bin/gcc - export CXX=/usr/bin/g++ - export PATH=/usr/bin:\$PATH - exec \"\$@\" - EOF - chmod +x /workspace/setup_env.sh - " - displayName: 'Verify and configure compilers in RHEL 9 ARM64 container' - - - script: | - # Install ODBC driver in the RHEL 9 ARM64 container - docker exec test-container-rhel9-arm64 bash -c " - # Add Microsoft repository for RHEL 9 - curl -sSL -o /etc/yum.repos.d/msprod.repo https://packages.microsoft.com/config/rhel/9/prod.repo - - # Install the driver (ARM64 version) - ACCEPT_EULA=Y dnf install -y msodbcsql18 - # optional: for bcp and sqlcmd - ACCEPT_EULA=Y dnf install -y mssql-tools18 - # optional: for unixODBC development headers - dnf install -y unixODBC-devel - " - displayName: 'Install ODBC Driver in RHEL 9 ARM64 container' - - - script: | - # Install Python dependencies in the container using virtual environment - docker exec test-container-rhel9-arm64 bash -c " - # Create a virtual environment with Python 3.9 - python3 -m venv myvenv - source myvenv/bin/activate - - # Install dependencies in the virtual environment - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - - # Make the virtual environment globally available - echo 'source myvenv/bin/activate' >> ~/.bashrc - " - displayName: 'Install Python dependencies in RHEL 9 ARM64 container' - - - script: | - # Build pybind bindings in the ARM64 container - docker exec test-container-rhel9-arm64 bash -c " - source myvenv/bin/activate - ls /usr/include/python3.9 - # Set compiler environment variables - export CC=/usr/bin/gcc - export CXX=/usr/bin/g++ - - cd mssql_python/pybind - chmod +x build.sh - ./build.sh - " - displayName: 'Build pybind bindings (.so) in RHEL 9 ARM64 container' - - - script: | - # Uninstall ODBC Driver before running tests - docker exec test-container-rhel9-arm64 bash -c " - dnf remove -y msodbcsql18 mssql-tools18 unixODBC-devel - rm -f /usr/bin/sqlcmd - rm -f /usr/bin/bcp - rm -rf /opt/microsoft/msodbcsql - rm -f /lib64/libodbcinst.so.2 - odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true - echo 'Uninstalled ODBC Driver and cleaned up libraries' - echo 'Verifying arm64 rhel driver library signatures:' - ldd mssql_python/libs/linux/rhel/arm64/lib/libmsodbcsql-18.5.so.1.1 - " - displayName: 'Uninstall ODBC Driver before running tests in RHEL 9 ARM64 container' - - - script: | - # Run tests in the ARM64 container - # Get SQL Server container IP - SQLSERVER_IP=$(docker inspect sqlserver-rhel9-arm64 --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}') - echo "SQL Server IP: $SQLSERVER_IP" - - docker exec \ - -e DB_CONNECTION_STRING="Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes" \ - -e DB_PASSWORD="$(DB_PASSWORD)" \ - test-container-rhel9-arm64 bash -c " - source myvenv/bin/activate - echo 'Build successful, running tests now on RHEL 9 ARM64' - echo 'Architecture:' \$(uname -m) - echo 'Using connection string: Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=***;TrustServerCertificate=yes' - python -m pytest -v --junitxml=test-results-rhel9-arm64.xml --cov=. --cov-report=xml:coverage-rhel9-arm64.xml --capture=tee-sys --cache-clear - " - displayName: 'Run pytest with coverage in RHEL 9 ARM64 container' - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - # Copy test results from container to host - docker cp test-container-rhel9-arm64:/workspace/test-results-rhel9-arm64.xml $(Build.SourcesDirectory)/ - docker cp test-container-rhel9-arm64:/workspace/coverage-rhel9-arm64.xml $(Build.SourcesDirectory)/ - displayName: 'Copy test results from RHEL 9 ARM64 container' - condition: always() - - - script: | - # Clean up containers - docker stop test-container-rhel9-arm64 || true - docker rm test-container-rhel9-arm64 || true - docker stop sqlserver-rhel9-arm64 || true - docker rm sqlserver-rhel9-arm64 || true - displayName: 'Clean up RHEL 9 ARM64 containers' - condition: always() - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-results-rhel9-arm64.xml' - testRunTitle: 'Publish pytest results on RHEL 9 ARM64' - -- job: PytestOnLinux_Alpine - displayName: 'Linux Alpine x86_64' - pool: - vmImage: 'ubuntu-latest' - - steps: - - script: | - # Set up Docker buildx for multi-architecture support - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker buildx create --name multiarch --driver docker-container --use - docker buildx inspect --bootstrap - displayName: 'Setup Docker buildx for multi-architecture support' - - - script: | - # Create a Docker container for testing on x86_64 - docker run -d --name test-container-alpine \ - --platform linux/amd64 \ - -v $(Build.SourcesDirectory):/workspace \ - -w /workspace \ - --network bridge \ - alpine:latest \ - tail -f /dev/null - displayName: 'Create Alpine x86_64 container' - - - script: | - # Start SQL Server container (x86_64) - docker run -d --name sqlserver-alpine \ - --platform linux/amd64 \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ - -p 1433:1433 \ - mcr.microsoft.com/mssql/server:2022-latest - - # Wait for SQL Server to be ready - echo "Waiting for SQL Server to start..." - for i in {1..60}; do - if docker exec sqlserver-alpine \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - break - fi - echo "Waiting... ($i/60)" - sleep 2 - done - - # Create test database - docker exec sqlserver-alpine \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "CREATE DATABASE TestDB" - displayName: 'Start SQL Server container for Alpine x86_64' - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - # Install dependencies in the Alpine x86_64 container - docker exec test-container-alpine sh -c " - # Update package index - apk update - - # Install build tools and system dependencies - apk add --no-cache \ - build-base \ - cmake \ - clang \ - git \ - bash \ - wget \ - curl \ - gnupg \ - unixodbc \ - unixodbc-dev \ - libffi-dev \ - openssl-dev \ - zlib-dev \ - py3-pip \ - python3-dev \ - patchelf - - # Create symlinks for Python compatibility - ln -sf python3 /usr/bin/python || true - ln -sf pip3 /usr/bin/pip || true - - # Verify installation and architecture - uname -m - python --version - which cmake - " - displayName: 'Install basic dependencies in Alpine x86_64 container' - - - script: | - # Install ODBC driver in the Alpine x86_64 container - docker exec test-container-alpine bash -c " - # Detect architecture for ODBC driver download - case \$(uname -m) in - x86_64) architecture='amd64' ;; - arm64|aarch64) architecture='arm64' ;; - *) architecture='unsupported' ;; - esac - - if [[ 'unsupported' == '\$architecture' ]]; then - echo 'Alpine architecture \$(uname -m) is not currently supported.' - exit 1 - fi - - echo 'Detected architecture: '\$architecture - - # Download the packages - curl -O https://download.microsoft.com/download/fae28b9a-d880-42fd-9b98-d779f0fdd77f/msodbcsql18_18.5.1.1-1_\$architecture.apk - curl -O https://download.microsoft.com/download/7/6d/76de322a-d860-4894-9945-f0cc5d6a45f8/mssql-tools18_18.4.1.1-1_\$architecture.apk - - # Download signatures for verification - curl -O https://download.microsoft.com/download/fae28b9a-d880-42fd-9b98-d779f0fdd77f/msodbcsql18_18.5.1.1-1_\$architecture.sig - curl -O https://download.microsoft.com/download/7/6d/76de322a-d860-4894-9945-f0cc5d6a45f8/mssql-tools18_18.4.1.1-1_\$architecture.sig - - # Import Microsoft GPG key and verify packages - curl https://packages.microsoft.com/keys/microsoft.asc | gpg --import - - gpg --verify msodbcsql18_18.5.1.1-1_\$architecture.sig msodbcsql18_18.5.1.1-1_\$architecture.apk - gpg --verify mssql-tools18_18.4.1.1-1_\$architecture.sig mssql-tools18_18.4.1.1-1_\$architecture.apk - - # Install the packages - apk add --allow-untrusted msodbcsql18_18.5.1.1-1_\$architecture.apk - apk add --allow-untrusted mssql-tools18_18.4.1.1-1_\$architecture.apk - - # Cleanup - rm -f msodbcsql18_18.5.1.1-1_\$architecture.* mssql-tools18_18.4.1.1-1_\$architecture.* - - # Add mssql-tools to PATH - export PATH=\"\$PATH:/opt/mssql-tools18/bin\" - echo 'export PATH=\"\$PATH:/opt/mssql-tools18/bin\"' >> ~/.bashrc - " - displayName: 'Install ODBC Driver in Alpine x86_64 container' - - - script: | - # Install Python dependencies in the Alpine x86_64 container using virtual environment - docker exec test-container-alpine bash -c " - # Create virtual environment - python -m venv /workspace/venv - - # Activate virtual environment and install dependencies - source /workspace/venv/bin/activate - - # Upgrade pip and install dependencies - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - - # Verify virtual environment is active - which python - which pip - " - displayName: 'Install Python dependencies in Alpine x86_64 container' - - - script: | - # Build pybind bindings in the Alpine x86_64 container - docker exec test-container-alpine bash -c " - # Activate virtual environment - source /workspace/venv/bin/activate - - cd mssql_python/pybind - chmod +x build.sh - ./build.sh - " - displayName: 'Build pybind bindings (.so) in Alpine x86_64 container' - - - script: | - # Uninstall ODBC Driver before running tests to use bundled libraries - docker exec test-container-alpine bash -c " - # Remove system ODBC installation - apk del msodbcsql18 mssql-tools18 unixodbc-dev || echo 'ODBC packages not installed via apk' - rm -f /usr/bin/sqlcmd - rm -f /usr/bin/bcp - rm -rf /opt/microsoft/msodbcsql18 - rm -f /usr/lib/libodbcinst.so.2 - odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true - echo 'Uninstalled system ODBC Driver and cleaned up libraries' - echo 'Verifying x86_64 alpine driver library signatures:' - ldd mssql_python/libs/linux/alpine/x86_64/lib/libmsodbcsql-18.5.so.1.1 || echo 'Driver library not found' - " - displayName: 'Uninstall system ODBC Driver before running tests in Alpine x86_64 container' - - - script: | - # Run tests in the Alpine x86_64 container - # Get SQL Server container IP - SQLSERVER_IP=$(docker inspect sqlserver-alpine --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}') - echo "SQL Server IP: $SQLSERVER_IP" - - docker exec \ - -e DB_CONNECTION_STRING="Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes" \ - -e DB_PASSWORD="$(DB_PASSWORD)" \ - test-container-alpine bash -c " - echo 'Build successful, running tests now on Alpine x86_64' - echo 'Architecture:' \$(uname -m) - echo 'Alpine version:' \$(cat /etc/alpine-release) - echo 'Using connection string: Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=***;TrustServerCertificate=yes' - - # Activate virtual environment - source /workspace/venv/bin/activate - - # Test basic Python import first - python -c 'import mssql_python; print(\"mssql_python imported successfully\")' - - # Run main.py if it exists - if [ -f main.py ]; then - echo 'Running main.py...' - python main.py - fi - - # Run pytest - python -m pytest -v --junitxml=test-results-alpine.xml --cov=. --cov-report=xml:coverage-alpine.xml --capture=tee-sys --cache-clear - " - displayName: 'Run pytest with coverage in Alpine x86_64 container' - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - # Copy test results from container to host - docker cp test-container-alpine:/workspace/test-results-alpine.xml $(Build.SourcesDirectory)/ || echo 'Failed to copy test results' - docker cp test-container-alpine:/workspace/coverage-alpine.xml $(Build.SourcesDirectory)/ || echo 'Failed to copy coverage results' - displayName: 'Copy test results from Alpine x86_64 container' - condition: always() - - - script: | - # Clean up containers - docker stop test-container-alpine || true - docker rm test-container-alpine || true - docker stop sqlserver-alpine || true - docker rm sqlserver-alpine || true - displayName: 'Clean up Alpine x86_64 containers' - condition: always() - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-results-alpine.xml' - testRunTitle: 'Publish pytest results on Alpine x86_64' - -- job: PytestOnLinux_Alpine_ARM64 - displayName: 'Linux Alpine ARM64' - pool: - vmImage: 'ubuntu-latest' - - steps: - - script: | - # Set up Docker buildx for multi-architecture support - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker buildx create --name multiarch --driver docker-container --use - docker buildx inspect --bootstrap - displayName: 'Setup Docker buildx for ARM64 emulation' - - - script: | - # Create a Docker container for testing on ARM64 - docker run -d --name test-container-alpine-arm64 \ - --platform linux/arm64 \ - -v $(Build.SourcesDirectory):/workspace \ - -w /workspace \ - --network bridge \ - alpine:latest \ - tail -f /dev/null - displayName: 'Create Alpine ARM64 container' - - - script: | - # Start SQL Server container (x86_64 - SQL Server doesn't support ARM64) - docker run -d --name sqlserver-alpine-arm64 \ - --platform linux/amd64 \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ - -p 1433:1433 \ - mcr.microsoft.com/mssql/server:2022-latest - - # Wait for SQL Server to be ready - echo "Waiting for SQL Server to start..." - for i in {1..60}; do - if docker exec sqlserver-alpine-arm64 \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - break - fi - echo "Waiting... ($i/60)" - sleep 2 - done - - # Create test database - docker exec sqlserver-alpine-arm64 \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "CREATE DATABASE TestDB" - displayName: 'Start SQL Server container for Alpine ARM64' - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - # Install dependencies in the Alpine ARM64 container - docker exec test-container-alpine-arm64 sh -c " - # Update package index - apk update - - # Install build tools and system dependencies - apk add --no-cache \ - build-base \ - cmake \ - clang \ - git \ - bash \ - wget \ - curl \ - gnupg \ - unixodbc \ - unixodbc-dev \ - libffi-dev \ - openssl-dev \ - zlib-dev \ - py3-pip \ - python3-dev \ - patchelf - - # Create symlinks for Python compatibility - ln -sf python3 /usr/bin/python || true - ln -sf pip3 /usr/bin/pip || true - - # Verify installation and architecture - uname -m - python --version - which cmake - " - displayName: 'Install basic dependencies in Alpine ARM64 container' - - - script: | - # Install ODBC driver in the Alpine ARM64 container - docker exec test-container-alpine-arm64 bash -c " - # Detect architecture for ODBC driver download - case \$(uname -m) in - x86_64) architecture='amd64' ;; - arm64|aarch64) architecture='arm64' ;; - *) architecture='unsupported' ;; - esac - - if [[ 'unsupported' == '\$architecture' ]]; then - echo 'Alpine architecture \$(uname -m) is not currently supported.' - exit 1 - fi - - echo 'Detected architecture: '\$architecture - - # Download the packages - curl -O https://download.microsoft.com/download/fae28b9a-d880-42fd-9b98-d779f0fdd77f/msodbcsql18_18.5.1.1-1_\$architecture.apk - curl -O https://download.microsoft.com/download/7/6d/76de322a-d860-4894-9945-f0cc5d6a45f8/mssql-tools18_18.4.1.1-1_\$architecture.apk - - # Download signatures for verification - curl -O https://download.microsoft.com/download/fae28b9a-d880-42fd-9b98-d779f0fdd77f/msodbcsql18_18.5.1.1-1_\$architecture.sig - curl -O https://download.microsoft.com/download/7/6d/76de322a-d860-4894-9945-f0cc5d6a45f8/mssql-tools18_18.4.1.1-1_\$architecture.sig - - # Import Microsoft GPG key and verify packages - curl https://packages.microsoft.com/keys/microsoft.asc | gpg --import - - gpg --verify msodbcsql18_18.5.1.1-1_\$architecture.sig msodbcsql18_18.5.1.1-1_\$architecture.apk - gpg --verify mssql-tools18_18.4.1.1-1_\$architecture.sig mssql-tools18_18.4.1.1-1_\$architecture.apk - - # Install the packages - apk add --allow-untrusted msodbcsql18_18.5.1.1-1_\$architecture.apk - apk add --allow-untrusted mssql-tools18_18.4.1.1-1_\$architecture.apk - - # Cleanup - rm -f msodbcsql18_18.5.1.1-1_\$architecture.* mssql-tools18_18.4.1.1-1_\$architecture.* - - # Add mssql-tools to PATH - export PATH=\"\$PATH:/opt/mssql-tools18/bin\" - echo 'export PATH=\"\$PATH:/opt/mssql-tools18/bin\"' >> ~/.bashrc - " - displayName: 'Install ODBC Driver in Alpine ARM64 container' - - - script: | - # Install Python dependencies in the Alpine ARM64 container using virtual environment - docker exec test-container-alpine-arm64 bash -c " - # Create virtual environment - python -m venv /workspace/venv - - # Activate virtual environment and install dependencies - source /workspace/venv/bin/activate - - # Upgrade pip and install dependencies - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - - # Verify virtual environment is active - which python - which pip - " - displayName: 'Install Python dependencies in Alpine ARM64 container' - - - script: | - # Build pybind bindings in the Alpine ARM64 container - docker exec test-container-alpine-arm64 bash -c " - # Activate virtual environment - source /workspace/venv/bin/activate - - cd mssql_python/pybind - chmod +x build.sh - ./build.sh - " - displayName: 'Build pybind bindings (.so) in Alpine ARM64 container' - - - script: | - # Uninstall ODBC Driver before running tests to use bundled libraries - docker exec test-container-alpine-arm64 bash -c " - # Remove system ODBC installation - apk del msodbcsql18 mssql-tools18 unixodbc-dev || echo 'ODBC packages not installed via apk' - rm -f /usr/bin/sqlcmd - rm -f /usr/bin/bcp - rm -rf /opt/microsoft/msodbcsql18 - rm -f /usr/lib/libodbcinst.so.2 - odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true - echo 'Uninstalled system ODBC Driver and cleaned up libraries' - echo 'Verifying arm64 alpine driver library signatures:' - ldd mssql_python/libs/linux/alpine/arm64/lib/libmsodbcsql-18.5.so.1.1 || echo 'Driver library not found' - " - displayName: 'Uninstall system ODBC Driver before running tests in Alpine ARM64 container' - - - script: | - # Run tests in the Alpine ARM64 container - # Get SQL Server container IP - SQLSERVER_IP=$(docker inspect sqlserver-alpine-arm64 --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}') - echo "SQL Server IP: $SQLSERVER_IP" - - docker exec \ - -e DB_CONNECTION_STRING="Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes" \ - -e DB_PASSWORD="$(DB_PASSWORD)" \ - test-container-alpine-arm64 bash -c " - echo 'Build successful, running tests now on Alpine ARM64' - echo 'Architecture:' \$(uname -m) - echo 'Alpine version:' \$(cat /etc/alpine-release) - echo 'Using connection string: Server=$SQLSERVER_IP;Database=TestDB;Uid=SA;Pwd=***;TrustServerCertificate=yes' - - # Activate virtual environment - source /workspace/venv/bin/activate - - # Test basic Python import first - python -c 'import mssql_python; print(\"mssql_python imported successfully\")' - - # Run main.py if it exists - if [ -f main.py ]; then - echo 'Running main.py...' - python main.py - fi - - # Run pytest - python -m pytest -v --junitxml=test-results-alpine-arm64.xml --cov=. --cov-report=xml:coverage-alpine-arm64.xml --capture=tee-sys --cache-clear - " - displayName: 'Run pytest with coverage in Alpine ARM64 container' - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - # Copy test results from container to host - docker cp test-container-alpine-arm64:/workspace/test-results-alpine-arm64.xml $(Build.SourcesDirectory)/ || echo 'Failed to copy test results' - docker cp test-container-alpine-arm64:/workspace/coverage-alpine-arm64.xml $(Build.SourcesDirectory)/ || echo 'Failed to copy coverage results' - displayName: 'Copy test results from Alpine ARM64 container' - condition: always() - - - script: | - # Clean up containers - docker stop test-container-alpine-arm64 || true - docker rm test-container-alpine-arm64 || true - docker stop sqlserver-alpine-arm64 || true - docker rm sqlserver-alpine-arm64 || true - displayName: 'Clean up Alpine ARM64 containers' - condition: always() - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-results-alpine-arm64.xml' - testRunTitle: 'Publish pytest results on Alpine ARM64' - -- job: CodeCoverageReport - displayName: 'Full Code Coverage Report in Ubuntu x86_64' - pool: - vmImage: 'ubuntu-latest' - - steps: - - script: | - # Install build dependencies - sudo apt-get update - sudo apt-get install -y cmake gcc g++ lcov unixodbc-dev llvm clang - displayName: 'Install build dependencies' - - - script: | - # Start SQL Server container - docker pull mcr.microsoft.com/mssql/server:2022-latest - docker run \ - --name sqlserver \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ - -p 1433:1433 \ - -d mcr.microsoft.com/mssql/server:2022-latest - - # Wait until SQL Server is ready - for i in {1..30}; do - docker exec sqlserver \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" && break - sleep 2 - done - displayName: 'Start SQL Server container' - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - # Install Python dependencies - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install coverage-lcov lcov-cobertura - displayName: 'Install Python dependencies' - - - script: | - # Build pybind bindings with coverage instrumentation - cd mssql_python/pybind - ./build.sh codecov - displayName: 'Build pybind bindings with coverage' - - - script: | - # Generate unified coverage (Python + C++) - chmod +x ./generate_codecov.sh - ./generate_codecov.sh - - # Convert unified LCOV to Cobertura XML for ADO reporting - lcov_cobertura total.info --output unified-coverage/coverage.xml - displayName: 'Generate unified coverage (Python + C++)' - env: - DB_CONNECTION_STRING: 'Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' - DB_PASSWORD: $(DB_PASSWORD) - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Publish pytest results with unified coverage' - - - task: PublishCodeCoverageResults@2 - condition: succeededOrFailed() - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: 'unified-coverage/coverage.xml' - reportDirectory: 'unified-coverage' - failIfCoverageEmpty: true - displayName: 'Publish unified code coverage results' From b52e0e2beea4de09e993a14b5ff8b70369df44e3 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 01:45:06 +0530 Subject: [PATCH 03/24] proper --- eng/pipelines/pr-validation-pipeline.yml | 140 ++++++++++++++++++----- 1 file changed, 111 insertions(+), 29 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index ab817d27..5493766a 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -60,16 +60,38 @@ jobs: env: DB_PASSWORD: $(DB_PASSWORD) - - task: DownloadPipelineArtifact@2 + - powershell: | + # Download Python 3.14 ARM64 libs from python.org + $pythonVersion = "3.14.0a2" # Update to latest alpha/beta/rc as available + $url = "https://www.python.org/ftp/python/$pythonVersion/python-$pythonVersion-embed-arm64.zip" + $output = "$(Build.SourcesDirectory)\python314-arm64-embed.zip" + + Write-Host "Downloading Python 3.14 ARM64 embed from: $url" + try { + Invoke-WebRequest -Uri $url -OutFile $output -UseBasicParsing + + # Extract to libs directory + $destPath = "$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64" + New-Item -ItemType Directory -Force -Path $destPath + Expand-Archive -Path $output -DestinationPath $destPath -Force + + Write-Host "Successfully downloaded and extracted Python 3.14 ARM64 libs" + ls $destPath + } catch { + Write-Warning "Could not download Python 3.14 ARM64 embed package. Will try alternative sources." + # Alternative: Download from GitHub releases if available + Write-Host "Attempting alternative download from GitHub..." + $ghUrl = "https://github.com/python/cpython/releases/download/v$pythonVersion/python-$pythonVersion-embed-arm64.zip" + try { + Invoke-WebRequest -Uri $ghUrl -OutFile $output -UseBasicParsing + Expand-Archive -Path $output -DestinationPath $destPath -Force + Write-Host "Successfully downloaded from GitHub" + } catch { + Write-Warning "ARM64 libs not available. Build may fail for ARM64." + } + } condition: eq(variables['targetArch'], 'arm64') - inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - definition: 2162 - buildVersionToDownload: 'latest' - artifactName: 'mssql-python-arm64-libs' - targetPath: '$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64' - displayName: 'Download ARM64 Python libs' + displayName: 'Download Python 3.14 ARM64 libs from python.org' - script: | echo "Python Version: $(pythonVersion)" @@ -545,38 +567,74 @@ jobs: fi if [[ "$(BASE_IMAGE)" == alpine* ]]; then - echo "Setting up Alpine Linux..." + echo "Setting up Alpine Linux with Python 3.14..." docker exec test-$(ARCH) sh -c " - apk update && apk add --no-cache python3 py3-pip python3-dev unixodbc-dev curl libtool libltdl krb5-libs + apk update && apk add --no-cache build-base openssl-dev zlib-dev libffi-dev readline-dev sqlite-dev bzip2-dev xz-dev unixodbc-dev curl wget tar gzip libtool libltdl krb5-libs + + # Download and build Python 3.14 + cd /tmp + wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a2.tar.xz || wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a1.tar.xz || echo 'Python 3.14 not available, will try git' + + if [ -f Python-3.14.0a*.tar.xz ]; then + tar -xf Python-3.14.0a*.tar.xz + cd Python-3.14.0a* + ./configure --prefix=/usr/local --enable-optimizations --with-ensurepip=install + make -j\$(nproc) + make install + ln -sf /usr/local/bin/python3.14 /usr/local/bin/python3 + else + echo 'Building from git source...' + apk add --no-cache git + git clone --depth 1 --branch v3.14.0a2 https://github.com/python/cpython.git || git clone --depth 1 --branch main https://github.com/python/cpython.git + cd cpython + ./configure --prefix=/usr/local --enable-optimizations --with-ensurepip=install + make -j\$(nproc) + make install + ln -sf /usr/local/bin/python3.14 /usr/local/bin/python3 + fi + + python3 --version python3 -m venv /venv /venv/bin/pip install pytest " PY_CMD="/venv/bin/python" elif [[ "$(BASE_IMAGE)" == *ubi* ]] || [[ "$(BASE_IMAGE)" == *rocky* ]] || [[ "$(BASE_IMAGE)" == *alma* ]]; then - echo "Setting up RHEL-based system..." + echo "Setting up RHEL-based system with Python 3.14..." docker exec test-$(ARCH) bash -c " set -euo pipefail - echo 'Installing Python on UBI/RHEL...' + echo 'Installing Python 3.14 on UBI/RHEL...' if command -v dnf >/dev/null; then dnf clean all rm -rf /var/cache/dnf dnf -y makecache - - dnf list --showduplicates python3.11 python3.12 || true - - if dnf -y install python3.12 python3.12-pip unixODBC-devel; then - PY=python3.12 - echo 'Installed Python 3.12' - elif dnf -y install python3.11 python3.11-pip unixODBC-devel; then - PY=python3.11 - echo 'Installed Python 3.11' + + # Install build dependencies + dnf -y install gcc gcc-c++ make openssl-devel zlib-devel libffi-devel readline-devel sqlite-devel bzip2-devel xz-devel unixODBC-devel wget tar gzip + + # Download and build Python 3.14 + cd /tmp + wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a2.tar.xz || wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a1.tar.xz || echo 'Python 3.14 not available, will try git' + + if [ -f Python-3.14.0a*.tar.xz ]; then + tar -xf Python-3.14.0a*.tar.xz + cd Python-3.14.0a* + ./configure --prefix=/usr/local --enable-optimizations --with-ensurepip=install + make -j\$(nproc) + make install + ln -sf /usr/local/bin/python3.14 /usr/local/bin/python3 else - dnf -y install python3 python3-pip unixODBC-devel - PY=python3 - echo 'Falling back to default Python' + echo 'Building from git source...' + dnf -y install git + git clone --depth 1 --branch v3.14.0a2 https://github.com/python/cpython.git || git clone --depth 1 --branch main https://github.com/python/cpython.git + cd cpython + ./configure --prefix=/usr/local --enable-optimizations --with-ensurepip=install + make -j\$(nproc) + make install + ln -sf /usr/local/bin/python3.14 /usr/local/bin/python3 fi - - \$PY -m venv /venv + + python3 --version + python3 -m venv /venv /venv/bin/python -m pip install -U 'pip>=25' pytest /venv/bin/python --version /venv/bin/pip --version @@ -587,11 +645,35 @@ jobs: " PY_CMD="/venv/bin/python" else - echo "Setting up Debian/Ubuntu..." + echo "Setting up Debian/Ubuntu with Python 3.14..." docker exec test-$(ARCH) bash -c " export DEBIAN_FRONTEND=noninteractive apt-get update - apt-get install -y python3 python3-pip python3-venv python3-full unixodbc-dev curl + apt-get install -y build-essential libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev liblzma-dev tk-dev libffi-dev unixodbc-dev curl wget tar gzip + + # Download and build Python 3.14 + cd /tmp + wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a2.tar.xz || wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a1.tar.xz || echo 'Python 3.14 not available, will try git' + + if [ -f Python-3.14.0a*.tar.xz ]; then + tar -xf Python-3.14.0a*.tar.xz + cd Python-3.14.0a* + ./configure --prefix=/usr/local --enable-optimizations --with-ensurepip=install + make -j\$(nproc) + make install + ln -sf /usr/local/bin/python3.14 /usr/local/bin/python3 + else + echo 'Building from git source...' + apt-get install -y git + git clone --depth 1 --branch v3.14.0a2 https://github.com/python/cpython.git || git clone --depth 1 --branch main https://github.com/python/cpython.git + cd cpython + ./configure --prefix=/usr/local --enable-optimizations --with-ensurepip=install + make -j\$(nproc) + make install + ln -sf /usr/local/bin/python3.14 /usr/local/bin/python3 + fi + + python3 --version python3 -m venv /venv /venv/bin/pip install pytest " From 5b881c5d5230ee7e6ef15405c09104f77659d984 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 09:11:49 +0530 Subject: [PATCH 04/24] fixes --- eng/pipelines/pr-validation-pipeline.yml | 89 +++++++++--------------- 1 file changed, 34 insertions(+), 55 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 5493766a..570d6342 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -569,43 +569,36 @@ jobs: if [[ "$(BASE_IMAGE)" == alpine* ]]; then echo "Setting up Alpine Linux with Python 3.14..." docker exec test-$(ARCH) sh -c " + set -ex apk update && apk add --no-cache build-base openssl-dev zlib-dev libffi-dev readline-dev sqlite-dev bzip2-dev xz-dev unixodbc-dev curl wget tar gzip libtool libltdl krb5-libs # Download and build Python 3.14 cd /tmp - wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a2.tar.xz || wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a1.tar.xz || echo 'Python 3.14 not available, will try git' + wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a2.tar.xz || echo 'Could not download a2' - if [ -f Python-3.14.0a*.tar.xz ]; then - tar -xf Python-3.14.0a*.tar.xz - cd Python-3.14.0a* - ./configure --prefix=/usr/local --enable-optimizations --with-ensurepip=install + if [ -f Python-3.14.0a2.tar.xz ]; then + tar -xf Python-3.14.0a2.tar.xz + cd /tmp/Python-3.14.0a2 + ./configure --prefix=/usr/local --with-ensurepip=install make -j\$(nproc) make install ln -sf /usr/local/bin/python3.14 /usr/local/bin/python3 + python3 --version + python3 -m venv /venv + /venv/bin/pip install pytest else - echo 'Building from git source...' - apk add --no-cache git - git clone --depth 1 --branch v3.14.0a2 https://github.com/python/cpython.git || git clone --depth 1 --branch main https://github.com/python/cpython.git - cd cpython - ./configure --prefix=/usr/local --enable-optimizations --with-ensurepip=install - make -j\$(nproc) - make install - ln -sf /usr/local/bin/python3.14 /usr/local/bin/python3 + echo 'ERROR: Could not download Python 3.14' + exit 1 fi - - python3 --version - python3 -m venv /venv - /venv/bin/pip install pytest " PY_CMD="/venv/bin/python" elif [[ "$(BASE_IMAGE)" == *ubi* ]] || [[ "$(BASE_IMAGE)" == *rocky* ]] || [[ "$(BASE_IMAGE)" == *alma* ]]; then echo "Setting up RHEL-based system with Python 3.14..." docker exec test-$(ARCH) bash -c " - set -euo pipefail + set -ex echo 'Installing Python 3.14 on UBI/RHEL...' if command -v dnf >/dev/null; then dnf clean all - rm -rf /var/cache/dnf dnf -y makecache # Install build dependencies @@ -613,31 +606,23 @@ jobs: # Download and build Python 3.14 cd /tmp - wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a2.tar.xz || wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a1.tar.xz || echo 'Python 3.14 not available, will try git' + wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a2.tar.xz || echo 'Could not download a2' - if [ -f Python-3.14.0a*.tar.xz ]; then - tar -xf Python-3.14.0a*.tar.xz - cd Python-3.14.0a* - ./configure --prefix=/usr/local --enable-optimizations --with-ensurepip=install + if [ -f Python-3.14.0a2.tar.xz ]; then + tar -xf Python-3.14.0a2.tar.xz + cd /tmp/Python-3.14.0a2 + ./configure --prefix=/usr/local --with-ensurepip=install make -j\$(nproc) make install ln -sf /usr/local/bin/python3.14 /usr/local/bin/python3 + python3 --version + python3 -m venv /venv + /venv/bin/python -m pip install -U pip pytest + /venv/bin/python --version else - echo 'Building from git source...' - dnf -y install git - git clone --depth 1 --branch v3.14.0a2 https://github.com/python/cpython.git || git clone --depth 1 --branch main https://github.com/python/cpython.git - cd cpython - ./configure --prefix=/usr/local --enable-optimizations --with-ensurepip=install - make -j\$(nproc) - make install - ln -sf /usr/local/bin/python3.14 /usr/local/bin/python3 + echo 'ERROR: Could not download Python 3.14' + exit 1 fi - - python3 --version - python3 -m venv /venv - /venv/bin/python -m pip install -U 'pip>=25' pytest - /venv/bin/python --version - /venv/bin/pip --version else echo 'ERROR: dnf not found' exit 1 @@ -647,35 +632,29 @@ jobs: else echo "Setting up Debian/Ubuntu with Python 3.14..." docker exec test-$(ARCH) bash -c " + set -ex export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y build-essential libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev liblzma-dev tk-dev libffi-dev unixodbc-dev curl wget tar gzip # Download and build Python 3.14 cd /tmp - wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a2.tar.xz || wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a1.tar.xz || echo 'Python 3.14 not available, will try git' + wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a2.tar.xz || echo 'Could not download a2' - if [ -f Python-3.14.0a*.tar.xz ]; then - tar -xf Python-3.14.0a*.tar.xz - cd Python-3.14.0a* - ./configure --prefix=/usr/local --enable-optimizations --with-ensurepip=install + if [ -f Python-3.14.0a2.tar.xz ]; then + tar -xf Python-3.14.0a2.tar.xz + cd /tmp/Python-3.14.0a2 + ./configure --prefix=/usr/local --with-ensurepip=install make -j\$(nproc) make install ln -sf /usr/local/bin/python3.14 /usr/local/bin/python3 + python3 --version + python3 -m venv /venv + /venv/bin/pip install pytest else - echo 'Building from git source...' - apt-get install -y git - git clone --depth 1 --branch v3.14.0a2 https://github.com/python/cpython.git || git clone --depth 1 --branch main https://github.com/python/cpython.git - cd cpython - ./configure --prefix=/usr/local --enable-optimizations --with-ensurepip=install - make -j\$(nproc) - make install - ln -sf /usr/local/bin/python3.14 /usr/local/bin/python3 + echo 'ERROR: Could not download Python 3.14' + exit 1 fi - - python3 --version - python3 -m venv /venv - /venv/bin/pip install pytest " PY_CMD="/venv/bin/python" fi From 75a8113ac2c5e462e03ee6b4c7ae2102bca16a9d Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 09:22:50 +0530 Subject: [PATCH 05/24] fix download links --- eng/pipelines/pr-validation-pipeline.yml | 51 ++++++++++++++---------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 570d6342..80aeca26 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -61,37 +61,44 @@ jobs: DB_PASSWORD: $(DB_PASSWORD) - powershell: | - # Download Python 3.14 ARM64 libs from python.org - $pythonVersion = "3.14.0a2" # Update to latest alpha/beta/rc as available - $url = "https://www.python.org/ftp/python/$pythonVersion/python-$pythonVersion-embed-arm64.zip" + # Download Python 3.14 ARM64 embed package from python.org + # Note: The FTP path uses the base version (3.14.0) not the full version (3.14.0a2) + $url = "https://www.python.org/ftp/python/3.14.0/python-3.14.0a2-embed-arm64.zip" $output = "$(Build.SourcesDirectory)\python314-arm64-embed.zip" + $destPath = "$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64" + + Write-Host "Downloading Python 3.14 ARM64 embed package from: $url" - Write-Host "Downloading Python 3.14 ARM64 embed from: $url" try { - Invoke-WebRequest -Uri $url -OutFile $output -UseBasicParsing + # Download the embed package + Invoke-WebRequest -Uri $url -OutFile $output -UseBasicParsing -ErrorAction Stop + Write-Host "Download successful! File size: $((Get-Item $output).Length / 1MB) MB" + + # Create destination directory + New-Item -ItemType Directory -Force -Path $destPath | Out-Null - # Extract to libs directory - $destPath = "$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64" - New-Item -ItemType Directory -Force -Path $destPath + # Extract the embed package Expand-Archive -Path $output -DestinationPath $destPath -Force - Write-Host "Successfully downloaded and extracted Python 3.14 ARM64 libs" - ls $destPath - } catch { - Write-Warning "Could not download Python 3.14 ARM64 embed package. Will try alternative sources." - # Alternative: Download from GitHub releases if available - Write-Host "Attempting alternative download from GitHub..." - $ghUrl = "https://github.com/python/cpython/releases/download/v$pythonVersion/python-$pythonVersion-embed-arm64.zip" - try { - Invoke-WebRequest -Uri $ghUrl -OutFile $output -UseBasicParsing - Expand-Archive -Path $output -DestinationPath $destPath -Force - Write-Host "Successfully downloaded from GitHub" - } catch { - Write-Warning "ARM64 libs not available. Build may fail for ARM64." + Write-Host "Successfully extracted Python 3.14 ARM64 libs to: $destPath" + Write-Host "Contents:" + Get-ChildItem $destPath | ForEach-Object { Write-Host " - $($_.Name)" } + + # Verify python314.dll exists + if (Test-Path "$destPath\python314.dll") { + Write-Host "✓ python314.dll found" + } else { + Write-Warning "python314.dll not found in extracted files" } + + } catch { + Write-Error "Failed to download Python 3.14 ARM64 embed package: $_" + Write-Host "URL attempted: $url" + Write-Host "This will likely cause the ARM64 build to fail." + exit 1 } condition: eq(variables['targetArch'], 'arm64') - displayName: 'Download Python 3.14 ARM64 libs from python.org' + displayName: 'Download Python 3.14 ARM64 embed package' - script: | echo "Python Version: $(pythonVersion)" From 56ab26856a1cf842a5342a2be83c89d1ef1f6acc Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 09:42:23 +0530 Subject: [PATCH 06/24] fix python libs --- eng/pipelines/pr-validation-pipeline.yml | 71 ++++++++++++++++-------- 1 file changed, 48 insertions(+), 23 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 80aeca26..ed88379a 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -61,44 +61,69 @@ jobs: DB_PASSWORD: $(DB_PASSWORD) - powershell: | - # Download Python 3.14 ARM64 embed package from python.org - # Note: The FTP path uses the base version (3.14.0) not the full version (3.14.0a2) - $url = "https://www.python.org/ftp/python/3.14.0/python-3.14.0a2-embed-arm64.zip" - $output = "$(Build.SourcesDirectory)\python314-arm64-embed.zip" + # Download Python 3.14 ARM64 installer and extract development files + # The installer contains python314.lib needed for linking + $installerUrl = "https://www.python.org/ftp/python/3.14.0/python-3.14.0a2-arm64.exe" + $installerPath = "$(Build.SourcesDirectory)\python314-arm64-installer.exe" + $extractPath = "$(Build.SourcesDirectory)\python314-arm64-extracted" $destPath = "$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64" - Write-Host "Downloading Python 3.14 ARM64 embed package from: $url" + Write-Host "Downloading Python 3.14 ARM64 installer from: $installerUrl" try { - # Download the embed package - Invoke-WebRequest -Uri $url -OutFile $output -UseBasicParsing -ErrorAction Stop - Write-Host "Download successful! File size: $((Get-Item $output).Length / 1MB) MB" + # Download the installer + Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing -ErrorAction Stop + Write-Host "Download successful! File size: $((Get-Item $installerPath).Length / 1MB) MB" - # Create destination directory + # Create directories + New-Item -ItemType Directory -Force -Path $extractPath | Out-Null New-Item -ItemType Directory -Force -Path $destPath | Out-Null - # Extract the embed package - Expand-Archive -Path $output -DestinationPath $destPath -Force + # Extract the installer (it's a self-extracting archive) + Write-Host "Extracting installer contents..." + & $installerPath /quiet /layout $extractPath - Write-Host "Successfully extracted Python 3.14 ARM64 libs to: $destPath" - Write-Host "Contents:" + # Wait for extraction to complete + Start-Sleep -Seconds 10 + + # Find and copy the libs directory (contains python314.lib) + $libsSource = Get-ChildItem -Path $extractPath -Recurse -Directory -Filter "libs" | Select-Object -First 1 + if ($libsSource) { + Write-Host "Found libs directory at: $($libsSource.FullName)" + Copy-Item -Path "$($libsSource.FullName)\*" -Destination $destPath -Recurse -Force + } + + # Also get the DLLs from the installer + $dllFiles = Get-ChildItem -Path $extractPath -Recurse -Filter "*.dll" + foreach ($dll in $dllFiles | Where-Object { $_.Name -like "python*" -or $_.Name -like "vcruntime*" }) { + Copy-Item -Path $dll.FullName -Destination $destPath -Force + Write-Host "Copied: $($dll.Name)" + } + + Write-Host "`nContents of $destPath after extraction:" Get-ChildItem $destPath | ForEach-Object { Write-Host " - $($_.Name)" } - # Verify python314.dll exists - if (Test-Path "$destPath\python314.dll") { - Write-Host "✓ python314.dll found" + # Verify python314.lib exists + if (Test-Path "$destPath\python314.lib") { + Write-Host "`n✓ python314.lib found (needed for linking)" } else { - Write-Warning "python314.dll not found in extracted files" + Write-Error "python314.lib not found! Build will fail." + Write-Host "Available files in libs directory:" + Get-ChildItem $destPath + exit 1 } + # Cleanup + Remove-Item -Path $installerPath -Force -ErrorAction SilentlyContinue + Remove-Item -Path $extractPath -Recurse -Force -ErrorAction SilentlyContinue + } catch { - Write-Error "Failed to download Python 3.14 ARM64 embed package: $_" - Write-Host "URL attempted: $url" - Write-Host "This will likely cause the ARM64 build to fail." + Write-Error "Failed to download/extract Python 3.14 ARM64 installer: $_" + Write-Host "URL attempted: $installerUrl" exit 1 } condition: eq(variables['targetArch'], 'arm64') - displayName: 'Download Python 3.14 ARM64 embed package' + displayName: 'Download Python 3.14 ARM64 development files' - script: | echo "Python Version: $(pythonVersion)" @@ -608,8 +633,8 @@ jobs: dnf clean all dnf -y makecache - # Install build dependencies - dnf -y install gcc gcc-c++ make openssl-devel zlib-devel libffi-devel readline-devel sqlite-devel bzip2-devel xz-devel unixODBC-devel wget tar gzip + # Install build dependencies (readline-devel may not be in UBI, but ncurses-devel provides similar functionality) + dnf -y install gcc gcc-c++ make openssl-devel zlib-devel libffi-devel ncurses-devel sqlite-devel bzip2-devel xz-devel unixODBC-devel wget tar gzip # Download and build Python 3.14 cd /tmp From 59280f22f10a9e045befd37d0619fe374179fbbc Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 10:08:22 +0530 Subject: [PATCH 07/24] added ubuntu directand fix windows arm64 builds --- eng/pipelines/pr-validation-pipeline.yml | 148 +++++++++++++++-------- 1 file changed, 99 insertions(+), 49 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index ed88379a..a6f1e550 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -60,70 +60,47 @@ jobs: env: DB_PASSWORD: $(DB_PASSWORD) + - task: UsePythonVersion@0 + condition: eq(variables['targetArch'], 'arm64') + inputs: + versionSpec: '3.14' + architecture: 'arm64' + addToPath: false + githubToken: $(GITHUB_TOKEN) + displayName: 'Download Python 3.14 ARM64 for development files' + - powershell: | - # Download Python 3.14 ARM64 installer and extract development files - # The installer contains python314.lib needed for linking - $installerUrl = "https://www.python.org/ftp/python/3.14.0/python-3.14.0a2-arm64.exe" - $installerPath = "$(Build.SourcesDirectory)\python314-arm64-installer.exe" - $extractPath = "$(Build.SourcesDirectory)\python314-arm64-extracted" - $destPath = "$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64" - - Write-Host "Downloading Python 3.14 ARM64 installer from: $installerUrl" + # Copy Python 3.14 ARM64 libs from the installed Python + $pythonArm64 = Get-ChildItem "C:\hostedtoolcache\windows\Python\3.14*\arm64" -Directory | Select-Object -First 1 - try { - # Download the installer - Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing -ErrorAction Stop - Write-Host "Download successful! File size: $((Get-Item $installerPath).Length / 1MB) MB" - - # Create directories - New-Item -ItemType Directory -Force -Path $extractPath | Out-Null - New-Item -ItemType Directory -Force -Path $destPath | Out-Null + if ($pythonArm64) { + $libsSource = Join-Path $pythonArm64.FullName "libs" + $destPath = "$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64" - # Extract the installer (it's a self-extracting archive) - Write-Host "Extracting installer contents..." - & $installerPath /quiet /layout $extractPath + Write-Host "Found Python 3.14 ARM64 at: $($pythonArm64.FullName)" + Write-Host "Copying libs from: $libsSource" - # Wait for extraction to complete - Start-Sleep -Seconds 10 - - # Find and copy the libs directory (contains python314.lib) - $libsSource = Get-ChildItem -Path $extractPath -Recurse -Directory -Filter "libs" | Select-Object -First 1 - if ($libsSource) { - Write-Host "Found libs directory at: $($libsSource.FullName)" - Copy-Item -Path "$($libsSource.FullName)\*" -Destination $destPath -Recurse -Force - } + New-Item -ItemType Directory -Force -Path $destPath | Out-Null + Copy-Item -Path "$libsSource\*" -Destination $destPath -Recurse -Force - # Also get the DLLs from the installer - $dllFiles = Get-ChildItem -Path $extractPath -Recurse -Filter "*.dll" - foreach ($dll in $dllFiles | Where-Object { $_.Name -like "python*" -or $_.Name -like "vcruntime*" }) { - Copy-Item -Path $dll.FullName -Destination $destPath -Force - Write-Host "Copied: $($dll.Name)" - } + # Also copy DLLs + Copy-Item -Path "$($pythonArm64.FullName)\*.dll" -Destination $destPath -Force -ErrorAction SilentlyContinue - Write-Host "`nContents of $destPath after extraction:" + Write-Host "`nContents of $destPath :" Get-ChildItem $destPath | ForEach-Object { Write-Host " - $($_.Name)" } - # Verify python314.lib exists if (Test-Path "$destPath\python314.lib") { - Write-Host "`n✓ python314.lib found (needed for linking)" + Write-Host "`n✓ python314.lib found" } else { - Write-Error "python314.lib not found! Build will fail." - Write-Host "Available files in libs directory:" - Get-ChildItem $destPath + Write-Error "python314.lib not found!" exit 1 } - - # Cleanup - Remove-Item -Path $installerPath -Force -ErrorAction SilentlyContinue - Remove-Item -Path $extractPath -Recurse -Force -ErrorAction SilentlyContinue - - } catch { - Write-Error "Failed to download/extract Python 3.14 ARM64 installer: $_" - Write-Host "URL attempted: $installerUrl" + } else { + Write-Error "Python 3.14 ARM64 not found in hostedtoolcache" exit 1 } condition: eq(variables['targetArch'], 'arm64') - displayName: 'Download Python 3.14 ARM64 development files' + displayName: 'Copy Python 3.14 ARM64 libs from hostedtoolcache' - script: | echo "Python Version: $(pythonVersion)" @@ -770,3 +747,76 @@ jobs: docker rm test-$(ARCH) sqlserver || true displayName: 'Cleanup containers' condition: always() + +# Direct Ubuntu testing job without Docker containers +- job: TestOnUbuntu_Python314 + displayName: 'Test Python 3.14 on Ubuntu Latest' + pool: + vmImage: 'ubuntu-latest' + + steps: + - checkout: self + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.14' + addToPath: true + githubToken: $(GITHUB_TOKEN) + displayName: 'Use Python 3.14' + + - script: | + python --version + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install cmake pybind11 pytest + displayName: 'Install dependencies' + + - script: | + set -ex + + # Pull and start SQL Server + docker pull mcr.microsoft.com/mssql/server:2022-latest + docker run -d --name sqlserver \ + -e ACCEPT_EULA=Y \ + -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ + -p 1433:1433 \ + mcr.microsoft.com/mssql/server:2022-latest + + # Wait for SQL Server to be ready + for i in {1..30}; do + if docker exec sqlserver /opt/mssql-tools18/bin/sqlcmd \ + -S localhost -U SA -P "$(DB_PASSWORD)" -C -Q "SELECT 1" >/dev/null 2>&1; then + echo "SQL Server is ready!" + break + fi + sleep 2 + done + + # Create test database + docker exec sqlserver /opt/mssql-tools18/bin/sqlcmd \ + -S localhost -U SA -P "$(DB_PASSWORD)" -C \ + -Q "CREATE DATABASE TestDB" + displayName: 'Start SQL Server' + env: + DB_PASSWORD: $(DB_PASSWORD) + + - script: | + set -ex + + cd $(Build.SourcesDirectory)/mssql_python/pybind + bash build.sh + + cd $(Build.SourcesDirectory) + python --version + python -m pytest -v + + displayName: 'Build and test with Python 3.14' + env: + DB_CONNECTION_STRING: 'Server=tcp:127.0.0.1,1433;Database=TestDB;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' + + - script: | + docker stop sqlserver || true + docker rm sqlserver || true + displayName: 'Cleanup SQL Server' + condition: always() + From b81481955bb77d3f98422797dd9bde429d91fd35 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 10:09:31 +0530 Subject: [PATCH 08/24] fix ubi --- eng/pipelines/pr-validation-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index a6f1e550..e8abe8e3 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -610,8 +610,8 @@ jobs: dnf clean all dnf -y makecache - # Install build dependencies (readline-devel may not be in UBI, but ncurses-devel provides similar functionality) - dnf -y install gcc gcc-c++ make openssl-devel zlib-devel libffi-devel ncurses-devel sqlite-devel bzip2-devel xz-devel unixODBC-devel wget tar gzip + # Install build dependencies (ncurses-devel may not be in UBI, but ncurses-devel provides similar functionality) + dnf -y install gcc gcc-c++ make openssl-devel zlib-devel libffi-devel ncurses-devel sqlite-devel bzip2-devel xz-devel unixODBC-devel wget tar gzip xz # Download and build Python 3.14 cd /tmp From 55772751bd068c5d0f04b27137903d1ed654fdae Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 10:53:25 +0530 Subject: [PATCH 09/24] streamline pipeline --- eng/pipelines/pr-validation-pipeline.yml | 422 ++++------------------- 1 file changed, 75 insertions(+), 347 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index e8abe8e3..5efea1dd 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -45,6 +45,10 @@ jobs: pip install -r requirements.txt pip install cmake pybind11 displayName: 'Install dependencies' + + - script: | + choco install sqlserver-odbcdriver -y + displayName: 'Install SQL Server ODBC Driver' - powershell: | sqllocaldb create MSSQLLocalDB @@ -439,384 +443,108 @@ jobs: find "$(Build.ArtifactStagingDirectory)/ddbc-bindings/$(LINUX_TAG)-$(ARCH)" -maxdepth 1 -type f ! -name "*.so" -delete || true displayName: 'Copy wheels and .so back to host' - - script: | - docker stop build-$(LINUX_TAG)-$(ARCH) || true - docker rm build-$(LINUX_TAG)-$(ARCH) || true - displayName: 'Clean up container' - condition: always() - - - task: PublishBuildArtifacts@1 - condition: succeededOrFailed() - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/dist' - ArtifactName: 'mssql-python-wheels-dist-py314' - publishLocation: 'Container' - displayName: 'Publish wheels as artifacts' - - - task: PublishBuildArtifacts@1 - condition: succeededOrFailed() - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/ddbc-bindings' - ArtifactName: 'mssql-python-ddbc-bindings-py314' - publishLocation: 'Container' - displayName: 'Publish .so files as artifacts' - -- job: TestWheelsOnLinux_Python314 - displayName: 'Pytests on Linux Python 3.14 -' - dependsOn: BuildLinuxWheels_Python314 - condition: succeeded('BuildLinuxWheels_Python314') - pool: { vmImage: 'ubuntu-latest' } - timeoutInMinutes: 60 - - strategy: - matrix: - debian12: - BASE_IMAGE: 'debian:12-slim' - ARCH: 'x86_64' - DOCKER_PLATFORM: 'linux/amd64' - rhel_ubi9: - BASE_IMAGE: 'registry.access.redhat.com/ubi9/ubi:latest' - ARCH: 'x86_64' - DOCKER_PLATFORM: 'linux/amd64' - alpine320: - BASE_IMAGE: 'alpine:3.20' - ARCH: 'x86_64' - DOCKER_PLATFORM: 'linux/amd64' - debian12_arm64: - BASE_IMAGE: 'debian:12-slim' - ARCH: 'arm64' - DOCKER_PLATFORM: 'linux/arm64' - rhel_ubi9_arm64: - BASE_IMAGE: 'registry.access.redhat.com/ubi9/ubi:latest' - ARCH: 'arm64' - DOCKER_PLATFORM: 'linux/arm64' - alpine320_arm64: - BASE_IMAGE: 'alpine:3.20' - ARCH: 'arm64' - DOCKER_PLATFORM: 'linux/arm64' - - steps: - - checkout: self - - - task: DownloadBuildArtifacts@0 - inputs: - buildType: 'current' - downloadType: 'single' - artifactName: 'mssql-python-wheels-dist-py314' - downloadPath: '$(System.ArtifactsDirectory)' - displayName: 'Download wheel artifacts from current build' - - script: | set -euxo pipefail - WHEEL_DIR="$(System.ArtifactsDirectory)/mssql-python-wheels-dist-py314" - if [ ! -d "$WHEEL_DIR" ] || [ -z "$(ls -A $WHEEL_DIR/*.whl 2>/dev/null)" ]; then - echo "ERROR: No wheel files found in $WHEEL_DIR" - echo "Contents of artifacts directory:" - find "$(System.ArtifactsDirectory)" -type f -name "*.whl" || echo "No .whl files found anywhere" - exit 1 - fi - echo "Found wheel files:" - ls -la "$WHEEL_DIR"/*.whl - displayName: 'Verify wheel artifacts exist' - - - script: | - set -euxo pipefail - docker run -d --name sqlserver \ - --network bridge \ + + # Start SQL Server on host + docker run -d --name sqlserver-$(LINUX_TAG)-$(ARCH) \ + --platform linux/amd64 \ -e ACCEPT_EULA=Y \ -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ -p 1433:1433 \ mcr.microsoft.com/mssql/server:2022-latest - - echo "Waiting for SQL Server to start..." + + # Wait for SQL Server to be ready for i in {1..30}; do - if docker exec sqlserver /opt/mssql-tools18/bin/sqlcmd \ + if docker exec sqlserver-$(LINUX_TAG)-$(ARCH) /opt/mssql-tools18/bin/sqlcmd \ -S localhost -U SA -P "$(DB_PASSWORD)" -C -Q "SELECT 1" >/dev/null 2>&1; then echo "SQL Server is ready!" break fi - echo "Attempt $i/30: SQL Server not ready yet..." - sleep 3 + sleep 2 done - - docker exec sqlserver /opt/mssql-tools18/bin/sqlcmd \ - -S localhost -U SA -P "$(DB_PASSWORD)" -C \ - -Q "CREATE DATABASE TestDB" - displayName: 'Start SQL Server and create test database' - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - set -euxo pipefail - if [[ "$(ARCH)" == "arm64" ]] || [[ "$(ARCH)" == "aarch64" ]]; then - sudo docker run --rm --privileged tonistiigi/binfmt --install all - fi + # Get SQL Server container IP + SQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sqlserver-$(LINUX_TAG)-$(ARCH)) + echo "SQL Server IP: $SQL_IP" - for i in {1..3}; do - if docker run -d --name test-$(ARCH) \ - --platform $(DOCKER_PLATFORM) \ - --network bridge \ - -v $(System.ArtifactsDirectory):/artifacts:ro \ - $(BASE_IMAGE) \ - tail -f /dev/null; then - echo "Container started successfully on attempt $i" - break - else - echo "Failed to start container on attempt $i, retrying..." - docker rm test-$(ARCH) 2>/dev/null || true - sleep 5 - fi - done - - if ! docker ps | grep -q test-$(ARCH); then - echo "ERROR: Container test-$(ARCH) is not running" - docker logs test-$(ARCH) || true - exit 1 - fi - - if [[ "$(BASE_IMAGE)" == alpine* ]]; then - echo "Setting up Alpine Linux with Python 3.14..." - docker exec test-$(ARCH) sh -c " + # Install wheel and run tests in build container + if [[ "$(LINUX_TAG)" == "manylinux" ]]; then + docker exec build-$(LINUX_TAG)-$(ARCH) bash -lc " set -ex - apk update && apk add --no-cache build-base openssl-dev zlib-dev libffi-dev readline-dev sqlite-dev bzip2-dev xz-dev unixodbc-dev curl wget tar gzip libtool libltdl krb5-libs + PY=/opt/python/cp314-cp314/bin/python - # Download and build Python 3.14 - cd /tmp - wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a2.tar.xz || echo 'Could not download a2' + # Install the built wheel in isolated directory + mkdir -p /test_isolated + cd /test_isolated + \$PY -m pip install /workspace/dist/*.whl - if [ -f Python-3.14.0a2.tar.xz ]; then - tar -xf Python-3.14.0a2.tar.xz - cd /tmp/Python-3.14.0a2 - ./configure --prefix=/usr/local --with-ensurepip=install - make -j\$(nproc) - make install - ln -sf /usr/local/bin/python3.14 /usr/local/bin/python3 - python3 --version - python3 -m venv /venv - /venv/bin/pip install pytest - else - echo 'ERROR: Could not download Python 3.14' - exit 1 - fi - " - PY_CMD="/venv/bin/python" - elif [[ "$(BASE_IMAGE)" == *ubi* ]] || [[ "$(BASE_IMAGE)" == *rocky* ]] || [[ "$(BASE_IMAGE)" == *alma* ]]; then - echo "Setting up RHEL-based system with Python 3.14..." - docker exec test-$(ARCH) bash -c " - set -ex - echo 'Installing Python 3.14 on UBI/RHEL...' - if command -v dnf >/dev/null; then - dnf clean all - dnf -y makecache - - # Install build dependencies (ncurses-devel may not be in UBI, but ncurses-devel provides similar functionality) - dnf -y install gcc gcc-c++ make openssl-devel zlib-devel libffi-devel ncurses-devel sqlite-devel bzip2-devel xz-devel unixODBC-devel wget tar gzip xz - - # Download and build Python 3.14 - cd /tmp - wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a2.tar.xz || echo 'Could not download a2' - - if [ -f Python-3.14.0a2.tar.xz ]; then - tar -xf Python-3.14.0a2.tar.xz - cd /tmp/Python-3.14.0a2 - ./configure --prefix=/usr/local --with-ensurepip=install - make -j\$(nproc) - make install - ln -sf /usr/local/bin/python3.14 /usr/local/bin/python3 - python3 --version - python3 -m venv /venv - /venv/bin/python -m pip install -U pip pytest - /venv/bin/python --version - else - echo 'ERROR: Could not download Python 3.14' - exit 1 - fi - else - echo 'ERROR: dnf not found' - exit 1 + # Verify installation + \$PY -c 'import mssql_python; print(\"Package imported successfully from:\", mssql_python.__file__)' + + # Install pytest + \$PY -m pip install pytest + + # Copy tests from workspace + cp -r /workspace/tests /test_isolated/ || echo 'No tests directory' + cp /workspace/requirements.txt /test_isolated/ || true + \$PY -m pip install -r /test_isolated/requirements.txt || true + + # Run pytest + if [ -d /test_isolated/tests ]; then + DB_CONNECTION_STRING='Server=$SQL_IP;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' \$PY -m pytest /test_isolated/tests -v || echo 'Some tests may have failed' fi " - PY_CMD="/venv/bin/python" else - echo "Setting up Debian/Ubuntu with Python 3.14..." - docker exec test-$(ARCH) bash -c " + docker exec build-$(LINUX_TAG)-$(ARCH) sh -lc " set -ex - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y build-essential libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev liblzma-dev tk-dev libffi-dev unixodbc-dev curl wget tar gzip + PY=/opt/python/cp314-cp314/bin/python - # Download and build Python 3.14 - cd /tmp - wget https://www.python.org/ftp/python/3.14.0/Python-3.14.0a2.tar.xz || echo 'Could not download a2' + # Install the built wheel in isolated directory + mkdir -p /test_isolated + cd /test_isolated + \$PY -m pip install /workspace/dist/*.whl - if [ -f Python-3.14.0a2.tar.xz ]; then - tar -xf Python-3.14.0a2.tar.xz - cd /tmp/Python-3.14.0a2 - ./configure --prefix=/usr/local --with-ensurepip=install - make -j\$(nproc) - make install - ln -sf /usr/local/bin/python3.14 /usr/local/bin/python3 - python3 --version - python3 -m venv /venv - /venv/bin/pip install pytest - else - echo 'ERROR: Could not download Python 3.14' - exit 1 + # Verify installation + \$PY -c 'import mssql_python; print(\"Package imported successfully from:\", mssql_python.__file__)' + + # Install pytest + \$PY -m pip install pytest + + # Copy tests from workspace + cp -r /workspace/tests /test_isolated/ || echo 'No tests directory' + cp /workspace/requirements.txt /test_isolated/ || true + \$PY -m pip install -r /test_isolated/requirements.txt || true + + # Run pytest + if [ -d /test_isolated/tests ]; then + DB_CONNECTION_STRING='Server=$SQL_IP;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' \$PY -m pytest /test_isolated/tests -v || echo 'Some tests may have failed' fi " - PY_CMD="/venv/bin/python" fi - - if [[ "$(BASE_IMAGE)" == alpine* ]]; then - SHELL_CMD="sh -c" - WHEEL_PATTERN="*musllinux*$(ARCH)*.whl" - else - SHELL_CMD="bash -c" - WHEEL_PATTERN="*manylinux*$(ARCH)*.whl" - fi - - docker exec test-$(ARCH) $SHELL_CMD " - mkdir -p /test_whl - cd /test_whl - - echo 'Available wheels:' - ls -la /artifacts/mssql-python-wheels-dist-py314/*.whl - echo 'Installing package (letting pip auto-select in isolated environment):' - $PY_CMD -m pip install mssql_python --find-links /artifacts/mssql-python-wheels-dist-py314 --no-index --no-deps - - echo 'Installed package location:' - $PY_CMD -c 'import mssql_python; print(\"Package location:\", mssql_python.__file__)' - - $PY_CMD -c 'import mssql_python; print(\"Package imported successfully\")' - " - - displayName: 'Test wheel installation on $(BASE_IMAGE)' - env: - DB_CONNECTION_STRING: 'Server=localhost;Database=TestDB;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' - - - script: | - set -euxo pipefail - - echo "Copying source code to container for pytest..." - docker cp $(Build.SourcesDirectory)/. test-$(ARCH):/workspace/ - - if [[ "$(BASE_IMAGE)" == alpine* ]]; then - SHELL_CMD="sh -c" - PY_CMD="/venv/bin/python" - else - SHELL_CMD="bash -c" - PY_CMD="/venv/bin/python" - fi - - docker exec test-$(ARCH) $SHELL_CMD " - cd /workspace - - echo 'Running pytest suite with installed wheel...' - echo 'Current directory:' \$(pwd) - echo 'Python version:' - $PY_CMD --version - - echo 'Package import verification:' - $PY_CMD -c 'import mssql_python; print(\"Testing installed wheel from:\", mssql_python.__file__)' - - if [ -f requirements.txt ]; then - echo 'Installing test requirements...' - $PY_CMD -m pip install -r requirements.txt || echo 'Failed to install some requirements' - fi - - $PY_CMD -m pip install pytest || echo 'pytest installation failed' - - echo 'Available test files:' - find tests/ -name 'test_*.py' 2>/dev/null || echo 'No test files found in tests/' - - if [ -d tests/ ]; then - echo 'Starting pytest...' - $PY_CMD -m pytest -v || echo 'Some tests failed - this may be expected in containerized environment' - else - echo 'No tests directory found, skipping pytest' - fi - " - displayName: 'Run pytest suite on $(BASE_IMAGE) $(ARCH)' + displayName: 'Install wheel and run pytests' env: - DB_CONNECTION_STRING: 'Server=localhost;Database=TestDB;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' + DB_PASSWORD: $(DB_PASSWORD) continueOnError: true - script: | - docker stop test-$(ARCH) sqlserver || true - docker rm test-$(ARCH) sqlserver || true - displayName: 'Cleanup containers' + docker stop build-$(LINUX_TAG)-$(ARCH) sqlserver-$(LINUX_TAG)-$(ARCH) || true + docker rm build-$(LINUX_TAG)-$(ARCH) sqlserver-$(LINUX_TAG)-$(ARCH) || true + displayName: 'Clean up containers' condition: always() -# Direct Ubuntu testing job without Docker containers -- job: TestOnUbuntu_Python314 - displayName: 'Test Python 3.14 on Ubuntu Latest' - pool: - vmImage: 'ubuntu-latest' - - steps: - - checkout: self - - - task: UsePythonVersion@0 + - task: PublishBuildArtifacts@1 + condition: succeededOrFailed() inputs: - versionSpec: '3.14' - addToPath: true - githubToken: $(GITHUB_TOKEN) - displayName: 'Use Python 3.14' - - - script: | - python --version - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install cmake pybind11 pytest - displayName: 'Install dependencies' - - - script: | - set -ex - - # Pull and start SQL Server - docker pull mcr.microsoft.com/mssql/server:2022-latest - docker run -d --name sqlserver \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ - -p 1433:1433 \ - mcr.microsoft.com/mssql/server:2022-latest - - # Wait for SQL Server to be ready - for i in {1..30}; do - if docker exec sqlserver /opt/mssql-tools18/bin/sqlcmd \ - -S localhost -U SA -P "$(DB_PASSWORD)" -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - break - fi - sleep 2 - done - - # Create test database - docker exec sqlserver /opt/mssql-tools18/bin/sqlcmd \ - -S localhost -U SA -P "$(DB_PASSWORD)" -C \ - -Q "CREATE DATABASE TestDB" - displayName: 'Start SQL Server' - env: - DB_PASSWORD: $(DB_PASSWORD) - - - script: | - set -ex - - cd $(Build.SourcesDirectory)/mssql_python/pybind - bash build.sh - - cd $(Build.SourcesDirectory) - python --version - python -m pytest -v - - displayName: 'Build and test with Python 3.14' - env: - DB_CONNECTION_STRING: 'Server=tcp:127.0.0.1,1433;Database=TestDB;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' - - - script: | - docker stop sqlserver || true - docker rm sqlserver || true - displayName: 'Cleanup SQL Server' - condition: always() + PathtoPublish: '$(Build.ArtifactStagingDirectory)/dist' + ArtifactName: 'mssql-python-wheels-dist-py314' + publishLocation: 'Container' + displayName: 'Publish wheels as artifacts' + - task: PublishBuildArtifacts@1 + condition: succeededOrFailed() + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/ddbc-bindings' + ArtifactName: 'mssql-python-ddbc-bindings-py314' + publishLocation: 'Container' + displayName: 'Publish .so files as artifacts' From 316265643dcb499415492430ef52afab2ea38643 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 10:58:35 +0530 Subject: [PATCH 10/24] fix arm64 installs and getting libs --- eng/pipelines/pr-validation-pipeline.yml | 99 ++++++++++++++---------- 1 file changed, 60 insertions(+), 39 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 5efea1dd..a7fdf894 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -32,13 +32,26 @@ jobs: targetArch: 'arm64' steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(pythonVersion)' - architecture: '$(architecture)' - addToPath: true - githubToken: $(GITHUB_TOKEN) - displayName: 'Use Python $(pythonVersion) ($(architecture))' + - powershell: | + # Download and install Python 3.14 x64 + $installerUrl = "https://www.python.org/ftp/python/3.14.0/python-3.14.0a2-amd64.exe" + $installerPath = "$(Build.SourcesDirectory)\python314-x64-installer.exe" + $installDir = "C:\Python314" + + Write-Host "Downloading Python 3.14 x64 installer from: $installerUrl" + Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing + + Write-Host "Installing Python 3.14 x64 to: $installDir" + Start-Process -FilePath $installerPath -ArgumentList "/quiet", "InstallAllUsers=1", "PrependPath=1", "Include_test=0", "TargetDir=$installDir" -Wait + + Write-Host "Adding Python to PATH" + $env:Path = "$installDir;$installDir\Scripts;" + $env:Path + [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine) + + python --version + + Remove-Item -Path $installerPath -Force + displayName: 'Install Python 3.14 x64' - script: | python -m pip install --upgrade pip @@ -64,47 +77,55 @@ jobs: env: DB_PASSWORD: $(DB_PASSWORD) - - task: UsePythonVersion@0 - condition: eq(variables['targetArch'], 'arm64') - inputs: - versionSpec: '3.14' - architecture: 'arm64' - addToPath: false - githubToken: $(GITHUB_TOKEN) - displayName: 'Download Python 3.14 ARM64 for development files' - - powershell: | - # Copy Python 3.14 ARM64 libs from the installed Python - $pythonArm64 = Get-ChildItem "C:\hostedtoolcache\windows\Python\3.14*\arm64" -Directory | Select-Object -First 1 + # Download Python 3.14 ARM64 installer and extract libs (without installing) + $installerUrl = "https://www.python.org/ftp/python/3.14.0/python-3.14.0a2-arm64.exe" + $installerPath = "$(Build.SourcesDirectory)\python314-arm64-installer.exe" + $extractPath = "$(Build.SourcesDirectory)\python314-arm64-temp" + $destPath = "$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64" - if ($pythonArm64) { - $libsSource = Join-Path $pythonArm64.FullName "libs" - $destPath = "$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64" - - Write-Host "Found Python 3.14 ARM64 at: $($pythonArm64.FullName)" - Write-Host "Copying libs from: $libsSource" - + Write-Host "Downloading Python 3.14 ARM64 installer from: $installerUrl" + Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing + + Write-Host "Extracting installer to temp folder: $extractPath" + # Use 7zip or built-in tools to extract the MSI from the installer + Start-Process -FilePath $installerPath -ArgumentList "/layout", $extractPath, "/quiet" -Wait + + Start-Sleep -Seconds 10 + + # Find libs directory in extracted files + Write-Host "Searching for libs directory in extracted files..." + $libsDir = Get-ChildItem -Path $extractPath -Recurse -Directory -Filter "libs" -ErrorAction SilentlyContinue | Select-Object -First 1 + + if ($libsDir) { + Write-Host "Found libs at: $($libsDir.FullName)" New-Item -ItemType Directory -Force -Path $destPath | Out-Null - Copy-Item -Path "$libsSource\*" -Destination $destPath -Recurse -Force + Copy-Item -Path "$($libsDir.FullName)\*" -Destination $destPath -Recurse -Force - # Also copy DLLs - Copy-Item -Path "$($pythonArm64.FullName)\*.dll" -Destination $destPath -Force -ErrorAction SilentlyContinue - - Write-Host "`nContents of $destPath :" - Get-ChildItem $destPath | ForEach-Object { Write-Host " - $($_.Name)" } - - if (Test-Path "$destPath\python314.lib") { - Write-Host "`n✓ python314.lib found" - } else { - Write-Error "python314.lib not found!" - exit 1 + # Also get DLLs + $dllFiles = Get-ChildItem -Path $extractPath -Recurse -Filter "*.dll" -ErrorAction SilentlyContinue | Where-Object { $_.Name -like "python*" -or $_.Name -like "vcruntime*" } + foreach ($dll in $dllFiles) { + Copy-Item -Path $dll.FullName -Destination $destPath -Force } } else { - Write-Error "Python 3.14 ARM64 not found in hostedtoolcache" + Write-Warning "libs directory not found in extracted files" + } + + Write-Host "`nContents of $destPath :" + Get-ChildItem $destPath -ErrorAction SilentlyContinue | ForEach-Object { Write-Host " - $($_.Name)" } + + if (Test-Path "$destPath\python314.lib") { + Write-Host "`n✓ python314.lib found" + } else { + Write-Error "python314.lib not found!" exit 1 } + + # Cleanup + Remove-Item -Path $installerPath -Force -ErrorAction SilentlyContinue + Remove-Item -Path $extractPath -Recurse -Force -ErrorAction SilentlyContinue condition: eq(variables['targetArch'], 'arm64') - displayName: 'Copy Python 3.14 ARM64 libs from hostedtoolcache' + displayName: 'Extract Python 3.14 ARM64 libs (no installation)' - script: | echo "Python Version: $(pythonVersion)" From 3b3c4e4787cc2ed0f36cf456181d029b2b36ebcf Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 11:06:22 +0530 Subject: [PATCH 11/24] copyfiles fix --- eng/pipelines/pr-validation-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index a7fdf894..136fea6c 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -159,7 +159,7 @@ jobs: - task: CopyFiles@2 inputs: - SourceFolder: '$(Build.SourcesDirectory)\mssql_python\pybind\build\$(targetArch)\py$(shortPyVer)\Release' + SourceFolder: '$(Build.SourcesDirectory)\mssql_python' Contents: 'ddbc_bindings.cp$(shortPyVer)-*.pyd' TargetFolder: '$(Build.ArtifactStagingDirectory)\ddbc-bindings\windows' displayName: 'Place PYD file into artifacts directory' From 3067bf0ded7e3446fa9524c63bfd16985bd1d562 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 11:07:38 +0530 Subject: [PATCH 12/24] ar64 installer extraction --- eng/pipelines/pr-validation-pipeline.yml | 71 ++++++++++++++++-------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 136fea6c..5b3a0270 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -78,41 +78,64 @@ jobs: DB_PASSWORD: $(DB_PASSWORD) - powershell: | - # Download Python 3.14 ARM64 installer and extract libs (without installing) + # Download Python 3.14 ARM64 embed package and extract libs + $embedUrl = "https://www.python.org/ftp/python/3.14.0/python-3.14.0a2-embed-arm64.zip" $installerUrl = "https://www.python.org/ftp/python/3.14.0/python-3.14.0a2-arm64.exe" + $embedZip = "$(Build.SourcesDirectory)\python314-arm64-embed.zip" $installerPath = "$(Build.SourcesDirectory)\python314-arm64-installer.exe" - $extractPath = "$(Build.SourcesDirectory)\python314-arm64-temp" + $embedExtract = "$(Build.SourcesDirectory)\python314-arm64-embed" + $installerExtract = "$(Build.SourcesDirectory)\python314-arm64-install" $destPath = "$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64" - Write-Host "Downloading Python 3.14 ARM64 installer from: $installerUrl" - Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing + New-Item -ItemType Directory -Force -Path $destPath | Out-Null - Write-Host "Extracting installer to temp folder: $extractPath" - # Use 7zip or built-in tools to extract the MSI from the installer - Start-Process -FilePath $installerPath -ArgumentList "/layout", $extractPath, "/quiet" -Wait + # Try embed package first for DLLs + Write-Host "Downloading Python 3.14 ARM64 embed package from: $embedUrl" + try { + Invoke-WebRequest -Uri $embedUrl -OutFile $embedZip -UseBasicParsing -ErrorAction Stop + Write-Host "Extracting embed package..." + Expand-Archive -Path $embedZip -DestinationPath $embedExtract -Force + + # Copy DLLs from embed package + Get-ChildItem -Path $embedExtract -Filter "*.dll" | ForEach-Object { + Write-Host " Copying $($_.Name) from embed package" + Copy-Item -Path $_.FullName -Destination $destPath -Force + } + + Remove-Item -Path $embedZip -Force -ErrorAction SilentlyContinue + Remove-Item -Path $embedExtract -Recurse -Force -ErrorAction SilentlyContinue + } catch { + Write-Warning "Could not download embed package: $_" + } - Start-Sleep -Seconds 10 + # Download full installer to extract python314.lib using 7zip + Write-Host "`nDownloading Python 3.14 ARM64 installer from: $installerUrl" + Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing - # Find libs directory in extracted files - Write-Host "Searching for libs directory in extracted files..." - $libsDir = Get-ChildItem -Path $extractPath -Recurse -Directory -Filter "libs" -ErrorAction SilentlyContinue | Select-Object -First 1 + Write-Host "Extracting installer with 7zip (it's a self-extracting archive)..." + # The installer is actually a self-extracting 7z archive + & "C:\Program Files\7-Zip\7z.exe" x "$installerPath" "-o$installerExtract" -y - if ($libsDir) { - Write-Host "Found libs at: $($libsDir.FullName)" - New-Item -ItemType Directory -Force -Path $destPath | Out-Null - Copy-Item -Path "$($libsDir.FullName)\*" -Destination $destPath -Recurse -Force - - # Also get DLLs - $dllFiles = Get-ChildItem -Path $extractPath -Recurse -Filter "*.dll" -ErrorAction SilentlyContinue | Where-Object { $_.Name -like "python*" -or $_.Name -like "vcruntime*" } - foreach ($dll in $dllFiles) { - Copy-Item -Path $dll.FullName -Destination $destPath -Force - } + # Find and copy python314.lib + Write-Host "Searching for python314.lib..." + $libFile = Get-ChildItem -Path $installerExtract -Recurse -Filter "python314.lib" -ErrorAction SilentlyContinue | Select-Object -First 1 + + if ($libFile) { + Write-Host "Found python314.lib at: $($libFile.FullName)" + Copy-Item -Path $libFile.FullName -Destination $destPath -Force } else { - Write-Warning "libs directory not found in extracted files" + Write-Warning "python314.lib not found in installer extraction" + } + + # Also look for other .lib files in libs directory + $libsFiles = Get-ChildItem -Path $installerExtract -Recurse -Filter "*.lib" -ErrorAction SilentlyContinue + foreach ($lib in $libsFiles) { + Write-Host " Copying $($lib.Name)" + Copy-Item -Path $lib.FullName -Destination $destPath -Force } Write-Host "`nContents of $destPath :" - Get-ChildItem $destPath -ErrorAction SilentlyContinue | ForEach-Object { Write-Host " - $($_.Name)" } + Get-ChildItem $destPath | ForEach-Object { Write-Host " - $($_.Name)" } if (Test-Path "$destPath\python314.lib") { Write-Host "`n✓ python314.lib found" @@ -123,7 +146,7 @@ jobs: # Cleanup Remove-Item -Path $installerPath -Force -ErrorAction SilentlyContinue - Remove-Item -Path $extractPath -Recurse -Force -ErrorAction SilentlyContinue + Remove-Item -Path $installerExtract -Recurse -Force -ErrorAction SilentlyContinue condition: eq(variables['targetArch'], 'arm64') displayName: 'Extract Python 3.14 ARM64 libs (no installation)' From 9bc2a447c24e742ccfe1edfdf1d39b59a0a10361 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 11:27:24 +0530 Subject: [PATCH 13/24] python arm64 installaition --- eng/pipelines/pr-validation-pipeline.yml | 75 ++++++++++-------------- 1 file changed, 30 insertions(+), 45 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 5b3a0270..5cdc421e 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -78,62 +78,45 @@ jobs: DB_PASSWORD: $(DB_PASSWORD) - powershell: | - # Download Python 3.14 ARM64 embed package and extract libs - $embedUrl = "https://www.python.org/ftp/python/3.14.0/python-3.14.0a2-embed-arm64.zip" + # Install Python 3.14 ARM64 to temp location to get libs, then uninstall $installerUrl = "https://www.python.org/ftp/python/3.14.0/python-3.14.0a2-arm64.exe" - $embedZip = "$(Build.SourcesDirectory)\python314-arm64-embed.zip" $installerPath = "$(Build.SourcesDirectory)\python314-arm64-installer.exe" - $embedExtract = "$(Build.SourcesDirectory)\python314-arm64-embed" - $installerExtract = "$(Build.SourcesDirectory)\python314-arm64-install" + $installDir = "$(Build.SourcesDirectory)\Python314-ARM64-Temp" $destPath = "$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64" - New-Item -ItemType Directory -Force -Path $destPath | Out-Null - - # Try embed package first for DLLs - Write-Host "Downloading Python 3.14 ARM64 embed package from: $embedUrl" - try { - Invoke-WebRequest -Uri $embedUrl -OutFile $embedZip -UseBasicParsing -ErrorAction Stop - Write-Host "Extracting embed package..." - Expand-Archive -Path $embedZip -DestinationPath $embedExtract -Force - - # Copy DLLs from embed package - Get-ChildItem -Path $embedExtract -Filter "*.dll" | ForEach-Object { - Write-Host " Copying $($_.Name) from embed package" - Copy-Item -Path $_.FullName -Destination $destPath -Force - } - - Remove-Item -Path $embedZip -Force -ErrorAction SilentlyContinue - Remove-Item -Path $embedExtract -Recurse -Force -ErrorAction SilentlyContinue - } catch { - Write-Warning "Could not download embed package: $_" - } - - # Download full installer to extract python314.lib using 7zip - Write-Host "`nDownloading Python 3.14 ARM64 installer from: $installerUrl" + Write-Host "Downloading Python 3.14 ARM64 installer from: $installerUrl" Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing - Write-Host "Extracting installer with 7zip (it's a self-extracting archive)..." - # The installer is actually a self-extracting 7z archive - & "C:\Program Files\7-Zip\7z.exe" x "$installerPath" "-o$installerExtract" -y + Write-Host "Installing Python 3.14 ARM64 to temp directory: $installDir" + Write-Host "(This won't modify PATH or system settings - using TargetDir and InstallAllUsers=0)" + Start-Process -FilePath $installerPath -ArgumentList "/quiet", "InstallAllUsers=0", "PrependPath=0", "Include_test=0", "TargetDir=$installDir" -Wait - # Find and copy python314.lib - Write-Host "Searching for python314.lib..." - $libFile = Get-ChildItem -Path $installerExtract -Recurse -Filter "python314.lib" -ErrorAction SilentlyContinue | Select-Object -First 1 + Start-Sleep -Seconds 5 - if ($libFile) { - Write-Host "Found python314.lib at: $($libFile.FullName)" - Copy-Item -Path $libFile.FullName -Destination $destPath -Force + Write-Host "`nChecking installation..." + if (Test-Path $installDir) { + Write-Host "Installation directory found at: $installDir" + Get-ChildItem $installDir | ForEach-Object { Write-Host " - $($_.Name)" } } else { - Write-Warning "python314.lib not found in installer extraction" + Write-Error "Installation failed - directory not found!" + exit 1 } - # Also look for other .lib files in libs directory - $libsFiles = Get-ChildItem -Path $installerExtract -Recurse -Filter "*.lib" -ErrorAction SilentlyContinue - foreach ($lib in $libsFiles) { - Write-Host " Copying $($lib.Name)" - Copy-Item -Path $lib.FullName -Destination $destPath -Force + Write-Host "`nCopying libs from: $installDir\libs" + New-Item -ItemType Directory -Force -Path $destPath | Out-Null + + if (Test-Path "$installDir\libs") { + Copy-Item -Path "$installDir\libs\*" -Destination $destPath -Recurse -Force + Write-Host "✓ Copied .lib files" + } else { + Write-Error "libs directory not found!" + exit 1 } + # Also copy DLLs + Write-Host "Copying DLLs..." + Copy-Item -Path "$installDir\*.dll" -Destination $destPath -Force -ErrorAction SilentlyContinue + Write-Host "`nContents of $destPath :" Get-ChildItem $destPath | ForEach-Object { Write-Host " - $($_.Name)" } @@ -144,9 +127,11 @@ jobs: exit 1 } - # Cleanup + # Cleanup temp installation + Write-Host "`nCleaning up temp installation..." Remove-Item -Path $installerPath -Force -ErrorAction SilentlyContinue - Remove-Item -Path $installerExtract -Recurse -Force -ErrorAction SilentlyContinue + Remove-Item -Path $installDir -Recurse -Force -ErrorAction SilentlyContinue + Write-Host "✓ Cleanup complete" condition: eq(variables['targetArch'], 'arm64') displayName: 'Extract Python 3.14 ARM64 libs (no installation)' From 504b7428c9befab8b3040f68c673ada1b2562420 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 11:32:06 +0530 Subject: [PATCH 14/24] python arm64 path --- eng/pipelines/pr-validation-pipeline.yml | 34 +++++++++++++++++++----- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 5cdc421e..d62330f6 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -44,16 +44,29 @@ jobs: Write-Host "Installing Python 3.14 x64 to: $installDir" Start-Process -FilePath $installerPath -ArgumentList "/quiet", "InstallAllUsers=1", "PrependPath=1", "Include_test=0", "TargetDir=$installDir" -Wait - Write-Host "Adding Python to PATH" + Start-Sleep -Seconds 10 + + Write-Host "`nPython installation complete. Setting up PATH..." $env:Path = "$installDir;$installDir\Scripts;" + $env:Path - [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine) - python --version + Write-Host "Verifying Python installation:" + & "$installDir\python.exe" --version + Write-Host "Python location:" + & "$installDir\python.exe" -c "import sys; print(sys.executable)" Remove-Item -Path $installerPath -Force displayName: 'Install Python 3.14 x64' - - script: | + - powershell: | + $env:Path = "C:\Python314;C:\Python314\Scripts;" + $env:Path + Write-Host "Current PATH (first 3 entries):" + ($env:Path -split ';')[0..2] | ForEach-Object { Write-Host " $_" } + + Write-Host "`nPython version:" + python --version + Write-Host "`nPython executable:" + python -c "import sys; print(sys.executable)" + python -m pip install --upgrade pip pip install -r requirements.txt pip install cmake pybind11 @@ -136,9 +149,16 @@ jobs: displayName: 'Extract Python 3.14 ARM64 libs (no installation)' - script: | - echo "Python Version: $(pythonVersion)" - echo "Short Tag: $(shortPyVer)" - echo "Architecture: Host=$(architecture), Target=$(targetArch)" + set PATH=C:\Python314;C:\Python314\Scripts;%PATH% + + echo Python Version: $(pythonVersion) + echo Short Tag: $(shortPyVer) + echo Architecture: Host=$(architecture), Target=$(targetArch) + + echo. + echo Verifying Python before build: + python --version + python -c "import sys; print('Python:', sys.executable); print('Version:', sys.version_info)" cd "$(Build.SourcesDirectory)\mssql_python\pybind" From f5e6f7c39465d73c597faa8c32c3b69eaebfe2b6 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 11:39:20 +0530 Subject: [PATCH 15/24] use nuget for py314 libs --- eng/pipelines/pr-validation-pipeline.yml | 68 +++++++++++------------- 1 file changed, 32 insertions(+), 36 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index d62330f6..bf84139d 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -91,62 +91,58 @@ jobs: DB_PASSWORD: $(DB_PASSWORD) - powershell: | - # Install Python 3.14 ARM64 to temp location to get libs, then uninstall - $installerUrl = "https://www.python.org/ftp/python/3.14.0/python-3.14.0a2-arm64.exe" - $installerPath = "$(Build.SourcesDirectory)\python314-arm64-installer.exe" - $installDir = "$(Build.SourcesDirectory)\Python314-ARM64-Temp" + # Download Python 3.14 ARM64 from NuGet (contains libs directory) + $nugetUrl = "https://www.nuget.org/api/v2/package/pythonarm64/3.14.0-a2" + $nugetZip = "$(Build.SourcesDirectory)\pythonarm64.nupkg" + $extractPath = "$(Build.SourcesDirectory)\pythonarm64-nuget" $destPath = "$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64" - Write-Host "Downloading Python 3.14 ARM64 installer from: $installerUrl" - Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing + Write-Host "Downloading Python 3.14 ARM64 NuGet package from: $nugetUrl" + Invoke-WebRequest -Uri $nugetUrl -OutFile $nugetZip -UseBasicParsing - Write-Host "Installing Python 3.14 ARM64 to temp directory: $installDir" - Write-Host "(This won't modify PATH or system settings - using TargetDir and InstallAllUsers=0)" - Start-Process -FilePath $installerPath -ArgumentList "/quiet", "InstallAllUsers=0", "PrependPath=0", "Include_test=0", "TargetDir=$installDir" -Wait + Write-Host "Extracting NuGet package..." + Expand-Archive -Path $nugetZip -DestinationPath $extractPath -Force - Start-Sleep -Seconds 5 + Write-Host "`nSearching for libs directory..." + $libsDir = Get-ChildItem -Path $extractPath -Recurse -Directory -Filter "libs" | Select-Object -First 1 - Write-Host "`nChecking installation..." - if (Test-Path $installDir) { - Write-Host "Installation directory found at: $installDir" - Get-ChildItem $installDir | ForEach-Object { Write-Host " - $($_.Name)" } + if ($libsDir) { + Write-Host "Found libs at: $($libsDir.FullName)" + New-Item -ItemType Directory -Force -Path $destPath | Out-Null + Copy-Item -Path "$($libsDir.FullName)\*" -Destination $destPath -Recurse -Force + Write-Host "✓ Copied .lib files from NuGet package" } else { - Write-Error "Installation failed - directory not found!" - exit 1 + Write-Host "libs directory not found, searching for .lib files..." + $libFiles = Get-ChildItem -Path $extractPath -Recurse -Filter "*.lib" + New-Item -ItemType Directory -Force -Path $destPath | Out-Null + foreach ($lib in $libFiles) { + Write-Host " Copying $($lib.Name)" + Copy-Item -Path $lib.FullName -Destination $destPath -Force + } } - Write-Host "`nCopying libs from: $installDir\libs" - New-Item -ItemType Directory -Force -Path $destPath | Out-Null - - if (Test-Path "$installDir\libs") { - Copy-Item -Path "$installDir\libs\*" -Destination $destPath -Recurse -Force - Write-Host "✓ Copied .lib files" - } else { - Write-Error "libs directory not found!" - exit 1 + # Also copy DLLs if available + $dllFiles = Get-ChildItem -Path $extractPath -Recurse -Filter "*.dll" | Where-Object { $_.Name -like "python*" -or $_.Name -like "vcruntime*" -or $_.Name -like "libcrypto*" -or $_.Name -like "libssl*" -or $_.Name -like "libffi*" } + foreach ($dll in $dllFiles) { + Write-Host " Copying $($dll.Name)" + Copy-Item -Path $dll.FullName -Destination $destPath -Force } - # Also copy DLLs - Write-Host "Copying DLLs..." - Copy-Item -Path "$installDir\*.dll" -Destination $destPath -Force -ErrorAction SilentlyContinue - Write-Host "`nContents of $destPath :" Get-ChildItem $destPath | ForEach-Object { Write-Host " - $($_.Name)" } if (Test-Path "$destPath\python314.lib") { Write-Host "`n✓ python314.lib found" } else { - Write-Error "python314.lib not found!" + Write-Error "python314.lib not found in NuGet package!" exit 1 } - # Cleanup temp installation - Write-Host "`nCleaning up temp installation..." - Remove-Item -Path $installerPath -Force -ErrorAction SilentlyContinue - Remove-Item -Path $installDir -Recurse -Force -ErrorAction SilentlyContinue - Write-Host "✓ Cleanup complete" + # Cleanup + Remove-Item -Path $nugetZip -Force -ErrorAction SilentlyContinue + Remove-Item -Path $extractPath -Recurse -Force -ErrorAction SilentlyContinue condition: eq(variables['targetArch'], 'arm64') - displayName: 'Extract Python 3.14 ARM64 libs (no installation)' + displayName: 'Download Python 3.14 ARM64 libs from NuGet' - script: | set PATH=C:\Python314;C:\Python314\Scripts;%PATH% From 8fc448f798454f44b8689dc665775d2dad37f849 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 11:40:31 +0530 Subject: [PATCH 16/24] poll for installation --- eng/pipelines/pr-validation-pipeline.yml | 35 ++++++++++++++++++++---- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index bf84139d..aa7b833d 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -42,14 +42,39 @@ jobs: Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing Write-Host "Installing Python 3.14 x64 to: $installDir" - Start-Process -FilePath $installerPath -ArgumentList "/quiet", "InstallAllUsers=1", "PrependPath=1", "Include_test=0", "TargetDir=$installDir" -Wait + Write-Host "This may take a minute..." + $process = Start-Process -FilePath $installerPath -ArgumentList "/quiet", "InstallAllUsers=1", "PrependPath=1", "Include_test=0", "TargetDir=$installDir" -Wait -PassThru - Start-Sleep -Seconds 10 + Write-Host "Installer exit code: $($process.ExitCode)" - Write-Host "`nPython installation complete. Setting up PATH..." - $env:Path = "$installDir;$installDir\Scripts;" + $env:Path + # Wait for installation to complete + Write-Host "Waiting for installation to complete..." + Start-Sleep -Seconds 15 - Write-Host "Verifying Python installation:" + # Verify installation + $maxAttempts = 10 + $attempt = 0 + $pythonFound = $false + + while ($attempt -lt $maxAttempts -and -not $pythonFound) { + $attempt++ + Write-Host "Verification attempt $attempt/$maxAttempts..." + + if (Test-Path "$installDir\python.exe") { + $pythonFound = $true + Write-Host "✓ Python executable found at: $installDir\python.exe" + } else { + Write-Host " Waiting for python.exe to appear..." + Start-Sleep -Seconds 3 + } + } + + if (-not $pythonFound) { + Write-Error "Python installation failed - python.exe not found after waiting" + exit 1 + } + + Write-Host "`nVerifying Python installation:" & "$installDir\python.exe" --version Write-Host "Python location:" & "$installDir\python.exe" -c "import sys; print(sys.executable)" From 419e96fbc2a7c818f01038e8df6dcab8e067b3f4 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 11:44:06 +0530 Subject: [PATCH 17/24] use nuget for py314 everywhere in windows --- eng/pipelines/pr-validation-pipeline.yml | 62 ++++++++---------------- 1 file changed, 21 insertions(+), 41 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index aa7b833d..70e88465 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -33,54 +33,34 @@ jobs: steps: - powershell: | - # Download and install Python 3.14 x64 - $installerUrl = "https://www.python.org/ftp/python/3.14.0/python-3.14.0a2-amd64.exe" - $installerPath = "$(Build.SourcesDirectory)\python314-x64-installer.exe" - $installDir = "C:\Python314" + # Download Python 3.14 x64 from NuGet (no installation needed) + $nugetUrl = "https://www.nuget.org/api/v2/package/python/3.14.0-a2" + $nugetZip = "$(Build.SourcesDirectory)\python-x64.nupkg" + $extractPath = "C:\Python314-NuGet" - Write-Host "Downloading Python 3.14 x64 installer from: $installerUrl" - Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing - - Write-Host "Installing Python 3.14 x64 to: $installDir" - Write-Host "This may take a minute..." - $process = Start-Process -FilePath $installerPath -ArgumentList "/quiet", "InstallAllUsers=1", "PrependPath=1", "Include_test=0", "TargetDir=$installDir" -Wait -PassThru - - Write-Host "Installer exit code: $($process.ExitCode)" + Write-Host "Downloading Python 3.14 x64 NuGet package from: $nugetUrl" + Invoke-WebRequest -Uri $nugetUrl -OutFile $nugetZip -UseBasicParsing - # Wait for installation to complete - Write-Host "Waiting for installation to complete..." - Start-Sleep -Seconds 15 + Write-Host "Extracting NuGet package to: $extractPath" + Expand-Archive -Path $nugetZip -DestinationPath $extractPath -Force - # Verify installation - $maxAttempts = 10 - $attempt = 0 - $pythonFound = $false + # Python executable is in tools directory + $pythonDir = "$extractPath\tools" - while ($attempt -lt $maxAttempts -and -not $pythonFound) { - $attempt++ - Write-Host "Verification attempt $attempt/$maxAttempts..." - - if (Test-Path "$installDir\python.exe") { - $pythonFound = $true - Write-Host "✓ Python executable found at: $installDir\python.exe" - } else { - Write-Host " Waiting for python.exe to appear..." - Start-Sleep -Seconds 3 - } - } + Write-Host "`nVerifying Python installation:" + & "$pythonDir\python.exe" --version + & "$pythonDir\python.exe" -c "import sys; print('Python executable:', sys.executable)" - if (-not $pythonFound) { - Write-Error "Python installation failed - python.exe not found after waiting" - exit 1 - } + # Set up for subsequent steps + Write-Host "`nPython is ready at: $pythonDir" - Write-Host "`nVerifying Python installation:" - & "$installDir\python.exe" --version - Write-Host "Python location:" - & "$installDir\python.exe" -c "import sys; print(sys.executable)" + # Create a simple redirect script at C:\Python314 + New-Item -ItemType Directory -Force -Path "C:\Python314" | Out-Null + Copy-Item -Path "$pythonDir\*" -Destination "C:\Python314" -Recurse -Force + Write-Host "Copied Python to C:\Python314 for consistent paths" - Remove-Item -Path $installerPath -Force - displayName: 'Install Python 3.14 x64' + Remove-Item -Path $nugetZip -Force + displayName: 'Download Python 3.14 x64 from NuGet' - powershell: | $env:Path = "C:\Python314;C:\Python314\Scripts;" + $env:Path From c43af9b7ea634d7a5757c23d564bcf0119acda7e Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 11:46:50 +0530 Subject: [PATCH 18/24] use nuget for py314 everywhere in windows - zip --- eng/pipelines/pr-validation-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 70e88465..31f72943 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -35,7 +35,7 @@ jobs: - powershell: | # Download Python 3.14 x64 from NuGet (no installation needed) $nugetUrl = "https://www.nuget.org/api/v2/package/python/3.14.0-a2" - $nugetZip = "$(Build.SourcesDirectory)\python-x64.nupkg" + $nugetZip = "$(Build.SourcesDirectory)\python-x64.zip" $extractPath = "C:\Python314-NuGet" Write-Host "Downloading Python 3.14 x64 NuGet package from: $nugetUrl" From f9b9cf224481c6ac03fb64c421b231b7aef14624 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 11:47:52 +0530 Subject: [PATCH 19/24] use nuget for py314 everywhere in windows - zip --- eng/pipelines/pr-validation-pipeline.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 31f72943..3c086044 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -35,14 +35,18 @@ jobs: - powershell: | # Download Python 3.14 x64 from NuGet (no installation needed) $nugetUrl = "https://www.nuget.org/api/v2/package/python/3.14.0-a2" - $nugetZip = "$(Build.SourcesDirectory)\python-x64.zip" + $nugetFile = "$(Build.SourcesDirectory)\python-x64.nupkg" + $zipFile = "$(Build.SourcesDirectory)\python-x64.zip" $extractPath = "C:\Python314-NuGet" Write-Host "Downloading Python 3.14 x64 NuGet package from: $nugetUrl" - Invoke-WebRequest -Uri $nugetUrl -OutFile $nugetZip -UseBasicParsing + Invoke-WebRequest -Uri $nugetUrl -OutFile $nugetFile -UseBasicParsing + + Write-Host "Renaming .nupkg to .zip for extraction..." + Move-Item -Path $nugetFile -Destination $zipFile -Force Write-Host "Extracting NuGet package to: $extractPath" - Expand-Archive -Path $nugetZip -DestinationPath $extractPath -Force + Expand-Archive -Path $zipFile -DestinationPath $extractPath -Force # Python executable is in tools directory $pythonDir = "$extractPath\tools" @@ -59,7 +63,7 @@ jobs: Copy-Item -Path "$pythonDir\*" -Destination "C:\Python314" -Recurse -Force Write-Host "Copied Python to C:\Python314 for consistent paths" - Remove-Item -Path $nugetZip -Force + Remove-Item -Path $zipFile -Force displayName: 'Download Python 3.14 x64 from NuGet' - powershell: | From 8bd5d8f110393f72bde97eae1ff073841798da42 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 12:16:42 +0530 Subject: [PATCH 20/24] arm64 libs fixed --- eng/pipelines/pr-validation-pipeline.yml | 25 ++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 3c086044..e04b8770 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -102,15 +102,19 @@ jobs: - powershell: | # Download Python 3.14 ARM64 from NuGet (contains libs directory) $nugetUrl = "https://www.nuget.org/api/v2/package/pythonarm64/3.14.0-a2" - $nugetZip = "$(Build.SourcesDirectory)\pythonarm64.nupkg" + $nugetFile = "$(Build.SourcesDirectory)\pythonarm64.nupkg" + $zipFile = "$(Build.SourcesDirectory)\pythonarm64.zip" $extractPath = "$(Build.SourcesDirectory)\pythonarm64-nuget" $destPath = "$(Build.SourcesDirectory)\mssql_python\pybind\python_libs\arm64" Write-Host "Downloading Python 3.14 ARM64 NuGet package from: $nugetUrl" - Invoke-WebRequest -Uri $nugetUrl -OutFile $nugetZip -UseBasicParsing + Invoke-WebRequest -Uri $nugetUrl -OutFile $nugetFile -UseBasicParsing + + Write-Host "Renaming .nupkg to .zip for extraction..." + Move-Item -Path $nugetFile -Destination $zipFile -Force Write-Host "Extracting NuGet package..." - Expand-Archive -Path $nugetZip -DestinationPath $extractPath -Force + Expand-Archive -Path $zipFile -DestinationPath $extractPath -Force Write-Host "`nSearching for libs directory..." $libsDir = Get-ChildItem -Path $extractPath -Recurse -Directory -Filter "libs" | Select-Object -First 1 @@ -148,7 +152,7 @@ jobs: } # Cleanup - Remove-Item -Path $nugetZip -Force -ErrorAction SilentlyContinue + Remove-Item -Path $zipFile -Force -ErrorAction SilentlyContinue Remove-Item -Path $extractPath -Recurse -Force -ErrorAction SilentlyContinue condition: eq(variables['targetArch'], 'arm64') displayName: 'Download Python 3.14 ARM64 libs from NuGet' @@ -180,11 +184,20 @@ jobs: continueOnError: false - powershell: | + set PATH=C:\Python314;C:\Python314\Scripts;%PATH% + Write-Host "Running pytests to validate bindings" if ("$(targetArch)" -eq "arm64") { - Write-Host "Skipping pytests on Windows ARM64" + Write-Host "Skipping tests for ARM64 (cannot run ARM64 binaries on x64 host)" + exit 0 } else { - python -m pytest -v + Write-Host "Using Python 3.14 for tests:" + C:\Python314\python.exe --version + C:\Python314\python.exe -m pytest tests/ -v --maxfail=5 + if ($LASTEXITCODE -ne 0) { + Write-Error "Tests failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE + } } displayName: 'Run pytests' env: From 88bc6d74fd66af36ef1ed2ecb0e24622780094aa Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 12:17:37 +0530 Subject: [PATCH 21/24] arm64 fixed --- eng/pipelines/pr-validation-pipeline.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index e04b8770..3a3f50e0 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -67,18 +67,18 @@ jobs: displayName: 'Download Python 3.14 x64 from NuGet' - powershell: | - $env:Path = "C:\Python314;C:\Python314\Scripts;" + $env:Path - Write-Host "Current PATH (first 3 entries):" - ($env:Path -split ';')[0..2] | ForEach-Object { Write-Host " $_" } + Write-Host "Installing dependencies with Python 3.14..." + Write-Host "Python version:" + C:\Python314\python.exe --version + Write-Host "Python executable:" + C:\Python314\python.exe -c "import sys; print(sys.executable)" - Write-Host "`nPython version:" - python --version - Write-Host "`nPython executable:" - python -c "import sys; print(sys.executable)" + C:\Python314\python.exe -m pip install --upgrade pip + C:\Python314\python.exe -m pip install -r requirements.txt + C:\Python314\python.exe -m pip install cmake pybind11 - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install cmake pybind11 + Write-Host "`nVerifying pytest installation:" + C:\Python314\python.exe -m pytest --version displayName: 'Install dependencies' - script: | From 1b8536cdea04e8d3d1d7c0a83e83e72333fefbf7 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 12:23:26 +0530 Subject: [PATCH 22/24] fix windows --- eng/pipelines/pr-validation-pipeline.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 3a3f50e0..fb3cb7f0 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -184,16 +184,14 @@ jobs: continueOnError: false - powershell: | - set PATH=C:\Python314;C:\Python314\Scripts;%PATH% - Write-Host "Running pytests to validate bindings" if ("$(targetArch)" -eq "arm64") { Write-Host "Skipping tests for ARM64 (cannot run ARM64 binaries on x64 host)" exit 0 } else { Write-Host "Using Python 3.14 for tests:" - C:\Python314\python.exe --version - C:\Python314\python.exe -m pytest tests/ -v --maxfail=5 + & C:\Python314\python.exe --version + & C:\Python314\python.exe -m pytest tests/ -v --maxfail=5 if ($LASTEXITCODE -ne 0) { Write-Error "Tests failed with exit code $LASTEXITCODE" exit $LASTEXITCODE @@ -211,6 +209,7 @@ jobs: displayName: 'Place PYD file into artifacts directory' - script: | + set PATH=C:\Python314;C:\Python314\Scripts;%PATH% python -m pip install --upgrade pip pip install wheel setuptools set ARCHITECTURE=$(targetArch) From b27c329477d9c2481bfc0f9ed96c809055ffc8ce Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 13 Nov 2025 12:24:03 +0530 Subject: [PATCH 23/24] fix windows --- eng/pipelines/pr-validation-pipeline.yml | 34 ++++++++++++++++++------ 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index fb3cb7f0..b11b938c 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -184,19 +184,37 @@ jobs: continueOnError: false - powershell: | - Write-Host "Running pytests to validate bindings" + Write-Host "Running pytests with Python 3.14 to validate bindings" + if ("$(targetArch)" -eq "arm64") { Write-Host "Skipping tests for ARM64 (cannot run ARM64 binaries on x64 host)" exit 0 + } + + Write-Host "`nPython version being used for tests:" + & C:\Python314\python.exe --version + & C:\Python314\python.exe -c "import sys; print('Executable:', sys.executable)" + + Write-Host "`nVerifying pytest is available:" + & C:\Python314\python.exe -m pytest --version + + Write-Host "`nVerifying DB_CONNECTION_STRING environment variable:" + if ($env:DB_CONNECTION_STRING) { + Write-Host "✓ DB_CONNECTION_STRING is set" } else { - Write-Host "Using Python 3.14 for tests:" - & C:\Python314\python.exe --version - & C:\Python314\python.exe -m pytest tests/ -v --maxfail=5 - if ($LASTEXITCODE -ne 0) { - Write-Error "Tests failed with exit code $LASTEXITCODE" - exit $LASTEXITCODE - } + Write-Error "DB_CONNECTION_STRING is not set!" + exit 1 } + + Write-Host "`nRunning pytest..." + & C:\Python314\python.exe -m pytest tests/ -v --maxfail=5 + + if ($LASTEXITCODE -ne 0) { + Write-Error "Tests failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE + } + + Write-Host "`n✓ All tests passed!" displayName: 'Run pytests' env: DB_CONNECTION_STRING: 'Server=(localdb)\MSSQLLocalDB;Database=TestDB;Uid=testuser;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' From af21161fcbee7e6897f4717ebd10ae9bcf62479d Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Fri, 14 Nov 2025 12:21:21 +0530 Subject: [PATCH 24/24] pytest.ini stuff --- eng/pipelines/pr-validation-pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index b11b938c..1fc10024 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -571,6 +571,7 @@ jobs: # Copy tests from workspace cp -r /workspace/tests /test_isolated/ || echo 'No tests directory' + cp /workspace/pytest.ini /test_isolated/ || echo 'No pytest.ini found' cp /workspace/requirements.txt /test_isolated/ || true \$PY -m pip install -r /test_isolated/requirements.txt || true @@ -597,6 +598,7 @@ jobs: # Copy tests from workspace cp -r /workspace/tests /test_isolated/ || echo 'No tests directory' + cp /workspace/pytest.ini /test_isolated/ || echo 'No pytest.ini found' cp /workspace/requirements.txt /test_isolated/ || true \$PY -m pip install -r /test_isolated/requirements.txt || true