From adf57c14b28a88d63dd46144d766fe90caaf48d6 Mon Sep 17 00:00:00 2001 From: Julien Langlois Date: Thu, 5 Mar 2026 12:52:49 -0800 Subject: [PATCH 1/3] SG-42304 Replace OS-specific binary scripts with install_binary.py install_binary.py is a single cross-platform Python script that replaces install_binary_linux.sh, install_binary_mac.sh and install_binary_windows.ps1. Platform detection is done via sys.platform at runtime. pipelines.yml already calls python install_binary.py - this commit adds the script and removes the three old OS-specific scripts. SG-42304 Clean up pipeline naming and move git add to pipeline steps --- .snyk | 1 + azure-pipelines.yml | 8 +- azure-pipelines/requirements.txt | 16 + resources/python/install_binary.py | 151 +++++ resources/python/install_binary_linux.sh | 48 -- resources/python/install_binary_mac.sh | 84 --- resources/python/install_binary_windows.ps1 | 37 -- resources/python/install_source_only.sh | 59 +- resources/python/pipelines/pipelines.yml | 613 ++++-------------- resources/python/requirements.txt | 122 ++++ .../python/requirements/3.10/requirements.txt | 16 - .../python/requirements/3.11/requirements.txt | 16 - .../python/requirements/3.13/requirements.txt | 30 - .../python/requirements/3.9/requirements.txt | 16 - resources/python/update_requirements.py | 39 +- 15 files changed, 481 insertions(+), 775 deletions(-) create mode 100644 azure-pipelines/requirements.txt create mode 100644 resources/python/install_binary.py delete mode 100755 resources/python/install_binary_linux.sh delete mode 100755 resources/python/install_binary_mac.sh delete mode 100644 resources/python/install_binary_windows.ps1 create mode 100644 resources/python/requirements.txt delete mode 100644 resources/python/requirements/3.10/requirements.txt delete mode 100644 resources/python/requirements/3.11/requirements.txt delete mode 100644 resources/python/requirements/3.13/requirements.txt delete mode 100644 resources/python/requirements/3.9/requirements.txt diff --git a/.snyk b/.snyk index 132067c26..e335fbf24 100644 --- a/.snyk +++ b/.snyk @@ -8,6 +8,7 @@ ignore: - resources/python/src/3.10/explicit_requirements.txt - resources/python/src/3.11/explicit_requirements.txt - resources/python/src/3.13/explicit_requirements.txt + # WHY &&&&??????? that's not what we want to do. Quite the opposite.... exclude: global: # Exclude Python 3.7 deps because Snyk only scans with supported versions diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9afd86abf..9ec75dc3b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -35,10 +35,10 @@ jobs: - name: tk-framework-desktopclient - name: tk-shotgun extra_test_dependencies: - # Required when binary dependencies are not bundled - - attrs==22.2.0 # Fix version. Otherwise tk-ci-tools will install latest - - Twisted==22.10.0 # Last version supporting Python 3.7 - - websocket-client==1.6.1 # Last version supporting Python 3.7 + # All runtime dependencies come from the unified requirements file. + # This guarantees CI-installed versions match what pkgs.zip bundles, + # preventing sys.modules version conflicts (see SG-42304). TODO + - --requirement=azure-pipelines/requirements.txt post_tests_steps: - task: Bash@3 displayName: Run interpreter integration tests diff --git a/azure-pipelines/requirements.txt b/azure-pipelines/requirements.txt new file mode 100644 index 000000000..c69ee5cdf --- /dev/null +++ b/azure-pipelines/requirements.txt @@ -0,0 +1,16 @@ +# Required when binary dependencies are not bundled +attrs==22.2.0 # Fix version. Otherwise tk-ci-tools will install latest +## TODO will see later if we can update this ..!.!.!.! + +# Twisted must match the version bundled in pkgs.zip/src/ for the corresponding +# Python version. If the test environment installs a different version, it gets +# loaded first and causes a sys.modules version conflict at runtime (see SG-42304). +Twisted==22.10.0; python_version < "3.9" +# Twisted==24.10.0; python_version >= "3.9" and python_version < "3.13" +# Twisted~=24.11.0; python_version >= "3.13" +Twisted~=25.5.0; python_version >= "3.9" + +# websocket-client is a test-only dependency (not bundled in pkgs.zip). +# 1.6.2+ dropped Python 3.7 support; our minimum is Python 3.9. +websocket-client==1.6.1; python_version < "3.9" # Last version supporting Python 3.7 +websocket-client~=1.9.0; python_version >= "3.9" diff --git a/resources/python/install_binary.py b/resources/python/install_binary.py new file mode 100644 index 000000000..306b59280 --- /dev/null +++ b/resources/python/install_binary.py @@ -0,0 +1,151 @@ +# Copyright (c) 2017 Shotgun Software Inc. +# +# CONFIDENTIAL AND PROPRIETARY +# +# This work is provided "AS IS" and subject to the Shotgun Pipeline Toolkit +# Source Code License included in this distribution package. See LICENSE. +# By accessing, using, copying or modifying this work you indicate your +# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights +# not expressly granted therein are reserved by Shotgun Software Inc. +"""Cross-platform binary requirements installer. + +Platform directory mapping: + linux -> bin//linux + darwin -> bin//mac + win32 -> bin//win + +Usage: python install_binary.py +""" + +import os +import pathlib +import platform +import shutil +import subprocess +import sys +import tempfile + +from pip._internal.cli.main import main as pip_main + +_PLATFORM_DIR = { + "linux": "linux", + "darwin": "mac", + "win32": "win", +} + +platform_dir = _PLATFORM_DIR[sys.platform] + +python_version = f"{sys.version_info.major}.{sys.version_info.minor}" +bin_dir = pathlib.Path("bin") / python_version / platform_dir +requirements = pathlib.Path("bin") / python_version / "explicit_requirements.txt" + +# Delete and recreate the bin directory +shutil.rmtree(bin_dir, ignore_errors=True) +bin_dir.mkdir(parents=True) + +# Build pip install arguments +pip_args = [ + "install", + "--target", + os.fspath(bin_dir), + "--no-deps", + "--requirement", + os.fspath(requirements), +] + +# Linux requires explicit manylinux platform tags so that pip downloads the +# correct pre-built wheels regardless of the host platform. +# See: https://deepwiki.com/pypa/manylinux#policy-and-platform-support-matrix +if sys.platform == "linux": + pip_args += [ + "--platform", + "manylinux_2_28_x86_64", + "--platform", + "manylinux2014_x86_64", + "--platform", + "manylinux2010_x86_64", + "--prefer-binary", + ] + +if pip_main(pip_args) != 0: + raise SystemExit("pip install failed") + +# Python 3.10+ is bundled with SGD Universal builds (SGD 1.9+), which run +# natively on both Intel and Apple Silicon. Some packages (e.g. cffi, +# zope.interface) do not publish universal wheels on PyPI, so we +# cross-compile them for the opposite architecture using ARCHFLAGS and +# combine the results into universal binaries with lipo. +# This works on any macOS machine without needing a dedicated arm64 runner. +# +# Python 3.9 and below are bundled with older Intel-only SGD builds. On +# Apple Silicon those SGD versions run entirely under Rosetta 2, so +# x86_64-only .so files are correct - no universal binaries needed. +if sys.platform == "darwin" and sys.version_info.minor >= 10: + native_arch = platform.machine() + cross_arch = "x86_64" if native_arch == "arm64" else "arm64" + print(f"Native arch: {native_arch} - cross-compiling for: {cross_arch}") + + with tempfile.TemporaryDirectory() as tmp_cross: + env = os.environ.copy() + env["ARCHFLAGS"] = f"-arch {cross_arch}" + result = subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--target", + tmp_cross, + "--no-deps", + "--no-binary", + "cffi", + "--no-binary", + "zope.interface", + "--requirement", + os.fspath(requirements), + ], + env=env, + ) + if result.returncode != 0: + raise SystemExit("Cross-arch pip install failed") + + tmp_cross_path = pathlib.Path(tmp_cross) + for cross_so in tmp_cross_path.rglob("*.so"): + rel_path = cross_so.relative_to(tmp_cross_path) + native_so = bin_dir / rel_path + if native_so.exists(): + file_output = subprocess.run( + ["file", str(native_so)], + capture_output=True, + text=True, + ).stdout + if "universal binary" not in file_output: + print(f"Creating universal binary: {rel_path}") + fat = native_so.with_suffix(".fat") + subprocess.run( + [ + "lipo", + "-create", + str(native_so), + str(cross_so), + "-output", + str(fat), + ], + check=True, + ) + fat.replace(native_so) + +# For some reason zope is missing a top-level __init__.py when installed +# with pip, so we add it manually. +(bin_dir / "zope" / "__init__.py").touch() + +# Remove test directories to reduce package size +for pattern in [ + "Crypto/SelfTest", + "zope/interface/tests", + "zope/interface/*/tests", +]: + for match in bin_dir.glob(pattern): + shutil.rmtree(match, ignore_errors=True) + + diff --git a/resources/python/install_binary_linux.sh b/resources/python/install_binary_linux.sh deleted file mode 100755 index 471e32805..000000000 --- a/resources/python/install_binary_linux.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2017 Shotgun Software Inc. -# -# CONFIDENTIAL AND PROPRIETARY -# -# This work is provided "AS IS" and subject to the Shotgun Pipeline Toolkit -# Source Code License included in this distribution package. See LICENSE. -# By accessing, using, copying or modifying this work you indicate your -# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights -# not expressly granted therein are reserved by Shotgun Software Inc. - -# Get python version -python_major_version=$(python -c "import sys; print(sys.version_info.major)") -python_minor_version=$(python -c "import sys; print(sys.version_info.minor)") -python_version="$python_major_version.$python_minor_version" - -# Set paths -bin_dir="bin/$python_version/linux" -requirements="bin/$python_version/explicit_requirements.txt" - -# Stops the script -set -e - -# Delete current files -rm -rf $bin_dir -mkdir $bin_dir - -# Install packages -pip install --target $bin_dir --no-deps \ - --platform manylinux_2_28_x86_64 \ - --platform manylinux2014_x86_64 \ - --platform manylinux2010_x86_64 \ - --prefer-binary \ - --requirement $requirements - -# Available manylinux platforms: https://deepwiki.com/pypa/manylinux#policy-and-platform-support-matrix - -# For some reason zope is missing a top level init file when installed with -# pip, so we're adding it. -touch $bin_dir/zope/__init__.py - -# Remove tests to thin out the packages -rm -rf $bin_dir/Crypto/SelfTest -rm -rf $bin_dir/zope/interface/tests -rm -rf $bin_dir/zope/interface/*/tests - -# Add bin dir to repo -git add $bin_dir diff --git a/resources/python/install_binary_mac.sh b/resources/python/install_binary_mac.sh deleted file mode 100755 index e453a99a3..000000000 --- a/resources/python/install_binary_mac.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2017 Shotgun Software Inc. -# -# CONFIDENTIAL AND PROPRIETARY -# -# This work is provided "AS IS" and subject to the Shotgun Pipeline Toolkit -# Source Code License included in this distribution package. See LICENSE. -# By accessing, using, copying or modifying this work you indicate your -# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights -# not expressly granted therein are reserved by Shotgun Software Inc. - -# Get python version -python_major_version=$(python -c "import sys; print(sys.version_info.major)") -python_minor_version=$(python -c "import sys; print(sys.version_info.minor)") -python_version="$python_major_version.$python_minor_version" - -# Set paths -bin_dir="bin/$python_version/mac" -requirements="bin/$python_version/explicit_requirements.txt" - -# Stops the script -set -e - -# Delete current files -rm -rf $bin_dir -mkdir $bin_dir - -# Install packages natively -pip install --target $bin_dir --no-deps -r $requirements - -# Python 3.10+ is bundled with SGD Universal builds (SGD 1.9+), which run -# natively on both Intel and Apple Silicon. Some packages (e.g. cffi, -# zope.interface) do not publish universal wheels on PyPI, so we -# cross-compile them for the opposite architecture using ARCHFLAGS and -# combine the results into universal binaries with lipo. -# This works on any macOS machine without needing an arm64 runner. -# -# Python 3.9 and below are bundled with older Intel-only SGD builds. On -# Apple Silicon those SGD versions run entirely under Rosetta 2, so -# x86_64-only .so files are correct - no universal binaries needed. -if [ "$python_minor_version" -ge 10 ]; then - tmp_cross=$(mktemp -d) - trap "rm -rf $tmp_cross" EXIT - - native_arch=$(python -c "import platform; print(platform.machine())") - if [ "$native_arch" = "arm64" ]; then - cross_arch="x86_64" - else - cross_arch="arm64" - fi - - echo "Native arch: $native_arch - cross-compiling for: $cross_arch" - - ARCHFLAGS="-arch $cross_arch" pip install \ - --target "$tmp_cross" \ - --no-deps \ - --no-binary cffi \ - --no-binary zope.interface \ - -r "$requirements" - - find "$tmp_cross" -name "*.so" | while read cross_so; do - rel_path="${cross_so#$tmp_cross/}" - native_so="$bin_dir/$rel_path" - if [ -f "$native_so" ]; then - if ! file "$native_so" | grep -q "universal binary"; then - echo "Creating universal binary: $rel_path" - lipo -create "$native_so" "$cross_so" -output "${native_so}.fat" - mv "${native_so}.fat" "$native_so" - fi - fi - done -fi - -# For some reason zope is missing a top level init file when installed with -# pip, so we're adding it. -touch $bin_dir/zope/__init__.py - -# Remove tests to thin out the packages -rm -rf $bin_dir/Crypto/SelfTest -rm -rf $bin_dir/zope/interface/tests -rm -rf $bin_dir/zope/interface/*/tests - -# Add bin dir to repo -git add $bin_dir diff --git a/resources/python/install_binary_windows.ps1 b/resources/python/install_binary_windows.ps1 deleted file mode 100644 index b2f8471b9..000000000 --- a/resources/python/install_binary_windows.ps1 +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2017 Shotgun Software Inc. -# -# CONFIDENTIAL AND PROPRIETARY -# -# This work is provided "AS IS" and subject to the Shotgun Pipeline Toolkit -# Source Code License included in this distribution package. See LICENSE. -# By accessing, using, copying or modifying this work you indicate your -# agreement to the Shotgun Pipeline Toolkit Source Code License. All rights -# not expressly granted therein are reserved by Shotgun Software Inc. - -# Get python version -$python_major_version = python -c "import sys; print(sys.version_info.major)" -$python_minor_version = python -c "import sys; print(sys.version_info.minor)" -$python_version = $python_major_version + "." + $python_minor_version - -# Set paths -$bin_dir = "bin/$python_version/win" -$requirements = "bin/$python_version/explicit_requirements.txt" - -# Delete current files -Remove-Item -LiteralPath $bin_dir -Force -Recurse -ErrorAction Ignore -mkdir $bin_dir - -# Install packages -pip install --target $bin_dir --no-deps -r $requirements - -# Remove tests to thin out the packages -Remove-Item -LiteralPath $bin_dir\Crypto\SelfTest -Force -Recurse -ErrorAction Ignore -Remove-Item -LiteralPath $bin_dir\zope\interface\tests -Force -Recurse -ErrorAction Ignore -Remove-Item -LiteralPath $bin_dir\zope\interface\common\tests -Force -Recurse -ErrorAction Ignore - -# For some reason zope is missing a top level init file when installed with -# pip, so we're adding it. -ni $bin_dir\zope\__init__.py - -# Add bin dir to repo -git add $bin_dir diff --git a/resources/python/install_source_only.sh b/resources/python/install_source_only.sh index 4eeb52eee..7d2b41261 100755 --- a/resources/python/install_source_only.sh +++ b/resources/python/install_source_only.sh @@ -8,12 +8,14 @@ # agreement to the Shotgun Pipeline Toolkit Source Code License. All rights # not expressly granted therein are reserved by Shotgun Software Inc. +set -o errexit + echo "----------------------------------------------------" echo "Get python version" -python_major_version=$(python -c "import sys; print(sys.version_info.major)") -python_minor_version=$(python -c "import sys; print(sys.version_info.minor)") -python_version="$python_major_version.$python_minor_version" +python_major_version="$(python -c "import sys; print(sys.version_info.major)")" +python_minor_version="$(python -c "import sys; print(sys.version_info.minor)")" +python_version="${python_major_version}.${python_minor_version}" echo "Python version is $python_version" @@ -22,8 +24,8 @@ echo "Set base paths" requirements_filename="explicit_requirements.txt" package_filename="pkgs.zip" -source_dir="src/$python_version" -source_requirements="$source_dir/$requirements_filename" +source_dir="src/${python_version}" +source_requirements="${source_dir}/${requirements_filename}" echo "Source Dir: $source_dir" echo "Source Requirements: $source_requirements" @@ -31,7 +33,11 @@ echo "Source Requirements: $source_requirements" echo "----------------------------------------------------" echo "Remove current packages" -find $source_dir/* ! -name $package_filename ! -name $requirements_filename -maxdepth 1 -exec rm -rf {} + +find "${source_dir}"/* \ + ! -name "${package_filename}" \ + ! -name "${requirements_filename}" \ + -maxdepth 1 \ + -exec rm --recursive --force {} + echo "----------------------------------------------------" echo "Install new packages" @@ -39,34 +45,37 @@ echo "Install new packages" pip install --upgrade pip setuptools wheel pip install \ - --target $source_dir \ + --target "${source_dir}" \ --no-deps \ - -r $source_requirements + --requirement "${source_requirements}" echo "----------------------------------------------------" echo "Remove unnecessary files" -rm -Rf $source_dir/autobahn/test -rm -Rf $source_dir/autobahn/*/test -rm -Rf $source_dir/twisted/test -rm -Rf $source_dir/twisted/*/test -rm -Rf $source_dir/twisted/*/*/test -rm -Rf $source_dir/automat/_test -rm -Rf $source_dir/hyperlink/test -rm -Rf $source_dir/incremental/tests +rm --recursive --force "${source_dir}/autobahn/test" +rm --recursive --force "${source_dir}/autobahn/*/test" +rm --recursive --force "${source_dir}/twisted/test" +rm --recursive --force "${source_dir}/twisted/*/test" +rm --recursive --force "${source_dir}/twisted/*/*/test" +rm --recursive --force "${source_dir}/automat/_test" +rm --recursive --force "${source_dir}/hyperlink/test" +rm --recursive --force "${source_dir}/incremental/tests" -# In twisted.internet.unix, there is a mixin which we don't use that allows to copy file descriptors -# into other processes, which we don't require. That module is compiled, so we'll delete it. -rm -Rf $source_dir/twisted/python/_sendmsg.so +# In twisted.internet.unix, there is a mixin which we don't use that allows to +# copy file descriptors into other processes, which we don't require. That module +# is compiled, so we'll delete it. +rm --recursive --force "${source_dir}/twisted/python/_sendmsg.so" # Compress all files -pushd $source_dir -zip -q -r pkgs.zip ./* +pushd "${source_dir}" +zip --quiet --recurse-paths pkgs.zip ./* popd # Remove files -find $source_dir/* ! -name $package_filename ! -name $requirements_filename -maxdepth 1 -exec rm -rf {} + +find "${source_dir}"/* \ + ! -name "${package_filename}" \ + ! -name "${requirements_filename}" \ + -maxdepth 1 \ + -exec rm --recursive --force {} + + -echo "----------------------------------------------------" -echo "Adding new files to git" -git add $source_dir diff --git a/resources/python/pipelines/pipelines.yml b/resources/python/pipelines/pipelines.yml index a0575bfa4..38d62bc73 100644 --- a/resources/python/pipelines/pipelines.yml +++ b/resources/python/pipelines/pipelines.yml @@ -7,8 +7,9 @@ parameters: branch: "" - # Runtime condition applied to every job. Declared here so azure-pipelines.yml - # owns the opt-in logic; the template applies it consistently to all jobs. + # Runtime condition applied to the first source job. All downstream jobs + # cascade-skip automatically via condition: succeeded() when the first job + # is skipped. Declared here so azure-pipelines.yml owns the opt-in logic. # Defaults to 'succeeded()' so the template is usable standalone. # # In production use (azure-pipelines.yml), this is set to skip all jobs when @@ -19,483 +20,133 @@ parameters: # packages (e.g. CI-only or documentation changes). job_condition: "succeeded()" -jobs: -- job: install_source_dependencies_3_7 - displayName: Install source dependencies Python 3.7 - condition: and(succeeded(), ${{ parameters.job_condition }}) - pool: - vmImage: ubuntu-22.04 - steps: - - template: template.yml - parameters: - python_version: 3.7 - - script: | - set -e - git checkout ${{ parameters.branch }} || git checkout $(System.PullRequest.SourceBranch) && git checkout -b ${{ parameters.branch }} - git merge origin/$(System.PullRequest.SourceBranch) - git push -u origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Create branch if not exists or update - - script: | - set -e - git checkout ${{ parameters.branch }} - python update_requirements.py --clean-pip - displayName: Generate all explicit_requirements 3.7 - workingDirectory: resources/python - - script: | - set -e - ./install_source_only.sh - git commit --allow-empty -a -m "Update source requirements 3.7" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Update source requirements 3.7 - workingDirectory: resources/python - -- job: install_source_dependencies_3_9 - displayName: Install source dependencies Python 3.9 - dependsOn: install_source_dependencies_3_7 - condition: succeeded() - pool: - vmImage: ubuntu-22.04 - steps: - - template: template.yml - parameters: - python_version: 3.9 - - script: | - set -e - git checkout ${{ parameters.branch }} - python update_requirements.py --clean-pip - displayName: Generate all explicit_requirements 3.9 - workingDirectory: resources/python - - script: | - set -e - ./install_source_only.sh - git commit --allow-empty -a -m "Update source requirements 3.9" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Update source requirements 3.9 - workingDirectory: resources/python - -- job: install_source_dependencies_3_10 - displayName: Install source dependencies Python 3.10 - dependsOn: install_source_dependencies_3_9 - condition: succeeded() - pool: - vmImage: ubuntu-22.04 - steps: - - template: template.yml - parameters: - python_version: 3.10 - - script: | - set -e - git checkout ${{ parameters.branch }} - python update_requirements.py --clean-pip - displayName: Generate all explicit_requirements 3.10 - workingDirectory: resources/python - - script: | - set -e - ./install_source_only.sh - git commit --allow-empty -a -m "Update source requirements 3.10" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Update source requirements 3.10 - workingDirectory: resources/python - -- job: install_source_dependencies_3_11 - displayName: Install source dependencies Python 3.11 - dependsOn: install_source_dependencies_3_10 - condition: succeeded() - pool: - vmImage: ubuntu-22.04 - steps: - - template: template.yml - parameters: - python_version: 3.11 - - script: | - set -e - git checkout ${{ parameters.branch }} - python update_requirements.py --clean-pip - displayName: Generate all explicit_requirements 3.11 - workingDirectory: resources/python - - script: | - set -e - ./install_source_only.sh - git commit --allow-empty -a -m "Update source requirements 3.11" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Update source requirements 3.11 - workingDirectory: resources/python - -- job: install_source_dependencies_3_13 - displayName: Install source dependencies Python 3.13 - dependsOn: install_source_dependencies_3_11 - condition: succeeded() - pool: - vmImage: ubuntu-22.04 - steps: - - template: template.yml - parameters: - python_version: 3.13 - - script: | - set -e - git checkout ${{ parameters.branch }} - python update_requirements.py --clean-pip - displayName: Generate all explicit_requirements 3.13 - workingDirectory: resources/python - - script: | - set -e - ./install_source_only.sh - git commit --allow-empty -a -m "Update source requirements 3.13" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Update source requirements 3.13 - workingDirectory: resources/python - -# --------------------------------------------- -# Binary dependencies run also in order to prevent git race conditions -# MacOS - -- job: install_binary_dependencies_mac_3_7 - displayName: Install binary dependencies Mac 3.7 - dependsOn: install_source_dependencies_3_13 - condition: succeeded() - pool: - vmImage: 'macOS-14' - steps: - - template: template.yml - parameters: - python_version: 3.7 - - script: | - set -e - git checkout ${{ parameters.branch }} - ./install_binary_mac.sh - git commit --allow-empty -a -m "Update binary requirements in Mac Python 3.7" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Update binary requirements - workingDirectory: resources/python - -- job: install_binary_dependencies_mac_3_9 - displayName: Install binary dependencies Mac 3.9 - dependsOn: install_binary_dependencies_mac_3_7 - condition: succeeded() - pool: - vmImage: 'macOS-14' - steps: - - template: template.yml - parameters: - python_version: 3.9 - - script: | - set -e - git checkout ${{ parameters.branch }} - ./install_binary_mac.sh - git commit --allow-empty -a -m "Update binary requirements in Mac Python 3.9" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Update binary requirements - workingDirectory: resources/python - -- job: install_binary_dependencies_mac_3_10 - displayName: Install binary dependencies Mac 3.10 - dependsOn: install_binary_dependencies_mac_3_9 - condition: succeeded() - pool: - vmImage: 'macOS-14' - steps: - - template: template.yml - parameters: - python_version: 3.10 - - script: | - set -e - git checkout ${{ parameters.branch }} - ./install_binary_mac.sh - git commit --allow-empty -a -m "Update binary requirements in Mac Python 3.10" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Update binary requirements - workingDirectory: resources/python - -- job: install_binary_dependencies_mac_3_11 - displayName: Install binary dependencies Mac 3.11 - dependsOn: install_binary_dependencies_mac_3_10 - condition: succeeded() - pool: - vmImage: 'macOS-14' - steps: - - template: template.yml - parameters: - python_version: 3.11 - - script: | - set -e - git checkout ${{ parameters.branch }} - ./install_binary_mac.sh - git commit --allow-empty -a -m "Update binary requirements in Mac Python 3.11" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Update binary requirements - workingDirectory: resources/python - -- job: install_binary_dependencies_mac_3_13 - displayName: Install binary dependencies Mac 3.13 - dependsOn: install_binary_dependencies_mac_3_11 - condition: succeeded() - pool: - vmImage: 'macOS-14' - steps: - - template: template.yml - parameters: - python_version: 3.13 - - script: | - set -e - git checkout ${{ parameters.branch }} - ./install_binary_mac.sh - git commit --allow-empty -a -m "Update binary requirements in Mac Python 3.13" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Update binary requirements - workingDirectory: resources/python - -# Linux + # Each entry describes one Python version to build requirements for. + # Per-version job order: source -> linux -> mac -> windows. + # Versions are chained sequentially: each version's source job depends on + # the previous version's windows job via prev. + python_versions: + - name: "3.7" + # No prev - this is the first version; it also creates the branch. + + - name: "3.9" + prev: "3.7" + + - name: "3.10" + prev: "3.9" + + - name: "3.11" + prev: "3.10" + + - name: "3.13" + prev: "3.11" + + # OS platforms to build binary requirements for, in order. + # The first platform (linux) depends on the source job for the same version. + # Subsequent platforms chain via prev. + # The last platform (windows) acts as the cross-version dependency anchor + # (source_ depends on windows_). + os_platforms: + - name: linux + display_name: Linux + short: Lin + vm_image: ubuntu-22.04 + + - name: mac + display_name: Mac + short: Mac + vm_image: macOS-14 + prev: linux + + - name: windows + display_name: Windows + short: Win + vm_image: windows-2022 + prev: mac -- job: install_binary_dependencies_linux_3_7 - displayName: Install binary dependencies Linux 3.7 - dependsOn: install_binary_dependencies_mac_3_13 - condition: succeeded() - pool: - vmImage: 'ubuntu-22.04' - steps: - - template: template.yml - parameters: - python_version: 3.7 - - script: | - set -e - git checkout ${{ parameters.branch }} - ./install_binary_linux.sh - git commit --allow-empty -a -m "Update binary requirements in Linux Python 3.7" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Update binary requirements - workingDirectory: resources/python - -- job: install_binary_dependencies_linux_3_9 - displayName: Install binary dependencies Linux 3.9 - dependsOn: install_binary_dependencies_linux_3_7 - condition: succeeded() - pool: - vmImage: 'ubuntu-22.04' - steps: - - template: template.yml - parameters: - python_version: 3.9 - - script: | - set -e - git checkout ${{ parameters.branch }} - ./install_binary_linux.sh - git commit --allow-empty -a -m "Update binary requirements in Linux Python 3.9" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Update binary requirements - workingDirectory: resources/python - -- job: install_binary_dependencies_linux_3_10 - displayName: Install binary dependencies Linux 3.10 - dependsOn: install_binary_dependencies_linux_3_9 - condition: succeeded() - pool: - vmImage: 'ubuntu-22.04' - steps: - - template: template.yml - parameters: - python_version: 3.10 - - script: | - set -e - git checkout ${{ parameters.branch }} - ./install_binary_linux.sh - git commit --allow-empty -a -m "Update binary requirements in Linux Python 3.10" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Update binary requirements - workingDirectory: resources/python - -- job: install_binary_dependencies_linux_3_11 - displayName: Install binary dependencies Linux 3.11 - dependsOn: install_binary_dependencies_linux_3_10 - condition: succeeded() - pool: - vmImage: 'ubuntu-22.04' - steps: - - template: template.yml - parameters: - python_version: 3.11 - - script: | - set -e - git checkout ${{ parameters.branch }} - ./install_binary_linux.sh - git commit --allow-empty -a -m "Update binary requirements in Linux Python 3.11" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Update binary requirements - workingDirectory: resources/python - -- job: install_binary_dependencies_linux_3_13 - displayName: Install binary dependencies Linux 3.13 - dependsOn: install_binary_dependencies_linux_3_11 - condition: succeeded() - pool: - vmImage: 'ubuntu-22.04' - steps: - - template: template.yml - parameters: - python_version: 3.13 - - script: | - set -e - git checkout ${{ parameters.branch }} - ./install_binary_linux.sh - git commit --allow-empty -a -m "Update binary requirements in Linux Python 3.13" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Update binary requirements - workingDirectory: resources/python - -# Windows - -- job: install_binary_dependencies_windows_3_7 - displayName: Install binary dependencies Windows 3.7 - dependsOn: install_binary_dependencies_linux_3_13 - condition: succeeded() - pool: - vmImage: 'windows-2022' - steps: - - template: template.yml - parameters: - python_version: 3.7 - - script: | - git checkout ${{ parameters.branch }} - displayName: Update binary requirements - workingDirectory: resources/python - - powershell: .\install_binary_windows.ps1 - displayName: Run PowerShell Scripts - workingDirectory: resources/python - - script: | - git commit --allow-empty -a -m "Update binary requirements in Windows Python 3.7" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Run Push Scripts - workingDirectory: resources/python - -- job: install_binary_dependencies_windows_3_9 - displayName: Install binary dependencies Windows 3.9 - dependsOn: install_binary_dependencies_windows_3_7 - condition: succeeded() - pool: - vmImage: 'windows-2022' - steps: - - template: template.yml - parameters: - python_version: 3.9 - - script: | - git checkout ${{ parameters.branch }} - displayName: Update binary requirements - workingDirectory: resources/python - - powershell: .\install_binary_windows.ps1 - displayName: Run PowerShell Scripts - workingDirectory: resources/python - - script: | - git commit --allow-empty -a -m "Update binary requirements in Windows Python 3.9" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Run Push Scripts - workingDirectory: resources/python - -- job: install_binary_dependencies_windows_3_10 - displayName: Install binary dependencies Windows 3.10 - dependsOn: install_binary_dependencies_windows_3_9 - condition: succeeded() - pool: - vmImage: 'windows-2022' - steps: - - template: template.yml - parameters: - python_version: 3.10 - - script: | - git checkout ${{ parameters.branch }} - displayName: Update binary requirements - workingDirectory: resources/python - - powershell: .\install_binary_windows.ps1 - displayName: Run PowerShell Scripts - workingDirectory: resources/python - - script: | - git commit --allow-empty -a -m "Update binary requirements in Windows Python 3.10" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Run Push Scripts - workingDirectory: resources/python - -- job: install_binary_dependencies_windows_3_11 - displayName: Install binary dependencies Windows 3.11 - dependsOn: install_binary_dependencies_windows_3_10 - condition: succeeded() - pool: - vmImage: 'windows-2022' - steps: - - template: template.yml - parameters: - python_version: 3.11 - - script: | - git checkout ${{ parameters.branch }} - displayName: Update binary requirements - workingDirectory: resources/python - - powershell: .\install_binary_windows.ps1 - displayName: Run PowerShell Scripts - workingDirectory: resources/python - - script: | - git commit --allow-empty -a -m "Update binary requirements in Windows Python 3.11" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Run Push Scripts - workingDirectory: resources/python - -- job: install_binary_dependencies_windows_3_13 - displayName: Install binary dependencies Windows 3.13 - dependsOn: install_binary_dependencies_windows_3_11 - condition: succeeded() - pool: - vmImage: 'windows-2022' - steps: - - template: template.yml - parameters: - python_version: 3.13 - - script: | - git checkout ${{ parameters.branch }} - displayName: Update binary requirements - workingDirectory: resources/python - - powershell: .\install_binary_windows.ps1 - displayName: Run PowerShell Scripts - workingDirectory: resources/python - - script: | - git commit --allow-empty -a -m "Update binary requirements in Windows Python 3.13" - git push origin ${{ parameters.branch }} - env: - GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) - displayName: Run Push Scripts - workingDirectory: resources/python +jobs: +- ${{ each pyver in parameters.python_versions }}: + + # ----------------------------------------------------------------- + # Source + # ----------------------------------------------------------------- + - job: source_${{ replace(pyver.name, '.', '_') }} + displayName: Py 3rd-Party Pkgs - ${{ pyver.name }} - Src + ${{ if not(pyver.prev) }}: + condition: and(succeeded(), ${{ parameters.job_condition }}) + pool: + vmImage: ubuntu-22.04 + ${{ if pyver.prev }}: + dependsOn: windows_${{ replace(pyver.prev, '.', '_') }} + steps: + - template: template.yml + parameters: + python_version: ${{ pyver.name }} + + - ${{ if not(pyver.prev) }}: + - task: Bash@3 + displayName: Create branch if not exists or update + inputs: + targetType: inline + script: | + set -e + git switch ${{ parameters.branch }} || git switch -c ${{ parameters.branch }} origin/$(System.PullRequest.SourceBranch) + git push -u origin ${{ parameters.branch }} + env: + GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) + + - task: Bash@3 + displayName: Generate all explicit_requirements ${{ pyver.name }} + inputs: + targetType: inline + script: | + set -e + git switch ${{ parameters.branch }} + python update_requirements.py --clean-pip + workingDirectory: resources/python + env: + GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) + + - task: Bash@3 + displayName: Update source requirements ${{ pyver.name }} + inputs: + targetType: inline + script: | + set -e + ./install_source_only.sh + git add src/ + git commit --allow-empty -a -m "Update source requirements ${{ pyver.name }}" + git push origin ${{ parameters.branch }} + workingDirectory: resources/python + env: + GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) + + # ----------------------------------------------------------------- + # Binary (linux -> mac -> windows) + # ----------------------------------------------------------------- + - ${{ each os in parameters.os_platforms }}: + - job: ${{ os.name }}_${{ replace(pyver.name, '.', '_') }} + displayName: Py 3rd-Party Pkgs - ${{ pyver.name }} - ${{ os.short }} + ${{ if os.prev }}: + dependsOn: ${{ os.prev }}_${{ replace(pyver.name, '.', '_') }} + ${{ else }}: + dependsOn: source_${{ replace(pyver.name, '.', '_') }} + pool: + vmImage: ${{ os.vm_image }} + steps: + - template: template.yml + parameters: + python_version: ${{ pyver.name }} + - task: Bash@3 + displayName: Install and commit binary requirements + inputs: + targetType: inline + workingDirectory: resources/python + script: | + set -e + git switch ${{ parameters.branch }} + python install_binary.py + git add bin/ + git commit --allow-empty -a -m "Update binary requirements in ${{ os.display_name }} Python ${{ pyver.name }}" + git push origin ${{ parameters.branch }} + env: + GH_ACCESS_TOKEN: $(GITHUB_ACCESS_TOKEN) diff --git a/resources/python/requirements.txt b/resources/python/requirements.txt new file mode 100644 index 000000000..446da6397 --- /dev/null +++ b/resources/python/requirements.txt @@ -0,0 +1,122 @@ +# Unified requirements file for tk-framework-desktopserver Python dependencies. +# +# This is the single source of truth for all Python dependency versions. +# It serves two purposes: +# +# 1. Building pkgs.zip - passed to update_requirements.py which calls pip +# under each target Python version. Pip env markers are resolved +# automatically based on the running interpreter. +# +# 2. CI test dependencies - referenced in azure-pipelines.yml as: +# extra_test_dependencies: +# - --requirement resources/python/requirements.txt +# +# Supported Python versions: +# - 3.7 (legacy; Twisted capped at 22.10.0) +# - 3.9 (VFX CY2022) +# - 3.10 (VFX CY2023) +# - 3.11 (VFX CY2024) +# - 3.13 (VFX CY2026; 3.12 was skipped - not a VFX platform year) +# +# When updating a dependency, run update_requirements.py for each Python +# version to regenerate the frozen explicit_requirements.txt files, then +# rebuild pkgs.zip with install_source_only.sh. + + +# ---------------------------------------------------------------------------- +# Core async / networking framework +# ---------------------------------------------------------------------------- + +# Twisted - main async framework. +# 22.10.0 is the last release supporting Python 3.7. +# 24.10.0 is used for 3.9-3.11. +# 24.11+ is required for 3.13 (dropped legacy code paths incompatible with +# the new interpreter). +Twisted==22.10.0; python_version == "3.7" +Twisted==24.10.0; python_version > "3.7" and python_version < "3.13" +Twisted~=24.11.0; python_version >= "3.13" + +# autobahn - WebSocket and WAMP protocol implementation on top of Twisted. +# 24.x required for 3.13: rewrote internals incompatible with stricter +# type checking in newer Python versions. +autobahn==22.12.1; python_version < "3.13" +autobahn~=24.4.2; python_version >= "3.13" + + +# ---------------------------------------------------------------------------- +# TLS / certificate handling +# ---------------------------------------------------------------------------- + +# pyOpenSSL - OpenSSL bindings. 25.x compatible with all supported versions. +pyOpenSSL==25.0.0 + +# service-identity - certificate verification for pyOpenSSL. +# 24.x required for 3.13 (updated type annotations, dropped older deps). +service-identity==21.1.0; python_version < "3.13" +service-identity~=24.2.0; python_version >= "3.13" + +# cryptography - low-level cryptography backend for pyOpenSSL. +# Binary package (wheel). NOTE: only needed for SAST scanning, not runtime. +# 44.x compatible across all supported Python versions. +cryptography==44.0.1 + +# cffi - C foreign function interface; dependency of cryptography. +# Binary package (wheel). 1.17.x required for Python 3.13 (updated C API +# adapters for the new interpreter ABI). +cffi==1.15.1; python_version < "3.13" +cffi~=1.17.1; python_version >= "3.13" + + +# ---------------------------------------------------------------------------- +# Utility libraries +# ---------------------------------------------------------------------------- + +# certifi - Mozilla CA certificate bundle. Updated alongside Desktop releases. +certifi==2026.1.4 + +# hyperlink - Pythonic URL parsing and manipulation. +hyperlink==21.0.0 + +# idna - Internationalized Domain Names in Applications. +idna==3.7; python_version < "3.13" +idna~=3.8; python_version >= "3.13" + +# six - Python 2/3 compatibility shim. +# Twisted 24.11 dropped six as a direct dependency, so not needed on 3.13. +# Still required transitively on 3.9-3.11. +six==1.16.0; python_version < "3.13" + +# zope.interface - component interface architecture; used internally by Twisted. +# Binary package (wheel). 7.x required for Python 3.13. +zope.interface==5.5.2; python_version < "3.13" +zope.interface~=7.1.0; python_version >= "3.13" + + +# ---------------------------------------------------------------------------- +# attrs +# ---------------------------------------------------------------------------- +# +# IMPORTANT: Pinned to 22.2.0 for ALL Python versions. +# Do not upgrade without reading this carefully. +# +# Root cause of SG-42304 (Python 3.13 CI failure): +# +# attrs ships two top-level packages: `attr/` and `attrs/`. Since attrs +# 23.2.0, `attrs/__init__.py` does `from attr import Converter`. +# +# During a pytest run, `attr` is imported early (as a pytest dependency) and +# cached in sys.modules. Our __init__.py then inserts pkgs.zip at the front +# of sys.path. However, Python never re-resolves modules already in the cache. +# If pkgs.zip bundles attrs >= 23.2.0 while sys.modules still holds the older +# `attr` module (without Converter), the import raises: +# +# ImportError: cannot import name 'Converter' from 'attr' +# +# Pinning to 22.2.0 here ensures that both pkgs.zip and the CI system-level +# install use the exact same version, so the cached module is always +# compatible with what pkgs.zip expects. +# +# Twisted 24.x and autobahn 24.x only require attrs >= 22.2.0, so this is +# fully compatible with all supported Python versions. +# +attrs==22.2.0 diff --git a/resources/python/requirements/3.10/requirements.txt b/resources/python/requirements/3.10/requirements.txt deleted file mode 100644 index c9c0b58dc..000000000 --- a/resources/python/requirements/3.10/requirements.txt +++ /dev/null @@ -1,16 +0,0 @@ -Twisted==24.10.0 -certifi==2026.1.4 -autobahn==22.12.1 -pyOpenSSL==25.0.0 -service-identity==21.1.0 - -# When updating certifi to match the version released with Desktop, some other unrelated modules (that are second -# level dependencies) were updating. The list below pins these versions for now until the next time we need to update -# modules and rebuild the binaries -attrs==22.2.0 -cffi==1.15.1 -cryptography==44.0.1 # Only for SAST. -hyperlink==21.0.0 -idna==3.7 -six==1.16.0 -zope.interface==5.5.2 diff --git a/resources/python/requirements/3.11/requirements.txt b/resources/python/requirements/3.11/requirements.txt deleted file mode 100644 index c9c0b58dc..000000000 --- a/resources/python/requirements/3.11/requirements.txt +++ /dev/null @@ -1,16 +0,0 @@ -Twisted==24.10.0 -certifi==2026.1.4 -autobahn==22.12.1 -pyOpenSSL==25.0.0 -service-identity==21.1.0 - -# When updating certifi to match the version released with Desktop, some other unrelated modules (that are second -# level dependencies) were updating. The list below pins these versions for now until the next time we need to update -# modules and rebuild the binaries -attrs==22.2.0 -cffi==1.15.1 -cryptography==44.0.1 # Only for SAST. -hyperlink==21.0.0 -idna==3.7 -six==1.16.0 -zope.interface==5.5.2 diff --git a/resources/python/requirements/3.13/requirements.txt b/resources/python/requirements/3.13/requirements.txt deleted file mode 100644 index 6a9c59b79..000000000 --- a/resources/python/requirements/3.13/requirements.txt +++ /dev/null @@ -1,30 +0,0 @@ -Twisted~=24.11.0 -autobahn~=24.4.2 -certifi==2026.1.4 -pyOpenSSL~=25.0.0 -service-identity~=24.2.0 - -# When updating certifi to match the version released with Desktop, some other unrelated modules (that are second -# level dependencies) were updating. The list below pins these versions for now until the next time we need to update -# modules and rebuild the binaries -# -# attrs is pinned to 22.2.0 intentionally - DO NOT upgrade it. -# -# The CI pipeline installs attrs==22.2.0 as a system-level test dependency -# (see extra_test_dependencies in azure-pipelines.yml). pytest itself also -# depends on attrs and will import it early, caching it in sys.modules. -# -# attrs ships two top-level packages: `attr/` and `attrs/`. Starting with -# attrs 23.2.0, `attrs/__init__.py` imports `Converter` from `attr`. If the -# version bundled in pkgs.zip is newer than the one already cached in -# sys.modules (from the system install), that import will fail with: -# ImportError: cannot import name 'Converter' from 'attr' -# -# All other Python versions (3.9, 3.10, 3.11) already bundle 22.2.0. -# Twisted 24.x and autobahn 24.x only require attrs>=22.2.0, so this is safe. -attrs==22.2.0 -cffi~=1.17.1 -cryptography~=44.0.1 # Only for SAST. -hyperlink~=21.0.0 -idna~=3.8 -zope.interface~=7.1.0 diff --git a/resources/python/requirements/3.9/requirements.txt b/resources/python/requirements/3.9/requirements.txt deleted file mode 100644 index c9c0b58dc..000000000 --- a/resources/python/requirements/3.9/requirements.txt +++ /dev/null @@ -1,16 +0,0 @@ -Twisted==24.10.0 -certifi==2026.1.4 -autobahn==22.12.1 -pyOpenSSL==25.0.0 -service-identity==21.1.0 - -# When updating certifi to match the version released with Desktop, some other unrelated modules (that are second -# level dependencies) were updating. The list below pins these versions for now until the next time we need to update -# modules and rebuild the binaries -attrs==22.2.0 -cffi==1.15.1 -cryptography==44.0.1 # Only for SAST. -hyperlink==21.0.0 -idna==3.7 -six==1.16.0 -zope.interface==5.5.2 diff --git a/resources/python/update_requirements.py b/resources/python/update_requirements.py index e6e40cfb1..afe8d7c3a 100755 --- a/resources/python/update_requirements.py +++ b/resources/python/update_requirements.py @@ -80,11 +80,11 @@ def __init__(self): def _pip_freeze(self): """List all packages installed.""" - output = self._pip("freeze").strip() - if output == "": + output = self._pip(["freeze"]).strip() + if not output: return [] - else: - return output.split("\n") + + return output.split("\n") def _clean_pip(self): """Uninstall all packages with pip.""" @@ -93,14 +93,16 @@ def _clean_pip(self): print("Cleaning PIP dependencies") for dependency in self._pip_freeze(): - cmd = "uninstall -y {}".format(dependency) - self._pip(cmd) + self._pip(["uninstall", "-y", dependency]) - def _pip(self, cmd): + def _pip(self, cmd: list): """Run the pip command.""" - pip_cmd = "python -m pip".split() + cmd.split() + pip_cmd = ["python", "-m", "pip"] + cmd try: - output = subprocess.check_output(pip_cmd) + output = subprocess.check_output( + pip_cmd, + text=True, + ) except subprocess.CalledProcessError as e: raise UpdateException( "Error running pip command: {}\nReturn Code:{}\n{}".format( @@ -109,15 +111,14 @@ def _pip(self, cmd): e.output, ) ) - - output = output.decode("utf-8") + return output @staticmethod - def _git(cmd): + def _git(cmd: list): """Run the git command.""" - git_cmd = ["git"] + cmd.split() - subprocess.check_output(git_cmd) + git_cmd = ["git"] + cmd + subprocess.check_call(git_cmd) def _get_dependencies_to_install(self): """Retrieve the full list of dependencies after a pip install.""" @@ -133,9 +134,11 @@ def _get_dependencies_to_install(self): try: # pip install all the requirements into the build subfolder. - self._pip("install -r requirements/{}/requirements.txt".format( - self._python_version_dot_format - )) + self._pip([ + "install", + "--requirement", + f"requirements.txt", + ]) print("All dependencies installed.") # list everything that was installed. @@ -192,7 +195,7 @@ def _update_requirements_file(self, dependencies): source_handler.writelines(requirement_to_add) # Add the new requirements files to git - self._git(f"add {source_reqs_path} {bin_reqs_path}") + self._git(["add", source_reqs_path, bin_reqs_path]) @staticmethod def _clean_before_update(): From a2b136bdeaa60d759d4071278b4d71c349190b56 Mon Sep 17 00:00:00 2001 From: Julien Langlois Date: Thu, 5 Mar 2026 17:54:44 -0800 Subject: [PATCH 2/3] cleanup --- .snyk | 2 -- azure-pipelines.yml | 2 +- azure-pipelines/requirements.txt | 7 ++----- resources/python/install_binary.py | 4 +--- resources/python/pipelines/pipelines.yml | 4 ++-- resources/python/requirements.txt | 22 +++++----------------- 6 files changed, 11 insertions(+), 30 deletions(-) diff --git a/.snyk b/.snyk index e335fbf24..deac7bc5d 100644 --- a/.snyk +++ b/.snyk @@ -8,10 +8,8 @@ ignore: - resources/python/src/3.10/explicit_requirements.txt - resources/python/src/3.11/explicit_requirements.txt - resources/python/src/3.13/explicit_requirements.txt - # WHY &&&&??????? that's not what we want to do. Quite the opposite.... exclude: global: # Exclude Python 3.7 deps because Snyk only scans with supported versions - - resources/python/requirements/3.7/requirements.txt - resources/python/bin/3.7/explicit_requirements.txt - resources/python/bin/3.7/explicit_requirements.txt diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9ec75dc3b..1b8587ed1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,7 +37,7 @@ jobs: extra_test_dependencies: # All runtime dependencies come from the unified requirements file. # This guarantees CI-installed versions match what pkgs.zip bundles, - # preventing sys.modules version conflicts (see SG-42304). TODO + # preventing sys.modules version conflicts. - --requirement=azure-pipelines/requirements.txt post_tests_steps: - task: Bash@3 diff --git a/azure-pipelines/requirements.txt b/azure-pipelines/requirements.txt index c69ee5cdf..6835ddbba 100644 --- a/azure-pipelines/requirements.txt +++ b/azure-pipelines/requirements.txt @@ -1,16 +1,13 @@ # Required when binary dependencies are not bundled attrs==22.2.0 # Fix version. Otherwise tk-ci-tools will install latest -## TODO will see later if we can update this ..!.!.!.! # Twisted must match the version bundled in pkgs.zip/src/ for the corresponding # Python version. If the test environment installs a different version, it gets # loaded first and causes a sys.modules version conflict at runtime (see SG-42304). Twisted==22.10.0; python_version < "3.9" -# Twisted==24.10.0; python_version >= "3.9" and python_version < "3.13" -# Twisted~=24.11.0; python_version >= "3.13" -Twisted~=25.5.0; python_version >= "3.9" +Twisted==24.10.0; python_version >= "3.9" and python_version < "3.13" +Twisted~=24.11.0; python_version >= "3.13" # websocket-client is a test-only dependency (not bundled in pkgs.zip). -# 1.6.2+ dropped Python 3.7 support; our minimum is Python 3.9. websocket-client==1.6.1; python_version < "3.9" # Last version supporting Python 3.7 websocket-client~=1.9.0; python_version >= "3.9" diff --git a/resources/python/install_binary.py b/resources/python/install_binary.py index 306b59280..f8cfd0e31 100644 --- a/resources/python/install_binary.py +++ b/resources/python/install_binary.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 Shotgun Software Inc. +# Copyright (c) 2026 Shotgun Software Inc. # # CONFIDENTIAL AND PROPRIETARY # @@ -147,5 +147,3 @@ ]: for match in bin_dir.glob(pattern): shutil.rmtree(match, ignore_errors=True) - - diff --git a/resources/python/pipelines/pipelines.yml b/resources/python/pipelines/pipelines.yml index 38d62bc73..7c4adbbac 100644 --- a/resources/python/pipelines/pipelines.yml +++ b/resources/python/pipelines/pipelines.yml @@ -70,7 +70,7 @@ jobs: # Source # ----------------------------------------------------------------- - job: source_${{ replace(pyver.name, '.', '_') }} - displayName: Py 3rd-Party Pkgs - ${{ pyver.name }} - Src + displayName: Pkg Req - Py ${{ pyver.name }} - Src ${{ if not(pyver.prev) }}: condition: and(succeeded(), ${{ parameters.job_condition }}) pool: @@ -125,7 +125,7 @@ jobs: # ----------------------------------------------------------------- - ${{ each os in parameters.os_platforms }}: - job: ${{ os.name }}_${{ replace(pyver.name, '.', '_') }} - displayName: Py 3rd-Party Pkgs - ${{ pyver.name }} - ${{ os.short }} + displayName: Pkg Req - Py ${{ pyver.name }} - ${{ os.short }} ${{ if os.prev }}: dependsOn: ${{ os.prev }}_${{ replace(pyver.name, '.', '_') }} ${{ else }}: diff --git a/resources/python/requirements.txt b/resources/python/requirements.txt index 446da6397..7b2d30750 100644 --- a/resources/python/requirements.txt +++ b/resources/python/requirements.txt @@ -1,22 +1,9 @@ -# Unified requirements file for tk-framework-desktopserver Python dependencies. -# -# This is the single source of truth for all Python dependency versions. -# It serves two purposes: -# -# 1. Building pkgs.zip - passed to update_requirements.py which calls pip -# under each target Python version. Pip env markers are resolved -# automatically based on the running interpreter. -# -# 2. CI test dependencies - referenced in azure-pipelines.yml as: -# extra_test_dependencies: -# - --requirement resources/python/requirements.txt -# # Supported Python versions: # - 3.7 (legacy; Twisted capped at 22.10.0) # - 3.9 (VFX CY2022) # - 3.10 (VFX CY2023) # - 3.11 (VFX CY2024) -# - 3.13 (VFX CY2026; 3.12 was skipped - not a VFX platform year) +# - 3.13 (VFX CY2026) # # When updating a dependency, run update_requirements.py for each Python # version to regenerate the frozen explicit_requirements.txt files, then @@ -32,8 +19,8 @@ # 24.10.0 is used for 3.9-3.11. # 24.11+ is required for 3.13 (dropped legacy code paths incompatible with # the new interpreter). -Twisted==22.10.0; python_version == "3.7" -Twisted==24.10.0; python_version > "3.7" and python_version < "3.13" +Twisted==22.10.0; python_version < "3.9" +Twisted==24.10.0; python_version >= "3.9" and python_version < "3.13" Twisted~=24.11.0; python_version >= "3.13" # autobahn - WebSocket and WAMP protocol implementation on top of Twisted. @@ -58,7 +45,8 @@ service-identity~=24.2.0; python_version >= "3.13" # cryptography - low-level cryptography backend for pyOpenSSL. # Binary package (wheel). NOTE: only needed for SAST scanning, not runtime. # 44.x compatible across all supported Python versions. -cryptography==44.0.1 +cryptography==44.0.3; python_version < "3.13" +cryptography~=44.0.1; python_version >= "3.13" # cffi - C foreign function interface; dependency of cryptography. # Binary package (wheel). 1.17.x required for Python 3.13 (updated C API From 88ae513a308ea0d4df22bc4c24f93bce6822dd6c Mon Sep 17 00:00:00 2001 From: Julien Langlois Date: Thu, 5 Mar 2026 19:27:00 -0800 Subject: [PATCH 3/3] fixup! cleanup --- resources/python/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/python/requirements.txt b/resources/python/requirements.txt index 7b2d30750..2cfdfd11d 100644 --- a/resources/python/requirements.txt +++ b/resources/python/requirements.txt @@ -45,8 +45,8 @@ service-identity~=24.2.0; python_version >= "3.13" # cryptography - low-level cryptography backend for pyOpenSSL. # Binary package (wheel). NOTE: only needed for SAST scanning, not runtime. # 44.x compatible across all supported Python versions. -cryptography==44.0.3; python_version < "3.13" -cryptography~=44.0.1; python_version >= "3.13" +cryptography==44.0.1; python_version < "3.13" +cryptography~=44.0.3; python_version >= "3.13" # cffi - C foreign function interface; dependency of cryptography. # Binary package (wheel). 1.17.x required for Python 3.13 (updated C API