diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9e9ad6bb..8d7f18d4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,12 +1,12 @@ name: OpenRV on: - push: - branches: - # This will run when PR is merged or direct pushes to main. - - main + # push: + # branches: + # # This will run when PR is merged or direct pushes to main. + # - main - pull_request: # This handles PR creation and subsequent commits. + # pull_request: # This handles PR creation and subsequent commits. schedule: # Midnight build every day @@ -22,7 +22,7 @@ on: jobs: detect-changes: - if: github.repository_owner == 'AcademySoftwareFoundation' + #if: github.repository_owner == 'AcademySoftwareFoundation' runs-on: ubuntu-latest outputs: cmake_changed: ${{ steps.filter.outputs.cmake }} diff --git a/.github/workflows/conan-upload-deps.yml b/.github/workflows/conan-upload-deps.yml new file mode 100644 index 000000000..677dc82fa --- /dev/null +++ b/.github/workflows/conan-upload-deps.yml @@ -0,0 +1,542 @@ +name: Conan Upload Deps + +# Batch-builds OpenRV's Conan deps declared in conan/packages.yml and +# uploads them to the ASWF Conan remote. Year-specific deps (CY2025.cmake) +# are published under @openrv/vfx2025; year-independent deps (CYCOMMON.cmake) +# under @openrv/common. Channel is driven by the `channel` field in +# packages.yml. Mirrors the phase pattern already proven in +# conan-upload-test.yml (openssl) and covers every platform in conan.yml's +# build matrix: +# +# rocky8 - amd64/rockylinux:8 + gcc-toolset-11 (profile x86_64_rocky8) +# macos - macos-14 (arm64) + Apple clang (profile arm64_apple_release) +# windows - windows-2022 + MSVC + MSYS2 (profile x86_64_windows) +# +# Rocky 9 is intentionally NOT uploaded. Its Conan profile is identical +# to Rocky 8's (os=Linux, compiler=gcc/11/libstdc++11), so binaries +# uploaded from a Rocky 9 runner share the same package_id as Rocky 8 +# but carry newer glibc / libstdc++ symbol requirements and break Rocky +# 8 consumers at link time. VFX CY2025 defines one Linux target (glibc +# 2.28 = Rocky 8); Rocky 9 consumes the Rocky 8 binaries, which are +# forward-compatible. +# +# Platform binaries live under the same @openrv/ ref; Conan keeps +# them separate by package_id derived from settings. +# +# Each matrix cell: +# 1. Exports every packages.yml recipe under @openrv/ so that +# replace_requires in conan/profiles/common can resolve transitive +# deps locally. +# 2. Runs `conan install --requires=@openrv/ --build=missing` +# with the full option union across packages.yml, plus forced local +# rebuild of glibc-sensitive host tools (b2, meson, nasm) to defend +# against CCI prebuilts built on newer hosts. +# 3. Uploads every @openrv/* ref in cache (target + transitives). +# +# pcre2 is windows_only and is only present in the Windows matrix. + +on: + push: + branches: + - main + + pull_request: + + workflow_dispatch: + inputs: + remote: + description: "Target Conan remote" + required: true + default: "aswftesting" + type: choice + options: + - aswftesting + - aswf + only: + description: "Single package name to run (empty = all)" + required: false + default: "" + platforms: + description: "Comma-separated subset of rocky8,macos,windows (empty = all)" + required: false + default: "rocky8,macos,windows" + +env: + REMOTE_NAME: ${{ inputs.remote || 'aswftesting' }} + REMOTE_URL: ${{ inputs.remote == 'aswf' && 'https://linuxfoundation.jfrog.io/artifactory/api/conan/aswf-conan' || 'https://linuxfoundation.jfrog.io/artifactory/api/conan/aswf-conan-dev' }} + CMAKE_VERSION: "3.31.6" + PYTHON_VERSION: "3.11" + PLATFORMS: ${{ inputs.platforms || 'rocky8,macos,windows' }} + +jobs: + # --------------------------------------------------------------------------- + # Validates that conan/profiles/common [replace_requires] is in sync with + # conan/packages.yml. Fails fast before any build jobs run. + # To fix: run `python3 conan/scripts/gen_profile_common.py` and commit. + # --------------------------------------------------------------------------- + check-profile-sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + submodules: false + + - name: Setup Python + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: "3.11" + + - name: Install PyYAML + run: python3 -m pip install --user pyyaml + + - name: Validate profiles/common matches packages.yml + run: python3 conan/scripts/gen_profile_common.py --check + + # --------------------------------------------------------------------------- + # Reads conan/packages.yml and emits two matrices: + # matrix_nonwindows - excludes windows_only entries (Linux + macOS jobs) + # matrix_windows - all entries, including windows_only (Windows job) + # Optionally narrows to a single package via the `only` input. + # --------------------------------------------------------------------------- + prepare-matrix: + needs: check-profile-sync + runs-on: ubuntu-latest + outputs: + matrix_nonwindows: ${{ steps.gen.outputs.matrix_nonwindows }} + matrix_windows: ${{ steps.gen.outputs.matrix_windows }} + count_nonwindows: ${{ steps.gen.outputs.count_nonwindows }} + count_windows: ${{ steps.gen.outputs.count_windows }} + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + submodules: false + + - name: Setup Python + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: "3.11" + + - name: Install PyYAML + run: python3 -m pip install --user pyyaml + + - name: Generate matrix from conan/packages.yml + id: gen + env: + ONLY: ${{ inputs.only || '' }} + run: | + if [ -n "$ONLY" ]; then + python3 conan/scripts/gen_matrix.py --only "$ONLY" >> "$GITHUB_OUTPUT" + else + python3 conan/scripts/gen_matrix.py >> "$GITHUB_OUTPUT" + fi + # --------------------------------------------------------------------------- + # Rocky Linux 8 - gcc-toolset-11, Python 3.11 from AppStream. + # --------------------------------------------------------------------------- + rocky8: + name: "rocky8: ${{ matrix.name }}/${{ matrix.version }} -> ${{ inputs.remote || 'aswftesting' }}" + needs: prepare-matrix + if: needs.prepare-matrix.outputs.count_nonwindows != '0' && contains(format(',{0},', inputs.platforms || 'rocky8,macos,windows'), ',rocky8,') + runs-on: ubuntu-latest + container: + image: amd64/rockylinux:8 + + strategy: + fail-fast: false + max-parallel: 8 + matrix: + include: ${{ fromJSON(needs.prepare-matrix.outputs.matrix_nonwindows) }} + + env: + CONAN_LOGIN_USERNAME: ${{ secrets.ARTIFACTORY_USER }} + CONAN_PASSWORD: ${{ secrets.ARTIFACTORY_TOKEN }} + CONAN_PROFILE: x86_64_rocky8 + IS_WINDOWS_JOB: "0" + CCI_DIR: /tmp/cci + + steps: + - name: Verify secrets are set + run: | + if [ -z "$CONAN_LOGIN_USERNAME" ] || [ -z "$CONAN_PASSWORD" ]; then + echo "::error::ARTIFACTORY_USER or ARTIFACTORY_TOKEN secret is not set on this repo." + exit 1 + fi + - name: Install base system packages + # VFX Reference Platform CY2025 mandates Python 3.11. Rocky 8's + # default python3 is 3.6.8, too old for meson (>=3.7). Install + # python3.11 from AppStream and expose it as the default `python3` + # via /usr/local/bin so every subsequent step picks it up. + run: | + dnf install -y epel-release + dnf config-manager --set-enabled powertools devel + dnf groupinstall "Development Tools" -y + dnf install -y which findutils git python3.11 python3.11-pip perl perl-IPC-Cmd perl-Digest-SHA nasm make + ln -sf /usr/bin/python3.11 /usr/local/bin/python3 + ln -sf /usr/bin/pip3.11 /usr/local/bin/pip3 + python3 --version + dnf clean all + - name: Install gcc-toolset-11 + run: | + for i in 1 2 3 4 5; do + dnf install -y gcc-toolset-11-toolchain && break + echo "attempt $i failed, retrying in 5s"; sleep 5 + done + - name: Enable gcc-toolset-11 + run: | + source /opt/rh/gcc-toolset-11/enable + echo "/opt/rh/gcc-toolset-11/root/usr/bin" >> $GITHUB_PATH + echo "CC=/opt/rh/gcc-toolset-11/root/usr/bin/gcc" >> $GITHUB_ENV + echo "CXX=/opt/rh/gcc-toolset-11/root/usr/bin/g++" >> $GITHUB_ENV + gcc --version + - name: Install CMake + # Install under /opt so the subsequent actions/checkout step does + # not wipe it, then symlink into /usr/local/bin so Conan's + # buildenv-isolated PATH can still find cmake. + run: | + curl -SL -o /tmp/cmake.tar.gz "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" + mkdir -p /opt + tar -xzf /tmp/cmake.tar.gz -C /opt + ln -sf "/opt/cmake-${CMAKE_VERSION}-linux-x86_64/bin/cmake" /usr/local/bin/cmake + ln -sf "/opt/cmake-${CMAKE_VERSION}-linux-x86_64/bin/ctest" /usr/local/bin/ctest + ln -sf "/opt/cmake-${CMAKE_VERSION}-linux-x86_64/bin/cpack" /usr/local/bin/cpack + echo "/opt/cmake-${CMAKE_VERSION}-linux-x86_64/bin" >> $GITHUB_PATH + cmake --version + - name: Install Conan 2.x + PyYAML + run: | + python3 -m pip install --upgrade pip + python3 -m pip install "conan>=2.0,<3.0" pyyaml + conan --version + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + submodules: false + + - name: Mark repo as safe git directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Configure Conan remote and authenticate + run: | + conan remote remove "$REMOTE_NAME" 2>/dev/null || true + conan remote add "$REMOTE_NAME" "$REMOTE_URL" --index 0 + conan remote list + conan remote login "$REMOTE_NAME" "$CONAN_LOGIN_USERNAME" -p "$CONAN_PASSWORD" + - name: Detect default Conan profile + run: conan profile detect --force + + - name: Clone conan-center-index + run: git clone --depth 1 --branch master https://github.com/conan-io/conan-center-index.git "$CCI_DIR" + + - name: Export every packages.yml recipe + run: python3 ./conan/scripts/export_recipes.py + + - name: Build ${{ matrix.name }}/${{ matrix.version }}@openrv/${{ matrix.channel }} + run: | + # Windows-only: pick up the MSVC bin dir that the earlier + # "Add MSVC cl.exe to msys2 PATH" step appended to bash_profile. + # Harmless no-op on Linux/macOS where the file does not exist. + [ -f ~/.bash_profile ] && source ~/.bash_profile || true + python3 ./conan/scripts/conan_install_target.py \ + "${{ matrix.name }}" "${{ matrix.version }}" "${CONAN_PROFILE}" + conan list "${{ matrix.name }}/${{ matrix.version }}@openrv/${{ matrix.channel }}:*" + - name: Upload all @openrv/* refs built in this cell + run: | + upload_fail=0 + CONAN_CMD="${CONAN_EXEC:-conan}" + for channel in common vfx2025; do + pattern="*@openrv/$channel" + set +e + output=$("$CONAN_CMD" upload "$pattern" --remote "$REMOTE_NAME" --confirm 2>&1) + rc=$? + set -e + echo "$output" + if [ $rc -eq 0 ]; then + echo "Upload of $pattern succeeded" + elif echo "$output" | grep -qi "permission denied\|403\|DELETE"; then + echo "::warning::Upload of $pattern hit 403 (recipe already on remote with different revision). Skipping." + else + echo "::error::Upload of $pattern failed with unexpected error" + upload_fail=1 + fi + done + [ "$upload_fail" -eq 0 ] || exit 1 + + - name: Verify server-side listing of target + run: conan search "${{ matrix.name }}/${{ matrix.version }}@openrv/${{ matrix.channel }}" -r "$REMOTE_NAME" + + # --------------------------------------------------------------------------- + # macOS arm64 - macos-14 native runner, Apple clang, Python 3.11 via setup-python. + # --------------------------------------------------------------------------- + macos: + name: "macos-arm64: ${{ matrix.name }}/${{ matrix.version }} -> ${{ inputs.remote || 'aswftesting' }}" + needs: prepare-matrix + if: needs.prepare-matrix.outputs.count_nonwindows != '0' && contains(format(',{0},', inputs.platforms || 'rocky8,macos,windows'), ',macos,') + runs-on: macos-14 + + strategy: + fail-fast: false + max-parallel: 4 + matrix: + include: ${{ fromJSON(needs.prepare-matrix.outputs.matrix_nonwindows) }} + + env: + CONAN_LOGIN_USERNAME: ${{ secrets.ARTIFACTORY_USER }} + CONAN_PASSWORD: ${{ secrets.ARTIFACTORY_TOKEN }} + CONAN_PROFILE: arm64_apple_release + IS_WINDOWS_JOB: "0" + CCI_DIR: /tmp/cci + # Matches conan.yml's pattern: actions/setup-python + `pip install + # --user` on macOS puts the conan CLI in ~/Library/Python/3.11/bin, + # which is NOT on PATH by default. Hardcode the full path so every + # step (and the helper scripts via $CONAN_EXEC) calls the right + # binary without relying on GITHUB_PATH ordering. + CONAN_EXEC: /Users/runner/Library/Python/3.11/bin/conan + + steps: + - name: Verify secrets are set + run: | + if [ -z "$CONAN_LOGIN_USERNAME" ] || [ -z "$CONAN_PASSWORD" ]; then + echo "::error::ARTIFACTORY_USER or ARTIFACTORY_TOKEN secret is not set." + exit 1 + fi + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + submodules: false + + - name: Install CMake + uses: jwlawson/actions-setup-cmake@802fa1a2c4e212495c05bf94dba2704a92a472be # v2 + with: + cmake-version: "${{ env.CMAKE_VERSION }}" + + - name: Setup Python + # Runs BEFORE `brew install` so setup-python's PATH prepend wins + # the resolution race. `brew install meson` pulls python@3.x as a + # dep; if brew runs first its python can end up winning `python3` + # for later steps, breaking the `import yaml` the helper scripts + # rely on. + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: "3.11" + + - name: Install Conan 2.x + PyYAML + # --user lands the CLI in $CONAN_EXEC (see job env). The import + # sanity check fails the step early if the Python that will run + # the helper scripts cannot see the packages we just installed. + run: | + python3 -m pip install --user --upgrade pip + python3 -m pip install --user "conan>=2.0,<3.0" pyyaml + python3 -c "import yaml, conan; print(f'yaml {yaml.__version__}, conan {conan.__version__}')" + "$CONAN_EXEC" --version + - name: Install Homebrew build tools + # nasm, meson: needed as fallbacks for recipes that do not + # tool_require them on macos. autoconf/automake/libtool: GNU + # autotools recipes need these. pkg-config: shared across recipes. + run: brew install nasm meson autoconf automake libtool pkg-config + + - name: Configure Conan remote and authenticate + run: | + "$CONAN_EXEC" remote remove "$REMOTE_NAME" 2>/dev/null || true + "$CONAN_EXEC" remote add "$REMOTE_NAME" "$REMOTE_URL" --index 0 + "$CONAN_EXEC" remote list + "$CONAN_EXEC" remote login "$REMOTE_NAME" "$CONAN_LOGIN_USERNAME" -p "$CONAN_PASSWORD" + - name: Detect default Conan profile + run: | + "$CONAN_EXEC" profile detect --force + - name: Clone conan-center-index + run: git clone --depth 1 --branch master https://github.com/conan-io/conan-center-index.git "$CCI_DIR" + + - name: Export every packages.yml recipe + run: python3 ./conan/scripts/export_recipes.py + + - name: Build ${{ matrix.name }}/${{ matrix.version }}@openrv/${{ matrix.channel }} + run: | + python3 ./conan/scripts/conan_install_target.py \ + "${{ matrix.name }}" "${{ matrix.version }}" "${CONAN_PROFILE}" + "$CONAN_EXEC" list "${{ matrix.name }}/${{ matrix.version }}@openrv/${{ matrix.channel }}:*" + - name: Upload all @openrv/* refs built in this cell + run: | + upload_fail=0 + CONAN_CMD="${CONAN_EXEC:-conan}" + for channel in common vfx2025; do + pattern="*@openrv/$channel" + set +e + output=$("$CONAN_CMD" upload "$pattern" --remote "$REMOTE_NAME" --confirm 2>&1) + rc=$? + set -e + echo "$output" + if [ $rc -eq 0 ]; then + echo "Upload of $pattern succeeded" + elif echo "$output" | grep -qi "permission denied\|403\|DELETE"; then + echo "::warning::Upload of $pattern hit 403 (recipe already on remote with different revision). Skipping." + else + echo "::error::Upload of $pattern failed with unexpected error" + upload_fail=1 + fi + done + [ "$upload_fail" -eq 0 ] || exit 1 + - name: Verify server-side listing of target + run: | + "$CONAN_EXEC" search "${{ matrix.name }}/${{ matrix.version }}@openrv/${{ matrix.channel }}" -r "$REMOTE_NAME" + # --------------------------------------------------------------------------- + # Windows x86_64 - windows-2022, MSVC 2022, MSYS2 shell. + # Includes the windows_only entry (pcre2). + # --------------------------------------------------------------------------- + windows: + name: "windows-x86_64: ${{ matrix.name }}/${{ matrix.version }} -> ${{ inputs.remote || 'aswftesting' }}" + needs: prepare-matrix + if: needs.prepare-matrix.outputs.count_windows != '0' && contains(format(',{0},', inputs.platforms || 'rocky8,macos,windows'), ',windows,') + runs-on: windows-2022 + + strategy: + fail-fast: false + max-parallel: 4 + matrix: + include: ${{ fromJSON(needs.prepare-matrix.outputs.matrix_windows) }} + + env: + CONAN_LOGIN_USERNAME: ${{ secrets.ARTIFACTORY_USER }} + CONAN_PASSWORD: ${{ secrets.ARTIFACTORY_TOKEN }} + CONAN_PROFILE: x86_64_windows + IS_WINDOWS_JOB: "1" + # Use a native Windows path; mingw Python and MSYS2 bash both + # agree on drive-letter paths, but disagree on /tmp. + CCI_DIR: "D:/cci" + # Tell MSYS2 to leave these Windows-style path lists alone. + # Without this, MSYS2 rewrites "C:\Foo;C:\Bar" to "/c/Foo:/c/Bar" + # and MSVC's cl.exe loses track of its headers / libs. + MSYS2_ENV_CONV_EXCL: "INCLUDE;LIB;LIBPATH" + + defaults: + run: + shell: msys2 {0} + + steps: + - name: Verify secrets are set + shell: bash + run: | + if [ -z "$CONAN_LOGIN_USERNAME" ] || [ -z "$CONAN_PASSWORD" ]; then + echo "::error::ARTIFACTORY_USER or ARTIFACTORY_TOKEN secret is not set." + exit 1 + fi + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + submodules: false + + - name: Setup MSVC + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Get Python install path + shell: powershell + run: | + $pythonDir = python -c "import sys, os; print(os.path.dirname(sys.executable))" + echo "WIN_PYTHON_DIR=$pythonDir" >> $env:GITHUB_ENV + + - name: Setup MSYS2 + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + update: true + cache: false + install: >- + mingw-w64-x86_64-toolchain + mingw-w64-x86_64-cmake + mingw-w64-x86_64-make + mingw-w64-x86_64-meson + autoconf + automake + libtool + nasm + perl + make + git + - name: Add MSVC + Windows SDK to msys2 PATH + # ilammy/msvc-dev-cmd puts cl.exe AND Windows SDK bins (mt.exe, + # rc.exe, signtool.exe, ...) on the runner's Windows PATH, but + # msys2 bash launches with its own PATH. Translate both roots to + # POSIX paths and persist them via ~/.bash_profile so every + # subsequent `msys2 {0}` step can invoke cl + mt (b2's bootstrap + # and the MSVC linker need both). + run: | + if [ -z "$VCToolsInstallDir" ]; then + echo "::error::VCToolsInstallDir not set; Setup MSVC step did not run." + exit 1 + fi + MSVC_BIN=$(cygpath -u "$VCToolsInstallDir/bin/Hostx64/x64") + echo "export PATH=\"$MSVC_BIN:\$PATH\"" >> ~/.bash_profile + echo "Added MSVC bin to PATH: $MSVC_BIN" + if [ -n "$WindowsSdkVerBinPath" ]; then + # $WindowsSdkVerBinPath ends with a trailing backslash; cygpath + # normalises and we append /x64 for the arch-specific tools. + WINSDK_BIN=$(cygpath -u "$WindowsSdkVerBinPath") + echo "export PATH=\"$WINSDK_BIN/x64:\$PATH\"" >> ~/.bash_profile + echo "Added Windows SDK bin to PATH: $WINSDK_BIN/x64" + else + echo "::error::WindowsSdkVerBinPath not set; mt.exe will be unreachable." + exit 1 + fi + WIN_PYTHON_DIR=$(cygpath -u "$WIN_PYTHON_DIR") + echo "export PATH=\"$WIN_PYTHON_DIR:\$PATH\"" >> ~/.bash_profile + echo "Added Python to PATH: $WIN_PYTHON_DIR" + - name: Install Conan 2.x + run: | + source ~/.bash_profile + python3 -m pip install --user --upgrade pip + python3 -m pip install --user "conan>=2.0,<3.0" pyyaml + PYTHON_SCRIPTS_DIR=$(python3 -c "import site, os; scripts = site.getusersitepackages(); print(os.path.join(os.path.dirname(scripts), 'Scripts'))") + PYTHON_SCRIPTS_MSYS=$(cygpath -u "$PYTHON_SCRIPTS_DIR") + echo "export PATH=\"\$PATH:$PYTHON_SCRIPTS_MSYS\"" >> ~/.bash_profile + source ~/.bash_profile + python3 --version + conan --version + - name: Configure Conan remote and authenticate + run: | + conan remote remove "$REMOTE_NAME" 2>/dev/null || true + conan remote add "$REMOTE_NAME" "$REMOTE_URL" --index 0 + conan remote list + conan remote login "$REMOTE_NAME" "$CONAN_LOGIN_USERNAME" -p "$CONAN_PASSWORD" + - name: Detect default Conan profile + run: conan profile detect --force + + - name: Clone conan-center-index + run: git clone --depth 1 --branch master https://github.com/conan-io/conan-center-index.git "$CCI_DIR" + + - name: Export every packages.yml recipe + run: python3 ./conan/scripts/export_recipes.py + + - name: Build ${{ matrix.name }}/${{ matrix.version }}@openrv/${{ matrix.channel }} + run: | + # Windows-only: pick up the MSVC bin dir that the earlier + # "Add MSVC cl.exe to msys2 PATH" step appended to bash_profile. + # Harmless no-op on Linux/macOS where the file does not exist. + [ -f ~/.bash_profile ] && source ~/.bash_profile || true + python3 ./conan/scripts/conan_install_target.py \ + "${{ matrix.name }}" "${{ matrix.version }}" "${CONAN_PROFILE}" + conan list "${{ matrix.name }}/${{ matrix.version }}@openrv/${{ matrix.channel }}:*" + - name: Upload all @openrv/* refs built in this cell + run: | + upload_fail=0 + CONAN_CMD="${CONAN_EXEC:-conan}" + for channel in common vfx2025; do + pattern="*@openrv/$channel" + set +e + output=$("$CONAN_CMD" upload "$pattern" --remote "$REMOTE_NAME" --confirm 2>&1) + rc=$? + set -e + echo "$output" + if [ $rc -eq 0 ]; then + echo "Upload of $pattern succeeded" + elif echo "$output" | grep -qi "permission denied\|403\|DELETE"; then + echo "::warning::Upload of $pattern hit 403 (recipe already on remote with different revision). Skipping." + else + echo "::error::Upload of $pattern failed with unexpected error" + upload_fail=1 + fi + done + [ "$upload_fail" -eq 0 ] || exit 1 + + - name: Verify server-side listing of target + run: conan search "${{ matrix.name }}/${{ matrix.version }}@openrv/${{ matrix.channel }}" -r "$REMOTE_NAME" \ No newline at end of file diff --git a/.github/workflows/conan-upload-test.yml b/.github/workflows/conan-upload-test.yml new file mode 100644 index 000000000..1f800908f --- /dev/null +++ b/.github/workflows/conan-upload-test.yml @@ -0,0 +1,681 @@ +name: Conan Upload Test (ASWF Artifactory) + +# Manual-only workflow for probing ASWF Artifactory write access and +# publishing prebuilt OpenRV packages under @openrv/vfxYYYY. +# +# Phases: +# probe = Phase A.5 only (upload a tiny dummy recipe to confirm +# the @openrv/* path is writable). ~2 min per platform. +# openssl = Phase A.5 + B + C + D + E (probe, then build openssl +# under @openrv/vfx2024, upload, round-trip install, +# profile graph verify). 10-20 min per platform. +# +# Target remote defaults to aswftesting (aswf-conan-dev) for safety. +# Switch to aswf (production) only after aswftesting succeeds and the +# namespace policy is confirmed. + +on: + workflow_dispatch: + inputs: + phase: + description: "Which phase to run" + required: true + default: "probe" + type: choice + options: + - probe + - openssl + remote: + description: "Target Conan remote" + required: true + default: "aswftesting" + type: choice + options: + - aswftesting + - aswf + platforms: + description: "Comma-separated subset of rocky8,rocky9,macos,windows (empty = all)" + required: false + default: "rocky8,rocky9,macos,windows" + +env: + # Shared across all jobs. Not platform-dependent. + OPENSSL_VERSION: "3.5.6" + CMAKE_VERSION: "3.31.6" + # inputs.* is only populated for workflow_dispatch. For push / PR + # triggers these are empty; each job applies its own fallback. + # TEMP: phase default is 'openssl' while dispatch is unavailable and + # the cross-platform openssl run is being validated. Flip back to + # 'probe' once the four-platform run passes. + PHASE: ${{ inputs.phase || 'openssl' }} + REMOTE_NAME: ${{ inputs.remote || 'aswftesting' }} + REMOTE_URL: ${{ inputs.remote == 'aswf' && 'https://linuxfoundation.jfrog.io/artifactory/api/conan/aswf-conan' || 'https://linuxfoundation.jfrog.io/artifactory/api/conan/aswf-conan-dev' }} + PLATFORMS: ${{ inputs.platforms || 'rocky8,rocky9,macos,windows' }} + +jobs: + # --------------------------------------------------------------------------- + # Rocky Linux (8 + 9). Matrix over rocky-version; job runs per row. + # --------------------------------------------------------------------------- + rocky-linux: + name: "Conan ${{ inputs.phase || 'probe' }} -> rocky${{ matrix.rocky-version }} -> ${{ inputs.remote || 'aswftesting' }}" + # GitHub Actions does not expose matrix.* in job-level if: expressions, + # so we cannot filter rocky8 vs rocky9 independently here. If either + # token appears in platforms, both matrix rows run; if neither does, + # neither runs. Selecting "rocky8 only" requires editing the matrix. + if: contains(format(',{0},', inputs.platforms || 'rocky8,rocky9,macos,windows'), ',rocky8,') || contains(format(',{0},', inputs.platforms || 'rocky8,rocky9,macos,windows'), ',rocky9,') + runs-on: ubuntu-latest + container: + image: ${{ matrix.image }} + + strategy: + fail-fast: false + matrix: + include: + - rocky-version: "8" + image: "amd64/rockylinux:8" + extra_repo: "powertools" + conan-profile: "x86_64_rocky8" + - rocky-version: "9" + image: "amd64/rockylinux:9" + extra_repo: "crb" + conan-profile: "x86_64_rocky9" + + env: + CONAN_LOGIN_USERNAME: ${{ secrets.ARTIFACTORY_USER }} + CONAN_PASSWORD: ${{ secrets.ARTIFACTORY_TOKEN }} + CONAN_PROFILE: ${{ matrix.conan-profile }} + + steps: + - name: Diagnose secret presence + env: + U: ${{ secrets.ARTIFACTORY_USER }} + T: ${{ secrets.ARTIFACTORY_TOKEN }} + run: | + echo "event: ${GITHUB_EVENT_NAME}" + echo "actor: ${GITHUB_ACTOR}" + echo "repo: ${GITHUB_REPOSITORY}" + echo "ref: ${GITHUB_REF}" + echo "platform: rocky${{ matrix.rocky-version }} (${{ matrix.conan-profile }})" + echo "ARTIFACTORY_USER length: ${#U}" + echo "ARTIFACTORY_TOKEN length: ${#T}" + echo "CONAN_LOGIN_USERNAME (env) length: ${#CONAN_LOGIN_USERNAME}" + echo "CONAN_PASSWORD (env) length: ${#CONAN_PASSWORD}" + + - name: Verify secrets are set + run: | + if [ -z "$CONAN_LOGIN_USERNAME" ] || [ -z "$CONAN_PASSWORD" ]; then + echo "::error::ARTIFACTORY_USER or ARTIFACTORY_TOKEN secret is not set on this repo." + exit 1 + fi + + - name: Install base system packages + run: | + dnf install -y epel-release + dnf config-manager --set-enabled ${{ matrix.extra_repo }} devel + dnf groupinstall "Development Tools" -y + dnf install -y which findutils git python3 python3-pip perl perl-IPC-Cmd perl-Digest-SHA nasm make + dnf clean all + + - name: Install gcc-toolset-11 (rocky8 only) + if: matrix.rocky-version == '8' + run: | + for i in 1 2 3 4 5; do + dnf install -y gcc-toolset-11-toolchain && break + echo "attempt $i failed, retrying in 5s"; sleep 5 + done + + - name: Enable gcc-toolset-11 (rocky8 only) + if: matrix.rocky-version == '8' + run: | + source /opt/rh/gcc-toolset-11/enable + echo "/opt/rh/gcc-toolset-11/root/usr/bin" >> $GITHUB_PATH + echo "CC=/opt/rh/gcc-toolset-11/root/usr/bin/gcc" >> $GITHUB_ENV + echo "CXX=/opt/rh/gcc-toolset-11/root/usr/bin/g++" >> $GITHUB_ENV + gcc --version + + - name: Install Rocky 9 perl CPAN (rocky9 only) + if: matrix.rocky-version == '9' + run: | + dnf install -y perl-CPAN + cpan FindBin || true + + - name: Install CMake + run: | + curl -SL -o cmake.tar.gz "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" + tar -xzf cmake.tar.gz + mv "cmake-${CMAKE_VERSION}-linux-x86_64" "./cmake-${CMAKE_VERSION}" + echo "$(pwd)/cmake-${CMAKE_VERSION}/bin" >> $GITHUB_PATH + "$(pwd)/cmake-${CMAKE_VERSION}/bin/cmake" --version + + - name: Install Conan 2.x + run: | + python3 -m pip install --upgrade pip + python3 -m pip install "conan>=2.0,<3.0" + conan --version + + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + submodules: false + + - name: Mark repo as safe git directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Configure Conan remote and authenticate + run: | + conan remote remove "$REMOTE_NAME" 2>/dev/null || true + conan remote add "$REMOTE_NAME" "$REMOTE_URL" --index 0 + conan remote list + conan remote login "$REMOTE_NAME" "$CONAN_LOGIN_USERNAME" -p "$CONAN_PASSWORD" + + - name: Detect default Conan profile + run: conan profile detect --force + + - name: Phase A.5 - Create probe recipe + run: | + mkdir -p /tmp/openrv-probe + cat > /tmp/openrv-probe/conanfile.py <<'PY' + from conan import ConanFile + + class Probe(ConanFile): + name = "openrv-probe" + version = "0.0.1" + package_type = "application" + settings = "os", "arch" + + def package(self): + pass + PY + conan create /tmp/openrv-probe --user openrv --channel test + + - name: Phase A.5 - Upload probe to ${{ inputs.remote || 'aswftesting' }} + id: probe_upload + run: | + set +e + out=$(conan upload "openrv-probe/0.0.1@openrv/test" --remote "$REMOTE_NAME" --confirm 2>&1) + status=$? + echo "$out" + echo "----" + if [ $status -eq 0 ]; then + echo "::notice::Probe upload succeeded. @openrv/* writes are permitted on $REMOTE_NAME." + else + if echo "$out" | grep -qiE "403|forbidden|unauthorized"; then + echo "::error::Probe upload rejected. Check LF Artifactory permission target for aswf-conan*/openrv/**." + fi + exit $status + fi + + - name: Phase A.5 - Clean up probe (local + best-effort remote) + if: always() && steps.probe_upload.outcome == 'success' + run: | + # Remote cleanup typically fails 403 on LF Artifactory because + # the OpenRV token has upload-only rights on @openrv/*. Treat + # as expected; an admin can clean the probe ref periodically. + if conan remove "openrv-probe/*" -r "$REMOTE_NAME" -c 2>/dev/null; then + echo "::notice::Remote probe cleaned from $REMOTE_NAME." + else + echo "::notice::Remote probe cleanup returned non-zero (expected for upload-only tokens). Leaving openrv-probe on $REMOTE_NAME." + fi + conan remove "openrv-probe/*" -c 2>/dev/null || true + + - name: Phase B - Clone conan-center-index and verify openssl version + if: ${{ env.PHASE == 'openssl' }} + run: | + git clone --depth 1 --branch master https://github.com/conan-io/conan-center-index.git /tmp/cci + echo "::group::CCI openssl config.yml" + cat /tmp/cci/recipes/openssl/config.yml + echo "::endgroup::" + if ! grep -q "\"${OPENSSL_VERSION}\"" /tmp/cci/recipes/openssl/config.yml; then + echo "::error::openssl ${OPENSSL_VERSION} not found in CCI config.yml. Update OPENSSL_VERSION env." + exit 1 + fi + + - name: Phase B - Build openssl/${{ env.OPENSSL_VERSION }}@openrv/vfx2024 + if: ${{ env.PHASE == 'openssl' }} + run: | + conan create /tmp/cci/recipes/openssl/3.x.x/ \ + --version "${OPENSSL_VERSION}" \ + --user openrv \ + --channel vfx2024 \ + --build=missing \ + -tf="" \ + -pr:a "./conan/profiles/${CONAN_PROFILE}" \ + -o 'openssl/*:shared=True' \ + -o 'openssl/*:no_zlib=True' + conan list "openssl/${OPENSSL_VERSION}@openrv/vfx2024:*" + + - name: Phase C - Upload openssl/${{ env.OPENSSL_VERSION }}@openrv/vfx2024 + if: ${{ env.PHASE == 'openssl' }} + run: conan upload "openssl/${OPENSSL_VERSION}@openrv/vfx2024" --remote "$REMOTE_NAME" --confirm + + - name: Verify server-side listing + if: ${{ env.PHASE == 'openssl' }} + run: conan search "openssl/${OPENSSL_VERSION}@openrv/vfx2024" -r "$REMOTE_NAME" + + - name: Phase D - Wipe local cache for openssl + if: ${{ env.PHASE == 'openssl' }} + run: conan remove "openssl/${OPENSSL_VERSION}@openrv/vfx2024" -c + + - name: Phase D - Round-trip install from ${{ inputs.remote || 'aswftesting' }} + if: ${{ env.PHASE == 'openssl' }} + run: | + set -o pipefail + conan install \ + --requires="openssl/${OPENSSL_VERSION}@openrv/vfx2024" \ + -r "$REMOTE_NAME" \ + --build=never \ + -pr:a "./conan/profiles/${CONAN_PROFILE}" \ + -o 'openssl/*:shared=True' \ + -o 'openssl/*:no_zlib=True' 2>&1 | tee /tmp/phase_d.log + if grep -qiE "Building from sources|building package" /tmp/phase_d.log; then + echo "::error::Phase D regressed - Conan is building instead of downloading." + exit 1 + fi + if ! grep -qiE "downloaded|Already installed" /tmp/phase_d.log; then + echo "::error::Phase D log does not show a download or cache hit from $REMOTE_NAME." + exit 1 + fi + echo "::notice::Phase D success - openssl/${OPENSSL_VERSION}@openrv/vfx2024 is consumable from $REMOTE_NAME without a rebuild." + + - name: Phase E - Verify profile rewrites openssl to @openrv/vfx2024 + if: ${{ env.PHASE == 'openssl' }} + run: | + set -o pipefail + conan graph info \ + --requires="openssl/3.5.0" \ + -r "$REMOTE_NAME" \ + -pr:a "./conan/profiles/${CONAN_PROFILE}" \ + -o 'openssl/*:shared=True' \ + -o 'openssl/*:no_zlib=True' 2>&1 | tee /tmp/phase_e.log + expected_ref="openssl/${OPENSSL_VERSION}@openrv/vfx2024" + if ! grep -qF "$expected_ref" /tmp/phase_e.log; then + echo "::error::Phase E failed - graph does not contain $expected_ref." + exit 1 + fi + echo "::notice::Phase E success - profile rewrites openssl/* to $expected_ref." + + # --------------------------------------------------------------------------- + # macOS arm64. GitHub-hosted macos-14 runner. Profile: arm64_apple_release. + # --------------------------------------------------------------------------- + macos: + name: "Conan ${{ inputs.phase || 'probe' }} -> macos-arm64 -> ${{ inputs.remote || 'aswftesting' }}" + if: contains(format(',{0},', inputs.platforms || 'rocky8,rocky9,macos,windows'), ',macos,') + runs-on: macos-14 + + env: + CONAN_LOGIN_USERNAME: ${{ secrets.ARTIFACTORY_USER }} + CONAN_PASSWORD: ${{ secrets.ARTIFACTORY_TOKEN }} + CONAN_PROFILE: arm64_apple_release + + steps: + - name: Diagnose secret presence + env: + U: ${{ secrets.ARTIFACTORY_USER }} + T: ${{ secrets.ARTIFACTORY_TOKEN }} + run: | + echo "event: ${GITHUB_EVENT_NAME}" + echo "platform: macos-arm64 (${CONAN_PROFILE})" + echo "ARTIFACTORY_USER length: ${#U}" + echo "ARTIFACTORY_TOKEN length: ${#T}" + + - name: Verify secrets are set + run: | + if [ -z "$CONAN_LOGIN_USERNAME" ] || [ -z "$CONAN_PASSWORD" ]; then + echo "::error::ARTIFACTORY_USER or ARTIFACTORY_TOKEN secret is not set." + exit 1 + fi + + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + submodules: false + + - name: Install CMake + uses: jwlawson/actions-setup-cmake@802fa1a2c4e212495c05bf94dba2704a92a472be # v2 + with: + cmake-version: "${{ env.CMAKE_VERSION }}" + + - name: Install nasm + run: brew install nasm + + - name: Setup Python + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: "3.11" + + - name: Install Conan 2.x + run: | + python3 -m pip install --upgrade pip + python3 -m pip install "conan>=2.0,<3.0" + conan --version + + - name: Configure Conan remote and authenticate + run: | + conan remote remove "$REMOTE_NAME" 2>/dev/null || true + conan remote add "$REMOTE_NAME" "$REMOTE_URL" --index 0 + conan remote list + conan remote login "$REMOTE_NAME" "$CONAN_LOGIN_USERNAME" -p "$CONAN_PASSWORD" + + - name: Detect default Conan profile + run: conan profile detect --force + + - name: Phase A.5 - Create probe recipe + run: | + mkdir -p /tmp/openrv-probe + cat > /tmp/openrv-probe/conanfile.py <<'PY' + from conan import ConanFile + + class Probe(ConanFile): + name = "openrv-probe" + version = "0.0.1" + package_type = "application" + settings = "os", "arch" + + def package(self): + pass + PY + conan create /tmp/openrv-probe --user openrv --channel test + + - name: Phase A.5 - Upload probe to ${{ inputs.remote || 'aswftesting' }} + id: probe_upload + run: | + set +e + out=$(conan upload "openrv-probe/0.0.1@openrv/test" --remote "$REMOTE_NAME" --confirm 2>&1) + status=$? + echo "$out" + if [ $status -eq 0 ]; then + echo "::notice::Probe upload succeeded (macos)." + else + if echo "$out" | grep -qiE "403|forbidden|unauthorized"; then + echo "::error::Probe upload rejected on macos. Check permissions." + fi + exit $status + fi + + - name: Phase A.5 - Clean up probe (local + best-effort remote) + if: always() && steps.probe_upload.outcome == 'success' + run: | + # Remote cleanup typically fails 403 on LF Artifactory because + # the OpenRV token has upload-only rights on @openrv/*. Treat + # as expected; an admin can clean the probe ref periodically. + if conan remove "openrv-probe/*" -r "$REMOTE_NAME" -c 2>/dev/null; then + echo "::notice::Remote probe cleaned from $REMOTE_NAME." + else + echo "::notice::Remote probe cleanup returned non-zero (expected for upload-only tokens). Leaving openrv-probe on $REMOTE_NAME." + fi + conan remove "openrv-probe/*" -c 2>/dev/null || true + + - name: Phase B - Clone conan-center-index and verify openssl version + if: ${{ env.PHASE == 'openssl' }} + run: | + git clone --depth 1 --branch master https://github.com/conan-io/conan-center-index.git /tmp/cci + echo "::group::CCI openssl config.yml" + cat /tmp/cci/recipes/openssl/config.yml + echo "::endgroup::" + if ! grep -q "\"${OPENSSL_VERSION}\"" /tmp/cci/recipes/openssl/config.yml; then + echo "::error::openssl ${OPENSSL_VERSION} not found in CCI config.yml." + exit 1 + fi + + - name: Phase B - Build openssl/${{ env.OPENSSL_VERSION }}@openrv/vfx2024 + if: ${{ env.PHASE == 'openssl' }} + run: | + conan create /tmp/cci/recipes/openssl/3.x.x/ \ + --version "${OPENSSL_VERSION}" \ + --user openrv \ + --channel vfx2024 \ + --build=missing \ + -tf="" \ + -pr:a "./conan/profiles/${CONAN_PROFILE}" \ + -o 'openssl/*:shared=True' \ + -o 'openssl/*:no_zlib=True' + conan list "openssl/${OPENSSL_VERSION}@openrv/vfx2024:*" + + - name: Phase C - Upload openssl/${{ env.OPENSSL_VERSION }}@openrv/vfx2024 + if: ${{ env.PHASE == 'openssl' }} + run: conan upload "openssl/${OPENSSL_VERSION}@openrv/vfx2024" --remote "$REMOTE_NAME" --confirm + + - name: Verify server-side listing + if: ${{ env.PHASE == 'openssl' }} + run: conan search "openssl/${OPENSSL_VERSION}@openrv/vfx2024" -r "$REMOTE_NAME" + + - name: Phase D - Wipe local cache for openssl + if: ${{ env.PHASE == 'openssl' }} + run: conan remove "openssl/${OPENSSL_VERSION}@openrv/vfx2024" -c + + - name: Phase D - Round-trip install from ${{ inputs.remote || 'aswftesting' }} + if: ${{ env.PHASE == 'openssl' }} + run: | + set -o pipefail + conan install \ + --requires="openssl/${OPENSSL_VERSION}@openrv/vfx2024" \ + -r "$REMOTE_NAME" \ + --build=never \ + -pr:a "./conan/profiles/${CONAN_PROFILE}" \ + -o 'openssl/*:shared=True' \ + -o 'openssl/*:no_zlib=True' 2>&1 | tee /tmp/phase_d.log + if grep -qiE "Building from sources|building package" /tmp/phase_d.log; then + echo "::error::Phase D regressed on macos - Conan is building instead of downloading." + exit 1 + fi + if ! grep -qiE "downloaded|Already installed" /tmp/phase_d.log; then + echo "::error::Phase D log does not show a download on macos." + exit 1 + fi + echo "::notice::Phase D success on macos." + + - name: Phase E - Verify profile rewrites openssl to @openrv/vfx2024 + if: ${{ env.PHASE == 'openssl' }} + run: | + set -o pipefail + conan graph info \ + --requires="openssl/3.5.0" \ + -r "$REMOTE_NAME" \ + -pr:a "./conan/profiles/${CONAN_PROFILE}" \ + -o 'openssl/*:shared=True' \ + -o 'openssl/*:no_zlib=True' 2>&1 | tee /tmp/phase_e.log + expected_ref="openssl/${OPENSSL_VERSION}@openrv/vfx2024" + if ! grep -qF "$expected_ref" /tmp/phase_e.log; then + echo "::error::Phase E failed on macos - graph missing $expected_ref." + exit 1 + fi + echo "::notice::Phase E success on macos." + + # --------------------------------------------------------------------------- + # Windows x86_64. windows-2022 + MSVC + MSYS2. Profile: x86_64_windows. + # --------------------------------------------------------------------------- + windows: + name: "Conan ${{ inputs.phase || 'probe' }} -> windows-x86_64 -> ${{ inputs.remote || 'aswftesting' }}" + if: contains(format(',{0},', inputs.platforms || 'rocky8,rocky9,macos,windows'), ',windows,') + runs-on: windows-2022 + + env: + CONAN_LOGIN_USERNAME: ${{ secrets.ARTIFACTORY_USER }} + CONAN_PASSWORD: ${{ secrets.ARTIFACTORY_TOKEN }} + CONAN_PROFILE: x86_64_windows + + defaults: + run: + shell: msys2 {0} + + steps: + - name: Diagnose secret presence + shell: bash + env: + U: ${{ secrets.ARTIFACTORY_USER }} + T: ${{ secrets.ARTIFACTORY_TOKEN }} + run: | + echo "event: ${GITHUB_EVENT_NAME}" + echo "platform: windows-x86_64 (${CONAN_PROFILE})" + echo "ARTIFACTORY_USER length: ${#U}" + echo "ARTIFACTORY_TOKEN length: ${#T}" + + - name: Verify secrets are set + shell: bash + run: | + if [ -z "$CONAN_LOGIN_USERNAME" ] || [ -z "$CONAN_PASSWORD" ]; then + echo "::error::ARTIFACTORY_USER or ARTIFACTORY_TOKEN secret is not set." + exit 1 + fi + + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + submodules: false + + - name: Setup MSVC + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Setup MSYS2 + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + update: true + cache: false + install: >- + mingw-w64-x86_64-toolchain + mingw-w64-x86_64-cmake + mingw-w64-x86_64-make + mingw-w64-x86_64-python + mingw-w64-x86_64-python-pip + nasm + perl + make + git + + - name: Install Conan 2.x (inside msys2) + run: | + python -m pip install --upgrade pip + python -m pip install "conan>=2.0,<3.0" + conan --version + + - name: Configure Conan remote and authenticate + run: | + conan remote remove "$REMOTE_NAME" 2>/dev/null || true + conan remote add "$REMOTE_NAME" "$REMOTE_URL" --index 0 + conan remote list + conan remote login "$REMOTE_NAME" "$CONAN_LOGIN_USERNAME" -p "$CONAN_PASSWORD" + + - name: Detect default Conan profile + run: conan profile detect --force + + - name: Phase A.5 - Create probe recipe + run: | + mkdir -p /tmp/openrv-probe + cat > /tmp/openrv-probe/conanfile.py <<'PY' + from conan import ConanFile + + class Probe(ConanFile): + name = "openrv-probe" + version = "0.0.1" + package_type = "application" + settings = "os", "arch" + + def package(self): + pass + PY + conan create /tmp/openrv-probe --user openrv --channel test + + - name: Phase A.5 - Upload probe to ${{ inputs.remote || 'aswftesting' }} + id: probe_upload + run: | + set +e + out=$(conan upload "openrv-probe/0.0.1@openrv/test" --remote "$REMOTE_NAME" --confirm 2>&1) + status=$? + echo "$out" + if [ $status -eq 0 ]; then + echo "::notice::Probe upload succeeded (windows)." + else + if echo "$out" | grep -qiE "403|forbidden|unauthorized"; then + echo "::error::Probe upload rejected on windows. Check permissions." + fi + exit $status + fi + + - name: Phase A.5 - Clean up probe (local + best-effort remote) + if: always() && steps.probe_upload.outcome == 'success' + run: | + # Remote cleanup typically fails 403 on LF Artifactory because + # the OpenRV token has upload-only rights on @openrv/*. Treat + # as expected; an admin can clean the probe ref periodically. + if conan remove "openrv-probe/*" -r "$REMOTE_NAME" -c 2>/dev/null; then + echo "::notice::Remote probe cleaned from $REMOTE_NAME." + else + echo "::notice::Remote probe cleanup returned non-zero (expected for upload-only tokens). Leaving openrv-probe on $REMOTE_NAME." + fi + conan remove "openrv-probe/*" -c 2>/dev/null || true + + - name: Phase B - Clone conan-center-index and verify openssl version + if: ${{ env.PHASE == 'openssl' }} + run: | + git clone --depth 1 --branch master https://github.com/conan-io/conan-center-index.git /tmp/cci + echo "::group::CCI openssl config.yml" + cat /tmp/cci/recipes/openssl/config.yml + echo "::endgroup::" + if ! grep -q "\"${OPENSSL_VERSION}\"" /tmp/cci/recipes/openssl/config.yml; then + echo "::error::openssl ${OPENSSL_VERSION} not found in CCI config.yml." + exit 1 + fi + + - name: Phase B - Build openssl/${{ env.OPENSSL_VERSION }}@openrv/vfx2024 + if: ${{ env.PHASE == 'openssl' }} + run: | + conan create /tmp/cci/recipes/openssl/3.x.x/ \ + --version "${OPENSSL_VERSION}" \ + --user openrv \ + --channel vfx2024 \ + --build=missing \ + -tf="" \ + -pr:a "./conan/profiles/${CONAN_PROFILE}" \ + -o 'openssl/*:shared=True' \ + -o 'openssl/*:no_zlib=True' + conan list "openssl/${OPENSSL_VERSION}@openrv/vfx2024:*" + + - name: Phase C - Upload openssl/${{ env.OPENSSL_VERSION }}@openrv/vfx2024 + if: ${{ env.PHASE == 'openssl' }} + run: conan upload "openssl/${OPENSSL_VERSION}@openrv/vfx2024" --remote "$REMOTE_NAME" --confirm + + - name: Verify server-side listing + if: ${{ env.PHASE == 'openssl' }} + run: conan search "openssl/${OPENSSL_VERSION}@openrv/vfx2024" -r "$REMOTE_NAME" + + - name: Phase D - Wipe local cache for openssl + if: ${{ env.PHASE == 'openssl' }} + run: conan remove "openssl/${OPENSSL_VERSION}@openrv/vfx2024" -c + + - name: Phase D - Round-trip install from ${{ inputs.remote || 'aswftesting' }} + if: ${{ env.PHASE == 'openssl' }} + run: | + set -o pipefail + conan install \ + --requires="openssl/${OPENSSL_VERSION}@openrv/vfx2024" \ + -r "$REMOTE_NAME" \ + --build=never \ + -pr:a "./conan/profiles/${CONAN_PROFILE}" \ + -o 'openssl/*:shared=True' \ + -o 'openssl/*:no_zlib=True' 2>&1 | tee /tmp/phase_d.log + if grep -qiE "Building from sources|building package" /tmp/phase_d.log; then + echo "::error::Phase D regressed on windows - Conan is building instead of downloading." + exit 1 + fi + if ! grep -qiE "downloaded|Already installed" /tmp/phase_d.log; then + echo "::error::Phase D log does not show a download on windows." + exit 1 + fi + echo "::notice::Phase D success on windows." + + - name: Phase E - Verify profile rewrites openssl to @openrv/vfx2024 + if: ${{ env.PHASE == 'openssl' }} + run: | + set -o pipefail + conan graph info \ + --requires="openssl/3.5.0" \ + -r "$REMOTE_NAME" \ + -pr:a "./conan/profiles/${CONAN_PROFILE}" \ + -o 'openssl/*:shared=True' \ + -o 'openssl/*:no_zlib=True' 2>&1 | tee /tmp/phase_e.log + expected_ref="openssl/${OPENSSL_VERSION}@openrv/vfx2024" + if ! grep -qF "$expected_ref" /tmp/phase_e.log; then + echo "::error::Phase E failed on windows - graph missing $expected_ref." + exit 1 + fi + echo "::notice::Phase E success on windows." diff --git a/.github/workflows/conan.yml b/.github/workflows/conan.yml index 9c3ac64bd..751c49aa0 100644 --- a/.github/workflows/conan.yml +++ b/.github/workflows/conan.yml @@ -13,7 +13,6 @@ on: env: SKIP_DEPS_CACHE: 'false' - ROCKY_QT6_MODULES: 'debug_info qt3d qt5compat qtcharts qtconnectivity qtdatavis3d qtgrpc qthttpserver qtimageformats qtlanguageserver qtlocation qtlottie qtmultimedia qtnetworkauth qtpdf qtpositioning qtquick3d qtquick3dphysics qtquickeffectmaker qtquicktimeline qtremoteobjects qtscxml qtsensors qtserialbus qtserialport qtshadertools qtspeech qtvirtualkeyboard qtwaylandcompositor qtwebchannel qtwebengine qtwebsockets qtwebview' ROCKY_QT6_ARCHIVES: 'icu qtbase qtdeclarative qtsvg qttools qttranslations qtwayland' MACOX_X86_64_QT6_MODULES: 'debug_info qt3d qt5compat qtcharts qtconnectivity qtdatavis3d qtgrpc qthttpserver qtimageformats qtlanguageserver qtlocation qtlottie qtmultimedia qtnetworkauth qtpdf qtpositioning qtquick3d qtquick3dphysics qtquickeffectmaker qtquicktimeline qtremoteobjects qtscxml qtsensors qtserialbus qtserialport qtshadertools qtspeech qtvirtualkeyboard qtwebchannel qtwebengine qtwebsockets qtwebview' @@ -60,7 +59,8 @@ jobs: python-version: "3.11.8" vfx-platform: "CY2024" extra_repo: "powertools" - + conan-profile: "x86_64_rocky8" + - os: "ubuntu-latest" rocky-version: "9" image: "amd64/rockylinux:9" @@ -71,6 +71,7 @@ jobs: python-version: "3.11.8" vfx-platform: "CY2024" extra_repo: "crb" + conan-profile: "x86_64_rocky9" steps: - name: Display disk space @@ -311,6 +312,18 @@ jobs: echo "Conan home path: $CONAN_HOME_OUTPUT" echo "CONAN_HOME=$CONAN_HOME_OUTPUT" >> $GITHUB_ENV + - name: Add ASWF Conan remote + # Required so conan/profiles/common [replace_requires] can resolve + # openssl/*@openrv/vfx2024 from the ASWF JFrog Artifactory. + # Anonymous read; no credentials needed for install. Uses aswftesting + # (aswf-conan-dev) for now; swap to the 'aswf' production remote + # once binaries are promoted there. + # Rocky 8 currently has a prebuilt binary; other profiles will + # build from source via the CCI recipe exported under this user/channel. + run: | + $CONAN_EXECUTABLE remote remove aswftesting 2>/dev/null || true + $CONAN_EXECUTABLE remote add aswftesting https://linuxfoundation.jfrog.io/artifactory/api/conan/aswf-conan-dev --index 0 + - name: Export OpenRVCore recipe to Conan's cache if: ${{ matrix.vfx-platform == 'CY2024' }} run: | @@ -319,6 +332,11 @@ jobs: $CONAN_EXECUTABLE export openrvcore-conanfile.py - name: Build OpenRV dependencies + # openexr (and every other dep covered by [replace_requires] in + # conan/profiles/common) now resolves to @openrv/vfx2024 and is + # pulled prebuilt from the ASWF remote on Rocky 8, built on the + # matching toolchain. The earlier Rocky 8-specific `--build=openexr/*` + # workaround is no longer needed. run: | # Rocky 8: force local rebuild of select packages whose Conan Center # prebuilts target a newer glibc / libstdc++ than Rocky 8 provides. @@ -339,7 +357,7 @@ jobs: - name: Build OpenRV main executable run: | - $CONAN_EXECUTABLE build conanfile.py --build=missing -pr:a ./conan/profiles/x86_64_rocky8 + $CONAN_EXECUTABLE build conanfile.py --build=missing -pr:a ./conan/profiles/${{ matrix.conan-profile }} - name: Tests run: | @@ -492,6 +510,12 @@ jobs: - name: Set Conan Home Path run: echo "CONAN_HOME=$(/Users/runner/Library/Python/3.11/bin/conan config home)" >> $GITHUB_ENV + - name: Add ASWF Conan remote + # See conan.yml Rocky step of the same name for rationale. + run: | + /Users/runner/Library/Python/3.11/bin/conan remote remove aswftesting 2>/dev/null || true + /Users/runner/Library/Python/3.11/bin/conan remote add aswftesting https://linuxfoundation.jfrog.io/artifactory/api/conan/aswf-conan-dev --index 0 + - name: Export OpenRVCore recipe if: ${{ matrix.vfx-platform == 'CY2024' }} run: | @@ -737,6 +761,14 @@ jobs: cat ~/.bash_profile | tail -5 shell: msys2 {0} + - name: Add ASWF Conan remote + # See conan.yml Rocky step of the same name for rationale. + run: | + source ~/.bash_profile + conan remote remove aswftesting 2>/dev/null || true + conan remote add aswftesting https://linuxfoundation.jfrog.io/artifactory/api/conan/aswf-conan-dev --index 0 + shell: msys2 {0} + - name: Export OpenRVCore recipe if: ${{ matrix.vfx-platform == 'CY2024' }} run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cdf2c621d..7c7c76716 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,5 +25,22 @@ repos: args: [--fix] description: "Lint and auto-fix markdown files" + - repo: local + hooks: + - id: conan-packages-yml-schema + name: validate conan/packages.yml schema + entry: python3 conan/scripts/validate_packages_yml.py + language: python + files: ^conan/packages\.yml$ + additional_dependencies: ['pyyaml'] + pass_filenames: false + - id: conan-profile-sync + name: conan profiles/common in sync with packages.yml + entry: python3 conan/scripts/gen_profile_common.py --check + language: python + files: ^conan/packages\.yml$ + additional_dependencies: ['pyyaml'] + pass_filenames: false + ci: autofix_prs: false diff --git a/conan/packages.yml b/conan/packages.yml new file mode 100644 index 000000000..60e04c41f --- /dev/null +++ b/conan/packages.yml @@ -0,0 +1,157 @@ +# OpenRV Conan dependencies - single source of truth for the Conan build path. +# +# Consumed by: +# - .github/workflows/conan-upload-deps.yml (CI upload) +# - openrvcore-conanfile.py requirements() (reads versions + options at runtime) +# - conan/scripts/gen_profile_common.py (generates profiles/common replace_requires) +# +# Year-specific deps (from CY2025.cmake) use channel "vfx2025"; +# year-independent deps (from CYCOMMON.cmake) use channel "common". +# +# Fields: +# name - Conan package name (matches CCI recipe folder). +# version - Pinned version; must be listed in CCI config.yml. +# channel - Conan channel: "vfx2025" for year-specific deps, +# "common" for year-independent deps. +# cci_folder - Subfolder under recipes// in conan-center-index. +# Most are "all"; a few like openssl use "3.x.x". +# options - Map of Conan option -> value. Passed as -o flags on +# conan install. Pattern strings like "foo/*:shared" match +# the install --requires ref. +# windows_only - Optional. If true, the dep is skipped at build time on +# non-Windows platforms (openrvcore-conanfile.py and +# conan_install_target.py). Recipes are still exported on +# all platforms so that replace_requires can always resolve. + +deps: + - name: zlib + version: "1.3.2" + channel: common + cci_folder: all + options: + "zlib/*:shared": "True" + + - name: libatomic_ops + version: "7.10.0" + channel: common + cci_folder: all + options: + "libatomic_ops/*:shared": "False" + + - name: libwebp + # openrvcore-conanfile.py:69 pins 1.2.1 because libwebp >=1.3.0 adds + # sharpyuv and causes issues with OIIO. CCI rotated 1.2.1 out; 1.2.4 + # is the closest available that remains pre-1.3.0. + version: "1.2.4" + channel: common + cci_folder: all + options: + "libwebp/*:shared": "False" + + - name: dav1d + version: "1.5.3" + channel: common + cci_folder: all + options: + "dav1d/*:shared": "True" + + - name: libjpeg-turbo + # 2.1.4 rotated out of CCI; 2.1.5 is the closest 2.1.x available. + # Avoiding 3.x to prevent ABI surprises against libraw/openjph. + version: "2.1.5" + channel: common + cci_folder: all + options: + "libjpeg-turbo/*:shared": "True" + + - name: boost + version: "1.85.0" + channel: vfx2025 + cci_folder: all + options: + "boost/*:shared": "True" + "boost/*:extra_b2_flags": "-d+0 -s NO_LZMA=1" + + - name: imath + version: "3.1.12" + channel: vfx2025 + cci_folder: all + source_url: "https://github.com/AcademySoftwareFoundation/Imath/archive/refs/tags/v3.1.12.tar.gz" + source_sha256: "8a1bc258f3149b5729c2f4f8ffd337c0e57f09096e4ba9784329f40c4a9035da" + options: + "imath/*:shared": "True" + + - name: libdeflate + version: "1.25" + channel: common + cci_folder: all + options: + "libdeflate/*:shared": "True" + + - name: openexr + version: "3.3.6" + channel: vfx2025 + cci_folder: "3.x" + source_url: "https://github.com/AcademySoftwareFoundation/openexr/archive/refs/tags/v3.3.6.tar.gz" + source_sha256: "2e3c14c0fd47938819cab0a1d9e0a6dc3d276cdec556745eb7521f6add4deed9" + options: + "openexr/*:shared": "True" + + - name: libpng + # 1.6.55 rotated; 1.6.58 is a patch bump within 1.6.x. + version: "1.6.58" + channel: common + cci_folder: all + options: + "libpng/*:shared": "True" + + - name: openjpeg + version: "2.5.4" + channel: common + cci_folder: all + options: + "openjpeg/*:shared": "True" + + - name: libraw + # 0.21.1 rotated; 0.21.2 is a patch bump within 0.21.x. + # 0.21.5b has a beta suffix, avoided. + version: "0.21.2" + channel: common + cci_folder: all + options: + "libraw/*:shared": "True" + "libraw/*:with_jpeg": "libjpeg-turbo" + "libraw/*:with_jasper": "False" + + - name: openjph + # 0.26.3 rotated; 0.27.0 is the only remaining version in CCI. + # Minor bump; validate CMake target name compatibility. + version: "0.27.0" + channel: common + cci_folder: all + options: + "openjph/*:shared": "True" + "openjph/*:with_tiff": "False" + + - name: openssl + # Used to override the openssl version pulled in transitively by ffmpeg. + # Not used directly by OpenRV at runtime. + version: "3.6.2" + channel: vfx2025 + cci_folder: "3.x.x" + options: + "openssl/*:shared": "True" + "openssl/*:no_zlib": "True" + + - name: pcre2 + version: "10.44" + channel: common + cci_folder: all + windows_only: true + options: + "pcre2/*:shared": "True" + "pcre2/*:with_zlib": "False" + "pcre2/*:with_bzip2": "False" + "pcre2/*:support_jit": "False" + "pcre2/*:build_pcre2_16": "False" + "pcre2/*:build_pcre2_32": "False" diff --git a/conan/profiles/common b/conan/profiles/common index 320bde179..90a1959d3 100644 --- a/conan/profiles/common +++ b/conan/profiles/common @@ -1,5 +1,34 @@ -# Common settings shared across all OpenRV profiles +# Common settings shared across all OpenRV profiles. +# +# Redirect every OpenRV-managed dependency to prebuilt binaries published +# on the ASWF Conan remote (aswf-conan-dev for testing, aswf-conan for +# production). Year-specific deps (CY2025.cmake) use @openrv/vfx2025; +# year-independent deps (CYCOMMON.cmake) use @openrv/common. Consumers +# must have one of those remotes +# configured before running `conan install`. The versions below track +# conan/packages.yml, which is the single source of truth for what the +# conan-upload-deps.yml workflow builds and uploads. +# +# With RV_DEPS_*_VERSION_MATCH=MINIMUM set in openrvcore-conanfile.py, these +# refs may be newer than what the conanfile pins; CMake's find_package will +# still accept them. replace_requires runs before force=True, so the bare +# version pins in the conanfile effectively become cosmetic for anything +# covered here. [replace_requires] -openssl/*: openssl/3.5.0 -libjpeg/*: libjpeg-turbo/2.1.4 +zlib/*: zlib/1.3.2@openrv/common +libatomic_ops/*: libatomic_ops/7.10.0@openrv/common +libwebp/*: libwebp/1.2.4@openrv/common +dav1d/*: dav1d/1.5.3@openrv/common +libjpeg/*: libjpeg-turbo/2.1.5@openrv/common +libjpeg-turbo/*: libjpeg-turbo/2.1.5@openrv/common +boost/*: boost/1.85.0@openrv/vfx2025 +imath/*: imath/3.1.12@openrv/vfx2025 +libdeflate/*: libdeflate/1.25@openrv/common +openexr/*: openexr/3.3.6@openrv/vfx2025 +libpng/*: libpng/1.6.58@openrv/common +openjpeg/*: openjpeg/2.5.4@openrv/common +libraw/*: libraw/0.21.2@openrv/common +openjph/*: openjph/0.27.0@openrv/common +openssl/*: openssl/3.6.2@openrv/vfx2025 +pcre2/*: pcre2/10.44@openrv/common diff --git a/conan/scripts/_common.py b/conan/scripts/_common.py new file mode 100644 index 000000000..c8daa6bac --- /dev/null +++ b/conan/scripts/_common.py @@ -0,0 +1,60 @@ +"""Shared helpers for conan/scripts/*. + +Centralizes the few values that the conan/scripts/* tools all need: +the repo root, the packages.yml path, the conan user namespace, and the +windows_only filter rule. Keeping these in one place prevents the kind +of silent drift that previously had two scripts each spelling out the +same filter against different env signals. +""" + +import os +import sys +from pathlib import Path + +try: + import yaml +except ImportError: + print("error: pyyaml is required. Install with: pip install pyyaml", file=sys.stderr) + sys.exit(1) + +REPO_ROOT = Path(__file__).resolve().parents[2] +PACKAGES_YML = REPO_ROOT / "conan" / "packages.yml" +PROFILES_DIR = REPO_ROOT / "conan" / "profiles" + +CONAN_USER = "openrv" + + +def conan_exec() -> str: + """Return the conan binary path (CONAN_EXEC env var, defaulting to "conan").""" + return os.environ.get("CONAN_EXEC", "conan") + + +def load_packages() -> dict: + """Load and return the packages.yml document as a dict.""" + pkgs = yaml.safe_load(PACKAGES_YML.read_text()) + if not isinstance(pkgs, dict): + print(f"error: {PACKAGES_YML} is empty or not a mapping", file=sys.stderr) + sys.exit(1) + return pkgs + + +def find_dep(pkgs: dict, name: str, version: str) -> dict | None: + """Find the dep entry in packages.yml matching the given name and version.""" + for dep in pkgs["deps"]: + if dep["name"] == name and dep["version"] == version: + return dep + return None + + +def is_active_dep(dep: dict, *, is_windows: bool) -> bool: + """Return True if `dep` should participate in the build on this platform. + + A dep flagged windows_only is only active on Windows; everything else + is active everywhere. This is the single source of truth for the + filter. openrvcore-conanfile.py mirrors the same key check at recipe + parse time (it cannot import this module due to Conan's recipe + sandboxing), so any rule change here must be reflected there. + """ + if dep.get("windows_only") and not is_windows: + return False + return True diff --git a/conan/scripts/conan_install_target.py b/conan/scripts/conan_install_target.py new file mode 100644 index 000000000..91d91db8d --- /dev/null +++ b/conan/scripts/conan_install_target.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Build a single target package from conan/packages.yml using conan install. + +Usage: + python3 conan/scripts/conan_install_target.py + +Reads packages.yml to look up the dep's channel and options, then runs: + conan install --requires=/@openrv/ + --build=missing --build=b2/* --build=meson/* --build=nasm/* + -pr:a ./conan/profiles/ + -o