From 6bc84bfe6c5c8db9bcf0739da7c123104e8eeb17 Mon Sep 17 00:00:00 2001 From: Trygve Wastvedt Date: Mon, 17 Mar 2025 12:18:26 -0500 Subject: [PATCH 01/10] Modify cicd for public github --- .github/workflows/ARM.yml | 2 +- .github/workflows/docs.yml | 40 ---------- .github/workflows/main.yml | 116 ++++++++++++++++++++++++++++ .github/workflows/nuget-preview.yml | 64 +++++++++++++++ pipeline.yml | 66 ---------------- scripts/test.sh | 15 ---- 6 files changed, 181 insertions(+), 122 deletions(-) delete mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/nuget-preview.yml delete mode 100755 scripts/test.sh diff --git a/.github/workflows/ARM.yml b/.github/workflows/ARM.yml index eef0e666d..b7065ef77 100644 --- a/.github/workflows/ARM.yml +++ b/.github/workflows/ARM.yml @@ -3,7 +3,7 @@ name: Main (ARM) on: push: branches: - - master + - dynamo_py3 pull_request: jobs: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 5b782c8b4..000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Documentation - -on: [push, pull_request] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - name: Doxygen Action - uses: mattnotmitt/doxygen-action@1.9.4 - with: - working-directory: "doc/" - - - name: Build Sphinx documentation - run: | - pip install -r doc/requirements.txt - sphinx-build doc/source/ ./doc/build/html/ - - - name: Upload artifact - # Automatically uploads an artifact from the './_site' directory by default - uses: actions/upload-pages-artifact@v1 - with: - path: doc/build/html/ - - deploy: - if: github.ref == 'refs/heads/master' - runs-on: ubuntu-latest - permissions: - contents: read - pages: write - id-token: write - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v1 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..450c47806 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,116 @@ +name: Main + +on: + push: + branches: + - dynamo_py3 + pull_request: + +jobs: + build-test: + name: Build and Test + runs-on: ${{ matrix.os.instance }} + timeout-minutes: 15 + + strategy: + fail-fast: false + matrix: + os: + - category: windows + platform: x86 + instance: windows-latest + + - category: windows + platform: x64 + instance: windows-latest + + - category: ubuntu + platform: x64 + instance: ubuntu-22.04 + + - category: macos + platform: x64 + instance: macos-13 + + python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + + # This fails in pytest with: + # CSC : error CS4023: /platform:anycpu32bitpreferred can only be used with /t:exe, /t:winexe and /t:appcontainerexe [D:\a\pythonnet\pythonnet\src\runtime\Python.Runtime.csproj] + exclude: + - os: + category: windows + platform: x86 + python: "3.13" + + steps: + - name: Set Environment on macOS + uses: maxim-lobanov/setup-xamarin@v1 + if: ${{ matrix.os.category == 'macos' }} + with: + mono-version: latest + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup .NET + uses: actions/setup-dotnet@v1 + with: + dotnet-version: '6.0.x' + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + architecture: ${{ matrix.os.platform }} + + - name: Install dependencies + run: | + pip install --upgrade -r requirements.txt + pip install numpy # for tests + + - name: Build and Install + run: | + pip install -v . + + - name: Set Python DLL path and PYTHONHOME (non Windows) + if: ${{ matrix.os.category != 'windows' }} + run: | + echo PYTHONNET_PYDLL=$(python -m find_libpython) >> $GITHUB_ENV + echo PYTHONHOME=$(python -c 'import sys; print(sys.prefix)') >> $GITHUB_ENV + + - name: Set Python DLL path and PYTHONHOME (Windows) + if: ${{ matrix.os.category == 'windows' }} + run: | + Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -InputObject "PYTHONNET_PYDLL=$(python -m find_libpython)" + Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -InputObject "PYTHONHOME=$(python -c 'import sys; print(sys.prefix)')" + + - name: Embedding tests + if: ${{ matrix.python != '3.13' }} + run: dotnet test --runtime any-${{ matrix.os.platform }} --logger "console;verbosity=detailed" src/embed_tests/ + env: + MONO_THREADS_SUSPEND: preemptive # https://github.com/mono/mono/issues/21466 + + - name: Python Tests (Mono) + if: ${{ matrix.os.category != 'windows' }} + run: pytest --runtime mono + + # TODO: Run these tests on Windows x86 + - name: Python Tests (.NET Core) + if: ${{ matrix.os.platform == 'x64' }} + run: pytest --runtime coreclr + + - name: Python Tests (.NET Framework) + if: ${{ matrix.os.category == 'windows' }} + run: pytest --runtime netfx + + - name: Python tests run from .NET + if: ${{ matrix.python != '3.13' }} + run: dotnet test --runtime any-${{ matrix.os.platform }} src/python_tests_runner/ + + - name: Perf tests + if: ${{ (matrix.python == '3.8') && (matrix.os.platform == 'x64') }} + run: | + pip install --force --no-deps --target src/perf_tests/baseline/ pythonnet==2.5.2 + dotnet test --configuration Release --runtime any-${{ matrix.os.platform }} --logger "console;verbosity=detailed" src/perf_tests/ + + # TODO: Run mono tests on Windows? diff --git a/.github/workflows/nuget-preview.yml b/.github/workflows/nuget-preview.yml new file mode 100644 index 000000000..7cffbf07b --- /dev/null +++ b/.github/workflows/nuget-preview.yml @@ -0,0 +1,64 @@ +name: NuGet Preview Release + +on: + push: + branches: + - dynamo_py3 + workflow_dispatch: + +jobs: + release: + name: Release Preview + runs-on: ubuntu-latest + environment: NuGet + timeout-minutes: 10 + + env: + PYTHONNET_SHUTDOWN_MODE: Normal + + steps: + - name: Get Date + run: | + echo "DATE_VER=$(date "+%Y-%m-%d")" >> $GITHUB_ENV + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup .NET + uses: actions/setup-dotnet@v1 + with: + dotnet-version: '6.0.x' + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + architecture: x64 + + - name: Install dependencies + run: | + pip install --upgrade -r requirements.txt + pip install numpy # for tests + + - name: Build and Install + run: | + pip install -v . + + - name: Set Python DLL path (non Windows) + if: ${{ matrix.os != 'windows' }} + run: | + echo PYTHONNET_PYDLL=$(python -m find_libpython) >> $GITHUB_ENV + + - name: Python Tests + run: pytest + + - name: Embedding tests + run: dotnet test --runtime any-ubuntu src/embed_tests/ + + - name: Pack + run: dotnet pack --configuration Release --version-suffix preview${{env.DATE_VER}} --output "Release-Preview" + + - name: Publish NuGet + run: | + dotnet nuget push --source https://api.nuget.org/v3/index.json --api-key ${{ secrets.NUGET_MONTHLY }} Release-Preview/*.nupkg + dotnet nuget push --skip-duplicate --source https://api.nuget.org/v3/index.json --api-key ${{ secrets.NUGET_MONTHLY }} Release-Preview/*.snupkg diff --git a/pipeline.yml b/pipeline.yml index 6a0c42fbb..ee349db01 100644 --- a/pipeline.yml +++ b/pipeline.yml @@ -1,26 +1,6 @@ version: 3.1.0 pipeline_os: linux main_branch: dynamo_py3 -update_changelog_on_release: true -create_github_release: true -create_pr_release_to_master: true - -env: - - GITHUB_ACCESS_TOKEN_ID: github_access_token_acsbuildguy - # - SLACK_QUANTUM_BUILD_CHANNEL: dynamo-vera-notify - -jenkins_creds: - - - type: secretText - credentialsId: svc_d_dyntest-apikey-artifactory-dynamo-team-nuget - secretText: API_KEY - -# code_analysis: -# sonarqube: -# source_encoding: UTF-8 -# src: . -# coverage_reports: -# - coverage.info build: - @@ -30,30 +10,6 @@ build: scripts: - scripts/install.sh -ci_test: - tests: - - - customized_builder_image: - registry: artifactory - url: artifactory.dev.adskengineer.net/dynamo/pythonnet-build:2.0.1 - scripts: - - scripts/install.sh - - scripts/test.sh - - publish_html_reports: - - - report_name: Embed Tests - report_index_file: src/embed_tests/TestResults/embed_tests-output.html - report_dir: . - - - report_name: Module Tests - report_index_file: src/module_tests/TestResults/module_tests-output.html - report_dir: . - - - report_name: Python Tests Runner - report_index_file: src/python_tests_runner/TestResults/python_tests_runner-output.html - report_dir: . - soc2: harmony: allow_branches: @@ -63,25 +19,3 @@ soc2: third_party_lib_paths: - build\lib\pythonnet\runtime analyze_results : true - -deployment: - # CILibrary doesn't yet support signing on Linux. I tried to get this going, but haven't figured it out yet. See the vera/nuget-deploy branch on the CILibrary repo. - # - - # type: sign - # files_to_sign: - # - pythonnet/runtime/Python.Runtime.dll - - - type: customized - allow_branches: dynamo_py3 - customized_builder_image: - registry: artifactory - url: artifactory.dev.adskengineer.net/dynamo/pythonnet-build:2.0.1 - scripts: - - dotnet pack --configuration Release --version-suffix preview-ADSK-$(date +%s) --output "Release-Preview" - - dotnet nuget push --source "Dynamo Artifactory" --api-key ${API_KEY} Release-Preview/*.nupkg - - dotnet nuget push --skip-duplicate --source "Dynamo Artifactory" --api-key ${API_KEY} Release-Preview/*.snupkg - - - - type: github - outputs: - - build\lib\pythonnet\runtime\Python.Runtime.dll diff --git a/scripts/test.sh b/scripts/test.sh deleted file mode 100755 index ff11a1ef8..000000000 --- a/scripts/test.sh +++ /dev/null @@ -1,15 +0,0 @@ -export PYTHONNET_PYDLL=$(python3 -m find_libpython) - -dotnet test --logger "html;verbosity=detailed;logfilename=embed_tests-output.html" -p:targetframeworks=net6.0 src/embed_tests/ -retVal=$? - -dotnet test --logger "html;verbosity=detailed;logfilename=module_tests-output.html" -p:targetframeworks=net6.0 src/module_tests/ -retVal=$(($retVal || $?)) - -python3 -m pytest --runtime coreclr -retVal=$(($retVal || $?)) - -dotnet test --logger "html;verbosity=detailed;logfilename=python_tests_runner-output.html" -p:targetframeworks=net6.0 src/python_tests_runner/ -retVal=$(($retVal || $?)) - -exit $retVal From 5a7f8ad223c96f629bf7ef3afeae6209efe2b71b Mon Sep 17 00:00:00 2001 From: Trygve Wastvedt Date: Mon, 17 Mar 2025 12:29:23 -0500 Subject: [PATCH 02/10] Haven't merged 3.13 yet --- .github/workflows/main.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 450c47806..22cc1d43b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,15 +32,7 @@ jobs: platform: x64 instance: macos-13 - python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] - - # This fails in pytest with: - # CSC : error CS4023: /platform:anycpu32bitpreferred can only be used with /t:exe, /t:winexe and /t:appcontainerexe [D:\a\pythonnet\pythonnet\src\runtime\Python.Runtime.csproj] - exclude: - - os: - category: windows - platform: x86 - python: "3.13" + python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - name: Set Environment on macOS @@ -85,7 +77,6 @@ jobs: Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -InputObject "PYTHONHOME=$(python -c 'import sys; print(sys.prefix)')" - name: Embedding tests - if: ${{ matrix.python != '3.13' }} run: dotnet test --runtime any-${{ matrix.os.platform }} --logger "console;verbosity=detailed" src/embed_tests/ env: MONO_THREADS_SUSPEND: preemptive # https://github.com/mono/mono/issues/21466 @@ -104,7 +95,6 @@ jobs: run: pytest --runtime netfx - name: Python tests run from .NET - if: ${{ matrix.python != '3.13' }} run: dotnet test --runtime any-${{ matrix.os.platform }} src/python_tests_runner/ - name: Perf tests From 95cdde52e1d3ddab780665fcdd2cb2ced1aaab30 Mon Sep 17 00:00:00 2001 From: Trygve Wastvedt Date: Mon, 17 Mar 2025 13:49:13 -0500 Subject: [PATCH 03/10] Remove custom nuget.config --- nuget.config | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 nuget.config diff --git a/nuget.config b/nuget.config deleted file mode 100644 index ea1faadee..000000000 --- a/nuget.config +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - From 9bb8c0d5f3bd03c0f361e5338ab42c11e4e312e5 Mon Sep 17 00:00:00 2001 From: Trygve Wastvedt Date: Mon, 17 Mar 2025 15:17:17 -0500 Subject: [PATCH 04/10] Disable perf tests --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 22cc1d43b..2a4a74f11 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -97,10 +97,10 @@ jobs: - name: Python tests run from .NET run: dotnet test --runtime any-${{ matrix.os.platform }} src/python_tests_runner/ - - name: Perf tests - if: ${{ (matrix.python == '3.8') && (matrix.os.platform == 'x64') }} - run: | - pip install --force --no-deps --target src/perf_tests/baseline/ pythonnet==2.5.2 - dotnet test --configuration Release --runtime any-${{ matrix.os.platform }} --logger "console;verbosity=detailed" src/perf_tests/ + # - name: Perf tests + # if: ${{ (matrix.python == '3.8') && (matrix.os.platform == 'x64') }} + # run: | + # pip install --force --no-deps --target src/perf_tests/baseline/ pythonnet==2.5.2 + # dotnet test --configuration Release --runtime any-${{ matrix.os.platform }} --logger "console;verbosity=detailed" src/perf_tests/ # TODO: Run mono tests on Windows? From 9f13d5da1eb956917993c7cf8ba0a817beb9b3a3 Mon Sep 17 00:00:00 2001 From: Trygve Wastvedt Date: Mon, 17 Mar 2025 15:46:49 -0500 Subject: [PATCH 05/10] ARM available publicly? --- .github/workflows/ARM.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ARM.yml b/.github/workflows/ARM.yml index b7065ef77..6e789cfe4 100644 --- a/.github/workflows/ARM.yml +++ b/.github/workflows/ARM.yml @@ -9,7 +9,7 @@ on: jobs: build-test-arm: name: Build and Test ARM64 - runs-on: [self-hosted, linux, ARM64] + runs-on: [linux, ARM64] timeout-minutes: 15 steps: From 778cce1a645e39c22b2028e53adff3ac0b3bcdc4 Mon Sep 17 00:00:00 2001 From: Trygve Wastvedt Date: Mon, 17 Mar 2025 15:55:43 -0500 Subject: [PATCH 06/10] Different arm id? --- .github/workflows/ARM.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ARM.yml b/.github/workflows/ARM.yml index 6e789cfe4..d9bf3a209 100644 --- a/.github/workflows/ARM.yml +++ b/.github/workflows/ARM.yml @@ -9,7 +9,7 @@ on: jobs: build-test-arm: name: Build and Test ARM64 - runs-on: [linux, ARM64] + runs-on: ubuntu-24.04-arm timeout-minutes: 15 steps: From a5d4a74b85fe45c9e9f6ac42773f3e7b2e4a1d8e Mon Sep 17 00:00:00 2001 From: Trygve Wastvedt Date: Tue, 18 Mar 2025 11:54:51 -0500 Subject: [PATCH 07/10] Use default python on ARM --- .github/workflows/ARM.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ARM.yml b/.github/workflows/ARM.yml index d9bf3a209..c56f04551 100644 --- a/.github/workflows/ARM.yml +++ b/.github/workflows/ARM.yml @@ -27,25 +27,25 @@ jobs: - name: Install dependencies run: | - pip3.8 install -r requirements.txt - pip3.8 install pytest numpy # for tests + pip install -r requirements.txt + pip install pytest numpy # for tests - name: Build and Install run: | - pip3.8 install -v . + pip install -v . - name: Set Python DLL path (non Windows) run: | - echo PYTHONNET_PYDLL=$(python3.8 -m find_libpython) >> $GITHUB_ENV + echo PYTHONNET_PYDLL=$(python -m find_libpython) >> $GITHUB_ENV - name: Embedding tests run: dotnet test --logger "console;verbosity=detailed" src/embed_tests/ - name: Python Tests (Mono) - run: python3.8 -m pytest --runtime mono + run: python -m pytest --runtime mono - name: Python Tests (.NET Core) - run: python3.8 -m pytest --runtime coreclr + run: python -m pytest --runtime coreclr - name: Python tests run from .NET run: dotnet test src/python_tests_runner/ From 54baed87deb3994267c568d3cd1d98d80eee40d7 Mon Sep 17 00:00:00 2001 From: Trygve Wastvedt Date: Tue, 18 Mar 2025 12:04:24 -0500 Subject: [PATCH 08/10] .net framework issue fixed in .NET 9? --- .github/workflows/ARM.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ARM.yml b/.github/workflows/ARM.yml index c56f04551..c7f01cd64 100644 --- a/.github/workflows/ARM.yml +++ b/.github/workflows/ARM.yml @@ -19,7 +19,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v1 with: - dotnet-version: '6.0.x' + dotnet-version: '9.0.x' - name: Clean previous install run: | From 4b75bba58c2b92fa86cb0166090b10a1babdce84 Mon Sep 17 00:00:00 2001 From: Trygve Wastvedt Date: Tue, 18 Mar 2025 12:13:45 -0500 Subject: [PATCH 09/10] Specify framework? --- .github/workflows/ARM.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ARM.yml b/.github/workflows/ARM.yml index c7f01cd64..8f2c1c65f 100644 --- a/.github/workflows/ARM.yml +++ b/.github/workflows/ARM.yml @@ -19,7 +19,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v1 with: - dotnet-version: '9.0.x' + dotnet-version: '6.0.x' - name: Clean previous install run: | @@ -39,7 +39,7 @@ jobs: echo PYTHONNET_PYDLL=$(python -m find_libpython) >> $GITHUB_ENV - name: Embedding tests - run: dotnet test --logger "console;verbosity=detailed" src/embed_tests/ + run: dotnet test --logger "console;verbosity=detailed" src/embed_tests/ --framework net6.0 - name: Python Tests (Mono) run: python -m pytest --runtime mono @@ -48,7 +48,7 @@ jobs: run: python -m pytest --runtime coreclr - name: Python tests run from .NET - run: dotnet test src/python_tests_runner/ + run: dotnet test src/python_tests_runner/ --framework net6.0 #- name: Perf tests # run: | From fb92b11b79c163553e0c59aad742a460048b68ad Mon Sep 17 00:00:00 2001 From: Trygve Wastvedt Date: Tue, 18 Mar 2025 13:16:06 -0500 Subject: [PATCH 10/10] Disable ARM for now --- .github/workflows/ARM.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ARM.yml b/.github/workflows/ARM.yml index 8f2c1c65f..e436c009e 100644 --- a/.github/workflows/ARM.yml +++ b/.github/workflows/ARM.yml @@ -3,13 +3,13 @@ name: Main (ARM) on: push: branches: - - dynamo_py3 - pull_request: + - master + # pull_request: jobs: build-test-arm: name: Build and Test ARM64 - runs-on: ubuntu-24.04-arm + runs-on: [self-hosted, linux, ARM64] timeout-minutes: 15 steps: @@ -27,28 +27,28 @@ jobs: - name: Install dependencies run: | - pip install -r requirements.txt - pip install pytest numpy # for tests + pip3.8 install -r requirements.txt + pip3.8 install pytest numpy # for tests - name: Build and Install run: | - pip install -v . + pip3.8 install -v . - name: Set Python DLL path (non Windows) run: | - echo PYTHONNET_PYDLL=$(python -m find_libpython) >> $GITHUB_ENV + echo PYTHONNET_PYDLL=$(python3.8 -m find_libpython) >> $GITHUB_ENV - name: Embedding tests - run: dotnet test --logger "console;verbosity=detailed" src/embed_tests/ --framework net6.0 + run: dotnet test --logger "console;verbosity=detailed" src/embed_tests/ - name: Python Tests (Mono) - run: python -m pytest --runtime mono + run: python3.8 -m pytest --runtime mono - name: Python Tests (.NET Core) - run: python -m pytest --runtime coreclr + run: python3.8 -m pytest --runtime coreclr - name: Python tests run from .NET - run: dotnet test src/python_tests_runner/ --framework net6.0 + run: dotnet test src/python_tests_runner/ #- name: Perf tests # run: |