ci: tidy3d-extras integration tests & docs

daquinteroflex · daquinteroflex · commit 0a8c0b9d5521 · 2025-12-09T13:28:45.000+01:00
diff --git a/.github/workflows/tidy3d-extras-python-client-tests-integration.yml b/.github/workflows/tidy3d-extras-python-client-tests-integration.yml
@@ -0,0 +1,145 @@
+name: "public/tidy3d-extras/python-client-integration-tests"
+
+on:
+  workflow_dispatch:
+    inputs:
+      release_tag:
+        description: 'Release Tag (v2.10.0, v2.10.0rc1)'
+        required: false
+        type: string
+        default: ''
+
+  workflow_call:
+    inputs:
+      release_tag:
+        description: 'Release Tag (v2.10.0, v2.10.0rc1)'
+        required: false
+        type: string
+        default: ''
+    outputs:
+      workflow_success:
+        description: 'Overall integration test workflow success status'
+        value: ${{ jobs.integration-tests.result == 'success' }}
+
+permissions:
+  contents: read
+
+jobs:
+  integration-tests:
+    name: python-${{ matrix.python-version }}-${{ matrix.platform }}-extras
+    runs-on: ${{ matrix.platform }}
+    permissions:
+      contents: read
+    concurrency:
+      group: tidy3d-extras-integration-${{ github.event.pull_request.number || github.sha }}-${{ matrix.platform }}-${{ matrix.python-version }}
+      cancel-in-progress: true
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.10', '3.13']
+        platform: [windows-latest, ubuntu-latest, macos-latest]
+    defaults:
+      run:
+        shell: bash
+    env:
+      PIP_ONLY_BINARY: gdstk
+      MPLBACKEND: agg
+      RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || inputs.release_tag }}
+
+    steps:
+    - name: checkout-head
+      if: ${{ !env.RELEASE_TAG }}
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+        submodules: false
+        persist-credentials: false
+
+    - name: checkout-tag
+      if: ${{ env.RELEASE_TAG }}
+      uses: actions/checkout@v4
+      with:
+        ref: refs/tags/${{ env.RELEASE_TAG }}
+        fetch-depth: 1
+        submodules: false
+        persist-credentials: false
+
+    - name: set-python-${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: install-poetry
+      uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a  # v1.4.1
+      with:
+        version: 2.1.1
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+
+    - name: configure-aws-credentials
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502  # v4.0.2
+      with:
+        aws-access-key-id: ${{ secrets.AWS_CODEARTIFACT_ACCESS_KEY }}
+        aws-secret-access-key: ${{ secrets.AWS_CODEARTIFACT_ACCESS_SECRET }}
+        aws-region: us-east-1
+
+    - name: configure-codeartifact-authentication
+      run: |
+        set -e
+        echo "Getting CodeArtifact token..."
+        CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token \
+          --domain flexcompute \
+          --domain-owner 625554095313 \
+          --query authorizationToken \
+          --output text)
+
+        echo "Configuring Poetry with CodeArtifact credentials..."
+        poetry config http-basic.codeartifact aws $CODEARTIFACT_AUTH_TOKEN
+        echo "✅ CodeArtifact authentication configured"
+
+    - name: install-project
+      shell: bash
+      run: |
+        poetry --version
+        python --version
+        python -m venv .venv
+        if [[ "${{ runner.os }}" == "Windows" ]]; then
+          source .venv/Scripts/activate
+          python --version
+        else
+          source .venv/bin/activate
+          which python
+        fi
+        poetry env use python
+        poetry env info
+        poetry run pip install --upgrade pip wheel setuptools
+        poetry run pip install gdstk --only-binary gdstk
+        poetry install -E extras -E dev
+    
+    - name: verify-extras-installation
+      run: |
+        export SIMCLOUD_APIKEY=${{ secrets.TIDY3D_API_KEY }}
+        poetry run tidy3d configure --apikey ${{ secrets.TIDY3D_API_KEY }}
+        poetry run python -c "import tidy3d; print(f'tidy3d version: {tidy3d.__version__}')"
+        poetry run python -c "import tidy3d_extras; print(f'tidy3d-extras version: {tidy3d_extras.__version__}')"
+        cd tests/
+        poetry run pytest tidy3d_extras_license_test.py
+
+    - name: run-doctests
+      run: |
+        export SIMCLOUD_APIKEY=${{ secrets.TIDY3D_API_KEY }}
+        poetry run tidy3d configure --apikey ${{ secrets.TIDY3D_API_KEY }}
+        poetry run pytest -rF --tb=short tidy3d
+
+    - name: run-tests-coverage
+      env:
+        PYTHONUNBUFFERED: "1"
+      run: |
+        export SIMCLOUD_APIKEY=${{ secrets.TIDY3D_API_KEY }}
+        poetry run tidy3d configure --apikey ${{ secrets.TIDY3D_API_KEY }}
+        poetry run pytest --cov=tidy3d -rF --tb=short tests/_test_data/_test_datasets_no_vtk.py
+        poetry run pytest --cov=tidy3d -rF --tb=short tests
+        poetry run coverage report -m
+        TOTAL_COVERAGE=$(poetry run coverage report --format=total)
+        echo "total=$TOTAL_COVERAGE" >> "$GITHUB_ENV"
+        echo "### Total coverage: ${TOTAL_COVERAGE}%"
diff --git a/.github/workflows/tidy3d-python-client-daily.yml b/.github/workflows/tidy3d-python-client-daily.yml
@@ -17,7 +17,18 @@ jobs:
     with:
       run-workflow: true
 
-  submodule-tests:
-    uses: ./.github/workflows/tidy3d-python-client-submodules-test.yml
+  daily-release:
+    uses: ./.github/workflows/tidy3d-python-client-release.yml
+    permissions:
+      contents: write
+      pull-requests: write
+      security-events: write
+      id-token: write
     with:
-      run-workflow: true
+      release_tag: 'daily-0.0.0'
+      release_type: 'draft'
+      workflow_control: 'start-tag'
+      client_tests: true
+      cli_tests: true
+      submodule_tests: false
+    secrets: inherit # zizmor: ignore[secrets-inherit]
diff --git a/.github/workflows/tidy3d-python-client-tests.yml b/.github/workflows/tidy3d-python-client-tests.yml
@@ -14,23 +14,27 @@ on:
         type: boolean
         default: false
       cli_tests:
-        description: 'Run develop-cli tests'
+        description: 'develop-cli'
         type: boolean
         default: false
       submodule_tests:
-        description: 'Run submodule tests'
+        description: 'submodule-tests'
         type: boolean
         default: false
       version_match_tests:
-        description: 'Run version consistency checks'
+        description: 'version-consistency-checks'
+        type: boolean
+        default: false
+      extras_integration_tests:
+        description: 'integration-tidy3d-extras'
         type: boolean
         default: false
       release_tag:
         description: 'Release Tag (v2.10.0, v2.10.0rc1)'
         required: false
         type: string
         default: ''
-  
+
   workflow_call:
     inputs:
       remote_tests:
@@ -58,6 +62,11 @@ on:
         type: boolean
         required: false
         default: false
+      extras_integration_tests:
+        description: 'Run tidy3d-extras integration tests'
+        type: boolean
+        required: false
+        default: false
       release_tag:
         description: 'Release Tag (v2.10.0, v2.10.0rc1)'
         required: false
@@ -95,6 +104,7 @@ jobs:
       cli_tests: ${{ steps.determine-test-type.outputs.cli_tests }}
       submodule_tests: ${{ steps.determine-test-type.outputs.submodule_tests }}
       version_match_tests: ${{ steps.determine-test-type.outputs.version_match_tests }}
+      extras_integration_tests: ${{ steps.determine-test-type.outputs.extras_integration_tests }}
     steps:
       - name: determine-test-type
         id: determine-test-type
@@ -108,6 +118,7 @@ jobs:
           INPUT_CLI: ${{ github.event.inputs.cli_tests || inputs.cli_tests }}
           INPUT_SUBMODULE: ${{ github.event.inputs.submodule_tests || inputs.submodule_tests }}
           INPUT_VERSION_MATCH: ${{ github.event.inputs.version_match_tests || inputs.version_match_tests }}
+          INPUT_EXTRAS_INTEGRATION: ${{ github.event.inputs.extras_integration_tests || inputs.extras_integration_tests }}
         run: |
           echo "Event: $EVENT_NAME"
           echo "Draft: $DRAFT_STATE"
@@ -118,6 +129,7 @@ jobs:
           echo "Input cli: $INPUT_CLI"
           echo "Input submodule: $INPUT_SUBMODULE"
           echo "Input version_match: $INPUT_VERSION_MATCH"
+          echo "Input extras_integration: $INPUT_EXTRAS_INTEGRATION"
 
           remote_tests=false
           local_tests=false
@@ -126,6 +138,7 @@ jobs:
           version_match_tests=false
           code_quality_tests=false
           pr_review_tests=false
+          extras_integration_tests=false
           
             # Workflow_dispatch input override
           if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then
@@ -150,6 +163,10 @@ jobs:
             if [[ "$INPUT_VERSION_MATCH" == "true" ]]; then
               version_match_tests=true
             fi
+
+            if [[ "$INPUT_EXTRAS_INTEGRATION" == "true" ]]; then
+              extras_integration_tests=true
+            fi
           fi
 
           # All PRs that have been triggered need local tests (remote reserved for merge queue/manual)
@@ -163,6 +180,7 @@ jobs:
               local_tests=true
               remote_tests=true
               code_quality_tests=true
+              extras_integration_tests=true
           fi
           
           echo "local_tests=$local_tests" >> $GITHUB_OUTPUT
@@ -172,13 +190,15 @@ jobs:
           echo "version_match_tests=$version_match_tests" >> $GITHUB_OUTPUT
           echo "code_quality_tests=$code_quality_tests" >> $GITHUB_OUTPUT
           echo "pr_review_tests=$pr_review_tests" >> $GITHUB_OUTPUT
+          echo "extras_integration_tests=$extras_integration_tests" >> $GITHUB_OUTPUT
           echo "code_quality_tests=$code_quality_tests"
           echo "pr_review_tests=$pr_review_tests"
           echo "local_tests=$local_tests"
           echo "remote_tests=$remote_tests"
           echo "cli_tests=$cli_tests"
           echo "submodule_tests=$submodule_tests"
           echo "version_match_tests=$version_match_tests"
+          echo "extras_integration_tests=$extras_integration_tests"
 
   lint:
     needs: determine-test-scope
@@ -919,7 +939,16 @@ jobs:
           
           echo ""
           echo "=== Submodule Checks Passed ==="
-  
+
+  extras-integration-tests:
+    name: extras-integration-tests
+    needs: determine-test-scope
+    if: needs.determine-test-scope.outputs.extras_integration_tests == 'true'
+    uses: ./.github/workflows/tidy3d-extras-python-client-tests-integration.yml
+    with:
+      release_tag: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || inputs.release_tag }}
+    secrets: inherit # zizmor: ignore[secrets-inherit]
+
   workflow-validation:
     name: workflow-validation
     if: always()
@@ -936,6 +965,7 @@ jobs:
       - develop-cli-tests
       - verify-version-consistency
       - test-submodules
+      - extras-integration-tests
     runs-on: ubuntu-latest
     steps:
       - name: check-linting-result
@@ -1004,6 +1034,12 @@ jobs:
           echo "❌ Submodule tests failed."
           exit 1
 
+      - name: check-extras-integration-tests-result
+        if: ${{ needs.determine-test-scope.outputs.extras_integration_tests == 'true' && needs.extras-integration-tests.result != 'success' && needs.extras-integration-tests.result != 'skipped' }}
+        run: |
+          echo "❌ tidy3d-extras integration tests failed."
+          exit 1
+
       - name: all-checks-passed
         if: ${{ success() }}
         run: echo "✅ All required jobs passed!"
diff --git a/.github/workflows/tidy3d-python-client-update-lockfile.yml b/.github/workflows/tidy3d-python-client-update-lockfile.yml
@@ -8,6 +8,11 @@ on:
         required: true
         type: boolean
         default: true
+      source_branch:
+        description: 'Source branch to checkout and update lockfile for'
+        required: false
+        type: string
+        default: 'develop'
 
   workflow_call:
     inputs:
@@ -16,6 +21,11 @@ on:
         required: true
         type: boolean
         default: true
+      source_branch:
+        description: 'Source branch to checkout and update lockfile for'
+        required: false
+        type: string
+        default: 'develop'
 
 permissions:
   contents: read
@@ -31,6 +41,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0
         with:
+          ref: ${{ github.event.inputs.source_branch || inputs.source_branch || 'develop' }}
           fetch-depth: 1
           submodules: false
           persist-credentials: false
@@ -76,12 +87,14 @@ jobs:
         uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e #v7.0.8
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: "chore(deps): :robot: Daily update `poetry.lock`"
-          title: "chore(deps): :robot: Daily update `poetry.lock`"
+          commit-message: "chore(deps): :robot: Update `poetry.lock` for ${{ github.event.inputs.source_branch || inputs.source_branch || 'develop' }}"
+          title: "chore(deps): :robot: Update `poetry.lock` for ${{ github.event.inputs.source_branch || inputs.source_branch || 'develop' }}"
           body: |
             This pull request was automatically generated by a GitHub Action.
 
             It updates the `poetry.lock` file to reflect changes in the pip state, and whether the package is broken by external dependency changes.
-          branch: "daily-chore/update-poetry-lock"
-          base: "develop"
+
+            Source branch: ${{ github.event.inputs.source_branch || inputs.source_branch || 'develop' }}
+          branch: "chore/update-poetry-lock-${{ github.event.inputs.source_branch || inputs.source_branch || 'develop' }}"
+          base: "${{ github.event.inputs.source_branch || inputs.source_branch || 'develop' }}"
           delete-branch: true
diff --git a/dev.Dockerfile b/dev.Dockerfile
@@ -8,6 +8,8 @@ RUN apt-get update && \
     git \
     pandoc \
     xsel \
+    groff \
+    mandoc \
     xclip
 
 RUN apt-get update && apt-get install -y zip unzip curl \
@@ -20,6 +22,7 @@ RUN apt-get update && apt-get install -y zip unzip curl \
 ENV POETRY_HOME=/opt/poetry
 RUN curl -sSL https://install.python-poetry.org | python3 -
 ENV PATH="/root/.local/bin:${POETRY_HOME}/bin:${PATH}"
+RUN poetry self add poetry-codeartifact-login
 
 RUN apt-get update && DEBIAN_FRONTEND="noninteractive" TZ="America/New_York" apt-get install -y curl \
     && curl -LO https://github.com/neovim/neovim/releases/latest/download/nvim-linux-x86_64.tar.gz \
diff --git a/docs/development/docker.rst b/docs/development/docker.rst
diff --git a/tests/tidy3d_extras_license_test.py b/tests/tidy3d_extras_license_test.py
