Allow publishing alpha tags to pypi

.github/workflows/backend_checks.yml

@@ -9,9 +9,13 @@ on:
       - "main"
       - "release-**"
 
+permissions:
+  checks: write
+  pull-requests: write
+
 env:
   IMAGE: ethyca/fides:local
-  DEFAULT_PYTHON_VERSION: "3.10.16"
+  DEFAULT_PYTHON_VERSION: "3.13.11"
   # Docker auth with read-only permissions.
   DOCKER_USER: ${{ secrets.DOCKER_USER }}
   DOCKER_RO_TOKEN: ${{ secrets.DOCKER_RO_TOKEN }}
@@ -55,6 +59,7 @@ jobs:
   Collect-Tests:
     needs: Check-Backend-Changes
     if: needs.Check-Backend-Changes.outputs.has_backend_changes == 'true'
+
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -69,16 +74,20 @@ jobs:
       - name: Install Nox
         run: pip install nox>=2022
 
+      - name: Cache Nox virtual environment
+        uses: actions/cache@v4
+        with:
+          path: .nox/
+          key: ${{ runner.os }}-nox-${{ github.job }}-${{ hashFiles('noxfile.py') }}-${{ hashFiles('noxfiles/**.py') }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-nox-${{ github.job }}-
+
       - name: Run Static Check
         run: nox -s collect_tests
 
   Build:
     needs: [Check-Backend-Changes, Collect-Tests]
     if: needs.Check-Backend-Changes.outputs.has_backend_changes == 'true'
-    strategy:
-      matrix:
-        # NOTE: These are the currently supported/tested Python Versions
-        python_version: ["3.9.21", "3.10.16"]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -93,17 +102,19 @@ jobs:
         with:
           builder: ${{ steps.buildx.outputs.name }}
           context: .
-          build-args: PYTHON_VERSION=${{ matrix.python_version }}
+          build-args: PYTHON_VERSION=${{ env.DEFAULT_PYTHON_VERSION }}
           target: prod
-          outputs: type=docker,dest=/tmp/python-${{ matrix.python_version }}.tar
+          outputs: type=docker,dest=/tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar
           push: false
           tags: ${{ env.IMAGE }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
       - name: Upload container
         uses: actions/upload-artifact@v4
         with:
-          name: python-${{ matrix.python_version }}
-          path: /tmp/python-${{ matrix.python_version }}.tar
+          name: python-${{ env.DEFAULT_PYTHON_VERSION }}
+          path: /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar
           retention-days: 1
 
   ##################
@@ -142,6 +153,14 @@ jobs:
       - name: Install Nox
         run: pip install nox>=2022
 
+      - name: Cache Nox virtual environment
+        uses: actions/cache@v4
+        with:
+          path: .nox/
+          key: ${{ runner.os }}-nox-${{ github.job }}-${{ hashFiles('noxfile.py') }}-${{ hashFiles('noxfiles/**.py') }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-nox-${{ github.job }}-
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
@@ -160,6 +179,13 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
+      - name: Pull Docker images in background
+        run: |
+          docker pull postgres:16 > /dev/null 2>&1 &
+          docker pull redis:8.0-alpine > /dev/null 2>&1 &
+          echo "Docker pull initiated in background."
+        shell: bash
+
       - name: Download container
         uses: actions/download-artifact@v4
         with:
@@ -181,6 +207,14 @@ jobs:
       - name: Install Nox
         run: pip install nox>=2022
 
+      - name: Cache Nox virtual environment
+        uses: actions/cache@v4
+        with:
+          path: .nox/
+          key: ${{ runner.os }}-nox-${{ github.job }}-${{ hashFiles('noxfile.py') }}-${{ hashFiles('noxfiles/**.py') }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-nox-${{ github.job }}-
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
@@ -227,6 +261,14 @@ jobs:
       - name: Install Nox
         run: pip install nox>=2022
 
+      - name: Cache Nox virtual environment
+        uses: actions/cache@v4
+        with:
+          path: .nox/
+          key: ${{ runner.os }}-nox-${{ github.job }}-${{ hashFiles('noxfile.py') }}-${{ hashFiles('noxfiles/**.py') }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-nox-${{ github.job }}-
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
@@ -272,6 +314,14 @@ jobs:
       - name: Install Nox
         run: pip install nox>=2022
 
+      - name: Cache Nox virtual environment
+        uses: actions/cache@v4
+        with:
+          path: .nox/
+          key: ${{ runner.os }}-nox-${{ github.job }}-${{ hashFiles('noxfile.py') }}-${{ hashFiles('noxfiles/**.py') }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-nox-${{ github.job }}-qq
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
@@ -281,6 +331,12 @@ jobs:
       - name: Run test suite
         run: nox -s "${{ matrix.test_selection }}"
 
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v5
+        if: success() || failure() # always run even if the previous step fails
+        with:
+          report_paths: '**/test_report.xml'
+
   ################
   ## Safe Tests ##
   ################
@@ -290,7 +346,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.9.21", "3.10.16"]
         test_selection:
           - "ctl-not-external"
           - "ops-unit-api"
@@ -304,14 +359,21 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 45
     steps:
+      - name: Pull Docker images in background
+        run: |
+          docker pull postgres:16 > /dev/null 2>&1 &
+          docker pull redis:8.0-alpine > /dev/null 2>&1 &
+          echo "Docker pull initiated in background."
+        shell: bash
+
       - name: Download container
         uses: actions/download-artifact@v4
         with:
-          name: python-${{ matrix.python_version }}
+          name: python-${{ env.DEFAULT_PYTHON_VERSION }}
           path: /tmp/
 
       - name: Load image
-        run: docker load --input /tmp/python-${{ matrix.python_version }}.tar
+        run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar
 
       - name: Checkout
         uses: actions/checkout@v4
@@ -325,6 +387,14 @@ jobs:
       - name: Install Nox
         run: pip install nox>=2022
 
+      - name: Cache Nox virtual environment
+        uses: actions/cache@v4
+        with:
+          path: .nox/
+          key: ${{ runner.os }}-nox-${{ github.job }}-${{ matrix.test_selection }}-${{ hashFiles('noxfile.py') }}-${{ hashFiles('noxfiles/**.py') }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-nox-${{ github.job }}-${{ matrix.test_selection }}
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
@@ -334,11 +404,17 @@ jobs:
       - name: Run test suite
         run: nox -s "pytest(${{ matrix.test_selection }})"
 
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v5
+        if: success() || failure() # always run even if the previous step fails
+        with:
+          report_paths: '**/test_report.xml'
+
       - name: Upload coverage
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          fail_ci_if_error: true
+          fail_ci_if_error: false
 
   ##################
   ## Unsafe Tests ##
@@ -350,19 +426,17 @@ jobs:
     if: needs.Check-Backend-Changes.outputs.has_backend_changes == 'true' && (contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push' || github.event_name == 'merge_group')
     strategy:
       max-parallel: 1 # This prevents collisions in shared external resources
-      matrix:
-        python_version: ["3.9.21", "3.10.16"]
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
       - name: Download container
         uses: actions/download-artifact@v4
         with:
-          name: python-${{ matrix.python_version }}
+          name: python-${{ env.DEFAULT_PYTHON_VERSION }}
           path: /tmp/
 
       - name: Load image
-        run: docker load --input /tmp/python-${{ matrix.python_version }}.tar
+        run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar
 
       - name: Checkout
         uses: actions/checkout@v4
@@ -376,6 +450,14 @@ jobs:
       - name: Install Nox
         run: pip install nox>=2022
 
+      - name: Cache Nox virtual environment
+        uses: actions/cache@v4
+        with:
+          path: .nox/
+          key: ${{ runner.os }}-nox-${{ github.job }}-${{ hashFiles('noxfile.py') }}-${{ hashFiles('noxfiles/**.py') }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-nox-${{ github.job }}-
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
@@ -411,19 +493,17 @@ jobs:
     if: needs.Check-Backend-Changes.outputs.has_backend_changes == 'true' && (contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push' || github.event_name == 'merge_group')
     strategy:
       max-parallel: 1 # This prevents collisions in shared external resources
-      matrix:
-        python_version: ["3.9.21", "3.10.16"]
     runs-on: ubuntu-latest
     timeout-minutes: 45
     steps:
       - name: Download container
         uses: actions/download-artifact@v4
         with:
-          name: python-${{ matrix.python_version }}
+          name: python-${{ env.DEFAULT_PYTHON_VERSION }}
           path: /tmp/
 
       - name: Load image
-        run: docker load --input /tmp/python-${{ matrix.python_version }}.tar
+        run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar
 
       - name: Checkout
         uses: actions/checkout@v4
@@ -437,6 +517,14 @@ jobs:
       - name: Install Nox
         run: pip install nox>=2022
 
+      - name: Cache Nox virtual environment
+        uses: actions/cache@v4
+        with:
+          path: .nox/
+          key: ${{ runner.os }}-nox-${{ github.job }}-${{ hashFiles('noxfile.py') }}-${{ hashFiles('noxfiles/**.py') }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-nox-${{ github.job }}-
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
@@ -520,17 +608,15 @@ jobs:
       id-token: write
     strategy:
       max-parallel: 1 # This prevents collisions in shared external resources
-      matrix:
-        python_version: ["3.9.21", "3.10.16"]
     steps:
       - name: Download container
         uses: actions/download-artifact@v4
         with:
-          name: python-${{ matrix.python_version }}
+          name: python-${{ env.DEFAULT_PYTHON_VERSION }}
           path: /tmp/
 
       - name: Load image
-        run: docker load --input /tmp/python-${{ matrix.python_version }}.tar
+        run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar
 
       - name: Checkout
         uses: actions/checkout@v4
@@ -544,6 +630,14 @@ jobs:
       - name: Install Nox
         run: pip install nox>=2022
 
+      - name: Cache Nox virtual environment
+        uses: actions/cache@v4
+        with:
+          path: .nox/
+          key: ${{ runner.os }}-nox-${{ github.job }}-${{ hashFiles('noxfile.py') }}-${{ hashFiles('noxfiles/**.py') }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-nox-${{ github.job }}-
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:

.github/workflows/cache_docker_image.yml

@@ -0,0 +1,30 @@
+name: Cache sidecar container image
+on:
+  workflow_call:
+    inputs:
+      image-name:
+        type: string
+        required: true
+        description: The name of the container image to cache
+      tag:
+        type: string
+        required: true
+        description: The tag of the container image to cache
+
+jobs:
+  cache-image:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Cache Docker images
+      uses: actions/cache@v4
+      with:
+        path: /tmp/docker-images # Path to store the tarball
+        key: docker-${{ runner.os }}-${{ inputs.image-name }}-${{ inputs.tag }}-${{ hashFiles('**/Dockerfile') }} # Key for the cache
+    - name: Pull and save image
+      run: |
+        docker pull ${{ inputs.image-name }}:${{ inputs.tag }}
+        docker save -o /tmp/docker-images/${{ inputs.image-name }}-${{ inputs.tag }}.tar ${{ inputs.image-name }}:${{ inputs.tag }}
+      if: steps.cache-image.outputs.cache-hit != 'true' # Only run if cache miss
+    - name: Load image from cache
+      run: docker load -i /tmp/docker-images/${{ inputs.image-name }}-${{ inputs.tag }}.tar
+      if: steps.cache-image.outputs.cache-hit == 'true' # Only run if cache hit

.github/workflows/cache_redis_and_postgres_images.yml

@@ -0,0 +1,27 @@
+name: Cache Redis and Postgres container images
+on:
+  workflow_call:
+    inputs:
+      redis-tag:
+        type: string
+        required: false
+        default: 8.0-alpine
+        description: The image tag to cache for Redis
+      postgres-tag:
+        type: string
+        required: false
+        default: 16
+        description: The image tag to cache for Postgres
+
+jobs:
+  cache-images:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: ./.github/workflows/cache_docker_image.yml
+        with:
+          image-name: postgres
+          tag: ${{ inputs.postgres-tag }}
+      - uses: ./.github/workflows/cache_docker_image.yml
+        with:
+          image-name: redis
+          tag: ${{ inputs.redis-tag }}