Add retry and caching to our GitHub actions

.github/actions/retry-command/action.yml

@@ -0,0 +1,55 @@
+name: Retry command
+description: Run a shell command with retries and exponential backoff.
+inputs:
+  command:
+    description: Command to run.
+    required: true
+  working-directory:
+    description: Directory to run the command in.
+    required: false
+    default: .
+  attempts:
+    description: Maximum number of attempts.
+    required: false
+    default: "3"
+  delay-seconds:
+    description: Initial delay between attempts.
+    required: false
+    default: "20"
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      env:
+        RETRY_COMMAND: ${{ inputs.command }}
+      working-directory: ${{ inputs.working-directory }}
+      run: |
+        set -euo pipefail
+
+        attempts="${{ inputs.attempts }}"
+        delay="${{ inputs.delay-seconds }}"
+        script="$(mktemp)"
+        trap 'rm -f "$script"' EXIT
+
+        printf '%s\n' "$RETRY_COMMAND" > "$script"
+
+        for attempt in $(seq 1 "$attempts"); do
+          echo "Running attempt ${attempt}/${attempts}"
+          set +e
+          bash -euo pipefail "$script"
+          status=$?
+          set -e
+
+          if [ "$status" -eq 0 ]; then
+            exit 0
+          fi
+
+          if [ "$attempt" -eq "$attempts" ]; then
+            echo "Command failed after ${attempts} attempts"
+            exit "$status"
+          fi
+
+          echo "Command failed with exit code ${status}; retrying in ${delay}s"
+          sleep "$delay"
+          delay=$((delay * 2))
+        done

.github/workflows/build_and_test_website.yml

@@ -29,10 +29,39 @@ jobs:
         uses: actions/setup-node@v6
         with:
           node-version: "20.x"
+          cache: yarn
+          cache-dependency-path: website/yarn.lock
+      - name: Restore wasm tool cache
+        id: wasm-tool-cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/wasm-pack
+            ~/.cargo/bin/wasm-opt
+          key: ${{ runner.os }}-${{ runner.arch }}-wasm-pack-0.15.0-wasm-opt-0.116.1
+      - name: Set up Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: pyrefly-website
+          cache-targets: false
       - name: Install yarn deps
-        run: cd website && yarn install
+        uses: ./.github/actions/retry-command
+        with:
+          working-directory: website
+          command: yarn install --frozen-lockfile
       - name: Install wasm-pack
-        run: cargo install wasm-pack wasm-opt
+        if: steps.wasm-tool-cache.outputs.cache-hit != 'true'
+        uses: ./.github/actions/retry-command
+        with:
+          command: |
+            cargo install wasm-pack --version 0.15.0 --locked
+            cargo install wasm-opt --version 0.116.1 --locked
+      - name: Fetch wasm Cargo dependencies
+        uses: ./.github/actions/retry-command
+        with:
+          command: |
+            rustup target add wasm32-unknown-unknown
+            cargo fetch --locked --manifest-path pyrefly_wasm/Cargo.toml --target wasm32-unknown-unknown
       - name: Build
         working-directory: ./website
         run: cargo version && chmod +x scripts/build.sh && scripts/build.sh && yarn build-wasm-for-test

.github/workflows/build_binaries.yml

@@ -25,6 +25,12 @@ jobs:
       - uses: actions/setup-python@v6
         with:
           python-version: ${{ env.PYTHON_VERSION }}
+      - name: Set up Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: pyrefly-build-binaries
+          shared-key: sdist
+          cache-targets: false
       - name: Build sdist
         uses: PyO3/maturin-action@v1
         with:
@@ -62,6 +68,12 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           architecture: ${{ matrix.platform.arch }}
+      - name: Set up Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: pyrefly-build-binaries
+          shared-key: macos
+          cache-targets: false
       - name: Set jemalloc page size for macOS ARM64
         if: ${{ matrix.platform.target == 'aarch64' }}
         run: echo "JEMALLOC_SYS_WITH_LG_PAGE=14" >> $GITHUB_ENV
@@ -109,6 +121,12 @@ jobs:
         # we need to set CARGO_HOME to a high-up directory on Windows machines, since some dependencies cloned
         # by Cargo have long paths and will cause builds/tests to fail
         run: echo "CARGO_HOME=C:\\cargo" >> $env:GITHUB_ENV
+      - name: Set up Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: pyrefly-build-binaries
+          shared-key: windows
+          cache-targets: false
       - name: Build wheels
         uses: PyO3/maturin-action@v1
         with:
@@ -148,6 +166,12 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           architecture: x64
+      - name: Set up Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: pyrefly-build-binaries
+          shared-key: linux
+          cache-targets: false
       - name: Build wheels
         if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' || matrix.target == 'i686-unknown-linux-gnu' }}
         uses: PyO3/maturin-action@v1
@@ -217,6 +241,12 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           architecture: x64
+      - name: Set up Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: pyrefly-build-binaries
+          shared-key: musllinux
+          cache-targets: false
       - name: Build wheels
         if: ${{ matrix.target == 'x86_64-unknown-linux-musl' }}
         uses: PyO3/maturin-action@v1
@@ -244,7 +274,18 @@ jobs:
             -v ${{ github.workspace }}:/io -w /io \
             --env PACKAGE_NAME --env BINARY_NAME \
             alpine:latest sh -c '
-              apk add --no-cache python3
+              attempt=1
+              delay=20
+              until apk add --no-cache python3; do
+                status=$?
+                if [ "$attempt" -eq 3 ]; then
+                  exit "$status"
+                fi
+                echo "apk add failed with exit code ${status}; retrying in ${delay}s"
+                sleep "$delay"
+                attempt=$((attempt + 1))
+                delay=$((delay * 2))
+              done
               python3 -m venv .venv
               .venv/bin/pip install "${PACKAGE_NAME}" --no-index --find-links pyrefly/dist/ --force-reinstall
               .venv/bin/"${BINARY_NAME}" --version

.github/workflows/build_extension.yml

@@ -107,9 +107,9 @@ jobs:
       - uses: actions/checkout@v6
       - name: install toolchain dependencies
         if: ${{ matrix.container == 'ubuntu:20.04' }}
-        shell: bash
-        run: |
-          apt-get update && apt-get -y install curl build-essential
+        uses: ./.github/actions/retry-command
+        with:
+          command: apt-get update && apt-get -y install curl build-essential
       - uses: dtolnay/rust-toolchain@stable
       - name: set windows cargo home
         # we need to set CARGO_HOME to a high-up directory on Windows machines, since some dependencies cloned
@@ -126,6 +126,17 @@ jobs:
       - name: set jemalloc page size for ARM64 macOS
         if: ${{ matrix.arch == 'arm64' && matrix.platform == 'darwin' }}
         run: echo "JEMALLOC_SYS_WITH_LG_PAGE=14" >> ${{ matrix.github_env }}
+      - name: Fetch Cargo dependencies
+        uses: ./.github/actions/retry-command
+        env:
+          RUST_TARGET: ${{ matrix.rust_target }}
+        with:
+          command: |
+            if [ -n "$RUST_TARGET" ]; then
+              cargo fetch --locked --manifest-path pyrefly/Cargo.toml --target "$RUST_TARGET"
+            else
+              cargo fetch --locked --manifest-path pyrefly/Cargo.toml
+            fi
       - name: build pyrefly binary (cross-compile)
         if: ${{ matrix.rust_target != '' }}
         uses: houseabsolute/actions-rust-cross@v1
@@ -155,10 +166,14 @@ jobs:
         with:
           node-version: 22
           cache: npm
+          cache-dependency-path: lsp/package-lock.json
       - name: save platform name
         run: echo "platform=${{ matrix.platform }}-${{ matrix.arch }}" >> ${{ matrix.github_env }}
-      - run: npm ci
-        working-directory: lsp/
+      - name: Install extension npm dependencies
+        uses: ./.github/actions/retry-command
+        with:
+          working-directory: lsp/
+          command: npm ci
       - run: npx vsce package --target ${{ env.platform }} ${{ needs.get_version.outputs.prerelease_flag }} ${{needs.get_version.outputs.marketplace_version}}
         working-directory: lsp/
       - uses: actions/upload-artifact@v6

.github/workflows/issue_ranking.yml

@@ -51,8 +51,21 @@ jobs:
         with:
           toolchain: stable
 
+      - name: Set up Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: pyrefly-issue-ranking-primer
+          cache-targets: false
+
       - name: Install type checkers
-        run: pip install pyright mypy
+        uses: ./.github/actions/retry-command
+        with:
+          command: pip install pyright mypy
+
+      - name: Fetch Cargo dependencies
+        uses: ./.github/actions/retry-command
+        with:
+          command: cargo fetch --locked --manifest-path pyrefly/Cargo.toml
 
       - name: Build pyrefly
         run: |
@@ -161,13 +174,15 @@ jobs:
       # Install type checkers for snippet checking
       - name: Install type checkers
         if: ${{ inputs.check_snippets }}
-        run: |
-          echo "=== Installing type checkers ==="
-          pip install pyrefly pyright mypy
-          echo "pyrefly version: $(pyrefly --version 2>&1 || echo 'not found')"
-          echo "pyright version: $(pyright --version 2>&1 || echo 'not found')"
-          echo "mypy version: $(mypy --version 2>&1 || echo 'not found')"
-          echo "pyrefly path: $(which pyrefly 2>&1 || echo 'not found')"
+        uses: ./.github/actions/retry-command
+        with:
+          command: |
+            echo "=== Installing type checkers ==="
+            pip install pyrefly pyright mypy
+            echo "pyrefly version: $(pyrefly --version 2>&1 || echo 'not found')"
+            echo "pyright version: $(pyright --version 2>&1 || echo 'not found')"
+            echo "mypy version: $(mypy --version 2>&1 || echo 'not found')"
+            echo "pyrefly path: $(which pyrefly 2>&1 || echo 'not found')"
 
       # Download primer results if available
       - name: Download primer results

.github/workflows/mypy_primer.yml

@@ -36,14 +36,22 @@ jobs:
         with:
           python-version: "3.13"
       - name: Install dependencies
-        run: |
-          python -m pip install -U pip
-          pip install git+https://github.com/hauntsaninja/mypy_primer.git
+        uses: ./pyrefly_to_test/.github/actions/retry-command
+        with:
+          command: |
+            python -m pip install -U pip
+            pip install git+https://github.com/hauntsaninja/mypy_primer.git
       - uses: dtolnay/rust-toolchain@master
         with:
           toolchain: stable
           components: clippy, rustfmt
-      - name: Run mypy_primer
+      - name: Set up Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: pyrefly-mypy-primer
+          cache-targets: false
+          workspaces: pyrefly_to_test
+      - name: Prepare mypy_primer comparison
         shell: bash
         run: |
           cd pyrefly_to_test
@@ -55,35 +63,62 @@ jobs:
           echo "base commit"
           git rev-list --format=%s --max-count=1 base_commit
 
+          git checkout "$GITHUB_SHA"
+
           echo ''
           cd ..
 
           BASE_DIR=$(pwd)/primer_base
           mkdir -p $BASE_DIR
+          echo "BASE_DIR=$BASE_DIR" >> "$GITHUB_ENV"
 
-          # Build pyrefly for "new" commit and install wrapper
           NEW_DIR=$BASE_DIR/pyrefly_new
           mkdir -p $NEW_DIR
-          cd pyrefly_to_test && git checkout $GITHUB_SHA
-          CARGO_TARGET_DIR=$NEW_DIR/target cargo build --release --manifest-path pyrefly/Cargo.toml
-          BINARY_DIR=$NEW_DIR/target/release
-          mv $BINARY_DIR/pyrefly $BINARY_DIR/pyrefly-real
-          cp scripts/pyrefly_primer_wrapper.sh $BINARY_DIR/pyrefly
-          chmod +x $BINARY_DIR/pyrefly
+          echo "NEW_DIR=$NEW_DIR" >> "$GITHUB_ENV"
 
-          # Build pyrefly for "old" commit and install wrapper
           OLD_DIR=$BASE_DIR/pyrefly_old
           mkdir -p $OLD_DIR
-          git checkout base_commit
-          CARGO_TARGET_DIR=$OLD_DIR/target cargo build --release --manifest-path pyrefly/Cargo.toml
+          echo "OLD_DIR=$OLD_DIR" >> "$GITHUB_ENV"
+      - name: Fetch new Pyrefly Cargo dependencies
+        uses: ./pyrefly_to_test/.github/actions/retry-command
+        with:
+          working-directory: pyrefly_to_test
+          command: cargo fetch --manifest-path pyrefly/Cargo.toml
+      - name: Build new Pyrefly
+        shell: bash
+        working-directory: pyrefly_to_test
+        run: CARGO_TARGET_DIR="$NEW_DIR/target" cargo build --release --manifest-path pyrefly/Cargo.toml
+      - name: Install new Pyrefly primer wrapper
+        shell: bash
+        run: |
+          BINARY_DIR=$NEW_DIR/target/release
+          mv $BINARY_DIR/pyrefly $BINARY_DIR/pyrefly-real
+          cp pyrefly_to_test/scripts/pyrefly_primer_wrapper.sh $BINARY_DIR/pyrefly
+          chmod +x $BINARY_DIR/pyrefly
+      - name: Fetch old Pyrefly Cargo dependencies
+        uses: ./pyrefly_to_test/.github/actions/retry-command
+        with:
+          working-directory: pyrefly_to_test
+          command: |
+            git checkout base_commit
+            cargo fetch --manifest-path pyrefly/Cargo.toml
+      - name: Build old Pyrefly
+        shell: bash
+        working-directory: pyrefly_to_test
+        run: CARGO_TARGET_DIR="$OLD_DIR/target" cargo build --release --manifest-path pyrefly/Cargo.toml
+      - name: Install old Pyrefly primer wrapper
+        shell: bash
+        run: |
           BINARY_DIR=$OLD_DIR/target/release
           mv $BINARY_DIR/pyrefly $BINARY_DIR/pyrefly-real
           # Use the wrapper from the new commit (old commit may not have it)
-          git checkout $GITHUB_SHA -- scripts/pyrefly_primer_wrapper.sh
+          cd pyrefly_to_test
+          git checkout "$GITHUB_SHA" -- scripts/pyrefly_primer_wrapper.sh
           cp scripts/pyrefly_primer_wrapper.sh $BINARY_DIR/pyrefly
           chmod +x $BINARY_DIR/pyrefly
-
-          cd ..
+      - name: Run mypy_primer
+        shell: bash
+        run: |
           # fail action if exit code isn't zero or one
           (
             MYPY_PRIMER_NO_REBUILD=1 mypy_primer \