diff --git a/.github/actions/retry-command/action.yml b/.github/actions/retry-command/action.yml new file mode 100644 index 0000000000..97241fd7ba --- /dev/null +++ b/.github/actions/retry-command/action.yml @@ -0,0 +1,55 @@ +name: Retry command +description: Run a shell command with retries and exponential backoff. +inputs: + command: + description: Command to run. + required: true + working-directory: + description: Directory to run the command in. + required: false + default: . + attempts: + description: Maximum number of attempts. + required: false + default: "3" + delay-seconds: + description: Initial delay between attempts. + required: false + default: "20" +runs: + using: composite + steps: + - shell: bash + env: + RETRY_COMMAND: ${{ inputs.command }} + working-directory: ${{ inputs.working-directory }} + run: | + set -euo pipefail + + attempts="${{ inputs.attempts }}" + delay="${{ inputs.delay-seconds }}" + script="$(mktemp)" + trap 'rm -f "$script"' EXIT + + printf '%s\n' "$RETRY_COMMAND" > "$script" + + for attempt in $(seq 1 "$attempts"); do + echo "Running attempt ${attempt}/${attempts}" + set +e + bash -euo pipefail "$script" + status=$? + set -e + + if [ "$status" -eq 0 ]; then + exit 0 + fi + + if [ "$attempt" -eq "$attempts" ]; then + echo "Command failed after ${attempts} attempts" + exit "$status" + fi + + echo "Command failed with exit code ${status}; retrying in ${delay}s" + sleep "$delay" + delay=$((delay * 2)) + done diff --git a/.github/workflows/build_and_test_website.yml b/.github/workflows/build_and_test_website.yml index d0754d0ea0..a0ed273eaa 100644 --- a/.github/workflows/build_and_test_website.yml +++ b/.github/workflows/build_and_test_website.yml @@ -29,10 +29,39 @@ jobs: uses: actions/setup-node@v6 with: node-version: "20.x" + cache: yarn + cache-dependency-path: website/yarn.lock + - name: Restore wasm tool cache + id: wasm-tool-cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/wasm-pack + ~/.cargo/bin/wasm-opt + key: ${{ runner.os }}-${{ runner.arch }}-wasm-pack-0.15.0-wasm-opt-0.116.1 + - name: Set up Rust cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: pyrefly-website + cache-targets: false - name: Install yarn deps - run: cd website && yarn install + uses: ./.github/actions/retry-command + with: + working-directory: website + command: yarn install --frozen-lockfile - name: Install wasm-pack - run: cargo install wasm-pack wasm-opt + if: steps.wasm-tool-cache.outputs.cache-hit != 'true' + uses: ./.github/actions/retry-command + with: + command: | + cargo install wasm-pack --version 0.15.0 --locked + cargo install wasm-opt --version 0.116.1 --locked + - name: Fetch wasm Cargo dependencies + uses: ./.github/actions/retry-command + with: + command: | + rustup target add wasm32-unknown-unknown + cargo fetch --locked --manifest-path pyrefly_wasm/Cargo.toml --target wasm32-unknown-unknown - name: Build working-directory: ./website run: cargo version && chmod +x scripts/build.sh && scripts/build.sh && yarn build-wasm-for-test diff --git a/.github/workflows/build_binaries.yml b/.github/workflows/build_binaries.yml index 09123181ae..c2199c8441 100644 --- a/.github/workflows/build_binaries.yml +++ b/.github/workflows/build_binaries.yml @@ -25,6 +25,12 @@ jobs: - uses: actions/setup-python@v6 with: python-version: ${{ env.PYTHON_VERSION }} + - name: Set up Rust cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: pyrefly-build-binaries + shared-key: sdist + cache-targets: false - name: Build sdist uses: PyO3/maturin-action@v1 with: @@ -62,6 +68,12 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} architecture: ${{ matrix.platform.arch }} + - name: Set up Rust cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: pyrefly-build-binaries + shared-key: macos + cache-targets: false - name: Set jemalloc page size for macOS ARM64 if: ${{ matrix.platform.target == 'aarch64' }} run: echo "JEMALLOC_SYS_WITH_LG_PAGE=14" >> $GITHUB_ENV @@ -109,6 +121,12 @@ jobs: # we need to set CARGO_HOME to a high-up directory on Windows machines, since some dependencies cloned # by Cargo have long paths and will cause builds/tests to fail run: echo "CARGO_HOME=C:\\cargo" >> $env:GITHUB_ENV + - name: Set up Rust cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: pyrefly-build-binaries + shared-key: windows + cache-targets: false - name: Build wheels uses: PyO3/maturin-action@v1 with: @@ -148,6 +166,12 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} architecture: x64 + - name: Set up Rust cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: pyrefly-build-binaries + shared-key: linux + cache-targets: false - name: Build wheels if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' || matrix.target == 'i686-unknown-linux-gnu' }} uses: PyO3/maturin-action@v1 @@ -217,6 +241,12 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} architecture: x64 + - name: Set up Rust cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: pyrefly-build-binaries + shared-key: musllinux + cache-targets: false - name: Build wheels if: ${{ matrix.target == 'x86_64-unknown-linux-musl' }} uses: PyO3/maturin-action@v1 @@ -244,7 +274,18 @@ jobs: -v ${{ github.workspace }}:/io -w /io \ --env PACKAGE_NAME --env BINARY_NAME \ alpine:latest sh -c ' - apk add --no-cache python3 + attempt=1 + delay=20 + until apk add --no-cache python3; do + status=$? + if [ "$attempt" -eq 3 ]; then + exit "$status" + fi + echo "apk add failed with exit code ${status}; retrying in ${delay}s" + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done python3 -m venv .venv .venv/bin/pip install "${PACKAGE_NAME}" --no-index --find-links pyrefly/dist/ --force-reinstall .venv/bin/"${BINARY_NAME}" --version diff --git a/.github/workflows/build_extension.yml b/.github/workflows/build_extension.yml index e75878fb1c..99f7b30bb4 100644 --- a/.github/workflows/build_extension.yml +++ b/.github/workflows/build_extension.yml @@ -107,9 +107,9 @@ jobs: - uses: actions/checkout@v6 - name: install toolchain dependencies if: ${{ matrix.container == 'ubuntu:20.04' }} - shell: bash - run: | - apt-get update && apt-get -y install curl build-essential + uses: ./.github/actions/retry-command + with: + command: apt-get update && apt-get -y install curl build-essential - uses: dtolnay/rust-toolchain@stable - name: set windows cargo home # we need to set CARGO_HOME to a high-up directory on Windows machines, since some dependencies cloned @@ -126,6 +126,17 @@ jobs: - name: set jemalloc page size for ARM64 macOS if: ${{ matrix.arch == 'arm64' && matrix.platform == 'darwin' }} run: echo "JEMALLOC_SYS_WITH_LG_PAGE=14" >> ${{ matrix.github_env }} + - name: Fetch Cargo dependencies + uses: ./.github/actions/retry-command + env: + RUST_TARGET: ${{ matrix.rust_target }} + with: + command: | + if [ -n "$RUST_TARGET" ]; then + cargo fetch --locked --manifest-path pyrefly/Cargo.toml --target "$RUST_TARGET" + else + cargo fetch --locked --manifest-path pyrefly/Cargo.toml + fi - name: build pyrefly binary (cross-compile) if: ${{ matrix.rust_target != '' }} uses: houseabsolute/actions-rust-cross@v1 @@ -155,10 +166,14 @@ jobs: with: node-version: 22 cache: npm + cache-dependency-path: lsp/package-lock.json - name: save platform name run: echo "platform=${{ matrix.platform }}-${{ matrix.arch }}" >> ${{ matrix.github_env }} - - run: npm ci - working-directory: lsp/ + - name: Install extension npm dependencies + uses: ./.github/actions/retry-command + with: + working-directory: lsp/ + command: npm ci - run: npx vsce package --target ${{ env.platform }} ${{ needs.get_version.outputs.prerelease_flag }} ${{needs.get_version.outputs.marketplace_version}} working-directory: lsp/ - uses: actions/upload-artifact@v6 diff --git a/.github/workflows/issue_ranking.yml b/.github/workflows/issue_ranking.yml index a26db83f9b..1bf04508b4 100644 --- a/.github/workflows/issue_ranking.yml +++ b/.github/workflows/issue_ranking.yml @@ -51,8 +51,21 @@ jobs: with: toolchain: stable + - name: Set up Rust cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: pyrefly-issue-ranking-primer + cache-targets: false + - name: Install type checkers - run: pip install pyright mypy + uses: ./.github/actions/retry-command + with: + command: pip install pyright mypy + + - name: Fetch Cargo dependencies + uses: ./.github/actions/retry-command + with: + command: cargo fetch --locked --manifest-path pyrefly/Cargo.toml - name: Build pyrefly run: | @@ -161,13 +174,15 @@ jobs: # Install type checkers for snippet checking - name: Install type checkers if: ${{ inputs.check_snippets }} - run: | - echo "=== Installing type checkers ===" - pip install pyrefly pyright mypy - echo "pyrefly version: $(pyrefly --version 2>&1 || echo 'not found')" - echo "pyright version: $(pyright --version 2>&1 || echo 'not found')" - echo "mypy version: $(mypy --version 2>&1 || echo 'not found')" - echo "pyrefly path: $(which pyrefly 2>&1 || echo 'not found')" + uses: ./.github/actions/retry-command + with: + command: | + echo "=== Installing type checkers ===" + pip install pyrefly pyright mypy + echo "pyrefly version: $(pyrefly --version 2>&1 || echo 'not found')" + echo "pyright version: $(pyright --version 2>&1 || echo 'not found')" + echo "mypy version: $(mypy --version 2>&1 || echo 'not found')" + echo "pyrefly path: $(which pyrefly 2>&1 || echo 'not found')" # Download primer results if available - name: Download primer results diff --git a/.github/workflows/mypy_primer.yml b/.github/workflows/mypy_primer.yml index bf6ed29334..e6604e66aa 100644 --- a/.github/workflows/mypy_primer.yml +++ b/.github/workflows/mypy_primer.yml @@ -36,14 +36,22 @@ jobs: with: python-version: "3.13" - name: Install dependencies - run: | - python -m pip install -U pip - pip install git+https://github.com/hauntsaninja/mypy_primer.git + uses: ./pyrefly_to_test/.github/actions/retry-command + with: + command: | + python -m pip install -U pip + pip install git+https://github.com/hauntsaninja/mypy_primer.git - uses: dtolnay/rust-toolchain@master with: toolchain: stable components: clippy, rustfmt - - name: Run mypy_primer + - name: Set up Rust cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: pyrefly-mypy-primer + cache-targets: false + workspaces: pyrefly_to_test + - name: Prepare mypy_primer comparison shell: bash run: | cd pyrefly_to_test @@ -55,35 +63,62 @@ jobs: echo "base commit" git rev-list --format=%s --max-count=1 base_commit + git checkout "$GITHUB_SHA" + echo '' cd .. BASE_DIR=$(pwd)/primer_base mkdir -p $BASE_DIR + echo "BASE_DIR=$BASE_DIR" >> "$GITHUB_ENV" - # Build pyrefly for "new" commit and install wrapper NEW_DIR=$BASE_DIR/pyrefly_new mkdir -p $NEW_DIR - cd pyrefly_to_test && git checkout $GITHUB_SHA - CARGO_TARGET_DIR=$NEW_DIR/target cargo build --release --manifest-path pyrefly/Cargo.toml - BINARY_DIR=$NEW_DIR/target/release - mv $BINARY_DIR/pyrefly $BINARY_DIR/pyrefly-real - cp scripts/pyrefly_primer_wrapper.sh $BINARY_DIR/pyrefly - chmod +x $BINARY_DIR/pyrefly + echo "NEW_DIR=$NEW_DIR" >> "$GITHUB_ENV" - # Build pyrefly for "old" commit and install wrapper OLD_DIR=$BASE_DIR/pyrefly_old mkdir -p $OLD_DIR - git checkout base_commit - CARGO_TARGET_DIR=$OLD_DIR/target cargo build --release --manifest-path pyrefly/Cargo.toml + echo "OLD_DIR=$OLD_DIR" >> "$GITHUB_ENV" + - name: Fetch new Pyrefly Cargo dependencies + uses: ./pyrefly_to_test/.github/actions/retry-command + with: + working-directory: pyrefly_to_test + command: cargo fetch --manifest-path pyrefly/Cargo.toml + - name: Build new Pyrefly + shell: bash + working-directory: pyrefly_to_test + run: CARGO_TARGET_DIR="$NEW_DIR/target" cargo build --release --manifest-path pyrefly/Cargo.toml + - name: Install new Pyrefly primer wrapper + shell: bash + run: | + BINARY_DIR=$NEW_DIR/target/release + mv $BINARY_DIR/pyrefly $BINARY_DIR/pyrefly-real + cp pyrefly_to_test/scripts/pyrefly_primer_wrapper.sh $BINARY_DIR/pyrefly + chmod +x $BINARY_DIR/pyrefly + - name: Fetch old Pyrefly Cargo dependencies + uses: ./pyrefly_to_test/.github/actions/retry-command + with: + working-directory: pyrefly_to_test + command: | + git checkout base_commit + cargo fetch --manifest-path pyrefly/Cargo.toml + - name: Build old Pyrefly + shell: bash + working-directory: pyrefly_to_test + run: CARGO_TARGET_DIR="$OLD_DIR/target" cargo build --release --manifest-path pyrefly/Cargo.toml + - name: Install old Pyrefly primer wrapper + shell: bash + run: | BINARY_DIR=$OLD_DIR/target/release mv $BINARY_DIR/pyrefly $BINARY_DIR/pyrefly-real # Use the wrapper from the new commit (old commit may not have it) - git checkout $GITHUB_SHA -- scripts/pyrefly_primer_wrapper.sh + cd pyrefly_to_test + git checkout "$GITHUB_SHA" -- scripts/pyrefly_primer_wrapper.sh cp scripts/pyrefly_primer_wrapper.sh $BINARY_DIR/pyrefly chmod +x $BINARY_DIR/pyrefly - - cd .. + - name: Run mypy_primer + shell: bash + run: | # fail action if exit code isn't zero or one ( MYPY_PRIMER_NO_REBUILD=1 mypy_primer \ diff --git a/.github/workflows/primer_comparison.yml b/.github/workflows/primer_comparison.yml index 056b1340b4..de32c5149b 100644 --- a/.github/workflows/primer_comparison.yml +++ b/.github/workflows/primer_comparison.yml @@ -37,8 +37,21 @@ jobs: with: toolchain: stable + - name: Set up Rust cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: pyrefly-primer-comparison + cache-targets: false + - name: Install type checkers - run: pip install pyright mypy + uses: ./.github/actions/retry-command + with: + command: pip install pyright mypy + + - name: Fetch Cargo dependencies + uses: ./.github/actions/retry-command + with: + command: cargo fetch --locked --manifest-path pyrefly/Cargo.toml - name: Build pyrefly run: | diff --git a/.github/workflows/pyrefly.yml b/.github/workflows/pyrefly.yml index e597d2d15a..1af02aae30 100644 --- a/.github/workflows/pyrefly.yml +++ b/.github/workflows/pyrefly.yml @@ -27,8 +27,24 @@ jobs: # by Cargo have long paths and will cause builds/tests to fail if: ${{ matrix.os == 'windows-latest' }} run: echo "CARGO_HOME=C:\\cargo" >> ${{ matrix.github_env }} - - name: install Scrut - run: cargo install scrut --locked + - name: Restore Scrut cache + uses: actions/cache@v4 + if: ${{ matrix.os != 'windows-latest' }} + with: + path: | + ~/.cargo/bin/scrut + ~/.cargo/.crates.toml + ~/.cargo/.crates2.json + key: ${{ runner.os }}-${{ runner.arch }}-scrut-0.4.3-v2 + - name: Install Scrut + uses: ./.github/actions/retry-command + with: + command: | + if command -v scrut >/dev/null 2>&1 && scrut --version | grep -q '0\.4\.3'; then + scrut --version + else + cargo install scrut --version 0.4.3 --locked --force + fi # Scrut doesn't support Windows yet if: ${{ matrix.os != 'windows-latest' }} - name: set up rust cache @@ -39,6 +55,10 @@ jobs: with: toolchain: stable components: clippy, rustfmt + - name: Fetch Cargo dependencies + uses: ./.github/actions/retry-command + with: + command: cargo fetch --locked - run: cargo fmt -- --check - run: cargo clippy --release - run: cargo build --release diff --git a/.github/workflows/release_binaries.yml b/.github/workflows/release_binaries.yml index da4170aa2e..3372b26bfe 100644 --- a/.github/workflows/release_binaries.yml +++ b/.github/workflows/release_binaries.yml @@ -94,14 +94,16 @@ jobs: - name: Install container dependencies if: matrix.container == 'ubuntu:20.04' - run: | - apt-get update && apt-get -y install curl build-essential git - curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg - chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null - apt-get update && apt-get -y install gh - # Fix git safe.directory for container builds (checkout creates repo with different owner) - git config --global --add safe.directory "$GITHUB_WORKSPACE" + uses: ./.github/actions/retry-command + with: + command: | + apt-get update && apt-get -y install curl build-essential git + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg + chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null + apt-get update && apt-get -y install gh + # Fix git safe.directory for container builds (checkout creates repo with different owner) + git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -172,20 +174,23 @@ jobs: - name: Upload to release (Unix) if: runner.os != 'Windows' - run: | - gh release upload ${{ env.TAG_NAME }} \ - ${{ matrix.artifact_name }}.tar.gz \ - ${{ matrix.artifact_name }}.tar.gz.sha256 \ - --clobber + uses: ./.github/actions/retry-command + with: + command: | + gh release upload "$TAG_NAME" \ + ${{ matrix.artifact_name }}.tar.gz \ + ${{ matrix.artifact_name }}.tar.gz.sha256 \ + --clobber - name: Upload to release (Windows) if: runner.os == 'Windows' - shell: pwsh - run: | - gh release upload ${{ env.TAG_NAME }} ` - "${{ matrix.artifact_name }}.zip" ` - "${{ matrix.artifact_name }}.zip.sha256" ` - --clobber + uses: ./.github/actions/retry-command + with: + command: | + gh release upload "$TAG_NAME" \ + "${{ matrix.artifact_name }}.zip" \ + "${{ matrix.artifact_name }}.zip.sha256" \ + --clobber wasm: name: Build pyrefly-wasm diff --git a/.github/workflows/test_extension.yml b/.github/workflows/test_extension.yml index 271047abc9..8e641cf55d 100644 --- a/.github/workflows/test_extension.yml +++ b/.github/workflows/test_extension.yml @@ -78,13 +78,37 @@ jobs: with: node-version: 22 cache: npm - - run: npm ci - working-directory: lsp/ + cache-dependency-path: lsp/package-lock.json + - name: Set VS Code cache week + shell: bash + run: echo "VSCODE_CACHE_WEEK=$(date -u +%G-%V)" >> "$GITHUB_ENV" + - name: Restore VS Code test binary cache + uses: actions/cache@v4 + with: + path: | + ~/.vscode-test + .vscode-test + lsp/.vscode-test + key: ${{ runner.os }}-${{ matrix.platform }}-${{ matrix.arch }}-vscode-test-${{ env.VSCODE_CACHE_WEEK }}-${{ hashFiles('lsp/package-lock.json', 'lsp/.vscode-test.mjs') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.platform }}-${{ matrix.arch }}-vscode-test-${{ env.VSCODE_CACHE_WEEK }}- + ${{ runner.os }}-${{ matrix.platform }}-${{ matrix.arch }}-vscode-test- + - name: Install extension npm dependencies + uses: ./.github/actions/retry-command + with: + working-directory: lsp/ + command: npm ci - run: npm run compile working-directory: lsp/ - - run: xvfb-run -a npm run test - working-directory: lsp/ + - name: Run extension tests on Linux + uses: ./.github/actions/retry-command if: runner.os == 'Linux' - - run: npm run test - working-directory: lsp/ + with: + working-directory: lsp/ + command: xvfb-run -a npm run test + - name: Run extension tests + uses: ./.github/actions/retry-command if: runner.os != 'Linux' + with: + working-directory: lsp/ + command: npm run test diff --git a/pyrefly/rust-toolchain b/pyrefly/rust-toolchain new file mode 100644 index 0000000000..292fe499e3 --- /dev/null +++ b/pyrefly/rust-toolchain @@ -0,0 +1,2 @@ +[toolchain] +channel = "stable"