From c4ff5df5d56b6f2b824f0a13c927644f567123bd Mon Sep 17 00:00:00 2001 From: Nick Carton Date: Mon, 18 Nov 2024 09:48:35 +0100 Subject: [PATCH 1/5] Add concurrency flag, add more targets --- .github/workflows/rust-build.yml | 46 ++++++++++++++++++------ .github/workflows/rust-publish.yml | 56 +++++++++++++++++++++--------- src/main.rs | 39 +++------------------ src/options/opts.rs | 7 ++++ 4 files changed, 86 insertions(+), 62 deletions(-) diff --git a/.github/workflows/rust-build.yml b/.github/workflows/rust-build.yml index 3cca5f2..b0cf6f2 100644 --- a/.github/workflows/rust-build.yml +++ b/.github/workflows/rust-build.yml @@ -1,31 +1,50 @@ name: Build on: - push: + pull_request: branches: - "*" jobs: build: - name: Publish for ${{ matrix.os }} + name: Build for ${{ matrix.target }} runs-on: ${{ matrix.os }} strategy: matrix: - name: [linux, windows, macos] - include: - - name: linux + # Tier 1 Platforms + - target: x86_64-unknown-linux-gnu os: ubuntu-latest artifact_name: bping - asset_name: bping-linux - - name: windows + - target: x86_64-pc-windows-msvc os: windows-latest artifact_name: bping.exe - asset_name: bping-windows - - name: macos + - target: x86_64-apple-darwin os: macos-latest artifact_name: bping - asset_name: bping-macos + - target: i686-pc-windows-msvc + os: windows-latest + artifact_name: bping.exe + - target: i686-unknown-linux-gnu + os: ubuntu-latest + artifact_name: bping + + # Tier 2 Platforms + - target: aarch64-unknown-linux-gnu + os: ubuntu-latest + artifact_name: bping + - target: aarch64-apple-darwin + os: macos-latest + artifact_name: bping + - target: arm-unknown-linux-gnueabi + os: ubuntu-latest + artifact_name: bping + - target: armv7-unknown-linux-gnueabihf + os: ubuntu-latest + artifact_name: bping + - target: x86_64-unknown-linux-musl + os: ubuntu-latest + artifact_name: bping steps: - uses: actions/checkout@v1 @@ -34,6 +53,11 @@ jobs: with: profile: minimal toolchain: stable + target: ${{ matrix.target }} + override: true - name: Build - run: cargo build --release + uses: actions-rs/cargo@v1 + with: + command: build + args: --release --target ${{ matrix.target }} diff --git a/.github/workflows/rust-publish.yml b/.github/workflows/rust-publish.yml index 20dd56f..6331e13 100644 --- a/.github/workflows/rust-publish.yml +++ b/.github/workflows/rust-publish.yml @@ -7,25 +7,44 @@ on: jobs: publish: - name: Publish for ${{ matrix.os }} + name: Publish for ${{ matrix.target }} runs-on: ${{ matrix.os }} strategy: matrix: - name: [linux, windows, macos] - include: - - name: linux + # Tier 1 Platforms + - target: x86_64-unknown-linux-gnu os: ubuntu-latest - artifact_name: bping - asset_name: bping-linux - - name: windows + asset_name: bping-x86_64-linux-gnu + - target: x86_64-pc-windows-msvc + os: windows-latest + asset_name: bping-x86_64-windows-msvc.exe + - target: x86_64-apple-darwin + os: macos-latest + asset_name: bping-x86_64-darwin + - target: i686-pc-windows-msvc os: windows-latest - artifact_name: bping.exe - asset_name: bping-windows - - name: macos + asset_name: bping-i686-windows-msvc.exe + - target: i686-unknown-linux-gnu + os: ubuntu-latest + asset_name: bping-i686-linux-gnu + + # Tier 2 Platforms + - target: aarch64-unknown-linux-gnu + os: ubuntu-latest + asset_name: bping-aarch64-linux-gnu + - target: aarch64-apple-darwin os: macos-latest - artifact_name: bping - asset_name: bping-macos + asset_name: bping-aarch64-darwin + - target: arm-unknown-linux-gnueabi + os: ubuntu-latest + asset_name: bping-arm-linux-gnueabi + - target: armv7-unknown-linux-gnueabihf + os: ubuntu-latest + asset_name: bping-armv7-linux-gnueabihf + - target: x86_64-unknown-linux-musl + os: ubuntu-latest + asset_name: bping-x86_64-linux-musl steps: - uses: actions/checkout@v1 @@ -34,18 +53,23 @@ jobs: with: profile: minimal toolchain: stable + target: ${{ matrix.target }} + override: true - name: Build - run: cargo build --release + uses: actions-rs/cargo@v1 + with: + command: build + args: --release --target ${{ matrix.target }} - - name: Strip - run: strip target/release/${{ matrix.artifact_name }} + - name: Strip binary if: runner.os != 'Windows' + run: strip target/${{ matrix.target }}/release/bping - name: Upload binaries to release uses: svenstaro/upload-release-action@v1-release with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: target/release/${{ matrix.artifact_name }} + file: target/${{ matrix.target }}/release/bping${{ runner.os == 'Windows' && '.exe' || '' }} asset_name: ${{ matrix.asset_name }} tag: ${{ github.ref }} diff --git a/src/main.rs b/src/main.rs index 4278e31..8873f57 100644 --- a/src/main.rs +++ b/src/main.rs @@ -72,7 +72,10 @@ async fn main() -> eyre::Result<()> { pb.set_style(spinner_style); for region in &APP_CONFIG.regions { - for chunk in (0..APP_CONFIG.attempts).collect::>().chunks(100) { + for chunk in (0..APP_CONFIG.attempts) + .collect::>() + .chunks(APP_CONFIG.concurrency) + { let mut chunk_set = JoinSet::new(); for _ in chunk { @@ -137,37 +140,3 @@ async fn main() -> eyre::Result<()> { pb.finish(); Ok(()) } - -// async fn perform_job( -// req: &models::CreateJobAPIRequest, -// token: &str, -// ) -> Result { -// let resp = api::post_job(req, token).await?; -// let job_id = resp.id; -// let res = api::get_job_results(&job_id, token).await?; -// Ok(res) -// } -// -// async fn check_node_availability(parsed_regions: Vec) -> Result<(), BpingErrors> { -// if let Ok(available_nodes) = api::get_available_nodes().await { -// for region in parsed_regions { -// if custom_validators::is_continent(®ion) { -// continue; -// } -// -// if let Some(country_code) = custom_validators::get_emoji_safe_region_code(®ion) { -// if let None = available_nodes -// .results -// .iter() -// .find(|x| x.countrycode == country_code) -// { -// return Err(BpingErrors::JobErrors( -// crate::api::JobErrors::UnableToFindNodes, -// )); -// } -// } -// } -// } -// -// Ok(()) -// } diff --git a/src/options/opts.rs b/src/options/opts.rs index a37b249..376e32f 100644 --- a/src/options/opts.rs +++ b/src/options/opts.rs @@ -11,6 +11,7 @@ pub struct Opts { pub count: usize, pub attempts: usize, pub api_key: String, + pub concurrency: usize, } impl Opts { @@ -47,12 +48,18 @@ impl Opts { .env("BITPING_API_KEY") .argument("api_key"); + let concurrency = bpaf::long("concurrency") + .help("Specifies how many concurrent requests to send at once. Defaults to 100.") + .argument::("concurrency") + .fallback(100); + bpaf::construct!(Opts { regions, count, attempts, api_key, endpoint, + concurrency }) .to_options() .descr("A command line utility to ping a website from anywhere in the world!") From f56731512de4b854645ae43476f9539f7d4e4a14 Mon Sep 17 00:00:00 2001 From: Nick Carton Date: Mon, 18 Nov 2024 09:54:54 +0100 Subject: [PATCH 2/5] cached rust builds, remove openssl dependency --- .github/workflows/rust-build.yml | 49 ++++++++++++++++++++++++++++-- .github/workflows/rust-publish.yml | 47 ++++++++++++++++++++++++++-- Cargo.toml | 7 +++-- 3 files changed, 96 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rust-build.yml b/.github/workflows/rust-build.yml index b0cf6f2..2b8be18 100644 --- a/.github/workflows/rust-build.yml +++ b/.github/workflows/rust-build.yml @@ -1,10 +1,16 @@ name: Build on: - pull_request: + push: branches: - "*" +env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: true + SCCACHE_CACHE_SIZE: 2G + SCCACHE_DIR: /home/runner/.cache/sccache + jobs: build: name: Build for ${{ matrix.target }} @@ -47,17 +53,54 @@ jobs: artifact_name: bping steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 + + - name: Install build dependencies + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y curl build-essential pkg-config libssl-dev - - uses: actions-rs/toolchain@v1 + - name: Install Rust + uses: actions-rs/toolchain@v1 with: profile: minimal toolchain: stable target: ${{ matrix.target }} override: true + - name: Configure sccache + run: | + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.6 + + - name: Rust Cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v1-rust" + shared-key: "${{ matrix.target }}-build" + cache-targets: "true" + cache-on-failure: "true" + cache-all-crates: "true" + save-if: ${{ github.ref == 'refs/heads/master' }} + workspaces: | + . -> target + - name: Build uses: actions-rs/cargo@v1 with: command: build args: --release --target ${{ matrix.target }} + + - name: Show sccache stats + run: sccache --show-stats + + # Optional: Upload artifacts if needed + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.target }}-binary + path: target/${{ matrix.target }}/release/${{ matrix.artifact_name }} diff --git a/.github/workflows/rust-publish.yml b/.github/workflows/rust-publish.yml index 6331e13..e80dc20 100644 --- a/.github/workflows/rust-publish.yml +++ b/.github/workflows/rust-publish.yml @@ -5,6 +5,12 @@ on: tags: - "*" +env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: true + SCCACHE_CACHE_SIZE: 2G + SCCACHE_DIR: /home/runner/.cache/sccache + jobs: publish: name: Publish for ${{ matrix.target }} @@ -47,21 +53,51 @@ jobs: asset_name: bping-x86_64-linux-musl steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 + + - name: Install build dependencies + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y curl build-essential pkg-config libssl-dev - - uses: actions-rs/toolchain@v1 + - name: Install Rust + uses: actions-rs/toolchain@v1 with: profile: minimal toolchain: stable target: ${{ matrix.target }} override: true + - name: Configure sccache + run: | + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.6 + + - name: Rust Cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v1-rust" + shared-key: "${{ matrix.target }}-release" + cache-targets: "true" + cache-on-failure: "true" + cache-all-crates: "true" + save-if: true # Always save cache for tagged releases + workspaces: | + . -> target + - name: Build uses: actions-rs/cargo@v1 with: command: build args: --release --target ${{ matrix.target }} + - name: Show sccache stats + run: sccache --show-stats + - name: Strip binary if: runner.os != 'Windows' run: strip target/${{ matrix.target }}/release/bping @@ -73,3 +109,10 @@ jobs: file: target/${{ matrix.target }}/release/bping${{ runner.os == 'Windows' && '.exe' || '' }} asset_name: ${{ matrix.asset_name }} tag: ${{ github.ref }} + + # Optional: Upload artifacts for debugging or additional processing + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.target }}-release-binary + path: target/${{ matrix.target }}/release/bping${{ runner.os == 'Windows' && '.exe' || '' }} diff --git a/Cargo.toml b/Cargo.toml index 3fc1678..f9a73c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,10 @@ opt-level = "z" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -reqwest = { version = "0.12.5", features = ["json"] } +reqwest = { version = "0.12", features = [ + "json", + "rustls-tls", +], default-features = false } spinners = "4.1.1" tokio = { version = "1.38.0", features = ["full"] } dirs = "5.0.1" @@ -24,7 +27,7 @@ colorful = "0.3.2" indicatif = "0.17.8" chrono = { version = "0.4", features = ["serde"] } celes = "2.4.0" -thiserror = "1.0.61" +thiserror = "2.0" async-stream = "0.3.5" console = "0.15.8" keshvar = { version = "0.5.0", features = ["serde", "emojis"] } From 34939a3e97edd2970195cce3385c931e9b79376c Mon Sep 17 00:00:00 2001 From: Nick Carton Date: Mon, 18 Nov 2024 09:59:15 +0100 Subject: [PATCH 3/5] build --- .github/workflows/rust-build.yml | 9 ++++++++- .github/workflows/rust-publish.yml | 18 ++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust-build.yml b/.github/workflows/rust-build.yml index 2b8be18..166bc9a 100644 --- a/.github/workflows/rust-build.yml +++ b/.github/workflows/rust-build.yml @@ -34,23 +34,28 @@ jobs: - target: i686-unknown-linux-gnu os: ubuntu-latest artifact_name: bping + gcc_package: gcc-multilib # Tier 2 Platforms - target: aarch64-unknown-linux-gnu os: ubuntu-latest artifact_name: bping + gcc_package: gcc-aarch64-linux-gnu - target: aarch64-apple-darwin os: macos-latest artifact_name: bping - target: arm-unknown-linux-gnueabi os: ubuntu-latest artifact_name: bping + gcc_package: gcc-arm-linux-gnueabi - target: armv7-unknown-linux-gnueabihf os: ubuntu-latest artifact_name: bping + gcc_package: gcc-arm-linux-gnueabihf - target: x86_64-unknown-linux-musl os: ubuntu-latest artifact_name: bping + gcc_package: musl-tools steps: - uses: actions/checkout@v4 @@ -60,6 +65,9 @@ jobs: run: | sudo apt-get update sudo apt-get install -y curl build-essential pkg-config libssl-dev + if [ ! -z "${{ matrix.gcc_package }}" ]; then + sudo apt-get install -y ${{ matrix.gcc_package }} + fi - name: Install Rust uses: actions-rs/toolchain@v1 @@ -98,7 +106,6 @@ jobs: - name: Show sccache stats run: sccache --show-stats - # Optional: Upload artifacts if needed - name: Upload artifacts uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/rust-publish.yml b/.github/workflows/rust-publish.yml index e80dc20..a39b5c4 100644 --- a/.github/workflows/rust-publish.yml +++ b/.github/workflows/rust-publish.yml @@ -34,23 +34,28 @@ jobs: - target: i686-unknown-linux-gnu os: ubuntu-latest asset_name: bping-i686-linux-gnu + gcc_package: gcc-multilib # Tier 2 Platforms - target: aarch64-unknown-linux-gnu os: ubuntu-latest asset_name: bping-aarch64-linux-gnu + gcc_package: gcc-aarch64-linux-gnu - target: aarch64-apple-darwin os: macos-latest asset_name: bping-aarch64-darwin - target: arm-unknown-linux-gnueabi os: ubuntu-latest asset_name: bping-arm-linux-gnueabi + gcc_package: gcc-arm-linux-gnueabi - target: armv7-unknown-linux-gnueabihf os: ubuntu-latest asset_name: bping-armv7-linux-gnueabihf + gcc_package: gcc-arm-linux-gnueabihf - target: x86_64-unknown-linux-musl os: ubuntu-latest asset_name: bping-x86_64-linux-musl + gcc_package: musl-tools steps: - uses: actions/checkout@v4 @@ -60,6 +65,9 @@ jobs: run: | sudo apt-get update sudo apt-get install -y curl build-essential pkg-config libssl-dev + if [ ! -z "${{ matrix.gcc_package }}" ]; then + sudo apt-get install -y ${{ matrix.gcc_package }} + fi - name: Install Rust uses: actions-rs/toolchain@v1 @@ -100,7 +108,14 @@ jobs: - name: Strip binary if: runner.os != 'Windows' - run: strip target/${{ matrix.target }}/release/bping + run: | + if [ "${{ matrix.target }}" != "aarch64-apple-darwin" ]; then + if [ -n "${{ matrix.gcc_package }}" ]; then + ${{ matrix.target }}-strip target/${{ matrix.target }}/release/bping + else + strip target/${{ matrix.target }}/release/bping + fi + fi - name: Upload binaries to release uses: svenstaro/upload-release-action@v1-release @@ -110,7 +125,6 @@ jobs: asset_name: ${{ matrix.asset_name }} tag: ${{ github.ref }} - # Optional: Upload artifacts for debugging or additional processing - name: Upload artifacts uses: actions/upload-artifact@v3 with: From ee8ca557d6efb79eae7005725ea07962b49580f5 Mon Sep 17 00:00:00 2001 From: Nick Carton Date: Mon, 18 Nov 2024 10:05:18 +0100 Subject: [PATCH 4/5] thing --- .github/workflows/rust-build.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/rust-build.yml b/.github/workflows/rust-build.yml index 166bc9a..499dc53 100644 --- a/.github/workflows/rust-build.yml +++ b/.github/workflows/rust-build.yml @@ -41,6 +41,7 @@ jobs: os: ubuntu-latest artifact_name: bping gcc_package: gcc-aarch64-linux-gnu + linker: aarch64-linux-gnu-gcc - target: aarch64-apple-darwin os: macos-latest artifact_name: bping @@ -48,10 +49,12 @@ jobs: os: ubuntu-latest artifact_name: bping gcc_package: gcc-arm-linux-gnueabi + linker: arm-linux-gnueabi-gcc - target: armv7-unknown-linux-gnueabihf os: ubuntu-latest artifact_name: bping gcc_package: gcc-arm-linux-gnueabihf + linker: arm-linux-gnueabihf-gcc - target: x86_64-unknown-linux-musl os: ubuntu-latest artifact_name: bping @@ -69,6 +72,13 @@ jobs: sudo apt-get install -y ${{ matrix.gcc_package }} fi + - name: Configure Cargo for cross-compilation + if: matrix.linker != '' + run: | + mkdir -p ~/.cargo + echo "[target.${{ matrix.target }}]" >> ~/.cargo/config.toml + echo "linker = \"${{ matrix.linker }}\"" >> ~/.cargo/config.toml + - name: Install Rust uses: actions-rs/toolchain@v1 with: From 82685d634a77af6473dd946b2915055045618327 Mon Sep 17 00:00:00 2001 From: Nick Carton Date: Mon, 18 Nov 2024 10:23:01 +0100 Subject: [PATCH 5/5] Add publish step with all Tier 1 targets, fix concurrency flag --- .github/workflows/rust-publish.yml | 12 +++++++++++- src/options/opts.rs | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust-publish.yml b/.github/workflows/rust-publish.yml index a39b5c4..d33119c 100644 --- a/.github/workflows/rust-publish.yml +++ b/.github/workflows/rust-publish.yml @@ -41,6 +41,7 @@ jobs: os: ubuntu-latest asset_name: bping-aarch64-linux-gnu gcc_package: gcc-aarch64-linux-gnu + linker: aarch64-linux-gnu-gcc - target: aarch64-apple-darwin os: macos-latest asset_name: bping-aarch64-darwin @@ -48,10 +49,12 @@ jobs: os: ubuntu-latest asset_name: bping-arm-linux-gnueabi gcc_package: gcc-arm-linux-gnueabi + linker: arm-linux-gnueabi-gcc - target: armv7-unknown-linux-gnueabihf os: ubuntu-latest asset_name: bping-armv7-linux-gnueabihf gcc_package: gcc-arm-linux-gnueabihf + linker: arm-linux-gnueabihf-gcc - target: x86_64-unknown-linux-musl os: ubuntu-latest asset_name: bping-x86_64-linux-musl @@ -69,6 +72,13 @@ jobs: sudo apt-get install -y ${{ matrix.gcc_package }} fi + - name: Configure Cargo for cross-compilation + if: matrix.linker != '' + run: | + mkdir -p ~/.cargo + echo "[target.${{ matrix.target }}]" >> ~/.cargo/config.toml + echo "linker = \"${{ matrix.linker }}\"" >> ~/.cargo/config.toml + - name: Install Rust uses: actions-rs/toolchain@v1 with: @@ -110,7 +120,7 @@ jobs: if: runner.os != 'Windows' run: | if [ "${{ matrix.target }}" != "aarch64-apple-darwin" ]; then - if [ -n "${{ matrix.gcc_package }}" ]; then + if [ -n "${{ matrix.linker }}" ]; then ${{ matrix.target }}-strip target/${{ matrix.target }}/release/bping else strip target/${{ matrix.target }}/release/bping diff --git a/src/options/opts.rs b/src/options/opts.rs index 376e32f..6c8c767 100644 --- a/src/options/opts.rs +++ b/src/options/opts.rs @@ -57,9 +57,9 @@ impl Opts { regions, count, attempts, + concurrency, api_key, endpoint, - concurrency }) .to_options() .descr("A command line utility to ping a website from anywhere in the world!")