From 700eea29b55a198eea3bca7507ba1aa85af65c63 Mon Sep 17 00:00:00 2001 From: Christian Treffs Date: Mon, 14 Oct 2024 09:52:22 +0200 Subject: [PATCH 01/12] Add Benchmark CI step --- .github/workflows/benchmark.yml | 38 +++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..56e986c2 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,38 @@ +name: Benchmark + +on: + workflow_dispatch: + + +# Make sure that we only benchmark the last version per branch, max one at the time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: macos-latest + env: + working_directory: ./Benchmarks/ + strategy: + matrix: + xcode: ["15.4"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies + run: brew install jemalloc + + - name: Run benchmark + run: swift package --allow-writing-to-package-directory benchmark --format jmh --path . + + - name: Upload artifacts + uses: actions/upload-artifact@v4.4.3 + with: + name: benchmarks-${{ matrix.xcode }}-${{ github.run_id }} + path: | + *.jmh.json + if-no-files-found: error + include-hidden-files: true From 486d7cfc313427ea5767caa8225008e94244418a Mon Sep 17 00:00:00 2001 From: Christian Treffs Date: Mon, 14 Oct 2024 09:57:07 +0200 Subject: [PATCH 02/12] Run when labeled with benchmark --- .github/workflows/benchmark.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 56e986c2..c6b545b4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,6 +1,8 @@ name: Benchmark on: + pull_request: + types: [ labeled ] workflow_dispatch: @@ -11,6 +13,7 @@ concurrency: jobs: build: + if: ${{ github.event.label.name == 'benchmark' }} runs-on: macos-latest env: working_directory: ./Benchmarks/ From 74561c9147d7ea9743e3e6a85e58d1a15e469981 Mon Sep 17 00:00:00 2001 From: Christian Treffs Date: Mon, 14 Oct 2024 10:00:23 +0200 Subject: [PATCH 03/12] Set working dir for job --- .github/workflows/benchmark.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c6b545b4..a8eb3cce 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -15,8 +15,9 @@ jobs: build: if: ${{ github.event.label.name == 'benchmark' }} runs-on: macos-latest - env: - working_directory: ./Benchmarks/ + defaults: + run: + working-directory: ./Benchmarks strategy: matrix: xcode: ["15.4"] From 52abffd12e069faf63e9df2c5e8603ebb85f16b7 Mon Sep 17 00:00:00 2001 From: Christian Treffs Date: Mon, 14 Oct 2024 10:03:37 +0200 Subject: [PATCH 04/12] Update CI --- .github/workflows/benchmark.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a8eb3cce..6020941c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -30,7 +30,9 @@ jobs: run: brew install jemalloc - name: Run benchmark - run: swift package --allow-writing-to-package-directory benchmark --format jmh --path . + run: | + swift package --allow-writing-to-package-directory benchmark --no-progress --format jmh + ls -la - name: Upload artifacts uses: actions/upload-artifact@v4.4.3 From fa3e55db02b4960e94dc3f16911f467066066ba0 Mon Sep 17 00:00:00 2001 From: Christian Treffs Date: Mon, 14 Oct 2024 10:09:26 +0200 Subject: [PATCH 05/12] Find artifact --- .github/workflows/benchmark.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 6020941c..7a12e378 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -39,6 +39,8 @@ jobs: with: name: benchmarks-${{ matrix.xcode }}-${{ github.run_id }} path: | + Current_run.jmh.json + Benchmarks/Current_run.jmh.json *.jmh.json - if-no-files-found: error + if-no-files-found: warn include-hidden-files: true From 74d5f2f5ae79eeec90c7c84971e9923338d84bdd Mon Sep 17 00:00:00 2001 From: Christian Treffs Date: Mon, 14 Oct 2024 10:20:37 +0200 Subject: [PATCH 06/12] Add JMH action --- .github/workflows/benchmark.yml | 53 ++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7a12e378..ec93c4c2 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,10 +1,9 @@ name: Benchmark on: - pull_request: - types: [ labeled ] - workflow_dispatch: - + pull_request: + types: [labeled] + workflow_dispatch: # Make sure that we only benchmark the last version per branch, max one at the time concurrency: @@ -16,31 +15,37 @@ jobs: if: ${{ github.event.label.name == 'benchmark' }} runs-on: macos-latest defaults: - run: - working-directory: ./Benchmarks + run: + working-directory: ./Benchmarks strategy: matrix: xcode: ["15.4"] steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install dependencies - run: brew install jemalloc - - - name: Run benchmark - run: | - swift package --allow-writing-to-package-directory benchmark --no-progress --format jmh - ls -la - - - name: Upload artifacts - uses: actions/upload-artifact@v4.4.3 - with: - name: benchmarks-${{ matrix.xcode }}-${{ github.run_id }} - path: | + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies + run: brew install jemalloc + + - name: Run benchmark + run: | + swift package --allow-writing-to-package-directory benchmark --no-progress --format jmh + ls -la + + - name: Upload artifacts + uses: actions/upload-artifact@v4.4.3 + with: + name: benchmarks-${{ matrix.xcode }}-${{ github.run_id }} + path: | Current_run.jmh.json Benchmarks/Current_run.jmh.json *.jmh.json - if-no-files-found: warn - include-hidden-files: true + if-no-files-found: warn + include-hidden-files: true + + - name: JMH Benchmark Action + uses: kitlangton/jmh-benchmark-action@v0.1.1 + with: + jmh-output-path: Benchmarks/Current_run.jmh.json + github-token: ${{ secrets.GITHUB_TOKEN }} From 72a7126852600de2644205e5f70df6ad8f52becf Mon Sep 17 00:00:00 2001 From: Christian Treffs Date: Mon, 14 Oct 2024 10:24:08 +0200 Subject: [PATCH 07/12] Give write permissions --- .github/workflows/benchmark.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ec93c4c2..fb93577e 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -10,6 +10,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: + contents: write + pull-requests: write + jobs: build: if: ${{ github.event.label.name == 'benchmark' }} From d571e5ffd86c3c3eadc1f14f2f7287198df92865 Mon Sep 17 00:00:00 2001 From: Christian Treffs Date: Mon, 14 Oct 2024 10:30:30 +0200 Subject: [PATCH 08/12] Use main --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index fb93577e..d57f096c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -49,7 +49,7 @@ jobs: include-hidden-files: true - name: JMH Benchmark Action - uses: kitlangton/jmh-benchmark-action@v0.1.1 + uses: kitlangton/jmh-benchmark-action@main with: jmh-output-path: Benchmarks/Current_run.jmh.json github-token: ${{ secrets.GITHUB_TOKEN }} From aa5c56b9ca80cf2dccae2ce33ac620cd92b1d91b Mon Sep 17 00:00:00 2001 From: Christian Treffs Date: Mon, 14 Oct 2024 10:34:35 +0200 Subject: [PATCH 09/12] Try local path --- .github/workflows/benchmark.yml | 2 +- Benchmarks/Package.resolved | 77 +++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 Benchmarks/Package.resolved diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d57f096c..ac3f4fb4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -51,5 +51,5 @@ jobs: - name: JMH Benchmark Action uses: kitlangton/jmh-benchmark-action@main with: - jmh-output-path: Benchmarks/Current_run.jmh.json + jmh-output-path: ./Current_run.jmh.json github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/Benchmarks/Package.resolved b/Benchmarks/Package.resolved new file mode 100644 index 00000000..3f0277ea --- /dev/null +++ b/Benchmarks/Package.resolved @@ -0,0 +1,77 @@ +{ + "pins" : [ + { + "identity" : "hdrhistogram-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/HdrHistogram/hdrhistogram-swift", + "state" : { + "revision" : "a69fa24d7b70421870cafa86340ece900489e17e", + "version" : "0.1.2" + } + }, + { + "identity" : "package-benchmark", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ordo-one/package-benchmark", + "state" : { + "revision" : "f9dcf0b540c4973ab1cef7893fb46e34a4e74099", + "version" : "1.27.2" + } + }, + { + "identity" : "package-jemalloc", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ordo-one/package-jemalloc", + "state" : { + "revision" : "e8a5db026963f5bfeac842d9d3f2cc8cde323b49", + "version" : "1.0.0" + } + }, + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser", + "state" : { + "revision" : "41982a3656a71c768319979febd796c6fd111d5c", + "version" : "1.5.0" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics", + "state" : { + "revision" : "cd142fd2f64be2100422d658e7411e39489da985", + "version" : "1.2.0" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics", + "state" : { + "revision" : "0a5bc04095a675662cf24757cc0640aa2204253b", + "version" : "1.0.2" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system", + "state" : { + "revision" : "d2ba781702a1d8285419c15ee62fd734a9437ff5", + "version" : "1.3.2" + } + }, + { + "identity" : "texttable", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ordo-one/TextTable", + "state" : { + "revision" : "a27a07300cf4ae322e0079ca0a475c5583dd575f", + "version" : "0.0.2" + } + } + ], + "version" : 2 +} From 613cb761a18c8d2be1fc9a535c719b634e1ef9b1 Mon Sep 17 00:00:00 2001 From: Christian Treffs Date: Mon, 14 Oct 2024 10:42:29 +0200 Subject: [PATCH 10/12] Use different tool --- .github/workflows/benchmark.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ac3f4fb4..8085c6bd 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -48,8 +48,15 @@ jobs: if-no-files-found: warn include-hidden-files: true - - name: JMH Benchmark Action - uses: kitlangton/jmh-benchmark-action@main + # Run `github-action-benchmark` action + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1.20.3 with: - jmh-output-path: ./Current_run.jmh.json - github-token: ${{ secrets.GITHUB_TOKEN }} + # What benchmark tool the output.txt came from + tool: 'jmh' + # Where the output from the benchmark tool is stored + output-file-path: Current_run.jmh.json + # Where the previous data file is stored + # external-data-json-path: ./cache/benchmark-data.json + # Workflow will fail when an alert happens + # fail-on-alert: true From 15b82ed36b21151f5a82e7cdf4316e2caf8c2f91 Mon Sep 17 00:00:00 2001 From: Christian Treffs Date: Mon, 14 Oct 2024 10:45:11 +0200 Subject: [PATCH 11/12] Correct path --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 8085c6bd..aeaf7a28 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -55,7 +55,7 @@ jobs: # What benchmark tool the output.txt came from tool: 'jmh' # Where the output from the benchmark tool is stored - output-file-path: Current_run.jmh.json + output-file-path: Benchmarks/Current_run.jmh.json # Where the previous data file is stored # external-data-json-path: ./cache/benchmark-data.json # Workflow will fail when an alert happens From fce014d5ee87966173c49f700b8a0a2fa8618bbd Mon Sep 17 00:00:00 2001 From: Christian Treffs Date: Mon, 14 Oct 2024 10:52:30 +0200 Subject: [PATCH 12/12] Try different config --- .github/workflows/benchmark.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index aeaf7a28..a1b0df28 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -53,10 +53,21 @@ jobs: uses: benchmark-action/github-action-benchmark@v1.20.3 with: # What benchmark tool the output.txt came from - tool: 'jmh' + tool: "jmh" # Where the output from the benchmark tool is stored output-file-path: Benchmarks/Current_run.jmh.json # Where the previous data file is stored # external-data-json-path: ./cache/benchmark-data.json # Workflow will fail when an alert happens # fail-on-alert: true + fail-on-alert: true + # GitHub API token to make a commit comment + github-token: ${{ secrets.GITHUB_TOKEN }} + # Enable alert commit comment + # comment-on-alert: true + # Enable Job Summary for PRs + summary-always: true + # Mention @rhysd in the commit comment + # alert-comment-cc-users: "@rhysd" + save-data-file: false + skip-fetch-gh-pages: true