diff --git a/.github/workflows/span_benchmark.yml b/.github/workflows/span_benchmark.yml new file mode 100644 index 00000000..aaccae78 --- /dev/null +++ b/.github/workflows/span_benchmark.yml @@ -0,0 +1,156 @@ +name: Span Benchmark + +on: + pull_request: + branches: [main] + +# Cancel stale runs when a new commit is pushed to the same PR. +concurrency: + group: span-bench-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + benchmark-linux: + name: Linux / ${{ format('{0}-cpp{1}', matrix.cxx, matrix.cppstd) }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + cxx: [g++-13, g++-14, clang++-16, clang++-17, clang++-18] + cppstd: [20, 23] + exclude: + - cxx: g++-13 + cppstd: 23 + - cxx: clang++-17 + cppstd: 23 + steps: + - uses: actions/checkout@v4 + + - name: Cache FetchContent dependencies + uses: actions/cache@v4 + with: + path: build/_deps + key: fetchcontent-linux-${{ matrix.cxx }}-${{ hashFiles('benchmark/CMakeLists.txt') }} + restore-keys: | + fetchcontent-linux-${{ matrix.cxx }}- + + - name: Configure + run: | + cmake -S . -B build \ + -DGSL_BENCHMARK=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ + -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }} \ + + - name: Build + run: cmake --build build --target span_bench -j$(nproc) + + - name: Run benchmark + run: | + ./build/span_bench \ + --benchmark_format=json \ + --benchmark_repetitions=10 \ + --benchmark_report_aggregates_only=true \ + --benchmark_out=results_${{ format('{0}-cpp{1}', matrix.cxx, matrix.cppstd) }}.json + + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: bench-${{ format('{0}-cpp{1}', matrix.cxx, matrix.cppstd) }} + path: results_${{ format('{0}-cpp{1}', matrix.cxx, matrix.cppstd) }}.json + retention-days: 7 + + benchmark-windows: + name: Windows / ${{ format('{0}-cpp{1}', matrix.toolset || 'MSVC', matrix.cppstd) }} + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + generator: [ 'Visual Studio 17 2022' ] + toolset: [ '', 'ClangCL' ] + cppstd: [ 20, 23 ] + steps: + - uses: actions/checkout@v4 + + - name: Cache FetchContent dependencies + uses: actions/cache@v4 + with: + path: build/_deps + key: fetchcontent-windows-${{ matrix.toolset }}-${{ hashFiles('benchmark/CMakeLists.txt') }} + restore-keys: | + fetchcontent-windows-${{ matrix.toolset }}- + + - name: Configure + shell: pwsh + run: | + $tsArg = if ("${{ matrix.toolset }}" -ne "") { @("-T", "${{ matrix.toolset }}") } else { @() } + + cmake -S . -B build ` + -DGSL_BENCHMARK=ON ` + -G "${{ matrix.generator }}" @tsArg ` + -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }} ` + + - name: Build + shell: pwsh + run: | + cmake --build build --target span_bench ` + --config Release -j $env:NUMBER_OF_PROCESSORS + + - name: Run benchmark + shell: pwsh + run: | + .\build\Release\span_bench.exe ` + --benchmark_format=json ` + --benchmark_repetitions=10 ` + --benchmark_report_aggregates_only=true ` + --benchmark_out=results_${{ format('{0}-cpp{1}', matrix.toolset || 'MSVC', matrix.cppstd) }}.json + + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: bench-${{ format('{0}-cpp{1}', matrix.toolset || 'MSVC', matrix.cppstd) }} + path: results_${{ format('{0}-cpp{1}', matrix.toolset || 'MSVC', matrix.cppstd) }}.json + retention-days: 7 + + benchmark-macos: + name: macOS / ${{ format('AppleClang-cpp{0}', matrix.cppstd) }} + runs-on: macos-latest + strategy: + fail-fast: false + matrix: + cppstd: [ 20, 23 ] + steps: + - uses: actions/checkout@v4 + + - name: Cache FetchContent dependencies + uses: actions/cache@v4 + with: + path: build/_deps + key: fetchcontent-macos-${{ matrix.cppstd }}-${{ hashFiles('benchmark/CMakeLists.txt') }} + restore-keys: | + fetchcontent-macos-${{ matrix.cppstd }}- + + - name: Configure + run: | + cmake -S . -B build \ + -DGSL_BENCHMARK=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_STANDARD=${{ matrix.cppstd }} \ + + - name: Build + run: cmake --build build --target span_bench -j$(sysctl -n hw.logicalcpu) + + - name: Run benchmark + run: | + ./build/span_bench \ + --benchmark_format=json \ + --benchmark_repetitions=10 \ + --benchmark_report_aggregates_only=true \ + --benchmark_out=results_${{ format('AppleClang-cpp{0}', matrix.cppstd) }}.json + + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: bench-${{ format('AppleClang-cpp{0}', matrix.cppstd) }} + path: results_${{ format('AppleClang-cpp{0}', matrix.cppstd) }}.json + retention-days: 7 \ No newline at end of file diff --git a/.github/workflows/span_benchmark_comment.yml b/.github/workflows/span_benchmark_comment.yml new file mode 100644 index 00000000..a7ab68c2 --- /dev/null +++ b/.github/workflows/span_benchmark_comment.yml @@ -0,0 +1,76 @@ +name: Span Benchmark Comment + +on: + workflow_run: + workflows: ["Span Benchmark"] + types: [completed] + +permissions: + pull-requests: write + +jobs: + comment: + name: Post PR comment + runs-on: ubuntu-latest + if: always() + + steps: + - uses: actions/checkout@v6 + + # Download all bench-* artifacts from the triggering workflow run. + - name: Download all result artifacts + uses: actions/download-artifact@v4 + with: + pattern: bench-* + merge-multiple: true + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Generate regression report + id: report + run: | + python3 benchmark/check_regression.py \ + --threshold 0.15 \ + --output comment.md \ + results_*.json + continue-on-error: true + + # Resolve the PR number from the triggering workflow run. + - name: Get PR number + id: pr + uses: actions/github-script@v7 + with: + script: | + const runs = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + }); + const pr = runs.data.find(p => + p.head.sha === context.payload.workflow_run.head_sha + ); + return pr ? pr.number : null; + + - name: Find existing bot comment + if: steps.pr.outputs.result != 'null' + uses: peter-evans/find-comment@v3 + id: find_comment + with: + issue-number: ${{ steps.pr.outputs.result }} + comment-author: github-actions[bot] + body-includes: "" + + - name: Create or update PR comment + if: steps.pr.outputs.result != 'null' + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.find_comment.outputs.comment-id }} + issue-number: ${{ steps.pr.outputs.result }} + body-path: comment.md + edit-mode: replace + + - name: Fail if regression detected + if: steps.report.outcome == 'failure' + run: | + echo "::error::Performance regression detected. See the PR comment for the full table." + exit 1 \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index b306dc1c..f0f9366b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,7 @@ string(COMPARE EQUAL ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR} PROJECT_IS_ option(GSL_INSTALL "Generate and install GSL target" ${PROJECT_IS_TOP_LEVEL}) option(GSL_TEST "Build and perform GSL tests" ${PROJECT_IS_TOP_LEVEL}) +option(GSL_BENCHMARK "Build span benchmarks" OFF) # The implementation generally assumes a platform that implements C++14 support target_compile_features(GSL INTERFACE "cxx_std_14") @@ -19,6 +20,10 @@ add_subdirectory(include) target_sources(GSL INTERFACE $) +if(GSL_BENCHMARK) + add_subdirectory(benchmark) +endif() + if (GSL_TEST) enable_testing() add_subdirectory(tests) diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt new file mode 100644 index 00000000..2bdf7da4 --- /dev/null +++ b/benchmark/CMakeLists.txt @@ -0,0 +1,57 @@ +cmake_minimum_required(VERSION 3.14...3.16) + +project(GSLBenchmarks LANGUAGES CXX) + +# ── C++ standard ─────────────────────────────────────────────────────────────── +# Minimum is 20 because std::span (required for this benchmark) is C++20 only. +# If a consumer passes GSL_CXX_STANDARD < 20 we clamp and warn. +set(GSL_CXX_STANDARD "20" CACHE STRING "Use c++ standard") + +if(GSL_CXX_STANDARD LESS 20) + message(WARNING + "GSL_CXX_STANDARD=${GSL_CXX_STANDARD} is too old for the span benchmark " + "(std::span requires C++20). Overriding to 20.") + set(GSL_CXX_STANDARD "20" CACHE STRING "Use c++ standard" FORCE) +endif() + +set(CMAKE_CXX_STANDARD ${GSL_CXX_STANDARD}) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# Makes Visual Studio organise targets into folders +set_property(GLOBAL PROPERTY USE_FOLDERS ON) + +include(FetchContent) + +# Suppress benchmark's own test suite +set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) +set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE) + +FetchContent_Declare( + googlebenchmark + GIT_REPOSITORY https://github.com/google/benchmark.git + GIT_TAG v1.9.5 + GIT_SHALLOW ON +) +FetchContent_MakeAvailable(googlebenchmark) + +add_executable(span_bench span_bench.cpp) + +target_link_libraries(span_bench PRIVATE + benchmark::benchmark + Microsoft.GSL::GSL +) + +set_target_properties(span_bench PROPERTIES FOLDER "benchmarks") + +# ── Optimisation flags ───────────────────────────────────────────────────────── +target_compile_options(span_bench PRIVATE + $<$: + -O3 + -march=native + > + $<$: + /O2 + /GL + > +) \ No newline at end of file diff --git a/benchmark/check_regression.py b/benchmark/check_regression.py new file mode 100644 index 00000000..f7a20123 --- /dev/null +++ b/benchmark/check_regression.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +""" +benchmark/check_regression.py + +Reads one or more Google Benchmark JSON result files (produced with +--benchmark_report_aggregates_only=true), pairs up gsl::span and std::span +benchmarks by name, computes the ratio gsl_ns / std_ns, and writes a +Markdown report to --output (default: stdout). + +Exits with code 1 if any ratio exceeds the threshold so CI fails cleanly. + +Usage: + python3 check_regression.py [--threshold 0.15] [--output report.md] results_*.json +""" + +import argparse +import json +import os +import sys +from pathlib import Path + + +# ─── helpers ────────────────────────────────────────────────────────────────── + +def load_json(path: str) -> dict: + with open(path) as f: + return json.load(f) + + +def label_from_filename(path: str) -> str: + """ + 'results_GCC-14-cpp20.json' → 'GCC-14-cpp20' + Falls back to the bare filename stem if the prefix isn't there. + """ + stem = Path(path).stem # e.g. 'results_GCC-14-cpp20' + if stem.startswith("results_"): + return stem[len("results_"):] + return stem + + +def parse_benchmarks(data: dict) -> dict[str, dict]: + """ + Returns a dict keyed by benchmark name. + For aggregated runs Google Benchmark emits multiple rows per benchmark + (_mean, _median, _stddev, _cv). We pull out mean and stddev. + + { "BM_IsSorted_StdSpan": {"mean": 124.3, "stddev": 2.1}, ... } + """ + result: dict[str, dict] = {} + for bm in data.get("benchmarks", []): + name: str = bm["name"] # e.g. "BM_IsSorted_StdSpan_mean" + agg = bm.get("aggregate_name") # "mean" | "stddev" | "median" | "cv" + + if agg not in ("mean", "stddev"): + continue + + # Strip the trailing _mean / _stddev to get the canonical name + suffix = f"_{agg}" + assert name.endswith(suffix) + base_name = name[:-len(suffix)] + + entry = result.setdefault(base_name, {"mean": None, "stddev": None}) + entry[agg] = bm.get("real_time") or bm.get("cpu_time", 0.0) + + return result + + +def pair_benchmarks(bm_dict: dict[str, dict]): + """ + Match GslSpan variants with their StdSpan counterparts. + + Actual names from span_bench.cpp: + BM_IsSortedStdSpan ↔ BM_IsSortedGslSpan + BM_IsSortedRangesStdSpan ↔ BM_IsSortedRangesGslSpan + BM_IsSortedCustomStdSpan ↔ BM_IsSortedCustomGslSpan + BM_MinElementAlgorithmStdSpan ↔ BM_MinElementAlgorithmGslSpan + BM_MinElementRangeForStdSpan ↔ BM_MinElementRangeForGslSpan + + All std variants contain 'StdSpan'; swapping it for 'GslSpan' gives + the paired name. Short display name strips 'BM_' prefix and 'StdSpan' + infix, leaving e.g. 'IsSortedRanges', 'MinElementAlgorithm'. + """ + pairs = [] + for name, vals in sorted(bm_dict.items()): + if "StdSpan" not in name: + continue + gsl_name = name.replace("StdSpan", "GslSpan") + if gsl_name not in bm_dict: + continue + + # Strip BM_ prefix and StdSpan infix for a clean display name. + # e.g. "BM_IsSortedRangesStdSpan" → "IsSortedRanges" + short = name.removeprefix("BM_").replace("StdSpan", "").rstrip("_") + + pairs.append({ + "short": short, + "std_name": name, + "gsl_name": gsl_name, + "std_mean": bm_dict[name]["mean"] or 0.0, + "std_stddev": bm_dict[name]["stddev"] or 0.0, + "gsl_mean": bm_dict[gsl_name]["mean"] or 0.0, + "gsl_stddev": bm_dict[gsl_name]["stddev"] or 0.0, + }) + return pairs + + +def ratio_and_status(gsl: float, std: float, threshold: float): + if std == 0: + return None, "⚠️ div/0" + r = gsl / std + if r > 1 + threshold: + return r, f"🔴 **{r:.2f}×** regression" + if r < 1 - threshold: + return r, f"🟢 **{r:.2f}×** faster" + return r, f"✅ {r:.2f}×" + + +def fmt(ns: float) -> str: + """Format nanoseconds nicely.""" + if ns >= 1_000_000: + return f"{ns/1_000_000:.1f} ms" + if ns >= 1_000: + return f"{ns/1_000:.1f} µs" + return f"{ns:.1f} ns" + + +def fmt_stddev(stddev: float, mean: float) -> str: + if mean == 0: + return "—" + pct = (stddev / mean) * 100 + return f"±{pct:.1f}%" + + +# ─── report builder ─────────────────────────────────────────────────────────── + +def build_report(json_paths: list[str], threshold: float) -> tuple[str, bool]: + """ + Returns (markdown_text, had_regression). + """ + lines = [] + had_regression = False + + lines.append("") + lines.append("## 📊 `gsl::span` vs `std::span` benchmark results") + lines.append("") + lines.append( + f"> Threshold: flag if `gsl_ns / std_ns > {1 + threshold:.2f}` " + f"(+{threshold*100:.0f}%) or `< {1 - threshold:.2f}` " + f"(-{threshold*100:.0f}%) \n" + f"> Each benchmark ran **10 repetitions**; times shown are the mean " + f"± stddev." + ) + lines.append("") + + found_any = False + + for path in sorted(json_paths): + if not os.path.exists(path): + lines.append(f"> ⚠️ Result file not found: `{path}`") + continue + + try: + data = load_json(path) + except Exception as e: + lines.append(f"> ⚠️ Could not parse `{path}`: {e}") + continue + + config_label = label_from_filename(path) + bm_dict = parse_benchmarks(data) + pairs = pair_benchmarks(bm_dict) + + if not pairs: + lines.append(f"### `{config_label}`") + lines.append("> ⚠️ No paired benchmarks found — check naming convention.") + lines.append("") + continue + + found_any = True + + # Extract context info from the JSON if present + ctx = data.get("context", {}) + # cpu_scaling_enabled=True means the OS was allowed to vary frequency + # — bad for stable benchmarks. Google Benchmark warns about this itself + # but we surface it in the report too. + cpu_scaling = ctx.get("cpu_scaling_enabled", None) + num_cpus = ctx.get("num_cpus", "?") + mhz = ctx.get("mhz_per_cpu", "?") + + lines.append(f"### `{config_label}`") + lines.append(f"CPUs: {num_cpus} @ {mhz} MHz") + lines.append("") + + if cpu_scaling is True: + lines.append("> ⚠️ CPU frequency scaling was **enabled** — results may be noisier than usual.") + + lines.append("") + lines.append( + "| Benchmark | std mean | std σ | gsl mean | gsl σ | ratio | status |" + ) + lines.append( + "|-----------|----------|-------|----------|-------|-------|--------|" + ) + + config_regression = False + for p in pairs: + ratio, status = ratio_and_status(p["gsl_mean"], p["std_mean"], threshold) + ratio_str = "—" if ratio is None else f"{ratio:.2f}×" + if "regression" in status: + had_regression = True + config_regression = True + + lines.append( + f"| `{p['short']}` " + f"| {fmt(p['std_mean'])} " + f"| {fmt_stddev(p['std_stddev'], p['std_mean'])} " + f"| {fmt(p['gsl_mean'])} " + f"| {fmt_stddev(p['gsl_stddev'], p['gsl_mean'])} " + f"| {ratio_str} " + f"| {status} |" + ) + + if config_regression: + lines.append("") + lines.append( + f"> 🔴 **Regression detected** in `{config_label}` — " + f"`gsl::span` is more than {threshold*100:.0f}% slower than " + f"`std::span` on one or more benchmarks." + ) + + lines.append("") + + if not found_any: + lines.append("> ❌ No benchmark results could be loaded.") + + return "\n".join(lines), had_regression + + +# ─── entry point ───────────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser( + description="Check gsl::span vs std::span benchmark regression." + ) + parser.add_argument( + "json_files", + nargs="+", + metavar="results.json", + help="Google Benchmark JSON output files (one per CI matrix config).", + ) + parser.add_argument( + "--threshold", + type=float, + default=0.15, + help="Fractional slowdown threshold before flagging a regression (default: 0.15 = 15%%).", + ) + parser.add_argument( + "--output", + default=None, + metavar="FILE", + help="Write Markdown report to FILE instead of stdout.", + ) + args = parser.parse_args() + + report, had_regression = build_report(args.json_files, args.threshold) + + if args.output: + Path(args.output).write_text(report, encoding="utf-8") + print(f"Report written to {args.output}") + else: + print(report) + + if had_regression: + print( + "\n[check_regression] ❌ Performance regression detected. " + "See the report above.", + file=sys.stderr, + ) + sys.exit(1) + + print("\n[check_regression] ✅ No regressions detected.", file=sys.stderr) + sys.exit(0) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/benchmark/span_bench.cpp b/benchmark/span_bench.cpp new file mode 100644 index 00000000..f8a92ad5 --- /dev/null +++ b/benchmark/span_bench.cpp @@ -0,0 +1,174 @@ +#include "gsl/span" +#include +#include +#include +#include +#include +#include + +static std::vector make_vector() +{ + std::vector v; + constexpr size_t vec_size = 1000; + v.reserve(vec_size); + for (int i = 0; i < vec_size; ++i) + v.push_back(i); + + return v; +} + +template +bool custom_is_sorted(ForwardIt first, ForwardIt last) +{ + if (first != last) + { + ForwardIt next = first; + while (++next != last) + { + if (*next < *first) + return false; + first = next; + } + } + return true; +} + +static void BM_IsSortedCustomStdSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + std::span sp = vec; + benchmark::DoNotOptimize(custom_is_sorted(sp.begin(), sp.end())); + } +} + +static void BM_IsSortedCustomGslSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + gsl::span sp = vec; + benchmark::DoNotOptimize(custom_is_sorted(sp.begin(), sp.end())); + } +} + +static void BM_IsSortedStdSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + std::span sp = vec; + benchmark::DoNotOptimize(std::is_sorted(sp.begin(), sp.end())); + } +} + +static void BM_IsSortedGslSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + gsl::span sp = vec; + benchmark::DoNotOptimize(std::is_sorted(sp.begin(), sp.end())); + } +} + +static void BM_IsSortedRangesStdSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + std::span sp = vec; + benchmark::DoNotOptimize(std::ranges::is_sorted(sp)); + } +} + + +static void BM_IsSortedRangesGslSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + gsl::span sp = vec; + benchmark::DoNotOptimize(std::ranges::is_sorted(sp)); + } +} + +BENCHMARK(BM_IsSortedStdSpan); +BENCHMARK(BM_IsSortedGslSpan); +BENCHMARK(BM_IsSortedRangesStdSpan); +BENCHMARK(BM_IsSortedRangesGslSpan); +BENCHMARK(BM_IsSortedCustomStdSpan); +BENCHMARK(BM_IsSortedCustomGslSpan); + +template +static int CustomMinElement(TSpan span) +{ + auto min = std::numeric_limits::max(); + for (int e : span) + { + if (e < min) + min = e; + } + return min; +} + +static void BM_MinElementAlgorithmStdSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + std::span sp = vec; + benchmark::DoNotOptimize(std::min_element(sp.begin(), sp.end())); + } +} + +static void BM_MinElementAlgorithmGslSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + gsl::span sp = vec; + benchmark::DoNotOptimize(std::min_element(sp.begin(), sp.end())); + } +} + +static void BM_MinElementRangeForStdSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + std::span sp = vec; + benchmark::DoNotOptimize(CustomMinElement(sp)); + } +} + + +static void BM_MinElementRangeForGslSpan(benchmark::State& state) +{ + auto vec = make_vector(); + + for (auto _ : state) + { + gsl::span sp = vec; + benchmark::DoNotOptimize(CustomMinElement(sp)); + } +} + + +BENCHMARK(BM_MinElementAlgorithmStdSpan); +BENCHMARK(BM_MinElementAlgorithmGslSpan); +BENCHMARK(BM_MinElementRangeForStdSpan); +BENCHMARK(BM_MinElementRangeForGslSpan); + +BENCHMARK_MAIN(); +