Improve benchmark structure and run targets

Author: art049

.github/workflows/codspeed.yml

@@ -19,21 +19,17 @@ jobs:
       - name: Install build tools and dependencies
         run: |
           sudo apt-get update
-          # Install necessary build tools
           sudo apt-get install -y cmake g++
-          # Install required Boost components
-          sudo apt-get install -y libboost-serialization-dev libboost-graph-dev
 
       - name: Build the benchmark target
         run: |
           mkdir build
           cd build
           cmake -DCODSPEED_MODE=instrumentation ..
-          make -j
+          cmake --build . -j
 
       - name: Run all benchmarks with CodSpeed
         uses: CodSpeedHQ/action@v3
         with:
-          run: build/boost_bench # Path to our executable
+          run: cd build && make run_all_benchmarks
           token: ${{ secrets.CODSPEED_TOKEN }}
-          # No filter - run all benchmarks

CMakeLists.txt

@@ -1,59 +1,71 @@
 cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
 project(boost-bench)
 
+# TODO: remove this once the CodSpeed Benchmark is fixed
+# Disable -Werror for CodSpeed Benchmark to avoid the unused private field error
+if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  add_compile_options(-Wno-error=unused-private-field)
+elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
+  add_compile_options(-Wno-error)
+endif()
+
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
+if(NOT DEFINED BOOST_VERSION)
+  set(BOOST_VERSION "1.87.0")
+endif()
 
-# Option for CodSpeed instrumentation mode
-option(CODSPEED_MODE "Build for CodSpeed instrumentation" OFF)
 
-# Include FetchContent for downloading Google Benchmark
 include(FetchContent)
+include(cmake/CPM.cmake)
 
-# Set benchmark download dependencies ON
 set(BENCHMARK_DOWNLOAD_DEPENDENCIES ON)
-
-# Download and configure CodSpeed C++ Benchmark
 FetchContent_Declare(
   google_benchmark
   GIT_REPOSITORY https://github.com/CodSpeedHQ/codspeed-cpp
   SOURCE_SUBDIR google_benchmark
   GIT_TAG v1.0.0
 )
+FetchContent_MakeAvailable(google_benchmark)
 
-# Configure benchmark options
-set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable benchmark testing" FORCE)
-if(CODSPEED_MODE)
-  set(BENCHMARK_CODSPEED_MODE "instrumentation" CACHE STRING "CodSpeed mode" FORCE)
-  message(STATUS "Building with CodSpeed instrumentation enabled")
-endif()
+CPMAddPackage(
+  NAME Boost
+  VERSION ${BOOST_VERSION} # Versions less than 1.85.0 may need patches for installation targets.
+  URL https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}-cmake.tar.xz
+  OPTIONS "BOOST_ENABLE_CMAKE ON" "BOOST_SKIP_INSTALL_RULES ON" # Set `OFF` for installation
+          "BUILD_SHARED_LIBS OFF" "BOOST_INCLUDE_LIBRARIES container\\\;asio\\\;format\\\;any\\\;uuid\\\;spirit\\\;serialization\\\;graph"
+          "CMAKE_BUILD_TYPE RelWithDebInfo"
+)
+set(BOOST_LIBRARIES Boost::container Boost::asio Boost::format Boost::any Boost::uuid Boost::spirit Boost::serialization Boost::graph)
 
-# Disable -Werror for CodSpeed Benchmark to avoid the unused private field error
-if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-  add_compile_options(-Wno-error=unused-private-field)
-elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
-  add_compile_options(-Wno-error)
-endif()
 
-FetchContent_MakeAvailable(google_benchmark)
+# Create individual benchmark executables
+set(BENCHMARKS string_bench container_bench utility_bench optional_bench spirit_bench multiindex_bench graph_bench serialization_bench)
 
-# Add Boost
-find_package(Boost REQUIRED 
-  COMPONENTS 
-    serialization 
-    graph
-)
-include_directories(${Boost_INCLUDE_DIRS})
+foreach(benchmark IN LISTS BENCHMARKS)
+  add_executable(${benchmark} src/${benchmark}.cpp)
+  target_link_libraries(${benchmark}
+    benchmark::benchmark
+    ${BOOST_LIBRARIES}
+  )
+endforeach()
 
-# Add benchmark executable
-add_executable(boost_bench src/boost_bench.cpp)
-target_link_libraries(boost_bench 
-  benchmark::benchmark 
-  ${Boost_LIBRARIES}
+add_custom_target(run_all_benchmarks
+  COMMAND ${CMAKE_COMMAND} -E echo "Running all benchmarks..."
 )
-target_include_directories(boost_bench PRIVATE include)
+
+# Make each benchmark depend on the run_all_benchmarks target
+foreach(benchmark IN LISTS BENCHMARKS)
+  add_custom_command(
+    TARGET run_all_benchmarks
+    POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E echo "Running ${benchmark}..."
+    COMMAND $<TARGET_FILE:${benchmark}>
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  )
+endforeach()
 
 # Print configuration info
 message(STATUS "C++ Standard: ${CMAKE_CXX_STANDARD}")
-message(STATUS "Boost version: ${Boost_VERSION}")
-message(STATUS "CodSpeed mode: ${CODSPEED_MODE}")
+message(STATUS "Boost version: ${BOOST_VERSION}")
+message(STATUS "CodSpeed mode: ${CODSPEED_MODE}")

README.md

@@ -1,15 +1,12 @@
 # Boost Benchmarks
 
-This project contains benchmarks for the Boost C++ library using Google Benchmark. It tests various components of the Boost library with different input sizes and configurations to measure performance characteristics.
+A benchmarking suite for comparing performance of various Boost C++ library components using Google Benchmark.
 
 ## Requirements
 
-- CMake (3.14 or higher)
-- C++ compiler with C++17 support
-- Boost library with the following components:
-  - serialization
-  - graph
-- Internet connection (for downloading Google Benchmark)
+- CMake 3.14+
+- C++17 compiler
+- Boost 1.87.0+ (with serialization and graph components)
 
 ## Building
 
@@ -19,7 +16,11 @@ mkdir -p build
 cd build
 
 # Configure and build
-cmake ..
+# For regular timing-based benchmarks:
+cmake -DCODSPEED_MODE=walltime ..
+# For CodSpeed instrumentation:
+cmake -DCODSPEED_MODE=instrumentation ..
+
 make
 ```
 
@@ -28,101 +29,56 @@ make
 From the build directory:
 
 ```bash
-# Run all benchmarks
-./boost_bench
-
-# Run specific benchmarks (using regex filter)
-./boost_bench --benchmark_filter="src/string_bench.h::BM_BoostStringSplit"
-
-# List all available benchmarks
-./boost_bench --benchmark_list_tests
+# Run individual component benchmarks
+./string_bench
+./container_bench
+./utility_bench
+./optional_bench
+./spirit_bench
+./multiindex_bench
+./graph_bench
+./serialization_bench
+
+# Run all benchmarks with a single command
+cmake --build . --target run_all_benchmarks
+
+# Pass arguments to all benchmarks
+cmake --build . --target run_all_benchmarks -- --benchmark_filter=BM_Boost
+
+# Run with regex filter
+./string_bench --benchmark_filter="BM_BoostLexicalCast"
+
+# List available benchmarks
+./string_bench --benchmark_list_tests
 ```
 
-### CodSpeed Integration
+## Benchmark Categories
 
-This project is set up to run benchmarks in CI using [CodSpeed](https://codspeed.io/). When a pull request is opened or code is pushed to the main branch, GitHub Actions will automatically build and run the benchmarks, reporting the results to CodSpeed.
+- **string_bench**: String operations, lexical_cast, regex, format
+- **container_bench**: Container performance comparisons
+- **utility_bench**: Type-safe any, algorithms, UUID generation
+- **optional_bench**: Boost vs std::optional
+- **spirit_bench**: Parsing operations (CSV, JSON, expressions)
+- **multiindex_bench**: Multi-index container operations
+- **graph_bench**: Graph algorithms (Dijkstra, A*, BFS, DFS)
+- **serialization_bench**: Serialization performance (text, binary, XML)
 
-#### CI Workflow
+## Custom Boost Version
 
-The GitHub Actions workflow:
-1. Installs necessary tools and dependencies:
-   - CMake and GCC for building
-   - Required Boost components (serialization, graph)
-2. Configures the project with CodSpeed instrumentation
-3. Builds the project
-4. Runs all benchmarks and reports results to CodSpeed
+Specify a custom Boost version:
 
-#### Local CodSpeed Testing
+```bash
+cmake -DBOOST_VERSION=1.82.0 -DCODSPEED_MODE=walltime ..
+```
 
-To enable CodSpeed instrumentation locally:
+## CodSpeed Integration
+
+This project integrates with [CodSpeed](https://codspeed.io/) for CI performance tracking. For local testing, use:
 
 ```bash
-mkdir -p build && cd build
 cmake -DCODSPEED_MODE=instrumentation ..
 make
-
-# Run the benchmarks with CodSpeed instrumentation
-./boost_bench
+./string_bench  # Will show notice about running outside CI
 ```
 
-You'll see a message like this when running with CodSpeed instrumentation locally:
-```
-NOTICE: codspeed is enabled, but no performance measurement will be made since it's running in an unknown environment.
-```
-
-This is normal as CodSpeed metrics are only collected in CI environments.
-
-You'll need to set up a `CODSPEED_TOKEN` secret in your GitHub repository settings for the CodSpeed integration to work in CI.
-
-## Notes
-
-- CodSpeed's fork of Google Benchmark is automatically downloaded and configured via CMake's FetchContent
-- The benchmarks are parameterized to test with different input sizes (small, medium, large)
-- The project is organized into modular benchmark files for better maintainability
-- CodSpeed integration allows performance tracking in CI
-
-## Benchmark Categories
-
-### String and Text Processing
-- String splitting via boost::algorithm::split
-- Integer and floating-point conversion via boost::lexical_cast
-- Regular expressions via boost::regex
-- String formatting via boost::format
-
-### Containers
-- Container performance comparisons:
-  - boost::container::flat_map vs std::map
-  - boost::container::vector vs std::vector
-
-### Utility Libraries
-- Type-safe any type via boost::any
-- Algorithms: boost::algorithm::all_of
-- UUID generation with boost::uuid
-- Optional types: boost::optional vs std::optional
-
-### Multi-Index Containers
-- Insertion performance vs standard containers
-- Lookup by different indices (ID, email)
-- Range-based queries
-- Modification with reindexing
-
-### Spirit (Parsing)
-- CSV data parsing
-- JSON structure parsing
-- Calculator expression parsing
-
-### Graph Algorithms
-- Dijkstra's shortest path algorithm
-- A* search on grid graphs
-- Breadth-First Search (BFS)
-- Depth-First Search (DFS)
-
-### Serialization
-- Text archive performance
-- Binary archive performance
-- XML archive performance
-- Format comparison (text vs binary vs XML)
-
-## Adding new benchmarks
-
-Add new benchmark functions to the corresponding header file in the `src/` directory, or create a new header file for a new category of benchmarks.
+For CI integration, ensure the `CODSPEED_TOKEN` secret is set in your GitHub repository.

cmake/CPM.cmake

@@ -1,4 +1,9 @@
-set(CPM_DOWNLOAD_VERSION 0.38.7)
+# SPDX-License-Identifier: MIT
+#
+# SPDX-FileCopyrightText: Copyright (c) 2019-2023 Lars Melchior and contributors
+
+set(CPM_DOWNLOAD_VERSION 0.40.8)
+set(CPM_HASH_SUM "78ba32abdf798bc616bab7c73aac32a17bbd7b06ad9e26a6add69de8f3ae4791")
 
 if(CPM_SOURCE_CACHE)
   set(CPM_DOWNLOAD_LOCATION "${CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
@@ -11,22 +16,9 @@ endif()
 # Expand relative path. This is important if the provided path contains a tilde (~)
 get_filename_component(CPM_DOWNLOAD_LOCATION ${CPM_DOWNLOAD_LOCATION} ABSOLUTE)
 
-function(download_cpm)
-  message(STATUS "Downloading CPM.cmake to ${CPM_DOWNLOAD_LOCATION}")
-  file(DOWNLOAD
-       https://github.com/cpm-cmake/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake
-       ${CPM_DOWNLOAD_LOCATION}
-  )
-endfunction()
-
-if(NOT EXISTS ${CPM_DOWNLOAD_LOCATION})
-  download_cpm()
-else()
-  # resume download if it previously failed
-  file(READ ${CPM_DOWNLOAD_LOCATION} check_result LIMIT 5 OFFSET 0)
-  if("${check_result}" STREQUAL "")
-    download_cpm()
-  endif()
-endif()
+file(DOWNLOAD
+     https://github.com/cpm-cmake/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake
+     ${CPM_DOWNLOAD_LOCATION} EXPECTED_HASH SHA256=${CPM_HASH_SUM}
+)
 
-include(${CPM_DOWNLOAD_LOCATION})
+include(${CPM_DOWNLOAD_LOCATION})

src/boost_bench.cpp

@@ -1,6 +1,3 @@
-#ifndef CONTAINER_BENCH_H
-#define CONTAINER_BENCH_H
-
 #include <benchmark/benchmark.h>
 #include <boost/container/flat_map.hpp>
 #include <boost/container/vector.hpp>
@@ -103,4 +100,4 @@ BENCHMARK(BM_BoostVector)
   ->Arg(10000)    // Medium vector
   ->Arg(100000);  // Large vector
 
-#endif // CONTAINER_BENCH_H
+BENCHMARK_MAIN();

src/container_bench.h src/container_bench.cppsrc/container_bench.h renamed to src/container_bench.cpp

@@ -1,6 +1,3 @@
-#ifndef GRAPH_BENCH_H
-#define GRAPH_BENCH_H
-
 #include <benchmark/benchmark.h>
 #include <boost/graph/adjacency_list.hpp>
 #include <boost/graph/dijkstra_shortest_paths.hpp>
@@ -456,4 +453,4 @@ BENCHMARK(BM_BoostGraphDFS)
   ->Args({1000, 10})   // Medium graph (1000 vertices, ~10 edges per vertex)
   ->Args({5000, 20});  // Large graph (5000 vertices, ~20 edges per vertex)
 
-#endif // GRAPH_BENCH_H
+BENCHMARK_MAIN();