From 0814565321866291ebb2e46a532cb72164dc38f3 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Fri, 10 Jul 2026 12:32:06 +0200 Subject: [PATCH] feat(cli): add --shard / to split the suite across runners (#739) Run a deterministic round-robin subset of the test files so a large suite can be split across parallel CI machines; the union of all shards is the full suite with no overlap. Index is 1-based and validated (bad input exits non-zero). Sharding happens on the collected file list before execution, so it composes with --parallel and --random-order. Closes #739 --- CHANGELOG.md | 1 + docs/command-line.md | 27 +++++++++++ src/console_header.sh | 1 + src/env.sh | 17 +++++++ src/main.sh | 55 +++++++++++++++++++++ tests/acceptance/bashunit_shard_test.sh | 58 +++++++++++++++++++++++ tests/acceptance/fixtures/test_shard_a.sh | 6 +++ tests/acceptance/fixtures/test_shard_b.sh | 6 +++ tests/acceptance/fixtures/test_shard_c.sh | 6 +++ tests/acceptance/fixtures/test_shard_d.sh | 6 +++ 10 files changed, 183 insertions(+) create mode 100644 tests/acceptance/bashunit_shard_test.sh create mode 100644 tests/acceptance/fixtures/test_shard_a.sh create mode 100644 tests/acceptance/fixtures/test_shard_b.sh create mode 100644 tests/acceptance/fixtures/test_shard_c.sh create mode 100644 tests/acceptance/fixtures/test_shard_d.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c4b0b53..c342e35b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- `--shard /` to run a deterministic, round-robin subset of the test files so a suite can be split across parallel CI runners (union of all shards is the full suite, no overlap); validates input and composes with `--parallel` (#739) - `--random-order` flag with `--seed ` / `BASHUNIT_SEED` to randomize test file and function execution order (surfaces inter-test coupling); the seed is printed for replay and the shuffle is reproducible and works with `--parallel`. Disabled by default (#738) - `--retry ` flag and `BASHUNIT_RETRY` env var to re-run a failed test up to N extra times (flaky-test mitigation); passes if any attempt passes, annotates tests that only passed on retry, and works with `--parallel` and `--stop-on-failure`. Disabled by default (#737) - `--report-tap ` writes a TAP version 13 report to a file (complements the streaming `--output tap`) (#740) diff --git a/docs/command-line.md b/docs/command-line.md index 829a5a03..75a4268d 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -80,6 +80,7 @@ bashunit test tests/ --parallel --simple | `--retry ` | Re-run a failed test up to N extra times | | `--random-order` | Randomize test execution order | | `--seed ` | Seed for `--random-order` (reproducible shuffle) | +| `--shard /` | Run shard i of n (split suite across runners) | | `--show-skipped` | Show skipped tests summary at end | | `--show-incomplete` | Show incomplete tests summary at end | | `-vvv, --verbose` | Show execution details | @@ -520,6 +521,32 @@ bashunit test tests/ --random-order --seed 12345 It can also be set via the `BASHUNIT_SEED` environment variable (see [configuration](/configuration#random-order)). +### Shard + +> `bashunit test --shard /` + +Run a deterministic subset (shard) of the test files, so a large suite can be +split across parallel CI machines. `index` is 1-based (`1 <= index <= total`); +invalid input exits non-zero with an error. Files are assigned round-robin, so +the union of all shards is the full suite with no overlap. Composes with +`--parallel` (shard first on each runner, then parallelize the slice). + +::: code-group +```bash [Split across 4 runners] +bashunit test tests/ --shard 1/4 +bashunit test tests/ --shard 2/4 +bashunit test tests/ --shard 3/4 +bashunit test tests/ --shard 4/4 +``` +```yaml [GitHub Actions matrix] +strategy: + matrix: + shard: [1, 2, 3, 4] +steps: + - run: ./bashunit tests/ --shard ${{ matrix.shard }}/4 +``` +::: + ### No Progress > `bashunit test --no-progress` diff --git a/src/console_header.sh b/src/console_header.sh index 2985eb6d..c401838c 100644 --- a/src/console_header.sh +++ b/src/console_header.sh @@ -133,6 +133,7 @@ Options: --retry Re-run a failed test up to N extra times (0 = off) --random-order Randomize test execution order --seed Seed for --random-order (reproducible shuffle) + --shard / Run shard i of n (split the suite across runners) -vvv, --verbose Show execution details --debug [file] Enable shell debug mode --no-output Suppress all output diff --git a/src/env.sh b/src/env.sh index ea568434..f86fc815 100644 --- a/src/env.sh +++ b/src/env.sh @@ -125,6 +125,9 @@ _BASHUNIT_DEFAULT_RETRY="0" _BASHUNIT_DEFAULT_RANDOM_ORDER="false" # Seed for --random-order (empty = generate one and print it) _BASHUNIT_DEFAULT_SEED="" +# Shard / to split the suite across runners (empty = disabled) +_BASHUNIT_DEFAULT_SHARD_INDEX="" +_BASHUNIT_DEFAULT_SHARD_TOTAL="" : "${BASHUNIT_PARALLEL_RUN:=${PARALLEL_RUN:=$_BASHUNIT_DEFAULT_PARALLEL_RUN}}" : "${BASHUNIT_PARALLEL_JOBS:=0}" @@ -158,6 +161,8 @@ _BASHUNIT_DEFAULT_SEED="" # up unrelated environment values. : "${BASHUNIT_RANDOM_ORDER:=$_BASHUNIT_DEFAULT_RANDOM_ORDER}" : "${BASHUNIT_SEED:=$_BASHUNIT_DEFAULT_SEED}" +: "${BASHUNIT_SHARD_INDEX:=$_BASHUNIT_DEFAULT_SHARD_INDEX}" +: "${BASHUNIT_SHARD_TOTAL:=$_BASHUNIT_DEFAULT_SHARD_TOTAL}" # Support NO_COLOR standard (https://no-color.org) if [ -n "${NO_COLOR:-}" ]; then BASHUNIT_NO_COLOR="true" @@ -212,6 +217,18 @@ function bashunit::env::seed() { printf '%s' "${BASHUNIT_SEED:-}" } +function bashunit::env::is_shard_enabled() { + [ -n "${BASHUNIT_SHARD_INDEX:-}" ] && [ -n "${BASHUNIT_SHARD_TOTAL:-}" ] +} + +function bashunit::env::shard_index() { + printf '%s' "${BASHUNIT_SHARD_INDEX:-}" +} + +function bashunit::env::shard_total() { + printf '%s' "${BASHUNIT_SHARD_TOTAL:-}" +} + function bashunit::env::is_show_header_enabled() { [ "$BASHUNIT_SHOW_HEADER" = "true" ] } diff --git a/src/main.sh b/src/main.sh index 7deedb22..8614db95 100644 --- a/src/main.sh +++ b/src/main.sh @@ -1,5 +1,36 @@ #!/usr/bin/env bash +## +# Validates a `--shard /` spec and exports the parts, or prints an +# error and exits non-zero. Requires numeric index/total with 1 <= index <= total. +## +function bashunit::main::set_shard_or_exit() { + local spec="${1:-}" + local index total + case "$spec" in + */*) + index="${spec%%/*}" + total="${spec##*/}" + ;; + *) + index="" + total="" + ;; + esac + case "$index" in '' | *[!0-9]*) index="" ;; esac + case "$total" in '' | *[!0-9]*) total="" ;; esac + + if [ -z "$index" ] || [ -z "$total" ] || + [ "$total" -lt 1 ] || [ "$index" -lt 1 ] || [ "$index" -gt "$total" ]; then + printf "%sError: --shard must be / with 1 <= index <= total (e.g. 1/4).%s\n" \ + "${_BASHUNIT_COLOR_FAILED}" "${_BASHUNIT_COLOR_DEFAULT}" >&2 + exit 1 + fi + + export BASHUNIT_SHARD_INDEX="$index" + export BASHUNIT_SHARD_TOTAL="$total" +} + ############################# # Subcommand: test ############################# @@ -89,6 +120,10 @@ function bashunit::main::cmd_test() { export BASHUNIT_SEED="$2" shift ;; + --shard) + bashunit::main::set_shard_or_exit "$2" + shift + ;; -w | --watch) export BASHUNIT_WATCH_MODE=true ;; @@ -682,6 +717,26 @@ function bashunit::main::exec_tests() { exit 1 fi + # Split the suite across runners: keep the files whose position matches this + # shard (round-robin), so all shards together cover the whole suite with no + # overlap. An empty shard (more shards than files) is valid and runs nothing. + if bashunit::env::is_shard_enabled; then + local _shard_index _shard_total + _shard_index=$(bashunit::env::shard_index) + _shard_total=$(bashunit::env::shard_total) + local -a _sharded=() + local _i=0 + while [ "$_i" -lt "$test_files_count" ]; do + if [ "$((_i % _shard_total))" -eq "$((_shard_index - 1))" ]; then + _sharded[${#_sharded[@]}]="${test_files[_i]}" + fi + _i=$((_i + 1)) + done + test_files=("${_sharded[@]+"${_sharded[@]}"}") + test_files_count=${#test_files[@]} + bashunit::internal_log "shard" "index:$_shard_index" "total:$_shard_total" "files:$test_files_count" + fi + # Trap SIGINT (Ctrl-C) and call the cleanup function trap 'bashunit::main::cleanup' SIGINT trap '[ $? -eq $EXIT_CODE_STOP_ON_FAILURE ] && bashunit::main::handle_stop_on_failure_sync' EXIT diff --git a/tests/acceptance/bashunit_shard_test.sh b/tests/acceptance/bashunit_shard_test.sh new file mode 100644 index 00000000..efe0d2b6 --- /dev/null +++ b/tests/acceptance/bashunit_shard_test.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +function set_up_before_script() { + TEST_ENV_FILE="tests/acceptance/fixtures/.env.default" + D="tests/acceptance/fixtures" + FILES="$D/test_shard_a.sh $D/test_shard_b.sh $D/test_shard_c.sh $D/test_shard_d.sh" +} + +# Runs the four shard fixtures through a shard and prints which ran, sorted. +function shard_run() { + local order_file + order_file="$(mktemp)" + # shellcheck disable=SC2086 + BASHUNIT_TEST_ORDER_FILE="$order_file" \ + ./bashunit --no-parallel --env "$TEST_ENV_FILE" "$@" $FILES >/dev/null 2>&1 + sort "$order_file" | tr '\n' ' ' | sed 's/ *$//' + rm -f "$order_file" +} + +function test_shard_1_of_2_runs_its_slice() { + assert_same "a c" "$(shard_run --shard 1/2)" +} + +function test_shard_2_of_2_runs_its_slice() { + assert_same "b d" "$(shard_run --shard 2/2)" +} + +function test_shards_are_disjoint_and_cover_the_whole_suite() { + local union + union="$(printf '%s %s' "$(shard_run --shard 1/2)" "$(shard_run --shard 2/2)" | + tr ' ' '\n' | sort | tr '\n' ' ' | sed 's/ *$//')" + + assert_same "a b c d" "$union" +} + +function test_shard_rejects_index_greater_than_total() { + # shellcheck disable=SC2086 + assert_general_error \ + "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" --shard 3/2 $FILES 2>&1)" +} + +function test_shard_rejects_non_numeric_input() { + # shellcheck disable=SC2086 + assert_general_error \ + "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" --shard x/2 $FILES 2>&1)" +} + +function test_shard_composes_with_parallel() { + local order_file count + order_file="$(mktemp)" + # shellcheck disable=SC2086 + BASHUNIT_TEST_ORDER_FILE="$order_file" \ + ./bashunit --parallel --env "$TEST_ENV_FILE" --shard 1/2 $FILES >/dev/null 2>&1 + count=$(tr -cd 'a-d' <"$order_file" | wc -c | tr -d ' ') + rm -f "$order_file" + + assert_same "2" "$count" +} diff --git a/tests/acceptance/fixtures/test_shard_a.sh b/tests/acceptance/fixtures/test_shard_a.sh new file mode 100644 index 00000000..60a12dd5 --- /dev/null +++ b/tests/acceptance/fixtures/test_shard_a.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +function test_shard_a() { + printf '%s\n' a >>"${BASHUNIT_TEST_ORDER_FILE:?order file required}" + assert_same 1 1 +} diff --git a/tests/acceptance/fixtures/test_shard_b.sh b/tests/acceptance/fixtures/test_shard_b.sh new file mode 100644 index 00000000..54e21b4f --- /dev/null +++ b/tests/acceptance/fixtures/test_shard_b.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +function test_shard_b() { + printf '%s\n' b >>"${BASHUNIT_TEST_ORDER_FILE:?order file required}" + assert_same 1 1 +} diff --git a/tests/acceptance/fixtures/test_shard_c.sh b/tests/acceptance/fixtures/test_shard_c.sh new file mode 100644 index 00000000..8b5c9b70 --- /dev/null +++ b/tests/acceptance/fixtures/test_shard_c.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +function test_shard_c() { + printf '%s\n' c >>"${BASHUNIT_TEST_ORDER_FILE:?order file required}" + assert_same 1 1 +} diff --git a/tests/acceptance/fixtures/test_shard_d.sh b/tests/acceptance/fixtures/test_shard_d.sh new file mode 100644 index 00000000..d4b28f34 --- /dev/null +++ b/tests/acceptance/fixtures/test_shard_d.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +function test_shard_d() { + printf '%s\n' d >>"${BASHUNIT_TEST_ORDER_FILE:?order file required}" + assert_same 1 1 +}