From d12ec0c407250559db8314dac0e2475538cc45d5 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Fri, 10 Jul 2026 12:03:27 +0200 Subject: [PATCH 1/2] feat(cli): add --random-order and --seed to randomize test order (#738) Shuffle test-file and per-file function execution order to surface hidden inter-test coupling. A seeded Fisher-Yates (LCG) makes the shuffle reproducible; each shuffle reseeds locally (mixing the seed with a per-file cksum), so it is subshell-safe and works under --parallel. The seed is printed for replay and generated when absent. --seed without --random-order is a no-op. Disabled by default. Closes #738 --- CHANGELOG.md | 1 + docs/command-line.md | 30 +++++++++++ docs/configuration.md | 22 ++++++++ src/console_header.sh | 12 +++++ src/env.sh | 19 +++++++ src/main.sh | 18 +++++++ src/math.sh | 40 ++++++++++++++ src/runner.sh | 27 ++++++++++ .../acceptance/bashunit_random_order_test.sh | 52 +++++++++++++++++++ .../fixtures/test_bashunit_random_order.sh | 43 +++++++++++++++ tests/unit/math_test.sh | 32 ++++++++++++ 11 files changed, 296 insertions(+) create mode 100644 tests/acceptance/bashunit_random_order_test.sh create mode 100644 tests/acceptance/fixtures/test_bashunit_random_order.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 830dd062..0c4b0b53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- `--random-order` flag with `--seed ` / `BASHUNIT_SEED` to randomize test file and function execution order (surfaces inter-test coupling); the seed is printed for replay and the shuffle is reproducible and works with `--parallel`. Disabled by default (#738) - `--retry ` flag and `BASHUNIT_RETRY` env var to re-run a failed test up to N extra times (flaky-test mitigation); passes if any attempt passes, annotates tests that only passed on retry, and works with `--parallel` and `--stop-on-failure`. Disabled by default (#737) - `--report-tap ` writes a TAP version 13 report to a file (complements the streaming `--output tap`) (#740) - `assert_within_delta ` asserts a number is within a tolerance of another; supports floats (#744) diff --git a/docs/command-line.md b/docs/command-line.md index d2f3ef54..829a5a03 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -78,6 +78,8 @@ bashunit test tests/ --parallel --simple | `-S, --stop-on-failure` | Stop on first failure | | `--test-timeout ` | Fail a test if it runs longer than N seconds | | `--retry ` | Re-run a failed test up to N extra times | +| `--random-order` | Randomize test execution order | +| `--seed ` | Seed for `--random-order` (reproducible shuffle) | | `--show-skipped` | Show skipped tests summary at end | | `--show-incomplete` | Show incomplete tests summary at end | | `-vvv, --verbose` | Show execution details | @@ -490,6 +492,34 @@ Tests: 1 passed, 1 total It can also be set via the `BASHUNIT_RETRY` environment variable (see [configuration](/configuration#retry)). +### Random order + +> `bashunit test --random-order [--seed ]` + +Randomize the order in which test files and the tests within each file run, to +surface hidden inter-test coupling (leaked globals, shared temp files, ordering +dependencies). Disabled by default. + +When enabled and no `--seed` is given, a seed is generated and printed in the +run header so a failing run can be replayed exactly with `--seed `. The same +seed always produces the same order, and it composes with `--parallel`. `--seed` +on its own (without `--random-order`) has no effect. + +::: code-group +```bash [Example] +bashunit test tests/ --random-order +``` +```[Output] +Randomized with seed: 12345 + +# replay the exact same order: +bashunit test tests/ --random-order --seed 12345 +``` +::: + +It can also be set via the `BASHUNIT_SEED` environment variable (see +[configuration](/configuration#random-order)). + ### No Progress > `bashunit test --no-progress` diff --git a/docs/configuration.md b/docs/configuration.md index 7b47c7d9..c01ea42b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -173,6 +173,28 @@ BASHUNIT_RETRY=0 ``` ::: +## Random order + +> `BASHUNIT_RANDOM_ORDER=true|false` and `BASHUNIT_SEED=` + +Randomize the order of test files and of the tests within each file to surface +hidden inter-test coupling. Disabled by default. `BASHUNIT_SEED` pins the +shuffle so a run can be replayed; when unset, a seed is generated and printed in +the run header. Composes with `--parallel`. + +Similar as using `--random-order` / `--seed` options on the +[command line](/command-line#random-order). + +::: code-group +```bash [Reproducible shuffle] +BASHUNIT_RANDOM_ORDER=true +BASHUNIT_SEED=12345 +``` +```bash [Disabled (default)] +BASHUNIT_RANDOM_ORDER=false +``` +::: + ## Stop on assertion failure > `BASHUNIT_STOP_ON_ASSERTION_FAILURE=true|false` diff --git a/src/console_header.sh b/src/console_header.sh index 221a8a9b..2985eb6d 100644 --- a/src/console_header.sh +++ b/src/console_header.sh @@ -16,6 +16,16 @@ function bashunit::console_header::print_version_with_env() { fi } +## +# Prints the seed used to randomize test order, so a failing run can be replayed +# with `--seed `. +## +function bashunit::console_header::print_random_order_seed() { + local seed=$1 + printf "%sRandomized with seed:%s %s\n" \ + "${_BASHUNIT_COLOR_INCOMPLETE}" "${_BASHUNIT_COLOR_DEFAULT}" "$seed" +} + function bashunit::console_header::print_version() { local filter=${1:-} shift || true @@ -121,6 +131,8 @@ Options: -S, --stop-on-failure Stop on first failure --test-timeout Fail a test if it runs longer than N seconds (0 = off) --retry Re-run a failed test up to N extra times (0 = off) + --random-order Randomize test execution order + --seed Seed for --random-order (reproducible shuffle) -vvv, --verbose Show execution details --debug [file] Enable shell debug mode --no-output Suppress all output diff --git a/src/env.sh b/src/env.sh index 39d89998..ea568434 100644 --- a/src/env.sh +++ b/src/env.sh @@ -121,6 +121,10 @@ _BASHUNIT_DEFAULT_PROFILE_COUNT="10" _BASHUNIT_DEFAULT_TEST_TIMEOUT="0" # Extra attempts for a failed test (0 = no retry) _BASHUNIT_DEFAULT_RETRY="0" +# Randomize test execution order to surface inter-test coupling +_BASHUNIT_DEFAULT_RANDOM_ORDER="false" +# Seed for --random-order (empty = generate one and print it) +_BASHUNIT_DEFAULT_SEED="" : "${BASHUNIT_PARALLEL_RUN:=${PARALLEL_RUN:=$_BASHUNIT_DEFAULT_PARALLEL_RUN}}" : "${BASHUNIT_PARALLEL_JOBS:=0}" @@ -150,6 +154,10 @@ _BASHUNIT_DEFAULT_RETRY="0" # No bare RETRY alias on purpose: it is too generic and would pick up unrelated # environment values. Only BASHUNIT_RETRY configures retries. : "${BASHUNIT_RETRY:=$_BASHUNIT_DEFAULT_RETRY}" +# Single alias on purpose: bare RANDOM_ORDER/SEED are too generic and would pick +# up unrelated environment values. +: "${BASHUNIT_RANDOM_ORDER:=$_BASHUNIT_DEFAULT_RANDOM_ORDER}" +: "${BASHUNIT_SEED:=$_BASHUNIT_DEFAULT_SEED}" # Support NO_COLOR standard (https://no-color.org) if [ -n "${NO_COLOR:-}" ]; then BASHUNIT_NO_COLOR="true" @@ -193,6 +201,17 @@ function bashunit::env::retry_count() { printf '%s' "${BASHUNIT_RETRY:-0}" } +function bashunit::env::is_random_order_enabled() { + [ "$BASHUNIT_RANDOM_ORDER" = "true" ] +} + +## +# Prints the configured random-order seed (empty when none set yet). +## +function bashunit::env::seed() { + printf '%s' "${BASHUNIT_SEED:-}" +} + function bashunit::env::is_show_header_enabled() { [ "$BASHUNIT_SHOW_HEADER" = "true" ] } diff --git a/src/main.sh b/src/main.sh index 0457c63d..7deedb22 100644 --- a/src/main.sh +++ b/src/main.sh @@ -82,6 +82,13 @@ function bashunit::main::cmd_test() { export BASHUNIT_RETRY="$2" shift ;; + --random-order) + export BASHUNIT_RANDOM_ORDER=true + ;; + --seed) + export BASHUNIT_SEED="$2" + shift + ;; -w | --watch) export BASHUNIT_WATCH_MODE=true ;; @@ -696,6 +703,17 @@ function bashunit::main::exec_tests() { bashunit::console_header::print_version_with_env "$filter" "${test_files[@]}" fi + # Resolve the shuffle seed once (generating one if absent) so it can be printed + # for replay and inherited by parallel test-file subshells. + if bashunit::env::is_random_order_enabled; then + if [ -z "${BASHUNIT_SEED:-}" ]; then + export BASHUNIT_SEED=$RANDOM + fi + if ! bashunit::env::is_tap_output_enabled; then + bashunit::console_header::print_random_order_seed "$BASHUNIT_SEED" + fi + fi + if bashunit::env::is_verbose_enabled; then if bashunit::env::is_simple_output_enabled; then echo "" diff --git a/src/math.sh b/src/math.sh index 66f83b66..b0b69462 100644 --- a/src/math.sh +++ b/src/math.sh @@ -25,3 +25,43 @@ function bashunit::math::calculate() { local result=$((expr)) echo "$result" } + +## +# Deterministically shuffles stdin lines (one item per line) with a Fisher-Yates +# driven by a seeded LCG (glibc constants). Same seed + same input always yields +# the same permutation, so a randomized run can be replayed via its seed. +# Self-contained (seeds a local state), so it is safe inside subshells/pipes and +# in --parallel where each test file shuffles in its own forked shell. +# Arguments: $1 - integer seed (non-numeric treated as 0) +## +function bashunit::math::shuffle() { + local seed=$1 + case "$seed" in '' | *[!0-9]*) seed=0 ;; esac + local state=$((seed & 2147483647)) + + local -a items=() + local n=0 + local line + # `|| [ -n "$line" ]` keeps the final item when stdin has no trailing newline. + while IFS= read -r line || [ -n "$line" ]; do + items[n]=$line + n=$((n + 1)) + done + + local i j tmp + i=$((n - 1)) + while [ "$i" -gt 0 ]; do + state=$(((1103515245 * state + 12345) & 2147483647)) + j=$((state % (i + 1))) + tmp=${items[i]} + items[i]=${items[j]} + items[j]=$tmp + i=$((i - 1)) + done + + local k=0 + while [ "$k" -lt "$n" ]; do + printf '%s\n' "${items[k]}" + k=$((k + 1)) + done +} diff --git a/src/runner.sh b/src/runner.sh index 28aa6688..fecd388e 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -333,6 +333,16 @@ function bashunit::runner::load_test_files() { local -a scripts_ids=() local scripts_ids_count=0 + # Randomize file execution order (deterministic for the resolved seed). + if bashunit::env::is_random_order_enabled; then + local -a _shuffled_files=() + local _sf + while IFS= read -r _sf; do + [ -n "$_sf" ] && _shuffled_files[${#_shuffled_files[@]}]=$_sf + done < <(printf '%s\n' "${files[@]+"${files[@]}"}" | bashunit::math::shuffle "$(bashunit::env::seed)") + files=("${_shuffled_files[@]+"${_shuffled_files[@]}"}") + fi + bashunit::runner::sync_coverage_flag # Initialize coverage tracking if enabled @@ -723,6 +733,23 @@ function bashunit::runner::call_test_functions() { fi fi + # Randomize function order within this file. The seed is mixed with a stable + # per-file value (cksum of the path) so different files get different orders + # while staying reproducible for the resolved seed. + if bashunit::env::is_random_order_enabled && [ "$functions_to_run_count" -gt 1 ]; then + local _base _crc _fn_seed + _base=$(bashunit::env::seed) + _crc=$(printf '%s' "$script" | cksum | cut -d' ' -f1) + _fn_seed=$(((_base + _crc) & 2147483647)) + local -a _shuffled_fns=() + local _sfn + while IFS= read -r _sfn; do + [ -n "$_sfn" ] && _shuffled_fns[${#_shuffled_fns[@]}]=$_sfn + done < <(printf '%s\n' "${functions_to_run[@]+"${functions_to_run[@]}"}" | bashunit::math::shuffle "$_fn_seed") + functions_to_run=("${_shuffled_fns[@]+"${_shuffled_fns[@]}"}") + functions_to_run_count=${#functions_to_run[@]} + fi + if [ "$functions_to_run_count" -le 0 ]; then return fi diff --git a/tests/acceptance/bashunit_random_order_test.sh b/tests/acceptance/bashunit_random_order_test.sh new file mode 100644 index 00000000..f3697de9 --- /dev/null +++ b/tests/acceptance/bashunit_random_order_test.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +function set_up_before_script() { + TEST_ENV_FILE="tests/acceptance/fixtures/.env.default" + FIXTURE="tests/acceptance/fixtures/test_bashunit_random_order.sh" +} + +# Prints the dispatch order of tests as a space-separated list of names. +function order_of() { + ./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" "$@" "$FIXTURE" | + grep "Passed:" | sed 's/.*Passed: //' | awk '{print $1}' | xargs +} + +function test_default_order_is_unchanged_without_the_flag() { + assert_same "Alpha Bravo Charlie Delta Echo Foxtrot Golf Hotel" "$(order_of)" +} + +function test_random_order_is_reproducible_with_a_given_seed() { + local first second + first="$(order_of --random-order --seed 42)" + second="$(order_of --random-order --seed 42)" + + assert_same "$first" "$second" +} + +function test_random_order_reorders_tests() { + assert_not_equals "$(order_of)" "$(order_of --random-order --seed 42)" +} + +function test_random_order_prints_the_seed_for_replay() { + local output + output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --random-order --seed 42 "$FIXTURE")" + + assert_contains "Randomized with seed: 42" "$output" +} + +function test_random_order_prints_a_generated_seed_when_none_given() { + local output + output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --random-order "$FIXTURE")" + + assert_contains "Randomized with seed:" "$output" +} + +function test_random_order_composes_with_parallel() { + local output + output="$(./bashunit --parallel --no-color --env "$TEST_ENV_FILE" \ + --random-order --seed 42 "$FIXTURE")" + + assert_contains "8 passed" "$output" +} diff --git a/tests/acceptance/fixtures/test_bashunit_random_order.sh b/tests/acceptance/fixtures/test_bashunit_random_order.sh new file mode 100644 index 00000000..0825b344 --- /dev/null +++ b/tests/acceptance/fixtures/test_bashunit_random_order.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Each test echoes an order marker so acceptance tests can read the dispatch +# order from the output independently of timing/formatting. +function test_alpha() { + echo "ORDER:alpha" + assert_same 1 1 +} + +function test_bravo() { + echo "ORDER:bravo" + assert_same 1 1 +} + +function test_charlie() { + echo "ORDER:charlie" + assert_same 1 1 +} + +function test_delta() { + echo "ORDER:delta" + assert_same 1 1 +} + +function test_echo() { + echo "ORDER:echo" + assert_same 1 1 +} + +function test_foxtrot() { + echo "ORDER:foxtrot" + assert_same 1 1 +} + +function test_golf() { + echo "ORDER:golf" + assert_same 1 1 +} + +function test_hotel() { + echo "ORDER:hotel" + assert_same 1 1 +} diff --git a/tests/unit/math_test.sh b/tests/unit/math_test.sh index a921c481..5d458c0f 100644 --- a/tests/unit/math_test.sh +++ b/tests/unit/math_test.sh @@ -72,3 +72,35 @@ function test_calculate_fallback_to_bash_arithmetic_for_decimal() { assert_equals "30" "$result" } + +function test_shuffle_is_deterministic_for_a_given_seed() { + local first second + first=$(printf '%s\n' a b c d e f g h | bashunit::math::shuffle 12345) + second=$(printf '%s\n' a b c d e f g h | bashunit::math::shuffle 12345) + + assert_equals "$first" "$second" +} + +function test_shuffle_differs_for_different_seeds() { + local one two + one=$(printf '%s\n' a b c d e f g h | bashunit::math::shuffle 1) + two=$(printf '%s\n' a b c d e f g h | bashunit::math::shuffle 2) + + assert_not_equals "$one" "$two" +} + +function test_shuffle_preserves_all_items() { + local shuffled_sorted input_sorted + shuffled_sorted=$(printf '%s\n' a b c d e f g h | bashunit::math::shuffle 7 | sort) + input_sorted=$(printf '%s\n' a b c d e f g h | sort) + + assert_equals "$input_sorted" "$shuffled_sorted" +} + +function test_shuffle_actually_reorders_for_a_known_seed() { + local original shuffled + original=$(printf '%s\n' a b c d e f g h) + shuffled=$(printf '%s\n' a b c d e f g h | bashunit::math::shuffle 12345) + + assert_not_equals "$original" "$shuffled" +} From d0b4327944991feaef7ecffc3b1740b56218eb60 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Fri, 10 Jul 2026 12:15:02 +0200 Subject: [PATCH 2/2] test(cli): make random-order acceptance tests output-independent (#738) The tests parsed colored console output, which varies by TTY/output-mode/locale and failed in CI (empty capture; a color reset split 'seed:' from the number). Record dispatch order to a file instead, and assert the seed label only. --- .../acceptance/bashunit_random_order_test.sh | 27 +++++++++++-------- .../fixtures/test_bashunit_random_order.sh | 25 ++++++++++------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/tests/acceptance/bashunit_random_order_test.sh b/tests/acceptance/bashunit_random_order_test.sh index f3697de9..fce16dd1 100644 --- a/tests/acceptance/bashunit_random_order_test.sh +++ b/tests/acceptance/bashunit_random_order_test.sh @@ -5,14 +5,19 @@ function set_up_before_script() { FIXTURE="tests/acceptance/fixtures/test_bashunit_random_order.sh" } -# Prints the dispatch order of tests as a space-separated list of names. +# Runs the fixture and prints the recorded dispatch order (space-separated), +# read from a file so it is independent of console output/color/locale. function order_of() { - ./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" "$@" "$FIXTURE" | - grep "Passed:" | sed 's/.*Passed: //' | awk '{print $1}' | xargs + local order_file + order_file="$(mktemp)" + BASHUNIT_TEST_ORDER_FILE="$order_file" \ + ./bashunit --no-parallel --env "$TEST_ENV_FILE" "$@" "$FIXTURE" >/dev/null 2>&1 + tr '\n' ' ' <"$order_file" | sed 's/ *$//' + rm -f "$order_file" } function test_default_order_is_unchanged_without_the_flag() { - assert_same "Alpha Bravo Charlie Delta Echo Foxtrot Golf Hotel" "$(order_of)" + assert_same "alpha bravo charlie delta echo foxtrot golf hotel" "$(order_of)" } function test_random_order_is_reproducible_with_a_given_seed() { @@ -29,24 +34,24 @@ function test_random_order_reorders_tests() { function test_random_order_prints_the_seed_for_replay() { local output - output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ - --random-order --seed 42 "$FIXTURE")" + output="$(BASHUNIT_TEST_ORDER_FILE="$(mktemp)" \ + ./bashunit --no-parallel --env "$TEST_ENV_FILE" --random-order --seed 42 "$FIXTURE")" - assert_contains "Randomized with seed: 42" "$output" + assert_contains "Randomized with seed:" "$output" } function test_random_order_prints_a_generated_seed_when_none_given() { local output - output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ - --random-order "$FIXTURE")" + output="$(BASHUNIT_TEST_ORDER_FILE="$(mktemp)" \ + ./bashunit --no-parallel --env "$TEST_ENV_FILE" --random-order "$FIXTURE")" assert_contains "Randomized with seed:" "$output" } function test_random_order_composes_with_parallel() { local output - output="$(./bashunit --parallel --no-color --env "$TEST_ENV_FILE" \ - --random-order --seed 42 "$FIXTURE")" + output="$(BASHUNIT_TEST_ORDER_FILE="$(mktemp)" \ + ./bashunit --parallel --env "$TEST_ENV_FILE" --random-order --seed 42 "$FIXTURE")" assert_contains "8 passed" "$output" } diff --git a/tests/acceptance/fixtures/test_bashunit_random_order.sh b/tests/acceptance/fixtures/test_bashunit_random_order.sh index 0825b344..f359775d 100644 --- a/tests/acceptance/fixtures/test_bashunit_random_order.sh +++ b/tests/acceptance/fixtures/test_bashunit_random_order.sh @@ -1,43 +1,48 @@ #!/usr/bin/env bash -# Each test echoes an order marker so acceptance tests can read the dispatch -# order from the output independently of timing/formatting. +# Each test appends its name to an order file, so acceptance tests can read the +# dispatch order from the filesystem instead of parsing console output (which +# varies with color, output mode, TTY and locale). +function record_order() { + printf '%s\n' "$1" >>"${BASHUNIT_TEST_ORDER_FILE:?order file required}" +} + function test_alpha() { - echo "ORDER:alpha" + record_order alpha assert_same 1 1 } function test_bravo() { - echo "ORDER:bravo" + record_order bravo assert_same 1 1 } function test_charlie() { - echo "ORDER:charlie" + record_order charlie assert_same 1 1 } function test_delta() { - echo "ORDER:delta" + record_order delta assert_same 1 1 } function test_echo() { - echo "ORDER:echo" + record_order echo assert_same 1 1 } function test_foxtrot() { - echo "ORDER:foxtrot" + record_order foxtrot assert_same 1 1 } function test_golf() { - echo "ORDER:golf" + record_order golf assert_same 1 1 } function test_hotel() { - echo "ORDER:hotel" + record_order hotel assert_same 1 1 }