feat(cli): add --retry <n> to re-run failed tests

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 ### Added
+- `--retry <n>` flag and `BASHUNIT_RETRY` env var to re-run a failed test up to N extra times (flaky-test mitigation); passes if any attempt passes, annotates tests that only passed on retry, and works with `--parallel` and `--stop-on-failure`. Disabled by default (#737)
 - `--report-tap <file>` writes a TAP version 13 report to a file (complements the streaming `--output tap`) (#740)
 - `assert_within_delta <expected> <actual> <delta>` asserts a number is within a tolerance of another; supports floats (#744)
 - `assert_array_length <n> <array>` asserts an array has exactly `n` elements (#743)

docs/command-line.md

@@ -77,6 +77,7 @@ bashunit test tests/ --parallel --simple
 | `--detailed`                   | Detailed output (default)                        |
 | `-S, --stop-on-failure`        | Stop on first failure                            |
 | `--test-timeout <seconds>`     | Fail a test if it runs longer than N seconds     |
+| `--retry <n>`                  | Re-run a failed test up to N extra times         |
 | `--show-skipped`               | Show skipped tests summary at end                |
 | `--show-incomplete`            | Show incomplete tests summary at end             |
 | `-vvv, --verbose`              | Show execution details                           |
@@ -461,6 +462,34 @@ Tests:      1 passed, 1 failed, 2 total
 It can also be set via the `BASHUNIT_TEST_TIMEOUT` environment variable (see
 [configuration](/configuration#test-timeout)).
 
+### Retry
+
+> `bashunit test --retry <n>`
+
+Re-run a **failed** test up to `n` extra times and report it as passed if any
+attempt passes; it only fails once every attempt has failed. This mitigates
+flaky tests (timing, network or filesystem races) in CI without hiding a test
+that is consistently broken.
+
+Retry is **disabled by default** (`0`). A test that recovered on retry is
+annotated so the flakiness stays visible, retries apply per test, and it works
+together with `--parallel` and `--stop-on-failure` (a test that recovers on
+retry does not trigger stop-on-failure).
+
+::: code-group
+```bash [Example]
+bashunit test tests/ --retry 2
+```
+```[Output]
+✓ Passed: A flaky test (retry 1/2)
+
+Tests:      1 passed, 1 total
+```
+:::
+
+It can also be set via the `BASHUNIT_RETRY` environment variable (see
+[configuration](/configuration#retry)).
+
 ### No Progress
 
 > `bashunit test --no-progress`

docs/configuration.md

@@ -151,6 +151,28 @@ BASHUNIT_TEST_TIMEOUT=0
 ```
 :::
 
+## Retry
+
+> `BASHUNIT_RETRY=<n>`
+
+Re-run a failed test up to `n` extra times and report it as passed if any
+attempt passes; it fails only after every attempt fails. `0` (disabled) by
+default. Mitigates flaky tests in CI without hiding a consistently broken one; a
+test that recovers on retry is annotated so the flakiness stays visible.
+
+Applies per test and works together with `--parallel` and `--stop-on-failure`.
+
+Similar as using `--retry` option on the [command line](/command-line#retry).
+
+::: code-group
+```bash [Retry up to 2 times]
+BASHUNIT_RETRY=2
+```
+```bash [Disabled (default)]
+BASHUNIT_RETRY=0
+```
+:::
+
 ## Stop on assertion failure
 
 > `BASHUNIT_STOP_ON_ASSERTION_FAILURE=true|false`

src/console_header.sh

@@ -120,6 +120,7 @@ Options:
   -R, --run-all               Run all assertions (don't stop on first failure)
   -S, --stop-on-failure       Stop on first failure
   --test-timeout <seconds>    Fail a test if it runs longer than N seconds (0 = off)
+  --retry <n>                 Re-run a failed test up to N extra times (0 = off)
   -vvv, --verbose             Show execution details
   --debug [file]              Enable shell debug mode
   --no-output                 Suppress all output

src/console_results.sh

@@ -244,6 +244,10 @@ function bashunit::console_results::print_successful_test() {
       "$_BASHUNIT_COLOR_PASSED" "$_BASHUNIT_COLOR_DEFAULT" "$test_name" "$quoted_args")
   fi
 
+  # Retry annotation (e.g. " (retry 1/2)") set by the runner when a test only
+  # passed after retrying; empty in the common no-retry path.
+  line="${line}${_BASHUNIT_RETRY_NOTE:-}"
+
   local full_line=$line
   if bashunit::env::is_show_execution_time_enabled; then
     local time_display

src/env.sh

@@ -119,6 +119,8 @@ _BASHUNIT_DEFAULT_PROFILE="false"
 _BASHUNIT_DEFAULT_PROFILE_COUNT="10"
 # Per-test timeout in seconds (0 = disabled)
 _BASHUNIT_DEFAULT_TEST_TIMEOUT="0"
+# Extra attempts for a failed test (0 = no retry)
+_BASHUNIT_DEFAULT_RETRY="0"
 
 : "${BASHUNIT_PARALLEL_RUN:=${PARALLEL_RUN:=$_BASHUNIT_DEFAULT_PARALLEL_RUN}}"
 : "${BASHUNIT_PARALLEL_JOBS:=0}"
@@ -145,6 +147,9 @@ _BASHUNIT_DEFAULT_TEST_TIMEOUT="0"
 : "${BASHUNIT_PROFILE:=${PROFILE:=$_BASHUNIT_DEFAULT_PROFILE}}"
 : "${BASHUNIT_PROFILE_COUNT:=${PROFILE_COUNT:=$_BASHUNIT_DEFAULT_PROFILE_COUNT}}"
 : "${BASHUNIT_TEST_TIMEOUT:=${TEST_TIMEOUT:=$_BASHUNIT_DEFAULT_TEST_TIMEOUT}}"
+# No bare RETRY alias on purpose: it is too generic and would pick up unrelated
+# environment values. Only BASHUNIT_RETRY configures retries.
+: "${BASHUNIT_RETRY:=$_BASHUNIT_DEFAULT_RETRY}"
 # Support NO_COLOR standard (https://no-color.org)
 if [ -n "${NO_COLOR:-}" ]; then
   BASHUNIT_NO_COLOR="true"
@@ -174,6 +179,20 @@ function bashunit::env::test_timeout_secs() {
   printf '%s' "${BASHUNIT_TEST_TIMEOUT:-0}"
 }
 
+##
+# Prints the number of extra attempts for a failed test (0 = no retry).
+# A non-numeric value is treated as 0.
+##
+function bashunit::env::retry_count() {
+  case "${BASHUNIT_RETRY:-0}" in
+  '' | *[!0-9]*)
+    printf '%s' "0"
+    return
+    ;;
+  esac
+  printf '%s' "${BASHUNIT_RETRY:-0}"
+}
+
 function bashunit::env::is_show_header_enabled() {
   [ "$BASHUNIT_SHOW_HEADER" = "true" ]
 }

src/main.sh

@@ -78,6 +78,10 @@ function bashunit::main::cmd_test() {
       export BASHUNIT_TEST_TIMEOUT="$2"
       shift
       ;;
+    --retry)
+      export BASHUNIT_RETRY="$2"
+      shift
+      ;;
     -w | --watch)
       export BASHUNIT_WATCH_MODE=true
       ;;

src/runner.sh

@@ -131,6 +131,14 @@ _BASHUNIT_RUNNER_TOTAL_OUT=""
 _BASHUNIT_RUNNER_TYPE_OUT=""
 _BASHUNIT_RUNNER_OUTPUT_OUT=""
 _BASHUNIT_RUNNER_INTERP_OUT=""
+_BASHUNIT_RUNNER_COUNTS_FAILED_OUT=0
+_BASHUNIT_RUNNER_COUNTS_PASSED_OUT=0
+_BASHUNIT_RUNNER_COUNTS_SKIPPED_OUT=0
+_BASHUNIT_RUNNER_COUNTS_INCOMPLETE_OUT=0
+_BASHUNIT_RUNNER_COUNTS_SNAPSHOT_OUT=0
+_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT=0
+# Suffix appended to a passed-test line when it only passed after retrying.
+_BASHUNIT_RETRY_NOTE=""
 
 # Writes the value of an encoded field (##KEY=value##) into _BASHUNIT_RUNNER_FIELD_OUT.
 # Arguments: $1 test_execution_result, $2 key
@@ -1000,7 +1008,6 @@ function bashunit::runner::run_with_timeout() {
 
 function bashunit::runner::run_test() {
   local start_time
-  start_time=$(bashunit::clock::now)
 
   local test_file="$1"
   shift
@@ -1025,13 +1032,37 @@ function bashunit::runner::run_test() {
 
   local test_execution_result
   local timed_out="false"
-  if bashunit::env::is_test_timeout_enabled; then
-    bashunit::runner::run_with_timeout "$test_file" "$fn_name" "$@"
-    test_execution_result="$_BASHUNIT_RUNNER_EXEC_OUT"
-    timed_out="$_BASHUNIT_RUNNER_TIMED_OUT"
-  else
-    test_execution_result=$(bashunit::runner::execute_test_body "$test_file" "$fn_name" "$@")
-  fi
+  local retry_max
+  retry_max=$(bashunit::env::retry_count)
+  local retries_used=0
+  # Retry wraps ONLY execution: a failed attempt is judged from its encoded
+  # result without committing, so the parse/report/counter path below still runs
+  # exactly once (on the final attempt) and nothing is double-counted. Each fork
+  # in --parallel retries itself before writing its single .result file.
+  while :; do
+    start_time=$(bashunit::clock::now)
+    if bashunit::env::is_test_timeout_enabled; then
+      bashunit::runner::run_with_timeout "$test_file" "$fn_name" "$@"
+      test_execution_result="$_BASHUNIT_RUNNER_EXEC_OUT"
+      timed_out="$_BASHUNIT_RUNNER_TIMED_OUT"
+    else
+      test_execution_result=$(bashunit::runner::execute_test_body "$test_file" "$fn_name" "$@")
+    fi
+
+    local attempt_runtime_output="${test_execution_result%%##ASSERTIONS_*}"
+    local attempt_runtime_error
+    attempt_runtime_error=$(bashunit::runner::detect_runtime_error "$attempt_runtime_output")
+    bashunit::runner::extract_result_counts "$test_execution_result"
+    # Mirror the commit-phase failure test exactly (runtime error, non-zero exit,
+    # or a failed assertion); snapshot/incomplete/skipped/risky are not failures.
+    if [ -z "$attempt_runtime_error" ] &&
+      [ "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" -eq 0 ] &&
+      [ "$_BASHUNIT_RUNNER_COUNTS_FAILED_OUT" -eq 0 ]; then
+      break
+    fi
+    [ "$retries_used" -ge "$retry_max" ] && break
+    retries_used=$((retries_used + 1))
+  done
 
   # Closes FD 3, which was used temporarily to hold the original stdout.
   exec 3>&-
@@ -1220,6 +1251,11 @@ function bashunit::runner::run_test() {
     return
   fi
 
+  # A test that only passed after retrying is annotated so flakiness stays visible.
+  _BASHUNIT_RETRY_NOTE=""
+  if [ "$retries_used" -gt 0 ]; then
+    _BASHUNIT_RETRY_NOTE=" (retry $retries_used/$retry_max)"
+  fi
   # In failures-only mode, suppress successful test output
   if ! bashunit::env::is_failures_only_enabled; then
     if [ "$fn_name" = "$interpolated_fn_name" ]; then
@@ -1228,6 +1264,7 @@ function bashunit::runner::run_test() {
       bashunit::console_results::print_successful_test "${label}" "$duration"
     fi
   fi
+  _BASHUNIT_RETRY_NOTE=""
   bashunit::state::add_tests_passed
   bashunit::reports::add_test_passed "$test_file" "$label" "$duration" "$total_assertions"
   bashunit::internal_log "Test passed" "$label"
@@ -1412,9 +1449,14 @@ function bashunit::runner::parse_result_parallel() {
 }
 
 # shellcheck disable=SC2295
-function bashunit::runner::parse_result_sync() {
-  local fn_name=$1
-  local execution_result=$2
+##
+# Parses the encoded per-test result's last line into the counts out-slots
+# (_BASHUNIT_RUNNER_COUNTS_*_OUT). Pure read: never mutates the cumulative
+# _BASHUNIT_ASSERTIONS_* / _BASHUNIT_TEST_EXIT_CODE state, so the retry loop can
+# judge an attempt's outcome without committing it.
+##
+function bashunit::runner::extract_result_counts() {
+  local execution_result=$1
 
   local result_line
   result_line="${execution_result##*$'\n'}"
@@ -1453,22 +1495,36 @@ function bashunit::runner::parse_result_sync() {
     ;;
   esac
 
+  _BASHUNIT_RUNNER_COUNTS_FAILED_OUT=$assertions_failed
+  _BASHUNIT_RUNNER_COUNTS_PASSED_OUT=$assertions_passed
+  _BASHUNIT_RUNNER_COUNTS_SKIPPED_OUT=$assertions_skipped
+  _BASHUNIT_RUNNER_COUNTS_INCOMPLETE_OUT=$assertions_incomplete
+  _BASHUNIT_RUNNER_COUNTS_SNAPSHOT_OUT=$assertions_snapshot
+  _BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT=$test_exit_code
+}
+
+function bashunit::runner::parse_result_sync() {
+  local fn_name=$1
+  local execution_result=$2
+
+  bashunit::runner::extract_result_counts "$execution_result"
+
   bashunit::internal_log "[SYNC]" "fn_name:$fn_name" "execution_result:$execution_result"
 
-  _BASHUNIT_ASSERTIONS_PASSED=$((_BASHUNIT_ASSERTIONS_PASSED + assertions_passed))
-  _BASHUNIT_ASSERTIONS_FAILED=$((_BASHUNIT_ASSERTIONS_FAILED + assertions_failed))
-  _BASHUNIT_ASSERTIONS_SKIPPED=$((_BASHUNIT_ASSERTIONS_SKIPPED + assertions_skipped))
-  _BASHUNIT_ASSERTIONS_INCOMPLETE=$((_BASHUNIT_ASSERTIONS_INCOMPLETE + assertions_incomplete))
-  _BASHUNIT_ASSERTIONS_SNAPSHOT=$((_BASHUNIT_ASSERTIONS_SNAPSHOT + assertions_snapshot))
-  _BASHUNIT_TEST_EXIT_CODE=$((_BASHUNIT_TEST_EXIT_CODE + test_exit_code))
+  _BASHUNIT_ASSERTIONS_PASSED=$((_BASHUNIT_ASSERTIONS_PASSED + _BASHUNIT_RUNNER_COUNTS_PASSED_OUT))
+  _BASHUNIT_ASSERTIONS_FAILED=$((_BASHUNIT_ASSERTIONS_FAILED + _BASHUNIT_RUNNER_COUNTS_FAILED_OUT))
+  _BASHUNIT_ASSERTIONS_SKIPPED=$((_BASHUNIT_ASSERTIONS_SKIPPED + _BASHUNIT_RUNNER_COUNTS_SKIPPED_OUT))
+  _BASHUNIT_ASSERTIONS_INCOMPLETE=$((_BASHUNIT_ASSERTIONS_INCOMPLETE + _BASHUNIT_RUNNER_COUNTS_INCOMPLETE_OUT))
+  _BASHUNIT_ASSERTIONS_SNAPSHOT=$((_BASHUNIT_ASSERTIONS_SNAPSHOT + _BASHUNIT_RUNNER_COUNTS_SNAPSHOT_OUT))
+  _BASHUNIT_TEST_EXIT_CODE=$((_BASHUNIT_TEST_EXIT_CODE + _BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT))
 
   bashunit::internal_log "result_summary" \
-    "failed:$assertions_failed" \
-    "passed:$assertions_passed" \
-    "skipped:$assertions_skipped" \
-    "incomplete:$assertions_incomplete" \
-    "snapshot:$assertions_snapshot" \
-    "exit_code:$test_exit_code"
+    "failed:$_BASHUNIT_RUNNER_COUNTS_FAILED_OUT" \
+    "passed:$_BASHUNIT_RUNNER_COUNTS_PASSED_OUT" \
+    "skipped:$_BASHUNIT_RUNNER_COUNTS_SKIPPED_OUT" \
+    "incomplete:$_BASHUNIT_RUNNER_COUNTS_INCOMPLETE_OUT" \
+    "snapshot:$_BASHUNIT_RUNNER_COUNTS_SNAPSHOT_OUT" \
+    "exit_code:$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT"
 }
 
 function bashunit::runner::write_failure_result_output() {

tests/acceptance/bashunit_retry_test.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+function set_up_before_script() {
+  TEST_ENV_FILE="tests/acceptance/fixtures/.env.default"
+  FIXTURE="tests/acceptance/fixtures/test_bashunit_retry.sh"
+}
+
+function set_up() {
+  COUNTER_FILE="$(mktemp)"
+  export BASHUNIT_RETRY_FIXTURE_COUNTER="$COUNTER_FILE"
+  printf '0' >"$COUNTER_FILE"
+}
+
+function tear_down() {
+  rm -f "$COUNTER_FILE"
+  unset BASHUNIT_RETRY_FIXTURE_COUNTER BASHUNIT_RETRY_FIXTURE_PASS_ON
+}
+
+function test_bashunit_retry_recovers_a_flaky_test() {
+  export BASHUNIT_RETRY_FIXTURE_PASS_ON=2
+  local output
+  output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \
+    --retry 1 --filter test_a_flaky "$FIXTURE")"
+
+  assert_contains "1 passed" "$output"
+  assert_same "2" "$(cat "$COUNTER_FILE")"
+}
+
+function test_bashunit_retry_annotates_a_test_that_only_passed_on_retry() {
+  export BASHUNIT_RETRY_FIXTURE_PASS_ON=2
+  local output
+  output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \
+    --retry 1 --filter test_a_flaky "$FIXTURE")"
+
+  assert_contains "retry" "$output"
+}
+
+function test_bashunit_without_retry_a_flaky_test_fails() {
+  export BASHUNIT_RETRY_FIXTURE_PASS_ON=2
+  local output
+  output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \
+    --retry 0 --filter test_a_flaky "$FIXTURE")"
+
+  assert_contains "1 failed" "$output"
+  assert_same "1" "$(cat "$COUNTER_FILE")"
+}
+
+function test_bashunit_retry_gives_up_after_exhausting_attempts() {
+  export BASHUNIT_RETRY_FIXTURE_PASS_ON=99
+  local output
+  output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \
+    --retry 2 --filter test_a_flaky "$FIXTURE")"
+
+  # Counted once, not once per attempt.
+  assert_contains "1 failed" "$output"
+  # 1 initial run + 2 retries.
+  assert_same "3" "$(cat "$COUNTER_FILE")"
+}
+
+function test_bashunit_retry_recovers_a_flaky_test_in_parallel() {
+  export BASHUNIT_RETRY_FIXTURE_PASS_ON=2
+  local output
+  output="$(./bashunit --parallel --env "$TEST_ENV_FILE" \
+    --retry 1 --filter test_a_flaky "$FIXTURE")"
+
+  assert_contains "1 passed" "$output"
+}
+
+function test_bashunit_retry_defers_stop_on_failure_until_attempts_exhausted() {
+  export BASHUNIT_RETRY_FIXTURE_PASS_ON=2
+  local output
+  output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \
+    --stop-on-failure --retry 1 "$FIXTURE")"
+
+  # The flaky test recovers on retry, so stop-on-failure never fires and the
+  # later test still runs.
+  assert_contains "2 passed" "$output"
+}