diff --git a/CHANGELOG.md b/CHANGELOG.md index 68cbad22..830dd062 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- `--retry ` flag and `BASHUNIT_RETRY` env var to re-run a failed test up to N extra times (flaky-test mitigation); passes if any attempt passes, annotates tests that only passed on retry, and works with `--parallel` and `--stop-on-failure`. Disabled by default (#737) - `--report-tap ` writes a TAP version 13 report to a file (complements the streaming `--output tap`) (#740) - `assert_within_delta ` asserts a number is within a tolerance of another; supports floats (#744) - `assert_array_length ` asserts an array has exactly `n` elements (#743) diff --git a/docs/command-line.md b/docs/command-line.md index 8ae5dd9d..d2f3ef54 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -77,6 +77,7 @@ bashunit test tests/ --parallel --simple | `--detailed` | Detailed output (default) | | `-S, --stop-on-failure` | Stop on first failure | | `--test-timeout ` | Fail a test if it runs longer than N seconds | +| `--retry ` | Re-run a failed test up to N extra times | | `--show-skipped` | Show skipped tests summary at end | | `--show-incomplete` | Show incomplete tests summary at end | | `-vvv, --verbose` | Show execution details | @@ -461,6 +462,34 @@ Tests: 1 passed, 1 failed, 2 total It can also be set via the `BASHUNIT_TEST_TIMEOUT` environment variable (see [configuration](/configuration#test-timeout)). +### Retry + +> `bashunit test --retry ` + +Re-run a **failed** test up to `n` extra times and report it as passed if any +attempt passes; it only fails once every attempt has failed. This mitigates +flaky tests (timing, network or filesystem races) in CI without hiding a test +that is consistently broken. + +Retry is **disabled by default** (`0`). A test that recovered on retry is +annotated so the flakiness stays visible, retries apply per test, and it works +together with `--parallel` and `--stop-on-failure` (a test that recovers on +retry does not trigger stop-on-failure). + +::: code-group +```bash [Example] +bashunit test tests/ --retry 2 +``` +```[Output] +✓ Passed: A flaky test (retry 1/2) + +Tests: 1 passed, 1 total +``` +::: + +It can also be set via the `BASHUNIT_RETRY` environment variable (see +[configuration](/configuration#retry)). + ### No Progress > `bashunit test --no-progress` diff --git a/docs/configuration.md b/docs/configuration.md index 267e72d0..7b47c7d9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -151,6 +151,28 @@ BASHUNIT_TEST_TIMEOUT=0 ``` ::: +## Retry + +> `BASHUNIT_RETRY=` + +Re-run a failed test up to `n` extra times and report it as passed if any +attempt passes; it fails only after every attempt fails. `0` (disabled) by +default. Mitigates flaky tests in CI without hiding a consistently broken one; a +test that recovers on retry is annotated so the flakiness stays visible. + +Applies per test and works together with `--parallel` and `--stop-on-failure`. + +Similar as using `--retry` option on the [command line](/command-line#retry). + +::: code-group +```bash [Retry up to 2 times] +BASHUNIT_RETRY=2 +``` +```bash [Disabled (default)] +BASHUNIT_RETRY=0 +``` +::: + ## Stop on assertion failure > `BASHUNIT_STOP_ON_ASSERTION_FAILURE=true|false` diff --git a/src/console_header.sh b/src/console_header.sh index 947fdc66..221a8a9b 100644 --- a/src/console_header.sh +++ b/src/console_header.sh @@ -120,6 +120,7 @@ Options: -R, --run-all Run all assertions (don't stop on first failure) -S, --stop-on-failure Stop on first failure --test-timeout Fail a test if it runs longer than N seconds (0 = off) + --retry Re-run a failed test up to N extra times (0 = off) -vvv, --verbose Show execution details --debug [file] Enable shell debug mode --no-output Suppress all output diff --git a/src/console_results.sh b/src/console_results.sh index c3149814..1df9f03a 100644 --- a/src/console_results.sh +++ b/src/console_results.sh @@ -244,6 +244,10 @@ function bashunit::console_results::print_successful_test() { "$_BASHUNIT_COLOR_PASSED" "$_BASHUNIT_COLOR_DEFAULT" "$test_name" "$quoted_args") fi + # Retry annotation (e.g. " (retry 1/2)") set by the runner when a test only + # passed after retrying; empty in the common no-retry path. + line="${line}${_BASHUNIT_RETRY_NOTE:-}" + local full_line=$line if bashunit::env::is_show_execution_time_enabled; then local time_display diff --git a/src/env.sh b/src/env.sh index a20e7c5e..39d89998 100644 --- a/src/env.sh +++ b/src/env.sh @@ -119,6 +119,8 @@ _BASHUNIT_DEFAULT_PROFILE="false" _BASHUNIT_DEFAULT_PROFILE_COUNT="10" # Per-test timeout in seconds (0 = disabled) _BASHUNIT_DEFAULT_TEST_TIMEOUT="0" +# Extra attempts for a failed test (0 = no retry) +_BASHUNIT_DEFAULT_RETRY="0" : "${BASHUNIT_PARALLEL_RUN:=${PARALLEL_RUN:=$_BASHUNIT_DEFAULT_PARALLEL_RUN}}" : "${BASHUNIT_PARALLEL_JOBS:=0}" @@ -145,6 +147,9 @@ _BASHUNIT_DEFAULT_TEST_TIMEOUT="0" : "${BASHUNIT_PROFILE:=${PROFILE:=$_BASHUNIT_DEFAULT_PROFILE}}" : "${BASHUNIT_PROFILE_COUNT:=${PROFILE_COUNT:=$_BASHUNIT_DEFAULT_PROFILE_COUNT}}" : "${BASHUNIT_TEST_TIMEOUT:=${TEST_TIMEOUT:=$_BASHUNIT_DEFAULT_TEST_TIMEOUT}}" +# No bare RETRY alias on purpose: it is too generic and would pick up unrelated +# environment values. Only BASHUNIT_RETRY configures retries. +: "${BASHUNIT_RETRY:=$_BASHUNIT_DEFAULT_RETRY}" # Support NO_COLOR standard (https://no-color.org) if [ -n "${NO_COLOR:-}" ]; then BASHUNIT_NO_COLOR="true" @@ -174,6 +179,20 @@ function bashunit::env::test_timeout_secs() { printf '%s' "${BASHUNIT_TEST_TIMEOUT:-0}" } +## +# Prints the number of extra attempts for a failed test (0 = no retry). +# A non-numeric value is treated as 0. +## +function bashunit::env::retry_count() { + case "${BASHUNIT_RETRY:-0}" in + '' | *[!0-9]*) + printf '%s' "0" + return + ;; + esac + printf '%s' "${BASHUNIT_RETRY:-0}" +} + function bashunit::env::is_show_header_enabled() { [ "$BASHUNIT_SHOW_HEADER" = "true" ] } diff --git a/src/main.sh b/src/main.sh index 84af70d8..0457c63d 100644 --- a/src/main.sh +++ b/src/main.sh @@ -78,6 +78,10 @@ function bashunit::main::cmd_test() { export BASHUNIT_TEST_TIMEOUT="$2" shift ;; + --retry) + export BASHUNIT_RETRY="$2" + shift + ;; -w | --watch) export BASHUNIT_WATCH_MODE=true ;; diff --git a/src/runner.sh b/src/runner.sh index dd91eb42..28aa6688 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -131,6 +131,14 @@ _BASHUNIT_RUNNER_TOTAL_OUT="" _BASHUNIT_RUNNER_TYPE_OUT="" _BASHUNIT_RUNNER_OUTPUT_OUT="" _BASHUNIT_RUNNER_INTERP_OUT="" +_BASHUNIT_RUNNER_COUNTS_FAILED_OUT=0 +_BASHUNIT_RUNNER_COUNTS_PASSED_OUT=0 +_BASHUNIT_RUNNER_COUNTS_SKIPPED_OUT=0 +_BASHUNIT_RUNNER_COUNTS_INCOMPLETE_OUT=0 +_BASHUNIT_RUNNER_COUNTS_SNAPSHOT_OUT=0 +_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT=0 +# Suffix appended to a passed-test line when it only passed after retrying. +_BASHUNIT_RETRY_NOTE="" # Writes the value of an encoded field (##KEY=value##) into _BASHUNIT_RUNNER_FIELD_OUT. # Arguments: $1 test_execution_result, $2 key @@ -1000,7 +1008,6 @@ function bashunit::runner::run_with_timeout() { function bashunit::runner::run_test() { local start_time - start_time=$(bashunit::clock::now) local test_file="$1" shift @@ -1025,13 +1032,37 @@ function bashunit::runner::run_test() { local test_execution_result local timed_out="false" - if bashunit::env::is_test_timeout_enabled; then - bashunit::runner::run_with_timeout "$test_file" "$fn_name" "$@" - test_execution_result="$_BASHUNIT_RUNNER_EXEC_OUT" - timed_out="$_BASHUNIT_RUNNER_TIMED_OUT" - else - test_execution_result=$(bashunit::runner::execute_test_body "$test_file" "$fn_name" "$@") - fi + local retry_max + retry_max=$(bashunit::env::retry_count) + local retries_used=0 + # Retry wraps ONLY execution: a failed attempt is judged from its encoded + # result without committing, so the parse/report/counter path below still runs + # exactly once (on the final attempt) and nothing is double-counted. Each fork + # in --parallel retries itself before writing its single .result file. + while :; do + start_time=$(bashunit::clock::now) + if bashunit::env::is_test_timeout_enabled; then + bashunit::runner::run_with_timeout "$test_file" "$fn_name" "$@" + test_execution_result="$_BASHUNIT_RUNNER_EXEC_OUT" + timed_out="$_BASHUNIT_RUNNER_TIMED_OUT" + else + test_execution_result=$(bashunit::runner::execute_test_body "$test_file" "$fn_name" "$@") + fi + + local attempt_runtime_output="${test_execution_result%%##ASSERTIONS_*}" + local attempt_runtime_error + attempt_runtime_error=$(bashunit::runner::detect_runtime_error "$attempt_runtime_output") + bashunit::runner::extract_result_counts "$test_execution_result" + # Mirror the commit-phase failure test exactly (runtime error, non-zero exit, + # or a failed assertion); snapshot/incomplete/skipped/risky are not failures. + if [ -z "$attempt_runtime_error" ] && + [ "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" -eq 0 ] && + [ "$_BASHUNIT_RUNNER_COUNTS_FAILED_OUT" -eq 0 ]; then + break + fi + [ "$retries_used" -ge "$retry_max" ] && break + retries_used=$((retries_used + 1)) + done # Closes FD 3, which was used temporarily to hold the original stdout. exec 3>&- @@ -1220,6 +1251,11 @@ function bashunit::runner::run_test() { return fi + # A test that only passed after retrying is annotated so flakiness stays visible. + _BASHUNIT_RETRY_NOTE="" + if [ "$retries_used" -gt 0 ]; then + _BASHUNIT_RETRY_NOTE=" (retry $retries_used/$retry_max)" + fi # In failures-only mode, suppress successful test output if ! bashunit::env::is_failures_only_enabled; then if [ "$fn_name" = "$interpolated_fn_name" ]; then @@ -1228,6 +1264,7 @@ function bashunit::runner::run_test() { bashunit::console_results::print_successful_test "${label}" "$duration" fi fi + _BASHUNIT_RETRY_NOTE="" bashunit::state::add_tests_passed bashunit::reports::add_test_passed "$test_file" "$label" "$duration" "$total_assertions" bashunit::internal_log "Test passed" "$label" @@ -1412,9 +1449,14 @@ function bashunit::runner::parse_result_parallel() { } # shellcheck disable=SC2295 -function bashunit::runner::parse_result_sync() { - local fn_name=$1 - local execution_result=$2 +## +# Parses the encoded per-test result's last line into the counts out-slots +# (_BASHUNIT_RUNNER_COUNTS_*_OUT). Pure read: never mutates the cumulative +# _BASHUNIT_ASSERTIONS_* / _BASHUNIT_TEST_EXIT_CODE state, so the retry loop can +# judge an attempt's outcome without committing it. +## +function bashunit::runner::extract_result_counts() { + local execution_result=$1 local result_line result_line="${execution_result##*$'\n'}" @@ -1453,22 +1495,36 @@ function bashunit::runner::parse_result_sync() { ;; esac + _BASHUNIT_RUNNER_COUNTS_FAILED_OUT=$assertions_failed + _BASHUNIT_RUNNER_COUNTS_PASSED_OUT=$assertions_passed + _BASHUNIT_RUNNER_COUNTS_SKIPPED_OUT=$assertions_skipped + _BASHUNIT_RUNNER_COUNTS_INCOMPLETE_OUT=$assertions_incomplete + _BASHUNIT_RUNNER_COUNTS_SNAPSHOT_OUT=$assertions_snapshot + _BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT=$test_exit_code +} + +function bashunit::runner::parse_result_sync() { + local fn_name=$1 + local execution_result=$2 + + bashunit::runner::extract_result_counts "$execution_result" + bashunit::internal_log "[SYNC]" "fn_name:$fn_name" "execution_result:$execution_result" - _BASHUNIT_ASSERTIONS_PASSED=$((_BASHUNIT_ASSERTIONS_PASSED + assertions_passed)) - _BASHUNIT_ASSERTIONS_FAILED=$((_BASHUNIT_ASSERTIONS_FAILED + assertions_failed)) - _BASHUNIT_ASSERTIONS_SKIPPED=$((_BASHUNIT_ASSERTIONS_SKIPPED + assertions_skipped)) - _BASHUNIT_ASSERTIONS_INCOMPLETE=$((_BASHUNIT_ASSERTIONS_INCOMPLETE + assertions_incomplete)) - _BASHUNIT_ASSERTIONS_SNAPSHOT=$((_BASHUNIT_ASSERTIONS_SNAPSHOT + assertions_snapshot)) - _BASHUNIT_TEST_EXIT_CODE=$((_BASHUNIT_TEST_EXIT_CODE + test_exit_code)) + _BASHUNIT_ASSERTIONS_PASSED=$((_BASHUNIT_ASSERTIONS_PASSED + _BASHUNIT_RUNNER_COUNTS_PASSED_OUT)) + _BASHUNIT_ASSERTIONS_FAILED=$((_BASHUNIT_ASSERTIONS_FAILED + _BASHUNIT_RUNNER_COUNTS_FAILED_OUT)) + _BASHUNIT_ASSERTIONS_SKIPPED=$((_BASHUNIT_ASSERTIONS_SKIPPED + _BASHUNIT_RUNNER_COUNTS_SKIPPED_OUT)) + _BASHUNIT_ASSERTIONS_INCOMPLETE=$((_BASHUNIT_ASSERTIONS_INCOMPLETE + _BASHUNIT_RUNNER_COUNTS_INCOMPLETE_OUT)) + _BASHUNIT_ASSERTIONS_SNAPSHOT=$((_BASHUNIT_ASSERTIONS_SNAPSHOT + _BASHUNIT_RUNNER_COUNTS_SNAPSHOT_OUT)) + _BASHUNIT_TEST_EXIT_CODE=$((_BASHUNIT_TEST_EXIT_CODE + _BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT)) bashunit::internal_log "result_summary" \ - "failed:$assertions_failed" \ - "passed:$assertions_passed" \ - "skipped:$assertions_skipped" \ - "incomplete:$assertions_incomplete" \ - "snapshot:$assertions_snapshot" \ - "exit_code:$test_exit_code" + "failed:$_BASHUNIT_RUNNER_COUNTS_FAILED_OUT" \ + "passed:$_BASHUNIT_RUNNER_COUNTS_PASSED_OUT" \ + "skipped:$_BASHUNIT_RUNNER_COUNTS_SKIPPED_OUT" \ + "incomplete:$_BASHUNIT_RUNNER_COUNTS_INCOMPLETE_OUT" \ + "snapshot:$_BASHUNIT_RUNNER_COUNTS_SNAPSHOT_OUT" \ + "exit_code:$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" } function bashunit::runner::write_failure_result_output() { diff --git a/tests/acceptance/bashunit_retry_test.sh b/tests/acceptance/bashunit_retry_test.sh new file mode 100644 index 00000000..ba826f32 --- /dev/null +++ b/tests/acceptance/bashunit_retry_test.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +function set_up_before_script() { + TEST_ENV_FILE="tests/acceptance/fixtures/.env.default" + FIXTURE="tests/acceptance/fixtures/test_bashunit_retry.sh" +} + +function set_up() { + COUNTER_FILE="$(mktemp)" + export BASHUNIT_RETRY_FIXTURE_COUNTER="$COUNTER_FILE" + printf '0' >"$COUNTER_FILE" +} + +function tear_down() { + rm -f "$COUNTER_FILE" + unset BASHUNIT_RETRY_FIXTURE_COUNTER BASHUNIT_RETRY_FIXTURE_PASS_ON +} + +function test_bashunit_retry_recovers_a_flaky_test() { + export BASHUNIT_RETRY_FIXTURE_PASS_ON=2 + local output + output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \ + --retry 1 --filter test_a_flaky "$FIXTURE")" + + assert_contains "1 passed" "$output" + assert_same "2" "$(cat "$COUNTER_FILE")" +} + +function test_bashunit_retry_annotates_a_test_that_only_passed_on_retry() { + export BASHUNIT_RETRY_FIXTURE_PASS_ON=2 + local output + output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \ + --retry 1 --filter test_a_flaky "$FIXTURE")" + + assert_contains "retry" "$output" +} + +function test_bashunit_without_retry_a_flaky_test_fails() { + export BASHUNIT_RETRY_FIXTURE_PASS_ON=2 + local output + output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \ + --retry 0 --filter test_a_flaky "$FIXTURE")" + + assert_contains "1 failed" "$output" + assert_same "1" "$(cat "$COUNTER_FILE")" +} + +function test_bashunit_retry_gives_up_after_exhausting_attempts() { + export BASHUNIT_RETRY_FIXTURE_PASS_ON=99 + local output + output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \ + --retry 2 --filter test_a_flaky "$FIXTURE")" + + # Counted once, not once per attempt. + assert_contains "1 failed" "$output" + # 1 initial run + 2 retries. + assert_same "3" "$(cat "$COUNTER_FILE")" +} + +function test_bashunit_retry_recovers_a_flaky_test_in_parallel() { + export BASHUNIT_RETRY_FIXTURE_PASS_ON=2 + local output + output="$(./bashunit --parallel --env "$TEST_ENV_FILE" \ + --retry 1 --filter test_a_flaky "$FIXTURE")" + + assert_contains "1 passed" "$output" +} + +function test_bashunit_retry_defers_stop_on_failure_until_attempts_exhausted() { + export BASHUNIT_RETRY_FIXTURE_PASS_ON=2 + local output + output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \ + --stop-on-failure --retry 1 "$FIXTURE")" + + # The flaky test recovers on retry, so stop-on-failure never fires and the + # later test still runs. + assert_contains "2 passed" "$output" +} diff --git a/tests/acceptance/fixtures/test_bashunit_retry.sh b/tests/acceptance/fixtures/test_bashunit_retry.sh new file mode 100644 index 00000000..95b6652d --- /dev/null +++ b/tests/acceptance/fixtures/test_bashunit_retry.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Deterministic stand-in for a flaky test: a persistent counter file survives +# across retry attempts (each attempt re-runs the body), so the test fails until +# the attempt count reaches BASHUNIT_RETRY_FIXTURE_PASS_ON. +function test_a_flaky_until_nth_attempt() { + local counter_file="${BASHUNIT_RETRY_FIXTURE_COUNTER:?counter file required}" + local pass_on="${BASHUNIT_RETRY_FIXTURE_PASS_ON:-2}" + + local attempts + attempts=$(cat "$counter_file" 2>/dev/null || echo 0) + attempts=$((attempts + 1)) + printf '%s' "$attempts" >"$counter_file" + + if [ "$attempts" -ge "$pass_on" ]; then + assert_same "ok" "ok" + else + assert_same "pass-on-attempt-$pass_on" "failed-on-attempt-$attempts" + fi +} + +function test_b_always_passes() { + assert_same "ran" "ran" +} diff --git a/tests/unit/env_test.sh b/tests/unit/env_test.sh index 19689277..64c773ae 100644 --- a/tests/unit/env_test.sh +++ b/tests/unit/env_test.sh @@ -96,7 +96,7 @@ function test_is_dev_mode_disabled_when_dev_log_empty() { } # @data_provider provide_test_timeout_enabled -function test_is_test_timeout_enabled(){ +function test_is_test_timeout_enabled() { local value="$1" local expected="$2" @@ -132,6 +132,28 @@ function test_test_timeout_secs_returns_the_configured_value() { assert_equals "7" "$result" } +function test_retry_count_returns_the_configured_value() { + local original="${BASHUNIT_RETRY:-0}" + export BASHUNIT_RETRY="3" + + local result + result=$(bashunit::env::retry_count) + + export BASHUNIT_RETRY="$original" + assert_equals "3" "$result" +} + +function test_retry_count_treats_non_numeric_as_zero() { + local original="${BASHUNIT_RETRY:-0}" + export BASHUNIT_RETRY="abc" + + local result + result=$(bashunit::env::retry_count) + + export BASHUNIT_RETRY="$original" + assert_equals "0" "$result" +} + function test_is_tap_output_enabled_when_format_is_tap() { local original="$BASHUNIT_OUTPUT_FORMAT" export BASHUNIT_OUTPUT_FORMAT="tap" diff --git a/tests/unit/runner_test.sh b/tests/unit/runner_test.sh index 62734c26..e4c04507 100644 --- a/tests/unit/runner_test.sh +++ b/tests/unit/runner_test.sh @@ -155,6 +155,43 @@ function test_compute_total_assertions_treats_missing_counters_as_zero() { assert_same "2" "$_BASHUNIT_RUNNER_TOTAL_OUT" } +# Builds a one-line encoded test result like execute_test_body emits. +# Args: failed passed skipped incomplete snapshot exit_code +function build_encoded_result() { + local out="##ASSERTIONS_FAILED=$1##ASSERTIONS_PASSED=$2" + out="$out##ASSERTIONS_SKIPPED=$3##ASSERTIONS_INCOMPLETE=$4" + out="$out##ASSERTIONS_SNAPSHOT=$5##TEST_EXIT_CODE=$6##" + printf '%s' "$out" +} + +function test_extract_result_counts_writes_counts_to_slots() { + bashunit::runner::extract_result_counts "$(build_encoded_result 2 3 0 0 0 5)" + + assert_same "2" "$_BASHUNIT_RUNNER_COUNTS_FAILED_OUT" + assert_same "3" "$_BASHUNIT_RUNNER_COUNTS_PASSED_OUT" + assert_same "5" "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" +} + +function test_extract_result_counts_does_not_mutate_cumulative_state() { + local before_failed="$_BASHUNIT_ASSERTIONS_FAILED" + local before_exit="$_BASHUNIT_TEST_EXIT_CODE" + + bashunit::runner::extract_result_counts "$(build_encoded_result 9 9 0 0 0 1)" + + assert_same "$before_failed" "$_BASHUNIT_ASSERTIONS_FAILED" + assert_same "$before_exit" "$_BASHUNIT_TEST_EXIT_CODE" +} + +function test_extract_result_counts_reads_only_the_last_line() { + local result + result="user output mentioning ASSERTIONS_FAILED=7 should be ignored +$(build_encoded_result 1 0 0 0 0 0)" + + bashunit::runner::extract_result_counts "$result" + + assert_same "1" "$_BASHUNIT_RUNNER_COUNTS_FAILED_OUT" +} + function test_extract_subshell_type_strips_brackets_into_slot() { bashunit::runner::extract_subshell_type "[failed] something happened"