diff --git a/CHANGELOG.md b/CHANGELOG.md index c342e35b..389f329b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- `--report-json ` writes machine-readable JSON results (summary counts + per-test records) with pure-Bash string escaping (no `jq` dependency) (#741) - `--shard /` to run a deterministic, round-robin subset of the test files so a suite can be split across parallel CI runners (union of all shards is the full suite, no overlap); validates input and composes with `--parallel` (#739) - `--random-order` flag with `--seed ` / `BASHUNIT_SEED` to randomize test file and function execution order (surfaces inter-test coupling); the seed is printed for replay and the shuffle is reproducible and works with `--parallel`. Disabled by default (#738) - `--retry ` flag and `BASHUNIT_RETRY` env var to re-run a failed test up to N extra times (flaky-test mitigation); passes if any attempt passes, annotates tests that only passed on retry, and works with `--parallel` and `--stop-on-failure`. Disabled by default (#737) diff --git a/docs/command-line.md b/docs/command-line.md index 75a4268d..9fcfa4b1 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -72,6 +72,7 @@ bashunit test tests/ --parallel --simple | `--no-parallel` | Run tests sequentially | | `-r, --report-html ` | Write HTML report | | `--report-tap ` | Write TAP version 13 report to a file | +| `--report-json ` | Write machine-readable JSON report to a file | | `-R, --run-all` | Run all assertions (don't stop on first failure) | | `-s, --simple` | Simple output (dots) | | `--detailed` | Detailed output (default) | @@ -373,10 +374,27 @@ bashunit test tests/ --report-html report.html # Stream annotations straight to the runner log: bashunit test tests/ --log-gha /dev/stdout ``` +```bash [JSON] +bashunit test tests/ --report-json report.json +``` ::: The `--log-gha` flag writes GitHub Actions workflow commands (`::error`, `::warning`, `::notice`) for failed, risky and incomplete tests, including the failing test's `file` and `line`. Point it at `/dev/stdout` (or stream a log file to stdout) on a runner and the failures appear as inline annotations in the "Files changed" tab of a pull request. +The `--report-json` flag writes machine-readable results for scripts, dashboards and bots. Strings are escaped in pure Bash, so no `jq` is needed to produce it. Its schema is: + +```json +{ + "summary": { "total": 3, "passed": 2, "failed": 1, "skipped": 0, "incomplete": 0, "duration_ms": 42 }, + "tests": [ + { "file": "tests/math_test.sh", "name": "it adds", "status": "passed", "duration_ms": 5, "message": "" }, + { "file": "tests/math_test.sh", "name": "it divides", "status": "failed", "duration_ms": 3, "message": "Expected 2 but got 3" } + ] +} +``` + +`status` is one of `passed`, `failed`, `skipped`, `incomplete` (`snapshot` and `risky` are also emitted per test and counted as passed in the summary). Like the other file reporters, per-test rows come from a sequential run; under `--parallel` the file is still valid JSON. + ### Show Output on Failure > `bashunit test --show-output` diff --git a/src/console_header.sh b/src/console_header.sh index c401838c..b0655722 100644 --- a/src/console_header.sh +++ b/src/console_header.sh @@ -124,6 +124,7 @@ Options: --no-parallel Run tests sequentially -r, --report-html Write HTML report --report-tap Write TAP version 13 report + --report-json Write machine-readable JSON report -s, --simple Simple output (dots) --detailed Detailed output (default) --output Output format: tap (TAP version 13) diff --git a/src/env.sh b/src/env.sh index f86fc815..7e1c646e 100644 --- a/src/env.sh +++ b/src/env.sh @@ -62,6 +62,7 @@ _BASHUNIT_DEFAULT_LOG_JUNIT="" _BASHUNIT_DEFAULT_LOG_GHA="" _BASHUNIT_DEFAULT_REPORT_HTML="" _BASHUNIT_DEFAULT_REPORT_TAP="" +_BASHUNIT_DEFAULT_REPORT_JSON="" # Coverage defaults (following kcov, bashcov, SimpleCov conventions) _BASHUNIT_DEFAULT_COVERAGE="false" @@ -81,6 +82,7 @@ _BASHUNIT_DEFAULT_COVERAGE_THRESHOLD_HIGH="80" : "${BASHUNIT_LOG_GHA:=${LOG_GHA:=$_BASHUNIT_DEFAULT_LOG_GHA}}" : "${BASHUNIT_REPORT_HTML:=${REPORT_HTML:=$_BASHUNIT_DEFAULT_REPORT_HTML}}" : "${BASHUNIT_REPORT_TAP:=${REPORT_TAP:=$_BASHUNIT_DEFAULT_REPORT_TAP}}" +: "${BASHUNIT_REPORT_JSON:=${REPORT_JSON:=$_BASHUNIT_DEFAULT_REPORT_JSON}}" # Coverage : "${BASHUNIT_COVERAGE:=${COVERAGE:=$_BASHUNIT_DEFAULT_COVERAGE}}" diff --git a/src/main.sh b/src/main.sh index 8614db95..e69b4ba1 100644 --- a/src/main.sh +++ b/src/main.sh @@ -158,6 +158,10 @@ function bashunit::main::cmd_test() { export BASHUNIT_REPORT_TAP="$2" shift ;; + --report-json) + export BASHUNIT_REPORT_JSON="$2" + shift + ;; --no-output) export BASHUNIT_NO_OUTPUT=true ;; @@ -822,6 +826,10 @@ function bashunit::main::exec_tests() { bashunit::reports::generate_report_tap "$BASHUNIT_REPORT_TAP" fi + if [ -n "$BASHUNIT_REPORT_JSON" ]; then + bashunit::reports::generate_report_json "$BASHUNIT_REPORT_JSON" + fi + # Generate coverage report if enabled if bashunit::env::is_coverage_enabled; then # Aggregate per-process coverage data from parallel runs diff --git a/src/reports.sh b/src/reports.sh index 0fb97e76..9a479a1d 100755 --- a/src/reports.sh +++ b/src/reports.sh @@ -39,7 +39,8 @@ function bashunit::reports::add_test() { [ -n "${BASHUNIT_LOG_JUNIT:-}" ] || [ -n "${BASHUNIT_REPORT_HTML:-}" ] || [ -n "${BASHUNIT_LOG_GHA:-}" ] || - [ -n "${BASHUNIT_REPORT_TAP:-}" ] + [ -n "${BASHUNIT_REPORT_TAP:-}" ] || + [ -n "${BASHUNIT_REPORT_JSON:-}" ] } || return 0 local file="$1" @@ -76,6 +77,20 @@ function bashunit::reports::__xml_escape() { | sed -e 's/&/\&/g' -e 's//\>/g' -e 's/"/\"/g' -e "s/'/\'/g" } +# Escapes a string for embedding in a JSON string literal (pure Bash, no jq). +# Strips ANSI/control chars that cannot appear inline, keeps \t\r\n as escapes. +function bashunit::reports::__json_escape() { + local text="$1" + text=$(printf '%s' "$text" | sed -e 's/\x1b\[[0-9;]*[a-zA-Z]//g' | tr -d '\000-\010\013\014\016-\037') + # Backslash first so escapes added below are not doubled. + text="${text//\\/\\\\}" + text="${text//\"/\\\"}" + text="${text//$'\t'/\\t}" + text="${text//$'\r'/\\r}" + text="${text//$'\n'/\\n}" + printf '%s' "$text" +} + function bashunit::reports::generate_junit_xml() { local output_file="$1" @@ -186,6 +201,50 @@ function bashunit::reports::generate_report_tap() { } >"$output_file" } +function bashunit::reports::generate_report_json() { + local output_file="$1" + local total="${#_BASHUNIT_REPORTS_TEST_NAMES[@]}" + + local passed=0 failed=0 skipped=0 incomplete=0 duration_total=0 + local i + for i in "${!_BASHUNIT_REPORTS_TEST_NAMES[@]}"; do + duration_total=$((duration_total + ${_BASHUNIT_REPORTS_TEST_DURATIONS[$i]:-0})) + case "${_BASHUNIT_REPORTS_TEST_STATUSES[$i]:-}" in + failed) failed=$((failed + 1)) ;; + skipped) skipped=$((skipped + 1)) ;; + incomplete) incomplete=$((incomplete + 1)) ;; + # snapshot and risky ran without failing, so they count as passed here; the + # per-test "status" field below preserves the exact category. + *) passed=$((passed + 1)) ;; + esac + done + + { + printf '{\n' + printf ' "summary": { "total": %d, "passed": %d, "failed": %d,' \ + "$total" "$passed" "$failed" + printf ' "skipped": %d, "incomplete": %d, "duration_ms": %d },\n' \ + "$skipped" "$incomplete" "$duration_total" + printf ' "tests": [\n' + local seq=0 + for i in "${!_BASHUNIT_REPORTS_TEST_NAMES[@]}"; do + local file name status duration message sep + file=$(bashunit::reports::__json_escape "${_BASHUNIT_REPORTS_TEST_FILES[$i]:-}") + name=$(bashunit::reports::__json_escape "${_BASHUNIT_REPORTS_TEST_NAMES[$i]:-}") + status="${_BASHUNIT_REPORTS_TEST_STATUSES[$i]:-}" + duration="${_BASHUNIT_REPORTS_TEST_DURATIONS[$i]:-0}" + message=$(bashunit::reports::__json_escape "${_BASHUNIT_REPORTS_TEST_FAILURES[$i]:-}") + sep="," + [ "$seq" -eq "$((total - 1))" ] && sep="" + printf ' { "file": "%s", "name": "%s", "status": "%s", "duration_ms": %d, "message": "%s" }%s\n' \ + "$file" "$name" "$status" "$duration" "$message" "$sep" + seq=$((seq + 1)) + done + printf ' ]\n' + printf '}\n' + } >"$output_file" +} + function bashunit::reports::__gha_encode() { local text="$1" # Strip ANSI escape sequences first (one sed call) diff --git a/tests/acceptance/bashunit_report_json_test.sh b/tests/acceptance/bashunit_report_json_test.sh new file mode 100644 index 00000000..a7fda219 --- /dev/null +++ b/tests/acceptance/bashunit_report_json_test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +function set_up_before_script() { + TEST_ENV_FILE="tests/acceptance/fixtures/.env.default" + FIXTURE="tests/acceptance/fixtures/test_bashunit_report_json.sh" + JQ_AVAILABLE=false + command -v jq >/dev/null 2>&1 && JQ_AVAILABLE=true +} + +function test_report_json_writes_valid_json_with_correct_counts() { + if [ "$JQ_AVAILABLE" = false ]; then bashunit::skip "jq required"; return; fi + local report + report="$(mktemp)" + ./bashunit --no-parallel --env "$TEST_ENV_FILE" --report-json "$report" "$FIXTURE" >/dev/null 2>&1 + + assert_successful_code "$(jq empty "$report" 2>&1)" + assert_same "2" "$(jq '.summary.total' "$report")" + assert_same "1" "$(jq '.summary.passed' "$report")" + assert_same "1" "$(jq '.summary.failed' "$report")" + rm -f "$report" +} + +function test_report_json_escapes_special_characters_in_messages() { + if [ "$JQ_AVAILABLE" = false ]; then bashunit::skip "jq required"; return; fi + local report + report="$(mktemp)" + ./bashunit --no-parallel --env "$TEST_ENV_FILE" --report-json "$report" "$FIXTURE" >/dev/null 2>&1 + + # A double quote inside the failure message must round-trip as valid JSON. + local message + message="$(jq -r '.tests[] | select(.status == "failed") | .message' "$report")" + assert_contains 'a"b' "$message" + rm -f "$report" +} + +# Under --parallel the per-test rows are not aggregated (a pre-existing limit +# shared by all file reporters), but the output must still be valid JSON. +function test_report_json_is_valid_json_under_parallel() { + if [ "$JQ_AVAILABLE" = false ]; then bashunit::skip "jq required"; return; fi + local report + report="$(mktemp)" + ./bashunit --parallel --env "$TEST_ENV_FILE" --report-json "$report" "$FIXTURE" >/dev/null 2>&1 + + assert_successful_code "$(jq empty "$report" 2>&1)" + rm -f "$report" +} + +function test_report_json_is_not_written_without_the_flag() { + local report + report="$(mktemp)" + rm -f "$report" + ./bashunit --no-parallel --env "$TEST_ENV_FILE" "$FIXTURE" >/dev/null 2>&1 + + assert_file_not_exists "$report" +} diff --git a/tests/acceptance/fixtures/test_bashunit_report_json.sh b/tests/acceptance/fixtures/test_bashunit_report_json.sh new file mode 100644 index 00000000..85b1e871 --- /dev/null +++ b/tests/acceptance/fixtures/test_bashunit_report_json.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +function test_json_pass() { + assert_same "ok" "ok" +} + +# The failure message contains a double quote, exercising JSON string escaping. +function test_json_fail_with_quote() { + assert_same 'a"b' 'c' +} diff --git a/tests/unit/reports_json_test.sh b/tests/unit/reports_json_test.sh new file mode 100644 index 00000000..8a2870bb --- /dev/null +++ b/tests/unit/reports_json_test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2329,SC2034 + +_JQ_AVAILABLE=false +if command -v jq >/dev/null 2>&1; then + _JQ_AVAILABLE=true +fi + +function test_json_escape_escapes_quotes_and_backslashes() { + assert_same 'a\"b\\c' "$(bashunit::reports::__json_escape 'a"b\c')" +} + +function test_json_escape_escapes_newlines_and_tabs() { + assert_same 'a\tb\nc' "$(bashunit::reports::__json_escape "$(printf 'a\tb\nc')")" +} + +function test_generate_report_json_summary_counts() { + if [ "$_JQ_AVAILABLE" = false ]; then bashunit::skip "jq required"; return; fi + local out + out="$(mktemp)" + set_up_report_fixture + bashunit::reports::generate_report_json "$out" + + assert_same "2" "$(jq '.summary.total' "$out")" + assert_same "1" "$(jq '.summary.passed' "$out")" + assert_same "1" "$(jq '.summary.failed' "$out")" + rm -f "$out" +} + +function test_generate_report_json_is_valid_and_escapes_messages() { + if [ "$_JQ_AVAILABLE" = false ]; then bashunit::skip "jq required"; return; fi + local out + out="$(mktemp)" + set_up_report_fixture + bashunit::reports::generate_report_json "$out" + + # jq parsing succeeds only if the embedded quote AND newline were escaped + # correctly; asserting the quote substring avoids a Windows CRLF round-trip. + assert_successful_code "$(jq empty "$out" 2>&1)" + assert_same 'failed' "$(jq -r '.tests[1].status' "$out")" + assert_contains 'say "hi"' "$(jq -r '.tests[1].message' "$out")" + rm -f "$out" +} + +# Populates the reports arrays with one passed and one failed test; the failed +# message contains a quote and a newline to exercise escaping. +function set_up_report_fixture() { + _BASHUNIT_REPORTS_TEST_FILES=("tests/math_test.sh" "tests/math_test.sh") + _BASHUNIT_REPORTS_TEST_NAMES=("it adds" "it divides") + _BASHUNIT_REPORTS_TEST_STATUSES=("passed" "failed") + _BASHUNIT_REPORTS_TEST_DURATIONS=("5" "3") + _BASHUNIT_REPORTS_TEST_ASSERTIONS=("1" "1") + _BASHUNIT_REPORTS_TEST_FAILURES=("" "$(printf 'say "hi"\nnext')") + _BASHUNIT_REPORTS_TEST_LINES=("10" "20") +}