From 4c58345a0d848876da621fa9a8c3f04fdcd0407e Mon Sep 17 00:00:00 2001 From: Michael Pawliszyn Date: Wed, 25 Feb 2026 21:34:34 -0500 Subject: [PATCH] feat: add coverage-report.sh with tests Shell script to generate a coverage summary from review-tree.md. Read-only, no file mutations. Output is structured markdown with a documented format contract for the orchestrator. Reports: - Node counts: pending, reviewed, accepted, total - Progress: decided/total with percentage - Confidence: examined in detail (reviewed) vs pattern-trusted (accepted) - Nodes with comments (flag-precise pattern), top-level concept count - File coverage: files in diff, files mapped, unmapped - Pending nodes list (only shown if any remain) Fixes from review: - Node-counting greps require [0-9] after status to prevent context block inflation (I1) - Comment flag pattern uses \{[^}]*comment\} for precision - File coverage data extracted from tree's Coverage section (E1) - Progress percentage added (E2) - Output format documented in script header as contract 18 bats tests covering: - Status counts, progress percentage, comment count - Confidence summary, top-level count, file coverage - Pending nodes (listed when present, hidden when none) - Section headers, input validation - Context inflation protection (regression test) - Edge cases (all pending with 0%, all reviewed with 100%) Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/coverage-report.sh | 116 ++++++++ tests/scripts/test-coverage-report.bats | 366 ++++++++++++++++++++++++ 2 files changed, 482 insertions(+) create mode 100755 scripts/coverage-report.sh create mode 100755 tests/scripts/test-coverage-report.bats diff --git a/scripts/coverage-report.sh b/scripts/coverage-report.sh new file mode 100755 index 0000000..6fbe32c --- /dev/null +++ b/scripts/coverage-report.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# +# coverage-report.sh -- Generate a coverage summary from a review-tree.md file. +# +# Usage: coverage-report.sh +# +# Output: Structured markdown text suitable for the orchestrator to present +# to the customer. The output format is a contract -- the orchestrator parses +# these field names and section headers. +# +# Output format: +# ## Status +# pending: +# reviewed: +# accepted: +# total: +# decided: / (%) +# with comments: +# examined in detail: +# pattern-trusted: +# top-level concepts: +# ## Files +# files in diff: +# files mapped: +# unmapped: +# ## Pending (only if pending > 0) +# Pending nodes: +# +# +# Exit codes: +# 0 -- success +# 1 -- invalid arguments or file not found + +set -euo pipefail + +# --- Input validation --- + +if [ $# -lt 1 ]; then + echo "Usage: coverage-report.sh " >&2 + exit 1 +fi + +TREE_FILE="$1" + +if [ ! -f "$TREE_FILE" ]; then + echo "Error: file not found: $TREE_FILE" >&2 + exit 1 +fi + +# --- Count nodes by status --- +# Pattern includes [0-9] after status to avoid matching status-like text in context blocks + +PENDING=$(grep -cE '^\s*- \[pending\] [0-9]' "$TREE_FILE" || true) +REVIEWED=$(grep -cE '^\s*- \[reviewed\] [0-9]' "$TREE_FILE" || true) +ACCEPTED=$(grep -cE '^\s*- \[accepted\] [0-9]' "$TREE_FILE" || true) +TOTAL=$((PENDING + REVIEWED + ACCEPTED)) + +# --- Progress --- + +DECIDED=$((REVIEWED + ACCEPTED)) +if [ "$TOTAL" -gt 0 ]; then + PCT=$((DECIDED * 100 / TOTAL)) +else + PCT=0 +fi + +# --- Count nodes with comments --- +# Pattern matches "comment" inside flag braces only, not in titles or context + +WITH_COMMENTS=$(grep -cE '\{[^}]*comment\}' "$TREE_FILE" || true) + +# --- Count top-level concepts --- + +TOP_LEVEL=$(grep -cE '^- \[(pending|reviewed|accepted)\] [0-9]' "$TREE_FILE" || true) + +# --- File coverage from tree's Coverage section --- + +TOTAL_FILES=$(grep 'Total files in diff:' "$TREE_FILE" | grep -oE '[0-9]+' || true) +MAPPED_FILES=$(grep 'Files mapped to tree:' "$TREE_FILE" | grep -oE '[0-9]+' || true) +UNMAPPED=$(grep 'Unmapped files:' "$TREE_FILE" | sed 's/.*: //' || true) + +# --- List pending nodes --- + +PENDING_LIST=$(grep -E '^\s*- \[pending\] [0-9]' "$TREE_FILE" | sed -E 's/^[[:space:]]*- \[pending\] / /' || true) + +# --- Output report --- + +echo "## Status" +echo "" +echo " pending: $PENDING" +echo " reviewed: $REVIEWED" +echo " accepted: $ACCEPTED" +echo " total: $TOTAL" +echo " decided: $DECIDED/$TOTAL ($PCT%)" +echo " with comments: $WITH_COMMENTS" +echo "" +echo " examined in detail: $REVIEWED" +echo " pattern-trusted: $ACCEPTED" +echo " top-level concepts: $TOP_LEVEL" + +if [ -n "$TOTAL_FILES" ]; then + echo "" + echo "## Files" + echo "" + echo " files in diff: $TOTAL_FILES" + echo " files mapped: $MAPPED_FILES" + echo " unmapped: $UNMAPPED" +fi + +if [ "$PENDING" -gt 0 ]; then + echo "" + echo "## Pending" + echo "" + echo " Pending nodes:" + echo "$PENDING_LIST" +fi diff --git a/tests/scripts/test-coverage-report.bats b/tests/scripts/test-coverage-report.bats new file mode 100755 index 0000000..4eb644b --- /dev/null +++ b/tests/scripts/test-coverage-report.bats @@ -0,0 +1,366 @@ +#!/usr/bin/env bats + +# Tests for scripts/coverage-report.sh +# TDD: write tests first, then implement the script. + +REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" +SAMPLE="$REPO_ROOT/tests/formats/sample-tree-hawksbury.md" +SCRIPT="$REPO_ROOT/scripts/coverage-report.sh" + +# --- Status counts --- + +@test "reports count of pending nodes" { + run "$SCRIPT" "$SAMPLE" + [ "$status" -eq 0 ] + [[ "$output" == *"pending: 1"* ]] +} + +@test "reports count of reviewed nodes" { + run "$SCRIPT" "$SAMPLE" + [ "$status" -eq 0 ] + [[ "$output" == *"reviewed: 14"* ]] +} + +@test "reports count of accepted nodes" { + run "$SCRIPT" "$SAMPLE" + [ "$status" -eq 0 ] + [[ "$output" == *"accepted: 28"* ]] +} + +@test "reports total node count" { + run "$SCRIPT" "$SAMPLE" + [ "$status" -eq 0 ] + [[ "$output" == *"total: 43"* ]] +} + +# --- Progress --- + +@test "reports progress percentage" { + run "$SCRIPT" "$SAMPLE" + [ "$status" -eq 0 ] + # 42 decided out of 43 = 97% + [[ "$output" == *"decided: 42/43 (97%)"* ]] +} + +# --- Comment counts --- + +@test "reports count of nodes with comments" { + run "$SCRIPT" "$SAMPLE" + [ "$status" -eq 0 ] + [[ "$output" == *"with comments: 4"* ]] +} + +# --- Pending nodes list --- + +@test "lists pending nodes" { + run "$SCRIPT" "$SAMPLE" + [ "$status" -eq 0 ] + [[ "$output" == *"5. CLAUDE.md"* ]] +} + +@test "no pending nodes listed when all reviewed or accepted" { + local tmpfile="$BATS_TEST_TMPDIR/tree.md" + cp "$SAMPLE" "$tmpfile" + "$REPO_ROOT/scripts/update-node-status.sh" "$tmpfile" "5" "reviewed" + run "$SCRIPT" "$tmpfile" + [ "$status" -eq 0 ] + [[ "$output" == *"pending: 0"* ]] + [[ "$output" != *"Pending nodes:"* ]] +} + +# --- Confidence summary --- + +@test "reports confidence summary" { + run "$SCRIPT" "$SAMPLE" + [ "$status" -eq 0 ] + [[ "$output" == *"examined in detail: 14"* ]] + [[ "$output" == *"pattern-trusted: 28"* ]] +} + +# --- Top-level summary --- + +@test "reports top-level concept count" { + run "$SCRIPT" "$SAMPLE" + [ "$status" -eq 0 ] + [[ "$output" == *"top-level concepts: 5"* ]] +} + +# --- File coverage --- + +@test "reports file coverage from tree" { + run "$SCRIPT" "$SAMPLE" + [ "$status" -eq 0 ] + [[ "$output" == *"files in diff: 32"* ]] + [[ "$output" == *"files mapped: 32"* ]] + [[ "$output" == *"unmapped: none"* ]] +} + +@test "output has Files section header" { + run "$SCRIPT" "$SAMPLE" + [ "$status" -eq 0 ] + [[ "$output" == *"## Files"* ]] +} + +# --- Output structure --- + +@test "output has section headers" { + run "$SCRIPT" "$SAMPLE" + [ "$status" -eq 0 ] + [[ "$output" == *"## Status"* ]] + [[ "$output" == *"## Pending"* ]] +} + +# --- Input validation --- + +@test "rejects missing file argument" { + run "$SCRIPT" + [ "$status" -ne 0 ] +} + +@test "rejects non-existent file" { + run "$SCRIPT" "/tmp/nonexistent-tree.md" + [ "$status" -ne 0 ] +} + +# --- Context inflation protection --- + +@test "context block with status-like text does not inflate counts" { + local tmpfile="$BATS_TEST_TMPDIR/tree.md" + cat > "$tmpfile" << 'EOF' +# Review Tree: Test + +| Field | Value | +|-------------|-------| +| PR | test/test#1 | +| HEAD | abc123 | +| Revision | 1 | +| Tree Built | 2026-02-25T10:00:00Z | +| Updated | 2026-02-25T10:00:00Z | + +## Tree + +- [reviewed] 1. Guard mechanism + context: | + Status markers in other systems: + - [pending] means waiting for review + - [accepted] means done + +## Description Verification + +| # | Claim | Status | Evidence | + +## Coverage + +Total files in diff: 0 +Files mapped to tree: 0 +Unmapped files: none +EOF + run "$SCRIPT" "$tmpfile" + [ "$status" -eq 0 ] + [[ "$output" == *"pending: 0"* ]] + [[ "$output" == *"reviewed: 1"* ]] + [[ "$output" == *"accepted: 0"* ]] + [[ "$output" == *"total: 1"* ]] +} + +# --- Edge cases --- + +@test "handles tree with all nodes pending" { + local tmpfile="$BATS_TEST_TMPDIR/tree.md" + cat > "$tmpfile" << 'EOF' +# Review Tree: Test + +| Field | Value | +|-------------|-------| +| PR | test/test#1 | +| HEAD | abc123 | +| Revision | 1 | +| Tree Built | 2026-02-25T10:00:00Z | +| Updated | 2026-02-25T10:00:00Z | + +## Tree + +- [pending] 1. First concept +- [pending] 2. Second concept + +## Description Verification + +| # | Claim | Status | Evidence | + +## Coverage + +Total files in diff: 0 +Files mapped to tree: 0 +Unmapped files: none +EOF + run "$SCRIPT" "$tmpfile" + [ "$status" -eq 0 ] + [[ "$output" == *"pending: 2"* ]] + [[ "$output" == *"reviewed: 0"* ]] + [[ "$output" == *"accepted: 0"* ]] + [[ "$output" == *"decided: 0/2 (0%)"* ]] +} + +@test "handles tree with all nodes reviewed" { + local tmpfile="$BATS_TEST_TMPDIR/tree.md" + cat > "$tmpfile" << 'EOF' +# Review Tree: Test + +| Field | Value | +|-------------|-------| +| PR | test/test#1 | +| HEAD | abc123 | +| Revision | 1 | +| Tree Built | 2026-02-25T10:00:00Z | +| Updated | 2026-02-25T10:00:00Z | + +## Tree + +- [reviewed] 1. First concept +- [reviewed] 2. Second concept + +## Description Verification + +| # | Claim | Status | Evidence | + +## Coverage + +Total files in diff: 0 +Files mapped to tree: 0 +Unmapped files: none +EOF + run "$SCRIPT" "$tmpfile" + [ "$status" -eq 0 ] + [[ "$output" == *"pending: 0"* ]] + [[ "$output" == *"reviewed: 2"* ]] + [[ "$output" == *"decided: 2/2 (100%)"* ]] + [[ "$output" != *"Pending nodes:"* ]] +} + +@test "empty tree with no nodes reports zeros" { + local tmpfile="$BATS_TEST_TMPDIR/tree.md" + cat > "$tmpfile" << 'EOF' +# Review Tree: Test + +| Field | Value | +|-------------|-------| +| PR | test/test#1 | +| HEAD | abc123 | +| Revision | 1 | +| Tree Built | 2026-02-25T10:00:00Z | +| Updated | 2026-02-25T10:00:00Z | + +## Tree + +## Description Verification + +| # | Claim | Status | Evidence | + +## Coverage + +Total files in diff: 0 +Files mapped to tree: 0 +Unmapped files: none +EOF + run "$SCRIPT" "$tmpfile" + [ "$status" -eq 0 ] + [[ "$output" == *"total: 0"* ]] + [[ "$output" == *"decided: 0/0 (0%)"* ]] +} + +@test "tree without Coverage section omits Files section" { + local tmpfile="$BATS_TEST_TMPDIR/tree.md" + cat > "$tmpfile" << 'EOF' +# Review Tree: Test + +| Field | Value | +|-------------|-------| +| PR | test/test#1 | +| HEAD | abc123 | +| Revision | 1 | +| Tree Built | 2026-02-25T10:00:00Z | +| Updated | 2026-02-25T10:00:00Z | + +## Tree + +- [reviewed] 1. Only concept + +## Description Verification + +| # | Claim | Status | Evidence | +EOF + run "$SCRIPT" "$tmpfile" + [ "$status" -eq 0 ] + [[ "$output" == *"reviewed: 1"* ]] + [[ "$output" != *"## Files"* ]] +} + +@test "reports unmapped files when present" { + local tmpfile="$BATS_TEST_TMPDIR/tree.md" + cat > "$tmpfile" << 'EOF' +# Review Tree: Test + +| Field | Value | +|-------------|-------| +| PR | test/test#1 | +| HEAD | abc123 | +| Revision | 1 | +| Tree Built | 2026-02-25T10:00:00Z | +| Updated | 2026-02-25T10:00:00Z | + +## Tree + +- [reviewed] 1. Only concept + +## Description Verification + +| # | Claim | Status | Evidence | + +## Coverage + +Total files in diff: 3 +Files mapped to tree: 1 +Unmapped files: src/util.java, src/config.yaml +EOF + run "$SCRIPT" "$tmpfile" + [ "$status" -eq 0 ] + [[ "$output" == *"files in diff: 3"* ]] + [[ "$output" == *"files mapped: 1"* ]] + [[ "$output" == *"unmapped: src/util.java, src/config.yaml"* ]] +} + +@test "deeply nested pending node listed correctly" { + local tmpfile="$BATS_TEST_TMPDIR/tree.md" + cat > "$tmpfile" << 'EOF' +# Review Tree: Test + +| Field | Value | +|-------------|-------| +| PR | test/test#1 | +| HEAD | abc123 | +| Revision | 1 | +| Tree Built | 2026-02-25T10:00:00Z | +| Updated | 2026-02-25T10:00:00Z | + +## Tree + +- [reviewed] 1. Top level + - [reviewed] 1.1. Second level + - [reviewed] 1.1.1. Third level + - [pending] 1.1.1.1. Deep pending node + +## Description Verification + +| # | Claim | Status | Evidence | + +## Coverage + +Total files in diff: 0 +Files mapped to tree: 0 +Unmapped files: none +EOF + run "$SCRIPT" "$tmpfile" + [ "$status" -eq 0 ] + [[ "$output" == *"pending: 1"* ]] + [[ "$output" == *"1.1.1.1. Deep pending node"* ]] +}