Skip to content

Commit 6455f41

Browse files
codeadictcodeadict
committed
fix: compliance test suite summary script
1 parent 06b966d commit 6455f41

1 file changed

Lines changed: 88 additions & 263 deletions

File tree

‎test/support/generate_compliance_summary.sh‎

Lines changed: 88 additions & 263 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
set -euo pipefail
33

44
# Generate GitHub Actions Summary for H2 Compliance Tests
5+
# Parses CT log HTML to discover groups and their pass/fail status.
56
# Usage: ./generate_compliance_summary.sh <ct_log_dir>
67

78
CT_LOG_DIR="${1:-_build/test/logs}"
@@ -11,290 +12,114 @@ OUTPUT_FILE="${GITHUB_STEP_SUMMARY:-/tmp/summary.md}"
1112
LATEST_RUN=$(find "$CT_LOG_DIR" -name "suite.log.html" -path "*h2_compliance_SUITE.logs/*" -type f 2>/dev/null | sort -r | head -1)
1213

1314
if [ -z "$LATEST_RUN" ]; then
14-
echo "⚠️ No test results found in $CT_LOG_DIR"
15-
echo "Tests may not have run yet."
15+
echo "No test results found in $CT_LOG_DIR"
1616
exit 1
1717
fi
1818

19-
echo "📊 Parsing test results from: $LATEST_RUN"
19+
echo "Parsing test results from: $LATEST_RUN"
2020

21-
# Parse test results - look for test group outcomes
22-
declare -A TEST_STATUS
21+
# Parse test results from CT log HTML into a temp file.
22+
# Each line: <group_name> <OK|FAILED>
23+
RESULTS_FILE=$(mktemp)
24+
trap 'rm -f "$RESULTS_FILE"' EXIT
2325

24-
# Parse the HTML log for test results
25-
# Look for patterns in HTML table rows like:
26-
# <td>...run_connection_preface...</td>...<td><font color="red">FAILED</font></td>
27-
# or <td>...run_connection_preface...</td>...<td><font color="green">Ok</font></td>
28-
29-
# Method 1: Try to parse from suite log
3026
while IFS= read -r line; do
31-
if [[ "$line" =~ h2_compliance_suite\.run_([a-z_]+) ]]; then
32-
group="${BASH_REMATCH[1]}"
33-
# Check if this line has FAILED or Ok/ok in it
34-
if [[ "$line" =~ \<font\ color=\"red\"\>FAILED\<\/font\> ]]; then
35-
TEST_STATUS["$group"]="❌"
36-
elif [[ "$line" =~ \<font\ color=\"[^\"]+\"\>(Ok|ok)\<\/font\> ]]; then
37-
TEST_STATUS["$group"]="✅"
38-
fi
27+
# Match lines like: h2_compliance_suite.run_connection_preface ... Ok/FAILED
28+
group=""
29+
status=""
30+
case "$line" in
31+
*h2_compliance_suite.run_*)
32+
# Extract group name after "run_"
33+
group=$(echo "$line" | sed -n 's/.*h2_compliance_suite\.run_\([a-z_0-9]*\).*/\1/p')
34+
;;
35+
esac
36+
37+
if [ -z "$group" ]; then
38+
continue
3939
fi
40-
done <"$LATEST_RUN"
4140

42-
# Method 2: If we didn't find results, check if "All tests passed"
43-
# Temporarily disable unbound variable check for array length check
44-
set +u
45-
array_size=${#TEST_STATUS[@]}
46-
set -u
47-
48-
if [ $array_size -eq 0 ]; then
49-
if grep -q "All.*tests passed" "$LATEST_RUN" 2>/dev/null; then
50-
echo "✓ All tests passed (detected from summary)"
51-
ALL_PASSED=true
52-
else
53-
echo "⚠️ Could not determine test status, assuming failures"
54-
ALL_PASSED=false
55-
fi
56-
else
57-
echo "✓ Parsed $array_size test group results"
58-
ALL_PASSED=true
59-
for status in "${TEST_STATUS[@]}"; do
60-
if [ "$status" = "❌" ]; then
61-
ALL_PASSED=false
62-
break
41+
case "$line" in
42+
*'<font color="red">FAILED</font>'*)
43+
status="FAILED" ;;
44+
*'<font color='*'>Ok</font>'*|*'<font color='*'>ok</font>'*)
45+
status="OK" ;;
46+
esac
47+
48+
if [ -n "$status" ]; then
49+
# Only record first result per group
50+
if ! grep -q "^${group} " "$RESULTS_FILE" 2>/dev/null; then
51+
echo "$group $status" >> "$RESULTS_FILE"
6352
fi
64-
done
65-
fi
66-
67-
# Helper function to get status for a group
68-
get_status() {
69-
local group="$1"
70-
set +u
71-
local array_size=${#TEST_STATUS[@]}
72-
set -u
73-
if [ $array_size -eq 0 ]; then
74-
# No parsed results, use ALL_PASSED flag
75-
if [ "$ALL_PASSED" = true ]; then
76-
echo "✅"
77-
else
78-
echo "❌"
79-
fi
80-
else
81-
# Use parsed status or default to ❌ if not found
82-
echo "${TEST_STATUS[$group]:-❌}"
83-
fi
84-
}
85-
86-
# Count totals
87-
TOTAL_GROUPS=37
88-
PASSED=0
89-
FAILED=0
90-
91-
for group in connection_preface frame_format frame_size header_compression \
92-
stream_states stream_identifiers stream_concurrency stream_priority \
93-
stream_dependencies error_handling connection_errors extensions \
94-
data_frames headers_frames priority_frames rst_stream_frames \
95-
settings_frames push_promise_frames ping_frames goaway_frames \
96-
window_update_frames continuation_frames http_exchange \
97-
http_header_fields pseudo_headers connection_headers \
98-
request_pseudo_headers malformed_requests server_push push_requests \
99-
hpack_dynamic_table hpack_decoding hpack_table_management \
100-
hpack_integer hpack_string hpack_binary generic_tests; do
101-
102-
status=$(get_status "$group")
103-
if [ "$status" = "✅" ]; then
104-
PASSED=$((PASSED + 1))
105-
else
106-
FAILED=$((FAILED + 1))
10753
fi
108-
done
109-
110-
SUCCESS_RATE=$((PASSED * 100 / TOTAL_GROUPS))
111-
112-
echo "📈 Results: $PASSED passed, $FAILED failed ($SUCCESS_RATE% success rate)"
113-
114-
# Generate summary header
115-
cat >"$OUTPUT_FILE" <<EOF
116-
# 🧪 HTTP/2 Compliance Test Results
54+
done < "$LATEST_RUN"
11755

118-
## Summary
56+
TOTAL=$(wc -l < "$RESULTS_FILE" | tr -d ' ')
11957

120-
EOF
58+
if [ "$TOTAL" -eq 0 ]; then
59+
echo "Could not parse any test results from the log."
60+
echo "# HTTP/2 Compliance Test Results" > "$OUTPUT_FILE"
61+
echo "" >> "$OUTPUT_FILE"
62+
echo "No test results could be parsed from CT logs." >> "$OUTPUT_FILE"
63+
exit 1
64+
fi
12165

122-
if [ "$FAILED" -eq 0 ]; then
123-
cat >>"$OUTPUT_FILE" <<EOF
124-
### ✅ All Tests Passed!
66+
echo "Found $TOTAL test groups"
12567

126-
- **Total Groups**: 37
127-
- **Individual Tests**: 156
128-
- **Success Rate**: 100%
129-
- **Status**: 🎉 Complete RFC 7540 & RFC 7541 Compliance
68+
PASSED=$(grep -c " OK$" "$RESULTS_FILE" || true)
69+
FAILED=$(grep -c " FAILED$" "$RESULTS_FILE" || true)
13070

131-
EOF
71+
if [ "$TOTAL" -gt 0 ]; then
72+
SUCCESS_RATE=$((PASSED * 100 / TOTAL))
13273
else
133-
cat >>"$OUTPUT_FILE" <<EOF
134-
### ⚠️ Some Tests Failed
135-
136-
- **Passed**: $PASSED / $TOTAL_GROUPS groups
137-
- **Failed**: $FAILED / $TOTAL_GROUPS groups
138-
- **Success Rate**: ${SUCCESS_RATE}%
139-
140-
EOF
74+
SUCCESS_RATE=0
14175
fi
14276

143-
# Generate detailed tables with actual status
144-
cat >>"$OUTPUT_FILE" <<EOF
145-
## Test Coverage by RFC Section
146-
147-
### RFC 7540 - HTTP/2 Protocol
148-
149-
<details open>
150-
<summary><b>Core Protocol</b></summary>
77+
echo "Results: $PASSED passed, $FAILED failed ($SUCCESS_RATE% pass rate)"
15178

152-
| Test ID | Description | RFC Section | Status |
153-
|---------|-------------|-------------|--------|
154-
| 3.5/1 | Valid connection preface | §3.5 | $(get_status "connection_preface") |
155-
| 3.5/2 | Invalid connection preface | §3.5 | $(get_status "connection_preface") |
156-
| 4.1/1 | Unknown frame type | §4.1 | $(get_status "frame_format") |
157-
| 4.1/2 | PING with undefined flags | §4.1 | $(get_status "frame_format") |
158-
| 4.1/3 | PING with reserved bit | §4.1 | $(get_status "frame_format") |
159-
| 4.2/1 | DATA frame 2^14 octets | §4.2 | $(get_status "frame_size") |
160-
| 4.2/2 | Oversized DATA frame | §4.2 | $(get_status "frame_size") |
161-
| 4.2/3 | Oversized HEADERS frame | §4.2 | $(get_status "frame_size") |
162-
| 4.3/1 | Invalid header block fragment | §4.3 | $(get_status "header_compression") |
163-
| 4.3/2 | Header block after END_HEADERS | §4.3 | $(get_status "header_compression") |
164-
165-
</details>
166-
167-
<details open>
168-
<summary><b>Stream Management</b></summary>
169-
170-
| Test ID | Description | RFC Section | Status |
171-
|---------|-------------|-------------|--------|
172-
| 5.1/1-17 | Stream states (17 tests) | §5.1 | $(get_status "stream_states") |
173-
| 5.1.1/1-4 | Stream identifiers (4 tests) | §5.1.1 | $(get_status "stream_identifiers") |
174-
| 5.1.2/1-2 | Stream concurrency (2 tests) | §5.1.2 | $(get_status "stream_concurrency") |
175-
| 5.3/1-2 | Stream priority (2 tests) | §5.3 | $(get_status "stream_priority") |
176-
| 5.3.1/1-3 | Stream dependencies (3 tests) | §5.3.1 | $(get_status "stream_dependencies") |
177-
| 5.4/1-4 | Error handling (4 tests) | §5.4 | $(get_status "error_handling") |
178-
| 5.4.1/1-3 | Connection errors (3 tests) | §5.4.1 | $(get_status "connection_errors") |
179-
| 5.5/1-2 | Extensions (2 tests) | §5.5 | $(get_status "extensions") |
180-
181-
</details>
182-
183-
<details open>
184-
<summary><b>Frame Definitions</b></summary>
185-
186-
| Test ID | Description | RFC Section | Status |
187-
|---------|-------------|-------------|--------|
188-
| 6.1/1-6 | DATA frames (6 tests) | §6.1 | $(get_status "data_frames") |
189-
| 6.2/1-6 | HEADERS frames (6 tests) | §6.2 | $(get_status "headers_frames") |
190-
| 6.3/1-4 | PRIORITY frames (4 tests) | §6.3 | $(get_status "priority_frames") |
191-
| 6.4/1-4 | RST_STREAM frames (4 tests) | §6.4 | $(get_status "rst_stream_frames") |
192-
| 6.5/1-10 | SETTINGS frames (10 tests) | §6.5 | $(get_status "settings_frames") |
193-
| 6.6/1-4 | PUSH_PROMISE frames (4 tests) | §6.6 | $(get_status "push_promise_frames") |
194-
| 6.7/1-4 | PING frames (4 tests) | §6.7 | $(get_status "ping_frames") |
195-
| 6.8/1-4 | GOAWAY frames (4 tests) | §6.8 | $(get_status "goaway_frames") |
196-
| 6.9/1-5 | WINDOW_UPDATE frames (5 tests) | §6.9 | $(get_status "window_update_frames") |
197-
| 6.10/1-4 | CONTINUATION frames (4 tests) | §6.10 | $(get_status "continuation_frames") |
198-
199-
</details>
200-
201-
<details open>
202-
<summary><b>HTTP Semantics</b></summary>
203-
204-
| Test ID | Description | RFC Section | Status |
205-
|---------|-------------|-------------|--------|
206-
| 8.1/1-5 | HTTP request/response (5 tests) | §8.1 | $(get_status "http_exchange") |
207-
| 8.1.2/1-6 | HTTP header fields (6 tests) | §8.1.2 | $(get_status "http_header_fields") |
208-
| 8.1.2.1/1-2 | Pseudo-header fields (2 tests) | §8.1.2.1 | $(get_status "pseudo_headers") |
209-
| 8.1.2.2/1-5 | Connection headers (5 tests) | §8.1.2.2 | $(get_status "connection_headers") |
210-
| 8.1.2.3/1-3 | Request pseudo-headers (3 tests) | §8.1.2.3 | $(get_status "request_pseudo_headers") |
211-
| 8.1.2.6/1-2 | Malformed requests (2 tests) | §8.1.2.6 | $(get_status "malformed_requests") |
212-
| 8.2/1-2 | Server push (2 tests) | §8.2 | $(get_status "server_push") |
213-
| 8.2.1/1-2 | Push requests (2 tests) | §8.2.1 | $(get_status "push_requests") |
214-
215-
</details>
216-
217-
### RFC 7541 - HPACK Compression
218-
219-
<details open>
220-
<summary><b>HPACK Tests</b></summary>
221-
222-
| Test ID | Description | RFC Section | Status |
223-
|---------|-------------|-------------|--------|
224-
| hpack/2.3.2/1-3 | Dynamic table (3 tests) | §2.3.2 | $(get_status "hpack_dynamic_table") |
225-
| hpack/3.2/1-3 | Header block decoding (3 tests) | §3.2 | $(get_status "hpack_decoding") |
226-
| hpack/4/1-3 | Table management (3 tests) | §4 | $(get_status "hpack_table_management") |
227-
| hpack/5.1/1-2 | Integer encoding (2 tests) | §5.1 | $(get_status "hpack_integer") |
228-
| hpack/5.2/1-3 | String encoding (3 tests) | §5.2 | $(get_status "hpack_string") |
229-
| hpack/6/1-2 | Binary format (2 tests) | §6 | $(get_status "hpack_binary") |
230-
231-
</details>
232-
233-
### Generic Protocol Tests
234-
235-
<details open>
236-
<summary><b>Additional Coverage</b></summary>
237-
238-
| Test ID | Description | Status |
239-
|---------|-------------|--------|
240-
| generic/1-15 | Connection lifecycle (15 tests) | $(get_status "generic_tests") |
241-
242-
</details>
243-
244-
## Test Groups Summary
245-
246-
| Group | Tests | Status |
247-
|-------|-------|--------|
248-
| Connection Preface | 2 | $(get_status "connection_preface") |
249-
| Frame Format | 3 | $(get_status "frame_format") |
250-
| Frame Size | 3 | $(get_status "frame_size") |
251-
| Header Compression | 2 | $(get_status "header_compression") |
252-
| Stream States | 17 | $(get_status "stream_states") |
253-
| Stream Identifiers | 4 | $(get_status "stream_identifiers") |
254-
| Stream Concurrency | 2 | $(get_status "stream_concurrency") |
255-
| Stream Priority | 2 | $(get_status "stream_priority") |
256-
| Stream Dependencies | 3 | $(get_status "stream_dependencies") |
257-
| Error Handling | 4 | $(get_status "error_handling") |
258-
| Connection Errors | 3 | $(get_status "connection_errors") |
259-
| Extensions | 2 | $(get_status "extensions") |
260-
| DATA Frames | 6 | $(get_status "data_frames") |
261-
| HEADERS Frames | 6 | $(get_status "headers_frames") |
262-
| PRIORITY Frames | 4 | $(get_status "priority_frames") |
263-
| RST_STREAM Frames | 4 | $(get_status "rst_stream_frames") |
264-
| SETTINGS Frames | 10 | $(get_status "settings_frames") |
265-
| PUSH_PROMISE Frames | 4 | $(get_status "push_promise_frames") |
266-
| PING Frames | 4 | $(get_status "ping_frames") |
267-
| GOAWAY Frames | 4 | $(get_status "goaway_frames") |
268-
| WINDOW_UPDATE Frames | 5 | $(get_status "window_update_frames") |
269-
| CONTINUATION Frames | 4 | $(get_status "continuation_frames") |
270-
| HTTP Exchange | 5 | $(get_status "http_exchange") |
271-
| HTTP Header Fields | 6 | $(get_status "http_header_fields") |
272-
| Pseudo Headers | 2 | $(get_status "pseudo_headers") |
273-
| Connection Headers | 5 | $(get_status "connection_headers") |
274-
| Request Pseudo Headers | 3 | $(get_status "request_pseudo_headers") |
275-
| Malformed Requests | 2 | $(get_status "malformed_requests") |
276-
| Server Push | 2 | $(get_status "server_push") |
277-
| Push Requests | 2 | $(get_status "push_requests") |
278-
| HPACK Dynamic Table | 3 | $(get_status "hpack_dynamic_table") |
279-
| HPACK Decoding | 3 | $(get_status "hpack_decoding") |
280-
| HPACK Table Management | 3 | $(get_status "hpack_table_management") |
281-
| HPACK Integer | 2 | $(get_status "hpack_integer") |
282-
| HPACK String | 3 | $(get_status "hpack_string") |
283-
| HPACK Binary | 2 | $(get_status "hpack_binary") |
284-
| Generic Tests | 15 | $(get_status "generic_tests") |
79+
# Pretty-print a group name: stream_states -> Stream States
80+
pretty_name() {
81+
echo "$1" | sed 's/_/ /g' | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2)} 1'
82+
}
28583

286-
---
84+
# Build the summary markdown
85+
{
86+
echo "# HTTP/2 Compliance Test Results"
87+
echo ""
28788

288-
## 📊 Statistics
89+
if [ "$FAILED" -eq 0 ]; then
90+
echo "## All $TOTAL Groups Passed"
91+
echo ""
92+
echo "- **Groups**: $TOTAL"
93+
echo "- **Pass Rate**: 100%"
94+
else
95+
echo "## $PASSED / $TOTAL Groups Passed ($SUCCESS_RATE%)"
96+
echo ""
97+
echo "- **Passed**: $PASSED"
98+
echo "- **Failed**: $FAILED"
99+
echo ""
100+
echo "### Failed Groups"
101+
echo ""
102+
grep " FAILED$" "$RESULTS_FILE" | while read -r g _; do
103+
echo "- $(pretty_name "$g")"
104+
done
105+
fi
289106

290-
- **Total Test Groups**: 37
291-
- **Passed**: $PASSED
292-
- **Failed**: $FAILED
293-
- **Success Rate**: ${SUCCESS_RATE}%
294-
- **Total Individual Tests**: 156
107+
echo ""
108+
echo "## All Groups"
109+
echo ""
110+
echo "| Group | Status |"
111+
echo "|-------|--------|"
112+
while read -r group status; do
113+
name=$(pretty_name "$group")
114+
if [ "$status" = "OK" ]; then
115+
echo "| $name | Pass |"
116+
else
117+
echo "| $name | **FAIL** |"
118+
fi
119+
done < "$RESULTS_FILE"
295120

296-
EOF
121+
} > "$OUTPUT_FILE"
297122

298-
echo "✓ Summary generated: $OUTPUT_FILE"
299-
echo " Passed: $PASSED/$TOTAL_GROUPS"
300-
echo " Failed: $FAILED/$TOTAL_GROUPS"
123+
echo "Summary written to: $OUTPUT_FILE"
124+
echo " Passed: $PASSED/$TOTAL"
125+
echo " Failed: $FAILED/$TOTAL"

0 commit comments

Comments
 (0)