From 1cdbda3fd8fa76984e875eae29807a48326d1254 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 27 Mar 2026 15:14:24 +0100 Subject: [PATCH 1/3] add batch testing (cherry picked from commit dd5e48f1ea62c70c29c409649a24e7d9c480804a) --- .github/scripts/update-test-dashboard.py | 231 +++++++++++++++++++ .github/workflows/monthly-full-test.yml | 278 +++++++++++++++++++++++ 2 files changed, 509 insertions(+) create mode 100755 .github/scripts/update-test-dashboard.py create mode 100644 .github/workflows/monthly-full-test.yml diff --git a/.github/scripts/update-test-dashboard.py b/.github/scripts/update-test-dashboard.py new file mode 100755 index 000000000000..6c129ea75257 --- /dev/null +++ b/.github/scripts/update-test-dashboard.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +""" +Update the monthly test dashboard issue with batch results. +""" + +import os +import sys +import json +from datetime import datetime +from typing import Optional +from urllib.request import Request, urlopen +from urllib.error import URLError + + +def github_api_request(method: str, endpoint: str, data: Optional[dict] = None) -> dict: + """Make a GitHub API request.""" + token = os.environ.get("GITHUB_TOKEN") + repo = os.environ.get("GITHUB_REPOSITORY") + + url = f"https://api.github.com/repos/{repo}/{endpoint}" + headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github.v3+json", + "Content-Type": "application/json", + } + + req = Request(url, headers=headers, method=method) + if data: + req.data = json.dumps(data).encode() + + try: + with urlopen(req) as response: + return json.loads(response.read().decode()) + except URLError as e: + print(f"API request failed: {e}", file=sys.stderr) + return {} + + +def get_issue_number() -> Optional[int]: + """Find the existing dashboard issue.""" + issues = github_api_request("GET", "issues?labels=test-dashboard&state=open&per_page=1") + return issues[0]["number"] if isinstance(issues, list) and issues else None + + +def get_issue_body(issue_number: int) -> str: + """Get the current issue body.""" + issue = github_api_request("GET", f"issues/{issue_number}") + return issue.get("body", "") + + +def create_batch_row( + batch_num: str, batch_label: str, status: str, passed: int, total: int, run_number: str, run_url: str +) -> str: + """Create a markdown table row for a batch.""" + status_emoji = {"passed": "✅", "failed": "❌", "cancelled": "⚠️", "skipped": "⏭️"}.get( + status, "❓" + ) + + timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M UTC") + + return f"| Batch {batch_num} ({batch_label}) | {status_emoji} {status} | {passed}/{total} | {timestamp} | [Run #{run_number}]({run_url}) |" + + +def create_failed_tests_section(batch_num: str, failed_count: int, failed_tests: list[str]) -> str: + """Create a collapsible section for failed tests.""" + if not failed_tests or failed_count == 0: + return "" + + failed_list = "\n".join(failed_tests) + return f""" +
+❌ {failed_count} failing test(s) in batch {batch_num} + +``` +{failed_list} +``` +
+""" + + +def update_body( + current_body: str, + batch_num: str, + batch_row: str, + failed_section: str, +) -> str: + """Update the issue body with new batch results.""" + lines = current_body.split("\n") + updated_lines = [] + in_table = False + batch_updated = False + in_failed_section = False + skip_until_detail_close = False + + for i, line in enumerate(lines): + # Track if we're in the status table + if "| Batch | Status |" in line: + in_table = True + updated_lines.append(line) + continue + + # End of table + if in_table and line.strip() and not line.startswith("|"): + in_table = False + + # Update or skip existing batch row + if in_table and line.strip().startswith(f"| Batch {batch_num}"): + updated_lines.append(batch_row) + batch_updated = True + continue + + # Track failed tests section + if "## Failed Tests" in line: + in_failed_section = True + updated_lines.append(line) + continue + + # Skip old failed section for this batch + if in_failed_section and f"batch {batch_num}" in line: + skip_until_detail_close = True + continue + + if skip_until_detail_close: + if "" in line: + skip_until_detail_close = False + continue + + updated_lines.append(line) + + # If batch wasn't in table, add it after table header + if not batch_updated: + for i, line in enumerate(updated_lines): + if "|-------|--------|" in line: + updated_lines.insert(i + 1, batch_row) + break + + # Add failed tests section if needed + if failed_section: + # Find the Failed Tests section + for i, line in enumerate(updated_lines): + if "## Failed Tests" in line: + # Insert after the heading + updated_lines.insert(i + 2, failed_section) + break + + return "\n".join(updated_lines) + + +def create_issue_body( + batch_row: str, failed_section: str, repo: str, workflow_url: str +) -> str: + """Create initial issue body.""" + failed_text = failed_section if failed_section else "_No failures currently tracked_" + + return f"""# 🧪 Monthly Module Test Dashboard + +This issue tracks the status of monthly full module testing across all batches. + +## Current Month Status + +| Batch | Status | Tests Passed | Last Run | Workflow | +|-------|--------|--------------|----------|----------| +{batch_row} + +## Failed Tests + +{failed_text} + +--- +*This dashboard is automatically updated by the [Monthly Full Test workflow]({workflow_url})* +""" + + +def main(): + # Get environment variables + batch_num = os.environ["BATCH_NUM"] + batch_label = os.environ["BATCH_LABEL"] + status = os.environ["STATUS"] + run_url = os.environ["RUN_URL"] + run_number = os.environ["RUN_NUMBER"] + failed_count = int(os.environ["FAILED_COUNT"]) + passed_count = int(os.environ["PASSED_COUNT"]) + total_count = int(os.environ["TOTAL_COUNT"]) + repo = os.environ["GITHUB_REPOSITORY"] + workflow_url = f"https://github.com/{repo}/actions/workflows/monthly-full-test.yml" + + # Read failed tests + failed_tests = [] + if os.path.exists("failed_tests.txt"): + with open("failed_tests.txt") as f: + failed_tests = [line.strip() for line in f if line.strip()] + + # Create batch row + batch_row = create_batch_row( + batch_num, batch_label, status, passed_count, total_count, run_number, run_url + ) + + # Create failed tests section + failed_section = create_failed_tests_section(batch_num, failed_count, failed_tests) if failed_count > 0 else "" + + # Find or create issue + issue_number = get_issue_number() + + if issue_number is None: + # Create new issue + body = create_issue_body(batch_row, failed_section, repo, workflow_url) + result = github_api_request( + "POST", + "issues", + {"title": "🧪 Monthly Module Test Dashboard", "body": body, "labels": ["test-dashboard"]}, + ) + if result: + print("✅ Created new dashboard issue") + else: + print("❌ Failed to create issue", file=sys.stderr) + sys.exit(1) + else: + # Update existing issue + current_body = get_issue_body(issue_number) + updated_body = update_body(current_body, batch_num, batch_row, failed_section) + + result = github_api_request("PATCH", f"issues/{issue_number}", {"body": updated_body}) + if result: + print(f"✅ Updated dashboard issue #{issue_number}") + else: + print(f"❌ Failed to update issue #{issue_number}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/monthly-full-test.yml b/.github/workflows/monthly-full-test.yml new file mode 100644 index 000000000000..b6b6585b97e7 --- /dev/null +++ b/.github/workflows/monthly-full-test.yml @@ -0,0 +1,278 @@ +name: Monthly Full Module Test +on: + schedule: + # Every Sunday at 2 AM UTC - will test 1/4 of modules each week + - cron: '0 2 * * 0' + workflow_dispatch: + inputs: + batch: + description: 'Batch to test (1=a-f, 2=g-m, 3=n-s, 4=t-z)' + required: true + type: choice + options: ['1', '2', '3', '4'] + default: '1' + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.inputs.batch || 'scheduled' }} + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # renovate: datasource=github-releases depName=askimed/nf-test versioning=semver + NFT_VER: "0.9.4" + NXF_ANSI_LOG: false + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver + NXF_VER: "25.10.2" + +jobs: + list-modules: + name: List modules for batch + runs-on: + - runs-on=${{ github.run_id }}-list-modules + - runner=4cpu-linux-x64 + - image=ubuntu22-full-x64 + outputs: + batch_number: ${{ steps.set-batch.outputs.batch_number }} + batch_label: ${{ steps.set-batch.outputs.batch_label }} + paths: ${{ steps.list.outputs.paths }} + modules: ${{ steps.list.outputs.modules }} + shard: ${{ steps.set-shards.outputs.shard }} + total_shards: ${{ steps.set-shards.outputs.total_shards }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Determine batch to test + id: set-batch + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + BATCH=${{ github.event.inputs.batch }} + else + # Calculate batch based on week of month (1-4) + WEEK_OF_MONTH=$(( ($(date +%-d) - 1) / 7 + 1 )) + # Ensure we stay within 1-4 range + BATCH=$(( (WEEK_OF_MONTH - 1) % 4 + 1 )) + fi + + # Set batch label + case $BATCH in + 1) LABEL="a-f" ;; + 2) LABEL="g-m" ;; + 3) LABEL="n-s" ;; + 4) LABEL="t-z" ;; + esac + + echo "batch_number=$BATCH" >> $GITHUB_OUTPUT + echo "batch_label=$LABEL" >> $GITHUB_OUTPUT + echo "Testing batch $BATCH: modules starting with $LABEL" + + - name: List all modules in batch ${{ steps.set-batch.outputs.batch_label }} + id: list + run: | + BATCH=${{ steps.set-batch.outputs.batch_number }} + + # Define regex patterns for each batch + case $BATCH in + 1) PATTERN='^[a-fA-F]' ;; + 2) PATTERN='^[g-mG-M]' ;; + 3) PATTERN='^[n-sN-S]' ;; + 4) PATTERN='^[t-zT-Z]' ;; + esac + + # Find all module test files matching the batch pattern + PATHS=$(find modules/nf-core -name "main.nf.test" -type f | \ + grep -E "modules/nf-core/[^/]*${PATTERN}" | \ + sed 's|/tests/main.nf.test||' | \ + jq -R -s -c 'split("\n") | map(select(length > 0))') + + echo "paths=$PATHS" >> $GITHUB_OUTPUT + + # Extract module names + MODULES=$(echo "$PATHS" | jq -c 'map(gsub("modules/nf-core/"; ""))') + echo "modules=$MODULES" >> $GITHUB_OUTPUT + + echo "Found $(echo "$PATHS" | jq 'length') modules in batch $BATCH" + echo "Sample modules: $(echo "$MODULES" | jq -r '.[0:5] | join(", ")')" + + - name: Get number of shards + id: set-shards + if: ${{ steps.list.outputs.paths != '[]' }} + uses: ./.github/actions/get-shards + env: + NFT_VER: ${{ env.NFT_VER }} + with: + max_shards: 32 + paths: "${{ join(fromJson(steps.list.outputs.paths), ' ') }}" + + - name: Debug output + run: | + echo "Batch: ${{ steps.set-batch.outputs.batch_number }} (${{ steps.set-batch.outputs.batch_label }})" + echo "Paths: ${{ steps.list.outputs.paths }}" + echo "Modules: ${{ steps.list.outputs.modules }}" + echo "Shards: ${{ steps.set-shards.outputs.total_shards }}" + + nf-test: + runs-on: + - runs-on=${{ github.run_id }} + - runner=4cpu-linux-x64 + - image=ubuntu24-full-x64 + name: "Batch ${{ needs.list-modules.outputs.batch_label }} | docker | shard ${{ matrix.shard }}" + needs: [list-modules] + if: ${{ needs.list-modules.outputs.total_shards != '0' }} + strategy: + fail-fast: false + matrix: + shard: ${{ fromJson(needs.list-modules.outputs.shard) }} + env: + NXF_ANSI_LOG: false + TOTAL_SHARDS: ${{ needs.list-modules.outputs.total_shards }} + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Run nf-test Action + id: nf-test + uses: ./.github/actions/nf-test-action + env: + SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} + SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} + SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} + SENTIEON_AUTH_MECH: "GitHub Actions - token" + with: + profile: docker + shard: ${{ matrix.shard }} + total_shards: ${{ env.TOTAL_SHARDS }} + paths: "${{ join(fromJson(needs.list-modules.outputs.paths), ' ') }}" + + - name: Upload test results + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: tap-results-${{ matrix.shard }} + path: test.tap + retention-days: 30 + + confirm-pass: + runs-on: + - runs-on=${{ github.run_id }}-confirm-pass + - runner=4cpu-linux-x64 + - image=ubuntu22-full-x64 + needs: [nf-test] + if: always() + outputs: + test_status: ${{ steps.set-status.outputs.status }} + steps: + - name: Set test status + id: set-status + run: | + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then + echo "status=failed" >> $GITHUB_OUTPUT + elif [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "status=cancelled" >> $GITHUB_OUTPUT + elif [[ "${{ contains(needs.*.result, 'success') }}" == "true" ]]; then + echo "status=passed" >> $GITHUB_OUTPUT + else + echo "status=skipped" >> $GITHUB_OUTPUT + fi + + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: One or more tests cancelled + if: ${{ contains(needs.*.result, 'cancelled') }} + run: exit 1 + + - name: All tests passed + if: ${{ contains(needs.*.result, 'success') }} + run: | + echo "✅ All tests passed for batch ${{ needs.list-modules.outputs.batch_number }}" + echo "Batch range: ${{ needs.list-modules.outputs.batch_label }}" + + - name: Debug output + if: always() + run: | + echo "::group::DEBUG: Results" + echo "Batch: ${{ needs.list-modules.outputs.batch_number }} (${{ needs.list-modules.outputs.batch_label }})" + echo "Results: ${{ toJSON(needs.*.result) }}" + echo "::endgroup::" + + update-dashboard: + runs-on: + - runs-on=${{ github.run_id }}-update-dashboard + - runner=4cpu-linux-x64 + - image=ubuntu22-full-x64 + needs: [list-modules, nf-test, confirm-pass] + if: always() && needs.list-modules.outputs.total_shards != '0' + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Download all test results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: tap-results-* + path: tap-results/ + + - name: Parse test results + id: parse + run: | + python3 - <<'PYTHON' + import os + import re + from pathlib import Path + + failed_tests = set() + total_tests = 0 + passed_tests = 0 + + # Parse all TAP files + tap_dir = Path("tap-results") + if tap_dir.exists(): + for tap_file in tap_dir.rglob("test.tap"): + with open(tap_file) as f: + for line in f: + line = line.strip() + if line.startswith("ok "): + total_tests += 1 + passed_tests += 1 + elif line.startswith("not ok "): + total_tests += 1 + # Extract test name (remove "not ok ") + test_name = re.sub(r'^not ok \d+ ', '', line) + failed_tests.add(test_name) + + failed_count = len(failed_tests) + + # Write to GitHub output + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"failed_count={failed_count}\n") + f.write(f"passed_count={passed_tests}\n") + f.write(f"total_count={total_tests}\n") + + # Write failed tests to file + with open("failed_tests.txt", "w") as f: + for test in sorted(failed_tests): + f.write(f"{test}\n") + + print(f"Parsed {total_tests} tests: {passed_tests} passed, {failed_count} failed") + PYTHON + + - name: Create or update dashboard issue + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BATCH_NUM: ${{ needs.list-modules.outputs.batch_number }} + BATCH_LABEL: ${{ needs.list-modules.outputs.batch_label }} + STATUS: ${{ needs.confirm-pass.outputs.test_status }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + RUN_NUMBER: ${{ github.run_number }} + FAILED_COUNT: ${{ steps.parse.outputs.failed_count }} + PASSED_COUNT: ${{ steps.parse.outputs.passed_count }} + TOTAL_COUNT: ${{ steps.parse.outputs.total_count }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + python3 .github/scripts/update-test-dashboard.py From 9afdf3ec2e1f05237c7591cdde1d6caeb6ae7550 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 27 Mar 2026 15:35:55 +0100 Subject: [PATCH 2/3] switch to batches of 100 (cherry picked from commit bf6fe107575a4e21471a78b48ce3aaa1684bc121) --- .github/workflows/monthly-full-test.yml | 80 +++++++++++-------------- 1 file changed, 34 insertions(+), 46 deletions(-) diff --git a/.github/workflows/monthly-full-test.yml b/.github/workflows/monthly-full-test.yml index b6b6585b97e7..0f80130c1dfa 100644 --- a/.github/workflows/monthly-full-test.yml +++ b/.github/workflows/monthly-full-test.yml @@ -1,16 +1,14 @@ name: Monthly Full Module Test on: schedule: - # Every Sunday at 2 AM UTC - will test 1/4 of modules each week - - cron: '0 2 * * 0' + # Every Sunday at 2 AM UTC - cycles through all batches of 100 modules + - cron: "0 2 * * 0" workflow_dispatch: inputs: batch: - description: 'Batch to test (1=a-f, 2=g-m, 3=n-s, 4=t-z)' + description: "Batch number to test (1-based; each batch is 100 modules)" required: true - type: choice - options: ['1', '2', '3', '4'] - default: '1' + default: "1" # Cancel if a newer run is started concurrency: @@ -35,8 +33,8 @@ jobs: - runner=4cpu-linux-x64 - image=ubuntu22-full-x64 outputs: - batch_number: ${{ steps.set-batch.outputs.batch_number }} - batch_label: ${{ steps.set-batch.outputs.batch_label }} + batch_number: ${{ steps.list.outputs.batch_number }} + batch_label: ${{ steps.list.outputs.batch_label }} paths: ${{ steps.list.outputs.paths }} modules: ${{ steps.list.outputs.modules }} shard: ${{ steps.set-shards.outputs.shard }} @@ -44,57 +42,47 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - name: Determine batch to test - id: set-batch + - name: List 100 modules for this batch + id: list run: | + BATCH_SIZE=100 + + # All module directories that have a main.nf, sorted deterministically + ALL_PATHS=$(find modules/nf-core -name "main.nf" -type f | \ + sed 's|/main.nf||' | sort) + + TOTAL=$(echo "$ALL_PATHS" | wc -l | tr -d ' ') + TOTAL_BATCHES=$(( (TOTAL + BATCH_SIZE - 1) / BATCH_SIZE )) + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then BATCH=${{ github.event.inputs.batch }} else - # Calculate batch based on week of month (1-4) - WEEK_OF_MONTH=$(( ($(date +%-d) - 1) / 7 + 1 )) - # Ensure we stay within 1-4 range - BATCH=$(( (WEEK_OF_MONTH - 1) % 4 + 1 )) + # Cycle through all batches using week-of-year + WEEK=$(date +%V | sed 's/^0//') + BATCH=$(( (WEEK - 1) % TOTAL_BATCHES + 1 )) fi - # Set batch label - case $BATCH in - 1) LABEL="a-f" ;; - 2) LABEL="g-m" ;; - 3) LABEL="n-s" ;; - 4) LABEL="t-z" ;; - esac + # Clamp to valid range + if [ "$BATCH" -lt 1 ]; then BATCH=1; fi + if [ "$BATCH" -gt "$TOTAL_BATCHES" ]; then BATCH=$TOTAL_BATCHES; fi - echo "batch_number=$BATCH" >> $GITHUB_OUTPUT - echo "batch_label=$LABEL" >> $GITHUB_OUTPUT - echo "Testing batch $BATCH: modules starting with $LABEL" + START=$(( (BATCH - 1) * BATCH_SIZE + 1 )) + END=$(( BATCH * BATCH_SIZE )) - - name: List all modules in batch ${{ steps.set-batch.outputs.batch_label }} - id: list - run: | - BATCH=${{ steps.set-batch.outputs.batch_number }} - - # Define regex patterns for each batch - case $BATCH in - 1) PATTERN='^[a-fA-F]' ;; - 2) PATTERN='^[g-mG-M]' ;; - 3) PATTERN='^[n-sN-S]' ;; - 4) PATTERN='^[t-zT-Z]' ;; - esac - - # Find all module test files matching the batch pattern - PATHS=$(find modules/nf-core -name "main.nf.test" -type f | \ - grep -E "modules/nf-core/[^/]*${PATTERN}" | \ - sed 's|/tests/main.nf.test||' | \ + PATHS=$(echo "$ALL_PATHS" | sed -n "${START},${END}p" | \ jq -R -s -c 'split("\n") | map(select(length > 0))') + BATCH_LABEL="modules ${START}-$(echo "$PATHS" | jq 'length + '${START}' - 1') of ${TOTAL}" + + echo "batch_number=$BATCH" >> $GITHUB_OUTPUT + echo "batch_label=$BATCH_LABEL" >> $GITHUB_OUTPUT echo "paths=$PATHS" >> $GITHUB_OUTPUT - # Extract module names MODULES=$(echo "$PATHS" | jq -c 'map(gsub("modules/nf-core/"; ""))') echo "modules=$MODULES" >> $GITHUB_OUTPUT - echo "Found $(echo "$PATHS" | jq 'length') modules in batch $BATCH" - echo "Sample modules: $(echo "$MODULES" | jq -r '.[0:5] | join(", ")')" + echo "Batch $BATCH of $TOTAL_BATCHES: $(echo "$PATHS" | jq 'length') modules" + echo "Sample: $(echo "$MODULES" | jq -r '.[0:5] | join(", ")')" - name: Get number of shards id: set-shards @@ -103,12 +91,12 @@ jobs: env: NFT_VER: ${{ env.NFT_VER }} with: - max_shards: 32 + max_shards: 100 paths: "${{ join(fromJson(steps.list.outputs.paths), ' ') }}" - name: Debug output run: | - echo "Batch: ${{ steps.set-batch.outputs.batch_number }} (${{ steps.set-batch.outputs.batch_label }})" + echo "Batch: ${{ steps.list.outputs.batch_number }} (${{ steps.list.outputs.batch_label }})" echo "Paths: ${{ steps.list.outputs.paths }}" echo "Modules: ${{ steps.list.outputs.modules }}" echo "Shards: ${{ steps.set-shards.outputs.total_shards }}" From 767c8598251bc98277cd2a15daae483f41789c9a Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 27 Mar 2026 16:16:23 +0100 Subject: [PATCH 3/3] simplify script --- .github/scripts/update-test-dashboard.py | 86 ++++++++---------------- 1 file changed, 29 insertions(+), 57 deletions(-) diff --git a/.github/scripts/update-test-dashboard.py b/.github/scripts/update-test-dashboard.py index 6c129ea75257..7cc992a2b5f8 100755 --- a/.github/scripts/update-test-dashboard.py +++ b/.github/scripts/update-test-dashboard.py @@ -3,17 +3,20 @@ Update the monthly test dashboard issue with batch results. """ +import json import os import sys -import json -from datetime import datetime +from datetime import datetime, timezone from typing import Optional -from urllib.request import Request, urlopen from urllib.error import URLError +from urllib.request import Request, urlopen + +TABLE_HEADER = "| Batch | Status |" +TABLE_SEPARATOR = "|-------|--------|" +FAILED_SECTION_HEADER = "## Failed Tests" def github_api_request(method: str, endpoint: str, data: Optional[dict] = None) -> dict: - """Make a GitHub API request.""" token = os.environ.get("GITHUB_TOKEN") repo = os.environ.get("GITHUB_REPOSITORY") @@ -36,33 +39,24 @@ def github_api_request(method: str, endpoint: str, data: Optional[dict] = None) return {} -def get_issue_number() -> Optional[int]: - """Find the existing dashboard issue.""" +def get_existing_issue() -> Optional[tuple[int, str]]: + """Return (number, body) of the existing dashboard issue, or None.""" issues = github_api_request("GET", "issues?labels=test-dashboard&state=open&per_page=1") - return issues[0]["number"] if isinstance(issues, list) and issues else None - - -def get_issue_body(issue_number: int) -> str: - """Get the current issue body.""" - issue = github_api_request("GET", f"issues/{issue_number}") - return issue.get("body", "") + if not isinstance(issues, list) or not issues: + return None + issue = issues[0] + return issue["number"], issue.get("body", "") def create_batch_row( batch_num: str, batch_label: str, status: str, passed: int, total: int, run_number: str, run_url: str ) -> str: - """Create a markdown table row for a batch.""" - status_emoji = {"passed": "✅", "failed": "❌", "cancelled": "⚠️", "skipped": "⏭️"}.get( - status, "❓" - ) - - timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M UTC") - + status_emoji = {"passed": "✅", "failed": "❌", "cancelled": "⚠️", "skipped": "⏭️"}.get(status, "❓") + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") return f"| Batch {batch_num} ({batch_label}) | {status_emoji} {status} | {passed}/{total} | {timestamp} | [Run #{run_number}]({run_url}) |" def create_failed_tests_section(batch_num: str, failed_count: int, failed_tests: list[str]) -> str: - """Create a collapsible section for failed tests.""" if not failed_tests or failed_count == 0: return "" @@ -84,7 +78,6 @@ def update_body( batch_row: str, failed_section: str, ) -> str: - """Update the issue body with new batch results.""" lines = current_body.split("\n") updated_lines = [] in_table = False @@ -92,30 +85,25 @@ def update_body( in_failed_section = False skip_until_detail_close = False - for i, line in enumerate(lines): - # Track if we're in the status table - if "| Batch | Status |" in line: + for line in lines: + if TABLE_HEADER in line: in_table = True updated_lines.append(line) continue - # End of table if in_table and line.strip() and not line.startswith("|"): in_table = False - # Update or skip existing batch row if in_table and line.strip().startswith(f"| Batch {batch_num}"): updated_lines.append(batch_row) batch_updated = True continue - # Track failed tests section - if "## Failed Tests" in line: + if FAILED_SECTION_HEADER in line: in_failed_section = True updated_lines.append(line) continue - # Skip old failed section for this batch if in_failed_section and f"batch {batch_num}" in line: skip_until_detail_close = True continue @@ -127,29 +115,22 @@ def update_body( updated_lines.append(line) - # If batch wasn't in table, add it after table header if not batch_updated: for i, line in enumerate(updated_lines): - if "|-------|--------|" in line: + if TABLE_SEPARATOR in line: updated_lines.insert(i + 1, batch_row) break - # Add failed tests section if needed if failed_section: - # Find the Failed Tests section for i, line in enumerate(updated_lines): - if "## Failed Tests" in line: - # Insert after the heading + if FAILED_SECTION_HEADER in line: updated_lines.insert(i + 2, failed_section) break return "\n".join(updated_lines) -def create_issue_body( - batch_row: str, failed_section: str, repo: str, workflow_url: str -) -> str: - """Create initial issue body.""" +def create_issue_body(batch_row: str, failed_section: str, repo: str, workflow_url: str) -> str: failed_text = failed_section if failed_section else "_No failures currently tracked_" return f"""# 🧪 Monthly Module Test Dashboard @@ -172,7 +153,6 @@ def create_issue_body( def main(): - # Get environment variables batch_num = os.environ["BATCH_NUM"] batch_label = os.environ["BATCH_LABEL"] status = os.environ["STATUS"] @@ -184,25 +164,19 @@ def main(): repo = os.environ["GITHUB_REPOSITORY"] workflow_url = f"https://github.com/{repo}/actions/workflows/monthly-full-test.yml" - # Read failed tests failed_tests = [] - if os.path.exists("failed_tests.txt"): + try: with open("failed_tests.txt") as f: failed_tests = [line.strip() for line in f if line.strip()] + except FileNotFoundError: + pass - # Create batch row - batch_row = create_batch_row( - batch_num, batch_label, status, passed_count, total_count, run_number, run_url - ) - - # Create failed tests section - failed_section = create_failed_tests_section(batch_num, failed_count, failed_tests) if failed_count > 0 else "" + batch_row = create_batch_row(batch_num, batch_label, status, passed_count, total_count, run_number, run_url) + failed_section = create_failed_tests_section(batch_num, failed_count, failed_tests) - # Find or create issue - issue_number = get_issue_number() + existing = get_existing_issue() - if issue_number is None: - # Create new issue + if existing is None: body = create_issue_body(batch_row, failed_section, repo, workflow_url) result = github_api_request( "POST", @@ -215,10 +189,8 @@ def main(): print("❌ Failed to create issue", file=sys.stderr) sys.exit(1) else: - # Update existing issue - current_body = get_issue_body(issue_number) + issue_number, current_body = existing updated_body = update_body(current_body, batch_num, batch_row, failed_section) - result = github_api_request("PATCH", f"issues/{issue_number}", {"body": updated_body}) if result: print(f"✅ Updated dashboard issue #{issue_number}")