From 406ab6f48cb0d6196ee827dd0c9b9a6f9b3489de Mon Sep 17 00:00:00 2001 From: Albert Malewski Date: Wed, 27 May 2026 19:18:32 +0000 Subject: [PATCH] [CI] Emit FrameworkWeb ingest JSON from parity workflow Add `.automation_scripts/pytorch-unit-test-scripts/summary.py`, a port of the FrameworkWeb `summary.py` that converts a directory of JUnit XML reports into the JSON schema accepted by the FrameworkWeb ingest endpoint: { build: {url, branch, commit, gfxArch, repoOwner, rocmVersion, pytorchVersion, testConfig, buildTimestamp}, results: [{file, classname, name, time, status}, ...] } The local copy accepts repeated `--input-dir` and a named `--output-json` so the workflow can group per-shard subdirectories (`test-default-i-N`, `test-distributed-i-N`, `test-inductor-i-N`) by test config and emit one JSON per (arch, config) pair. Update `parity.yml`: * Add optional `rocm_version` / `pytorch_version` workflow_dispatch inputs (recorded in the JSON; default ""). * After the existing log-failure detection step, group shard dirs under `rocm_xml/` by config and invoke `summary.py` per config. Output is named `${DATE}_${ARCH}_${CFG}_ingest.json` in the per-arch result folder. * Include `*.json` in the per-arch artifact upload paths. Closes ROCm/frameworks-internal#16703 --- .../pytorch-unit-test-scripts/summary.py | 134 ++++++++++++++++++ .github/workflows/parity.yml | 75 ++++++++++ 2 files changed, 209 insertions(+) create mode 100644 .automation_scripts/pytorch-unit-test-scripts/summary.py diff --git a/.automation_scripts/pytorch-unit-test-scripts/summary.py b/.automation_scripts/pytorch-unit-test-scripts/summary.py new file mode 100644 index 0000000000000..388408a83ba3e --- /dev/null +++ b/.automation_scripts/pytorch-unit-test-scripts/summary.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Convert JUnit XML test reports to a JSON ingest record for FrameworkWeb. + +Output schema matches the ingest endpoint described in +https://github.com/ROCm/frameworks-internal/tree/ingesting_ut_tracker/pytorch-unit-test-scripts/ut_results_ingestion/FrameworkWeb +(see utt/models.py). +""" + +import argparse +import json +import xml.etree.ElementTree as ET +from datetime import datetime, timezone +from pathlib import Path + + +class Status: + PASSED = "passed" + SKIPPED = "skipped" + FAILED = "failed" + MISSED = "missed" + XFAILED = "xfailed" + ERROR = "error" + + +def _classify(testcase): + skipped = testcase.find("skipped") + if skipped is not None: + if skipped.attrib.get("type") == "pytest.xfail": + return Status.XFAILED + return Status.SKIPPED + if testcase.find("failure") is not None: + return Status.FAILED + if testcase.find("error") is not None: + return Status.ERROR + return Status.PASSED + + +def _collect(node, results): + for child in node: + if child.tag == "testcase": + try: + results.append({ + "file": child.attrib["file"], + "classname": child.attrib["classname"], + "name": child.attrib["name"], + "time": child.attrib["time"], + "status": _classify(child), + }) + except KeyError: + # Skip testcases missing required attributes (e.g., aggregated + # entries with no file/classname/name/time). + pass + _collect(child, results) + + +def import_dirs(input_dirs): + results = [] + for input_dir in input_dirs: + before = len(results) + for xml_path in Path(input_dir).rglob("*.xml"): + try: + tree = ET.parse(xml_path) + except ET.ParseError as exc: + print(f"WARNING: skipping malformed XML {xml_path}: {exc}") + continue + _collect(tree.getroot(), results) + print(f"Collected {len(results) - before} test results from {input_dir}") + return results + + +def build_info(args): + return { + "url": args.build_url or "", + "branch": args.branch_name or "", + "commit": args.commit_sha or "", + "gfxArch": args.gfx_arch or "", + "repoOwner": args.repo_owner or "", + "rocmVersion": args.rocm_version or "", + "pytorchVersion": args.pytorch_version or "", + "testConfig": args.test_config or "", + "buildTimestamp": ( + args.build_timestamp + if args.build_timestamp + else str(datetime.now(tz=timezone.utc)) + ), + } + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Convert XML test reports to a FrameworkWeb JSON ingest record" + ) + parser.add_argument( + "--input-dir", + dest="input_dirs", + type=str, + action="append", + required=True, + help="Directory containing JUnit XML files (recursive). May be repeated.", + ) + parser.add_argument( + "--output-json", + dest="output_json", + type=str, + required=True, + help="Path to write JSON output", + ) + parser.add_argument("--build_url", type=str, default="", help="CI build URL") + parser.add_argument("--branch_name", type=str, default="", help="Source branch name") + parser.add_argument("--commit_sha", type=str, default="", help="Commit SHA under test") + parser.add_argument("--gfx_arch", type=str, default="", help="GPU architecture (e.g. gfx942)") + parser.add_argument("--repo_owner", type=str, default="", help="Repository owner") + parser.add_argument("--rocm_version", type=str, default="", help="ROCm version") + parser.add_argument("--pytorch_version", type=str, default="", help="PyTorch version") + parser.add_argument("--test_config", type=str, default="", help="Test config (default, distributed, inductor)") + parser.add_argument("--build_timestamp", type=str, default="", help="Override build timestamp (default: now UTC)") + return parser.parse_args() + + +def main(): + args = parse_args() + data = { + "build": build_info(args), + "results": import_dirs(args.input_dirs), + } + output_path = Path(args.output_json) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + json.dump(data, f, default=str) + print(f"Wrote {output_path} ({output_path.stat().st_size} bytes)") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/parity.yml b/.github/workflows/parity.yml index 5f88548712818..9dd20558272d2 100644 --- a/.github/workflows/parity.yml +++ b/.github/workflows/parity.yml @@ -82,6 +82,17 @@ on: required: false default: false type: boolean + # FrameworkWeb ingest JSON metadata + rocm_version: + description: 'ROCm version recorded in the FrameworkWeb ingest JSON. Optional.' + required: false + default: '' + type: string + pytorch_version: + description: 'PyTorch version recorded in the FrameworkWeb ingest JSON. Optional.' + required: false + default: '' + type: string jobs: setup-matrix: @@ -256,11 +267,75 @@ jobs: echo "No log files found in $FOLDER, skipping log failure detection" fi + - name: Generate FrameworkWeb ingest JSON + if: ${{ !inputs.skip_rocm }} + working-directory: .automation_scripts/pytorch-unit-test-scripts + run: | + set -euo pipefail + FOLDER="${{ steps.folder.outputs.folder }}" + SHA="${{ steps.folder.outputs.sha }}" + DATE="${{ steps.folder.outputs.date }}" + ARCH="${{ matrix.arch }}" + XML_ROOT="$FOLDER/rocm_xml" + + if [ ! -d "$XML_ROOT" ]; then + echo "No $XML_ROOT directory; nothing to ingest." + exit 0 + fi + + # Map ROCm arch label to gfx target for FrameworkWeb's gfxArch field. + case "$ARCH" in + mi300) GFX=gfx942 ;; + mi355) GFX=gfx950 ;; + mi200) GFX=gfx90a ;; + navi31) GFX=gfx1100 ;; + nightly) GFX=gfx942 ;; + *) GFX="$ARCH" ;; + esac + + BUILD_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + BRANCH="${{ github.ref_name }}" + REPO_OWNER="${{ github.repository_owner }}" + + # download_testlogs extracts per-shard subdirs like test-default-1-6, + # test-distributed-1-3, test-inductor-1-N directly under rocm_xml/. + # Group them by config and emit one JSON per (arch, config). + for CFG in default distributed inductor; do + DIRS=() + while IFS= read -r d; do + [ -n "$d" ] && DIRS+=("$d") + done < <(find "$XML_ROOT" -maxdepth 1 -mindepth 1 -type d -name "test-${CFG}-*" 2>/dev/null | sort) + if [ "${#DIRS[@]}" -eq 0 ]; then + echo "No shards found for config '$CFG', skipping." + continue + fi + + OUT="$FOLDER/${DATE}_${ARCH}_${CFG}_ingest.json" + INPUT_ARGS=() + for d in "${DIRS[@]}"; do + INPUT_ARGS+=(--input-dir "$d") + done + + echo "Generating $OUT from ${#DIRS[@]} shard dir(s)" + python3 summary.py \ + "${INPUT_ARGS[@]}" \ + --output-json "$OUT" \ + --build_url "$BUILD_URL" \ + --branch_name "$BRANCH" \ + --commit_sha "$SHA" \ + --gfx_arch "$GFX" \ + --repo_owner "$REPO_OWNER" \ + --rocm_version "${{ inputs.rocm_version }}" \ + --pytorch_version "${{ inputs.pytorch_version }}" \ + --test_config "$CFG" + done + - name: Collect upload paths id: upload-paths run: | FOLDER=".automation_scripts/pytorch-unit-test-scripts/${{ steps.folder.outputs.folder }}" PATHS="${FOLDER}/*.csv + ${FOLDER}/*.json ${FOLDER}/*.log ${FOLDER}/*.txt ${FOLDER}/inductor_periodic_rocm_dir/