From 123b68785b03fa6cca035026a93c01fd0ced2f72 Mon Sep 17 00:00:00 2001 From: Liang Juhao Date: Fri, 15 May 2026 13:18:10 +0800 Subject: [PATCH] fix(suite_C): describe the actual reason a precision level is skipped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the cleanup in #45. That PR removed the runner-declared quantization-backend gating logic and renamed the obvious skip-reason in the headline `print` (line 101), but two sibling references to the old strategy were missed: * The function-level docstring still claimed format selection intersects with `runner.SUPPORTED_QUANTIZATIONS` and warns on any format the runner doesn't declare. * The per-format final-summary line printed `skipped (backend not in SUPPORTED_QUANTIZATION_BACKENDS)` even though the `skipped` list now only ever holds the *other* full-precision baseline (e.g. FP16 on Ampere where the hw baseline is BF16). Rewrite both so the docstring describes today's policy (always include the hw-supported full-precision baseline; dispatch every quantized level; let the inference subprocess decide hardware compatibility) and the skip-reason print matches what actually causes the entry. The result.json field name `precision_levels_skipped` is **kept** — it's a stable schema field already indexed by the leaderboard and used by older results, so the name stays; only the human-readable strings around it are corrected. No functional change. Co-authored-by: Cursor --- suites/suite_C/suite.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/suites/suite_C/suite.py b/suites/suite_C/suite.py index d360226..7200ab9 100644 --- a/suites/suite_C/suite.py +++ b/suites/suite_C/suite.py @@ -53,10 +53,13 @@ def _run_suite_c(br, args, suite: dict, env_info: dict) -> None: without blocking. Missing baselines (placeholders) are allowed. Format selection: - - Always includes BF16 (baseline) - - Other formats: intersection of suite["precision_levels"] and - runner.SUPPORTED_QUANTIZATIONS - - Formats the runner doesn't declare are skipped with a warning + - Always includes the hardware-supported full-precision baseline + (BF16 on Ampere+, FP16 on V100/T4); the *other* full-precision + baseline is recorded under "skipped". + - All quantized formats listed in suite["precision_levels"] are + dispatched. Hardware/engine compatibility is left to the inference + subprocess — if the engine cannot run a format, its own error is + captured in the per-format summary. """ base_dir = Path(args.output_dir) base_dir.mkdir(parents=True, exist_ok=True) @@ -218,7 +221,7 @@ def _run_suite_c(br, args, suite: dict, env_info: dict) -> None: print(f" [{icon}] {precision:6s} — {status}") if skipped: for p in skipped: - print(f" [—] {p:6s} — skipped (backend not in SUPPORTED_QUANTIZATION_BACKENDS)") + print(f" [—] {p:6s} — skipped (other full-precision baseline)") print() successful = [p for p, status, _ in results_summary if status == "SUCCESS"]