From 427cac18f29a0d200132866ca50ba1ce5420b94d Mon Sep 17 00:00:00 2001 From: Christos Kotsalos Date: Thu, 30 Apr 2026 14:39:40 +0200 Subject: [PATCH 1/4] Infrastructure to test DaCe's codegen (in)deterministic behavior --- ci/dace_deterministic_codegen/README.md | 172 +++++ .../bootstrap_icon4py.py | 182 +++++ .../dace_deterministic_codegen.py | 626 ++++++++++++++++++ 3 files changed, 980 insertions(+) create mode 100644 ci/dace_deterministic_codegen/README.md create mode 100644 ci/dace_deterministic_codegen/bootstrap_icon4py.py create mode 100644 ci/dace_deterministic_codegen/dace_deterministic_codegen.py diff --git a/ci/dace_deterministic_codegen/README.md b/ci/dace_deterministic_codegen/README.md new file mode 100644 index 0000000000..b47d4ee1cf --- /dev/null +++ b/ci/dace_deterministic_codegen/README.md @@ -0,0 +1,172 @@ +# dace_deterministic_codegen + +Determinism check for gt4py's DaCe backend. Runs an icon4py test selection +through `nox` **twice** with isolated gt4py build caches, then compares +the generated source code under each program's `src/` between the two +runs. Exit 0 = identical (deterministic), exit 1 = different. + +Currently supports the **cpu**, **cuda**, and **HIP** dace backends. +HIP is supported transparently: dace emits HIP code under `src/cuda/hip/` +(target_name="cuda", target_type="hip"), and the harness's recursive +sweep of `src/cuda/` picks it up automatically. If a run emits anything +else under `src/` (mpi, sve, mlir, snitch, …) the harness fails +immediately with a clear message — silently ignoring an unfamiliar +backend would mean reporting "deterministic" without actually checking +the relevant code. + +Valid `--selection` and `--component` values are read from icon4py's +own `noxfile.py` at runtime — no hardcoding here, so the harness +auto-tracks any future changes to icon4py's parametrization. + +Mirrors icon4py's `ci/dace.yml`, with the session name configurable: + +```bash +nox -r -s "-(, )" -- +``` + +Default `` is `test_model` — what `ci/dace.yml` itself uses. + +## A note on paths + +Every `--*` flag that takes a path (`--icon4py`, `--gt4py`, `--dace`) +accepts **both absolute and relative** paths. Relative paths are +resolved against the current working directory — i.e. wherever you +invoke the script from, not where the script lives. The script prints +the resolved absolute path on startup whenever you pass a relative one, +so you can confirm what it landed on. + +## Setup (one-time) + +Done once per machine, before any check is run. + +**1. Activate the gt4py venv** with editable gt4py (and dace, if on a +custom branch): + +```bash +source /path/to/gt4py-venv/bin/activate +uv pip install -e /path/to/gt4py +uv pip install -e /path/to/dace # optional, if custom dace branch +``` + +**2. Bootstrap icon4py into that same venv.** This patches icon4py's +`[tool.uv.sources]` so the editable gt4py / dace are what `uv sync` +installs into nox's session venv: + +```bash +uv pip install tomli_w +python /path/to/gt4py/ci/dace_deterministic_codegen/bootstrap_icon4py.py \ + --icon4py /path/to/icon4py \ + --gt4py /path/to/gt4py \ + --dace /path/to/dace # omit if upstream dace +``` + +**3. Sanity check:** + +```bash +python -c "import gt4py.next; print(gt4py.next.__file__)" +# must print a path inside your gt4py checkout, NOT site-packages/ +``` + +## Run the check + +With the venv from step 1 active: + +```bash +python /path/to/gt4py/ci/dace_deterministic_codegen/dace_deterministic_codegen.py \ + --icon4py /path/to/icon4py \ + --selection \ + --component \ + --posarg=--backend=dace_cpu \ + --posarg=--grid=icon_regional +``` + +The valid values for `--selection` and `--component` are read directly +from icon4py's `noxfile.py` at runtime. As of icon4py main, that's: + +- `--selection`: `datatest`, `stencils`, `basic` +- `--component`: `advection`, `diffusion`, `dycore`, `microphysics`, + `muphys`, `common`, `driver`, `standalone_driver`, `testing` + +If icon4py adds or renames these, the harness picks it up automatically; +no update needed here. If you pass an invalid value, the error message +lists the actual valid set extracted from your icon4py checkout. + +## Examples + +**Stencils for muphys, CPU** — mirrors `ci/dace.yml`'s stencil pattern: + +```bash +python $GT4PY/ci/dace_deterministic_codegen/dace_deterministic_codegen.py \ + --icon4py $ICON4PY \ + --selection stencils \ + --component muphys \ + --posarg=--backend=dace_cpu \ + --posarg=--grid=icon_regional +``` + +**Datatest for dycore, GPU** — mirrors the datatest pattern: + +```bash +python $GT4PY/ci/dace_deterministic_codegen/dace_deterministic_codegen.py \ + --icon4py $ICON4PY \ + --selection datatest \ + --component dycore \ + --posarg=--backend=dace_gpu \ + --posarg=--level=integration +``` + +**Custom session** — say a future icon4py defines a `test_other` +session with the same parametrization shape: + +```bash +python $GT4PY/ci/dace_deterministic_codegen/dace_deterministic_codegen.py \ + --icon4py $ICON4PY \ + --session test_other \ + --selection stencils \ + --component muphys \ + --posarg=--backend=dace_cpu +``` + +## Output + +By default, everything lands at `/_dace_deterministic_codegen/`. +Override with `--workdir PATH` (absolute or relative): + +``` +/ +├── run1/.gt4py_cache/... run1/test.log +├── run2/.gt4py_cache/... run2/test.log +├── diffs//.diff (only on mismatch) +└── report.txt (human-readable summary) +``` + +**Re-running wipes the workdir.** Whatever was there before — old logs, +old caches, an old `report.txt` from yesterday — is removed before the +new run starts. No merging, no appending. If you want to keep history +across invocations, copy the directory before re-running. + +## Exit codes + +| Code | Meaning | +|------|---------| +| 0 | Codegen is deterministic. | +| 1 | Codegen differs (see `report.txt` and `diffs/`). | +| 2 | Bad arguments (path doesn't exist, missing noxfile, …). | +| 3 | No programs observed in either run (test selection collected nothing). | +| 4 | A `nox` invocation itself failed (see `run1/test.log` / `run2/test.log`). | + +## Flags + +``` +--icon4py PATH icon4py checkout, abs or rel (required) +--session NAME nox session name (default: test_model) +--selection NAME noxfile selection (required); validated against + icon4py's actual noxfile at runtime +--component NAME leaf subpackage name (required); validated + against icon4py's actual noxfile at runtime +--python X.Y python version for the nox session (default: 3.10) +--workdir PATH where run1/, run2/, diffs/, report.txt land, + abs or rel (default: /_dace_deterministic_codegen/). + Wiped before each run. +--posarg ARG forwarded to pytest. Repeatable. +``` diff --git a/ci/dace_deterministic_codegen/bootstrap_icon4py.py b/ci/dace_deterministic_codegen/bootstrap_icon4py.py new file mode 100644 index 0000000000..4ceb05e6db --- /dev/null +++ b/ci/dace_deterministic_codegen/bootstrap_icon4py.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# GT4Py - GridTools Framework +# +# Copyright (c) 2014-2024, ETH Zurich +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +"""Bootstrap icon4py into the *currently activated* venv (the gt4py CI venv). + +Edits icon4py's `pyproject.toml` so that `[tool.uv.sources]` points +`gt4py` (and optionally `dace`) at local-path editable installs, regenerates +the lockfile, and runs `uv sync --active` to install icon4py + its other +dependencies into `$VIRTUAL_ENV`. + +This is what makes the editable gt4py / dace branches survive everything +downstream — including the icon4py noxfile's own `uv sync` call when our +dace_deterministic_codegen harness runs `nox --no-venv`. + +Usage (run from anywhere): + + python ci/dace_deterministic_codegen/bootstrap_icon4py.py \\ + --icon4py /path/to/icon4py \\ + --gt4py /path/to/gt4py-dace_toolchain_deterministic \\ + --dace /path/to/dace # optional + +Idempotent: re-running it is safe; the [tool.uv.sources] entries are +overwritten in place. +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + +try: + import tomllib # Python 3.11+ +except ModuleNotFoundError: + import tomli as tomllib # type: ignore[import-not-found] + +try: + import tomli_w +except ModuleNotFoundError: + print( + "error: this script needs `tomli_w`. install with:\n" + " uv pip install tomli_w # or pip install tomli_w", + file=sys.stderr, + ) + sys.exit(2) + + +def _is_python_project(path: Path) -> bool: + """A directory is installable by uv if it has any of these markers.""" + return any((path / m).is_file() for m in ("pyproject.toml", "setup.py", "setup.cfg")) + + +def patch_sources(pyproject: Path, overrides: dict[str, Path]) -> None: + """Set `[tool.uv.sources][] = {path = "...", editable = true}` for + every (pkg, path) in overrides. Other entries are preserved.""" + with pyproject.open("rb") as f: + doc = tomllib.load(f) + + sources = ( + doc.setdefault("tool", {}) + .setdefault("uv", {}) + .setdefault("sources", {}) + ) + for pkg, path in overrides.items(): + sources[pkg] = {"path": str(path), "editable": True} + + # Make a backup once. Idempotent: don't overwrite an existing backup, + # which would clobber the pristine original after a re-run. + backup = pyproject.with_suffix(pyproject.suffix + ".dace_deterministic_codegen.bak") + if not backup.exists(): + shutil.copy2(pyproject, backup) + + with pyproject.open("wb") as f: + tomli_w.dump(doc, f) + print(f"patched {pyproject} (backup at {backup.name})") + + +def run(cmd: list[str], cwd: Path) -> None: + print(f"+ {' '.join(cmd)} (cwd={cwd})") + rc = subprocess.run(cmd, cwd=str(cwd)).returncode + if rc != 0: + sys.exit(rc) + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0]) + p.add_argument( + "--icon4py", required=True, type=Path, metavar="PATH", + help=( + "Path to icon4py checkout. Accepts BOTH absolute and relative " + "paths. Relative paths are resolved against the current working " + "directory." + ), + ) + p.add_argument( + "--gt4py", required=True, type=Path, metavar="PATH", + help=( + "Path to gt4py checkout to install editable. Accepts BOTH " + "absolute and relative paths (resolved against cwd)." + ), + ) + p.add_argument( + "--dace", type=Path, default=None, metavar="PATH", + help=( + "Optional path to dace checkout (absolute or relative). If " + "omitted, dace resolves through icon4py's existing source pin." + ), + ) + p.add_argument("--no-lock", action="store_true", + help="Skip `uv lock`. Useful if you already locked.") + p.add_argument("--no-sync", action="store_true", + help="Skip `uv sync`. Useful for CI steps that sync later.") + args = p.parse_args() + + # Resolve every path NOW. The script can be run from any cwd. + icon4py = args.icon4py.expanduser().resolve() + gt4py = args.gt4py.expanduser().resolve() + dace = args.dace.expanduser().resolve() if args.dace else None + + pyproject = icon4py / "pyproject.toml" + if not pyproject.is_file(): + print(f"error: no pyproject.toml at {pyproject}", file=sys.stderr) + return 2 + if not _is_python_project(gt4py): + print( + f"error: --gt4py path is not a python project (no pyproject.toml, " + f"setup.py, or setup.cfg): {gt4py}", + file=sys.stderr, + ) + return 2 + if dace and not _is_python_project(dace): + print( + f"error: --dace path is not a python project (no pyproject.toml, " + f"setup.py, or setup.cfg): {dace}", + file=sys.stderr, + ) + return 2 + + # Loud warning if no venv is active — the whole point of this script + # is to install INTO the gt4py CI venv. Without VIRTUAL_ENV set, uv + # would create a new .venv and we'd get nowhere. + if not os.environ.get("VIRTUAL_ENV"): + print( + "warning: VIRTUAL_ENV is not set. This script is meant to install " + "icon4py into the *currently activated* venv (typically your " + "gt4py CI venv). Activate it first, then re-run.", + file=sys.stderr, + ) + + overrides: dict[str, Path] = {"gt4py": gt4py} + if dace: + overrides["dace"] = dace + patch_sources(pyproject, overrides) + + if not args.no_lock: + # Regenerate uv.lock so it matches the new [tool.uv.sources]. + run(["uv", "lock"], cwd=icon4py) + if not args.no_sync: + # --active = use $VIRTUAL_ENV (the gt4py venv) instead of ./venv/. + run(["uv", "sync", "--active"], cwd=icon4py) + + print() + print("done. quick sanity check:") + print(' python -c "import gt4py.next; print(gt4py.next.__file__)"') + print(f' # should print a path inside {gt4py}') + if dace: + print(' python -c "import dace; print(dace.__file__)"') + print(f' # should print a path inside {dace}') + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/dace_deterministic_codegen/dace_deterministic_codegen.py b/ci/dace_deterministic_codegen/dace_deterministic_codegen.py new file mode 100644 index 0000000000..402dcce18f --- /dev/null +++ b/ci/dace_deterministic_codegen/dace_deterministic_codegen.py @@ -0,0 +1,626 @@ +#!/usr/bin/env python3 +# GT4Py - GridTools Framework +# +# Copyright (c) 2014-2024, ETH Zurich +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +"""GT4Py / DaCe codegen determinism check. + +Drives an icon4py test selection through nox **twice** with isolated +gt4py build caches, then checks that the generated source files under +each program's `src/` are byte-identical between the two runs. A diff +is a determinism bug. + +Compares only the contents of `/src/` — the actual generated +backend code. Currently supports cpu, cuda, and hip (hip is emitted by +dace under `src/cuda/hip/`). Any other top-level backend under `src/` +(mpi, sve, mlir, snitch, …) causes the harness to fail with a clear +message rather than silently ignore it. + +Valid `--selection` and `--component` values are read from icon4py's +own `noxfile.py` at runtime (no hardcoding here), so the harness +tracks any future changes to icon4py's parametrization automatically. + +Mirrors icon4py's `ci/dace.yml` invocation pattern, with the session +name configurable: + + nox -r -s "-(, )" -- + +Defaults to `=test_model`, which is the icon4py main test +entry point and what `ci/dace.yml` uses. + +Outputs land at `/_dace_deterministic_codegen/`: + run1/.gt4py_cache/... run1/test.log + run2/.gt4py_cache/... run2/test.log + diffs//.diff (only on mismatch) + report.txt +""" + +from __future__ import annotations + +import argparse +import ast +import dataclasses +import difflib +import hashlib +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Optional + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +#: GT4Py names each cached program folder `_`. +PROGRAM_FOLDER_RE = re.compile(r"^(?P.+)_(?P[0-9a-f]{64})$") + +#: The single directory under each program folder we compare. Only `src/`, +#: nothing else — by design. dace also writes `include/`, `sample/`, +#: `program.sdfg`, source maps under `map/`, runtime metadata +#: (`dace.conf`, `*.csv`), and build artifacts under `build/`. None of +#: those are the codegen surface we care about for this check. +CODEGEN_ROOT = "src" + +#: Backends recognized as direct children of `src/`. dace lays out +#: codegen as `src//[/]`: +#: +#: - CPU codegen → src/cpu/.cpp +#: - CUDA codegen → src/cuda/.cu +#: - HIP codegen → src/cuda/hip/.cpp (NOTE: under cuda/) +#: +#: HIP is dispatched by dace's CUDA target with `target_type="hip"`, so +#: it lands as a *subdirectory* of `src/cuda/`, not its own top-level +#: backend folder. That means {cpu, cuda} as a top-level allowlist is +#: enough to cover all three: cpu via `cpu/`, cuda + hip both via +#: `cuda/` (with `rglob` picking up the nested hip files). +#: +#: If a snapshot ever encounters another top-level backend (mpi, sve, +#: mlir, snitch, …), the harness fails loudly rather than silently +#: ignoring — those would need explicit support added here. +SUPPORTED_BACKENDS: frozenset[str] = frozenset({"cpu", "cuda"}) + +#: Where outputs are written, relative to the icon4py checkout. +WORKDIR_NAME = "_dace_deterministic_codegen" + + +# --------------------------------------------------------------------------- +# icon4py noxfile introspection +# --------------------------------------------------------------------------- + +class NoxfileIntrospectionError(RuntimeError): + """Raised when we can't extract sessions/components from the noxfile.""" + + +def introspect_icon4py_noxfile( + noxfile: Path, +) -> tuple[frozenset[str], frozenset[str]]: + """Parse icon4py's noxfile.py and extract the valid `selection` and + `component` values. Returns `(selections, components)`. + + Reads the noxfile as AST — does not execute it. Two reasons: + importing would require `nox` in this script's environment, and + noxfile imports often have side effects (icon4py's pulls in a + handful of typing imports plus nox's own session machinery). + + Looks for two type-alias definitions matching icon4py main: + + ModelTestsSubset: TypeAlias = Literal["datatest", "stencils", "basic"] + ModelSubpackagePath: TypeAlias = Literal["atmosphere/advection", ...] + + Components are derived from the *leaf name* of each subpackage path + (`subpackage.split("/")[-1]`), matching the `id=...` icon4py uses + in nox.param. So `atmosphere/subgrid_scale_physics/muphys` becomes + the component `muphys`. + """ + if not noxfile.is_file(): + raise NoxfileIntrospectionError( + f"no noxfile.py at {noxfile} — is --icon4py the icon4py repo root?" + ) + + try: + tree = ast.parse(noxfile.read_text()) + except SyntaxError as e: + raise NoxfileIntrospectionError( + f"could not parse {noxfile} as Python: {e}" + ) from e + + selections = _extract_literal_strings(tree, "ModelTestsSubset") + subpackages = _extract_literal_strings(tree, "ModelSubpackagePath") + + if not selections: + raise NoxfileIntrospectionError( + f"could not find `ModelTestsSubset: TypeAlias = Literal[...]` " + f"in {noxfile}. icon4py's noxfile structure may have changed." + ) + if not subpackages: + raise NoxfileIntrospectionError( + f"could not find `ModelSubpackagePath: TypeAlias = Literal[...]` " + f"in {noxfile}. icon4py's noxfile structure may have changed." + ) + + components = frozenset(p.rsplit("/", 1)[-1] for p in subpackages) + return frozenset(selections), components + + +def _extract_literal_strings(tree: ast.AST, alias_name: str) -> list[str]: + """Find `: TypeAlias = Literal["a", "b", ...]` in the AST + and return the string literals. Returns [] if not found or shape is + unexpected (caller decides whether that's fatal).""" + for node in ast.walk(tree): + if not (isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name)): + continue + if node.target.id != alias_name: + continue + # Expect: value = Subscript(value=Name('Literal'), slice=Tuple(elts=[Constant, ...])) + v = node.value + if not isinstance(v, ast.Subscript): + continue + elts: list[ast.expr] = [] + if isinstance(v.slice, ast.Tuple): + elts = list(v.slice.elts) + else: + # Single-arg Literal["foo"] + elts = [v.slice] + out: list[str] = [] + for e in elts: + if isinstance(e, ast.Constant) and isinstance(e.value, str): + out.append(e.value) + return out + return [] + + +# --------------------------------------------------------------------------- +# Snapshot +# --------------------------------------------------------------------------- + +@dataclasses.dataclass(frozen=True) +class FileEntry: + relpath: str + sha256: str + + +@dataclasses.dataclass +class ProgramSnapshot: + name: str + folder: Path + files: dict[str, FileEntry] + + +class UnsupportedBackendError(RuntimeError): + """A program's `src/` contained a top-level backend other than cpu/cuda.""" + + +def snapshot_run(cache_root: Path) -> dict[str, ProgramSnapshot]: + """Walk a `.gt4py_cache` and snapshot every program's generated source. + + For each `_/` folder, we read everything under + `/src/` recursively. dace lays this out as + `src//[/]`: + + src/cpu/.cpp + src/cuda/.cu (CUDA — target_type="") + src/cuda/hip/.cpp (HIP — target_type="hip", under cuda/) + + Currently supports cpu and cuda as top-level backends. HIP is + handled implicitly because dace nests it inside `src/cuda/hip/`, + not as a separate top-level directory; the recursive walk picks + it up automatically. + + If we encounter any *other* top-level backend under `src/` (mpi, + sve, mlir, snitch, ...), raises UnsupportedBackendError so the + user knows immediately rather than silently skipping. + """ + if not cache_root.exists(): + return {} + + out: dict[str, ProgramSnapshot] = {} + for folder in sorted(p for p in cache_root.iterdir() if p.is_dir()): + m = PROGRAM_FOLDER_RE.match(folder.name) + if not m: + continue + name = m.group("name") + + src_root = folder / CODEGEN_ROOT + if not src_root.is_dir(): + # No src/ at all — record an empty snapshot. Pairing logic + # downstream will flag it if its counterpart in the other run + # has files. + out[name] = ProgramSnapshot(name=name, folder=folder, files={}) + continue + + # Backend check: every direct child of src/ must be a supported + # top-level backend. HIP lives nested under cuda/, so cuda is + # what matters here, not "hip". + backend_dirs = sorted(d for d in src_root.iterdir() if d.is_dir()) + for bd in backend_dirs: + if bd.name not in SUPPORTED_BACKENDS: + raise UnsupportedBackendError( + f"unsupported dace backend `{bd.name}/` found under " + f"{src_root} — this harness currently supports " + f"{sorted(SUPPORTED_BACKENDS)} as top-level backends " + f"(HIP is handled under `cuda/hip/`). Add explicit " + f"support in dace_deterministic_codegen.py before " + f"running this selection." + ) + + # rglob recursively descends — picks up `cuda/hip/` along + # with `cpu/` and `cuda/`, no special-casing needed. + files: dict[str, FileEntry] = {} + for fpath in sorted(src_root.rglob("*")): + if not fpath.is_file(): + continue + rel = fpath.relative_to(folder).as_posix() + files[rel] = FileEntry(relpath=rel, sha256=_sha256(fpath)) + out[name] = ProgramSnapshot(name=name, folder=folder, files=files) + return out + + +def _sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1 << 16), b""): + h.update(chunk) + return h.hexdigest() + + +# --------------------------------------------------------------------------- +# Compare +# --------------------------------------------------------------------------- + +@dataclasses.dataclass +class ProgramResult: + name: str + match: bool + differing_files: list[str] + only_in_run1: list[str] + only_in_run2: list[str] + + +def compare( + snap1: dict[str, ProgramSnapshot], + snap2: dict[str, ProgramSnapshot], +) -> list[ProgramResult]: + results: list[ProgramResult] = [] + for name in sorted(set(snap1) | set(snap2)): + s1 = snap1.get(name) + s2 = snap2.get(name) + + if s1 is None or s2 is None: + results.append(ProgramResult( + name=name, match=False, differing_files=[], + only_in_run1=sorted((s1.files if s1 else {}).keys()), + only_in_run2=sorted((s2.files if s2 else {}).keys()), + )) + continue + + keys1, keys2 = set(s1.files), set(s2.files) + only1 = sorted(keys1 - keys2) + only2 = sorted(keys2 - keys1) + differing = sorted( + rel for rel in keys1 & keys2 + if s1.files[rel].sha256 != s2.files[rel].sha256 + ) + results.append(ProgramResult( + name=name, + match=not (differing or only1 or only2), + differing_files=differing, + only_in_run1=only1, + only_in_run2=only2, + )) + return results + + +# --------------------------------------------------------------------------- +# Diff + report +# --------------------------------------------------------------------------- + +def write_diffs( + results: list[ProgramResult], + snap1: dict[str, ProgramSnapshot], + snap2: dict[str, ProgramSnapshot], + diffs_dir: Path, +) -> None: + for r in results: + if r.match: + continue + s1, s2 = snap1.get(r.name), snap2.get(r.name) + prog_dir = diffs_dir / r.name + for rel in r.differing_files: + f1 = (s1.folder / rel) if s1 else None + f2 = (s2.folder / rel) if s2 else None + if not (f1 and f2 and f1.exists() and f2.exists()): + continue + try: + t1 = f1.read_text().splitlines(keepends=True) + t2 = f2.read_text().splitlines(keepends=True) + except UnicodeDecodeError: + prog_dir.mkdir(parents=True, exist_ok=True) + (prog_dir / f"{rel.replace('/', '__')}.binary-differs").write_text( + f"binary content differs:\n run1: {f1}\n run2: {f2}\n" + ) + continue + udiff = "".join(difflib.unified_diff( + t1, t2, fromfile=f"run1/{rel}", tofile=f"run2/{rel}", n=3, + )) + prog_dir.mkdir(parents=True, exist_ok=True) + (prog_dir / f"{rel.replace('/', '__')}.diff").write_text(udiff) + + +def render_report(results: list[ProgramResult]) -> str: + n_total = len(results) + n_match = sum(1 for r in results if r.match) + n_diff = n_total - n_match + + lines = [f"Programs: {n_total} matches: {n_match} mismatches: {n_diff}", ""] + for r in results: + lines.append(f" [{'MATCH ' if r.match else 'DIFFER'}] {r.name}") + if not r.match: + for rel in r.differing_files: + lines.append(f" differs: {rel}") + for rel in r.only_in_run1: + lines.append(f" only in run1: {rel}") + for rel in r.only_in_run2: + lines.append(f" only in run2: {rel}") + + lines.append("") + if n_total == 0: + lines.append("RESULT: no programs observed (nothing was cached).") + elif n_diff == 0: + lines.append(f"RESULT: codegen deterministic — {n_match} program(s) match.") + else: + lines.append(f"RESULT: NON-DETERMINISTIC CODEGEN — {n_diff}/{n_total} program(s) differ.") + return "\n".join(lines) + "\n" + + +# --------------------------------------------------------------------------- +# Nox runner +# --------------------------------------------------------------------------- + +def run_nox( + icon4py: Path, run_dir: Path, log_path: Path, + session: str, selection: str, component: str, python: str, posargs: list[str], +) -> int: + """Run nox once with `GT4PY_BUILD_CACHE_DIR=run_dir`. Returns the exit code. + + Mirrors `ci/dace.yml`: positional session ID, `-r` to reuse the venv + between runs (so run1 and run2 see identical venv state — important + for the determinism check). + + NOTE: gt4py's config appends `.gt4py_cache` to GT4PY_BUILD_CACHE_DIR, + so `run_dir` is the *parent*: gt4py creates + `run_dir/.gt4py_cache/_/` inside it. + """ + session_id = f"{session}-{python}({selection}, {component})" + argv = ["nox", "-r", "-s", session_id] + if posargs: + argv.append("--") + argv.extend(posargs) + + env = dict(os.environ.items()) + env["GT4PY_BUILD_CACHE_DIR"] = str(run_dir) + env["GT4PY_BUILD_CACHE_LIFETIME"] = "persistent" + + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("w") as logf: + logf.write( + f"# cwd: {icon4py}\n" + "# command:\n " + "\n ".join(repr(a) for a in argv) + "\n" + f"# GT4PY_BUILD_CACHE_DIR={run_dir}\n" + f"# (gt4py appends .gt4py_cache; cache lands at {run_dir}/.gt4py_cache/)\n" + "# ---\n" + ) + logf.flush() + proc = subprocess.run(argv, cwd=str(icon4py), env=env, + stdout=logf, stderr=subprocess.STDOUT) + return proc.returncode + + +# --------------------------------------------------------------------------- +# Workdir +# --------------------------------------------------------------------------- + +@dataclasses.dataclass +class Workdir: + """Two parent dirs for gt4py's cache + a place for logs/diffs/report.""" + + root: Path + + @property + def run1_dir(self) -> Path: return self.root / "run1" + @property + def run2_dir(self) -> Path: return self.root / "run2" + @property + def cache1(self) -> Path: return self.run1_dir / ".gt4py_cache" + @property + def cache2(self) -> Path: return self.run2_dir / ".gt4py_cache" + @property + def log1(self) -> Path: return self.run1_dir / "test.log" + @property + def log2(self) -> Path: return self.run2_dir / "test.log" + @property + def diffs(self) -> Path: return self.root / "diffs" + @property + def report(self) -> Path: return self.root / "report.txt" + + def prepare(self) -> None: + """Wipe stale state from previous invocations.""" + for d in (self.run1_dir, self.run2_dir, self.diffs): + if d.exists(): + shutil.rmtree(d) + for d in (self.run1_dir, self.run2_dir): + d.mkdir(parents=True, exist_ok=True) + if self.report.exists(): + self.report.unlink() + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace: + p = argparse.ArgumentParser( + prog="dace_deterministic_codegen", + description=( + "Run an icon4py test selection twice via nox with isolated gt4py " + "caches and check that the generated source code is byte-identical." + ), + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p.add_argument( + "--icon4py", required=True, type=Path, metavar="PATH", + help=( + "Path to icon4py checkout. Accepts BOTH absolute and relative " + "paths. Relative paths are resolved against the current working " + "directory (i.e. wherever you invoke this script from)." + ), + ) + p.add_argument( + "--session", default="test_model", metavar="NAME", + help=( + "Nox session name. Composed with --python/--selection/--component " + "into the final session ID `-(, " + ")`. Default matches icon4py's main test entry point." + ), + ) + p.add_argument( + "--selection", required=True, metavar="NAME", + help=( + "icon4py noxfile selection (e.g. stencils, datatest, basic). " + "Validated at runtime against icon4py's actual noxfile." + ), + ) + p.add_argument( + "--component", required=True, metavar="NAME", + help=( + "icon4py noxfile subpackage leaf name (e.g. muphys, dycore). " + "Validated at runtime against icon4py's actual noxfile." + ), + ) + p.add_argument( + "--python", default="3.10", metavar="X.Y", + help="Python version for the nox session.", + ) + p.add_argument( + "--workdir", type=Path, default=None, metavar="PATH", + help=( + "Where run1/, run2/, diffs/, and report.txt are written. " + "Accepts absolute or relative paths (resolved against cwd). " + "If the directory already exists from a prior run, its contents " + "are wiped before this run starts — no merging or appending. " + "Default: /_dace_deterministic_codegen/" + ), + ) + p.add_argument( + "--posarg", action="append", default=[], dest="posargs", metavar="ARG", + help=( + "Forwarded to pytest via `nox -- ARG`. Repeatable. " + "Example: --posarg=--backend=dace_cpu --posarg=--grid=icon_regional" + ), + ) + return p.parse_args(argv) + + +def main(argv: Optional[list[str]] = None) -> int: + args = parse_args(argv) + + # Resolve every path to absolute up-front, so the harness can be run + # from any cwd. We print what the path resolved to — `--icon4py ../foo` + # behaves intuitively but it's nice to confirm what it landed on. + icon4py = args.icon4py.expanduser().resolve() + if not args.icon4py.is_absolute(): + print(f"--icon4py resolved to: {icon4py}") + if not icon4py.is_dir(): + print(f"error: --icon4py path is not a directory: {icon4py}", file=sys.stderr) + return 2 + noxfile_path = icon4py / "noxfile.py" + if not noxfile_path.is_file(): + print(f"error: no noxfile.py at {noxfile_path} — is --icon4py " + f"the icon4py repo root?", file=sys.stderr) + return 2 + + # Introspect icon4py's noxfile to discover the legal selection / + # component values. This avoids hardcoding the lists, so the harness + # auto-tracks any future changes to icon4py's noxfile structure. + try: + valid_selections, valid_components = introspect_icon4py_noxfile(noxfile_path) + except NoxfileIntrospectionError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + + if args.selection not in valid_selections: + print( + f"error: --selection {args.selection!r} is not one of " + f"{sorted(valid_selections)} (extracted from {noxfile_path})", + file=sys.stderr, + ) + return 2 + if args.component not in valid_components: + print( + f"error: --component {args.component!r} is not one of " + f"{sorted(valid_components)} (extracted from {noxfile_path})", + file=sys.stderr, + ) + return 2 + + workdir_root = ( + args.workdir.expanduser().resolve() + if args.workdir is not None + else icon4py / WORKDIR_NAME + ) + if args.workdir is not None and not args.workdir.is_absolute(): + print(f"--workdir resolved to: {workdir_root}") + workdir = Workdir(root=workdir_root) + workdir.prepare() # wipes run1/, run2/, diffs/, report.txt — see prepare() + + session_id = f"{args.session}-{args.python}({args.selection}, {args.component})" + + # ----- Run 1 + print(f"[1/2] nox -s '{session_id}' (cache: {workdir.run1_dir})", flush=True) + rc1 = run_nox(icon4py, workdir.run1_dir, workdir.log1, + args.session, args.selection, args.component, args.python, args.posargs) + if rc1 != 0: + print(f"error: run 1 failed (exit {rc1}). See log: {workdir.log1}", + file=sys.stderr) + return 4 + + # ----- Run 2 + print(f"[2/2] nox -s '{session_id}' (cache: {workdir.run2_dir})", flush=True) + rc2 = run_nox(icon4py, workdir.run2_dir, workdir.log2, + args.session, args.selection, args.component, args.python, args.posargs) + if rc2 != 0: + print(f"error: run 2 failed (exit {rc2}). See log: {workdir.log2}", + file=sys.stderr) + return 4 + + # ----- Snapshot + compare + report + try: + snap1 = snapshot_run(workdir.cache1) + snap2 = snapshot_run(workdir.cache2) + except UnsupportedBackendError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + results = compare(snap1, snap2) + write_diffs(results, snap1, snap2, workdir.diffs) + report = render_report(results) + workdir.report.write_text(report) + print(report) + print(f"workdir: {workdir.root}") + + if not results: + print(f"error: no programs observed in either run — check the logs:\n" + f" {workdir.log1}\n {workdir.log2}", file=sys.stderr) + return 3 + + return 1 if any(not r.match for r in results) else 0 + + +if __name__ == "__main__": + sys.exit(main()) From 655fa5f30e1da7d56b19a85b64c55654fde43479 Mon Sep 17 00:00:00 2001 From: Christos Kotsalos Date: Mon, 4 May 2026 08:00:26 +0200 Subject: [PATCH 2/4] Infrastructure to test DaCe's codegen (in)deterministic behavior: CI/CD integration [WIP] --- ci/cscs-ci-dace-determinism.yml | 136 +++++++++++ ci/cscs-ci.yml | 1 + ci/dace_deterministic_codegen/README.md | 38 +++ .../bootstrap_icon4py.py | 44 ++-- .../dace_deterministic_codegen.py | 185 +++++++++----- ci/dace_deterministic_codegen/run_in_ci.sh | 231 ++++++++++++++++++ 6 files changed, 563 insertions(+), 72 deletions(-) create mode 100644 ci/cscs-ci-dace-determinism.yml create mode 100644 ci/dace_deterministic_codegen/run_in_ci.sh diff --git a/ci/cscs-ci-dace-determinism.yml b/ci/cscs-ci-dace-determinism.yml new file mode 100644 index 0000000000..b71e209211 --- /dev/null +++ b/ci/cscs-ci-dace-determinism.yml @@ -0,0 +1,136 @@ +# +# GT4Py - GridTools Framework +# +# Copyright (c) 2014-2024, ETH Zurich +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause +# + +# DaCe codegen determinism check (icon4py-driven) +# =============================================== +# +# Drives an icon4py test selection through nox twice with isolated +# GT4PY_BUILD_CACHE_DIR per run, then asserts the generated source +# under each /src/ is byte-identical between the two runs. +# A diff means the gt4py + dace toolchain is non-deterministic for +# that selection. +# +# The actual logic lives in: +# ci/dace_deterministic_codegen/dace_deterministic_codegen.py (harness) +# ci/dace_deterministic_codegen/bootstrap_icon4py.py (venv prep) +# ci/dace_deterministic_codegen/run_in_ci.sh (CI driver) +# +# This file just wires those into GitLab CI: when to run, on which +# runners, with which selection × component matrix. +# +# Failure semantics +# ----------------- +# `allow_failure: true` while the determinism work stabilizes — +# surface regressions on the dashboard without gating merges. Drop +# `allow_failure` once we have a sustained green stretch on `main`. +# +# Custom dace branch +# ------------------ +# To test a dace fork/branch (e.g. unmerged determinism work), set +# DACE_REPO and DACE_REF in the variables block below. When unset, +# dace resolves through icon4py's existing source pin (currently the +# GridTools/pypi published wheel). See run_in_ci.sh for the details. + +stages: + - dace-determinism + +variables: + ICON4PY_REPO: 'https://github.com/C2SM/icon4py.git' + ICON4PY_REF: 'main' + + # Custom dace fork/branch under test. Leave both empty to resolve + # dace through icon4py's existing source pin. Set both together to + # install editable dace from a clone of DACE_REPO @ DACE_REF. + # Example for the deterministic codegen branch: + # DACE_REPO: 'https://github.com/GridTools/dace.git' + # DACE_REF: 'dace_toolchain_deterministic' + DACE_REPO: '' + DACE_REF: '' + +# Shared template for all dace-determinism jobs +.dace_determinism_common: + stage: dace-determinism + image: ${CSCS_REGISTRY_PATH}/public/${ARCH}/base/gt4py-ci-${PY_VERSION}:${DOCKER_TAG} + variables: + PY_VERSION: '3.10' + DACE_DETERMINISM_PYTHON: '${PY_VERSION}' + DACE_DETERMINISM_SELECTION: 'stencils' + DACE_DETERMINISM_GRID: 'icon_regional' + SLURM_JOB_NUM_NODES: 1 + SLURM_TIMELIMIT: 30 + # Don't block merges on determinism regressions — yet. Flip to + # `false` once the toolchain is reliably converged. + allow_failure: true + artifacts: + when: always + paths: + - _dace_deterministic_codegen/${DACE_DETERMINISM_BACKEND}/${DACE_DETERMINISM_COMPONENT}/ + expire_in: 1 month + script: + - mkdir -p "${WORKDIR}/gt4py" && git clone --depth 1 "${CSCS_CI_ORIG_CLONE_URL}" "${WORKDIR}/gt4py" + - cd "${WORKDIR}/gt4py" && git fetch --depth 1 origin "${CI_COMMIT_SHA}" && git checkout "${CI_COMMIT_SHA}" + # The gt4py CI Docker image already sets VIRTUAL_ENV and prepends + # ${VIRTUAL_ENV}/bin to PATH (see ci/Dockerfile), so + # `python`, `uv`, `nox` etc. resolve to the venv binaries without + # sourcing activate. Matches the .test_common pattern. + - export GT4PY_PATH="${WORKDIR}/gt4py" + - export ICON4PY_PATH="${WORKDIR}/icon4py" + # Custom dace branch: only set DACE_PATH if DACE_REPO is non-empty. + # run_in_ci.sh treats DACE_PATH (along with DACE_REPO/DACE_REF) as + # the trigger to clone + install dace editable. + - if [ -n "${DACE_REPO}" ]; then export DACE_PATH="${WORKDIR}/dace"; fi + # Per-cell artifact subdirectory (matches artifacts.paths above). + - export DACE_DETERMINISM_ARTIFACT_DIR="${CI_PROJECT_DIR}/_dace_deterministic_codegen/${DACE_DETERMINISM_BACKEND}/${DACE_DETERMINISM_COMPONENT}" + - bash "${WORKDIR}/gt4py/ci/dace_deterministic_codegen/run_in_ci.sh" + +dace_determinism_cscs_gh200_cuda: + extends: + - .container-runner-santis-gh200 + - .dace_determinism_common + needs: + - job: build_cscs_gh200 + parallel: + matrix: + - PY_VERSION: '3.10' + variables: + DACE_DETERMINISM_BACKEND: 'dace_gpu' + SLURM_GPUS_PER_NODE: 1 + SLURM_PARTITION: 'shared' + GT4PY_BUILD_JOBS: 8 + PYTEST_XDIST_AUTO_NUM_WORKERS: 32 + parallel: + matrix: + - DACE_DETERMINISM_COMPONENT: + - dycore + - advection + - diffusion + - muphys + +dace_determinism_cscs_gh200_cpu: + extends: + - .container-runner-santis-gh200 + - .dace_determinism_common + needs: + - job: build_cscs_gh200 + parallel: + matrix: + - PY_VERSION: '3.10' + variables: + DACE_DETERMINISM_BACKEND: 'dace_cpu' + SLURM_PARTITION: 'shared' + GT4PY_BUILD_JOBS: 8 + PYTEST_XDIST_AUTO_NUM_WORKERS: 32 + parallel: + matrix: + - DACE_DETERMINISM_COMPONENT: + - dycore + - advection + - diffusion + - muphys diff --git a/ci/cscs-ci.yml b/ci/cscs-ci.yml index 0f4475ad61..6ec2e1e96c 100644 --- a/ci/cscs-ci.yml +++ b/ci/cscs-ci.yml @@ -11,6 +11,7 @@ include: - remote: 'https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml' - local: 'ci/cscs-ci-ext-config.yml' + - local: 'ci/cscs-ci-dace-determinism.yml' # Note: # block-name-with-dashes -> defined in remote cscs-ci ext include diff --git a/ci/dace_deterministic_codegen/README.md b/ci/dace_deterministic_codegen/README.md index b47d4ee1cf..371ce446ba 100644 --- a/ci/dace_deterministic_codegen/README.md +++ b/ci/dace_deterministic_codegen/README.md @@ -170,3 +170,41 @@ across invocations, copy the directory before re-running. Wiped before each run. --posarg ARG forwarded to pytest. Repeatable. ``` + +## CI integration + +The harness runs in CSCS CI as a separate `dace-determinism` stage, +defined in `ci/cscs-ci-dace-determinism.yml` and wired into the +pipeline via `ci/cscs-ci.yml`. A small driver script, +`ci/dace_deterministic_codegen/run_in_ci.sh`, encapsulates the +clone + bootstrap + harness invocation so the YAML stays minimal and +the same flow can be reproduced locally. + +### Reproducing a CI run locally + +The driver script reads only env vars, so a green or red CI run can +be reproduced one-to-one by exporting the same variables and invoking +`run_in_ci.sh`: + +```bash +# gt4py CI venv with editable gt4py already there +source /path/to/gt4py-venv/bin/activate + +export GT4PY_PATH=/path/to/gt4py +export ICON4PY_REPO=https://github.com/C2SM/icon4py.git +export ICON4PY_REF=main # or the SHA from the failing run +export ICON4PY_PATH=/tmp/icon4py + +# Optional: custom dace branch +export DACE_REPO=https://github.com/GridTools/dace.git +export DACE_REF=dace_toolchain_deterministic +export DACE_PATH=/tmp/dace + +export DACE_DETERMINISM_SELECTION=stencils +export DACE_DETERMINISM_COMPONENT=muphys +export DACE_DETERMINISM_PYTHON=3.10 +export DACE_DETERMINISM_BACKEND=dace_cpu +export DACE_DETERMINISM_GRID=icon_regional + +bash $GT4PY_PATH/ci/dace_deterministic_codegen/run_in_ci.sh +``` diff --git a/ci/dace_deterministic_codegen/bootstrap_icon4py.py b/ci/dace_deterministic_codegen/bootstrap_icon4py.py index 4ceb05e6db..87a8fd125a 100644 --- a/ci/dace_deterministic_codegen/bootstrap_icon4py.py +++ b/ci/dace_deterministic_codegen/bootstrap_icon4py.py @@ -38,8 +38,9 @@ import sys from pathlib import Path + try: - import tomllib # Python 3.11+ + import tomllib # Python 3.11+ except ModuleNotFoundError: import tomli as tomllib # type: ignore[import-not-found] @@ -65,11 +66,7 @@ def patch_sources(pyproject: Path, overrides: dict[str, Path]) -> None: with pyproject.open("rb") as f: doc = tomllib.load(f) - sources = ( - doc.setdefault("tool", {}) - .setdefault("uv", {}) - .setdefault("sources", {}) - ) + sources = doc.setdefault("tool", {}).setdefault("uv", {}).setdefault("sources", {}) for pkg, path in overrides.items(): sources[pkg] = {"path": str(path), "editable": True} @@ -94,7 +91,10 @@ def run(cmd: list[str], cwd: Path) -> None: def main() -> int: p = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0]) p.add_argument( - "--icon4py", required=True, type=Path, metavar="PATH", + "--icon4py", + required=True, + type=Path, + metavar="PATH", help=( "Path to icon4py checkout. Accepts BOTH absolute and relative " "paths. Relative paths are resolved against the current working " @@ -102,29 +102,39 @@ def main() -> int: ), ) p.add_argument( - "--gt4py", required=True, type=Path, metavar="PATH", + "--gt4py", + required=True, + type=Path, + metavar="PATH", help=( "Path to gt4py checkout to install editable. Accepts BOTH " "absolute and relative paths (resolved against cwd)." ), ) p.add_argument( - "--dace", type=Path, default=None, metavar="PATH", + "--dace", + type=Path, + default=None, + metavar="PATH", help=( "Optional path to dace checkout (absolute or relative). If " "omitted, dace resolves through icon4py's existing source pin." ), ) - p.add_argument("--no-lock", action="store_true", - help="Skip `uv lock`. Useful if you already locked.") - p.add_argument("--no-sync", action="store_true", - help="Skip `uv sync`. Useful for CI steps that sync later.") + p.add_argument( + "--no-lock", action="store_true", help="Skip `uv lock`. Useful if you already locked." + ) + p.add_argument( + "--no-sync", + action="store_true", + help="Skip `uv sync`. Useful for CI steps that sync later.", + ) args = p.parse_args() # Resolve every path NOW. The script can be run from any cwd. icon4py = args.icon4py.expanduser().resolve() - gt4py = args.gt4py.expanduser().resolve() - dace = args.dace.expanduser().resolve() if args.dace else None + gt4py = args.gt4py.expanduser().resolve() + dace = args.dace.expanduser().resolve() if args.dace else None pyproject = icon4py / "pyproject.toml" if not pyproject.is_file(): @@ -171,10 +181,10 @@ def main() -> int: print() print("done. quick sanity check:") print(' python -c "import gt4py.next; print(gt4py.next.__file__)"') - print(f' # should print a path inside {gt4py}') + print(f" # should print a path inside {gt4py}") if dace: print(' python -c "import dace; print(dace.__file__)"') - print(f' # should print a path inside {dace}') + print(f" # should print a path inside {dace}") return 0 diff --git a/ci/dace_deterministic_codegen/dace_deterministic_codegen.py b/ci/dace_deterministic_codegen/dace_deterministic_codegen.py index 402dcce18f..67b9923b46 100644 --- a/ci/dace_deterministic_codegen/dace_deterministic_codegen.py +++ b/ci/dace_deterministic_codegen/dace_deterministic_codegen.py @@ -95,6 +95,7 @@ # icon4py noxfile introspection # --------------------------------------------------------------------------- + class NoxfileIntrospectionError(RuntimeError): """Raised when we can't extract sessions/components from the noxfile.""" @@ -128,9 +129,7 @@ def introspect_icon4py_noxfile( try: tree = ast.parse(noxfile.read_text()) except SyntaxError as e: - raise NoxfileIntrospectionError( - f"could not parse {noxfile} as Python: {e}" - ) from e + raise NoxfileIntrospectionError(f"could not parse {noxfile} as Python: {e}") from e selections = _extract_literal_strings(tree, "ModelTestsSubset") subpackages = _extract_literal_strings(tree, "ModelSubpackagePath") @@ -159,7 +158,9 @@ def _extract_literal_strings(tree: ast.AST, alias_name: str) -> list[str]: continue if node.target.id != alias_name: continue - # Expect: value = Subscript(value=Name('Literal'), slice=Tuple(elts=[Constant, ...])) + # Match the AST pattern for Literal["a", "b", ...]: + # a Subscript whose value is the Name "Literal" and whose slice is + # either a Tuple of string Constants or a single string Constant. v = node.value if not isinstance(v, ast.Subscript): continue @@ -181,6 +182,7 @@ def _extract_literal_strings(tree: ast.AST, alias_name: str) -> list[str]: # Snapshot # --------------------------------------------------------------------------- + @dataclasses.dataclass(frozen=True) class FileEntry: relpath: str @@ -275,6 +277,7 @@ def _sha256(path: Path) -> str: # Compare # --------------------------------------------------------------------------- + @dataclasses.dataclass class ProgramResult: name: str @@ -294,27 +297,32 @@ def compare( s2 = snap2.get(name) if s1 is None or s2 is None: - results.append(ProgramResult( - name=name, match=False, differing_files=[], - only_in_run1=sorted((s1.files if s1 else {}).keys()), - only_in_run2=sorted((s2.files if s2 else {}).keys()), - )) + results.append( + ProgramResult( + name=name, + match=False, + differing_files=[], + only_in_run1=sorted((s1.files if s1 else {}).keys()), + only_in_run2=sorted((s2.files if s2 else {}).keys()), + ) + ) continue keys1, keys2 = set(s1.files), set(s2.files) only1 = sorted(keys1 - keys2) only2 = sorted(keys2 - keys1) differing = sorted( - rel for rel in keys1 & keys2 - if s1.files[rel].sha256 != s2.files[rel].sha256 + rel for rel in keys1 & keys2 if s1.files[rel].sha256 != s2.files[rel].sha256 + ) + results.append( + ProgramResult( + name=name, + match=not (differing or only1 or only2), + differing_files=differing, + only_in_run1=only1, + only_in_run2=only2, + ) ) - results.append(ProgramResult( - name=name, - match=not (differing or only1 or only2), - differing_files=differing, - only_in_run1=only1, - only_in_run2=only2, - )) return results @@ -322,6 +330,7 @@ def compare( # Diff + report # --------------------------------------------------------------------------- + def write_diffs( results: list[ProgramResult], snap1: dict[str, ProgramSnapshot], @@ -347,9 +356,15 @@ def write_diffs( f"binary content differs:\n run1: {f1}\n run2: {f2}\n" ) continue - udiff = "".join(difflib.unified_diff( - t1, t2, fromfile=f"run1/{rel}", tofile=f"run2/{rel}", n=3, - )) + udiff = "".join( + difflib.unified_diff( + t1, + t2, + fromfile=f"run1/{rel}", + tofile=f"run2/{rel}", + n=3, + ) + ) prog_dir.mkdir(parents=True, exist_ok=True) (prog_dir / f"{rel.replace('/', '__')}.diff").write_text(udiff) @@ -384,9 +399,16 @@ def render_report(results: list[ProgramResult]) -> str: # Nox runner # --------------------------------------------------------------------------- + def run_nox( - icon4py: Path, run_dir: Path, log_path: Path, - session: str, selection: str, component: str, python: str, posargs: list[str], + icon4py: Path, + run_dir: Path, + log_path: Path, + session: str, + selection: str, + component: str, + python: str, + posargs: list[str], ) -> int: """Run nox once with `GT4PY_BUILD_CACHE_DIR=run_dir`. Returns the exit code. @@ -418,8 +440,9 @@ def run_nox( "# ---\n" ) logf.flush() - proc = subprocess.run(argv, cwd=str(icon4py), env=env, - stdout=logf, stderr=subprocess.STDOUT) + proc = subprocess.run( + argv, cwd=str(icon4py), env=env, stdout=logf, stderr=subprocess.STDOUT + ) return proc.returncode @@ -427,6 +450,7 @@ def run_nox( # Workdir # --------------------------------------------------------------------------- + @dataclasses.dataclass class Workdir: """Two parent dirs for gt4py's cache + a place for logs/diffs/report.""" @@ -434,21 +458,36 @@ class Workdir: root: Path @property - def run1_dir(self) -> Path: return self.root / "run1" + def run1_dir(self) -> Path: + return self.root / "run1" + @property - def run2_dir(self) -> Path: return self.root / "run2" + def run2_dir(self) -> Path: + return self.root / "run2" + @property - def cache1(self) -> Path: return self.run1_dir / ".gt4py_cache" + def cache1(self) -> Path: + return self.run1_dir / ".gt4py_cache" + @property - def cache2(self) -> Path: return self.run2_dir / ".gt4py_cache" + def cache2(self) -> Path: + return self.run2_dir / ".gt4py_cache" + @property - def log1(self) -> Path: return self.run1_dir / "test.log" + def log1(self) -> Path: + return self.run1_dir / "test.log" + @property - def log2(self) -> Path: return self.run2_dir / "test.log" + def log2(self) -> Path: + return self.run2_dir / "test.log" + @property - def diffs(self) -> Path: return self.root / "diffs" + def diffs(self) -> Path: + return self.root / "diffs" + @property - def report(self) -> Path: return self.root / "report.txt" + def report(self) -> Path: + return self.root / "report.txt" def prepare(self) -> None: """Wipe stale state from previous invocations.""" @@ -465,6 +504,7 @@ def prepare(self) -> None: # CLI # --------------------------------------------------------------------------- + def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace: p = argparse.ArgumentParser( prog="dace_deterministic_codegen", @@ -475,7 +515,10 @@ def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace: formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p.add_argument( - "--icon4py", required=True, type=Path, metavar="PATH", + "--icon4py", + required=True, + type=Path, + metavar="PATH", help=( "Path to icon4py checkout. Accepts BOTH absolute and relative " "paths. Relative paths are resolved against the current working " @@ -483,7 +526,9 @@ def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace: ), ) p.add_argument( - "--session", default="test_model", metavar="NAME", + "--session", + default="test_model", + metavar="NAME", help=( "Nox session name. Composed with --python/--selection/--component " "into the final session ID `-(, " @@ -491,25 +536,34 @@ def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace: ), ) p.add_argument( - "--selection", required=True, metavar="NAME", + "--selection", + required=True, + metavar="NAME", help=( "icon4py noxfile selection (e.g. stencils, datatest, basic). " "Validated at runtime against icon4py's actual noxfile." ), ) p.add_argument( - "--component", required=True, metavar="NAME", + "--component", + required=True, + metavar="NAME", help=( "icon4py noxfile subpackage leaf name (e.g. muphys, dycore). " "Validated at runtime against icon4py's actual noxfile." ), ) p.add_argument( - "--python", default="3.10", metavar="X.Y", + "--python", + default="3.10", + metavar="X.Y", help="Python version for the nox session.", ) p.add_argument( - "--workdir", type=Path, default=None, metavar="PATH", + "--workdir", + type=Path, + default=None, + metavar="PATH", help=( "Where run1/, run2/, diffs/, and report.txt are written. " "Accepts absolute or relative paths (resolved against cwd). " @@ -519,7 +573,11 @@ def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace: ), ) p.add_argument( - "--posarg", action="append", default=[], dest="posargs", metavar="ARG", + "--posarg", + action="append", + default=[], + dest="posargs", + metavar="ARG", help=( "Forwarded to pytest via `nox -- ARG`. Repeatable. " "Example: --posarg=--backend=dace_cpu --posarg=--grid=icon_regional" @@ -542,8 +600,10 @@ def main(argv: Optional[list[str]] = None) -> int: return 2 noxfile_path = icon4py / "noxfile.py" if not noxfile_path.is_file(): - print(f"error: no noxfile.py at {noxfile_path} — is --icon4py " - f"the icon4py repo root?", file=sys.stderr) + print( + f"error: no noxfile.py at {noxfile_path} — is --icon4py the icon4py repo root?", + file=sys.stderr, + ) return 2 # Introspect icon4py's noxfile to discover the legal selection / @@ -571,9 +631,7 @@ def main(argv: Optional[list[str]] = None) -> int: return 2 workdir_root = ( - args.workdir.expanduser().resolve() - if args.workdir is not None - else icon4py / WORKDIR_NAME + args.workdir.expanduser().resolve() if args.workdir is not None else icon4py / WORKDIR_NAME ) if args.workdir is not None and not args.workdir.is_absolute(): print(f"--workdir resolved to: {workdir_root}") @@ -584,20 +642,34 @@ def main(argv: Optional[list[str]] = None) -> int: # ----- Run 1 print(f"[1/2] nox -s '{session_id}' (cache: {workdir.run1_dir})", flush=True) - rc1 = run_nox(icon4py, workdir.run1_dir, workdir.log1, - args.session, args.selection, args.component, args.python, args.posargs) + rc1 = run_nox( + icon4py, + workdir.run1_dir, + workdir.log1, + args.session, + args.selection, + args.component, + args.python, + args.posargs, + ) if rc1 != 0: - print(f"error: run 1 failed (exit {rc1}). See log: {workdir.log1}", - file=sys.stderr) + print(f"error: run 1 failed (exit {rc1}). See log: {workdir.log1}", file=sys.stderr) return 4 # ----- Run 2 print(f"[2/2] nox -s '{session_id}' (cache: {workdir.run2_dir})", flush=True) - rc2 = run_nox(icon4py, workdir.run2_dir, workdir.log2, - args.session, args.selection, args.component, args.python, args.posargs) + rc2 = run_nox( + icon4py, + workdir.run2_dir, + workdir.log2, + args.session, + args.selection, + args.component, + args.python, + args.posargs, + ) if rc2 != 0: - print(f"error: run 2 failed (exit {rc2}). See log: {workdir.log2}", - file=sys.stderr) + print(f"error: run 2 failed (exit {rc2}). See log: {workdir.log2}", file=sys.stderr) return 4 # ----- Snapshot + compare + report @@ -615,8 +687,11 @@ def main(argv: Optional[list[str]] = None) -> int: print(f"workdir: {workdir.root}") if not results: - print(f"error: no programs observed in either run — check the logs:\n" - f" {workdir.log1}\n {workdir.log2}", file=sys.stderr) + print( + f"error: no programs observed in either run — check the logs:\n" + f" {workdir.log1}\n {workdir.log2}", + file=sys.stderr, + ) return 3 return 1 if any(not r.match for r in results) else 0 diff --git a/ci/dace_deterministic_codegen/run_in_ci.sh b/ci/dace_deterministic_codegen/run_in_ci.sh new file mode 100644 index 0000000000..b04cd1d366 --- /dev/null +++ b/ci/dace_deterministic_codegen/run_in_ci.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env bash +# GT4Py - GridTools Framework +# +# Copyright (c) 2014-2024, ETH Zurich +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +# Driver for running the dace_deterministic_codegen harness in CI. +# +# Encapsulates the clone + bootstrap + harness invocation so the YAML +# stays minimal and the logic is easy to reproduce locally (just set +# the env vars and run the script). +# +# Required environment variables (CI sets all of these via job vars): +# GT4PY_PATH Existing gt4py checkout (the commit under test). +# ICON4PY_REPO Git URL to clone icon4py from. +# ICON4PY_REF Git ref (branch, tag, or SHA) to checkout. +# ICON4PY_PATH Where to clone icon4py to (created if missing). +# DACE_DETERMINISM_SELECTION icon4py noxfile selection: stencils|datatest|basic. +# DACE_DETERMINISM_COMPONENT icon4py subpackage leaf: muphys|dycore|... +# DACE_DETERMINISM_PYTHON Python version for the nox session: 3.10, 3.14, ... +# DACE_DETERMINISM_BACKEND dace_cpu | dace_gpu (passed to pytest as --backend=...) +# DACE_DETERMINISM_GRID Grid name passed to pytest as --grid=... +# +# Optional environment variables: +# DACE_REPO Git URL for a custom dace fork. When set, +# DACE_REF and DACE_PATH must also be set. +# DACE_REF Git ref of the custom dace branch. +# DACE_PATH Where to clone dace to (created if missing). +# DACE_DETERMINISM_WORKDIR Where run1/, run2/, diffs/, report.txt land. +# Default: ${ICON4PY_PATH}/_dace_deterministic_codegen +# DACE_DETERMINISM_ARTIFACT_DIR If set, the workdir is copied here at the end +# (success or failure). Set to a path under +# ${CI_PROJECT_DIR} for GitLab CI artifact upload. +# +# Custom dace branch behaviour: +# - If DACE_REPO is unset, dace lands in the nox session venv via +# icon4py's existing [tool.uv.sources] pin (currently the +# GridTools/pypi published wheel). The parent venv may have its +# own dace from the gt4py CI venv setup, but that's separate — +# nox creates a fresh isolated venv and uv sync's into it from +# icon4py's pyproject.toml. +# - If DACE_REPO is set, the dace repo is cloned at DACE_REF and +# icon4py's [tool.uv.sources] is patched to point at the clone. +# Both the parent venv (Step 2) and the nox session venv (Step 3 +# onwards, via the patched source pin) end up with editable dace +# from the same local path. +# +# Exit codes: passed through from dace_deterministic_codegen.py. +# 0 = deterministic, 1 = differs, 2/3/4 = harness errors. +# See the harness README for the full table. + +set -euo pipefail + +# --- Validate required env vars ------------------------------------------- +required=( + GT4PY_PATH + ICON4PY_REPO + ICON4PY_REF + ICON4PY_PATH + DACE_DETERMINISM_SELECTION + DACE_DETERMINISM_COMPONENT + DACE_DETERMINISM_PYTHON + DACE_DETERMINISM_BACKEND + DACE_DETERMINISM_GRID +) +missing=() +for v in "${required[@]}"; do + if [[ -z "${!v:-}" ]]; then + missing+=("$v") + fi +done +if (( ${#missing[@]} > 0 )); then + echo "error: missing required env vars: ${missing[*]}" >&2 + exit 2 +fi + +# Custom dace branch is all-or-nothing: setting one of the three +# DACE_* vars without the others would leave us in a half-configured +# state where it's unclear whether the local dace is supposed to win +# over icon4py's source pin. +if [[ -n "${DACE_REPO:-}" ]]; then + if [[ -z "${DACE_REF:-}" || -z "${DACE_PATH:-}" ]]; then + echo "error: DACE_REPO is set but DACE_REF and/or DACE_PATH are not." >&2 + echo " To use a custom dace branch, set all three together:" >&2 + echo " DACE_REPO - git URL of the dace fork" >&2 + echo " DACE_REF - branch, tag, or SHA to check out" >&2 + echo " DACE_PATH - where to clone dace (typically \${WORKDIR}/dace)" >&2 + exit 2 + fi +fi + +# Active venv check. The Docker image sets VIRTUAL_ENV; bare local runs +# might not. We don't auto-activate — that's the caller's responsibility — +# but warn if it's missing, since installing into the system Python is +# almost never what's wanted. +if [[ -z "${VIRTUAL_ENV:-}" ]]; then + echo "warning: VIRTUAL_ENV is not set. Activate the gt4py CI venv first" >&2 + echo " (the gt4py CI Docker image sets this automatically; only" >&2 + echo " relevant when running this script outside the CI image)." >&2 +fi + +DACE_DETERMINISM_WORKDIR_DEFAULT="${ICON4PY_PATH}/_dace_deterministic_codegen" +DACE_DETERMINISM_WORKDIR="${DACE_DETERMINISM_WORKDIR:-${DACE_DETERMINISM_WORKDIR_DEFAULT}}" + +HARNESS_DIR="${GT4PY_PATH}/ci/dace_deterministic_codegen" +HARNESS="${HARNESS_DIR}/dace_deterministic_codegen.py" +BOOTSTRAP="${HARNESS_DIR}/bootstrap_icon4py.py" + +if [[ ! -f "$HARNESS" ]]; then + echo "error: harness not found at $HARNESS" >&2 + echo " (is GT4PY_PATH=$GT4PY_PATH the gt4py repo root?)" >&2 + exit 2 +fi +if [[ ! -f "$BOOTSTRAP" ]]; then + echo "error: bootstrap not found at $BOOTSTRAP" >&2 + exit 2 +fi + +# --- Helper: shallow-clone a repo at a ref, with SHA fallback ------------ +# Some git versions can't combine --depth 1 with arbitrary commit SHAs in +# `clone -b`. If -b fails, fall back to a full clone + explicit checkout. +clone_at_ref() { + local repo="$1" ref="$2" dest="$3" label="$4" + if [[ -d "${dest}/.git" ]]; then + echo " (${label} already cloned at ${dest}; fetching ${ref})" + git -C "${dest}" fetch --depth 1 origin "${ref}" + git -C "${dest}" checkout FETCH_HEAD + return + fi + if ! git clone --depth 1 -b "${ref}" "${repo}" "${dest}" 2>/dev/null; then + echo " (-b ${ref} failed; ${ref} may be a SHA — doing full clone + checkout)" + git clone "${repo}" "${dest}" + git -C "${dest}" checkout "${ref}" + fi +} + +# --- Step 1: clone icon4py at the pinned ref ----------------------------- +echo "==> [1/4] cloning icon4py @ ${ICON4PY_REF} from ${ICON4PY_REPO}" +clone_at_ref "${ICON4PY_REPO}" "${ICON4PY_REF}" "${ICON4PY_PATH}" "icon4py" + +# --- Step 1b (optional): clone custom dace ------------------------------- +if [[ -n "${DACE_REPO:-}" ]]; then + echo "==> [1b/4] cloning dace @ ${DACE_REF} from ${DACE_REPO}" + clone_at_ref "${DACE_REPO}" "${DACE_REF}" "${DACE_PATH}" "dace" +fi + +# --- Step 2: install editable gt4py (+ dace) + tomli_w into the venv ----- +# The gt4py CI Docker image already has gt4py's deps installed via uv +# sync --no-install-project. We add gt4py itself (editable, pointing at +# our checkout), tomli_w (which bootstrap_icon4py.py imports), and +# optionally dace (editable, when a custom branch is being tested). +# --no-deps skips re-resolving heavy transitive deps; the icon4py +# bootstrap below will handle anything missing via uv sync --active. +if [[ -n "${DACE_PATH:-}" ]]; then + echo "==> [2/4] installing editable gt4py + dace + tomli_w into ${VIRTUAL_ENV:-system Python}" +else + echo "==> [2/4] installing editable gt4py + tomli_w into ${VIRTUAL_ENV:-system Python}" +fi +uv pip install --no-deps -e "${GT4PY_PATH}" +if [[ -n "${DACE_PATH:-}" ]]; then + uv pip install --no-deps -e "${DACE_PATH}" +fi +uv pip install tomli_w + +# --- Step 3: bootstrap icon4py into the active venv ---------------------- +# Patches icon4py's [tool.uv.sources] so gt4py (and optionally dace) +# resolve to our local checkouts, then `uv lock` + `uv sync --active`. +# This is what makes the editable installs survive when icon4py's noxfile +# creates its session venv and runs `uv sync` inside it — that uv sync +# sees the patched source pins and installs editable from the same paths. +echo "==> [3/4] bootstrapping icon4py into the active venv" +bootstrap_args=( --icon4py "${ICON4PY_PATH}" --gt4py "${GT4PY_PATH}" ) +if [[ -n "${DACE_PATH:-}" ]]; then + bootstrap_args+=( --dace "${DACE_PATH}" ) +fi +python "${BOOTSTRAP}" "${bootstrap_args[@]}" + +# --- Step 4: run the determinism harness --------------------------------- +echo "==> [4/4] running the determinism harness" +echo " selection=${DACE_DETERMINISM_SELECTION} component=${DACE_DETERMINISM_COMPONENT}" +echo " python=${DACE_DETERMINISM_PYTHON} backend=${DACE_DETERMINISM_BACKEND} grid=${DACE_DETERMINISM_GRID}" +echo " workdir=${DACE_DETERMINISM_WORKDIR}" + +# Run with `set +e` and capture the exit code so the artifact-copy step +# below runs whether the harness reported determinism, non-determinism, +# or a tooling error. The harness is the source of truth on the exit +# code; we just defer reacting to it. +set +e +python "${HARNESS}" \ + --icon4py "${ICON4PY_PATH}" \ + --selection "${DACE_DETERMINISM_SELECTION}" \ + --component "${DACE_DETERMINISM_COMPONENT}" \ + --python "${DACE_DETERMINISM_PYTHON}" \ + --workdir "${DACE_DETERMINISM_WORKDIR}" \ + --posarg=--backend="${DACE_DETERMINISM_BACKEND}" \ + --posarg=--grid="${DACE_DETERMINISM_GRID}" +harness_rc=$? +set -e + +# --- Step 5 (optional): publish artifacts -------------------------------- +# If DACE_DETERMINISM_ARTIFACT_DIR is set (typically in CI to a path +# under ${CI_PROJECT_DIR}), copy the workdir there so GitLab can pick +# it up as a build artifact. We do this whether the harness passed or +# failed — both outcomes have a useful report.txt. +if [[ -n "${DACE_DETERMINISM_ARTIFACT_DIR:-}" ]]; then + echo "==> publishing artifacts to ${DACE_DETERMINISM_ARTIFACT_DIR}" + rm -rf "${DACE_DETERMINISM_ARTIFACT_DIR}" + mkdir -p "$(dirname "${DACE_DETERMINISM_ARTIFACT_DIR}")" + if [[ -d "${DACE_DETERMINISM_WORKDIR}" ]]; then + cp -r "${DACE_DETERMINISM_WORKDIR}" "${DACE_DETERMINISM_ARTIFACT_DIR}" + else + # Harness errored before creating the workdir — leave a note so + # the artifact upload still has something for diagnosis from the + # GitLab UI without ssh'ing to the runner. + mkdir -p "${DACE_DETERMINISM_ARTIFACT_DIR}" + cat > "${DACE_DETERMINISM_ARTIFACT_DIR}/MISSING_WORKDIR.txt" < Date: Mon, 4 May 2026 08:05:48 +0200 Subject: [PATCH 3/4] Infrastructure to test DaCe's codegen (in)deterministic behavior: CI/CD integration [WIP] --- ci/cscs-ci-dace-determinism.yml | 2 +- ci/dace_deterministic_codegen/README.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/cscs-ci-dace-determinism.yml b/ci/cscs-ci-dace-determinism.yml index b71e209211..6742fbf82f 100644 --- a/ci/cscs-ci-dace-determinism.yml +++ b/ci/cscs-ci-dace-determinism.yml @@ -52,7 +52,7 @@ variables: # DACE_REPO: 'https://github.com/GridTools/dace.git' # DACE_REF: 'dace_toolchain_deterministic' DACE_REPO: '' - DACE_REF: '' + DACE_REF: '' # Shared template for all dace-determinism jobs .dace_determinism_common: diff --git a/ci/dace_deterministic_codegen/README.md b/ci/dace_deterministic_codegen/README.md index 371ce446ba..9dababdf57 100644 --- a/ci/dace_deterministic_codegen/README.md +++ b/ci/dace_deterministic_codegen/README.md @@ -147,12 +147,12 @@ across invocations, copy the directory before re-running. ## Exit codes -| Code | Meaning | -|------|---------| -| 0 | Codegen is deterministic. | -| 1 | Codegen differs (see `report.txt` and `diffs/`). | -| 2 | Bad arguments (path doesn't exist, missing noxfile, …). | -| 3 | No programs observed in either run (test selection collected nothing). | +| Code | Meaning | +| ---- | ------------------------------------------------------------------------- | +| 0 | Codegen is deterministic. | +| 1 | Codegen differs (see `report.txt` and `diffs/`). | +| 2 | Bad arguments (path doesn't exist, missing noxfile, …). | +| 3 | No programs observed in either run (test selection collected nothing). | | 4 | A `nox` invocation itself failed (see `run1/test.log` / `run2/test.log`). | ## Flags From 09842978fab24684ee95880f1607b5667898e6c8 Mon Sep 17 00:00:00 2001 From: Christos Kotsalos Date: Mon, 18 May 2026 14:37:38 +0200 Subject: [PATCH 4/4] WIP: wording --- ci/cscs-ci-dace-determinism.yml | 2 +- ci/dace_deterministic_codegen/README.md | 12 +++--- .../bootstrap_icon4py.py | 2 +- .../dace_deterministic_codegen.py | 12 +++--- ci/dace_deterministic_codegen/run_in_ci.sh | 38 +++++++++---------- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/ci/cscs-ci-dace-determinism.yml b/ci/cscs-ci-dace-determinism.yml index 6742fbf82f..f04b6fa17e 100644 --- a/ci/cscs-ci-dace-determinism.yml +++ b/ci/cscs-ci-dace-determinism.yml @@ -18,7 +18,7 @@ # that selection. # # The actual logic lives in: -# ci/dace_deterministic_codegen/dace_deterministic_codegen.py (harness) +# ci/dace_deterministic_codegen/dace_deterministic_codegen.py (checker) # ci/dace_deterministic_codegen/bootstrap_icon4py.py (venv prep) # ci/dace_deterministic_codegen/run_in_ci.sh (CI driver) # diff --git a/ci/dace_deterministic_codegen/README.md b/ci/dace_deterministic_codegen/README.md index 9dababdf57..c5073cb352 100644 --- a/ci/dace_deterministic_codegen/README.md +++ b/ci/dace_deterministic_codegen/README.md @@ -7,15 +7,15 @@ runs. Exit 0 = identical (deterministic), exit 1 = different. Currently supports the **cpu**, **cuda**, and **HIP** dace backends. HIP is supported transparently: dace emits HIP code under `src/cuda/hip/` -(target_name="cuda", target_type="hip"), and the harness's recursive +(target_name="cuda", target_type="hip"), and the checker's recursive sweep of `src/cuda/` picks it up automatically. If a run emits anything -else under `src/` (mpi, sve, mlir, snitch, …) the harness fails +else under `src/` (mpi, sve, mlir, snitch, …) the checker fails immediately with a clear message — silently ignoring an unfamiliar backend would mean reporting "deterministic" without actually checking the relevant code. Valid `--selection` and `--component` values are read from icon4py's -own `noxfile.py` at runtime — no hardcoding here, so the harness +own `noxfile.py` at runtime — no hardcoding here, so the checker auto-tracks any future changes to icon4py's parametrization. Mirrors icon4py's `ci/dace.yml`, with the session name configurable: @@ -87,7 +87,7 @@ from icon4py's `noxfile.py` at runtime. As of icon4py main, that's: - `--component`: `advection`, `diffusion`, `dycore`, `microphysics`, `muphys`, `common`, `driver`, `standalone_driver`, `testing` -If icon4py adds or renames these, the harness picks it up automatically; +If icon4py adds or renames these, the checker picks it up automatically; no update needed here. If you pass an invalid value, the error message lists the actual valid set extracted from your icon4py checkout. @@ -173,11 +173,11 @@ across invocations, copy the directory before re-running. ## CI integration -The harness runs in CSCS CI as a separate `dace-determinism` stage, +The checker runs in CSCS CI as a separate `dace-determinism` stage, defined in `ci/cscs-ci-dace-determinism.yml` and wired into the pipeline via `ci/cscs-ci.yml`. A small driver script, `ci/dace_deterministic_codegen/run_in_ci.sh`, encapsulates the -clone + bootstrap + harness invocation so the YAML stays minimal and +clone + bootstrap + checker invocation so the YAML stays minimal and the same flow can be reproduced locally. ### Reproducing a CI run locally diff --git a/ci/dace_deterministic_codegen/bootstrap_icon4py.py b/ci/dace_deterministic_codegen/bootstrap_icon4py.py index 87a8fd125a..6a743c5c35 100644 --- a/ci/dace_deterministic_codegen/bootstrap_icon4py.py +++ b/ci/dace_deterministic_codegen/bootstrap_icon4py.py @@ -16,7 +16,7 @@ This is what makes the editable gt4py / dace branches survive everything downstream — including the icon4py noxfile's own `uv sync` call when our -dace_deterministic_codegen harness runs `nox --no-venv`. +dace_deterministic_codegen checker runs `nox --no-venv`. Usage (run from anywhere): diff --git a/ci/dace_deterministic_codegen/dace_deterministic_codegen.py b/ci/dace_deterministic_codegen/dace_deterministic_codegen.py index 67b9923b46..5fd908b3b2 100644 --- a/ci/dace_deterministic_codegen/dace_deterministic_codegen.py +++ b/ci/dace_deterministic_codegen/dace_deterministic_codegen.py @@ -17,11 +17,11 @@ Compares only the contents of `/src/` — the actual generated backend code. Currently supports cpu, cuda, and hip (hip is emitted by dace under `src/cuda/hip/`). Any other top-level backend under `src/` -(mpi, sve, mlir, snitch, …) causes the harness to fail with a clear +(mpi, sve, mlir, snitch, …) causes the checker to fail with a clear message rather than silently ignore it. Valid `--selection` and `--component` values are read from icon4py's -own `noxfile.py` at runtime (no hardcoding here), so the harness +own `noxfile.py` at runtime (no hardcoding here), so the checker tracks any future changes to icon4py's parametrization automatically. Mirrors icon4py's `ci/dace.yml` invocation pattern, with the session @@ -83,7 +83,7 @@ #: `cuda/` (with `rglob` picking up the nested hip files). #: #: If a snapshot ever encounters another top-level backend (mpi, sve, -#: mlir, snitch, …), the harness fails loudly rather than silently +#: mlir, snitch, …), the checker fails loudly rather than silently #: ignoring — those would need explicit support added here. SUPPORTED_BACKENDS: frozenset[str] = frozenset({"cpu", "cuda"}) @@ -246,7 +246,7 @@ def snapshot_run(cache_root: Path) -> dict[str, ProgramSnapshot]: if bd.name not in SUPPORTED_BACKENDS: raise UnsupportedBackendError( f"unsupported dace backend `{bd.name}/` found under " - f"{src_root} — this harness currently supports " + f"{src_root} — this checker currently supports " f"{sorted(SUPPORTED_BACKENDS)} as top-level backends " f"(HIP is handled under `cuda/hip/`). Add explicit " f"support in dace_deterministic_codegen.py before " @@ -589,7 +589,7 @@ def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace: def main(argv: Optional[list[str]] = None) -> int: args = parse_args(argv) - # Resolve every path to absolute up-front, so the harness can be run + # Resolve every path to absolute up-front, so the checker can be run # from any cwd. We print what the path resolved to — `--icon4py ../foo` # behaves intuitively but it's nice to confirm what it landed on. icon4py = args.icon4py.expanduser().resolve() @@ -607,7 +607,7 @@ def main(argv: Optional[list[str]] = None) -> int: return 2 # Introspect icon4py's noxfile to discover the legal selection / - # component values. This avoids hardcoding the lists, so the harness + # component values. This avoids hardcoding the lists, so the checker # auto-tracks any future changes to icon4py's noxfile structure. try: valid_selections, valid_components = introspect_icon4py_noxfile(noxfile_path) diff --git a/ci/dace_deterministic_codegen/run_in_ci.sh b/ci/dace_deterministic_codegen/run_in_ci.sh index b04cd1d366..6d9e3ec4c0 100644 --- a/ci/dace_deterministic_codegen/run_in_ci.sh +++ b/ci/dace_deterministic_codegen/run_in_ci.sh @@ -7,9 +7,9 @@ # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause -# Driver for running the dace_deterministic_codegen harness in CI. +# Driver for running the dace_deterministic_codegen checker in CI. # -# Encapsulates the clone + bootstrap + harness invocation so the YAML +# Encapsulates the clone + bootstrap + checker invocation so the YAML # stays minimal and the logic is easy to reproduce locally (just set # the env vars and run the script). # @@ -49,8 +49,8 @@ # from the same local path. # # Exit codes: passed through from dace_deterministic_codegen.py. -# 0 = deterministic, 1 = differs, 2/3/4 = harness errors. -# See the harness README for the full table. +# 0 = deterministic, 1 = differs, 2/3/4 = checker errors. +# See the checker README for the full table. set -euo pipefail @@ -105,12 +105,12 @@ fi DACE_DETERMINISM_WORKDIR_DEFAULT="${ICON4PY_PATH}/_dace_deterministic_codegen" DACE_DETERMINISM_WORKDIR="${DACE_DETERMINISM_WORKDIR:-${DACE_DETERMINISM_WORKDIR_DEFAULT}}" -HARNESS_DIR="${GT4PY_PATH}/ci/dace_deterministic_codegen" -HARNESS="${HARNESS_DIR}/dace_deterministic_codegen.py" -BOOTSTRAP="${HARNESS_DIR}/bootstrap_icon4py.py" +CHECKER_DIR="${GT4PY_PATH}/ci/dace_deterministic_codegen" +CHECKER="${CHECKER_DIR}/dace_deterministic_codegen.py" +BOOTSTRAP="${CHECKER_DIR}/bootstrap_icon4py.py" -if [[ ! -f "$HARNESS" ]]; then - echo "error: harness not found at $HARNESS" >&2 +if [[ ! -f "$CHECKER" ]]; then + echo "error: checker not found at $CHECKER" >&2 echo " (is GT4PY_PATH=$GT4PY_PATH the gt4py repo root?)" >&2 exit 2 fi @@ -178,18 +178,18 @@ if [[ -n "${DACE_PATH:-}" ]]; then fi python "${BOOTSTRAP}" "${bootstrap_args[@]}" -# --- Step 4: run the determinism harness --------------------------------- -echo "==> [4/4] running the determinism harness" +# --- Step 4: run the determinism checker --------------------------------- +echo "==> [4/4] running the determinism checker" echo " selection=${DACE_DETERMINISM_SELECTION} component=${DACE_DETERMINISM_COMPONENT}" echo " python=${DACE_DETERMINISM_PYTHON} backend=${DACE_DETERMINISM_BACKEND} grid=${DACE_DETERMINISM_GRID}" echo " workdir=${DACE_DETERMINISM_WORKDIR}" # Run with `set +e` and capture the exit code so the artifact-copy step -# below runs whether the harness reported determinism, non-determinism, -# or a tooling error. The harness is the source of truth on the exit +# below runs whether the checker reported determinism, non-determinism, +# or a tooling error. The checker is the source of truth on the exit # code; we just defer reacting to it. set +e -python "${HARNESS}" \ +python "${CHECKER}" \ --icon4py "${ICON4PY_PATH}" \ --selection "${DACE_DETERMINISM_SELECTION}" \ --component "${DACE_DETERMINISM_COMPONENT}" \ @@ -197,13 +197,13 @@ python "${HARNESS}" \ --workdir "${DACE_DETERMINISM_WORKDIR}" \ --posarg=--backend="${DACE_DETERMINISM_BACKEND}" \ --posarg=--grid="${DACE_DETERMINISM_GRID}" -harness_rc=$? +checker_rc=$? set -e # --- Step 5 (optional): publish artifacts -------------------------------- # If DACE_DETERMINISM_ARTIFACT_DIR is set (typically in CI to a path # under ${CI_PROJECT_DIR}), copy the workdir there so GitLab can pick -# it up as a build artifact. We do this whether the harness passed or +# it up as a build artifact. We do this whether the checker passed or # failed — both outcomes have a useful report.txt. if [[ -n "${DACE_DETERMINISM_ARTIFACT_DIR:-}" ]]; then echo "==> publishing artifacts to ${DACE_DETERMINISM_ARTIFACT_DIR}" @@ -212,12 +212,12 @@ if [[ -n "${DACE_DETERMINISM_ARTIFACT_DIR:-}" ]]; then if [[ -d "${DACE_DETERMINISM_WORKDIR}" ]]; then cp -r "${DACE_DETERMINISM_WORKDIR}" "${DACE_DETERMINISM_ARTIFACT_DIR}" else - # Harness errored before creating the workdir — leave a note so + # Checker errored before creating the workdir — leave a note so # the artifact upload still has something for diagnosis from the # GitLab UI without ssh'ing to the runner. mkdir -p "${DACE_DETERMINISM_ARTIFACT_DIR}" cat > "${DACE_DETERMINISM_ARTIFACT_DIR}/MISSING_WORKDIR.txt" <