From a026ed84d98daa182824b4d791d2e7594e9e34ef Mon Sep 17 00:00:00 2001 From: Stevengre Date: Wed, 18 Mar 2026 01:53:11 +0000 Subject: [PATCH 1/3] ci: split integration tests into parallel LLVM and Haskell jobs - Phase 1: Add `test-integration-llvm` and `test-integration-haskell` Makefile targets that run the respective test subsets in parallel CI jobs. - Phase 2: Fix redundant llvm-kompile calls in test_exec_smir by replacing function-scoped `tmp_path` with session-scoped fixtures (`kompile_cache_dir`, `exec_smir_kompile_dirs`) so KompileDigest caching hits for repeated (smir_json, symbolic) combinations within a test session. A file-lock guards against concurrent pytest-xdist workers racing on the same dir. - Phase 3: Add a shared `build` CI job that runs `make stable-mir-json` + `make build` once and commits the result as a Docker image; downstream `integration-tests-llvm` and `integration-tests-haskell` jobs start containers from that image, avoiding a redundant full rebuild per job. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/test.yml | 87 ++++++++++++++++--- Makefile | 10 +++ kmir/src/tests/integration/conftest.py | 75 ++++++++++++++++ .../src/tests/integration/test_integration.py | 8 +- 4 files changed, 168 insertions(+), 12 deletions(-) create mode 100644 kmir/src/tests/integration/conftest.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 76e6a75dc..81085e004 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,10 +49,12 @@ jobs: - name: 'Run unit tests' run: make test-unit - integration-tests: + build: needs: code-quality-checks - name: 'Integration Tests' + name: 'Build' runs-on: [self-hosted, linux, normal] + outputs: + image: ${{ steps.commit-image.outputs.image }} steps: - name: 'Check out code' uses: actions/checkout@v4 @@ -62,14 +64,80 @@ jobs: - name: 'Set up Docker' uses: ./.github/actions/with-docker with: - container-name: mir-semantics-ci-${{ github.sha }} - - name: 'Build stable-mir-json and kmir' - run: docker exec --user github-user mir-semantics-ci-${GITHUB_SHA} make build - - name: 'Run integration tests' - run: docker exec --user github-user mir-semantics-ci-${GITHUB_SHA} make test-integration + container-name: mir-semantics-build-${{ github.sha }} + - name: 'Build stable-mir-json' + run: docker exec --user github-user mir-semantics-build-${GITHUB_SHA} make stable-mir-json + - name: 'Build kmir' + run: docker exec --user github-user mir-semantics-build-${GITHUB_SHA} make build + - name: 'Commit built container as image' + id: commit-image + run: | + IMAGE="mir-semantics-built:${GITHUB_SHA}" + docker commit mir-semantics-build-${GITHUB_SHA} ${IMAGE} + echo "image=${IMAGE}" >> "${GITHUB_OUTPUT}" + - name: 'Tear down build container' + if: always() + run: docker stop --time 0 mir-semantics-build-${GITHUB_SHA} || true + + integration-tests-llvm: + needs: [code-quality-checks, build] + name: 'Integration Tests (LLVM)' + runs-on: [self-hosted, linux, normal] + steps: + - name: 'Check out code' + uses: actions/checkout@v4 + with: + token: ${{ secrets.JENKINS_GITHUB_PAT }} + submodules: recursive + - name: 'Start container from pre-built image' + run: | + IMAGE="${{ needs.build.outputs.image }}" + docker run \ + --name mir-semantics-llvm-${GITHUB_SHA} \ + --rm \ + --interactive \ + --tty \ + --detach \ + --workdir /home/github-user/workspace \ + ${IMAGE} + docker cp . mir-semantics-llvm-${GITHUB_SHA}:/home/github-user/workspace + docker exec --user root mir-semantics-llvm-${GITHUB_SHA} \ + chown -R github-user:github-user /home/github-user/workspace + - name: 'Run LLVM integration tests' + run: docker exec --user github-user mir-semantics-llvm-${GITHUB_SHA} make test-integration-llvm - name: 'Tear down Docker' if: always() - run: docker stop --time 0 mir-semantics-ci-${GITHUB_SHA} + run: docker stop --time 0 mir-semantics-llvm-${GITHUB_SHA} || true + + integration-tests-haskell: + needs: [code-quality-checks, build] + name: 'Integration Tests (Haskell)' + runs-on: [self-hosted, linux, normal] + steps: + - name: 'Check out code' + uses: actions/checkout@v4 + with: + token: ${{ secrets.JENKINS_GITHUB_PAT }} + submodules: recursive + - name: 'Start container from pre-built image' + run: | + IMAGE="${{ needs.build.outputs.image }}" + docker run \ + --name mir-semantics-haskell-${GITHUB_SHA} \ + --rm \ + --interactive \ + --tty \ + --detach \ + --workdir /home/github-user/workspace \ + ${IMAGE} + docker cp . mir-semantics-haskell-${GITHUB_SHA}:/home/github-user/workspace + docker exec --user root mir-semantics-haskell-${GITHUB_SHA} \ + chown -R github-user:github-user /home/github-user/workspace + - name: 'Run Haskell integration tests' + run: docker exec --user github-user mir-semantics-haskell-${GITHUB_SHA} make test-integration-haskell + - name: 'Tear down Docker' + if: always() + run: docker stop --time 0 mir-semantics-haskell-${GITHUB_SHA} || true stable-mir-json-integration-tests: needs: code-quality-checks @@ -181,7 +249,7 @@ jobs: strategy: fail-fast: false matrix: - runner: [normal, MacM1] # MacM1 / normal are self-hosted, + runner: [normal, MacM1] # MacM1 / normal are self-hosted, runs-on: ${{ matrix.runner }} timeout-minutes: 20 steps: @@ -212,4 +280,3 @@ jobs: set -euxo pipefail nix --version nix flake check # build and run smoke test - diff --git a/Makefile b/Makefile index 9bd7f8f2d..6d00bae73 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,16 @@ test-integration: stable-mir-json build $(UV_RUN) pytest $(TOP_DIR)/kmir/src/tests/integration --maxfail=1 --verbose \ --durations=0 --numprocesses=$(PARALLEL) --dist=worksteal $(TEST_ARGS) +test-integration-llvm: stable-mir-json build + $(UV_RUN) pytest $(TOP_DIR)/kmir/src/tests/integration --maxfail=1 --verbose \ + --durations=0 --numprocesses=$(PARALLEL) --dist=worksteal \ + -k "llvm or test_decode_value or test_run_smir_random or test_python_decode_value" $(TEST_ARGS) + +test-integration-haskell: stable-mir-json build + $(UV_RUN) pytest $(TOP_DIR)/kmir/src/tests/integration --maxfail=1 --verbose \ + --durations=0 --numprocesses=$(PARALLEL) --dist=worksteal \ + -k "haskell or test_prove or test_crate_examples or test_schema_parse or test_schema_kapply_parse or test_prove_termination or test_functions or test_cli" $(TEST_ARGS) + .PHONY: test-stable-mir-ui test-stable-mir-ui: stable-mir-json build @test -n "$(RUST_DIR_ROOT)" || (echo "RUST_DIR_ROOT is required. Example: RUST_DIR_ROOT=/path/to/rust make test-stable-mir-ui"; exit 2) diff --git a/kmir/src/tests/integration/conftest.py b/kmir/src/tests/integration/conftest.py new file mode 100644 index 000000000..5ecdc0312 --- /dev/null +++ b/kmir/src/tests/integration/conftest.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import hashlib +import time +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + + from pytest import TempPathFactory + + +def _smir_kompile_key(smir_json: Path, symbolic: bool) -> str: + """Return a short stable key for a (smir_json, symbolic) pair.""" + path_hash = hashlib.sha256(str(smir_json).encode()).hexdigest()[:16] + suffix = 'symbolic' if symbolic else 'concrete' + return f'{path_hash}-{suffix}' + + +@pytest.fixture(scope='session') +def kompile_cache_dir(tmp_path_factory: TempPathFactory) -> Path: + """Session-scoped base directory for kompile cache. + + Returns a stable temporary directory that persists across the test session, + allowing KompileDigest caching in kompile_smir to avoid redundant llvm-kompile calls + when multiple test_exec_smir parametrizations share the same smir_json + symbolic combo. + """ + return tmp_path_factory.mktemp('kompile-cache', numbered=False) + + +@pytest.fixture(scope='session') +def exec_smir_kompile_dirs(kompile_cache_dir: Path) -> dict[str, Path]: + """Session-scoped mapping from (smir_json, symbolic) key to shared kompile output dir. + + Multiple test_exec_smir invocations that share the same smir_json + symbolic flag + will reuse the same target_dir, letting KompileDigest skip redundant kompile calls. + """ + return {} + + +def get_exec_smir_target_dir( + smir_json: Path, + symbolic: bool, + kompile_cache_dir: Path, + exec_smir_kompile_dirs: dict[str, Path], +) -> Path: + """Return a shared target directory for a given (smir_json, symbolic) pair. + + Uses a file lock to avoid concurrent kompile races when running under pytest-xdist. + """ + key = _smir_kompile_key(smir_json, symbolic) + if key in exec_smir_kompile_dirs: + return exec_smir_kompile_dirs[key] + + target = kompile_cache_dir / key + lock_file = kompile_cache_dir / f'{key}.lock' + + try: + # Attempt to claim the lock (atomic create) + with open(lock_file, 'x'): + target.mkdir(parents=True, exist_ok=True) + exec_smir_kompile_dirs[key] = target + lock_file.unlink(missing_ok=True) + except FileExistsError: + # Another worker is building; wait for the lock to be released (max 5 min) + target.mkdir(parents=True, exist_ok=True) + secs = 0 + while lock_file.exists() and secs < 300: + time.sleep(1) + secs += 1 + exec_smir_kompile_dirs[key] = target + + return target diff --git a/kmir/src/tests/integration/test_integration.py b/kmir/src/tests/integration/test_integration.py index 8c59cce9b..e95f946af 100644 --- a/kmir/src/tests/integration/test_integration.py +++ b/kmir/src/tests/integration/test_integration.py @@ -390,11 +390,15 @@ def test_exec_smir( test_case: tuple[str, Path, Path, int], symbolic: bool, update_expected_output: bool, - tmp_path: Path, + kompile_cache_dir: Path, + exec_smir_kompile_dirs: dict[str, Path], ) -> None: + from .conftest import get_exec_smir_target_dir + _, input_json, output_kast, depth = test_case smir_info = SMIRInfo.from_file(input_json) - kmir_backend = KMIR.from_kompiled_kore(smir_info, target_dir=tmp_path, symbolic=symbolic) + target_dir = get_exec_smir_target_dir(input_json, symbolic, kompile_cache_dir, exec_smir_kompile_dirs) + kmir_backend = KMIR.from_kompiled_kore(smir_info, target_dir=target_dir, symbolic=symbolic) result = kmir_backend.run_smir(smir_info, depth=depth) result_pretty = kmir_backend.kore_to_pretty(result).rstrip() assert_or_update_show_output(result_pretty, output_kast, update=update_expected_output) From 2e30d10730dc1d7b8d9d917c45e17a09a8917c64 Mon Sep 17 00:00:00 2001 From: Stevengre Date: Wed, 18 Mar 2026 02:33:44 +0000 Subject: [PATCH 2/3] ci: fix Docker image sharing across self-hosted runners The previous approach used `docker commit` to save a built image and share it across jobs. This fails because self-hosted runners are separate machines and local Docker images are not shared between them. Switch to having each test job independently run `with-docker` + build, matching the pattern used by other jobs in this workflow. The tests themselves still run in parallel across two separate jobs. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/test.yml | 74 ++++++++------------------------------ 1 file changed, 14 insertions(+), 60 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 81085e004..e2836cfc6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,12 +49,10 @@ jobs: - name: 'Run unit tests' run: make test-unit - build: + integration-tests-llvm: needs: code-quality-checks - name: 'Build' + name: 'Integration Tests (LLVM)' runs-on: [self-hosted, linux, normal] - outputs: - image: ${{ steps.commit-image.outputs.image }} steps: - name: 'Check out code' uses: actions/checkout@v4 @@ -64,53 +62,17 @@ jobs: - name: 'Set up Docker' uses: ./.github/actions/with-docker with: - container-name: mir-semantics-build-${{ github.sha }} - - name: 'Build stable-mir-json' - run: docker exec --user github-user mir-semantics-build-${GITHUB_SHA} make stable-mir-json - - name: 'Build kmir' - run: docker exec --user github-user mir-semantics-build-${GITHUB_SHA} make build - - name: 'Commit built container as image' - id: commit-image - run: | - IMAGE="mir-semantics-built:${GITHUB_SHA}" - docker commit mir-semantics-build-${GITHUB_SHA} ${IMAGE} - echo "image=${IMAGE}" >> "${GITHUB_OUTPUT}" - - name: 'Tear down build container' - if: always() - run: docker stop --time 0 mir-semantics-build-${GITHUB_SHA} || true - - integration-tests-llvm: - needs: [code-quality-checks, build] - name: 'Integration Tests (LLVM)' - runs-on: [self-hosted, linux, normal] - steps: - - name: 'Check out code' - uses: actions/checkout@v4 - with: - token: ${{ secrets.JENKINS_GITHUB_PAT }} - submodules: recursive - - name: 'Start container from pre-built image' - run: | - IMAGE="${{ needs.build.outputs.image }}" - docker run \ - --name mir-semantics-llvm-${GITHUB_SHA} \ - --rm \ - --interactive \ - --tty \ - --detach \ - --workdir /home/github-user/workspace \ - ${IMAGE} - docker cp . mir-semantics-llvm-${GITHUB_SHA}:/home/github-user/workspace - docker exec --user root mir-semantics-llvm-${GITHUB_SHA} \ - chown -R github-user:github-user /home/github-user/workspace + container-name: mir-semantics-llvm-${{ github.sha }} + - name: 'Build stable-mir-json and kmir' + run: docker exec --user github-user mir-semantics-llvm-${GITHUB_SHA} make stable-mir-json build - name: 'Run LLVM integration tests' run: docker exec --user github-user mir-semantics-llvm-${GITHUB_SHA} make test-integration-llvm - name: 'Tear down Docker' if: always() - run: docker stop --time 0 mir-semantics-llvm-${GITHUB_SHA} || true + run: docker stop --time 0 mir-semantics-llvm-${GITHUB_SHA} integration-tests-haskell: - needs: [code-quality-checks, build] + needs: code-quality-checks name: 'Integration Tests (Haskell)' runs-on: [self-hosted, linux, normal] steps: @@ -119,25 +81,17 @@ jobs: with: token: ${{ secrets.JENKINS_GITHUB_PAT }} submodules: recursive - - name: 'Start container from pre-built image' - run: | - IMAGE="${{ needs.build.outputs.image }}" - docker run \ - --name mir-semantics-haskell-${GITHUB_SHA} \ - --rm \ - --interactive \ - --tty \ - --detach \ - --workdir /home/github-user/workspace \ - ${IMAGE} - docker cp . mir-semantics-haskell-${GITHUB_SHA}:/home/github-user/workspace - docker exec --user root mir-semantics-haskell-${GITHUB_SHA} \ - chown -R github-user:github-user /home/github-user/workspace + - name: 'Set up Docker' + uses: ./.github/actions/with-docker + with: + container-name: mir-semantics-haskell-${{ github.sha }} + - name: 'Build stable-mir-json and kmir' + run: docker exec --user github-user mir-semantics-haskell-${GITHUB_SHA} make stable-mir-json build - name: 'Run Haskell integration tests' run: docker exec --user github-user mir-semantics-haskell-${GITHUB_SHA} make test-integration-haskell - name: 'Tear down Docker' if: always() - run: docker stop --time 0 mir-semantics-haskell-${GITHUB_SHA} || true + run: docker stop --time 0 mir-semantics-haskell-${GITHUB_SHA} stable-mir-json-integration-tests: needs: code-quality-checks From 5cba7069be7fb5355cfd64fb009555e9dd33373b Mon Sep 17 00:00:00 2001 From: Stevengre Date: Wed, 18 Mar 2026 03:03:50 +0000 Subject: [PATCH 3/3] ci: share build via GHCR to avoid duplicate kompile across test jobs Build once in a dedicated job, commit the container image with all kompile artifacts to GHCR, then pull it in the LLVM and Haskell test jobs. This avoids rebuilding stable-mir-json and kdist definitions twice. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/test.yml | 85 ++++++++++++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e2836cfc6..e84ac77a7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,6 +2,9 @@ name: 'Test' on: pull_request: workflow_dispatch: +permissions: + contents: read + packages: write concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -49,10 +52,14 @@ jobs: - name: 'Run unit tests' run: make test-unit - integration-tests-llvm: + build: needs: code-quality-checks - name: 'Integration Tests (LLVM)' + name: 'Build' runs-on: [self-hosted, linux, normal] + permissions: + packages: write + outputs: + image: ${{ steps.commit-image.outputs.image }} steps: - name: 'Check out code' uses: actions/checkout@v4 @@ -62,36 +69,88 @@ jobs: - name: 'Set up Docker' uses: ./.github/actions/with-docker with: - container-name: mir-semantics-llvm-${{ github.sha }} + container-name: mir-semantics-build-${{ github.sha }} - name: 'Build stable-mir-json and kmir' - run: docker exec --user github-user mir-semantics-llvm-${GITHUB_SHA} make stable-mir-json build + run: docker exec --user github-user mir-semantics-build-${GITHUB_SHA} make stable-mir-json build + - name: 'Commit and push built image to GHCR' + id: commit-image + run: | + IMAGE="ghcr.io/runtimeverification/mir-semantics/ci:${GITHUB_SHA}" + docker commit mir-semantics-build-${GITHUB_SHA} ${IMAGE} + echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin + docker push ${IMAGE} + echo "image=${IMAGE}" >> "${GITHUB_OUTPUT}" + - name: 'Tear down Docker' + if: always() + run: | + docker stop --time 0 mir-semantics-build-${GITHUB_SHA} || true + docker rmi ${{ steps.commit-image.outputs.image }} || true + + integration-tests-llvm: + needs: [code-quality-checks, build] + name: 'Integration Tests (LLVM)' + runs-on: [self-hosted, linux, normal] + permissions: + packages: read + steps: + - name: 'Check out code' + uses: actions/checkout@v4 + with: + token: ${{ secrets.JENKINS_GITHUB_PAT }} + submodules: recursive + - name: 'Pull pre-built image and start container' + run: | + echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin + docker pull ${{ needs.build.outputs.image }} + docker run \ + --name mir-semantics-llvm-${GITHUB_SHA} \ + --rm \ + --interactive \ + --tty \ + --detach \ + --workdir /home/github-user/workspace \ + ${{ needs.build.outputs.image }} + docker cp . mir-semantics-llvm-${GITHUB_SHA}:/home/github-user/workspace + docker exec --user root mir-semantics-llvm-${GITHUB_SHA} \ + chown -R github-user:github-user /home/github-user/workspace - name: 'Run LLVM integration tests' run: docker exec --user github-user mir-semantics-llvm-${GITHUB_SHA} make test-integration-llvm - name: 'Tear down Docker' if: always() - run: docker stop --time 0 mir-semantics-llvm-${GITHUB_SHA} + run: docker stop --time 0 mir-semantics-llvm-${GITHUB_SHA} || true integration-tests-haskell: - needs: code-quality-checks + needs: [code-quality-checks, build] name: 'Integration Tests (Haskell)' runs-on: [self-hosted, linux, normal] + permissions: + packages: read steps: - name: 'Check out code' uses: actions/checkout@v4 with: token: ${{ secrets.JENKINS_GITHUB_PAT }} submodules: recursive - - name: 'Set up Docker' - uses: ./.github/actions/with-docker - with: - container-name: mir-semantics-haskell-${{ github.sha }} - - name: 'Build stable-mir-json and kmir' - run: docker exec --user github-user mir-semantics-haskell-${GITHUB_SHA} make stable-mir-json build + - name: 'Pull pre-built image and start container' + run: | + echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin + docker pull ${{ needs.build.outputs.image }} + docker run \ + --name mir-semantics-haskell-${GITHUB_SHA} \ + --rm \ + --interactive \ + --tty \ + --detach \ + --workdir /home/github-user/workspace \ + ${{ needs.build.outputs.image }} + docker cp . mir-semantics-haskell-${GITHUB_SHA}:/home/github-user/workspace + docker exec --user root mir-semantics-haskell-${GITHUB_SHA} \ + chown -R github-user:github-user /home/github-user/workspace - name: 'Run Haskell integration tests' run: docker exec --user github-user mir-semantics-haskell-${GITHUB_SHA} make test-integration-haskell - name: 'Tear down Docker' if: always() - run: docker stop --time 0 mir-semantics-haskell-${GITHUB_SHA} + run: docker stop --time 0 mir-semantics-haskell-${GITHUB_SHA} || true stable-mir-json-integration-tests: needs: code-quality-checks