ci: split integration tests into parallel LLVM and Haskell jobs #993
+180
−11
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| from __future__ import annotations | ||
|
|
||
| import hashlib | ||
| import time | ||
| from typing import TYPE_CHECKING | ||
|
|
||
| import pytest | ||
|
|
||
| if TYPE_CHECKING: | ||
| from pathlib import Path | ||
|
|
||
| from pytest import TempPathFactory | ||
|
|
||
|
|
||
| def _smir_kompile_key(smir_json: Path, symbolic: bool) -> str: | ||
| """Return a short stable key for a (smir_json, symbolic) pair.""" | ||
| path_hash = hashlib.sha256(str(smir_json).encode()).hexdigest()[:16] | ||
| suffix = 'symbolic' if symbolic else 'concrete' | ||
| return f'{path_hash}-{suffix}' | ||
|
|
||
|
|
||
| @pytest.fixture(scope='session') | ||
| def kompile_cache_dir(tmp_path_factory: TempPathFactory) -> Path: | ||
| """Session-scoped base directory for kompile cache. | ||
|
|
||
| Returns a stable temporary directory that persists across the test session, | ||
| allowing KompileDigest caching in kompile_smir to avoid redundant llvm-kompile calls | ||
| when multiple test_exec_smir parametrizations share the same smir_json + symbolic combo. | ||
| """ | ||
| return tmp_path_factory.mktemp('kompile-cache', numbered=False) | ||
|
|
||
|
|
||
| @pytest.fixture(scope='session') | ||
| def exec_smir_kompile_dirs(kompile_cache_dir: Path) -> dict[str, Path]: | ||
| """Session-scoped mapping from (smir_json, symbolic) key to shared kompile output dir. | ||
|
|
||
| Multiple test_exec_smir invocations that share the same smir_json + symbolic flag | ||
| will reuse the same target_dir, letting KompileDigest skip redundant kompile calls. | ||
| """ | ||
| return {} | ||
|
|
||
|
|
||
| def get_exec_smir_target_dir( | ||
| smir_json: Path, | ||
| symbolic: bool, | ||
| kompile_cache_dir: Path, | ||
| exec_smir_kompile_dirs: dict[str, Path], | ||
| ) -> Path: | ||
| """Return a shared target directory for a given (smir_json, symbolic) pair. | ||
|
|
||
| Uses a file lock to avoid concurrent kompile races when running under pytest-xdist. | ||
| """ | ||
| key = _smir_kompile_key(smir_json, symbolic) | ||
| if key in exec_smir_kompile_dirs: | ||
| return exec_smir_kompile_dirs[key] | ||
|
|
||
| target = kompile_cache_dir / key | ||
| lock_file = kompile_cache_dir / f'{key}.lock' | ||
|
|
||
| try: | ||
| # Attempt to claim the lock (atomic create) | ||
| with open(lock_file, 'x'): | ||
| target.mkdir(parents=True, exist_ok=True) | ||
| exec_smir_kompile_dirs[key] = target | ||
| lock_file.unlink(missing_ok=True) | ||
|
Comment on lines
+62
to
+65
There was a problem hiding this comment.
The lock file is removed immediately after creating the target directory, but the expensive work ( Useful? React with 👍 / 👎. |
||
| except FileExistsError: | ||
| # Another worker is building; wait for the lock to be released (max 5 min) | ||
| target.mkdir(parents=True, exist_ok=True) | ||
| secs = 0 | ||
| while lock_file.exists() and secs < 300: | ||
| time.sleep(1) | ||
| secs += 1 | ||
| exec_smir_kompile_dirs[key] = target | ||
|
Comment on lines
+70
to
+73
There was a problem hiding this comment.
When another worker holds the lock longer than 300 seconds, this code still records and returns Useful? React with 👍 / 👎. |
||
|
|
||
| return target | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This job checks out with
actions/checkout@v4usingsecrets.JENKINS_GITHUB_PATand then commits/pushes the running container, which already has the copied workspace under/home/github-user/workspace; in v4, checkout persists auth credentials for git commands, so the committed layer can include those credentials (for example via repo git config) and expose them to anyone who can pullghcr.io/.../ci:<sha>. Avoid committing that workspace as-is (or setpersist-credentials: falseand scrub.gitbeforedocker commit).Useful? React with 👍 / 👎.