From 374b5800f39a8d051a2927717e609ad1a1743cdf Mon Sep 17 00:00:00 2001 From: timurcarstensen Date: Fri, 1 Aug 2025 18:00:02 +0200 Subject: [PATCH 1/9] Add GitHub Actions CI workflows for linting and formatting - Add main CI workflow with Ruff for formatting and linting - Add pre-commit workflow for automated checks - Configure Ruff in pyproject.toml for consistent code style - Add .pre-commit-config.yaml for local pre-commit hooks - Include YAML and shell script linting - Support Python 3.8-3.11 in CI matrix --- .github/workflows/ci.yml | 79 ++++++++++++++++++++++++++++++++ .github/workflows/pre-commit.yml | 16 +++++++ .pre-commit-config.yaml | 27 +++++++++++ pyproject.toml | 32 ++++++++++++- 4 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/pre-commit.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f9a8804 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,79 @@ +name: CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + lint-and-format: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache pip packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ruff + + - name: Check code formatting with Ruff + run: | + ruff format --check --diff . + + - name: Lint with Ruff + run: | + ruff check . + + - name: Type check with mypy + run: | + mypy . --ignore-missing-imports || true + continue-on-error: true + + yaml-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install yamllint + run: | + pip install yamllint + + - name: Lint YAML files + run: | + yamllint . || true + continue-on-error: true + + shell-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run ShellCheck + uses: ludeeus/action-shellcheck@master + with: + severity: warning + check_together: yes + scandir: './oellm' \ No newline at end of file diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..4e32936 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,16 @@ +name: Pre-commit + +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + - uses: pre-commit/action@v3.0.0 \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..3de1803 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - id: check-merge-conflict + - id: check-toml + - id: check-json + - id: debug-statements + - id: mixed-line-ending + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.13 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.8.0 + hooks: + - id: mypy + additional_dependencies: [types-all] + args: [--ignore-missing-imports] \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 57b68e2..bae32ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,4 +38,34 @@ torchvision = [ [[tool.uv.index]] name = "pytorch-cpu" url = "https://download.pytorch.org/whl/cpu" -explicit = true \ No newline at end of file +explicit = true + +[tool.ruff] +line-length = 88 +target-version = "py38" + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "UP", # pyupgrade +] +ignore = [ + "E501", # line too long + "B008", # do not perform function calls in argument defaults + "C901", # too complex + "W191", # indentation contains tabs +] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401"] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" \ No newline at end of file From 770d87300de27df7f28135de5e258b51acde25cf Mon Sep 17 00:00:00 2001 From: timurcarstensen Date: Fri, 1 Aug 2025 18:03:10 +0200 Subject: [PATCH 2/9] Update CI to use Python 3.12 only - Remove matrix strategy, use single Python 3.12 version - Update all workflows to use Python 3.12 consistently --- .github/workflows/ci.yml | 9 +++------ .github/workflows/pre-commit.yml | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9a8804..1e49531 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,17 +9,14 @@ on: jobs: lint-and-format: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version }} + python-version: "3.12" - name: Cache pip packages uses: actions/cache@v3 @@ -55,7 +52,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "3.12" - name: Install yamllint run: | diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 4e32936..fee2a54 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -12,5 +12,5 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "3.12" - uses: pre-commit/action@v3.0.0 \ No newline at end of file From 7d37da0b625a7d8a101b6583ec657b049775785d Mon Sep 17 00:00:00 2001 From: timurcarstensen Date: Fri, 1 Aug 2025 18:04:07 +0200 Subject: [PATCH 3/9] Use uv for faster dependency installation in CI - Replace pip with uv for all dependency installations - Use uv venv and uv run for consistent environment management - Significant speed improvements expected in CI runs --- .github/workflows/ci.yml | 40 ++++++++++++++++---------------- .github/workflows/pre-commit.yml | 7 ++++-- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e49531..2a02495 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,35 +13,31 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Python 3.12 - uses: actions/setup-python@v4 + - name: Install uv + uses: astral-sh/setup-uv@v3 with: - python-version: "3.12" + version: "latest" - - name: Cache pip packages - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-pip- + - name: Set up Python 3.12 + run: uv python install 3.12 - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install ruff + uv venv + uv pip install ruff - name: Check code formatting with Ruff run: | - ruff format --check --diff . + uv run ruff format --check --diff . - name: Lint with Ruff run: | - ruff check . + uv run ruff check . - name: Type check with mypy run: | - mypy . --ignore-missing-imports || true + uv pip install mypy + uv run mypy . --ignore-missing-imports || true continue-on-error: true yaml-lint: @@ -49,18 +45,22 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v4 + - name: Install uv + uses: astral-sh/setup-uv@v3 with: - python-version: "3.12" + version: "latest" + + - name: Set up Python 3.12 + run: uv python install 3.12 - name: Install yamllint run: | - pip install yamllint + uv venv + uv pip install yamllint - name: Lint YAML files run: | - yamllint . || true + uv run yamllint . || true continue-on-error: true shell-lint: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index fee2a54..63c853d 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -10,7 +10,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - name: Install uv + uses: astral-sh/setup-uv@v3 with: - python-version: "3.12" + version: "latest" + - name: Set up Python 3.12 + run: uv python install 3.12 - uses: pre-commit/action@v3.0.0 \ No newline at end of file From 7f2b345365c327d2df4155ce2117fe89cab0af86 Mon Sep 17 00:00:00 2001 From: timurcarstensen Date: Fri, 1 Aug 2025 18:05:49 +0200 Subject: [PATCH 4/9] misc --- .github/workflows/ci.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a02495..66f223f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,14 +63,3 @@ jobs: uv run yamllint . || true continue-on-error: true - shell-lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Run ShellCheck - uses: ludeeus/action-shellcheck@master - with: - severity: warning - check_together: yes - scandir: './oellm' \ No newline at end of file From 43424a90c98c43b5ebab7684afaed9b6f3ea4449 Mon Sep 17 00:00:00 2001 From: timurcarstensen Date: Fri, 1 Aug 2025 18:07:21 +0200 Subject: [PATCH 5/9] making ruff happy --- oellm/main.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/oellm/main.py b/oellm/main.py index 30ec779..4ae97a3 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -1,5 +1,4 @@ # Standard library imports -import yaml import logging import os import re @@ -12,6 +11,7 @@ from typing import Iterable import pandas as pd +import yaml from huggingface_hub import hf_hub_download, snapshot_download from jsonargparse import auto_cli from rich.console import Console @@ -22,7 +22,7 @@ def ensure_singularity_image(image_name: str) -> None: # TODO: switch to OELLM dataset repo once it is created hf_repo = os.environ.get("HF_SIF_REPO", "timurcarstensen/testing") image_path = Path(os.getenv("EVAL_BASE_DIR")) / image_name - + try: hf_hub_download( repo_id=hf_repo, @@ -30,15 +30,15 @@ def ensure_singularity_image(image_name: str) -> None: repo_type="dataset", local_dir=os.getenv("EVAL_BASE_DIR"), ) - logging.info("Successfully downloaded latest Singularity image from HuggingFace") + logging.info( + "Successfully downloaded latest Singularity image from HuggingFace" + ) except Exception as e: logging.warning( "Failed to fetch latest container image from HuggingFace: %s", str(e) ) if image_path.exists(): - logging.info( - "Using existing Singularity image at %s", image_path - ) + logging.info("Using existing Singularity image at %s", image_path) else: raise RuntimeError( f"No container image found at {image_path} and failed to download from HuggingFace. " @@ -79,7 +79,7 @@ def _load_cluster_env() -> None: """ Loads the correct cluster environment variables from `clusters.yaml` based on the hostname. """ - with open(Path(__file__).parent / "clusters.yaml", "r") as f: + with open(Path(__file__).parent / "clusters.yaml") as f: clusters = yaml.safe_load(f) hostname = socket.gethostname() @@ -181,10 +181,7 @@ def _process_model_paths(models: Iterable[str]) -> dict[str, list[Path | str]]: ) if "," in model: - model_kwargs = { - k: v - for k, v in [kv.split("=") for kv in model.split(",") if "=" in kv] - } + model_kwargs = dict([kv.split("=") for kv in model.split(",") if "=" in kv]) # The first element before the comma is the repository ID on the 🤗 Hub repo_id = model.split(",")[0] @@ -359,16 +356,20 @@ def schedule_evals( expanded_rows.append(new_row) df = pd.DataFrame(expanded_rows) else: - logging.info("Skipping model path processing and validation (--skip-checks enabled)") + logging.info( + "Skipping model path processing and validation (--skip-checks enabled)" + ) elif models and tasks and n_shot is not None: if not skip_checks: model_path_map = _process_model_paths(models.split(",")) model_paths = [p for paths in model_path_map.values() for p in paths] else: - logging.info("Skipping model path processing and validation (--skip-checks enabled)") + logging.info( + "Skipping model path processing and validation (--skip-checks enabled)" + ) model_paths = models.split(",") - + tasks_list = tasks.split(",") # cross product of model_paths and tasks into a dataframe @@ -425,7 +426,7 @@ def schedule_evals( logging.debug(f"Saved evaluation dataframe to temporary CSV: {csv_path}") - with open(Path(__file__).parent / "template.sbatch", "r") as f: + with open(Path(__file__).parent / "template.sbatch") as f: sbatch_template = f.read() # replace the placeholders in the template with the actual values @@ -468,17 +469,18 @@ def schedule_evals( ) logging.info("Job submitted successfully.") logging.info(result.stdout) - + # Provide helpful information about job monitoring and file locations logging.info(f"📁 Evaluation directory: {evals_dir}") logging.info(f"📄 SLURM script: {sbatch_script_path}") logging.info(f"📋 Job configuration: {csv_path}") logging.info(f"📜 SLURM logs will be stored in: {slurm_logs_dir}") logging.info(f"📊 Results will be stored in: {evals_dir / 'results'}") - + # Extract job ID from sbatch output for monitoring commands import re - job_id_match = re.search(r'Submitted batch job (\d+)', result.stdout) + + job_id_match = re.search(r"Submitted batch job (\d+)", result.stdout) if job_id_match: job_id = job_id_match.group(1) logging.info(f"🔍 Monitor job status: squeue -j {job_id}") From b5d2149bb64d3e02d7ea784d5ded06e84075e30a Mon Sep 17 00:00:00 2001 From: timurcarstensen Date: Fri, 1 Aug 2025 18:13:04 +0200 Subject: [PATCH 6/9] don't need precommit workflow --- .github/workflows/pre-commit.yml | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 .github/workflows/pre-commit.yml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml deleted file mode 100644 index 63c853d..0000000 --- a/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Pre-commit - -on: - pull_request: - push: - branches: [main] - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install uv - uses: astral-sh/setup-uv@v3 - with: - version: "latest" - - name: Set up Python 3.12 - run: uv python install 3.12 - - uses: pre-commit/action@v3.0.0 \ No newline at end of file From be6eac4ff75887502754530c6e0b0447b4655667 Mon Sep 17 00:00:00 2001 From: timurcarstensen Date: Fri, 1 Aug 2025 18:15:00 +0200 Subject: [PATCH 7/9] Use uvx for tool execution instead of virtual environments - Replace uv venv/pip install with uvx for direct tool execution - Eliminates unnecessary virtual environment creation - Faster CI runs by avoiding full dependency installation - Uses uvx ruff, uvx mypy, uvx yamllint directly --- .github/workflows/ci.yml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66f223f..8a03dd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,23 +21,17 @@ jobs: - name: Set up Python 3.12 run: uv python install 3.12 - - name: Install dependencies - run: | - uv venv - uv pip install ruff - - name: Check code formatting with Ruff run: | - uv run ruff format --check --diff . + uvx ruff format --check --diff . - name: Lint with Ruff run: | - uv run ruff check . + uvx ruff check . - name: Type check with mypy run: | - uv pip install mypy - uv run mypy . --ignore-missing-imports || true + uvx mypy . --ignore-missing-imports || true continue-on-error: true yaml-lint: @@ -53,13 +47,8 @@ jobs: - name: Set up Python 3.12 run: uv python install 3.12 - - name: Install yamllint - run: | - uv venv - uv pip install yamllint - - name: Lint YAML files run: | - uv run yamllint . || true + uvx yamllint . || true continue-on-error: true From 1eba0f98f446e84bcac160278844a5e0c72a1713 Mon Sep 17 00:00:00 2001 From: timurcarstensen Date: Fri, 1 Aug 2025 18:16:06 +0200 Subject: [PATCH 8/9] making ruff happy --- oellm/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/oellm/main.py b/oellm/main.py index 4ae97a3..1feeaa5 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -181,7 +181,9 @@ def _process_model_paths(models: Iterable[str]) -> dict[str, list[Path | str]]: ) if "," in model: - model_kwargs = dict([kv.split("=") for kv in model.split(",") if "=" in kv]) + model_kwargs = dict( + [kv.split("=") for kv in model.split(",") if "=" in kv] + ) # The first element before the comma is the repository ID on the 🤗 Hub repo_id = model.split(",")[0] From b3b070bb15e6500f35aae257d8993461e51f455a Mon Sep 17 00:00:00 2001 From: timurcarstensen Date: Fri, 1 Aug 2025 18:17:34 +0200 Subject: [PATCH 9/9] making ruff happy --- oellm/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oellm/main.py b/oellm/main.py index 1feeaa5..af60e33 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -115,7 +115,7 @@ def _load_cluster_env() -> None: if len(e.args) > 1: raise ValueError( f"Env. variable substitution for {k} failed. Missing keys: {', '.join(e.args)}" - ) + ) from e missing_key: str = e.args[0] os.environ[k] = str(v).format(