H-6664: Python monorepo infrastructure (uv workspace + turbo integration) — experimental spike

.github/actions/prune-repository/action.yml

@@ -17,7 +17,7 @@ runs:
     - name: Copy required files
       shell: bash
       run: |
-        cp -R Cargo.toml Cargo.lock rust-toolchain.toml oxfmt.config.ts infra/ out/
+        cp -R Cargo.toml Cargo.lock rust-toolchain.toml oxfmt.config.ts pyproject.toml uv.lock infra/ out/
 
         # Globs are fun, especially in Bash. Covers all dot-files except `.`, `..`, and `.git`.
         shopt -s extglob

.github/actions/prune-repository/prune.py

@@ -6,6 +6,7 @@
 
 import argparse
 import json
+import shutil
 import subprocess
 from collections.abc import Iterable
 from glob import glob
@@ -168,6 +169,71 @@ def stub_missing_members(*, dry_run: bool = False) -> None:
         out_cargo.write_text(f'[package]\nname = "{name}"\nedition.workspace = true\n')
 
 
+def resolve_python_workspace_members() -> list[Path]:
+    """Read uv workspace members from pyproject.toml and resolve globs to member directories."""
+
+    with open("pyproject.toml", "rb") as fh:
+        pyproject = tomllib.load(fh)
+
+    members: list[str] = pyproject["tool"]["uv"]["workspace"]["members"]
+    paths: list[Path] = []
+
+    for member in members:
+        for match in glob(member):
+            if (Path(match) / "pyproject.toml").exists():
+                paths.append(Path(match))
+
+    return paths
+
+
+def python_workspace_scopes() -> frozenset[str]:
+    """Return the turbo package names of all uv workspace members.
+
+    uv always resolves the whole workspace: `uv sync --locked` and `uv run --frozen`
+    fail when a member directory referenced by the root `pyproject.toml`/`uv.lock` is
+    missing. Python packages are therefore exempt from pruning by adding them to the
+    scopes (which also keeps their entries in the pruned `yarn.lock`). The root
+    `pyproject.toml` and `uv.lock` are restored by the action's copy step.
+    """
+
+    scopes: set[str] = set()
+
+    for directory in resolve_python_workspace_members():
+        package_json = directory / "package.json"
+        if package_json.exists():
+            with open(package_json, "rb") as fh:
+                scopes.add(json.load(fh)["name"])
+
+    return frozenset(scopes)
+
+
+def preserve_python_members(*, dry_run: bool = False) -> None:
+    """Copy uv workspace members without a `package.json` into the pruned tree.
+
+    Members with a `package.json` are turbo workspaces and are kept via
+    `python_workspace_scopes`; anything else is invisible to turbo and copied here.
+    """
+
+    for directory in resolve_python_workspace_members():
+        out_dir = Path("out") / directory
+
+        if out_dir.exists():
+            continue
+
+        if dry_run:
+            print(f"[dry-run] Would preserve Python workspace member: {directory}")
+            continue
+
+        print(f"Preserving Python workspace member: {directory}")
+        shutil.copytree(
+            directory,
+            out_dir,
+            ignore=shutil.ignore_patterns(
+                "__pycache__", ".pytest_cache", ".ruff_cache", ".venv"
+            ),
+        )
+
+
 def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     parser = argparse.ArgumentParser(
         description="Prune the turbo workspace to only the packages required for the given scopes.",
@@ -192,9 +258,10 @@ def main(argv: list[str] | None = None) -> None:
     }
 
     dependencies = turbo_dependency_map()
-    scopes = fixpoint_expand(initial, dependencies)
+    scopes = fixpoint_expand(initial | python_workspace_scopes(), dependencies)
     turbo_prune(scopes, dry_run=args.dry_run)
     stub_missing_members(dry_run=args.dry_run)
+    preserve_python_members(dry_run=args.dry_run)
 
 
 if __name__ == "__main__":

.github/workflows/lint.yml

@@ -97,6 +97,14 @@ jobs:
             | jq '[.tasks[] | select(.task == "lint:clippy" and .command != "<NONEXISTENT>")] != []' || echo 'false')
           echo "clippy=$CLIPPY" | tee -a $GITHUB_OUTPUT
 
+          RUFF=$(turbo run lint:ruff --filter '${{ matrix.name }}' --dry-run=json \
+            | jq '[.tasks[] | select(.task == "lint:ruff" and .command != "<NONEXISTENT>")] != []' || echo 'false')
+          echo "ruff=$RUFF" | tee -a $GITHUB_OUTPUT
+
+          BASEDPYRIGHT=$(turbo run lint:basedpyright --filter '${{ matrix.name }}' --dry-run=json \
+            | jq '[.tasks[] | select(.task == "lint:basedpyright" and .command != "<NONEXISTENT>")] != []' || echo 'false')
+          echo "basedpyright=$BASEDPYRIGHT" | tee -a $GITHUB_OUTPUT
+
           HAS_RUST=$([[ -f "${{ matrix.path }}/Cargo.toml" || ${{ matrix.path }} = "apps/hash-graph" ]] && echo 'true' || echo 'false')
           echo "has-rust=$HAS_RUST" | tee -a $GITHUB_OUTPUT
 
@@ -129,6 +137,14 @@ jobs:
         if: always() && steps.lints.outputs.tsc == 'true'
         run: turbo run lint:tsc --filter "${{ matrix.name }}"
 
+      - name: Run ruff
+        if: always() && steps.lints.outputs.ruff == 'true'
+        run: turbo run lint:ruff --filter "${{ matrix.name }}"
+
+      - name: Run basedpyright
+        if: always() && steps.lints.outputs.basedpyright == 'true'
+        run: turbo run lint:basedpyright --filter "${{ matrix.name }}"
+
       - name: Run rustfmt
         if: always() && steps.lints.outputs.has-rust == 'true'
         working-directory: ${{ matrix.path }}