From 35bc0a2449bfa47f966129eec07d4e91a7998eac Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Tue, 22 Jul 2025 11:39:03 +0200 Subject: [PATCH 01/71] chores(ci): opted-out nix-fast-build --- .github/workflows/nix-build.yml | 4 +--- flake.lock | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 0a5d51800..6f2127a42 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -69,9 +69,7 @@ jobs: - name: Build psql bundle run: > - nix run "github:Mic92/nix-fast-build?rev=b1dae483ab7d4139a6297e02b6de9e5d30e43d48" - -- --skip-cached --no-nom ${{ matrix.runner == 'macos-latest-xlarge' && '--max-jobs 1' || '' }} --copy-to "s3://nix-postgres-artifacts?secret-key=/etc/nix/nix-secret-key" - --flake ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" + nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" env: AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} diff --git a/flake.lock b/flake.lock index b2d4ff6e3..8152dbb73 100644 --- a/flake.lock +++ b/flake.lock @@ -131,11 +131,11 @@ ] }, "locked": { - "lastModified": 1749427739, - "narHash": "sha256-Nm0oMqFNRnJsiZYeNChmefmjeVCOzngikpSQhgs7iXI=", + "lastModified": 1763868605, + "narHash": "sha256-wO8Lk66GPQeSpzXUzXCBpe2Pj1De17xByrROXxnwIPY=", "owner": "Mic92", "repo": "nix-fast-build", - "rev": "b1dae483ab7d4139a6297e02b6de9e5d30e43d48", + "rev": "0c7fd47fb587250e931a01c6645ab5c0cd737af8", "type": "github" }, "original": { From 81834ab82da9d0c202bd5e822fc4813623da0e3e Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Thu, 24 Jul 2025 09:42:53 +0200 Subject: [PATCH 02/71] chores(ci): set up nix-github-actions --- .github/workflows/nix-build.yml | 31 ++++++++++++++++++++----------- flake.lock | 21 +++++++++++++++++++++ flake.nix | 3 +++ nix/github-actions.nix | 6 ++++++ 4 files changed, 50 insertions(+), 11 deletions(-) create mode 100644 nix/github-actions.nix diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 6f2127a42..b2e354483 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -19,18 +19,27 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + nix-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v30 + - id: set-matrix + name: Generate Nix Matrix + run: | + set -Eeu + matrix="$(nix eval --json '.#githubActions.matrix')" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + build-run-image: + name: ${{ matrix.name }} (${{ matrix.system }}) + needs: nix-matrix + runs-on: ${{ matrix.os }} strategy: fail-fast: false - matrix: - include: - - runner: blacksmith-32vcpu-ubuntu-2404 - arch: amd64 - - runner: blacksmith-32vcpu-ubuntu-2404-arm - arch: arm64 - - runner: macos-latest-xlarge - arch: arm64 - runs-on: ${{ matrix.runner }} + matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} timeout-minutes: 180 steps: - name: Checkout Repo @@ -66,10 +75,10 @@ jobs: sudo rm -rf /tmp/* 2>/dev/null || true echo "=== AFTER CLEANUP ===" df -h - - - name: Build psql bundle + - name: Build psql bundle run: > nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" + - run: nix build -L '.#${{ matrix.attr }}' env: AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} diff --git a/flake.lock b/flake.lock index 8152dbb73..a8d1fac7c 100644 --- a/flake.lock +++ b/flake.lock @@ -144,6 +144,26 @@ "type": "github" } }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1737420293, + "narHash": "sha256-F1G5ifvqTpJq7fdkT34e/Jy9VCyzd5XfJ9TO8fHhJWE=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "f4158fa080ef4503c8f4c820967d946c2af31ec9", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, "nix2container": { "inputs": { "flake-utils": [ @@ -237,6 +257,7 @@ "git-hooks": "git-hooks", "nix-editor": "nix-editor", "nix-fast-build": "nix-fast-build", + "nix-github-actions": "nix-github-actions", "nix2container": "nix2container", "nixpkgs": "nixpkgs", "nixpkgs-go124": "nixpkgs-go124", diff --git a/flake.nix b/flake.nix index 8f83a0a25..4b4f18230 100644 --- a/flake.nix +++ b/flake.nix @@ -28,6 +28,8 @@ git-hooks.inputs.nixpkgs.follows = "nixpkgs"; nixpkgs-go124.url = "github:Nixos/nixpkgs/d2ac4dfa61fba987a84a0a81555da57ae0b9a2b0"; nixpkgs-pgbackrest.url = "github:nixos/nixpkgs/nixos-unstable-small"; + nix-github-actions.url = "github:nix-community/nix-github-actions"; + nix-github-actions.inputs.nixpkgs.follows = "nixpkgs"; }; outputs = @@ -49,6 +51,7 @@ nix/nixpkgs.nix nix/packages nix/overlays + nix/github-actions.nix ]; }); } diff --git a/nix/github-actions.nix b/nix/github-actions.nix new file mode 100644 index 000000000..b2f04a6c2 --- /dev/null +++ b/nix/github-actions.nix @@ -0,0 +1,6 @@ +{ inputs, ... }: +{ + flake.githubActions = inputs.nix-github-actions.lib.mkGithubMatrix { + checks = inputs.nixpkgs.lib.getAttrs [ "x86_64-linux" ] inputs.self.checks; + }; +} From 97f2d72061f7ac25ff99a70004b6f01fccc61be1 Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Mon, 11 Aug 2025 09:11:45 +0200 Subject: [PATCH 03/71] chores(ci): comment out vestigal testinfra-ami-build --- .github/workflows/nix-build.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index b2e354483..168ef0119 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -10,7 +10,7 @@ on: permissions: id-token: write - # required by testinfra-ami-build dependent workflows + # required by dependent workflows contents: write packages: write @@ -84,13 +84,6 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} - run-testinfra: - needs: build-run-image - if: ${{ success() }} - uses: ./.github/workflows/testinfra-ami-build.yml - secrets: - DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} - run-tests: needs: build-run-image if: ${{ success() }} From 777ebc0a8525a036c8d01d8492e7bd04c8cb5149 Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Mon, 11 Aug 2025 12:10:27 +0200 Subject: [PATCH 04/71] chores(ci): use custom github runners --- nix/github-actions.nix | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/nix/github-actions.nix b/nix/github-actions.nix index b2f04a6c2..752f0b41a 100644 --- a/nix/github-actions.nix +++ b/nix/github-actions.nix @@ -1,6 +1,18 @@ { inputs, ... }: +let + githubPlatforms = { + "x86_64-linux" = "large-linux-x86"; + "aarch64-linux" = "large-linux-arm"; + "aarch64-darwin" = "macos-latest-xlarge"; + }; +in { flake.githubActions = inputs.nix-github-actions.lib.mkGithubMatrix { - checks = inputs.nixpkgs.lib.getAttrs [ "x86_64-linux" ] inputs.self.checks; + checks = inputs.nixpkgs.lib.getAttrs [ + "x86_64-linux" + "aarch64-linux" + "aarch64-darwin" + ] inputs.self.checks; + platforms = githubPlatforms; }; } From 82ed2f5bf3f6f03477b60d68140f97861b2be17b Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Mon, 29 Sep 2025 14:38:39 +0200 Subject: [PATCH 05/71] chores(ci): use nix-eval-jobs and sets AWS creds to /etc/nix/aws --- .github/workflows/nix-build.yml | 80 +++++++++++++++------------------ nix/github-actions.nix | 6 +-- 2 files changed, 40 insertions(+), 46 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 168ef0119..d4b5d1a49 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -20,70 +20,64 @@ concurrency: jobs: nix-matrix: - runs-on: ubuntu-latest + runs-on: + group: self-hosted-runners-nix + labels: + - aarch64-darwin outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v30 + - name: Checkout Repo + uses: actions/checkout@v4 - id: set-matrix name: Generate Nix Matrix run: | set -Eeu - matrix="$(nix eval --json '.#githubActions.matrix')" - echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + echo matrix="$(nix eval --json '.#githubActions.matrix')" >> "$GITHUB_OUTPUT" build-run-image: name: ${{ matrix.name }} (${{ matrix.system }}) needs: nix-matrix - runs-on: ${{ matrix.os }} + runs-on: + group: ${{ contains(matrix.os, 'blacksmith-32vcpu-ubuntu-2404') && '' || 'self-hosted-runners-nix' }} + labels: + - ${{ matrix.os }} strategy: fail-fast: false matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} - timeout-minutes: 180 steps: - name: Checkout Repo - uses: supabase/postgres/.github/actions/shared-checkout@HEAD - - uses: ./.github/actions/nix-install-ephemeral + uses: actions/checkout@v4 + - name: aws-oidc + uses: aws-actions/configure-aws-credentials@v4.3.1 + with: + aws-region: us-east-2 + role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role # Shared Services + role-session-name: gha-oidc-${{ github.run_id }} + - name: aws-creds + uses: aws-actions/configure-aws-credentials@v4.3.1 with: - push-to-cache: ${{ github.secret_source == 'Actions' && 'true' || 'false' }} - env: - DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} - NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - - name: Aggressive disk cleanup for DuckDB build - if: matrix.runner == 'macos-latest-xlarge' + disable-retry: true + aws-region: us-east-2 + role-to-assume: arn:aws:iam::436098097459:role/nix-artifacts-deploy-role # supabase-dev + role-session-name: gha-oidc-${{ github.run_id }} + role-chaining: true + role-skip-session-tagging: true + role-duration-seconds: 900 # TODO: switch to 18000 (5 hours) + - name: Write creds files run: | - nix --version - echo "=== BEFORE CLEANUP ===" - df -h - # Remove major space consumers - sudo rm -rf /usr/share/dotnet || true - sudo rm -rf /usr/local/lib/android || true - sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform || true - sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/watchOS.platform || true - sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/tvOS.platform || true - # Clean everything possible - sudo rm -rf /opt/ghc || true - sudo rm -rf /usr/local/share/boost || true - sudo rm -rf /opt/homebrew || true - sudo xcrun simctl delete all 2>/dev/null || true - # Aggressive cache cleanup - sudo rm -rf /System/Library/Caches/* 2>/dev/null || true - sudo rm -rf /Library/Caches/* 2>/dev/null || true - sudo rm -rf ~/Library/Caches/* 2>/dev/null || true - sudo rm -rf /private/var/log/* 2>/dev/null || true - sudo rm -rf /tmp/* 2>/dev/null || true - echo "=== AFTER CLEANUP ===" - df -h + umask 006 + cat > /etc/nix/aws/nix-aws-credentials < nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - - run: nix build -L '.#${{ matrix.attr }}' - env: - AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} - AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} - + - name: nix-fast-build + run: nix build -L run-tests: needs: build-run-image if: ${{ success() }} diff --git a/nix/github-actions.nix b/nix/github-actions.nix index 752f0b41a..ad43f2a6f 100644 --- a/nix/github-actions.nix +++ b/nix/github-actions.nix @@ -1,9 +1,9 @@ { inputs, ... }: let githubPlatforms = { - "x86_64-linux" = "large-linux-x86"; - "aarch64-linux" = "large-linux-arm"; - "aarch64-darwin" = "macos-latest-xlarge"; + "aarch64-linux" = "aarch64-linux"; + "aarch64-darwin" = "aarch64-darwin"; + "x86_64-linux" = "blacksmith-32vcpu-ubuntu-2404"; }; in { From ecf0b4e0182455f3e68e2d627a531b663277318a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 29 Sep 2025 22:15:14 +0200 Subject: [PATCH 06/71] feat(ci): do not build if already cached --- .github/workflows/nix-build.yml | 21 +++-- scripts/github-matrix.py | 151 ++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 11 deletions(-) create mode 100644 scripts/github-matrix.py diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index d4b5d1a49..0c953b98f 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -14,10 +14,6 @@ permissions: contents: write packages: write -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - jobs: nix-matrix: runs-on: @@ -33,15 +29,12 @@ jobs: name: Generate Nix Matrix run: | set -Eeu - echo matrix="$(nix eval --json '.#githubActions.matrix')" >> "$GITHUB_OUTPUT" + echo matrix="$(python scripts/github-matrix.py)" >> "$GITHUB_OUTPUT" build-run-image: name: ${{ matrix.name }} (${{ matrix.system }}) needs: nix-matrix - runs-on: - group: ${{ contains(matrix.os, 'blacksmith-32vcpu-ubuntu-2404') && '' || 'self-hosted-runners-nix' }} - labels: - - ${{ matrix.os }} + runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} strategy: fail-fast: false matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} @@ -76,8 +69,14 @@ jobs: - name: Build psql bundle run: > nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - - name: nix-fast-build - run: nix build -L + - name: nix build + run: | + if ${{ matrix.already_cached }}; then + echo "${{ matrix.attr }} already cached, skipping build" + exit 0 + fi + nix build -L .#${{ matrix.attr }} + run-tests: needs: build-run-image if: ${{ success() }} diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py new file mode 100644 index 000000000..53deb4f96 --- /dev/null +++ b/scripts/github-matrix.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 + +import json +import os +import subprocess +import sys +from typing import ( + Any, + Dict, + Generator, + List, + Literal, + NotRequired, + Optional, + Set, + TypedDict, +) + + +class NixEvalJobsOutput(TypedDict): + """Raw output from nix-eval-jobs command.""" + + attr: str + attrPath: List[str] + cacheStatus: Literal["notBuilt", "cached", "local"] + drvPath: str + isCached: bool + name: str + system: str + neededBuilds: NotRequired[List[Any]] + neededSubstitutes: NotRequired[List[Any]] + outputs: NotRequired[Dict[str, str]] + + +class RunsOnConfig(TypedDict): + """GitHub Actions runs-on configuration.""" + + group: NotRequired[str] + labels: List[str] + + +class GitHubActionPackage(TypedDict): + """Processed package for GitHub Actions matrix.""" + + attr: str + name: str + system: str + already_cached: bool + runs_on: RunsOnConfig + + +BUILD_RUNNER_MAP: Dict[str, RunsOnConfig] = { + "aarch64-linux": { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-linux"], + }, + "aarch64-darwin": { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-darwin"], + }, + "x86_64-linux": { + "labels": ["blacksmith-32vcpu-ubuntu-2404"], + }, +} + + +def get_worker_count() -> int: + """Get optimal worker count based on CPU cores.""" + try: + return max(1, int(os.cpu_count())) + except (OSError, AttributeError): + print( + "Warning: Unable to get CPU count, using default max_workers=1", + file=sys.stderr, + ) + return 1 + + +def build_nix_eval_command(max_workers: int) -> List[str]: + """Build the nix-eval-jobs command with appropriate flags.""" + return [ + "nix-eval-jobs", + "--flake", + ".#checks", + "--check-cache-status", + "--force-recurse", + "--quiet", + "--workers", + str(max_workers), + ] + + +def parse_nix_eval_line( + line: str, drv_paths: Set[str] +) -> Optional[GitHubActionPackage]: + """Parse a single line of nix-eval-jobs output""" + if not line.strip(): + return None + + try: + data: NixEvalJobsOutput = json.loads(line) + if data["drvPath"] in drv_paths: + return None + drv_paths.add(data["drvPath"]) + print(f"Processing package: {data}", file=sys.stderr) + + runs_on_config = BUILD_RUNNER_MAP[data["system"]] + + return { + "attr": "checks." + data["attr"], + "name": data["name"], + "system": data["system"], + "already_cached": data.get("cacheStatus") != "notBuilt", + "runs_on": runs_on_config, + } + except json.JSONDecodeError: + print(f"Skipping invalid JSON line: {line}", file=sys.stderr) + return None + + +def run_nix_eval_jobs(cmd: List[str]) -> Generator[GitHubActionPackage, None, None]: + """Run nix-eval-jobs and yield parsed package data.""" + print(f"Running command: {' '.join(cmd)}", file=sys.stderr) + + with subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) as process: + drv_paths = set() + + for line in process.stdout: + package = parse_nix_eval_line(line, drv_paths) + if package: + yield package + + if process.returncode and process.returncode != 0: + print("Error: Evaluation failed", file=sys.stderr) + sys.stderr.write(process.stderr.read()) + sys.exit(process.returncode) + + +def main() -> None: + max_workers = get_worker_count() + cmd = build_nix_eval_command(max_workers) + + gh_action_packages = list(run_nix_eval_jobs(cmd)) + gh_output = {"include": gh_action_packages} + print(json.dumps(gh_output)) + + +if __name__ == "__main__": + main() From 005432608f76e939b2438ab5b1ac66bf805b04d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 29 Sep 2025 23:45:46 +0200 Subject: [PATCH 07/71] chore: improve reproducibility of postgresql builds Do not create postgresql package that depends on the runtime evaluation context as that harms reproducibility. --- nix/postgresql/generic.nix | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/nix/postgresql/generic.nix b/nix/postgresql/generic.nix index 76904ced7..ea67cbc91 100644 --- a/nix/postgresql/generic.nix +++ b/nix/postgresql/generic.nix @@ -21,7 +21,6 @@ let libxml2, tzdata, libkrb5, - substituteAll, darwin, linux-pam, #orioledb specific @@ -193,11 +192,7 @@ let ./patches/paths-for-split-outputs.patch ./patches/specify_pkglibdir_at_runtime.patch ./patches/paths-with-postgresql-suffix.patch - - (substituteAll { - src = ./patches/locale-binary-path.patch; - locale = "${if stdenv.isDarwin then darwin.adv_cmds else lib.getBin stdenv.cc.libc}/bin/locale"; - }) + ./patches/locale-binary-path.patch ] ++ lib.optionals stdenv'.hostPlatform.isMusl ( # Using fetchurl instead of fetchpatch on purpose: https://github.com/NixOS/nixpkgs/issues/240141 @@ -213,6 +208,9 @@ let '' # Hardcode the path to pgxs so pg_config returns the path in $out substituteInPlace "src/common/config_info.c" --subst-var out + substituteInPlace "src/backend/commands/collationcmds.c" --replace-fail '@locale@' '${ + if stdenv.isDarwin then darwin.adv_cmds else lib.getBin stdenv.cc.libc + }/bin/locale' '' + lib.optionalString jitSupport '' # Force lookup of jit stuff in $out instead of $lib From 02ecbc5530c6dc5454736d2299085d24e6ae2fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 29 Sep 2025 23:55:01 +0200 Subject: [PATCH 08/71] chore: remove nix-github-actions dependency --- flake.lock | 21 --------------------- flake.nix | 3 --- nix/github-actions.nix | 18 ------------------ 3 files changed, 42 deletions(-) delete mode 100644 nix/github-actions.nix diff --git a/flake.lock b/flake.lock index a8d1fac7c..8152dbb73 100644 --- a/flake.lock +++ b/flake.lock @@ -144,26 +144,6 @@ "type": "github" } }, - "nix-github-actions": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1737420293, - "narHash": "sha256-F1G5ifvqTpJq7fdkT34e/Jy9VCyzd5XfJ9TO8fHhJWE=", - "owner": "nix-community", - "repo": "nix-github-actions", - "rev": "f4158fa080ef4503c8f4c820967d946c2af31ec9", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nix-github-actions", - "type": "github" - } - }, "nix2container": { "inputs": { "flake-utils": [ @@ -257,7 +237,6 @@ "git-hooks": "git-hooks", "nix-editor": "nix-editor", "nix-fast-build": "nix-fast-build", - "nix-github-actions": "nix-github-actions", "nix2container": "nix2container", "nixpkgs": "nixpkgs", "nixpkgs-go124": "nixpkgs-go124", diff --git a/flake.nix b/flake.nix index 4b4f18230..8f83a0a25 100644 --- a/flake.nix +++ b/flake.nix @@ -28,8 +28,6 @@ git-hooks.inputs.nixpkgs.follows = "nixpkgs"; nixpkgs-go124.url = "github:Nixos/nixpkgs/d2ac4dfa61fba987a84a0a81555da57ae0b9a2b0"; nixpkgs-pgbackrest.url = "github:nixos/nixpkgs/nixos-unstable-small"; - nix-github-actions.url = "github:nix-community/nix-github-actions"; - nix-github-actions.inputs.nixpkgs.follows = "nixpkgs"; }; outputs = @@ -51,7 +49,6 @@ nix/nixpkgs.nix nix/packages nix/overlays - nix/github-actions.nix ]; }); } diff --git a/nix/github-actions.nix b/nix/github-actions.nix deleted file mode 100644 index ad43f2a6f..000000000 --- a/nix/github-actions.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ inputs, ... }: -let - githubPlatforms = { - "aarch64-linux" = "aarch64-linux"; - "aarch64-darwin" = "aarch64-darwin"; - "x86_64-linux" = "blacksmith-32vcpu-ubuntu-2404"; - }; -in -{ - flake.githubActions = inputs.nix-github-actions.lib.mkGithubMatrix { - checks = inputs.nixpkgs.lib.getAttrs [ - "x86_64-linux" - "aarch64-linux" - "aarch64-darwin" - ] inputs.self.checks; - platforms = githubPlatforms; - }; -} From 12a3bad8870c65eaab4145208710f6773af4df38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 11:23:47 +0200 Subject: [PATCH 09/71] feat(ci): split nix build workflow into separate extensions and checks jobs Split the monolithic nix-build job into two workflows: one for building PostgreSQL extensions and another for checks. Building extensions can be resource-intensive and time-consuming, so isolating them allows for better resource allocation and parallelism. Once they are built, the checks job can run tests and validations on the already built extensions. --- .github/workflows/nix-build.yml | 77 ++++++++++++++++++++++++++++----- nix/packages/postgres.nix | 6 +-- scripts/github-matrix.py | 56 +++++++++++++++++++----- 3 files changed, 114 insertions(+), 25 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 0c953b98f..a1a72dd92 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -15,7 +15,7 @@ permissions: packages: write jobs: - nix-matrix: + extensions-matrix: runs-on: group: self-hosted-runners-nix labels: @@ -29,15 +29,74 @@ jobs: name: Generate Nix Matrix run: | set -Eeu - echo matrix="$(python scripts/github-matrix.py)" >> "$GITHUB_OUTPUT" + echo matrix="$(python scripts/github-matrix.py extensions)" >> "$GITHUB_OUTPUT" - build-run-image: + build-extensions: + name: ${{matrix.postgresql_version}}.${{ matrix.name }} (${{ matrix.system }}) + needs: extensions-matrix + runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + strategy: + fail-fast: false + max-parallel: 3 + matrix: ${{fromJSON(needs.extensions-matrix.outputs.matrix)}} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: aws-oidc + uses: aws-actions/configure-aws-credentials@v4.3.1 + with: + aws-region: us-east-2 + role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role # Shared Services + role-session-name: gha-oidc-${{ github.run_id }} + - name: aws-creds + uses: aws-actions/configure-aws-credentials@v4.3.1 + with: + disable-retry: true + aws-region: us-east-2 + role-to-assume: arn:aws:iam::436098097459:role/nix-artifacts-deploy-role # supabase-dev + role-session-name: gha-oidc-${{ github.run_id }} + role-chaining: true + role-skip-session-tagging: true + role-duration-seconds: 3600 + - name: Write creds files + run: | + umask 006 + cat > /etc/nix/aws/nix-aws-credentials <> "$GITHUB_OUTPUT" + + + build-checks: name: ${{ matrix.name }} (${{ matrix.system }}) - needs: nix-matrix + needs: [checks-matrix, build-extensions] runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} strategy: fail-fast: false - matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} + matrix: ${{fromJSON(needs.checks-matrix.outputs.matrix)}} steps: - name: Checkout Repo uses: actions/checkout@v4 @@ -56,7 +115,7 @@ jobs: role-session-name: gha-oidc-${{ github.run_id }} role-chaining: true role-skip-session-tagging: true - role-duration-seconds: 900 # TODO: switch to 18000 (5 hours) + role-duration-seconds: 3600 - name: Write creds files run: | umask 006 @@ -71,13 +130,9 @@ jobs: nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - name: nix build run: | - if ${{ matrix.already_cached }}; then - echo "${{ matrix.attr }} already cached, skipping build" - exit 0 - fi nix build -L .#${{ matrix.attr }} run-tests: - needs: build-run-image + needs: build-checks if: ${{ success() }} uses: ./.github/workflows/test.yml diff --git a/nix/packages/postgres.nix b/nix/packages/postgres.nix index 8472e05fc..6d4c254df 100644 --- a/nix/packages/postgres.nix +++ b/nix/packages/postgres.nix @@ -1,7 +1,7 @@ { inputs, ... }: { perSystem = - { pkgs, ... }: + { pkgs, lib, ... }: let # Custom extensions that exist in our repository. These aren't upstream # either because nobody has done the work, maintaining them here is @@ -154,10 +154,9 @@ # install. # - exts: an attrset containing all the extensions, mapped to their # package names. - makePostgres = version: { + makePostgres = version: lib.recurseIntoAttrs { bin = makePostgresBin version; exts = makeOurPostgresPkgsSet version; - recurseForDerivations = true; }; basePackages = { psql_15 = makePostgres "15"; @@ -167,5 +166,6 @@ in { packages = inputs.flake-utils.lib.flattenTree basePackages; + legacyPackages = basePackages; }; } diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 53deb4f96..7adef8cfa 100644 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import argparse import json import os import subprocess @@ -47,6 +48,7 @@ class GitHubActionPackage(TypedDict): system: str already_cached: bool runs_on: RunsOnConfig + postgresql_version: NotRequired[str] BUILD_RUNNER_MAP: Dict[str, RunsOnConfig] = { @@ -76,22 +78,23 @@ def get_worker_count() -> int: return 1 -def build_nix_eval_command(max_workers: int) -> List[str]: +def build_nix_eval_command(max_workers: int, target: str) -> List[str]: """Build the nix-eval-jobs command with appropriate flags.""" - return [ + nix_eval_cmd = [ "nix-eval-jobs", "--flake", - ".#checks", + f".#{target}", "--check-cache-status", "--force-recurse", "--quiet", "--workers", str(max_workers), ] + return nix_eval_cmd def parse_nix_eval_line( - line: str, drv_paths: Set[str] + line: str, drv_paths: Set[str], target: str ) -> Optional[GitHubActionPackage]: """Parse a single line of nix-eval-jobs output""" if not line.strip(): @@ -102,12 +105,11 @@ def parse_nix_eval_line( if data["drvPath"] in drv_paths: return None drv_paths.add(data["drvPath"]) - print(f"Processing package: {data}", file=sys.stderr) runs_on_config = BUILD_RUNNER_MAP[data["system"]] return { - "attr": "checks." + data["attr"], + "attr": f"{target}.{data['attr']}", "name": data["name"], "system": data["system"], "already_cached": data.get("cacheStatus") != "notBuilt", @@ -118,7 +120,9 @@ def parse_nix_eval_line( return None -def run_nix_eval_jobs(cmd: List[str]) -> Generator[GitHubActionPackage, None, None]: +def run_nix_eval_jobs( + cmd: List[str], target: str +) -> Generator[GitHubActionPackage, None, None]: """Run nix-eval-jobs and yield parsed package data.""" print(f"Running command: {' '.join(cmd)}", file=sys.stderr) @@ -128,8 +132,8 @@ def run_nix_eval_jobs(cmd: List[str]) -> Generator[GitHubActionPackage, None, No drv_paths = set() for line in process.stdout: - package = parse_nix_eval_line(line, drv_paths) - if package: + package = parse_nix_eval_line(line, drv_paths, target) + if package and not package["already_cached"]: yield package if process.returncode and process.returncode != 0: @@ -138,11 +142,41 @@ def run_nix_eval_jobs(cmd: List[str]) -> Generator[GitHubActionPackage, None, No sys.exit(process.returncode) +def is_extension_pkg(pkg: GitHubActionPackage) -> bool: + """Check if the package is a postgresql extension package.""" + attrs = pkg["attr"].split(".") + return attrs[-2] == "exts" + + def main() -> None: + parser = argparse.ArgumentParser( + description="Generate GitHub Actions matrix for Nix builds" + ) + parser.add_argument( + "target", choices=["checks", "extensions"], help="Type of matrix to generate" + ) + + args = parser.parse_args() + max_workers = get_worker_count() - cmd = build_nix_eval_command(max_workers) - gh_action_packages = list(run_nix_eval_jobs(cmd)) + if args.target == "checks": + flake_output = "checks" + else: + flake_output = "legacyPackages" + + cmd = build_nix_eval_command(max_workers, flake_output) + + gh_action_packages = list(run_nix_eval_jobs(cmd, flake_output)) + + if args.target == "extensions": + # filter to only include extension packages and add postgresql_version field + gh_action_packages = [ + {**pkg, "postgresql_version": pkg["attr"].split(".")[-3]} + for pkg in gh_action_packages + if is_extension_pkg(pkg) + ] + gh_output = {"include": gh_action_packages} print(json.dumps(gh_output)) From cb09c1b13135d06ca8fe1b3d3e0c831b77d94b39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 12:54:58 +0200 Subject: [PATCH 10/71] feat(ci): extract nix build setup into reusable action and split builds by architecture Extract AWS credential setup and nix build steps into a composite action to reduce duplication. Split extension builds into separate jobs per architecture (aarch64-linux, aarch64-darwin, x86_64-linux) and update matrix generation to group packages by system. --- .github/actions/nix-build-setup/action.yml | 46 +++++++++ .github/workflows/nix-build.yml | 108 +++++++++------------ scripts/github-matrix.py | 17 +++- 3 files changed, 109 insertions(+), 62 deletions(-) create mode 100644 .github/actions/nix-build-setup/action.yml diff --git a/.github/actions/nix-build-setup/action.yml b/.github/actions/nix-build-setup/action.yml new file mode 100644 index 000000000..f6a8c450a --- /dev/null +++ b/.github/actions/nix-build-setup/action.yml @@ -0,0 +1,46 @@ +name: 'Nix Build Setup' +description: 'Sets up AWS credentials and builds a Nix package' +inputs: + attr: + description: 'The Nix attribute to build' + required: true + aws-role-duration: + description: 'AWS role session duration in seconds' + required: false + default: '3600' + +runs: + using: 'composite' + steps: + - name: aws-oidc + uses: aws-actions/configure-aws-credentials@v4.3.1 + with: + aws-region: us-east-2 + role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role # Shared Services + role-session-name: gha-oidc-${{ github.run_id }} + - name: aws-creds + uses: aws-actions/configure-aws-credentials@v4.3.1 + with: + disable-retry: true + aws-region: us-east-2 + role-to-assume: arn:aws:iam::436098097459:role/nix-artifacts-deploy-role # supabase-dev + role-session-name: gha-oidc-${{ github.run_id }} + role-chaining: true + role-skip-session-tagging: true + role-duration-seconds: ${{ inputs.aws-role-duration }} + - name: Write creds files + shell: bash + run: | + umask 006 + cat > /etc/nix/aws/nix-aws-credentials < + nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" + - name: nix build + shell: bash + run: nix build -L .#${{ inputs.attr }} diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index a1a72dd92..6a4ca7aad 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -30,50 +30,63 @@ jobs: run: | set -Eeu echo matrix="$(python scripts/github-matrix.py extensions)" >> "$GITHUB_OUTPUT" + # XXX debugging + exit 1 - build-extensions: - name: ${{matrix.postgresql_version}}.${{ matrix.name }} (${{ matrix.system }}) + build-extensions-aarch64-linux: + name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-linux) needs: extensions-matrix runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_linux != null }} strategy: fail-fast: false max-parallel: 3 - matrix: ${{fromJSON(needs.extensions-matrix.outputs.matrix)}} + matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_linux }} steps: - name: Checkout Repo uses: actions/checkout@v4 - - name: aws-oidc - uses: aws-actions/configure-aws-credentials@v4.3.1 + - name: Build Nix Package + uses: ./.github/actions/nix-build-setup with: - aws-region: us-east-2 - role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role # Shared Services - role-session-name: gha-oidc-${{ github.run_id }} - - name: aws-creds - uses: aws-actions/configure-aws-credentials@v4.3.1 + attr: ${{ matrix.attr }} + + build-extensions-aarch64-darwin: + name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) + needs: extensions-matrix + runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_darwin != null }} + strategy: + fail-fast: false + max-parallel: 3 + matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_darwin }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Build Nix Package + uses: ./.github/actions/nix-build-setup with: - disable-retry: true - aws-region: us-east-2 - role-to-assume: arn:aws:iam::436098097459:role/nix-artifacts-deploy-role # supabase-dev - role-session-name: gha-oidc-${{ github.run_id }} - role-chaining: true - role-skip-session-tagging: true - role-duration-seconds: 3600 - - name: Write creds files - run: | - umask 006 - cat > /etc/nix/aws/nix-aws-credentials < /etc/nix/aws/nix-aws-credentials < - nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - - name: nix build - run: | - nix build -L .#${{ matrix.attr }} + attr: ${{ matrix.attr }} run-tests: needs: build-checks diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 7adef8cfa..562c6b9ec 100644 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -134,6 +134,7 @@ def run_nix_eval_jobs( for line in process.stdout: package = parse_nix_eval_line(line, drv_paths, target) if package and not package["already_cached"]: + print(f"Found package: {package['attr']}", file=sys.stderr) yield package if process.returncode and process.returncode != 0: @@ -177,7 +178,21 @@ def main() -> None: if is_extension_pkg(pkg) ] - gh_output = {"include": gh_action_packages} + # Group packages by system + grouped_by_system = {} + for pkg in gh_action_packages: + system = pkg["system"] + if system not in grouped_by_system: + grouped_by_system[system] = [] + grouped_by_system[system].append(pkg) + + # Create output with system-specific matrices + gh_output = {} + for system, packages in grouped_by_system.items(): + gh_output[system.replace("-", "_")] = {"include": packages} + else: + gh_output = {"include": gh_action_packages} + print(json.dumps(gh_output)) From c8edd1d267f1887bf8d3fecf4bc4d646aaf1e1b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 15:30:23 +0200 Subject: [PATCH 11/71] fix(ci): do not hide cached builds --- .github/actions/nix-build-setup/action.yml | 12 ++++++++++++ .github/workflows/nix-build.yml | 6 ++++-- nix/packages/postgres.nix | 10 ++++++---- scripts/github-matrix.py | 18 ++++++++++++------ 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/.github/actions/nix-build-setup/action.yml b/.github/actions/nix-build-setup/action.yml index f6a8c450a..308bf492c 100644 --- a/.github/actions/nix-build-setup/action.yml +++ b/.github/actions/nix-build-setup/action.yml @@ -8,17 +8,23 @@ inputs: description: 'AWS role session duration in seconds' required: false default: '3600' + cached: + description: 'Whether to use cached builds' + required: false + default: 'false' runs: using: 'composite' steps: - name: aws-oidc + if: ${{ inputs.cached == 'false' }} uses: aws-actions/configure-aws-credentials@v4.3.1 with: aws-region: us-east-2 role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role # Shared Services role-session-name: gha-oidc-${{ github.run_id }} - name: aws-creds + if: ${{ inputs.cached == 'false' }} uses: aws-actions/configure-aws-credentials@v4.3.1 with: disable-retry: true @@ -29,6 +35,7 @@ runs: role-skip-session-tagging: true role-duration-seconds: ${{ inputs.aws-role-duration }} - name: Write creds files + if: ${{ inputs.cached == 'false' }} shell: bash run: | umask 006 @@ -42,5 +49,10 @@ runs: run: > nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - name: nix build + if: ${{ inputs.cached == 'false' }} shell: bash run: nix build -L .#${{ inputs.attr }} + - name: nix build + if: ${{ inputs.cached == 'true' }} + shell: bash + run: echo "Skipping build of ${{ inputs.attr }} as it's marked cached" diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 6a4ca7aad..73628319d 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -30,8 +30,6 @@ jobs: run: | set -Eeu echo matrix="$(python scripts/github-matrix.py extensions)" >> "$GITHUB_OUTPUT" - # XXX debugging - exit 1 build-extensions-aarch64-linux: name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-linux) @@ -49,6 +47,7 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} + cached: ${{ matrix.is_cached }} build-extensions-aarch64-darwin: name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) @@ -66,6 +65,7 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} + cached: ${{ matrix.is_cached }} build-extensions-x86_64-linux: name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) @@ -83,6 +83,7 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} + cached: ${{ matrix.is_cached }} checks-matrix: @@ -117,6 +118,7 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} + cached: ${{ matrix.is_cached }} run-tests: needs: build-checks diff --git a/nix/packages/postgres.nix b/nix/packages/postgres.nix index 6d4c254df..f98bd3607 100644 --- a/nix/packages/postgres.nix +++ b/nix/packages/postgres.nix @@ -154,10 +154,12 @@ # install. # - exts: an attrset containing all the extensions, mapped to their # package names. - makePostgres = version: lib.recurseIntoAttrs { - bin = makePostgresBin version; - exts = makeOurPostgresPkgsSet version; - }; + makePostgres = + version: + lib.recurseIntoAttrs { + bin = makePostgresBin version; + exts = makeOurPostgresPkgsSet version; + }; basePackages = { psql_15 = makePostgres "15"; psql_17 = makePostgres "17"; diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 562c6b9ec..6c601fabd 100644 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -46,7 +46,7 @@ class GitHubActionPackage(TypedDict): attr: str name: str system: str - already_cached: bool + is_cached: bool runs_on: RunsOnConfig postgresql_version: NotRequired[str] @@ -103,8 +103,10 @@ def parse_nix_eval_line( try: data: NixEvalJobsOutput = json.loads(line) if data["drvPath"] in drv_paths: - return None - drv_paths.add(data["drvPath"]) + print(f"Skipping duplicate drvPath: {data['drvPath']}", file=sys.stderr) + data["cacheStatus"] = "cached" + else: + drv_paths.add(data["drvPath"]) runs_on_config = BUILD_RUNNER_MAP[data["system"]] @@ -112,8 +114,10 @@ def parse_nix_eval_line( "attr": f"{target}.{data['attr']}", "name": data["name"], "system": data["system"], - "already_cached": data.get("cacheStatus") != "notBuilt", + "is_cached": data.get("cacheStatus") != "notBuilt", "runs_on": runs_on_config, + "drvPath": data["drvPath"], # For debugging purposes + "outputs": data.get("outputs", {}), # For debugging purposes } except json.JSONDecodeError: print(f"Skipping invalid JSON line: {line}", file=sys.stderr) @@ -133,8 +137,7 @@ def run_nix_eval_jobs( for line in process.stdout: package = parse_nix_eval_line(line, drv_paths, target) - if package and not package["already_cached"]: - print(f"Found package: {package['attr']}", file=sys.stderr) + if package: yield package if process.returncode and process.returncode != 0: @@ -186,6 +189,9 @@ def main() -> None: grouped_by_system[system] = [] grouped_by_system[system].append(pkg) + print("Grouped packages by system:", file=sys.stderr) + print(json.dumps(grouped_by_system, indent=2), file=sys.stderr) + # Create output with system-specific matrices gh_output = {} for system, packages in grouped_by_system.items(): From d1217f225610a639992a4dea4f09f72933953c2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 16:19:24 +0200 Subject: [PATCH 12/71] Revert "fix(ci): do not hide cached builds" This reverts commit 9d41b1e45fd92df646baaf078b9889cf308ac32b. --- .github/actions/nix-build-setup/action.yml | 12 ------------ .github/workflows/nix-build.yml | 6 ++---- nix/packages/postgres.nix | 10 ++++------ scripts/github-matrix.py | 18 ++++++------------ 4 files changed, 12 insertions(+), 34 deletions(-) diff --git a/.github/actions/nix-build-setup/action.yml b/.github/actions/nix-build-setup/action.yml index 308bf492c..f6a8c450a 100644 --- a/.github/actions/nix-build-setup/action.yml +++ b/.github/actions/nix-build-setup/action.yml @@ -8,23 +8,17 @@ inputs: description: 'AWS role session duration in seconds' required: false default: '3600' - cached: - description: 'Whether to use cached builds' - required: false - default: 'false' runs: using: 'composite' steps: - name: aws-oidc - if: ${{ inputs.cached == 'false' }} uses: aws-actions/configure-aws-credentials@v4.3.1 with: aws-region: us-east-2 role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role # Shared Services role-session-name: gha-oidc-${{ github.run_id }} - name: aws-creds - if: ${{ inputs.cached == 'false' }} uses: aws-actions/configure-aws-credentials@v4.3.1 with: disable-retry: true @@ -35,7 +29,6 @@ runs: role-skip-session-tagging: true role-duration-seconds: ${{ inputs.aws-role-duration }} - name: Write creds files - if: ${{ inputs.cached == 'false' }} shell: bash run: | umask 006 @@ -49,10 +42,5 @@ runs: run: > nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - name: nix build - if: ${{ inputs.cached == 'false' }} shell: bash run: nix build -L .#${{ inputs.attr }} - - name: nix build - if: ${{ inputs.cached == 'true' }} - shell: bash - run: echo "Skipping build of ${{ inputs.attr }} as it's marked cached" diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 73628319d..6a4ca7aad 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -30,6 +30,8 @@ jobs: run: | set -Eeu echo matrix="$(python scripts/github-matrix.py extensions)" >> "$GITHUB_OUTPUT" + # XXX debugging + exit 1 build-extensions-aarch64-linux: name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-linux) @@ -47,7 +49,6 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} - cached: ${{ matrix.is_cached }} build-extensions-aarch64-darwin: name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) @@ -65,7 +66,6 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} - cached: ${{ matrix.is_cached }} build-extensions-x86_64-linux: name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) @@ -83,7 +83,6 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} - cached: ${{ matrix.is_cached }} checks-matrix: @@ -118,7 +117,6 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} - cached: ${{ matrix.is_cached }} run-tests: needs: build-checks diff --git a/nix/packages/postgres.nix b/nix/packages/postgres.nix index f98bd3607..6d4c254df 100644 --- a/nix/packages/postgres.nix +++ b/nix/packages/postgres.nix @@ -154,12 +154,10 @@ # install. # - exts: an attrset containing all the extensions, mapped to their # package names. - makePostgres = - version: - lib.recurseIntoAttrs { - bin = makePostgresBin version; - exts = makeOurPostgresPkgsSet version; - }; + makePostgres = version: lib.recurseIntoAttrs { + bin = makePostgresBin version; + exts = makeOurPostgresPkgsSet version; + }; basePackages = { psql_15 = makePostgres "15"; psql_17 = makePostgres "17"; diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 6c601fabd..562c6b9ec 100644 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -46,7 +46,7 @@ class GitHubActionPackage(TypedDict): attr: str name: str system: str - is_cached: bool + already_cached: bool runs_on: RunsOnConfig postgresql_version: NotRequired[str] @@ -103,10 +103,8 @@ def parse_nix_eval_line( try: data: NixEvalJobsOutput = json.loads(line) if data["drvPath"] in drv_paths: - print(f"Skipping duplicate drvPath: {data['drvPath']}", file=sys.stderr) - data["cacheStatus"] = "cached" - else: - drv_paths.add(data["drvPath"]) + return None + drv_paths.add(data["drvPath"]) runs_on_config = BUILD_RUNNER_MAP[data["system"]] @@ -114,10 +112,8 @@ def parse_nix_eval_line( "attr": f"{target}.{data['attr']}", "name": data["name"], "system": data["system"], - "is_cached": data.get("cacheStatus") != "notBuilt", + "already_cached": data.get("cacheStatus") != "notBuilt", "runs_on": runs_on_config, - "drvPath": data["drvPath"], # For debugging purposes - "outputs": data.get("outputs", {}), # For debugging purposes } except json.JSONDecodeError: print(f"Skipping invalid JSON line: {line}", file=sys.stderr) @@ -137,7 +133,8 @@ def run_nix_eval_jobs( for line in process.stdout: package = parse_nix_eval_line(line, drv_paths, target) - if package: + if package and not package["already_cached"]: + print(f"Found package: {package['attr']}", file=sys.stderr) yield package if process.returncode and process.returncode != 0: @@ -189,9 +186,6 @@ def main() -> None: grouped_by_system[system] = [] grouped_by_system[system].append(pkg) - print("Grouped packages by system:", file=sys.stderr) - print(json.dumps(grouped_by_system, indent=2), file=sys.stderr) - # Create output with system-specific matrices gh_output = {} for system, packages in grouped_by_system.items(): From 3ba1801874a6a6b928aa8e73b978a430db0aa4dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 16:38:29 +0200 Subject: [PATCH 13/71] chore: Temporarily disable x86_64-linux builds We don't have a self hosted runner for x86_64-linux at the moment --- .github/workflows/nix-build.yml | 35 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 6a4ca7aad..3157b3a1b 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -67,26 +67,27 @@ jobs: with: attr: ${{ matrix.attr }} - build-extensions-x86_64-linux: - name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) - needs: extensions-matrix - runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).x86_64_linux != null }} - strategy: - fail-fast: false - max-parallel: 3 - matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).x86_64_linux }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Build Nix Package - uses: ./.github/actions/nix-build-setup - with: - attr: ${{ matrix.attr }} + # TODO + # build-extensions-x86_64-linux: + # name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) + # needs: extensions-matrix + # runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + # if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).x86_64_linux != null }} + # strategy: + # fail-fast: false + # max-parallel: 3 + # matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).x86_64_linux }} + # steps: + # - name: Checkout Repo + # uses: actions/checkout@v4 + # - name: Build Nix Package + # uses: ./.github/actions/nix-build-setup + # with: + # attr: ${{ matrix.attr }} checks-matrix: - needs: [build-extensions-aarch64-linux, build-extensions-aarch64-darwin, build-extensions-x86_64-linux] + needs: [build-extensions-aarch64-linux, build-extensions-aarch64-darwin] #, build-extensions-x86_64-linux] runs-on: group: self-hosted-runners-nix labels: From 155165fd2c1b529bb8abe4c0b10c5e0236d54732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 16:38:29 +0200 Subject: [PATCH 14/71] fix: sort packages and filter out cached ones --- .github/workflows/nix-build.yml | 2 -- nix/packages/postgres.nix | 10 ++++--- scripts/github-matrix.py | 52 +++++++++++++++++++++++++++------ 3 files changed, 49 insertions(+), 15 deletions(-) mode change 100644 => 100755 scripts/github-matrix.py diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 3157b3a1b..884196206 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -30,8 +30,6 @@ jobs: run: | set -Eeu echo matrix="$(python scripts/github-matrix.py extensions)" >> "$GITHUB_OUTPUT" - # XXX debugging - exit 1 build-extensions-aarch64-linux: name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-linux) diff --git a/nix/packages/postgres.nix b/nix/packages/postgres.nix index 6d4c254df..f98bd3607 100644 --- a/nix/packages/postgres.nix +++ b/nix/packages/postgres.nix @@ -154,10 +154,12 @@ # install. # - exts: an attrset containing all the extensions, mapped to their # package names. - makePostgres = version: lib.recurseIntoAttrs { - bin = makePostgresBin version; - exts = makeOurPostgresPkgsSet version; - }; + makePostgres = + version: + lib.recurseIntoAttrs { + bin = makePostgresBin version; + exts = makeOurPostgresPkgsSet version; + }; basePackages = { psql_15 = makePostgres "15"; psql_17 = makePostgres "17"; diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py old mode 100644 new mode 100755 index 562c6b9ec..607755da9 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import argparse +from collections import defaultdict +import graphlib import json import os import subprocess @@ -48,6 +50,9 @@ class GitHubActionPackage(TypedDict): system: str already_cached: bool runs_on: RunsOnConfig + drvPath: str + neededSubstitutes: List[str] + neededBuilds: List[str] postgresql_version: NotRequired[str] @@ -114,6 +119,9 @@ def parse_nix_eval_line( "system": data["system"], "already_cached": data.get("cacheStatus") != "notBuilt", "runs_on": runs_on_config, + "drvPath": data["drvPath"], + "neededSubstitutes": data.get("neededSubstitutes", []), + "neededBuilds": data.get("neededBuilds", []), } except json.JSONDecodeError: print(f"Skipping invalid JSON line: {line}", file=sys.stderr) @@ -133,8 +141,7 @@ def run_nix_eval_jobs( for line in process.stdout: package = parse_nix_eval_line(line, drv_paths, target) - if package and not package["already_cached"]: - print(f"Found package: {package['attr']}", file=sys.stderr) + if package: yield package if process.returncode and process.returncode != 0: @@ -149,6 +156,34 @@ def is_extension_pkg(pkg: GitHubActionPackage) -> bool: return attrs[-2] == "exts" +# thank you buildbot-nix https://github.com/nix-community/buildbot-nix/blob/985d069a2a45cf4a571a4346107671adc2bd2a16/buildbot_nix/buildbot_nix/build_trigger.py#L297 +def sort_pkgs_by_closures(jobs: list[GitHubActionPackage]) -> list[GitHubActionPackage]: + sorted_jobs = [] + + # Prepare job dependencies + job_set = {job["drvPath"] for job in jobs} + job_closures = { + k["drvPath"]: set(k["neededSubstitutes"]) + .union(set(k["neededBuilds"])) + .intersection(job_set) + .difference({k["drvPath"]}) + for k in jobs + } + + sorter = graphlib.TopologicalSorter(job_closures) + + for item in sorter.static_order(): + i = 0 + while i < len(jobs): + if item == jobs[i]["drvPath"]: + sorted_jobs.append(jobs[i]) + del jobs[i] + else: + i += 1 + + return sorted_jobs + + def main() -> None: parser = argparse.ArgumentParser( description="Generate GitHub Actions matrix for Nix builds" @@ -168,23 +203,22 @@ def main() -> None: cmd = build_nix_eval_command(max_workers, flake_output) - gh_action_packages = list(run_nix_eval_jobs(cmd, flake_output)) + gh_action_packages = sort_pkgs_by_closures( + list(run_nix_eval_jobs(cmd, flake_output)) + ) if args.target == "extensions": # filter to only include extension packages and add postgresql_version field gh_action_packages = [ {**pkg, "postgresql_version": pkg["attr"].split(".")[-3]} for pkg in gh_action_packages - if is_extension_pkg(pkg) + if is_extension_pkg(pkg) and not pkg["already_cached"] ] # Group packages by system - grouped_by_system = {} + grouped_by_system = defaultdict(list) for pkg in gh_action_packages: - system = pkg["system"] - if system not in grouped_by_system: - grouped_by_system[system] = [] - grouped_by_system[system].append(pkg) + grouped_by_system[pkg["system"]].append(pkg) # Create output with system-specific matrices gh_output = {} From d2054e0052b8672ff4395a5c160fcbbd5f2c81a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 17:37:09 +0200 Subject: [PATCH 15/71] fix: do not skip checks-matrix if dependencies are skipped --- .github/workflows/nix-build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 884196206..c1bdf9334 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -86,6 +86,10 @@ jobs: checks-matrix: needs: [build-extensions-aarch64-linux, build-extensions-aarch64-darwin] #, build-extensions-x86_64-linux] + if: | + always() && + (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && + (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') runs-on: group: self-hosted-runners-nix labels: From 1bc9b174a1e13ed174a3a75da74b056f3a3776ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 17:47:30 +0200 Subject: [PATCH 16/71] fix: do not return debug fields in GitHub Actions matrix output --- scripts/github-matrix.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 607755da9..b0b3d2a0f 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -207,6 +207,16 @@ def main() -> None: list(run_nix_eval_jobs(cmd, flake_output)) ) + def clean_package_for_output(pkg: GitHubActionPackage) -> dict: + """Remove debug fields from package for final output""" + return { + "attr": pkg["attr"], + "name": pkg["name"], + "system": pkg["system"], + "runs_on": pkg["runs_on"], + **({ "postgresql_version": pkg["postgresql_version"]} if "postgresql_version" in pkg else {}) + } + if args.target == "extensions": # filter to only include extension packages and add postgresql_version field gh_action_packages = [ @@ -218,14 +228,15 @@ def main() -> None: # Group packages by system grouped_by_system = defaultdict(list) for pkg in gh_action_packages: - grouped_by_system[pkg["system"]].append(pkg) + grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) # Create output with system-specific matrices gh_output = {} for system, packages in grouped_by_system.items(): gh_output[system.replace("-", "_")] = {"include": packages} else: - gh_output = {"include": gh_action_packages} + cleaned_packages = [clean_package_for_output(pkg) for pkg in gh_action_packages] + gh_output = {"include": cleaned_packages} print(json.dumps(gh_output)) From 6f86ed457caf17e44d7eb5b3d624cdba31c57eb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 18:24:50 +0200 Subject: [PATCH 17/71] debugging --- .github/workflows/nix-build.yml | 4 ++-- scripts/github-matrix.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index c1bdf9334..1c7d0993c 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -38,7 +38,7 @@ jobs: if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_linux != null }} strategy: fail-fast: false - max-parallel: 3 + max-parallel: 5 matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_linux }} steps: - name: Checkout Repo @@ -55,7 +55,7 @@ jobs: if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_darwin != null }} strategy: fail-fast: false - max-parallel: 3 + max-parallel: 5 matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_darwin }} steps: - name: Checkout Repo diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index b0b3d2a0f..0829afab9 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -238,6 +238,7 @@ def clean_package_for_output(pkg: GitHubActionPackage) -> dict: cleaned_packages = [clean_package_for_output(pkg) for pkg in gh_action_packages] gh_output = {"include": cleaned_packages} + print(f"debug: Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}", file=sys.stderr) print(json.dumps(gh_output)) From b8124bf2a8a3f1421a68a3ffce2fe27b88f88b53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 2 Oct 2025 17:00:31 +0200 Subject: [PATCH 18/71] fix(ci): use !cancelled() instead of always() for dependent job conditions Replace always() with !cancelled() to prevent jobs from running when the workflow is cancelled. Add explicit success/skip conditions for dependent jobs to ensure proper workflow execution flow. --- .github/workflows/nix-build.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 1c7d0993c..780dad7c4 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -87,7 +87,7 @@ jobs: checks-matrix: needs: [build-extensions-aarch64-linux, build-extensions-aarch64-darwin] #, build-extensions-x86_64-linux] if: | - always() && + !cancelled() && (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') runs-on: @@ -110,6 +110,9 @@ jobs: name: ${{ matrix.name }} (${{ matrix.system }}) needs: [checks-matrix] runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: | + !cancelled() && + (needs.checks-matrix.result == 'skipped' || needs.checks-matrix.result == 'success') strategy: fail-fast: false matrix: ${{fromJSON(needs.checks-matrix.outputs.matrix)}} @@ -123,5 +126,7 @@ jobs: run-tests: needs: build-checks - if: ${{ success() }} + if: | + !cancelled() && + (needs.build-checks.result == 'skipped' || needs.build-checks.result == 'success') uses: ./.github/workflows/test.yml From 479ab0f8dbb5f2ec0f3c0d2c0723450ce13b10a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 2 Oct 2025 17:20:26 +0200 Subject: [PATCH 19/71] fix(ci): stop chaining aws roles We couldn't set duration to 18000 seconds when chaining roles. --- .github/actions/nix-build-setup/action.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/actions/nix-build-setup/action.yml b/.github/actions/nix-build-setup/action.yml index f6a8c450a..afa06499f 100644 --- a/.github/actions/nix-build-setup/action.yml +++ b/.github/actions/nix-build-setup/action.yml @@ -12,12 +12,6 @@ inputs: runs: using: 'composite' steps: - - name: aws-oidc - uses: aws-actions/configure-aws-credentials@v4.3.1 - with: - aws-region: us-east-2 - role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role # Shared Services - role-session-name: gha-oidc-${{ github.run_id }} - name: aws-creds uses: aws-actions/configure-aws-credentials@v4.3.1 with: @@ -25,9 +19,8 @@ runs: aws-region: us-east-2 role-to-assume: arn:aws:iam::436098097459:role/nix-artifacts-deploy-role # supabase-dev role-session-name: gha-oidc-${{ github.run_id }} - role-chaining: true - role-skip-session-tagging: true - role-duration-seconds: ${{ inputs.aws-role-duration }} + role-duration-seconds: 18000 + - name: Write creds files shell: bash run: | From 3467170f99c9335151c725d22189ab46c688b009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 2 Oct 2025 21:18:13 +0200 Subject: [PATCH 20/71] ci: run nixos test on aarch64-linux --- nix/checks.nix | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/nix/checks.nix b/nix/checks.nix index 0bc5261b7..7558b92ce 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -396,24 +396,22 @@ pg_regress ; } - // pkgs.lib.optionalAttrs (system == "aarch64-linux") { - inherit (self'.packages) - postgresql_15_debug - postgresql_15_src - postgresql_orioledb-17_debug - postgresql_orioledb-17_src - postgresql_17_debug - postgresql_17_src - ; - } - // pkgs.lib.optionalAttrs (system == "x86_64-linux") ( + // pkgs.lib.optionalAttrs (system == "aarch64-linux") ( { - devShell = self'.devShells.default; + inherit (self'.packages) + postgresql_15_debug + postgresql_15_src + postgresql_orioledb-17_debug + postgresql_orioledb-17_src + postgresql_17_debug + postgresql_17_src + ; } // (import ./ext/tests { inherit self; inherit pkgs; }) + // pkgs.lib.optionalAttrs (system == "x86_64-linux") { devShell = self'.devShells.default; } ); }; } From 5c6801e766f35d3feeb01a49ea7d9b3a18fc8057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 2 Oct 2025 21:18:13 +0200 Subject: [PATCH 21/71] fix(ci): disable eval-cache and accept-flake-config --- scripts/github-matrix.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 0829afab9..769ebb1c7 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -92,6 +92,12 @@ def build_nix_eval_command(max_workers: int, target: str) -> List[str]: "--check-cache-status", "--force-recurse", "--quiet", + "--option", + "eval-cache", + "false", + "--option", + "accept-flake-config", + "true", "--workers", str(max_workers), ] @@ -214,7 +220,11 @@ def clean_package_for_output(pkg: GitHubActionPackage) -> dict: "name": pkg["name"], "system": pkg["system"], "runs_on": pkg["runs_on"], - **({ "postgresql_version": pkg["postgresql_version"]} if "postgresql_version" in pkg else {}) + **( + {"postgresql_version": pkg["postgresql_version"]} + if "postgresql_version" in pkg + else {} + ), } if args.target == "extensions": @@ -238,7 +248,10 @@ def clean_package_for_output(pkg: GitHubActionPackage) -> dict: cleaned_packages = [clean_package_for_output(pkg) for pkg in gh_action_packages] gh_output = {"include": cleaned_packages} - print(f"debug: Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}", file=sys.stderr) + print( + f"debug: Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}", + file=sys.stderr, + ) print(json.dumps(gh_output)) From fc722b6b1e9538819cf22bc4abd2fe0ff478a1c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 2 Oct 2025 21:18:13 +0200 Subject: [PATCH 22/71] ci: split checks build jobs by system architecture Refactor GitHub Actions workflow to run build checks in parallel across different architectures (aarch64-linux, aarch64-darwin) with separate job matrices. --- .github/workflows/nix-build.yml | 58 ++++++++++++++++++++++++++++----- scripts/github-matrix.py | 20 +++++------- 2 files changed, 58 insertions(+), 20 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 780dad7c4..70309d28c 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -87,7 +87,7 @@ jobs: checks-matrix: needs: [build-extensions-aarch64-linux, build-extensions-aarch64-darwin] #, build-extensions-x86_64-linux] if: | - !cancelled() && + !cancelled() && (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') runs-on: @@ -105,17 +105,40 @@ jobs: set -Eeu echo matrix="$(python scripts/github-matrix.py checks)" >> "$GITHUB_OUTPUT" + build-checks-aarch64-linux: + name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) + needs: checks-matrix + runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: | + !cancelled() && + ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_linux != null }} && + (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && + (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') + strategy: + fail-fast: false + max-parallel: 5 + matrix: ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_linux }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Build Nix Package + uses: ./.github/actions/nix-build-setup + with: + attr: ${{ matrix.attr }} - build-checks: - name: ${{ matrix.name }} (${{ matrix.system }}) - needs: [checks-matrix] + build-checks-aarch64-darwin: + name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) + needs: checks-matrix runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: | !cancelled() && - (needs.checks-matrix.result == 'skipped' || needs.checks-matrix.result == 'success') + ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_darwin != null }} && + (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && + (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') strategy: fail-fast: false - matrix: ${{fromJSON(needs.checks-matrix.outputs.matrix)}} + max-parallel: 5 + matrix: ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_darwin }} steps: - name: Checkout Repo uses: actions/checkout@v4 @@ -124,9 +147,28 @@ jobs: with: attr: ${{ matrix.attr }} + # TODO + # build-checks-x86_64-linux: + # name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) + # needs: checks-matrix + # runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + # if: ${{ fromJSON(needs.checks-matrix.outputs.matrix).x86_64_linux != null }} + # strategy: + # fail-fast: false + # max-parallel: 5 + # matrix: ${{ fromJSON(needs.checks-matrix.outputs.matrix).x86_64_linux }} + # steps: + # - name: Checkout Repo + # uses: actions/checkout@v4 + # - name: Build Nix Package + # uses: ./.github/actions/nix-build-setup + # with: + # attr: ${{ matrix.attr }} + run-tests: - needs: build-checks + needs: [build-checks-aarch64-linux, build-checks-aarch64-darwin] #, build-checks-x86_64-linux] if: | !cancelled() && - (needs.build-checks.result == 'skipped' || needs.build-checks.result == 'success') + (needs.build-checks-aarch64-linux.result == 'skipped' || needs.build-checks-aarch64-linux.result == 'success') && + (needs.build-checks-aarch64-darwin.result == 'skipped' || needs.build-checks-aarch64-darwin.result == 'success') uses: ./.github/workflows/test.yml diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 769ebb1c7..4e23bf7f4 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -144,7 +144,6 @@ def run_nix_eval_jobs( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) as process: drv_paths = set() - for line in process.stdout: package = parse_nix_eval_line(line, drv_paths, target) if package: @@ -235,18 +234,15 @@ def clean_package_for_output(pkg: GitHubActionPackage) -> dict: if is_extension_pkg(pkg) and not pkg["already_cached"] ] - # Group packages by system - grouped_by_system = defaultdict(list) - for pkg in gh_action_packages: - grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) + # Group packages by system + grouped_by_system = defaultdict(list) + for pkg in gh_action_packages: + grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) - # Create output with system-specific matrices - gh_output = {} - for system, packages in grouped_by_system.items(): - gh_output[system.replace("-", "_")] = {"include": packages} - else: - cleaned_packages = [clean_package_for_output(pkg) for pkg in gh_action_packages] - gh_output = {"include": cleaned_packages} + # Create output with system-specific matrices + gh_output = {} + for system, packages in grouped_by_system.items(): + gh_output[system.replace("-", "_")] = {"include": packages} print( f"debug: Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}", From 24981dfd426eef132f343cc7b44ebdc616727776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 3 Oct 2025 11:47:49 +0200 Subject: [PATCH 23/71] fix(ci): use correct architecture name in aarch64-linux builds --- .github/workflows/nix-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 70309d28c..9b1775bb7 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -106,7 +106,7 @@ jobs: echo matrix="$(python scripts/github-matrix.py checks)" >> "$GITHUB_OUTPUT" build-checks-aarch64-linux: - name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) + name: ${{ matrix.name }} (aarch64-linux) needs: checks-matrix runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: | @@ -127,7 +127,7 @@ jobs: attr: ${{ matrix.attr }} build-checks-aarch64-darwin: - name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) + name: ${{ matrix.name }} (aarch64-darwin) needs: checks-matrix runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: | From 152c07adb5aff08ceca48dfe699fc4033665e252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 3 Oct 2025 11:47:49 +0200 Subject: [PATCH 24/71] fix(ci): do not try to build already cached checks --- scripts/github-matrix.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 4e23bf7f4..b3b003afc 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -231,13 +231,14 @@ def clean_package_for_output(pkg: GitHubActionPackage) -> dict: gh_action_packages = [ {**pkg, "postgresql_version": pkg["attr"].split(".")[-3]} for pkg in gh_action_packages - if is_extension_pkg(pkg) and not pkg["already_cached"] + if is_extension_pkg(pkg) ] # Group packages by system grouped_by_system = defaultdict(list) for pkg in gh_action_packages: - grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) + if not pkg["already_cached"]: + grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) # Create output with system-specific matrices gh_output = {} From cd829fb09a35b4b78d91a272325479d750c1e835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 3 Oct 2025 14:53:50 +0200 Subject: [PATCH 25/71] fix(ci): simplify GitHub Actions workflow for Nix builds Create a single nix-eval job to determine packages to build, removing redundant extension and check matrices. --- .github/actionlint.yaml | 4 ++ .github/workflows/nix-build.yml | 118 +++++--------------------------- scripts/github-matrix.py | 60 +++++----------- 3 files changed, 40 insertions(+), 142 deletions(-) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..0a2631f7d --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,4 @@ +self-hosted-runner: + labels: + - aarch64-darwin + - aarch64-linux diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 9b1775bb7..56e1235a6 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -15,7 +15,7 @@ permissions: packages: write jobs: - extensions-matrix: + nix-eval: runs-on: group: self-hosted-runners-nix labels: @@ -29,95 +29,17 @@ jobs: name: Generate Nix Matrix run: | set -Eeu - echo matrix="$(python scripts/github-matrix.py extensions)" >> "$GITHUB_OUTPUT" + echo matrix="$(python scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" - build-extensions-aarch64-linux: - name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-linux) - needs: extensions-matrix - runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_linux != null }} - strategy: - fail-fast: false - max-parallel: 5 - matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_linux }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Build Nix Package - uses: ./.github/actions/nix-build-setup - with: - attr: ${{ matrix.attr }} - - build-extensions-aarch64-darwin: - name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) - needs: extensions-matrix - runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_darwin != null }} - strategy: - fail-fast: false - max-parallel: 5 - matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_darwin }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Build Nix Package - uses: ./.github/actions/nix-build-setup - with: - attr: ${{ matrix.attr }} - - # TODO - # build-extensions-x86_64-linux: - # name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) - # needs: extensions-matrix - # runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - # if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).x86_64_linux != null }} - # strategy: - # fail-fast: false - # max-parallel: 3 - # matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).x86_64_linux }} - # steps: - # - name: Checkout Repo - # uses: actions/checkout@v4 - # - name: Build Nix Package - # uses: ./.github/actions/nix-build-setup - # with: - # attr: ${{ matrix.attr }} - - - checks-matrix: - needs: [build-extensions-aarch64-linux, build-extensions-aarch64-darwin] #, build-extensions-x86_64-linux] - if: | - !cancelled() && - (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && - (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') - runs-on: - group: self-hosted-runners-nix - labels: - - aarch64-darwin - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - id: set-matrix - name: Generate Nix Matrix - run: | - set -Eeu - echo matrix="$(python scripts/github-matrix.py checks)" >> "$GITHUB_OUTPUT" - - build-checks-aarch64-linux: + nix-build-aarch64-linux: name: ${{ matrix.name }} (aarch64-linux) - needs: checks-matrix + needs: nix-eval runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: | - !cancelled() && - ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_linux != null }} && - (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && - (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') + if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_linux != null }} strategy: fail-fast: false max-parallel: 5 - matrix: ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_linux }} + matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_linux }} steps: - name: Checkout Repo uses: actions/checkout@v4 @@ -126,19 +48,15 @@ jobs: with: attr: ${{ matrix.attr }} - build-checks-aarch64-darwin: + nix-build-aarch64-darwin: name: ${{ matrix.name }} (aarch64-darwin) - needs: checks-matrix + needs: nix-eval runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: | - !cancelled() && - ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_darwin != null }} && - (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && - (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') + if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin != null }} strategy: fail-fast: false max-parallel: 5 - matrix: ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_darwin }} + matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin }} steps: - name: Checkout Repo uses: actions/checkout@v4 @@ -148,15 +66,15 @@ jobs: attr: ${{ matrix.attr }} # TODO - # build-checks-x86_64-linux: + # nix-build-x86_64-linux: # name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) - # needs: checks-matrix + # needs: nix-eval # runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - # if: ${{ fromJSON(needs.checks-matrix.outputs.matrix).x86_64_linux != null }} + # if: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux != null }} # strategy: # fail-fast: false - # max-parallel: 5 - # matrix: ${{ fromJSON(needs.checks-matrix.outputs.matrix).x86_64_linux }} + # max-parallel: 3 + # matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux }} # steps: # - name: Checkout Repo # uses: actions/checkout@v4 @@ -166,9 +84,9 @@ jobs: # attr: ${{ matrix.attr }} run-tests: - needs: [build-checks-aarch64-linux, build-checks-aarch64-darwin] #, build-checks-x86_64-linux] + needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] if: | !cancelled() && - (needs.build-checks-aarch64-linux.result == 'skipped' || needs.build-checks-aarch64-linux.result == 'success') && - (needs.build-checks-aarch64-darwin.result == 'skipped' || needs.build-checks-aarch64-darwin.result == 'success') + (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && + (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') uses: ./.github/workflows/test.yml diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index b3b003afc..d6aa80ef3 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -71,24 +71,12 @@ class GitHubActionPackage(TypedDict): } -def get_worker_count() -> int: - """Get optimal worker count based on CPU cores.""" - try: - return max(1, int(os.cpu_count())) - except (OSError, AttributeError): - print( - "Warning: Unable to get CPU count, using default max_workers=1", - file=sys.stderr, - ) - return 1 - - -def build_nix_eval_command(max_workers: int, target: str) -> List[str]: +def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[str]: """Build the nix-eval-jobs command with appropriate flags.""" nix_eval_cmd = [ "nix-eval-jobs", "--flake", - f".#{target}", + ".", "--check-cache-status", "--force-recurse", "--quiet", @@ -100,12 +88,14 @@ def build_nix_eval_command(max_workers: int, target: str) -> List[str]: "true", "--workers", str(max_workers), + "--select", + f"outputs: {{ inherit (outputs) {' '.join(flake_outputs)}; }}", ] return nix_eval_cmd def parse_nix_eval_line( - line: str, drv_paths: Set[str], target: str + line: str, drv_paths: Set[str] ) -> Optional[GitHubActionPackage]: """Parse a single line of nix-eval-jobs output""" if not line.strip(): @@ -120,7 +110,7 @@ def parse_nix_eval_line( runs_on_config = BUILD_RUNNER_MAP[data["system"]] return { - "attr": f"{target}.{data['attr']}", + "attr": f"{data['attr']}", "name": data["name"], "system": data["system"], "already_cached": data.get("cacheStatus") != "notBuilt", @@ -134,22 +124,23 @@ def parse_nix_eval_line( return None -def run_nix_eval_jobs( - cmd: List[str], target: str -) -> Generator[GitHubActionPackage, None, None]: +def run_nix_eval_jobs(cmd: List[str]) -> Generator[GitHubActionPackage, None, None]: """Run nix-eval-jobs and yield parsed package data.""" print(f"Running command: {' '.join(cmd)}", file=sys.stderr) with subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) as process: - drv_paths = set() + drv_paths: Set[str] = set() + assert process.stdout is not None # for mypy + assert process.stderr is not None # for mypy for line in process.stdout: - package = parse_nix_eval_line(line, drv_paths, target) + package = parse_nix_eval_line(line, drv_paths) if package: yield package - if process.returncode and process.returncode != 0: + process.wait() + if process.returncode != 0: print("Error: Evaluation failed", file=sys.stderr) sys.stderr.write(process.stderr.read()) sys.exit(process.returncode) @@ -162,7 +153,7 @@ def is_extension_pkg(pkg: GitHubActionPackage) -> bool: # thank you buildbot-nix https://github.com/nix-community/buildbot-nix/blob/985d069a2a45cf4a571a4346107671adc2bd2a16/buildbot_nix/buildbot_nix/build_trigger.py#L297 -def sort_pkgs_by_closures(jobs: list[GitHubActionPackage]) -> list[GitHubActionPackage]: +def sort_pkgs_by_closures(jobs: List[GitHubActionPackage]) -> List[GitHubActionPackage]: sorted_jobs = [] # Prepare job dependencies @@ -194,23 +185,16 @@ def main() -> None: description="Generate GitHub Actions matrix for Nix builds" ) parser.add_argument( - "target", choices=["checks", "extensions"], help="Type of matrix to generate" + "flake_outputs", nargs="+", help="Nix flake outputs to evaluate" ) args = parser.parse_args() - max_workers = get_worker_count() + max_workers: int = os.cpu_count() or 1 - if args.target == "checks": - flake_output = "checks" - else: - flake_output = "legacyPackages" + cmd = build_nix_eval_command(max_workers, args.flake_outputs) - cmd = build_nix_eval_command(max_workers, flake_output) - - gh_action_packages = sort_pkgs_by_closures( - list(run_nix_eval_jobs(cmd, flake_output)) - ) + gh_action_packages = sort_pkgs_by_closures(list(run_nix_eval_jobs(cmd))) def clean_package_for_output(pkg: GitHubActionPackage) -> dict: """Remove debug fields from package for final output""" @@ -226,14 +210,6 @@ def clean_package_for_output(pkg: GitHubActionPackage) -> dict: ), } - if args.target == "extensions": - # filter to only include extension packages and add postgresql_version field - gh_action_packages = [ - {**pkg, "postgresql_version": pkg["attr"].split(".")[-3]} - for pkg in gh_action_packages - if is_extension_pkg(pkg) - ] - # Group packages by system grouped_by_system = defaultdict(list) for pkg in gh_action_packages: From 333f031637c530f5155b6986c991d9285f916c4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 3 Oct 2025 16:04:41 +0200 Subject: [PATCH 26/71] Revert "chores(ci): comment out vestigal testinfra-ami-build" This reverts commit e2db36800b8a2a0ad9e132fdad688613aeb5aa37. --- .github/workflows/nix-build.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 56e1235a6..86f40c64d 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -10,7 +10,7 @@ on: permissions: id-token: write - # required by dependent workflows + # required by testinfra-ami-build dependent workflows contents: write packages: write @@ -83,6 +83,16 @@ jobs: # with: # attr: ${{ matrix.attr }} + run-testinfra: + needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] + if: | + !cancelled() && + (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && + (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') + uses: ./.github/workflows/testinfra-ami-build.yml + secrets: + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} + run-tests: needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] if: | From df79c71156f1cfc2b643e548cd1b816b900840e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 3 Oct 2025 21:04:09 +0200 Subject: [PATCH 27/71] feat(ci): eval on blacksmith-32vcpu-ubuntu-2404 --- .github/actionlint.yaml | 1 + .../action.yml | 2 +- .github/workflows/nix-build.yml | 21 ++++++++++--------- 3 files changed, 13 insertions(+), 11 deletions(-) rename .github/actions/{nix-build-setup => nix-build-self-hosted}/action.yml (94%) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 0a2631f7d..6eea24e6a 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -2,3 +2,4 @@ self-hosted-runner: labels: - aarch64-darwin - aarch64-linux + - blacksmith-32vcpu-ubuntu-2404 diff --git a/.github/actions/nix-build-setup/action.yml b/.github/actions/nix-build-self-hosted/action.yml similarity index 94% rename from .github/actions/nix-build-setup/action.yml rename to .github/actions/nix-build-self-hosted/action.yml index afa06499f..f7b202f1c 100644 --- a/.github/actions/nix-build-setup/action.yml +++ b/.github/actions/nix-build-self-hosted/action.yml @@ -36,4 +36,4 @@ runs: nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - name: nix build shell: bash - run: nix build -L .#${{ inputs.attr }} + run: nix build --accept-flake-config -L .#${{ inputs.attr }} diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 86f40c64d..d78bb9e6f 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -16,20 +16,21 @@ permissions: jobs: nix-eval: - runs-on: - group: self-hosted-runners-nix - labels: - - aarch64-darwin + runs-on: blacksmith-32vcpu-ubuntu-2404 outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - name: Checkout Repo uses: actions/checkout@v4 + - name: Install nix + uses: cachix/install-nix-action@v31 + with: + install_url: https://releases.nixos.org/nix/nix-2.31.2/install - id: set-matrix name: Generate Nix Matrix run: | set -Eeu - echo matrix="$(python scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" + echo matrix="$(nix shell nixpkgs/405fc615369e0ea1b9c284c107ca4c3e1bc15774#nix-eval-jobs --command scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" nix-build-aarch64-linux: name: ${{ matrix.name }} (aarch64-linux) @@ -44,7 +45,7 @@ jobs: - name: Checkout Repo uses: actions/checkout@v4 - name: Build Nix Package - uses: ./.github/actions/nix-build-setup + uses: ./.github/actions/nix-build-self-hosted with: attr: ${{ matrix.attr }} @@ -61,7 +62,7 @@ jobs: - name: Checkout Repo uses: actions/checkout@v4 - name: Build Nix Package - uses: ./.github/actions/nix-build-setup + uses: ./.github/actions/nix-build-self-hosted with: attr: ${{ matrix.attr }} @@ -79,14 +80,14 @@ jobs: # - name: Checkout Repo # uses: actions/checkout@v4 # - name: Build Nix Package - # uses: ./.github/actions/nix-build-setup + # uses: ./.github/actions/nix-build-self-hosted # with: # attr: ${{ matrix.attr }} run-testinfra: needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] if: | - !cancelled() && + !cancelled() && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') uses: ./.github/workflows/testinfra-ami-build.yml @@ -96,7 +97,7 @@ jobs: run-tests: needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] if: | - !cancelled() && + !cancelled() && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') uses: ./.github/workflows/test.yml From fee4471f74a58e6f88fa7035672b72e2e23d92f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 3 Oct 2025 21:04:09 +0200 Subject: [PATCH 28/71] feat: add ephemeral Nix install action for GitHub runners --- .github/workflows/nix-build.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index d78bb9e6f..5f3d6b781 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -23,14 +23,12 @@ jobs: - name: Checkout Repo uses: actions/checkout@v4 - name: Install nix - uses: cachix/install-nix-action@v31 - with: - install_url: https://releases.nixos.org/nix/nix-2.31.2/install + uses: ./.github/actions/nix-install-ephemeral - id: set-matrix name: Generate Nix Matrix run: | set -Eeu - echo matrix="$(nix shell nixpkgs/405fc615369e0ea1b9c284c107ca4c3e1bc15774#nix-eval-jobs --command scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" + echo matrix="$(nix shell github:nix-community/nix-eval-jobs --command scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" nix-build-aarch64-linux: name: ${{ matrix.name }} (aarch64-linux) @@ -97,7 +95,7 @@ jobs: run-tests: needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] if: | - !cancelled() && + !cancelled() && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') uses: ./.github/workflows/test.yml From 1bbdf90c7416858f5b7d408cb8acb6d47b26dca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Sat, 4 Oct 2025 01:57:06 +0200 Subject: [PATCH 29/71] refactor(ci): extract nix eval into reusable workflow --- .github/workflows/nix-build.yml | 17 ++++------------- .github/workflows/nix-eval.yml | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/nix-eval.yml diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 5f3d6b781..f15920795 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -16,19 +16,10 @@ permissions: jobs: nix-eval: - runs-on: blacksmith-32vcpu-ubuntu-2404 - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Install nix - uses: ./.github/actions/nix-install-ephemeral - - id: set-matrix - name: Generate Nix Matrix - run: | - set -Eeu - echo matrix="$(nix shell github:nix-community/nix-eval-jobs --command scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" + uses: ./.github/workflows/nix-eval.yml + secrets: + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} + NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} nix-build-aarch64-linux: name: ${{ matrix.name }} (aarch64-linux) diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml new file mode 100644 index 000000000..f5436ce46 --- /dev/null +++ b/.github/workflows/nix-eval.yml @@ -0,0 +1,32 @@ +name: Nix Eval + +on: + workflow_call: + outputs: + matrix: + description: 'Generated build matrix' + value: ${{ jobs.eval.outputs.matrix }} + secrets: + DEV_AWS_ROLE: + required: false + NIX_SIGN_SECRET_KEY: + required: false + +jobs: + eval: + runs-on: blacksmith-32vcpu-ubuntu-2404 + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Install nix + uses: ./.github/actions/nix-install-ephemeral + env: + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} + NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} + - id: set-matrix + name: Generate Nix Matrix + run: | + set -Eeu + echo matrix="$(nix shell github:nix-community/nix-eval-jobs/v2.31.0 --command scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" From d27ecb5624e89b39332a01fb4b58dcbaeee84549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 13:15:02 +0200 Subject: [PATCH 30/71] feat: enable x86_64-linux builds in CI --- .../action.yml | 17 ++---- .github/workflows/nix-build.yml | 58 +++++++++++-------- 2 files changed, 37 insertions(+), 38 deletions(-) rename .github/actions/{nix-build-self-hosted => nix-install-self-hosted}/action.yml (62%) diff --git a/.github/actions/nix-build-self-hosted/action.yml b/.github/actions/nix-install-self-hosted/action.yml similarity index 62% rename from .github/actions/nix-build-self-hosted/action.yml rename to .github/actions/nix-install-self-hosted/action.yml index f7b202f1c..52e49db9d 100644 --- a/.github/actions/nix-build-self-hosted/action.yml +++ b/.github/actions/nix-install-self-hosted/action.yml @@ -1,13 +1,10 @@ -name: 'Nix Build Setup' -description: 'Sets up AWS credentials and builds a Nix package' +name: 'Configure Nix on self hosted runners' +description: 'Sets up AWS credentials to push to the Nix binary cache' inputs: - attr: - description: 'The Nix attribute to build' - required: true aws-role-duration: description: 'AWS role session duration in seconds' required: false - default: '3600' + default: '18000' runs: using: 'composite' @@ -19,7 +16,7 @@ runs: aws-region: us-east-2 role-to-assume: arn:aws:iam::436098097459:role/nix-artifacts-deploy-role # supabase-dev role-session-name: gha-oidc-${{ github.run_id }} - role-duration-seconds: 18000 + role-duration-seconds: ${{ inputs.aws-role-duration }} - name: Write creds files shell: bash @@ -31,9 +28,3 @@ runs: aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY} aws_session_token = ${AWS_SESSION_TOKEN} EOF - - name: Build psql bundle - run: > - nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - - name: nix build - shell: bash - run: nix build --accept-flake-config -L .#${{ inputs.attr }} diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index f15920795..8d4b43c17 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -34,9 +34,10 @@ jobs: - name: Checkout Repo uses: actions/checkout@v4 - name: Build Nix Package - uses: ./.github/actions/nix-build-self-hosted - with: - attr: ${{ matrix.attr }} + uses: ./.github/actions/nix-install-self-hosted + - name: nix build + shell: bash + run: nix build --accept-flake-config -L .#${{ matrix.attr }} nix-build-aarch64-darwin: name: ${{ matrix.name }} (aarch64-darwin) @@ -51,30 +52,37 @@ jobs: - name: Checkout Repo uses: actions/checkout@v4 - name: Build Nix Package - uses: ./.github/actions/nix-build-self-hosted - with: - attr: ${{ matrix.attr }} + uses: ./.github/actions/nix-install-self-hosted + - name: nix build + shell: bash + run: nix build --accept-flake-config -L .#${{ matrix.attr }} - # TODO - # nix-build-x86_64-linux: - # name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) - # needs: nix-eval - # runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - # if: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux != null }} - # strategy: - # fail-fast: false - # max-parallel: 3 - # matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux }} - # steps: - # - name: Checkout Repo - # uses: actions/checkout@v4 - # - name: Build Nix Package - # uses: ./.github/actions/nix-build-self-hosted - # with: - # attr: ${{ matrix.attr }} + nix-build-x86_64-linux: + name: ${{ matrix.name }} (x86_64-linux) + needs: nix-eval + runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux != null }} + strategy: + fail-fast: false + max-parallel: 5 + matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Install nix + uses: ./.github/actions/nix-install-ephemeral + env: + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} + NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} + - name: Build psql bundle + run: > + nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" + - name: nix build + shell: bash + run: nix build --accept-flake-config -L .#${{ matrix.attr }} run-testinfra: - needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] + needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] if: | !cancelled() && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && @@ -84,7 +92,7 @@ jobs: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} run-tests: - needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] + needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] if: | !cancelled() && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && From 19cdcbcbcce5f773470524357a9accccec8b0cd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 14:32:56 +0200 Subject: [PATCH 31/71] feat: add PostgreSQL version to GitHub Actions job names When building a postgres extension, the build matrix may include multiple time the same extension for different PostgreSQL versions. This change makes it easier to identify which job corresponds to which PostgreSQL version in the workflow runs. --- .github/workflows/nix-build.yml | 12 +++++++++--- scripts/github-matrix.py | 12 ++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 8d4b43c17..0c3e78bc2 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -22,7 +22,9 @@ jobs: NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} nix-build-aarch64-linux: - name: ${{ matrix.name }} (aarch64-linux) + name: >- + ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} + (aarch64-linux) needs: nix-eval runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_linux != null }} @@ -40,7 +42,9 @@ jobs: run: nix build --accept-flake-config -L .#${{ matrix.attr }} nix-build-aarch64-darwin: - name: ${{ matrix.name }} (aarch64-darwin) + name: >- + ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} + (aarch64-darwin) needs: nix-eval runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin != null }} @@ -58,7 +62,9 @@ jobs: run: nix build --accept-flake-config -L .#${{ matrix.attr }} nix-build-x86_64-linux: - name: ${{ matrix.name }} (x86_64-linux) + name: >- + ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} + (x86_64-linux) needs: nix-eval runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux != null }} diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index d6aa80ef3..b80a9f6c0 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -198,17 +198,17 @@ def main() -> None: def clean_package_for_output(pkg: GitHubActionPackage) -> dict: """Remove debug fields from package for final output""" - return { + returned_pkg = { "attr": pkg["attr"], "name": pkg["name"], "system": pkg["system"], "runs_on": pkg["runs_on"], - **( - {"postgresql_version": pkg["postgresql_version"]} - if "postgresql_version" in pkg - else {} - ), } + if is_extension_pkg(pkg): + # Extract PostgreSQL version from attribute path + attrs = pkg["attr"].split(".") + returned_pkg["postgresql_version"] = attrs[-3].split("_")[-1] + return returned_pkg # Group packages by system grouped_by_system = defaultdict(list) From 424e7b8ba96222fc9923f16804f9e2e93faea3b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 15:05:41 +0200 Subject: [PATCH 32/71] fix: disable treefmt flake check treefmt is already included in the pre-commit hooks check. --- nix/fmt.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/fmt.nix b/nix/fmt.nix index 562c3b3c5..08763e5b8 100644 --- a/nix/fmt.nix +++ b/nix/fmt.nix @@ -4,6 +4,7 @@ perSystem = { pkgs, ... }: { + treefmt.flakeCheck = false; treefmt.programs = { deadnix.enable = true; nixfmt = { From 994bd62c303727865e207dc1f463fd7b04727ec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 15:05:41 +0200 Subject: [PATCH 33/71] feat: run actionlint on new GitHub Actions workflows --- nix/hooks.nix | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/nix/hooks.nix b/nix/hooks.nix index 896c262ba..b659635f8 100644 --- a/nix/hooks.nix +++ b/nix/hooks.nix @@ -1,4 +1,11 @@ { inputs, ... }: +let + ghWorkflows = builtins.attrNames (builtins.readDir ../.github/workflows); + lintedWorkflows = [ + "nix-eval.yml" + "nix-build.yml" + ]; +in { imports = [ inputs.git-hooks.flakeModule ]; perSystem = @@ -8,9 +15,17 @@ check.enable = true; settings = { hooks = { + actionlint = { + enable = true; + excludes = builtins.filter (name: !builtins.elem name lintedWorkflows) ghWorkflows; + verbose = true; + }; + treefmt = { enable = true; package = config.treefmt.build.wrapper; + pass_filenames = false; + verbose = true; }; }; }; From b0067a70c01674c9dfb409ef3c14a76fc6c4703f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 16:08:25 +0200 Subject: [PATCH 34/71] chore: improve github matrix script type annotations --- scripts/github-matrix.py | 43 +++++++++++----------------------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index b80a9f6c0..cc072f33e 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -43,16 +43,12 @@ class RunsOnConfig(TypedDict): class GitHubActionPackage(TypedDict): - """Processed package for GitHub Actions matrix.""" + """Final package output for GitHub Actions matrix.""" attr: str name: str system: str - already_cached: bool runs_on: RunsOnConfig - drvPath: str - neededSubstitutes: List[str] - neededBuilds: List[str] postgresql_version: NotRequired[str] @@ -94,9 +90,7 @@ def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[s return nix_eval_cmd -def parse_nix_eval_line( - line: str, drv_paths: Set[str] -) -> Optional[GitHubActionPackage]: +def parse_nix_eval_line(line: str, drv_paths: Set[str]) -> Optional[NixEvalJobsOutput]: """Parse a single line of nix-eval-jobs output""" if not line.strip(): return None @@ -106,25 +100,13 @@ def parse_nix_eval_line( if data["drvPath"] in drv_paths: return None drv_paths.add(data["drvPath"]) - - runs_on_config = BUILD_RUNNER_MAP[data["system"]] - - return { - "attr": f"{data['attr']}", - "name": data["name"], - "system": data["system"], - "already_cached": data.get("cacheStatus") != "notBuilt", - "runs_on": runs_on_config, - "drvPath": data["drvPath"], - "neededSubstitutes": data.get("neededSubstitutes", []), - "neededBuilds": data.get("neededBuilds", []), - } + return data except json.JSONDecodeError: print(f"Skipping invalid JSON line: {line}", file=sys.stderr) return None -def run_nix_eval_jobs(cmd: List[str]) -> Generator[GitHubActionPackage, None, None]: +def run_nix_eval_jobs(cmd: List[str]) -> Generator[NixEvalJobsOutput, None, None]: """Run nix-eval-jobs and yield parsed package data.""" print(f"Running command: {' '.join(cmd)}", file=sys.stderr) @@ -146,21 +128,21 @@ def run_nix_eval_jobs(cmd: List[str]) -> Generator[GitHubActionPackage, None, No sys.exit(process.returncode) -def is_extension_pkg(pkg: GitHubActionPackage) -> bool: +def is_extension_pkg(pkg: NixEvalJobsOutput) -> bool: """Check if the package is a postgresql extension package.""" attrs = pkg["attr"].split(".") return attrs[-2] == "exts" # thank you buildbot-nix https://github.com/nix-community/buildbot-nix/blob/985d069a2a45cf4a571a4346107671adc2bd2a16/buildbot_nix/buildbot_nix/build_trigger.py#L297 -def sort_pkgs_by_closures(jobs: List[GitHubActionPackage]) -> List[GitHubActionPackage]: +def sort_pkgs_by_closures(jobs: List[NixEvalJobsOutput]) -> List[NixEvalJobsOutput]: sorted_jobs = [] # Prepare job dependencies job_set = {job["drvPath"] for job in jobs} job_closures = { - k["drvPath"]: set(k["neededSubstitutes"]) - .union(set(k["neededBuilds"])) + k["drvPath"]: set(k.get("neededSubstitutes", [])) + .union(set(k.get("neededBuilds", []))) .intersection(job_set) .difference({k["drvPath"]}) for k in jobs @@ -196,13 +178,12 @@ def main() -> None: gh_action_packages = sort_pkgs_by_closures(list(run_nix_eval_jobs(cmd))) - def clean_package_for_output(pkg: GitHubActionPackage) -> dict: - """Remove debug fields from package for final output""" - returned_pkg = { + def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: + """Convert nix-eval-jobs output to GitHub Actions matrix package""" + returned_pkg: GitHubActionPackage = { "attr": pkg["attr"], "name": pkg["name"], "system": pkg["system"], - "runs_on": pkg["runs_on"], } if is_extension_pkg(pkg): # Extract PostgreSQL version from attribute path @@ -213,7 +194,7 @@ def clean_package_for_output(pkg: GitHubActionPackage) -> dict: # Group packages by system grouped_by_system = defaultdict(list) for pkg in gh_action_packages: - if not pkg["already_cached"]: + if pkg.get("cacheStatus") == "notBuilt": grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) # Create output with system-specific matrices From 8e7744a391d15542e3dfe32efc859728a40fb86c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 16:31:07 +0200 Subject: [PATCH 35/71] feat: optimize CI runner selection based on package size Dynamically assign larger runners (32vcpu) for Rust and PostGIS extensions while using smaller runners (8vcpu) for standard packages. --- scripts/github-matrix.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index cc072f33e..f43280552 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -54,15 +54,14 @@ class GitHubActionPackage(TypedDict): BUILD_RUNNER_MAP: Dict[str, RunsOnConfig] = { "aarch64-linux": { - "group": "self-hosted-runners-nix", - "labels": ["aarch64-linux"], + "labels": ["blacksmith-8vcpu-ubuntu-2404-arm"], }, "aarch64-darwin": { "group": "self-hosted-runners-nix", "labels": ["aarch64-darwin"], }, "x86_64-linux": { - "labels": ["blacksmith-32vcpu-ubuntu-2404"], + "labels": ["blacksmith-8vcpu-ubuntu-2404"], }, } @@ -162,6 +161,30 @@ def sort_pkgs_by_closures(jobs: List[NixEvalJobsOutput]) -> List[NixEvalJobsOutp return sorted_jobs +def is_large_pkg(pkg: NixEvalJobsOutput) -> bool: + """Determine if a package is considered large based on its attribute path.""" + RUST_EXTENSIONS = ["exts.wrappers", "exts.pg_jsonschema", "exts.pg_graphql"] + LARGE_C_EXTENSION = ["exts.postgis"] + return any( + indicator in pkg["attr"] for indicator in RUST_EXTENSIONS + LARGE_C_EXTENSION + ) + + +def get_runner_for_package(pkg: NixEvalJobsOutput) -> RunsOnConfig: + """Determine the appropriate GitHub Actions runner for a package.""" + system = pkg["system"] + if is_large_pkg(pkg): + # Use larger runners for large packages for x86_64-linux and aarch64-linux + if system == "x86_64-linux": + return {"labels": ["blacksmith-32vcpu-ubuntu-2404"]} + elif system == "aarch64-linux": + return {"labels": ["blacksmith-32vcpu-ubuntu-2404-arm"]} + if system in BUILD_RUNNER_MAP: + return BUILD_RUNNER_MAP[system] + else: + raise ValueError(f"No runner configuration for system: {system}") + + def main() -> None: parser = argparse.ArgumentParser( description="Generate GitHub Actions matrix for Nix builds" @@ -184,6 +207,7 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: "attr": pkg["attr"], "name": pkg["name"], "system": pkg["system"], + "runs_on": get_runner_for_package(pkg), } if is_extension_pkg(pkg): # Extract PostgreSQL version from attribute path From 65f4e04a8ede87d93355dc441ed8fcf1b58d92f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 20:12:21 +0200 Subject: [PATCH 36/71] chore: fix package meta maintainers format --- nix/docs/adding-new-package.md | 2 +- nix/ext/pgvector.nix | 2 +- scripts/github-matrix.py | 15 ++++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/nix/docs/adding-new-package.md b/nix/docs/adding-new-package.md index 7d9fbdae9..d5620de09 100644 --- a/nix/docs/adding-new-package.md +++ b/nix/docs/adding-new-package.md @@ -42,7 +42,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Open-source vector similarity search for Postgres"; homepage = "https://github.com/${src.owner}/${src.repo}"; - maintainers = with maintainers; [ olirice ]; + maintainers = [ "olirice" ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index fa6dba77d..74571890e 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -69,7 +69,7 @@ let meta = with lib; { description = "Open-source vector similarity search for Postgres"; homepage = "https://github.com/${src.owner}/${src.repo}"; - maintainers = with maintainers; [ olirice ]; + maintainers = [ "olirice" ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index f43280552..cf0256968 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -33,6 +33,7 @@ class NixEvalJobsOutput(TypedDict): neededBuilds: NotRequired[List[Any]] neededSubstitutes: NotRequired[List[Any]] outputs: NotRequired[Dict[str, str]] + error: NotRequired[str] class RunsOnConfig(TypedDict): @@ -96,6 +97,10 @@ def parse_nix_eval_line(line: str, drv_paths: Set[str]) -> Optional[NixEvalJobsO try: data: NixEvalJobsOutput = json.loads(line) + if "error" in data: + raise ValueError( + f"Error in nix-eval-jobs output for {data['attr']}: {data['error']}" + ) if data["drvPath"] in drv_paths: return None drv_paths.add(data["drvPath"]) @@ -149,14 +154,10 @@ def sort_pkgs_by_closures(jobs: List[NixEvalJobsOutput]) -> List[NixEvalJobsOutp sorter = graphlib.TopologicalSorter(job_closures) + job_by_drv = {job["drvPath"]: job for job in jobs} for item in sorter.static_order(): - i = 0 - while i < len(jobs): - if item == jobs[i]["drvPath"]: - sorted_jobs.append(jobs[i]) - del jobs[i] - else: - i += 1 + if item in job_by_drv: + sorted_jobs.append(job_by_drv[item]) return sorted_jobs From 236ff3bf3678282cd9c65d9cd0cd3634ee9b8e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 21:32:36 +0200 Subject: [PATCH 37/71] chore: create a nix package for generating GitHub Actions matrix Add pytest tests for the package Add nix-eval-jobs in path for the package --- .github/workflows/nix-eval.yml | 2 +- flake.lock | 143 ++++++++++- flake.nix | 1 + nix/packages/default.nix | 3 + nix/packages/github-matrix/default.nix | 27 +++ .../packages/github-matrix/github_matrix.py | 78 ++++-- .../github-matrix/tests/test_github_matrix.py | 226 ++++++++++++++++++ 7 files changed, 453 insertions(+), 27 deletions(-) create mode 100644 nix/packages/github-matrix/default.nix rename scripts/github-matrix.py => nix/packages/github-matrix/github_matrix.py (77%) create mode 100644 nix/packages/github-matrix/tests/test_github_matrix.py diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml index f5436ce46..e97fe9171 100644 --- a/.github/workflows/nix-eval.yml +++ b/.github/workflows/nix-eval.yml @@ -29,4 +29,4 @@ jobs: name: Generate Nix Matrix run: | set -Eeu - echo matrix="$(nix shell github:nix-community/nix-eval-jobs/v2.31.0 --command scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" + echo matrix="$(nix run .\#github-matrix checks legacyPackages)" >> "$GITHUB_OUTPUT" diff --git a/flake.lock b/flake.lock index 8152dbb73..19db22fe0 100644 --- a/flake.lock +++ b/flake.lock @@ -34,6 +34,27 @@ "type": "github" } }, + "flake-parts_2": { + "inputs": { + "nixpkgs-lib": [ + "nix-eval-jobs", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1741352980, + "narHash": "sha256-+u2UunDA4Cl5Fci3m7S643HzKmIDAe+fiXrLqYsR2fs=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "f4330d22f1c5d2ba72d3d22df5597d123fdb60a9", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -95,6 +116,22 @@ "type": "github" } }, + "nix": { + "flake": false, + "locked": { + "lastModified": 1759793966, + "narHash": "sha256-h95Lz2j19/Sb9nuBM3BXUqlJgBkIZd15/QTkufajaLY=", + "owner": "NixOS", + "repo": "nix", + "rev": "eea6d75783e10e6057a097af29741c9a47cbb7e0", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nix", + "type": "github" + } + }, "nix-editor": { "inputs": { "nixpkgs": [ @@ -144,6 +181,28 @@ "type": "github" } }, + "nix-eval-jobs": { + "inputs": { + "flake-parts": "flake-parts_2", + "nix": "nix", + "nixpkgs": "nixpkgs_2", + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1760477819, + "narHash": "sha256-/hfZp5kjN2ratoMtv5JTvekLBUcaXu2i7JgqbNOsctw=", + "owner": "jfroche", + "repo": "nix-eval-jobs", + "rev": "1fe1983db41fbb3e148e8eb26b7ed4cb6f4cf49e", + "type": "github" + }, + "original": { + "owner": "jfroche", + "ref": "show-required-system-features", + "repo": "nix-eval-jobs", + "type": "github" + } + }, "nix2container": { "inputs": { "flake-utils": [ @@ -230,6 +289,66 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 315532800, + "narHash": "sha256-yDxtm0PESdgNetiJN5+MFxgubBcLDTiuSjjrJiyvsvM=", + "rev": "d7f52a7a640bc54c7bb414cca603835bf8dd4b10", + "type": "tarball", + "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre871443.d7f52a7a640b/nixexprs.tar.xz" + }, + "original": { + "type": "tarball", + "url": "https://nixos.org/channels/nixpkgs-unstable/nixexprs.tar.xz" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1697269602, + "narHash": "sha256-dSzV7Ud+JH4DPVD9od53EgDrxUVQOcSj4KGjggCDVJI=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9cb540e9c1910d74a7e10736277f6eb9dff51c81", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1712666087, + "narHash": "sha256-WwjUkWsjlU8iUImbivlYxNyMB1L5YVqE8QotQdL9jWc=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "a76c4553d7e741e17f289224eda135423de0491d", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_5": { + "locked": { + "lastModified": 1744536153, + "narHash": "sha256-awS2zRgF4uTwrOKwwiJcByDzDOdo3Q1rPZbiHQg/N38=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "18dd725c29603f582cf1900e0d25f9f1063dbf11", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { "flake-parts": "flake-parts", @@ -239,10 +358,11 @@ "nix-fast-build": "nix-fast-build", "nix2container": "nix2container", "nixpkgs": "nixpkgs", + "nix-eval-jobs": "nix-eval-jobs", "nixpkgs-go124": "nixpkgs-go124", "nixpkgs-pgbackrest": "nixpkgs-pgbackrest", "rust-overlay": "rust-overlay", - "treefmt-nix": "treefmt-nix" + "treefmt-nix": "treefmt-nix_2" } }, "rust-overlay": { @@ -281,6 +401,27 @@ } }, "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "nix-eval-jobs", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1752055615, + "narHash": "sha256-19m7P4O/Aw/6+CzncWMAJu89JaKeMh3aMle1CNQSIwM=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "c9d477b5d5bd7f26adddd3f96cfd6a904768d4f9", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + }, + "treefmt-nix_2": { "inputs": { "nixpkgs": [ "nixpkgs" diff --git a/flake.nix b/flake.nix index 8f83a0a25..aa1539553 100644 --- a/flake.nix +++ b/flake.nix @@ -28,6 +28,7 @@ git-hooks.inputs.nixpkgs.follows = "nixpkgs"; nixpkgs-go124.url = "github:Nixos/nixpkgs/d2ac4dfa61fba987a84a0a81555da57ae0b9a2b0"; nixpkgs-pgbackrest.url = "github:nixos/nixpkgs/nixos-unstable-small"; + nix-eval-jobs.url = "github:jfroche/nix-eval-jobs/show-required-system-features"; }; outputs = diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 0f5f43ffe..399b34865 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -33,6 +33,9 @@ cleanup-ami = pkgs.callPackage ./cleanup-ami.nix { }; dbmate-tool = pkgs.callPackage ./dbmate-tool.nix { inherit (self.supabase) defaults; }; docs = pkgs.callPackage ./docs.nix { }; + github-matrix = pkgs.callPackage ./github-matrix { + nix-eval-jobs = inputs'.nix-eval-jobs.packages.default; + }; supabase-groonga = pkgs.callPackage ./groonga { }; http-mock-server = pkgs.callPackage ./http-mock-server.nix { }; local-infra-bootstrap = pkgs.callPackage ./local-infra-bootstrap.nix { }; diff --git a/nix/packages/github-matrix/default.nix b/nix/packages/github-matrix/default.nix new file mode 100644 index 000000000..0d755c977 --- /dev/null +++ b/nix/packages/github-matrix/default.nix @@ -0,0 +1,27 @@ +{ + lib, + nix-eval-jobs, + python3Packages, +}: +let + pname = "github-matrix"; +in + +python3Packages.buildPythonApplication { + inherit pname; + version = "0.1.0"; + pyproject = false; + + src = ./.; + + makeWrapperArgs = [ "--suffix PATH : ${lib.makeBinPath [ nix-eval-jobs ]}" ]; + + nativeCheckInputs = with python3Packages; [ + pytestCheckHook + pytest-mypy + ]; + + installPhase = '' + install -Dm755 github_matrix.py "$out/bin/${pname}" + ''; +} diff --git a/scripts/github-matrix.py b/nix/packages/github-matrix/github_matrix.py similarity index 77% rename from scripts/github-matrix.py rename to nix/packages/github-matrix/github_matrix.py index cf0256968..2c4681670 100755 --- a/scripts/github-matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -19,6 +19,9 @@ TypedDict, ) +System = Literal["x86_64-linux", "aarch64-linux", "aarch64-darwin"] +RunnerType = Literal["ephemeral", "self-hosted"] + class NixEvalJobsOutput(TypedDict): """Raw output from nix-eval-jobs command.""" @@ -27,9 +30,8 @@ class NixEvalJobsOutput(TypedDict): attrPath: List[str] cacheStatus: Literal["notBuilt", "cached", "local"] drvPath: str - isCached: bool name: str - system: str + system: System neededBuilds: NotRequired[List[Any]] neededSubstitutes: NotRequired[List[Any]] outputs: NotRequired[Dict[str, str]] @@ -48,21 +50,29 @@ class GitHubActionPackage(TypedDict): attr: str name: str - system: str + system: System runs_on: RunsOnConfig postgresql_version: NotRequired[str] -BUILD_RUNNER_MAP: Dict[str, RunsOnConfig] = { - "aarch64-linux": { - "labels": ["blacksmith-8vcpu-ubuntu-2404-arm"], - }, - "aarch64-darwin": { - "group": "self-hosted-runners-nix", - "labels": ["aarch64-darwin"], +BUILD_RUNNER_MAP: Dict[RunnerType, Dict[System, RunsOnConfig]] = { + "ephemeral": { + "aarch64-linux": { + "labels": ["blacksmith-8vcpu-ubuntu-2404-arm"], + }, + "x86_64-linux": { + "labels": ["blacksmith-8vcpu-ubuntu-2404"], + }, }, - "x86_64-linux": { - "labels": ["blacksmith-8vcpu-ubuntu-2404"], + "self-hosted": { + "aarch64-darwin": { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-darwin"], + }, + "aarch64-linux": { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-linux"], + }, }, } @@ -76,6 +86,7 @@ def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[s "--check-cache-status", "--force-recurse", "--quiet", + "--show-required-system-features", "--option", "eval-cache", "false", @@ -171,19 +182,33 @@ def is_large_pkg(pkg: NixEvalJobsOutput) -> bool: ) -def get_runner_for_package(pkg: NixEvalJobsOutput) -> RunsOnConfig: - """Determine the appropriate GitHub Actions runner for a package.""" +def is_kvm_pkg(pkg: NixEvalJobsOutput) -> bool: + """Determine if a package requires KVM""" + return "kvm" in pkg.get("requiredSystemFeatures", []) + + +def get_runner_for_package(pkg: NixEvalJobsOutput) -> RunsOnConfig | None: + """Determine the appropriate GitHub Actions runner for a package. + + Priority order: + 1. KVM packages → self-hosted runners + 2. Large packages on Linux → 32vcpu ephemeral runners + 3. Darwin packages → self-hosted runners + 4. Default → ephemeral runners + """ system = pkg["system"] - if is_large_pkg(pkg): - # Use larger runners for large packages for x86_64-linux and aarch64-linux - if system == "x86_64-linux": - return {"labels": ["blacksmith-32vcpu-ubuntu-2404"]} - elif system == "aarch64-linux": - return {"labels": ["blacksmith-32vcpu-ubuntu-2404-arm"]} - if system in BUILD_RUNNER_MAP: - return BUILD_RUNNER_MAP[system] - else: - raise ValueError(f"No runner configuration for system: {system}") + + if is_kvm_pkg(pkg): + return BUILD_RUNNER_MAP["self-hosted"].get(system) + + if is_large_pkg(pkg) and system in ("x86_64-linux", "aarch64-linux"): + suffix = "-arm" if system == "aarch64-linux" else "" + return {"labels": [f"blacksmith-32vcpu-ubuntu-2404{suffix}"]} + + if system == "aarch64-darwin": + return BUILD_RUNNER_MAP["self-hosted"]["aarch64-darwin"] + + return BUILD_RUNNER_MAP["ephemeral"].get(system) def main() -> None: @@ -204,11 +229,14 @@ def main() -> None: def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: """Convert nix-eval-jobs output to GitHub Actions matrix package""" + runner = get_runner_for_package(pkg) + if runner is None: + raise ValueError(f"No runner configuration for system: {pkg['system']}") returned_pkg: GitHubActionPackage = { "attr": pkg["attr"], "name": pkg["name"], "system": pkg["system"], - "runs_on": get_runner_for_package(pkg), + "runs_on": runner, } if is_extension_pkg(pkg): # Extract PostgreSQL version from attribute path diff --git a/nix/packages/github-matrix/tests/test_github_matrix.py b/nix/packages/github-matrix/tests/test_github_matrix.py new file mode 100644 index 000000000..0e5069fdf --- /dev/null +++ b/nix/packages/github-matrix/tests/test_github_matrix.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 + +import pytest + +from github_matrix import ( + NixEvalJobsOutput, + get_runner_for_package, + is_extension_pkg, + is_kvm_pkg, + is_large_pkg, + sort_pkgs_by_closures, +) + + +class TestIsExtensionPkg: + def test_extension_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15.exts.pg_cron", + "attrPath": ["packages", "x86_64-linux", "psql_15", "exts", "pg_cron"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "pg_cron", + "system": "x86_64-linux", + } + assert is_extension_pkg(pkg) is True + + def test_non_extension_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15", + "attrPath": ["packages", "x86_64-linux", "psql_15"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgresql-16.0", + "system": "x86_64-linux", + } + assert is_extension_pkg(pkg) is False + + +class TestIsLargePkg: + @pytest.mark.parametrize( + "attr,expected", + [ + ("packages.x86_64-linux.psql_15.exts.wrappers", True), + ("packages.x86_64-linux.psql_15.exts.pg_jsonschema", True), + ("packages.x86_64-linux.psql_15.exts.pg_graphql", True), + ("packages.x86_64-linux.psql_15.exts.postgis", True), + ("packages.x86_64-linux.psql_15.exts.pg_cron", False), + ("packages.x86_64-linux.psql_15", False), + ], + ) + def test_large_package_detection(self, attr: str, expected: bool): + pkg: NixEvalJobsOutput = { + "attr": attr, + "attrPath": attr.split("."), + "cacheStatus": "notBuilt", + "drvPath": f"/nix/store/{attr}.drv", + "name": attr.split(".")[-1], + "system": "x86_64-linux", + } + assert is_large_pkg(pkg) is expected + + +class TestIsKvmPkg: + def test_kvm_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.vm-test", + "attrPath": ["packages", "x86_64-linux", "vm-test"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "vm-test", + "system": "x86_64-linux", + "requiredSystemFeatures": ["kvm"], + } + assert is_kvm_pkg(pkg) is True + + def test_non_kvm_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15", + "attrPath": ["packages", "x86_64-linux", "psql_15"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgresql-16.0", + "system": "x86_64-linux", + } + assert is_kvm_pkg(pkg) is False + + +class TestGetRunnerForPackage: + def test_kvm_package_x86_64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.vm-test", + "attrPath": ["packages", "x86_64-linux", "vm-test"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "vm-test", + "system": "x86_64-linux", + "requiredSystemFeatures": ["kvm"], + } + result = get_runner_for_package(pkg) + assert result is None # x86_64-linux not in self-hosted map + + def test_kvm_package_aarch64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.aarch64-linux.vm-test", + "attrPath": ["packages", "aarch64-linux", "vm-test"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "vm-test", + "system": "aarch64-linux", + "requiredSystemFeatures": ["kvm"], + } + result = get_runner_for_package(pkg) + assert result == { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-linux"], + } + + def test_large_package_x86_64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15.exts.postgis", + "attrPath": ["packages", "x86_64-linux", "psql_15", "exts", "postgis"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgis", + "system": "x86_64-linux", + } + result = get_runner_for_package(pkg) + assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404"]} + + def test_large_package_aarch64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.aarch64-linux.psql_15.exts.pg_graphql", + "attrPath": ["packages", "aarch64-linux", "psql_15", "exts", "pg_graphql"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "pg_graphql", + "system": "aarch64-linux", + } + result = get_runner_for_package(pkg) + assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404-arm"]} + + def test_darwin_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.aarch64-darwin.psql_15", + "attrPath": ["packages", "aarch64-darwin", "psql_15"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgresql-16.0", + "system": "aarch64-darwin", + } + result = get_runner_for_package(pkg) + assert result == { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-darwin"], + } + + def test_default_x86_64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15.exts.pg_cron", + "attrPath": ["packages", "x86_64-linux", "psql_15", "exts", "pg_cron"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "pg_cron", + "system": "x86_64-linux", + } + result = get_runner_for_package(pkg) + assert result == {"labels": ["blacksmith-8vcpu-ubuntu-2404"]} + + def test_default_aarch64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.aarch64-linux.psql_15.exts.pg_cron", + "attrPath": ["packages", "aarch64-linux", "psql_15", "exts", "pg_cron"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "pg_cron", + "system": "aarch64-linux", + } + result = get_runner_for_package(pkg) + assert result == {"labels": ["blacksmith-8vcpu-ubuntu-2404-arm"]} + + +class TestSortPkgsByClosures: + def test_empty_list(self): + result = sort_pkgs_by_closures([]) + assert result == [] + + def test_single_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15", + "attrPath": ["packages", "x86_64-linux", "psql_15"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgresql-16.0", + "system": "x86_64-linux", + } + result = sort_pkgs_by_closures([pkg]) + assert result == [pkg] + + def test_dependency_order(self): + pkg1: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.lib", + "attrPath": ["packages", "x86_64-linux", "lib"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/lib.drv", + "name": "lib", + "system": "x86_64-linux", + "neededBuilds": [], + "neededSubstitutes": [], + } + pkg2: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.app", + "attrPath": ["packages", "x86_64-linux", "app"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/app.drv", + "name": "app", + "system": "x86_64-linux", + "neededBuilds": ["/nix/store/lib.drv"], + "neededSubstitutes": [], + } + + # Regardless of input order, lib should come before app + result = sort_pkgs_by_closures([pkg2, pkg1]) + assert result == [pkg1, pkg2] + + result = sort_pkgs_by_closures([pkg1, pkg2]) + assert result == [pkg1, pkg2] From 5672ff9ad93b4f2e09bcc5a36dd5964dd9ca9bb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 21:32:36 +0200 Subject: [PATCH 38/71] fix: configure runner according to the matrix job The matrix job returns the type of runner, so we can configure the nix installation step accordingly. --- .github/workflows/nix-build.yml | 11 +++++++++-- .github/workflows/nix-eval.yml | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 0c3e78bc2..f326a29d3 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -35,7 +35,14 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@v4 - - name: Build Nix Package + - name: Install nix (ephemeral) + if: ${{ matrix.runs_on.group != 'self-hosted-runners-nix' }} + uses: ./.github/actions/nix-install-ephemeral + env: + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} + NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} + - name: Install nix (self-hosted) + if: ${{ matrix.runs_on.group == 'self-hosted-runners-nix' }} uses: ./.github/actions/nix-install-self-hosted - name: nix build shell: bash @@ -55,7 +62,7 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@v4 - - name: Build Nix Package + - name: Install nix uses: ./.github/actions/nix-install-self-hosted - name: nix build shell: bash diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml index e97fe9171..ada3ec844 100644 --- a/.github/workflows/nix-eval.yml +++ b/.github/workflows/nix-eval.yml @@ -29,4 +29,4 @@ jobs: name: Generate Nix Matrix run: | set -Eeu - echo matrix="$(nix run .\#github-matrix checks legacyPackages)" >> "$GITHUB_OUTPUT" + echo matrix="$(nix run --accept-flake-config .\#github-matrix checks legacyPackages)" >> "$GITHUB_OUTPUT" From 6c1aa191f282890c2af289f95f99e5f3997ce88f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Wed, 15 Oct 2025 00:57:43 +0200 Subject: [PATCH 39/71] Update nix-eval-jobs Our changes were merged upstream, so we can now track the original repository again. --- flake.lock | 35 ++++++++++----------- flake.nix | 2 +- nix/packages/github-matrix/github_matrix.py | 1 - 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/flake.lock b/flake.lock index 19db22fe0..a28edf942 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ ] }, "locked": { - "lastModified": 1741352980, - "narHash": "sha256-+u2UunDA4Cl5Fci3m7S643HzKmIDAe+fiXrLqYsR2fs=", + "lastModified": 1759362264, + "narHash": "sha256-wfG0S7pltlYyZTM+qqlhJ7GMw2fTF4mLKCIVhLii/4M=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "f4330d22f1c5d2ba72d3d22df5597d123fdb60a9", + "rev": "758cf7296bee11f1706a574c77d072b8a7baa881", "type": "github" }, "original": { @@ -119,11 +119,11 @@ "nix": { "flake": false, "locked": { - "lastModified": 1759793966, - "narHash": "sha256-h95Lz2j19/Sb9nuBM3BXUqlJgBkIZd15/QTkufajaLY=", + "lastModified": 1760472641, + "narHash": "sha256-BuKtM7Vr5EcxBXxUENBQPlOBwmNd5mkTRkSmlJi/iQ4=", "owner": "NixOS", "repo": "nix", - "rev": "eea6d75783e10e6057a097af29741c9a47cbb7e0", + "rev": "4041bfdb401ad6d1c31a292fab90392254be506a", "type": "github" }, "original": { @@ -189,16 +189,15 @@ "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1760477819, - "narHash": "sha256-/hfZp5kjN2ratoMtv5JTvekLBUcaXu2i7JgqbNOsctw=", - "owner": "jfroche", + "lastModified": 1760478325, + "narHash": "sha256-hA+NOH8KDcsuvH7vJqSwk74PyZP3MtvI/l+CggZcnTc=", + "owner": "nix-community", "repo": "nix-eval-jobs", - "rev": "1fe1983db41fbb3e148e8eb26b7ed4cb6f4cf49e", + "rev": "daa42f9e9c84aeff1e325dd50fda321f53dfd02c", "type": "github" }, "original": { - "owner": "jfroche", - "ref": "show-required-system-features", + "owner": "nix-community", "repo": "nix-eval-jobs", "type": "github" } @@ -292,10 +291,10 @@ "nixpkgs_2": { "locked": { "lastModified": 315532800, - "narHash": "sha256-yDxtm0PESdgNetiJN5+MFxgubBcLDTiuSjjrJiyvsvM=", - "rev": "d7f52a7a640bc54c7bb414cca603835bf8dd4b10", + "narHash": "sha256-vhAtaRMIQiEghARviANBmSnhGz9Qf2IQJ+nQgsDXnVs=", + "rev": "c12c63cd6c5eb34c7b4c3076c6a99e00fcab86ec", "type": "tarball", - "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre871443.d7f52a7a640b/nixexprs.tar.xz" + "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre877036.c12c63cd6c5e/nixexprs.tar.xz" }, "original": { "type": "tarball", @@ -408,11 +407,11 @@ ] }, "locked": { - "lastModified": 1752055615, - "narHash": "sha256-19m7P4O/Aw/6+CzncWMAJu89JaKeMh3aMle1CNQSIwM=", + "lastModified": 1760120816, + "narHash": "sha256-gq9rdocpmRZCwLS5vsHozwB6b5nrOBDNc2kkEaTXHfg=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "c9d477b5d5bd7f26adddd3f96cfd6a904768d4f9", + "rev": "761ae7aff00907b607125b2f57338b74177697ed", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index aa1539553..efd0b7e9e 100644 --- a/flake.nix +++ b/flake.nix @@ -28,7 +28,7 @@ git-hooks.inputs.nixpkgs.follows = "nixpkgs"; nixpkgs-go124.url = "github:Nixos/nixpkgs/d2ac4dfa61fba987a84a0a81555da57ae0b9a2b0"; nixpkgs-pgbackrest.url = "github:nixos/nixpkgs/nixos-unstable-small"; - nix-eval-jobs.url = "github:jfroche/nix-eval-jobs/show-required-system-features"; + nix-eval-jobs.url = "github:nix-community/nix-eval-jobs"; }; outputs = diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 2c4681670..22caa7d1c 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -86,7 +86,6 @@ def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[s "--check-cache-status", "--force-recurse", "--quiet", - "--show-required-system-features", "--option", "eval-cache", "false", From dfa3e75e57f3ffe0e5fb5c0c2f00e6dd4242233b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 27 Oct 2025 16:23:37 +0100 Subject: [PATCH 40/71] refactor(ci): standardize nix installation and disable cache push by default - Replace DeterminateSystems/nix-installer-action with custom nix-install-ephemeral action across all workflows - Change default push-to-cache from 'true' to 'false' to prevent unnecessary nix/aws configurations - Explicitly enable push-to-cache only for nix-build and nix-eval workflows where caching is beneficial --- .github/workflows/nix-build.yml | 4 ++++ .github/workflows/nix-eval.yml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index f326a29d3..c5eb06d84 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -38,6 +38,8 @@ jobs: - name: Install nix (ephemeral) if: ${{ matrix.runs_on.group != 'self-hosted-runners-nix' }} uses: ./.github/actions/nix-install-ephemeral + with: + push-to-cache: 'true' env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} @@ -84,6 +86,8 @@ jobs: uses: actions/checkout@v4 - name: Install nix uses: ./.github/actions/nix-install-ephemeral + with: + push-to-cache: 'true' env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml index ada3ec844..a899e2371 100644 --- a/.github/workflows/nix-eval.yml +++ b/.github/workflows/nix-eval.yml @@ -22,6 +22,8 @@ jobs: uses: actions/checkout@v4 - name: Install nix uses: ./.github/actions/nix-install-ephemeral + with: + push-to-cache: 'true' env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} From 7f185c39799bc53092e37332ee2f83def4313e58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 27 Oct 2025 18:02:20 +0100 Subject: [PATCH 41/71] feat: use big-parallel to identify large packages --- nix/ext/pg_graphql/default.nix | 7 +++++-- nix/ext/pg_jsonschema/default.nix | 7 +++++-- nix/ext/postgis.nix | 7 +++++-- nix/ext/wrappers/default.nix | 7 +++++-- nix/packages/github-matrix/github_matrix.py | 14 ++++++++------ .../github-matrix/tests/test_github_matrix.py | 10 ++++++++-- 6 files changed, 36 insertions(+), 16 deletions(-) diff --git a/nix/ext/pg_graphql/default.nix b/nix/ext/pg_graphql/default.nix index d944d5ede..a63fcd84e 100644 --- a/nix/ext/pg_graphql/default.nix +++ b/nix/ext/pg_graphql/default.nix @@ -129,7 +129,7 @@ let lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions ); in -buildEnv { +(buildEnv { name = pname; paths = packages; pathsToLink = [ @@ -177,4 +177,7 @@ buildEnv { version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; -} +}).overrideAttrs + (_: { + requiredSystemFeatures = [ "big-parallel" ]; + }) diff --git a/nix/ext/pg_jsonschema/default.nix b/nix/ext/pg_jsonschema/default.nix index 74742bed5..9939b733a 100644 --- a/nix/ext/pg_jsonschema/default.nix +++ b/nix/ext/pg_jsonschema/default.nix @@ -130,7 +130,7 @@ let lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions ); in -pkgs.buildEnv { +(pkgs.buildEnv { name = pname; paths = packages; pathsToLink = [ @@ -170,4 +170,7 @@ pkgs.buildEnv { version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; -} +}).overrideAttrs + (_: { + requiredSystemFeatures = [ "big-parallel" ]; + }) diff --git a/nix/ext/postgis.nix b/nix/ext/postgis.nix index ed1b738e4..e5b19ccfb 100644 --- a/nix/ext/postgis.nix +++ b/nix/ext/postgis.nix @@ -179,7 +179,7 @@ let }; }; in -buildEnv { +(buildEnv { name = pname; paths = packages; @@ -209,4 +209,7 @@ buildEnv { version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; -} +}).overrideAttrs + (_: { + requiredSystemFeatures = [ "big-parallel" ]; + }) diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index b37980844..dd06834c9 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -224,7 +224,7 @@ let lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions ); in -buildEnv { +(buildEnv { name = pname; paths = packages; pathsToLink = [ @@ -309,4 +309,7 @@ buildEnv { version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; -} +}).overrideAttrs + (_: { + requiredSystemFeatures = [ "big-parallel" ]; + }) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 22caa7d1c..324459706 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -36,6 +36,7 @@ class NixEvalJobsOutput(TypedDict): neededSubstitutes: NotRequired[List[Any]] outputs: NotRequired[Dict[str, str]] error: NotRequired[str] + requiredSystemFeatures: NotRequired[List[str]] class RunsOnConfig(TypedDict): @@ -174,11 +175,7 @@ def sort_pkgs_by_closures(jobs: List[NixEvalJobsOutput]) -> List[NixEvalJobsOutp def is_large_pkg(pkg: NixEvalJobsOutput) -> bool: """Determine if a package is considered large based on its attribute path.""" - RUST_EXTENSIONS = ["exts.wrappers", "exts.pg_jsonschema", "exts.pg_graphql"] - LARGE_C_EXTENSION = ["exts.postgis"] - return any( - indicator in pkg["attr"] for indicator in RUST_EXTENSIONS + LARGE_C_EXTENSION - ) + return "big-parallel" in pkg.get("requiredSystemFeatures", []) def is_kvm_pkg(pkg: NixEvalJobsOutput) -> bool: @@ -198,7 +195,12 @@ def get_runner_for_package(pkg: NixEvalJobsOutput) -> RunsOnConfig | None: system = pkg["system"] if is_kvm_pkg(pkg): - return BUILD_RUNNER_MAP["self-hosted"].get(system) + runConfig = BUILD_RUNNER_MAP["self-hosted"].get(system) + if runConfig is None: + raise ValueError( + f"No self-hosted with kvm support available for system: {system}" + ) + return runConfig if is_large_pkg(pkg) and system in ("x86_64-linux", "aarch64-linux"): suffix = "-arm" if system == "aarch64-linux" else "" diff --git a/nix/packages/github-matrix/tests/test_github_matrix.py b/nix/packages/github-matrix/tests/test_github_matrix.py index 0e5069fdf..3c8c9188d 100644 --- a/nix/packages/github-matrix/tests/test_github_matrix.py +++ b/nix/packages/github-matrix/tests/test_github_matrix.py @@ -56,6 +56,7 @@ def test_large_package_detection(self, attr: str, expected: bool): "drvPath": f"/nix/store/{attr}.drv", "name": attr.split(".")[-1], "system": "x86_64-linux", + "requiredSystemFeatures": ["big-parallel"] if expected else [], } assert is_large_pkg(pkg) is expected @@ -96,8 +97,11 @@ def test_kvm_package_x86_64_linux(self): "system": "x86_64-linux", "requiredSystemFeatures": ["kvm"], } - result = get_runner_for_package(pkg) - assert result is None # x86_64-linux not in self-hosted map + with pytest.raises( + ValueError, + match=r"No self-hosted with kvm support available for system: x86_64-linux", + ): + get_runner_for_package(pkg) def test_kvm_package_aarch64_linux(self): pkg: NixEvalJobsOutput = { @@ -123,6 +127,7 @@ def test_large_package_x86_64_linux(self): "drvPath": "/nix/store/test.drv", "name": "postgis", "system": "x86_64-linux", + "requiredSystemFeatures": ["big-parallel"], } result = get_runner_for_package(pkg) assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404"]} @@ -135,6 +140,7 @@ def test_large_package_aarch64_linux(self): "drvPath": "/nix/store/test.drv", "name": "pg_graphql", "system": "aarch64-linux", + "requiredSystemFeatures": ["big-parallel"], } result = get_runner_for_package(pkg) assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404-arm"]} From 6733d10b0f379dcd7e38293fbea6ce8fa65799f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 27 Oct 2025 18:02:20 +0100 Subject: [PATCH 42/71] fix(ci): ensure x86_64-linux build is considered in testinfra and test workflows --- .github/workflows/nix-build.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index c5eb06d84..da96d336a 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -103,7 +103,8 @@ jobs: if: | !cancelled() && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && - (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') + (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') && + (needs.nix-build-x86_64-linux.result == 'skipped' || needs.nix-build-x86_64-linux.result == 'success') uses: ./.github/workflows/testinfra-ami-build.yml secrets: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} @@ -113,5 +114,6 @@ jobs: if: | !cancelled() && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && - (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') + (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') && + (needs.nix-build-x86_64-linux.result == 'skipped' || needs.nix-build-x86_64-linux.result == 'success') uses: ./.github/workflows/test.yml From 3ad21bfca290fd744f3c28673aaad2735b780c1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 27 Oct 2025 18:02:20 +0100 Subject: [PATCH 43/71] fix: nix devShell inclusion condition --- nix/checks.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/checks.nix b/nix/checks.nix index 7558b92ce..cf0fe4c2f 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -411,7 +411,7 @@ inherit self; inherit pkgs; }) - // pkgs.lib.optionalAttrs (system == "x86_64-linux") { devShell = self'.devShells.default; } - ); + ) + // pkgs.lib.optionalAttrs (system == "x86_64-linux") ({ devShell = self'.devShells.default; }); }; } From 8a4dcf9e75d1217b9bdf57bde67e963f330863fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Wed, 29 Oct 2025 23:53:08 +0100 Subject: [PATCH 44/71] fix(ci): eval should fail if github-matrix run fails --- .github/workflows/nix-eval.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml index a899e2371..8f5776ebd 100644 --- a/.github/workflows/nix-eval.yml +++ b/.github/workflows/nix-eval.yml @@ -30,5 +30,5 @@ jobs: - id: set-matrix name: Generate Nix Matrix run: | - set -Eeu - echo matrix="$(nix run --accept-flake-config .\#github-matrix checks legacyPackages)" >> "$GITHUB_OUTPUT" + set -Eeu -o pipefail + echo matrix="$(nix run --accept-flake-config .\#github-matrix -- checks legacyPackages)" >> "$GITHUB_OUTPUT" From d2709bcb7d5e628448d3360ff5944356ebed1a9c Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Fri, 14 Nov 2025 10:54:26 +0100 Subject: [PATCH 45/71] fix(ci): remove redundant build psql bundle step --- .github/workflows/nix-build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index da96d336a..0f8331f7c 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -91,9 +91,6 @@ jobs: env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - - name: Build psql bundle - run: > - nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - name: nix build shell: bash run: nix build --accept-flake-config -L .#${{ matrix.attr }} From b0c8818688ad77b0978090e8ba831d7c3a79a6e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 14 Nov 2025 15:39:15 +0100 Subject: [PATCH 46/71] fix: reduce ARM runner size from 8vcpu to 4vcpu for ephemeral builds We might not need the full 8vcpu for aarch64-linux builds, so this change reduces the runner size to 4vcpu to wait less for available blacksmith runners. --- nix/packages/github-matrix/github_matrix.py | 2 +- nix/packages/github-matrix/tests/test_github_matrix.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 324459706..68daa0edf 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -59,7 +59,7 @@ class GitHubActionPackage(TypedDict): BUILD_RUNNER_MAP: Dict[RunnerType, Dict[System, RunsOnConfig]] = { "ephemeral": { "aarch64-linux": { - "labels": ["blacksmith-8vcpu-ubuntu-2404-arm"], + "labels": ["blacksmith-4vcpu-ubuntu-2404-arm"], }, "x86_64-linux": { "labels": ["blacksmith-8vcpu-ubuntu-2404"], diff --git a/nix/packages/github-matrix/tests/test_github_matrix.py b/nix/packages/github-matrix/tests/test_github_matrix.py index 3c8c9188d..d5fed5732 100644 --- a/nix/packages/github-matrix/tests/test_github_matrix.py +++ b/nix/packages/github-matrix/tests/test_github_matrix.py @@ -182,7 +182,7 @@ def test_default_aarch64_linux(self): "system": "aarch64-linux", } result = get_runner_for_package(pkg) - assert result == {"labels": ["blacksmith-8vcpu-ubuntu-2404-arm"]} + assert result == {"labels": ["blacksmith-4vcpu-ubuntu-2404-arm"]} class TestSortPkgsByClosures: From 19e461b2c42ba67c76d249145d8998e16c1b0925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 24 Nov 2025 16:24:48 +0100 Subject: [PATCH 47/71] feat: do not return empty matrices if no package has to be built --- .github/workflows/nix-build.yml | 16 ++++++++++++---- nix/packages/github-matrix/github_matrix.py | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 0f8331f7c..379488878 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -34,9 +34,10 @@ jobs: matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_linux }} steps: - name: Checkout Repo + if: ${{ matrix.attr != '' }} uses: actions/checkout@v4 - name: Install nix (ephemeral) - if: ${{ matrix.runs_on.group != 'self-hosted-runners-nix' }} + if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} uses: ./.github/actions/nix-install-ephemeral with: push-to-cache: 'true' @@ -44,9 +45,10 @@ jobs: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - name: Install nix (self-hosted) - if: ${{ matrix.runs_on.group == 'self-hosted-runners-nix' }} + if: ${{ matrix.attr != '' && matrix.runs_on.group == 'self-hosted-runners-nix' }} uses: ./.github/actions/nix-install-self-hosted - name: nix build + if: ${{ matrix.attr != '' }} shell: bash run: nix build --accept-flake-config -L .#${{ matrix.attr }} @@ -55,7 +57,7 @@ jobs: ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} (aarch64-darwin) needs: nix-eval - runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin != null }} strategy: fail-fast: false @@ -63,10 +65,13 @@ jobs: matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin }} steps: - name: Checkout Repo + if: ${{ matrix.attr != '' }} uses: actions/checkout@v4 - name: Install nix + if: ${{ matrix.attr != '' }} uses: ./.github/actions/nix-install-self-hosted - name: nix build + if: ${{ matrix.attr != '' }} shell: bash run: nix build --accept-flake-config -L .#${{ matrix.attr }} @@ -75,7 +80,7 @@ jobs: ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} (x86_64-linux) needs: nix-eval - runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux != null }} strategy: fail-fast: false @@ -83,8 +88,10 @@ jobs: matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux }} steps: - name: Checkout Repo + if: ${{ matrix.attr != '' }} uses: actions/checkout@v4 - name: Install nix + if: ${{ matrix.attr != '' }} uses: ./.github/actions/nix-install-ephemeral with: push-to-cache: 'true' @@ -92,6 +99,7 @@ jobs: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - name: nix build + if: ${{ matrix.attr != '' }} shell: bash run: nix build --accept-flake-config -L .#${{ matrix.attr }} diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 68daa0edf..e691dc2be 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -17,6 +17,7 @@ Optional, Set, TypedDict, + get_args, ) System = Literal["x86_64-linux", "aarch64-linux", "aarch64-darwin"] @@ -252,10 +253,23 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) # Create output with system-specific matrices + # Ensure that we have at least one entry per system gh_output = {} for system, packages in grouped_by_system.items(): gh_output[system.replace("-", "_")] = {"include": packages} + for system in get_args(System): + if system not in gh_output: + gh_output[system.replace("-", "_")] = { + "include": [ + { + "attr": "", + "name": "skipped", + "system": system, + "runs_on": {"labels": "ubuntu-latest"}, + } + ] + } print( f"debug: Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}", file=sys.stderr, From 2fa06221b34274c0c9d07fcb9dbaebcf5a7c0916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 24 Nov 2025 16:24:48 +0100 Subject: [PATCH 48/71] feat: fail pipeline if nix evaluation fails --- .github/workflows/nix-eval.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml index 8f5776ebd..ca3f98a5e 100644 --- a/.github/workflows/nix-eval.yml +++ b/.github/workflows/nix-eval.yml @@ -31,4 +31,5 @@ jobs: name: Generate Nix Matrix run: | set -Eeu -o pipefail - echo matrix="$(nix run --accept-flake-config .\#github-matrix -- checks legacyPackages)" >> "$GITHUB_OUTPUT" + result=$(nix run --accept-flake-config .\#github-matrix -- checks legacyPackages) + echo matrix="$result" >> "$GITHUB_OUTPUT" From 33662eabf7c63f3b2c41974a531d382f77824207 Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Mon, 24 Nov 2025 23:01:17 +0100 Subject: [PATCH 49/71] Update nix/ext/pgvector.nix Co-authored-by: samrose --- nix/ext/pgvector.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index 74571890e..b8f8347a3 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -69,7 +69,6 @@ let meta = with lib; { description = "Open-source vector similarity search for Postgres"; homepage = "https://github.com/${src.owner}/${src.repo}"; - maintainers = [ "olirice" ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; From 608ada685f05bdc0cadaae8c40140ac5d17447fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 25 Nov 2025 18:44:09 +0100 Subject: [PATCH 50/71] fix: add skip job only for systems that don't have any job --- nix/packages/github-matrix/github_matrix.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index e691dc2be..24c97ab74 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -259,8 +259,9 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: gh_output[system.replace("-", "_")] = {"include": packages} for system in get_args(System): - if system not in gh_output: - gh_output[system.replace("-", "_")] = { + s = system.replace("-", "_") + if s not in gh_output: + gh_output[s] = { "include": [ { "attr": "", From ef309621b449bb8f1e4994af762195c213bc49fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 25 Nov 2025 20:47:29 +0100 Subject: [PATCH 51/71] fix(github-matrix): handle evaluation errors without deadlock Fix github-matrix that would hang when nix-eval-jobs encountered errors due to subprocess pipe deadlock - stderr buffer would fill while reading stdout. This change ensure that evaluation errors are visible and the workflow fails properly while still showing which packages succeeded. --- .github/workflows/nix-eval.yml | 4 ++ nix/packages/github-matrix/github_matrix.py | 44 +++++++++++++++------ 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml index ca3f98a5e..adf74eaa1 100644 --- a/.github/workflows/nix-eval.yml +++ b/.github/workflows/nix-eval.yml @@ -32,4 +32,8 @@ jobs: run: | set -Eeu -o pipefail result=$(nix run --accept-flake-config .\#github-matrix -- checks legacyPackages) + if [ -z "$result" ]; then + echo "Error: github-matrix returned empty output" >&2 + exit 1 + fi echo matrix="$result" >> "$GITHUB_OUTPUT" diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 24c97ab74..ba3c35410 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -102,7 +102,9 @@ def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[s return nix_eval_cmd -def parse_nix_eval_line(line: str, drv_paths: Set[str]) -> Optional[NixEvalJobsOutput]: +def parse_nix_eval_line( + line: str, drv_paths: Set[str], errors: List[str] +) -> Optional[NixEvalJobsOutput]: """Parse a single line of nix-eval-jobs output""" if not line.strip(): return None @@ -110,38 +112,44 @@ def parse_nix_eval_line(line: str, drv_paths: Set[str]) -> Optional[NixEvalJobsO try: data: NixEvalJobsOutput = json.loads(line) if "error" in data: - raise ValueError( + error_msg = ( f"Error in nix-eval-jobs output for {data['attr']}: {data['error']}" ) + errors.append(error_msg) + return None if data["drvPath"] in drv_paths: return None drv_paths.add(data["drvPath"]) return data except json.JSONDecodeError: - print(f"Skipping invalid JSON line: {line}", file=sys.stderr) + error_msg = f"Skipping invalid JSON line: {line}" + print(error_msg, file=sys.stderr) + errors.append(error_msg) return None -def run_nix_eval_jobs(cmd: List[str]) -> Generator[NixEvalJobsOutput, None, None]: +def run_nix_eval_jobs( + cmd: List[str], errors: List[str] +) -> Generator[NixEvalJobsOutput, None, None]: """Run nix-eval-jobs and yield parsed package data.""" print(f"Running command: {' '.join(cmd)}", file=sys.stderr) with subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + cmd, stdout=subprocess.PIPE, stderr=None, text=True ) as process: drv_paths: Set[str] = set() assert process.stdout is not None # for mypy - assert process.stderr is not None # for mypy for line in process.stdout: - package = parse_nix_eval_line(line, drv_paths) + package = parse_nix_eval_line(line, drv_paths, errors) if package: yield package process.wait() if process.returncode != 0: - print("Error: Evaluation failed", file=sys.stderr) - sys.stderr.write(process.stderr.read()) - sys.exit(process.returncode) + error_msg = "Error: nix-eval-jobs process failed with non-zero exit code" + print(error_msg, file=sys.stderr) + errors.append(error_msg) + # Don't exit here - let main() handle it after reporting all errors def is_extension_pkg(pkg: NixEvalJobsOutput) -> bool: @@ -227,7 +235,9 @@ def main() -> None: cmd = build_nix_eval_command(max_workers, args.flake_outputs) - gh_action_packages = sort_pkgs_by_closures(list(run_nix_eval_jobs(cmd))) + # Collect all evaluation errors + errors: List[str] = [] + gh_action_packages = sort_pkgs_by_closures(list(run_nix_eval_jobs(cmd, errors))) def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: """Convert nix-eval-jobs output to GitHub Actions matrix package""" @@ -277,6 +287,18 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: ) print(json.dumps(gh_output)) + # Check if any errors occurred during evaluation + if errors: + print("\n=== Evaluation Errors ===", file=sys.stderr) + for i, error in enumerate(errors, 1): + print(f"\nError {i}:", file=sys.stderr) + print(error, file=sys.stderr) + print( + f"\n=== Total: {len(errors)} error(s) occurred during evaluation ===", + file=sys.stderr, + ) + sys.exit(1) + if __name__ == "__main__": main() From e513eb8a9d3491c73eac871d570612d54bde44d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 25 Nov 2025 21:47:43 +0100 Subject: [PATCH 52/71] feat(github-matrix): integrate github-action-utils for better error visibility Integrates github-action-utils library to improve error and warning visibility in GitHub Actions UI through workflow command annotations. --- .github/workflows/nix-eval.yml | 7 +- nix/packages/github-matrix/default.nix | 24 ++++ nix/packages/github-matrix/github_matrix.py | 137 ++++++++++++-------- 3 files changed, 107 insertions(+), 61 deletions(-) diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml index adf74eaa1..cc092b355 100644 --- a/.github/workflows/nix-eval.yml +++ b/.github/workflows/nix-eval.yml @@ -31,9 +31,4 @@ jobs: name: Generate Nix Matrix run: | set -Eeu -o pipefail - result=$(nix run --accept-flake-config .\#github-matrix -- checks legacyPackages) - if [ -z "$result" ]; then - echo "Error: github-matrix returned empty output" >&2 - exit 1 - fi - echo matrix="$result" >> "$GITHUB_OUTPUT" + nix run --accept-flake-config .\#github-matrix -- checks legacyPackages diff --git a/nix/packages/github-matrix/default.nix b/nix/packages/github-matrix/default.nix index 0d755c977..3e1be2df3 100644 --- a/nix/packages/github-matrix/default.nix +++ b/nix/packages/github-matrix/default.nix @@ -5,6 +5,25 @@ }: let pname = "github-matrix"; + + github-action-utils = python3Packages.buildPythonPackage rec { + pname = "github-action-utils"; + version = "1.1.0"; + pyproject = true; + + src = python3Packages.fetchPypi { + inherit pname version; + sha256 = "0q9xrb4jcvbn6954lvpn85gva1yc885ykdqb2q2410cxp280v94a"; + }; + + build-system = with python3Packages; [ setuptools ]; + + meta = with lib; { + description = "Collection of Python functions for GitHub Action Workflow Commands"; + homepage = "https://github.com/saadmk11/github-action-utils"; + license = licenses.mit; + }; + }; in python3Packages.buildPythonApplication { @@ -14,6 +33,11 @@ python3Packages.buildPythonApplication { src = ./.; + propagatedBuildInputs = [ + github-action-utils + python3Packages.result + ]; + makeWrapperArgs = [ "--suffix PATH : ${lib.makeBinPath [ nix-eval-jobs ]}" ]; nativeCheckInputs = with python3Packages; [ diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index ba3c35410..deac5bc1d 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import argparse -from collections import defaultdict +from collections import Counter, defaultdict import graphlib import json import os @@ -10,16 +10,19 @@ from typing import ( Any, Dict, - Generator, List, Literal, NotRequired, Optional, Set, + Tuple, TypedDict, get_args, ) +from github_action_utils import debug, error, set_output, warning +from result import Err, Ok, Result + System = Literal["x86_64-linux", "aarch64-linux", "aarch64-darwin"] RunnerType = Literal["ephemeral", "self-hosted"] @@ -103,11 +106,16 @@ def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[s def parse_nix_eval_line( - line: str, drv_paths: Set[str], errors: List[str] -) -> Optional[NixEvalJobsOutput]: - """Parse a single line of nix-eval-jobs output""" + line: str, drv_paths: Set[str] +) -> Result[Optional[NixEvalJobsOutput], str]: + """Parse a single line of nix-eval-jobs output. + + Returns: + Ok(package_data) if successful (None for empty/duplicate lines) + Err(error_message) if a nix evaluation error occurred + """ if not line.strip(): - return None + return Ok(None) try: data: NixEvalJobsOutput = json.loads(line) @@ -115,41 +123,58 @@ def parse_nix_eval_line( error_msg = ( f"Error in nix-eval-jobs output for {data['attr']}: {data['error']}" ) - errors.append(error_msg) - return None + error(error_msg, title="Nix Evaluation Error") + return Err(error_msg) if data["drvPath"] in drv_paths: - return None + return Ok(None) drv_paths.add(data["drvPath"]) - return data - except json.JSONDecodeError: - error_msg = f"Skipping invalid JSON line: {line}" - print(error_msg, file=sys.stderr) - errors.append(error_msg) - return None + return Ok(data) + except json.JSONDecodeError as e: + warning(f"Skipping invalid JSON line: {line}", title="JSON Parse Warning") + return Ok(None) def run_nix_eval_jobs( - cmd: List[str], errors: List[str] -) -> Generator[NixEvalJobsOutput, None, None]: - """Run nix-eval-jobs and yield parsed package data.""" - print(f"Running command: {' '.join(cmd)}", file=sys.stderr) - - with subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=None, text=True - ) as process: - drv_paths: Set[str] = set() - assert process.stdout is not None # for mypy - for line in process.stdout: - package = parse_nix_eval_line(line, drv_paths, errors) - if package: - yield package - - process.wait() - if process.returncode != 0: - error_msg = "Error: nix-eval-jobs process failed with non-zero exit code" - print(error_msg, file=sys.stderr) - errors.append(error_msg) - # Don't exit here - let main() handle it after reporting all errors + cmd: List[str], +) -> Tuple[List[NixEvalJobsOutput], List[str], bool]: + """Run nix-eval-jobs and return parsed package data, warnings, and error status. + + Returns: + Tuple of (packages, warnings_list, had_errors) + """ + debug(f"Running command: {' '.join(cmd)}") + + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + stdout_data, stderr_data = process.communicate() + + # Parse stdout for packages + packages: List[NixEvalJobsOutput] = [] + drv_paths: Set[str] = set() + had_errors = False + for line in stdout_data.splitlines(): + result = parse_nix_eval_line(line, drv_paths) + if result.is_err(): + had_errors = True + elif result.ok_value is not None: + packages.append(result.ok_value) + + # Parse stderr for warnings (lines starting with "warning:") + warnings_list: List[str] = [] + for line in stderr_data.splitlines(): + line = line.strip() + if line.startswith("warning:") or line.startswith("evaluation warning:"): + # Remove "warning:" prefix for cleaner messages + warnings_list.append(line[8:].strip()) + + if process.returncode != 0: + error( + "nix-eval-jobs process failed with non-zero exit code", + title="Process Failure", + ) + + return packages, warnings_list, had_errors def is_extension_pkg(pkg: NixEvalJobsOutput) -> bool: @@ -235,9 +260,9 @@ def main() -> None: cmd = build_nix_eval_command(max_workers, args.flake_outputs) - # Collect all evaluation errors - errors: List[str] = [] - gh_action_packages = sort_pkgs_by_closures(list(run_nix_eval_jobs(cmd, errors))) + # Run evaluation and collect packages and warnings + packages, warnings_list, had_errors = run_nix_eval_jobs(cmd) + gh_action_packages = sort_pkgs_by_closures(packages) def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: """Convert nix-eval-jobs output to GitHub Actions matrix package""" @@ -281,22 +306,24 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: } ] } - print( - f"debug: Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}", - file=sys.stderr, - ) - print(json.dumps(gh_output)) - - # Check if any errors occurred during evaluation - if errors: - print("\n=== Evaluation Errors ===", file=sys.stderr) - for i, error in enumerate(errors, 1): - print(f"\nError {i}:", file=sys.stderr) - print(error, file=sys.stderr) - print( - f"\n=== Total: {len(errors)} error(s) occurred during evaluation ===", - file=sys.stderr, - ) + + if warnings_list: + warning_counts = Counter(warnings_list) + for warn_msg, count in warning_counts.items(): + if count > 1: + warning( + f"{warn_msg} (occurred {count} times)", + title="Nix Evaluation Warning", + ) + else: + warning(warn_msg, title="Nix Evaluation Warning") + + # Output matrix to GitHub Actions + debug(f"Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}") + set_output("matrix", json.dumps(gh_output)) + + # Exit with error code if any evaluation errors occurred + if had_errors: sys.exit(1) From cb95579006b591ed7fc41d4ab50501c8e3e9e729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 25 Nov 2025 22:50:05 +0100 Subject: [PATCH 53/71] feat(github-matrix): group evaluation errors by message Refactor error handling to collect and group evaluation errors similar to warnings. Errors with the same message are now displayed together with a list of affected attributes. --- nix/packages/github-matrix/github_matrix.py | 66 ++++++++++++++------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index deac5bc1d..478935769 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -20,7 +20,7 @@ get_args, ) -from github_action_utils import debug, error, set_output, warning +from github_action_utils import debug, notice, error, set_output, warning from result import Err, Ok, Result System = Literal["x86_64-linux", "aarch64-linux", "aarch64-darwin"] @@ -60,6 +60,13 @@ class GitHubActionPackage(TypedDict): postgresql_version: NotRequired[str] +class NixEvalError(TypedDict): + """Error information from nix evaluation.""" + + attr: str + error: str + + BUILD_RUNNER_MAP: Dict[RunnerType, Dict[System, RunsOnConfig]] = { "ephemeral": { "aarch64-linux": { @@ -107,12 +114,12 @@ def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[s def parse_nix_eval_line( line: str, drv_paths: Set[str] -) -> Result[Optional[NixEvalJobsOutput], str]: +) -> Result[Optional[NixEvalJobsOutput], NixEvalError]: """Parse a single line of nix-eval-jobs output. Returns: Ok(package_data) if successful (None for empty/duplicate lines) - Err(error_message) if a nix evaluation error occurred + Err(NixEvalError) if a nix evaluation error occurred """ if not line.strip(): return Ok(None) @@ -120,11 +127,12 @@ def parse_nix_eval_line( try: data: NixEvalJobsOutput = json.loads(line) if "error" in data: - error_msg = ( - f"Error in nix-eval-jobs output for {data['attr']}: {data['error']}" - ) - error(error_msg, title="Nix Evaluation Error") - return Err(error_msg) + error_msg = data["error"] + # Strip the redundant first line if it contains "does not have valid outputs" + error_lines = error_msg.split("\n") + if len(error_lines) > 1 and "does not have valid outputs" in error_lines[0]: + error_msg = "\n".join(error_lines[1:]).strip() + return Err({"attr": data["attr"], "error": error_msg}) if data["drvPath"] in drv_paths: return Ok(None) drv_paths.add(data["drvPath"]) @@ -136,11 +144,11 @@ def parse_nix_eval_line( def run_nix_eval_jobs( cmd: List[str], -) -> Tuple[List[NixEvalJobsOutput], List[str], bool]: - """Run nix-eval-jobs and return parsed package data, warnings, and error status. +) -> Tuple[List[NixEvalJobsOutput], List[str], List[NixEvalError]]: + """Run nix-eval-jobs and return parsed package data, warnings, and errors. Returns: - Tuple of (packages, warnings_list, had_errors) + Tuple of (packages, warnings_list, errors_list) """ debug(f"Running command: {' '.join(cmd)}") @@ -152,11 +160,11 @@ def run_nix_eval_jobs( # Parse stdout for packages packages: List[NixEvalJobsOutput] = [] drv_paths: Set[str] = set() - had_errors = False + errors_list: List[NixEvalError] = [] for line in stdout_data.splitlines(): result = parse_nix_eval_line(line, drv_paths) if result.is_err(): - had_errors = True + errors_list.append(result.err_value) elif result.ok_value is not None: packages.append(result.ok_value) @@ -174,7 +182,7 @@ def run_nix_eval_jobs( title="Process Failure", ) - return packages, warnings_list, had_errors + return packages, warnings_list, errors_list def is_extension_pkg(pkg: NixEvalJobsOutput) -> bool: @@ -260,8 +268,8 @@ def main() -> None: cmd = build_nix_eval_command(max_workers, args.flake_outputs) - # Run evaluation and collect packages and warnings - packages, warnings_list, had_errors = run_nix_eval_jobs(cmd) + # Run evaluation and collect packages, warnings, and errors + packages, warnings_list, errors_list = run_nix_eval_jobs(cmd) gh_action_packages = sort_pkgs_by_closures(packages) def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: @@ -318,13 +326,31 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: else: warning(warn_msg, title="Nix Evaluation Warning") - # Output matrix to GitHub Actions - debug(f"Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}") - set_output("matrix", json.dumps(gh_output)) + if errors_list: + # Group errors by error message + errors_by_message: Dict[str, List[str]] = defaultdict(list) + for err in errors_list: + errors_by_message[err["error"]].append(err["attr"]) + + for error_msg, attrs in errors_by_message.items(): + if len(attrs) > 1: + error( + f"{error_msg}\nAffected attributes: {', '.join(attrs)}", + title="Nix Evaluation Error", + ) + else: + error( + f"{error_msg}\nAttribute: {attrs[0]}", + title="Nix Evaluation Error", + ) # Exit with error code if any evaluation errors occurred - if had_errors: + if errors_list: sys.exit(1) + else: + # Output matrix to GitHub Actions + notice(f"Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}") + set_output("matrix", json.dumps(gh_output)) if __name__ == "__main__": From 021f2436bbc685f3aa58587417c2f57ce455d714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 25 Nov 2025 23:18:10 +0100 Subject: [PATCH 54/71] fix(github-matrix): improve multiline error display in GitHub Actions Extract core error messages and format them better for GitHub Actions annotations. --- nix/packages/github-matrix/github_matrix.py | 45 ++++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 478935769..d0b696ad0 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -128,10 +128,23 @@ def parse_nix_eval_line( data: NixEvalJobsOutput = json.loads(line) if "error" in data: error_msg = data["error"] - # Strip the redundant first line if it contains "does not have valid outputs" + + # Extract the core error message (last "error:" line and following context) error_lines = error_msg.split("\n") - if len(error_lines) > 1 and "does not have valid outputs" in error_lines[0]: - error_msg = "\n".join(error_lines[1:]).strip() + core_error_idx = -1 + for i in range(len(error_lines) - 1, -1, -1): + if error_lines[i].strip().startswith("error:"): + core_error_idx = i + break + + if core_error_idx >= 0: + # Take the last error line and up to 3 lines of context after it + error_msg = "\n".join( + error_lines[ + core_error_idx : min(core_error_idx + 4, len(error_lines)) + ] + ).strip() + return Err({"attr": data["attr"], "error": error_msg}) if data["drvPath"] in drv_paths: return Ok(None) @@ -152,8 +165,12 @@ def run_nix_eval_jobs( """ debug(f"Running command: {' '.join(cmd)}") + # Disable colors in nix output + env = os.environ.copy() + env["NO_COLOR"] = "1" + process = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env ) stdout_data, stderr_data = process.communicate() @@ -333,23 +350,21 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: errors_by_message[err["error"]].append(err["attr"]) for error_msg, attrs in errors_by_message.items(): + # Format message with attributes on first line, then error details if len(attrs) > 1: - error( - f"{error_msg}\nAffected attributes: {', '.join(attrs)}", - title="Nix Evaluation Error", - ) + formatted_msg = f"Affected attributes ({len(attrs)}): {', '.join(attrs)}\n\n{error_msg}" else: - error( - f"{error_msg}\nAttribute: {attrs[0]}", - title="Nix Evaluation Error", - ) + formatted_msg = f"Attribute: {attrs[0]}\n\n{error_msg}" + formatted_msg = formatted_msg.replace("\n", "%0A") + error(formatted_msg, title="Nix Evaluation Error") - # Exit with error code if any evaluation errors occurred if errors_list: sys.exit(1) else: - # Output matrix to GitHub Actions - notice(f"Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}") + formatted_msg = f"Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}".replace( + "\n", "%0A" + ) + notice(formatted_msg, title="GitHub Actions Matrix") set_output("matrix", json.dumps(gh_output)) From 399cfd76c90a3e0acd3b93618df8ddb57b4accfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 25 Nov 2025 23:20:07 +0100 Subject: [PATCH 55/71] fix(ci): skip run-testinfra and run-tests when nix-eval fails Add nix-eval to needs dependencies and check its result in conditional expressions to prevent downstream test jobs from running when evaluation fails. --- .github/workflows/nix-build.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 379488878..3a163bd6f 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -104,9 +104,10 @@ jobs: run: nix build --accept-flake-config -L .#${{ matrix.attr }} run-testinfra: - needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] + needs: [nix-eval, nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] if: | !cancelled() && + needs.nix-eval.result == 'success' && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') && (needs.nix-build-x86_64-linux.result == 'skipped' || needs.nix-build-x86_64-linux.result == 'success') @@ -115,9 +116,10 @@ jobs: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} run-tests: - needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] + needs: [nix-eval, nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] if: | - !cancelled() && + !cancelled() && + needs.nix-eval.result == 'success' && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') && (needs.nix-build-x86_64-linux.result == 'skipped' || needs.nix-build-x86_64-linux.result == 'success') From b6d84e6ed4cec0df7760b6d507672f98d854f163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 25 Nov 2025 23:37:54 +0100 Subject: [PATCH 56/71] chore(github-matrix): update message when there are no build for a system --- nix/packages/github-matrix/github_matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index d0b696ad0..68fdc0411 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -325,7 +325,7 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: "include": [ { "attr": "", - "name": "skipped", + "name": "no packages to build", "system": system, "runs_on": {"labels": "ubuntu-latest"}, } From 41e344c73ce3a81dc8d67da5cf1a4c35a6e9e1fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Wed, 26 Nov 2025 00:22:56 +0100 Subject: [PATCH 57/71] fix(github-matrix): backward compatibility for Result access We are running an older version of the 'result' library that uses '_value' instead of 'ok_value' to access the successful result of a computation. --- nix/packages/github-matrix/github_matrix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 68fdc0411..7743753e5 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -182,8 +182,8 @@ def run_nix_eval_jobs( result = parse_nix_eval_line(line, drv_paths) if result.is_err(): errors_list.append(result.err_value) - elif result.ok_value is not None: - packages.append(result.ok_value) + elif result._value is not None: + packages.append(result._value) # Parse stderr for warnings (lines starting with "warning:") warnings_list: List[str] = [] From a694685d6c763eb38a1a5b4166808b4d9bcd32f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 1 Dec 2025 11:39:41 +0100 Subject: [PATCH 58/71] refactor: migrate from packages to legacyPackages for PostgreSQL extensions To be able to build extensions versions packages separately in CI, we need to expose them in a nested structure. It is currently not possible to do so with the flattened packages structure, as the individual extension packages are not directly accessible. In this change, we replace the flattened package structure with nested legacyPackages to improve discoverability of individual extension packages. --- nix/checks.nix | 2 +- nix/ext/tests/default.nix | 3 +- nix/ext/tests/http.nix | 3 +- nix/ext/tests/orioledb.nix | 2 +- nix/ext/tests/pg_plan_filter.nix | 3 +- nix/ext/tests/pg_repack.nix | 3 +- nix/ext/tests/pg_safeupdate.nix | 3 +- nix/ext/tests/pgjwt.nix | 3 +- nix/ext/tests/pgmq.nix | 3 +- nix/ext/tests/pgroonga.nix | 3 +- nix/ext/tests/pgrouting.nix | 6 +- nix/ext/tests/pgsodium.nix | 4 +- nix/ext/tests/plpgsql_check.nix | 3 +- nix/ext/tests/plv8.nix | 3 +- nix/ext/tests/postgis.nix | 3 +- nix/ext/tests/timescaledb.nix | 3 +- nix/ext/tests/vault.nix | 4 +- nix/ext/wrappers/default.nix | 16 +++-- nix/packages/default.nix | 2 +- .../github-matrix/tests/test_github_matrix.py | 70 +++++++++++++------ nix/packages/postgres.nix | 33 ++++++--- 21 files changed, 119 insertions(+), 56 deletions(-) diff --git a/nix/checks.nix b/nix/checks.nix index cf0fe4c2f..2f66871e7 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -88,7 +88,7 @@ builtins.trace "Major version result: ${result}" result; # Select the appropriate pgroonga package for this PostgreSQL version - pgroonga = self'.packages."psql_${majorVersion}/exts/pgroonga-all"; + pgroonga = self'.legacyPackages."psql_${majorVersion}".exts.pgroonga-all; pgPort = if (majorVersion == "17") then diff --git a/nix/ext/tests/default.nix b/nix/ext/tests/default.nix index a3e67d6da..a975116a7 100644 --- a/nix/ext/tests/default.nix +++ b/nix/ext/tests/default.nix @@ -11,7 +11,8 @@ let pname = extension_name; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/http.nix b/nix/ext/tests/http.nix index 09075c374..52cc371aa 100644 --- a/nix/ext/tests/http.nix +++ b/nix/ext/tests/http.nix @@ -3,7 +3,8 @@ let pname = "http"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/orioledb.nix b/nix/ext/tests/orioledb.nix index a6f5484c2..7d0475cef 100644 --- a/nix/ext/tests/orioledb.nix +++ b/nix/ext/tests/orioledb.nix @@ -11,7 +11,7 @@ let paths = [ postgresql postgresql.lib - self.packages.${pkgs.system}."psql_orioledb-17/exts/orioledb" + (self.legacyPackages.${pkgs.system}."psql_orioledb-17".exts.orioledb) ]; passthru = { inherit (postgresql) version psqlSchema; diff --git a/nix/ext/tests/pg_plan_filter.nix b/nix/ext/tests/pg_plan_filter.nix index fe0073770..47332eb48 100644 --- a/nix/ext/tests/pg_plan_filter.nix +++ b/nix/ext/tests/pg_plan_filter.nix @@ -3,7 +3,8 @@ let pname = "plan_filter"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/pg_repack.nix b/nix/ext/tests/pg_repack.nix index dae534d2e..dce50c26c 100644 --- a/nix/ext/tests/pg_repack.nix +++ b/nix/ext/tests/pg_repack.nix @@ -3,7 +3,8 @@ let pname = "pg_repack"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/pg_safeupdate.nix b/nix/ext/tests/pg_safeupdate.nix index 19a0132a8..4f01793bb 100644 --- a/nix/ext/tests/pg_safeupdate.nix +++ b/nix/ext/tests/pg_safeupdate.nix @@ -3,7 +3,8 @@ let pname = "safeupdate"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/pgjwt.nix b/nix/ext/tests/pgjwt.nix index 98884f4c3..2f60a01c5 100644 --- a/nix/ext/tests/pgjwt.nix +++ b/nix/ext/tests/pgjwt.nix @@ -3,7 +3,8 @@ let pname = "pgjwt"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/pgmq.nix b/nix/ext/tests/pgmq.nix index 94438d612..29be574f1 100644 --- a/nix/ext/tests/pgmq.nix +++ b/nix/ext/tests/pgmq.nix @@ -3,7 +3,8 @@ let pname = "pgmq"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/pgroonga.nix b/nix/ext/tests/pgroonga.nix index 83af20ac8..1e3919521 100644 --- a/nix/ext/tests/pgroonga.nix +++ b/nix/ext/tests/pgroonga.nix @@ -3,7 +3,8 @@ let pname = "pgroonga"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/pgrouting.nix b/nix/ext/tests/pgrouting.nix index f8775e4aa..3b2d8e75c 100644 --- a/nix/ext/tests/pgrouting.nix +++ b/nix/ext/tests/pgrouting.nix @@ -3,7 +3,7 @@ let pname = "pgrouting"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + postgresMajorVersion: self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: @@ -16,10 +16,10 @@ let postgresql postgresql.lib (installedExtension majorVersion) - self.packages.${pkgs.system}."psql_${majorVersion}/exts/postgis-all" + (self.legacyPackages.${pkgs.system}."psql_${majorVersion}".exts.postgis-all) ] ++ lib.optional (postgresql.isOrioleDB - ) self.packages.${pkgs.system}."psql_orioledb-17/exts/orioledb"; + ) (self.legacyPackages.${pkgs.system}."psql_orioledb-17".exts.orioledb); passthru = { inherit (postgresql) version psqlSchema; lib = pkg; diff --git a/nix/ext/tests/pgsodium.nix b/nix/ext/tests/pgsodium.nix index 9ad1aec4b..419417a07 100644 --- a/nix/ext/tests/pgsodium.nix +++ b/nix/ext/tests/pgsodium.nix @@ -3,7 +3,7 @@ let pname = "pgsodium"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + postgresMajorVersion: self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: @@ -15,7 +15,7 @@ let postgresql postgresql.lib (installedExtension majorVersion) - self.packages.${pkgs.system}."psql_${majorVersion}/exts/hypopg-all" + (self.legacyPackages.${pkgs.system}."psql_${majorVersion}".exts.hypopg-all) ]; passthru = { inherit (postgresql) version psqlSchema; diff --git a/nix/ext/tests/plpgsql_check.nix b/nix/ext/tests/plpgsql_check.nix index 105ed8a88..ceae9cb84 100644 --- a/nix/ext/tests/plpgsql_check.nix +++ b/nix/ext/tests/plpgsql_check.nix @@ -3,7 +3,8 @@ let pname = "plpgsql_check"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/plv8.nix b/nix/ext/tests/plv8.nix index 085ddd42f..81bcd33a5 100644 --- a/nix/ext/tests/plv8.nix +++ b/nix/ext/tests/plv8.nix @@ -4,7 +4,8 @@ let pname = "plv8"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/postgis.nix b/nix/ext/tests/postgis.nix index ab6a4b3f8..e928f729f 100644 --- a/nix/ext/tests/postgis.nix +++ b/nix/ext/tests/postgis.nix @@ -3,7 +3,8 @@ let pname = "postgis"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/timescaledb.nix b/nix/ext/tests/timescaledb.nix index 1bab187d1..07ac51786 100644 --- a/nix/ext/tests/timescaledb.nix +++ b/nix/ext/tests/timescaledb.nix @@ -3,7 +3,8 @@ let pname = "timescaledb"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; versions = (installedExtension "15").versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/vault.nix b/nix/ext/tests/vault.nix index 7b16247a5..9d49f7fac 100644 --- a/nix/ext/tests/vault.nix +++ b/nix/ext/tests/vault.nix @@ -3,7 +3,7 @@ let pname = "supabase_vault"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + postgresMajorVersion: self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: @@ -15,7 +15,7 @@ let postgresql postgresql.lib (installedExtension majorVersion) - self.packages.${pkgs.system}."psql_${majorVersion}/exts/pgsodium-all" # dependency + (self.legacyPackages.${pkgs.system}."psql_${majorVersion}".exts.pgsodium-all) # dependency ]; passthru = { inherit (postgresql) version psqlSchema; diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index dd06834c9..638a83365 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -220,9 +220,13 @@ let v: !(builtins.elem v versions) ) allPreviouslyPackagedVersions; numberOfPreviouslyPackagedVersions = builtins.length previouslyPackagedVersions; - packages = builtins.attrValues ( - lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions - ); + packagesAttrSet = lib.mapAttrs' ( + name: value: { + name = lib.replaceStrings ["."] ["_"] name; + value = build name value.hash value.rust value.pgrx; + } + ) supportedVersions; + packages = builtins.attrValues packagesAttrSet; in (buildEnv { name = pname; @@ -305,9 +309,13 @@ in ''; passthru = { inherit versions numberOfVersions; - pname = "${pname}-all"; + pname = "${pname}"; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); + # Expose individual packages for CI to build separately + packages = packagesAttrSet // { + recurseForDerivations = true; + }; }; }).overrideAttrs (_: { diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 399b34865..d2c3b904e 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -63,7 +63,7 @@ start-server = pkgs-lib.makePostgresDevSetup { inherit pkgs; name = "start-postgres-server"; - pgroonga = self'.packages."psql_${activeVersion}/exts/pgroonga-all"; + pgroonga = self'.legacyPackages."psql_${activeVersion}".exts.pgroonga-all; }; switch-ext-version = pkgs.callPackage ./switch-ext-version.nix { inherit (self'.packages) overlayfs-on-package; diff --git a/nix/packages/github-matrix/tests/test_github_matrix.py b/nix/packages/github-matrix/tests/test_github_matrix.py index d5fed5732..e977c2a6a 100644 --- a/nix/packages/github-matrix/tests/test_github_matrix.py +++ b/nix/packages/github-matrix/tests/test_github_matrix.py @@ -15,8 +15,14 @@ class TestIsExtensionPkg: def test_extension_package(self): pkg: NixEvalJobsOutput = { - "attr": "packages.x86_64-linux.psql_15.exts.pg_cron", - "attrPath": ["packages", "x86_64-linux", "psql_15", "exts", "pg_cron"], + "attr": "legacyPackages.x86_64-linux.psql_15.exts.pg_cron", + "attrPath": [ + "legacyPackages", + "x86_64-linux", + "psql_15", + "exts", + "pg_cron", + ], "cacheStatus": "notBuilt", "drvPath": "/nix/store/test.drv", "name": "pg_cron", @@ -26,8 +32,8 @@ def test_extension_package(self): def test_non_extension_package(self): pkg: NixEvalJobsOutput = { - "attr": "packages.x86_64-linux.psql_15", - "attrPath": ["packages", "x86_64-linux", "psql_15"], + "attr": "legacyPackages.x86_64-linux.psql_15", + "attrPath": ["legacyPackages", "x86_64-linux", "psql_15"], "cacheStatus": "notBuilt", "drvPath": "/nix/store/test.drv", "name": "postgresql-16.0", @@ -40,12 +46,12 @@ class TestIsLargePkg: @pytest.mark.parametrize( "attr,expected", [ - ("packages.x86_64-linux.psql_15.exts.wrappers", True), - ("packages.x86_64-linux.psql_15.exts.pg_jsonschema", True), - ("packages.x86_64-linux.psql_15.exts.pg_graphql", True), - ("packages.x86_64-linux.psql_15.exts.postgis", True), - ("packages.x86_64-linux.psql_15.exts.pg_cron", False), - ("packages.x86_64-linux.psql_15", False), + ("legacyPackages.x86_64-linux.psql_15.exts.wrappers", True), + ("legacyPackages.x86_64-linux.psql_15.exts.pg_jsonschema", True), + ("legacyPackages.x86_64-linux.psql_15.exts.pg_graphql", True), + ("legacyPackages.x86_64-linux.psql_15.exts.postgis", True), + ("legacyPackages.x86_64-linux.psql_15.exts.pg_cron", False), + ("legacyPackages.x86_64-linux.psql_15", False), ], ) def test_large_package_detection(self, attr: str, expected: bool): @@ -121,8 +127,14 @@ def test_kvm_package_aarch64_linux(self): def test_large_package_x86_64_linux(self): pkg: NixEvalJobsOutput = { - "attr": "packages.x86_64-linux.psql_15.exts.postgis", - "attrPath": ["packages", "x86_64-linux", "psql_15", "exts", "postgis"], + "attr": "legacyPackages.x86_64-linux.psql_15.exts.postgis", + "attrPath": [ + "legacyPackages", + "x86_64-linux", + "psql_15", + "exts", + "postgis", + ], "cacheStatus": "notBuilt", "drvPath": "/nix/store/test.drv", "name": "postgis", @@ -134,8 +146,14 @@ def test_large_package_x86_64_linux(self): def test_large_package_aarch64_linux(self): pkg: NixEvalJobsOutput = { - "attr": "packages.aarch64-linux.psql_15.exts.pg_graphql", - "attrPath": ["packages", "aarch64-linux", "psql_15", "exts", "pg_graphql"], + "attr": "legacyPackages.aarch64-linux.psql_15.exts.pg_graphql", + "attrPath": [ + "legacyPackages", + "aarch64-linux", + "psql_15", + "exts", + "pg_graphql", + ], "cacheStatus": "notBuilt", "drvPath": "/nix/store/test.drv", "name": "pg_graphql", @@ -147,8 +165,8 @@ def test_large_package_aarch64_linux(self): def test_darwin_package(self): pkg: NixEvalJobsOutput = { - "attr": "packages.aarch64-darwin.psql_15", - "attrPath": ["packages", "aarch64-darwin", "psql_15"], + "attr": "legacyPackages.aarch64-darwin.psql_15", + "attrPath": ["legacyPackages", "aarch64-darwin", "psql_15"], "cacheStatus": "notBuilt", "drvPath": "/nix/store/test.drv", "name": "postgresql-16.0", @@ -162,8 +180,14 @@ def test_darwin_package(self): def test_default_x86_64_linux(self): pkg: NixEvalJobsOutput = { - "attr": "packages.x86_64-linux.psql_15.exts.pg_cron", - "attrPath": ["packages", "x86_64-linux", "psql_15", "exts", "pg_cron"], + "attr": "legacyPackages.x86_64-linux.psql_15.exts.pg_cron", + "attrPath": [ + "legacyPackages", + "x86_64-linux", + "psql_15", + "exts", + "pg_cron", + ], "cacheStatus": "notBuilt", "drvPath": "/nix/store/test.drv", "name": "pg_cron", @@ -174,8 +198,14 @@ def test_default_x86_64_linux(self): def test_default_aarch64_linux(self): pkg: NixEvalJobsOutput = { - "attr": "packages.aarch64-linux.psql_15.exts.pg_cron", - "attrPath": ["packages", "aarch64-linux", "psql_15", "exts", "pg_cron"], + "attr": "legacyPackages.aarch64-linux.psql_15.exts.pg_cron", + "attrPath": [ + "legacyPackages", + "aarch64-linux", + "psql_15", + "exts", + "pg_cron", + ], "cacheStatus": "notBuilt", "drvPath": "/nix/store/test.drv", "name": "pg_cron", diff --git a/nix/packages/postgres.nix b/nix/packages/postgres.nix index f98bd3607..f6f521d60 100644 --- a/nix/packages/postgres.nix +++ b/nix/packages/postgres.nix @@ -108,15 +108,24 @@ # Create an attrset that contains all the extensions included in a server. makeOurPostgresPkgsSet = version: - (builtins.listToAttrs ( - map (drv: { - name = drv.pname; - value = drv; - }) (makeOurPostgresPkgs version) - )) - // { - recurseForDerivations = true; - }; + let + pkgsList = makeOurPostgresPkgs version; + baseAttrs = builtins.listToAttrs ( + map (drv: { + name = drv.pname; + value = drv; + }) pkgsList + ); + # Expose individual packages from extensions that have them in passthru.packages + # This makes them discoverable by nix-eval-jobs --force-recurse + individualPkgs = lib.concatMapAttrs ( + name: drv: + lib.optionalAttrs (drv ? passthru.packages) { + "${name}-pkgs" = drv.passthru.packages; + } + ) baseAttrs; + in + baseAttrs // individualPkgs // { recurseForDerivations = true; }; # Create a binary distribution of PostgreSQL, given a version. # @@ -165,9 +174,13 @@ psql_17 = makePostgres "17"; psql_orioledb-17 = makePostgres "orioledb-17"; }; + binPackages = lib.mapAttrs' (name: value: { + name = "${name}/bin"; + value = value.bin; + }) basePackages; in { - packages = inputs.flake-utils.lib.flattenTree basePackages; + packages = binPackages; legacyPackages = basePackages; }; } From aa9ac48b2bbe9266bef404bc5a6ef2da38c589c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 1 Dec 2025 12:26:40 +0100 Subject: [PATCH 59/71] refactor(nix): remove "-all" suffix from extension package names Simplify extension package naming by removing the redundant "-all" suffix that was appended to pname attributes. --- nix/checks.nix | 2 +- nix/ext/hypopg.nix | 3 +-- nix/ext/index_advisor.nix | 3 +-- nix/ext/pg-safeupdate.nix | 3 +-- nix/ext/pg_cron/default.nix | 1 - nix/ext/pg_graphql/default.nix | 3 +-- nix/ext/pg_hashids.nix | 3 +-- nix/ext/pg_jsonschema/default.nix | 3 +-- nix/ext/pg_net.nix | 3 +-- nix/ext/pg_plan_filter.nix | 3 +-- nix/ext/pg_repack.nix | 3 +-- nix/ext/pg_stat_monitor.nix | 3 +-- nix/ext/pg_tle.nix | 3 +-- nix/ext/pgaudit.nix | 3 +-- nix/ext/pgjwt.nix | 3 +-- nix/ext/pgmq/default.nix | 3 +-- nix/ext/pgroonga/default.nix | 3 +-- nix/ext/pgrouting.nix | 3 +-- nix/ext/pgsodium.nix | 3 +-- nix/ext/pgsql-http.nix | 3 +-- nix/ext/pgvector.nix | 3 +-- nix/ext/plpgsql-check.nix | 1 - nix/ext/plv8/default.nix | 3 +-- nix/ext/postgis.nix | 3 +-- nix/ext/rum.nix | 3 +-- nix/ext/tests/default.nix | 2 +- nix/ext/tests/http.nix | 2 +- nix/ext/tests/pg_plan_filter.nix | 2 +- nix/ext/tests/pg_repack.nix | 2 +- nix/ext/tests/pg_safeupdate.nix | 2 +- nix/ext/tests/pgjwt.nix | 2 +- nix/ext/tests/pgmq.nix | 2 +- nix/ext/tests/pgroonga.nix | 2 +- nix/ext/tests/pgrouting.nix | 10 ++++++---- nix/ext/tests/pgsodium.nix | 5 +++-- nix/ext/tests/plpgsql_check.nix | 2 +- nix/ext/tests/plv8.nix | 2 +- nix/ext/tests/postgis.nix | 2 +- nix/ext/tests/timescaledb.nix | 2 +- nix/ext/tests/vault.nix | 5 +++-- nix/ext/timescaledb.nix | 1 - nix/ext/vault.nix | 3 +-- nix/ext/wal2json.nix | 3 +-- nix/packages/default.nix | 2 +- nix/packages/postgres.nix | 9 +++------ 45 files changed, 53 insertions(+), 79 deletions(-) diff --git a/nix/checks.nix b/nix/checks.nix index 2f66871e7..81e0b4117 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -88,7 +88,7 @@ builtins.trace "Major version result: ${result}" result; # Select the appropriate pgroonga package for this PostgreSQL version - pgroonga = self'.legacyPackages."psql_${majorVersion}".exts.pgroonga-all; + pgroonga = self'.legacyPackages."psql_${majorVersion}".exts.pgroonga; pgPort = if (majorVersion == "17") then diff --git a/nix/ext/hypopg.nix b/nix/ext/hypopg.nix index 28e847d46..92784a8de 100644 --- a/nix/ext/hypopg.nix +++ b/nix/ext/hypopg.nix @@ -87,8 +87,7 @@ buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/index_advisor.nix b/nix/ext/index_advisor.nix index 9e6a54aea..85a23bfd0 100644 --- a/nix/ext/index_advisor.nix +++ b/nix/ext/index_advisor.nix @@ -78,8 +78,7 @@ pkgs.buildEnv { ]; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/pg-safeupdate.nix b/nix/ext/pg-safeupdate.nix index bc3430378..ee31f4371 100644 --- a/nix/ext/pg-safeupdate.nix +++ b/nix/ext/pg-safeupdate.nix @@ -74,8 +74,7 @@ pkgs.buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; defaultSettings = { shared_preload_libraries = [ "safeupdate" ]; }; diff --git a/nix/ext/pg_cron/default.nix b/nix/ext/pg_cron/default.nix index 75215d56a..cec9d8ec4 100644 --- a/nix/ext/pg_cron/default.nix +++ b/nix/ext/pg_cron/default.nix @@ -110,7 +110,6 @@ buildEnv { passthru = { inherit versions numberOfVersions switch-ext-version; - pname = "${pname}-all"; hasBackgroundWorker = true; defaultSettings = { shared_preload_libraries = [ "pg_cron" ]; diff --git a/nix/ext/pg_graphql/default.nix b/nix/ext/pg_graphql/default.nix index a63fcd84e..a7f6d1065 100644 --- a/nix/ext/pg_graphql/default.nix +++ b/nix/ext/pg_graphql/default.nix @@ -172,8 +172,7 @@ in ) ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/pg_hashids.nix b/nix/ext/pg_hashids.nix index 79bba3dcf..b11c5ce68 100644 --- a/nix/ext/pg_hashids.nix +++ b/nix/ext/pg_hashids.nix @@ -93,8 +93,7 @@ buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/pg_jsonschema/default.nix b/nix/ext/pg_jsonschema/default.nix index 9939b733a..1a2e8ee58 100644 --- a/nix/ext/pg_jsonschema/default.nix +++ b/nix/ext/pg_jsonschema/default.nix @@ -165,8 +165,7 @@ in ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/pg_net.nix b/nix/ext/pg_net.nix index 62dad4386..229971a81 100644 --- a/nix/ext/pg_net.nix +++ b/nix/ext/pg_net.nix @@ -116,8 +116,7 @@ pkgs.buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; hasBackgroundWorker = true; defaultSettings = { shared_preload_libraries = [ "pg_net" ]; diff --git a/nix/ext/pg_plan_filter.nix b/nix/ext/pg_plan_filter.nix index 4e52bf22f..dfb3262b7 100644 --- a/nix/ext/pg_plan_filter.nix +++ b/nix/ext/pg_plan_filter.nix @@ -75,8 +75,7 @@ pkgs.buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; defaultSettings = { shared_preload_libraries = [ "plan_filter" ]; }; diff --git a/nix/ext/pg_repack.nix b/nix/ext/pg_repack.nix index 4a62b68b4..153cebd76 100644 --- a/nix/ext/pg_repack.nix +++ b/nix/ext/pg_repack.nix @@ -129,8 +129,7 @@ buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/pg_stat_monitor.nix b/nix/ext/pg_stat_monitor.nix index 2a2ce0097..ddf46de30 100644 --- a/nix/ext/pg_stat_monitor.nix +++ b/nix/ext/pg_stat_monitor.nix @@ -97,8 +97,7 @@ buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/pg_tle.nix b/nix/ext/pg_tle.nix index f2fc5d1da..7101952a2 100644 --- a/nix/ext/pg_tle.nix +++ b/nix/ext/pg_tle.nix @@ -100,8 +100,7 @@ buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; defaultSettings = { shared_preload_libraries = [ "pg_tle" ]; }; diff --git a/nix/ext/pgaudit.nix b/nix/ext/pgaudit.nix index 9953e12fa..7660f2bd1 100644 --- a/nix/ext/pgaudit.nix +++ b/nix/ext/pgaudit.nix @@ -231,8 +231,7 @@ buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); defaultSettings = { diff --git a/nix/ext/pgjwt.nix b/nix/ext/pgjwt.nix index 72f1614d2..6bac9dcd8 100644 --- a/nix/ext/pgjwt.nix +++ b/nix/ext/pgjwt.nix @@ -77,8 +77,7 @@ buildEnv { pathsToLink = [ "/share/postgresql/extension" ]; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/pgmq/default.nix b/nix/ext/pgmq/default.nix index 32a7354e2..518308211 100644 --- a/nix/ext/pgmq/default.nix +++ b/nix/ext/pgmq/default.nix @@ -99,11 +99,10 @@ buildEnv { pathsToLink = [ "/share/postgresql/extension" ]; passthru = { - inherit versions numberOfVersions; + inherit versions numberOfVersions pname; defaultSettings = { search_path = "\"$user\", public, auth, extensions"; }; - pname = "${pname}-all"; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/pgroonga/default.nix b/nix/ext/pgroonga/default.nix index 55c0abedd..22a8c2dfc 100644 --- a/nix/ext/pgroonga/default.nix +++ b/nix/ext/pgroonga/default.nix @@ -170,8 +170,7 @@ buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/pgrouting.nix b/nix/ext/pgrouting.nix index b20b89c43..48833791e 100644 --- a/nix/ext/pgrouting.nix +++ b/nix/ext/pgrouting.nix @@ -136,8 +136,7 @@ buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/pgsodium.nix b/nix/ext/pgsodium.nix index 5c8b07df6..fa111d8a5 100644 --- a/nix/ext/pgsodium.nix +++ b/nix/ext/pgsodium.nix @@ -101,8 +101,7 @@ pkgs.buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/pgsql-http.nix b/nix/ext/pgsql-http.nix index ba669e22d..3ad03b80c 100644 --- a/nix/ext/pgsql-http.nix +++ b/nix/ext/pgsql-http.nix @@ -103,8 +103,7 @@ pkgs.buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index b8f8347a3..bcf86ebfb 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -83,8 +83,7 @@ pkgs.buildEnv { ]; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); pgRegressTestName = "pgvector"; diff --git a/nix/ext/plpgsql-check.nix b/nix/ext/plpgsql-check.nix index dc4cd946e..2ee225cad 100644 --- a/nix/ext/plpgsql-check.nix +++ b/nix/ext/plpgsql-check.nix @@ -131,7 +131,6 @@ buildEnv { passthru = { inherit versions numberOfVersions switch-ext-version; - pname = "${pname}-all"; hasBackgroundWorker = true; defaultSettings = { shared_preload_libraries = [ diff --git a/nix/ext/plv8/default.nix b/nix/ext/plv8/default.nix index 188ad99b5..d858405ed 100644 --- a/nix/ext/plv8/default.nix +++ b/nix/ext/plv8/default.nix @@ -240,8 +240,7 @@ buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/postgis.nix b/nix/ext/postgis.nix index e5b19ccfb..cdf48fb24 100644 --- a/nix/ext/postgis.nix +++ b/nix/ext/postgis.nix @@ -204,8 +204,7 @@ in ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/rum.nix b/nix/ext/rum.nix index f0839241f..6a9492e38 100644 --- a/nix/ext/rum.nix +++ b/nix/ext/rum.nix @@ -94,8 +94,7 @@ buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; diff --git a/nix/ext/tests/default.nix b/nix/ext/tests/default.nix index a975116a7..57644ee7f 100644 --- a/nix/ext/tests/default.nix +++ b/nix/ext/tests/default.nix @@ -12,7 +12,7 @@ let inherit (pkgs) lib; installedExtension = postgresMajorVersion: - self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/http.nix b/nix/ext/tests/http.nix index 52cc371aa..f465da254 100644 --- a/nix/ext/tests/http.nix +++ b/nix/ext/tests/http.nix @@ -4,7 +4,7 @@ let inherit (pkgs) lib; installedExtension = postgresMajorVersion: - self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/pg_plan_filter.nix b/nix/ext/tests/pg_plan_filter.nix index 47332eb48..c9efb4809 100644 --- a/nix/ext/tests/pg_plan_filter.nix +++ b/nix/ext/tests/pg_plan_filter.nix @@ -4,7 +4,7 @@ let inherit (pkgs) lib; installedExtension = postgresMajorVersion: - self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/pg_repack.nix b/nix/ext/tests/pg_repack.nix index dce50c26c..ff06a9b45 100644 --- a/nix/ext/tests/pg_repack.nix +++ b/nix/ext/tests/pg_repack.nix @@ -4,7 +4,7 @@ let inherit (pkgs) lib; installedExtension = postgresMajorVersion: - self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/pg_safeupdate.nix b/nix/ext/tests/pg_safeupdate.nix index 4f01793bb..1c7e09c7f 100644 --- a/nix/ext/tests/pg_safeupdate.nix +++ b/nix/ext/tests/pg_safeupdate.nix @@ -4,7 +4,7 @@ let inherit (pkgs) lib; installedExtension = postgresMajorVersion: - self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/pgjwt.nix b/nix/ext/tests/pgjwt.nix index 2f60a01c5..ef2e6b935 100644 --- a/nix/ext/tests/pgjwt.nix +++ b/nix/ext/tests/pgjwt.nix @@ -4,7 +4,7 @@ let inherit (pkgs) lib; installedExtension = postgresMajorVersion: - self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/pgmq.nix b/nix/ext/tests/pgmq.nix index 29be574f1..b0c432921 100644 --- a/nix/ext/tests/pgmq.nix +++ b/nix/ext/tests/pgmq.nix @@ -4,7 +4,7 @@ let inherit (pkgs) lib; installedExtension = postgresMajorVersion: - self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/pgroonga.nix b/nix/ext/tests/pgroonga.nix index 1e3919521..abfef8804 100644 --- a/nix/ext/tests/pgroonga.nix +++ b/nix/ext/tests/pgroonga.nix @@ -4,7 +4,7 @@ let inherit (pkgs) lib; installedExtension = postgresMajorVersion: - self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/pgrouting.nix b/nix/ext/tests/pgrouting.nix index 3b2d8e75c..f4550b576 100644 --- a/nix/ext/tests/pgrouting.nix +++ b/nix/ext/tests/pgrouting.nix @@ -3,7 +3,8 @@ let pname = "pgrouting"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: @@ -16,10 +17,11 @@ let postgresql postgresql.lib (installedExtension majorVersion) - (self.legacyPackages.${pkgs.system}."psql_${majorVersion}".exts.postgis-all) + (self.legacyPackages.${pkgs.system}."psql_${majorVersion}".exts.postgis) ] - ++ lib.optional (postgresql.isOrioleDB - ) (self.legacyPackages.${pkgs.system}."psql_orioledb-17".exts.orioledb); + ++ lib.optional (postgresql.isOrioleDB) ( + self.legacyPackages.${pkgs.system}."psql_orioledb-17".exts.orioledb + ); passthru = { inherit (postgresql) version psqlSchema; lib = pkg; diff --git a/nix/ext/tests/pgsodium.nix b/nix/ext/tests/pgsodium.nix index 419417a07..0eae3c0a0 100644 --- a/nix/ext/tests/pgsodium.nix +++ b/nix/ext/tests/pgsodium.nix @@ -3,7 +3,8 @@ let pname = "pgsodium"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: @@ -15,7 +16,7 @@ let postgresql postgresql.lib (installedExtension majorVersion) - (self.legacyPackages.${pkgs.system}."psql_${majorVersion}".exts.hypopg-all) + (self.legacyPackages.${pkgs.system}."psql_${majorVersion}".exts.hypopg) ]; passthru = { inherit (postgresql) version psqlSchema; diff --git a/nix/ext/tests/plpgsql_check.nix b/nix/ext/tests/plpgsql_check.nix index ceae9cb84..4d964ad9f 100644 --- a/nix/ext/tests/plpgsql_check.nix +++ b/nix/ext/tests/plpgsql_check.nix @@ -4,7 +4,7 @@ let inherit (pkgs) lib; installedExtension = postgresMajorVersion: - self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/plv8.nix b/nix/ext/tests/plv8.nix index 81bcd33a5..26a349637 100644 --- a/nix/ext/tests/plv8.nix +++ b/nix/ext/tests/plv8.nix @@ -5,7 +5,7 @@ let inherit (pkgs) lib; installedExtension = postgresMajorVersion: - self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/postgis.nix b/nix/ext/tests/postgis.nix index e928f729f..f20e2de24 100644 --- a/nix/ext/tests/postgis.nix +++ b/nix/ext/tests/postgis.nix @@ -4,7 +4,7 @@ let inherit (pkgs) lib; installedExtension = postgresMajorVersion: - self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/timescaledb.nix b/nix/ext/tests/timescaledb.nix index 07ac51786..0cea7a507 100644 --- a/nix/ext/tests/timescaledb.nix +++ b/nix/ext/tests/timescaledb.nix @@ -4,7 +4,7 @@ let inherit (pkgs) lib; installedExtension = postgresMajorVersion: - self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; versions = (installedExtension "15").versions; postgresqlWithExtension = postgresql: diff --git a/nix/ext/tests/vault.nix b/nix/ext/tests/vault.nix index 9d49f7fac..a3f9cb24f 100644 --- a/nix/ext/tests/vault.nix +++ b/nix/ext/tests/vault.nix @@ -3,7 +3,8 @@ let pname = "supabase_vault"; inherit (pkgs) lib; installedExtension = - postgresMajorVersion: self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}-all"; + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; postgresqlWithExtension = postgresql: @@ -15,7 +16,7 @@ let postgresql postgresql.lib (installedExtension majorVersion) - (self.legacyPackages.${pkgs.system}."psql_${majorVersion}".exts.pgsodium-all) # dependency + (self.legacyPackages.${pkgs.system}."psql_${majorVersion}".exts.pgsodium) # dependency ]; passthru = { inherit (postgresql) version psqlSchema; diff --git a/nix/ext/timescaledb.nix b/nix/ext/timescaledb.nix index a58e8e2b4..91605a48c 100644 --- a/nix/ext/timescaledb.nix +++ b/nix/ext/timescaledb.nix @@ -141,7 +141,6 @@ buildEnv { passthru = { inherit versions numberOfVersions switch-ext-version; - pname = "${pname}-all"; hasBackgroundWorker = true; defaultSettings = { shared_preload_libraries = [ "timescaledb" ]; diff --git a/nix/ext/vault.nix b/nix/ext/vault.nix index c518e1e14..cb90882a7 100644 --- a/nix/ext/vault.nix +++ b/nix/ext/vault.nix @@ -87,8 +87,7 @@ pkgs.buildEnv { ]; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); pgRegressTestName = "vault"; diff --git a/nix/ext/wal2json.nix b/nix/ext/wal2json.nix index f66e689d4..bd1e8ddd8 100644 --- a/nix/ext/wal2json.nix +++ b/nix/ext/wal2json.nix @@ -95,8 +95,7 @@ pkgs.buildEnv { ''; passthru = { - inherit versions numberOfVersions; - pname = "${pname}-all"; + inherit versions numberOfVersions pname; version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); defaultSettings = { diff --git a/nix/packages/default.nix b/nix/packages/default.nix index d2c3b904e..b3fc83a0f 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -63,7 +63,7 @@ start-server = pkgs-lib.makePostgresDevSetup { inherit pkgs; name = "start-postgres-server"; - pgroonga = self'.legacyPackages."psql_${activeVersion}".exts.pgroonga-all; + pgroonga = self'.legacyPackages."psql_${activeVersion}".exts.pgroonga; }; switch-ext-version = pkgs.callPackage ./switch-ext-version.nix { inherit (self'.packages) overlayfs-on-package; diff --git a/nix/packages/postgres.nix b/nix/packages/postgres.nix index f6f521d60..aa6cc9e4c 100644 --- a/nix/packages/postgres.nix +++ b/nix/packages/postgres.nix @@ -112,17 +112,14 @@ pkgsList = makeOurPostgresPkgs version; baseAttrs = builtins.listToAttrs ( map (drv: { - name = drv.pname; + name = drv.name; value = drv; }) pkgsList ); # Expose individual packages from extensions that have them in passthru.packages # This makes them discoverable by nix-eval-jobs --force-recurse individualPkgs = lib.concatMapAttrs ( - name: drv: - lib.optionalAttrs (drv ? passthru.packages) { - "${name}-pkgs" = drv.passthru.packages; - } + name: drv: lib.optionalAttrs (drv ? passthru.packages) { "${name}-pkgs" = drv.passthru.packages; } ) baseAttrs; in baseAttrs // individualPkgs // { recurseForDerivations = true; }; @@ -140,7 +137,7 @@ let postgresql = getPostgresqlPackage version; ourExts = map (ext: { - name = ext.pname; + name = ext.name; version = ext.version; }) (makeOurPostgresPkgs version); From 1cc01030522c6f6411eb463a742d28d750b57dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 1 Dec 2025 12:57:02 +0100 Subject: [PATCH 60/71] refactor(ci): split nix build jobs into separate packages and checks workflows To make sure we only build what is necessary, we start building packages first, then run checks once all packages are built successfully. --- .github/workflows/nix-build.yml | 122 +++++++++++++++++--- .github/workflows/nix-eval.yml | 12 +- nix/ext/wrappers/default.nix | 11 +- nix/packages/github-matrix/github_matrix.py | 50 +++++--- 4 files changed, 155 insertions(+), 40 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 3a163bd6f..c1f9c35f5 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -21,17 +21,17 @@ jobs: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - nix-build-aarch64-linux: + nix-build-packages-aarch64-linux: name: >- ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} (aarch64-linux) needs: nix-eval runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_linux != null }} + if: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).aarch64_linux != null }} strategy: fail-fast: false max-parallel: 5 - matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_linux }} + matrix: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).aarch64_linux }} steps: - name: Checkout Repo if: ${{ matrix.attr != '' }} @@ -52,17 +52,71 @@ jobs: shell: bash run: nix build --accept-flake-config -L .#${{ matrix.attr }} - nix-build-aarch64-darwin: + nix-build-checks-aarch64-linux: + name: >- + ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} + (aarch64-linux) + needs: [nix-eval, nix-build-packages-aarch64-linux] + runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).aarch64_linux != null }} + strategy: + fail-fast: false + max-parallel: 5 + matrix: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).aarch64_linux }} + steps: + - name: Checkout Repo + if: ${{ matrix.attr != '' }} + uses: actions/checkout@v4 + - name: Install nix (ephemeral) + if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} + uses: ./.github/actions/nix-install-ephemeral + with: + push-to-cache: 'true' + env: + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} + NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} + - name: Install nix (self-hosted) + if: ${{ matrix.attr != '' && matrix.runs_on.group == 'self-hosted-runners-nix' }} + uses: ./.github/actions/nix-install-self-hosted + - name: nix build + if: ${{ matrix.attr != '' }} + shell: bash + run: nix build --accept-flake-config -L .#${{ matrix.attr }} + + nix-build-packages-aarch64-darwin: name: >- ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} (aarch64-darwin) needs: nix-eval runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin != null }} + if: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).aarch64_darwin != null }} + strategy: + fail-fast: false + max-parallel: 5 + matrix: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).aarch64_darwin }} + steps: + - name: Checkout Repo + if: ${{ matrix.attr != '' }} + uses: actions/checkout@v4 + - name: Install nix + if: ${{ matrix.attr != '' }} + uses: ./.github/actions/nix-install-self-hosted + - name: nix build + if: ${{ matrix.attr != '' }} + shell: bash + run: nix build --accept-flake-config -L .#${{ matrix.attr }} + + nix-build-checks-aarch64-darwin: + name: >- + ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} + (aarch64-darwin) + needs: [nix-eval, nix-build-packages-aarch64-darwin] + runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).aarch64_darwin != null }} strategy: fail-fast: false max-parallel: 5 - matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin }} + matrix: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).aarch64_darwin }} steps: - name: Checkout Repo if: ${{ matrix.attr != '' }} @@ -75,17 +129,45 @@ jobs: shell: bash run: nix build --accept-flake-config -L .#${{ matrix.attr }} - nix-build-x86_64-linux: + nix-build-packages-x86_64-linux: name: >- ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} (x86_64-linux) needs: nix-eval runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux != null }} + if: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).x86_64_linux != null }} + strategy: + fail-fast: false + max-parallel: 5 + matrix: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).x86_64_linux }} + steps: + - name: Checkout Repo + if: ${{ matrix.attr != '' }} + uses: actions/checkout@v4 + - name: Install nix + if: ${{ matrix.attr != '' }} + uses: ./.github/actions/nix-install-ephemeral + with: + push-to-cache: 'true' + env: + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} + NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} + - name: nix build + if: ${{ matrix.attr != '' }} + shell: bash + run: nix build --accept-flake-config -L .#${{ matrix.attr }} + + nix-build-checks-x86_64-linux: + name: >- + ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} + (x86_64-linux) + needs: [nix-eval, nix-build-packages-x86_64-linux] + runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).x86_64_linux != null }} strategy: fail-fast: false max-parallel: 5 - matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux }} + matrix: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).x86_64_linux }} steps: - name: Checkout Repo if: ${{ matrix.attr != '' }} @@ -104,23 +186,29 @@ jobs: run: nix build --accept-flake-config -L .#${{ matrix.attr }} run-testinfra: - needs: [nix-eval, nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] + needs: [nix-eval, nix-build-packages-aarch64-linux, nix-build-checks-aarch64-linux, nix-build-packages-aarch64-darwin, nix-build-checks-aarch64-darwin, nix-build-packages-x86_64-linux, nix-build-checks-x86_64-linux] if: | !cancelled() && needs.nix-eval.result == 'success' && - (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && - (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') && - (needs.nix-build-x86_64-linux.result == 'skipped' || needs.nix-build-x86_64-linux.result == 'success') + (needs.nix-build-packages-aarch64-linux.result == 'skipped' || needs.nix-build-packages-aarch64-linux.result == 'success') && + (needs.nix-build-checks-aarch64-linux.result == 'skipped' || needs.nix-build-checks-aarch64-linux.result == 'success') && + (needs.nix-build-packages-aarch64-darwin.result == 'skipped' || needs.nix-build-packages-aarch64-darwin.result == 'success') && + (needs.nix-build-checks-aarch64-darwin.result == 'skipped' || needs.nix-build-checks-aarch64-darwin.result == 'success') && + (needs.nix-build-packages-x86_64-linux.result == 'skipped' || needs.nix-build-packages-x86_64-linux.result == 'success') && + (needs.nix-build-checks-x86_64-linux.result == 'skipped' || needs.nix-build-checks-x86_64-linux.result == 'success') uses: ./.github/workflows/testinfra-ami-build.yml secrets: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} run-tests: - needs: [nix-eval, nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] + needs: [nix-eval, nix-build-packages-aarch64-linux, nix-build-checks-aarch64-linux, nix-build-packages-aarch64-darwin, nix-build-checks-aarch64-darwin, nix-build-packages-x86_64-linux, nix-build-checks-x86_64-linux] if: | !cancelled() && needs.nix-eval.result == 'success' && - (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && - (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') && - (needs.nix-build-x86_64-linux.result == 'skipped' || needs.nix-build-x86_64-linux.result == 'success') + (needs.nix-build-packages-aarch64-linux.result == 'skipped' || needs.nix-build-packages-aarch64-linux.result == 'success') && + (needs.nix-build-checks-aarch64-linux.result == 'skipped' || needs.nix-build-checks-aarch64-linux.result == 'success') && + (needs.nix-build-packages-aarch64-darwin.result == 'skipped' || needs.nix-build-packages-aarch64-darwin.result == 'success') && + (needs.nix-build-checks-aarch64-darwin.result == 'skipped' || needs.nix-build-checks-aarch64-darwin.result == 'success') && + (needs.nix-build-packages-x86_64-linux.result == 'skipped' || needs.nix-build-packages-x86_64-linux.result == 'success') && + (needs.nix-build-checks-x86_64-linux.result == 'skipped' || needs.nix-build-checks-x86_64-linux.result == 'success') uses: ./.github/workflows/test.yml diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml index cc092b355..7f47be3df 100644 --- a/.github/workflows/nix-eval.yml +++ b/.github/workflows/nix-eval.yml @@ -3,9 +3,12 @@ name: Nix Eval on: workflow_call: outputs: - matrix: - description: 'Generated build matrix' - value: ${{ jobs.eval.outputs.matrix }} + packages_matrix: + description: 'Generated build matrix for packages' + value: ${{ jobs.eval.outputs.packages_matrix }} + checks_matrix: + description: 'Generated build matrix for checks' + value: ${{ jobs.eval.outputs.checks_matrix }} secrets: DEV_AWS_ROLE: required: false @@ -16,7 +19,8 @@ jobs: eval: runs-on: blacksmith-32vcpu-ubuntu-2404 outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + packages_matrix: ${{ steps.set-matrix.outputs.packages_matrix }} + checks_matrix: ${{ steps.set-matrix.outputs.checks_matrix }} steps: - name: Checkout Repo uses: actions/checkout@v4 diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 638a83365..c9f22dc4a 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -154,6 +154,7 @@ let doCheck = false; postInstall = '' + create_control_files() { sed -e "/^default_version =/d" \ -e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}-${version}'|" \ @@ -220,12 +221,10 @@ let v: !(builtins.elem v versions) ) allPreviouslyPackagedVersions; numberOfPreviouslyPackagedVersions = builtins.length previouslyPackagedVersions; - packagesAttrSet = lib.mapAttrs' ( - name: value: { - name = lib.replaceStrings ["."] ["_"] name; - value = build name value.hash value.rust value.pgrx; - } - ) supportedVersions; + packagesAttrSet = lib.mapAttrs' (name: value: { + name = lib.replaceStrings [ "." ] [ "_" ] name; + value = build name value.hash value.rust value.pgrx; + }) supportedVersions; packages = builtins.attrValues packagesAttrSet; in (buildEnv { diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 7743753e5..7a3bb08d6 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -181,7 +181,7 @@ def run_nix_eval_jobs( for line in stdout_data.splitlines(): result = parse_nix_eval_line(line, drv_paths) if result.is_err(): - errors_list.append(result.err_value) + errors_list.append(result._value) elif result._value is not None: packages.append(result._value) @@ -306,32 +306,55 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: returned_pkg["postgresql_version"] = attrs[-3].split("_")[-1] return returned_pkg - # Group packages by system - grouped_by_system = defaultdict(list) + # Group packages by system and type (checks vs packages) + packages_by_system: Dict[System, List[GitHubActionPackage]] = defaultdict(list) + checks_by_system: Dict[System, List[GitHubActionPackage]] = defaultdict(list) for pkg in gh_action_packages: if pkg.get("cacheStatus") == "notBuilt": - grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) + cleaned_pkg = clean_package_for_output(pkg) + if pkg["attr"].startswith("checks."): + checks_by_system[pkg["system"]].append(cleaned_pkg) + elif pkg["attr"].startswith("legacyPackages."): + packages_by_system[pkg["system"]].append(cleaned_pkg) - # Create output with system-specific matrices - # Ensure that we have at least one entry per system - gh_output = {} - for system, packages in grouped_by_system.items(): - gh_output[system.replace("-", "_")] = {"include": packages} + packages_output: Dict[str, Dict[str, List[GitHubActionPackage]]] = {} + for pkg_system, pkg_list in packages_by_system.items(): + packages_output[pkg_system.replace("-", "_")] = {"include": pkg_list} + + checks_output: Dict[str, Dict[str, List[GitHubActionPackage]]] = {} + for check_system, check_list in checks_by_system.items(): + checks_output[check_system.replace("-", "_")] = {"include": check_list} for system in get_args(System): s = system.replace("-", "_") - if s not in gh_output: - gh_output[s] = { + if s not in checks_output: + checks_output[s] = { + "include": [ + { + "attr": "", + "name": "no checks to build", + "system": system, + "runs_on": {"labels": ["ubuntu-latest"]}, + } + ] + } + if s not in packages_output: + packages_output[s] = { "include": [ { "attr": "", "name": "no packages to build", "system": system, - "runs_on": {"labels": "ubuntu-latest"}, + "runs_on": {"labels": ["ubuntu-latest"]}, } ] } + gh_output = { + "packages": packages_output, + "checks": checks_output, + } + if warnings_list: warning_counts = Counter(warnings_list) for warn_msg, count in warning_counts.items(): @@ -365,7 +388,8 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: "\n", "%0A" ) notice(formatted_msg, title="GitHub Actions Matrix") - set_output("matrix", json.dumps(gh_output)) + set_output("packages_matrix", json.dumps(gh_output["packages"])) + set_output("checks_matrix", json.dumps(gh_output["checks"])) if __name__ == "__main__": From 16ec6f8ff740719dc62330c6e76dcad9463205f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 2 Dec 2025 21:57:01 +0100 Subject: [PATCH 61/71] fix(ci): use depot runners --- nix/packages/github-matrix/github_matrix.py | 8 ++++---- nix/packages/github-matrix/tests/test_github_matrix.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 7a3bb08d6..f7e08568c 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -70,10 +70,10 @@ class NixEvalError(TypedDict): BUILD_RUNNER_MAP: Dict[RunnerType, Dict[System, RunsOnConfig]] = { "ephemeral": { "aarch64-linux": { - "labels": ["blacksmith-4vcpu-ubuntu-2404-arm"], + "labels": ["depot-ubuntu-24.04-arm-8"], }, "x86_64-linux": { - "labels": ["blacksmith-8vcpu-ubuntu-2404"], + "labels": ["depot-ubuntu-24.04-8"], }, }, "self-hosted": { @@ -262,8 +262,8 @@ def get_runner_for_package(pkg: NixEvalJobsOutput) -> RunsOnConfig | None: return runConfig if is_large_pkg(pkg) and system in ("x86_64-linux", "aarch64-linux"): - suffix = "-arm" if system == "aarch64-linux" else "" - return {"labels": [f"blacksmith-32vcpu-ubuntu-2404{suffix}"]} + arch = "arm-" if system == "aarch64-linux" else "" + return {"labels": [f"depot-ubuntu-24.04-{arch}32"]} if system == "aarch64-darwin": return BUILD_RUNNER_MAP["self-hosted"]["aarch64-darwin"] diff --git a/nix/packages/github-matrix/tests/test_github_matrix.py b/nix/packages/github-matrix/tests/test_github_matrix.py index e977c2a6a..f92e9b44d 100644 --- a/nix/packages/github-matrix/tests/test_github_matrix.py +++ b/nix/packages/github-matrix/tests/test_github_matrix.py @@ -142,7 +142,7 @@ def test_large_package_x86_64_linux(self): "requiredSystemFeatures": ["big-parallel"], } result = get_runner_for_package(pkg) - assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404"]} + assert result == {"labels": ["depot-ubuntu-24.04-32"]} def test_large_package_aarch64_linux(self): pkg: NixEvalJobsOutput = { @@ -161,7 +161,7 @@ def test_large_package_aarch64_linux(self): "requiredSystemFeatures": ["big-parallel"], } result = get_runner_for_package(pkg) - assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404-arm"]} + assert result == {"labels": ["depot-ubuntu-24.04-arm-32"]} def test_darwin_package(self): pkg: NixEvalJobsOutput = { @@ -194,7 +194,7 @@ def test_default_x86_64_linux(self): "system": "x86_64-linux", } result = get_runner_for_package(pkg) - assert result == {"labels": ["blacksmith-8vcpu-ubuntu-2404"]} + assert result == {"labels": ["depot-ubuntu-24.04-8"]} def test_default_aarch64_linux(self): pkg: NixEvalJobsOutput = { @@ -212,7 +212,7 @@ def test_default_aarch64_linux(self): "system": "aarch64-linux", } result = get_runner_for_package(pkg) - assert result == {"labels": ["blacksmith-4vcpu-ubuntu-2404-arm"]} + assert result == {"labels": ["depot-ubuntu-24.04-arm-8"]} class TestSortPkgsByClosures: From 5161d38172f919d1af70bd94dd76063e94309614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 2 Dec 2025 21:57:01 +0100 Subject: [PATCH 62/71] chore: test ci performance --- nix/ext/wrappers/default.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index c9f22dc4a..a443bb9e5 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -154,7 +154,6 @@ let doCheck = false; postInstall = '' - create_control_files() { sed -e "/^default_version =/d" \ -e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}-${version}'|" \ @@ -163,6 +162,9 @@ let } create_control_files + + + ''; meta = with lib; { From 81df1541647df35597cb2f5bea5aeec61d81b335 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 4 Dec 2025 10:49:28 -0500 Subject: [PATCH 63/71] Revert "fix(ci): use depot runners" This reverts commit 55505ce8abd840fb12c6f14e8f9789e1e786e98b. --- nix/packages/github-matrix/github_matrix.py | 8 ++++---- nix/packages/github-matrix/tests/test_github_matrix.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index f7e08568c..7a3bb08d6 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -70,10 +70,10 @@ class NixEvalError(TypedDict): BUILD_RUNNER_MAP: Dict[RunnerType, Dict[System, RunsOnConfig]] = { "ephemeral": { "aarch64-linux": { - "labels": ["depot-ubuntu-24.04-arm-8"], + "labels": ["blacksmith-4vcpu-ubuntu-2404-arm"], }, "x86_64-linux": { - "labels": ["depot-ubuntu-24.04-8"], + "labels": ["blacksmith-8vcpu-ubuntu-2404"], }, }, "self-hosted": { @@ -262,8 +262,8 @@ def get_runner_for_package(pkg: NixEvalJobsOutput) -> RunsOnConfig | None: return runConfig if is_large_pkg(pkg) and system in ("x86_64-linux", "aarch64-linux"): - arch = "arm-" if system == "aarch64-linux" else "" - return {"labels": [f"depot-ubuntu-24.04-{arch}32"]} + suffix = "-arm" if system == "aarch64-linux" else "" + return {"labels": [f"blacksmith-32vcpu-ubuntu-2404{suffix}"]} if system == "aarch64-darwin": return BUILD_RUNNER_MAP["self-hosted"]["aarch64-darwin"] diff --git a/nix/packages/github-matrix/tests/test_github_matrix.py b/nix/packages/github-matrix/tests/test_github_matrix.py index f92e9b44d..e977c2a6a 100644 --- a/nix/packages/github-matrix/tests/test_github_matrix.py +++ b/nix/packages/github-matrix/tests/test_github_matrix.py @@ -142,7 +142,7 @@ def test_large_package_x86_64_linux(self): "requiredSystemFeatures": ["big-parallel"], } result = get_runner_for_package(pkg) - assert result == {"labels": ["depot-ubuntu-24.04-32"]} + assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404"]} def test_large_package_aarch64_linux(self): pkg: NixEvalJobsOutput = { @@ -161,7 +161,7 @@ def test_large_package_aarch64_linux(self): "requiredSystemFeatures": ["big-parallel"], } result = get_runner_for_package(pkg) - assert result == {"labels": ["depot-ubuntu-24.04-arm-32"]} + assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404-arm"]} def test_darwin_package(self): pkg: NixEvalJobsOutput = { @@ -194,7 +194,7 @@ def test_default_x86_64_linux(self): "system": "x86_64-linux", } result = get_runner_for_package(pkg) - assert result == {"labels": ["depot-ubuntu-24.04-8"]} + assert result == {"labels": ["blacksmith-8vcpu-ubuntu-2404"]} def test_default_aarch64_linux(self): pkg: NixEvalJobsOutput = { @@ -212,7 +212,7 @@ def test_default_aarch64_linux(self): "system": "aarch64-linux", } result = get_runner_for_package(pkg) - assert result == {"labels": ["depot-ubuntu-24.04-arm-8"]} + assert result == {"labels": ["blacksmith-4vcpu-ubuntu-2404-arm"]} class TestSortPkgsByClosures: From 47e3fb76a7cee71eba58e6549335b1f9a817b666 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 4 Dec 2025 11:00:22 -0500 Subject: [PATCH 64/71] test: trigger full rebuild --- nix/ext/wrappers/default.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index a443bb9e5..8a9867dae 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -32,6 +32,9 @@ let inherit hash; }; + + + nativeBuildInputs = [ pkg-config cargo From daf3cada0c55337aebf34c60fbc5e381f1eedd2c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 4 Dec 2025 11:12:00 -0500 Subject: [PATCH 65/71] Revert "test: trigger full rebuild" This reverts commit c67cc4f429960db1cde19103b75f39b3b911e8f7. --- nix/ext/wrappers/default.nix | 3 --- 1 file changed, 3 deletions(-) diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 8a9867dae..a443bb9e5 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -32,9 +32,6 @@ let inherit hash; }; - - - nativeBuildInputs = [ pkg-config cargo From 1fafab13357bbebaf875464e12fc1c757bc3a388 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 4 Dec 2025 11:17:00 -0500 Subject: [PATCH 66/71] tests: drop 1 extension to trigger rebuild --- nix/ext/versions.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/nix/ext/versions.json b/nix/ext/versions.json index b35c6c40b..c7862183b 100644 --- a/nix/ext/versions.json +++ b/nix/ext/versions.json @@ -582,14 +582,6 @@ } }, "wrappers": { - "0.3.0": { - "postgresql": [ - "15" - ], - "hash": "sha256-ogpF8NJ7kW3Ut8jaKMDiKYIXnI38nfRq2mMK4rqFAIA=", - "pgrx": "0.11.3", - "rust": "1.76.0" - }, "0.4.1": { "postgresql": [ "15" From 9bd9a14218fc1caffe9efd45092a0ec2e31774ea Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 4 Dec 2025 11:23:29 -0500 Subject: [PATCH 67/71] test: trigger full rebuild of wrappers --- nix/ext/wrappers/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index a443bb9e5..72b88a806 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -165,6 +165,7 @@ let + ''; meta = with lib; { From 97cb18db32fc6fad98fee328d38eb130c62311f6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 4 Dec 2025 19:13:14 -0500 Subject: [PATCH 68/71] test: retrigger build --- nix/ext/wrappers/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 72b88a806..079ec0a3c 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -304,6 +304,7 @@ in create_lib_files create_migration_sql_files + # Verify library count matches expected (test "$(ls -A $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)" = "${ toString (numberOfVersions + numberOfPreviouslyPackagedVersions + 1) From 7df180033eab9921641bafd50e1e9037a8b27928 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 5 Dec 2025 09:10:21 -0500 Subject: [PATCH 69/71] test: re-add oldest to trigger build again --- nix/ext/versions.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nix/ext/versions.json b/nix/ext/versions.json index c7862183b..b35c6c40b 100644 --- a/nix/ext/versions.json +++ b/nix/ext/versions.json @@ -582,6 +582,14 @@ } }, "wrappers": { + "0.3.0": { + "postgresql": [ + "15" + ], + "hash": "sha256-ogpF8NJ7kW3Ut8jaKMDiKYIXnI38nfRq2mMK4rqFAIA=", + "pgrx": "0.11.3", + "rust": "1.76.0" + }, "0.4.1": { "postgresql": [ "15" From e3172aa9513bfa677892cf32b7a6bd25e7e22ae2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 5 Dec 2025 15:19:56 +0100 Subject: [PATCH 70/71] chore: trigger CI --- nix/ext/wrappers/default.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 079ec0a3c..769ace525 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -154,18 +154,14 @@ let doCheck = false; postInstall = '' + create_control_files() { sed -e "/^default_version =/d" \ -e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}-${version}'|" \ $out/share/postgresql/extension/${pname}.control > $out/share/postgresql/extension/${pname}--${version}.control rm $out/share/postgresql/extension/${pname}.control } - create_control_files - - - - ''; meta = with lib; { From 0b8979d5fc5fb8ed7f9ab71e6eec832e389d7448 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 5 Dec 2025 13:36:47 -0500 Subject: [PATCH 71/71] test: retrigger build --- nix/ext/wrappers/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 769ace525..2e7a2af01 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -298,6 +298,8 @@ in create_control_files create_lib_files + + create_migration_sql_files