From 8f18482668941385ddbc37a913a8a8fff855b58b Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Tue, 22 Jul 2025 11:39:03 +0200 Subject: [PATCH 01/58] chores(ci): opted-out nix-fast-build --- .github/workflows/nix-build.yml | 4 +--- flake.lock | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 6a416709b..cc3c0b285 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -69,9 +69,7 @@ jobs: - name: Build psql bundle run: > - nix run "github:Mic92/nix-fast-build?rev=b1dae483ab7d4139a6297e02b6de9e5d30e43d48" - -- --skip-cached --no-nom ${{ matrix.runner == 'macos-latest-xlarge' && '--max-jobs 1' || '' }} - --flake ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" + nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" env: AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} diff --git a/flake.lock b/flake.lock index 0b1ec8191..ad09ae311 100644 --- a/flake.lock +++ b/flake.lock @@ -131,11 +131,11 @@ ] }, "locked": { - "lastModified": 1749427739, - "narHash": "sha256-Nm0oMqFNRnJsiZYeNChmefmjeVCOzngikpSQhgs7iXI=", + "lastModified": 1763868605, + "narHash": "sha256-wO8Lk66GPQeSpzXUzXCBpe2Pj1De17xByrROXxnwIPY=", "owner": "Mic92", "repo": "nix-fast-build", - "rev": "b1dae483ab7d4139a6297e02b6de9e5d30e43d48", + "rev": "0c7fd47fb587250e931a01c6645ab5c0cd737af8", "type": "github" }, "original": { From 985854bf20835dc9836f156d98917d5e26304066 Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Thu, 24 Jul 2025 09:42:53 +0200 Subject: [PATCH 02/58] chores(ci): set up nix-github-actions --- .github/workflows/nix-build.yml | 31 ++++++++++++++++++++----------- flake.lock | 21 +++++++++++++++++++++ flake.nix | 3 +++ nix/github-actions.nix | 6 ++++++ 4 files changed, 50 insertions(+), 11 deletions(-) create mode 100644 nix/github-actions.nix diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index cc3c0b285..b2e354483 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -19,18 +19,27 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + nix-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v30 + - id: set-matrix + name: Generate Nix Matrix + run: | + set -Eeu + matrix="$(nix eval --json '.#githubActions.matrix')" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + build-run-image: + name: ${{ matrix.name }} (${{ matrix.system }}) + needs: nix-matrix + runs-on: ${{ matrix.os }} strategy: fail-fast: false - matrix: - include: - - runner: blacksmith-32vcpu-ubuntu-2404 - arch: amd64 - - runner: blacksmith-32vcpu-ubuntu-2404-arm - arch: arm64 - - runner: macos-latest-xlarge - arch: arm64 - runs-on: ${{ matrix.runner }} + matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} timeout-minutes: 180 steps: - name: Checkout Repo @@ -66,10 +75,10 @@ jobs: sudo rm -rf /tmp/* 2>/dev/null || true echo "=== AFTER CLEANUP ===" df -h - - - name: Build psql bundle + - name: Build psql bundle run: > nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" + - run: nix build -L '.#${{ matrix.attr }}' env: AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} diff --git a/flake.lock b/flake.lock index ad09ae311..1d3909126 100644 --- a/flake.lock +++ b/flake.lock @@ -144,6 +144,26 @@ "type": "github" } }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1737420293, + "narHash": "sha256-F1G5ifvqTpJq7fdkT34e/Jy9VCyzd5XfJ9TO8fHhJWE=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "f4158fa080ef4503c8f4c820967d946c2af31ec9", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, "nix2container": { "inputs": { "flake-utils": [ @@ -237,6 +257,7 @@ "git-hooks": "git-hooks", "nix-editor": "nix-editor", "nix-fast-build": "nix-fast-build", + "nix-github-actions": "nix-github-actions", "nix2container": "nix2container", "nixpkgs": "nixpkgs", "nixpkgs-go124": "nixpkgs-go124", diff --git a/flake.nix b/flake.nix index 8f83a0a25..4b4f18230 100644 --- a/flake.nix +++ b/flake.nix @@ -28,6 +28,8 @@ git-hooks.inputs.nixpkgs.follows = "nixpkgs"; nixpkgs-go124.url = "github:Nixos/nixpkgs/d2ac4dfa61fba987a84a0a81555da57ae0b9a2b0"; nixpkgs-pgbackrest.url = "github:nixos/nixpkgs/nixos-unstable-small"; + nix-github-actions.url = "github:nix-community/nix-github-actions"; + nix-github-actions.inputs.nixpkgs.follows = "nixpkgs"; }; outputs = @@ -49,6 +51,7 @@ nix/nixpkgs.nix nix/packages nix/overlays + nix/github-actions.nix ]; }); } diff --git a/nix/github-actions.nix b/nix/github-actions.nix new file mode 100644 index 000000000..b2f04a6c2 --- /dev/null +++ b/nix/github-actions.nix @@ -0,0 +1,6 @@ +{ inputs, ... }: +{ + flake.githubActions = inputs.nix-github-actions.lib.mkGithubMatrix { + checks = inputs.nixpkgs.lib.getAttrs [ "x86_64-linux" ] inputs.self.checks; + }; +} From 173f556dc0368d23d8db4a7c38cc05f51ecb2b2d Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Mon, 11 Aug 2025 09:11:45 +0200 Subject: [PATCH 03/58] chores(ci): comment out vestigal testinfra-ami-build --- .github/workflows/nix-build.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index b2e354483..168ef0119 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -10,7 +10,7 @@ on: permissions: id-token: write - # required by testinfra-ami-build dependent workflows + # required by dependent workflows contents: write packages: write @@ -84,13 +84,6 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} - run-testinfra: - needs: build-run-image - if: ${{ success() }} - uses: ./.github/workflows/testinfra-ami-build.yml - secrets: - DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} - run-tests: needs: build-run-image if: ${{ success() }} From d2855d08677e662e462033f138c746bafcf6cedc Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Mon, 11 Aug 2025 12:10:27 +0200 Subject: [PATCH 04/58] chores(ci): use custom github runners --- nix/github-actions.nix | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/nix/github-actions.nix b/nix/github-actions.nix index b2f04a6c2..752f0b41a 100644 --- a/nix/github-actions.nix +++ b/nix/github-actions.nix @@ -1,6 +1,18 @@ { inputs, ... }: +let + githubPlatforms = { + "x86_64-linux" = "large-linux-x86"; + "aarch64-linux" = "large-linux-arm"; + "aarch64-darwin" = "macos-latest-xlarge"; + }; +in { flake.githubActions = inputs.nix-github-actions.lib.mkGithubMatrix { - checks = inputs.nixpkgs.lib.getAttrs [ "x86_64-linux" ] inputs.self.checks; + checks = inputs.nixpkgs.lib.getAttrs [ + "x86_64-linux" + "aarch64-linux" + "aarch64-darwin" + ] inputs.self.checks; + platforms = githubPlatforms; }; } From 84b0e7d450a00c6b1d0fcaf792375b587dd53be0 Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Mon, 29 Sep 2025 14:38:39 +0200 Subject: [PATCH 05/58] chores(ci): use nix-eval-jobs and sets AWS creds to /etc/nix/aws --- .github/workflows/nix-build.yml | 80 +++++++++++++++------------------ nix/github-actions.nix | 6 +-- 2 files changed, 40 insertions(+), 46 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 168ef0119..d4b5d1a49 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -20,70 +20,64 @@ concurrency: jobs: nix-matrix: - runs-on: ubuntu-latest + runs-on: + group: self-hosted-runners-nix + labels: + - aarch64-darwin outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v30 + - name: Checkout Repo + uses: actions/checkout@v4 - id: set-matrix name: Generate Nix Matrix run: | set -Eeu - matrix="$(nix eval --json '.#githubActions.matrix')" - echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + echo matrix="$(nix eval --json '.#githubActions.matrix')" >> "$GITHUB_OUTPUT" build-run-image: name: ${{ matrix.name }} (${{ matrix.system }}) needs: nix-matrix - runs-on: ${{ matrix.os }} + runs-on: + group: ${{ contains(matrix.os, 'blacksmith-32vcpu-ubuntu-2404') && '' || 'self-hosted-runners-nix' }} + labels: + - ${{ matrix.os }} strategy: fail-fast: false matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} - timeout-minutes: 180 steps: - name: Checkout Repo - uses: supabase/postgres/.github/actions/shared-checkout@HEAD - - uses: ./.github/actions/nix-install-ephemeral + uses: actions/checkout@v4 + - name: aws-oidc + uses: aws-actions/configure-aws-credentials@v4.3.1 + with: + aws-region: us-east-2 + role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role # Shared Services + role-session-name: gha-oidc-${{ github.run_id }} + - name: aws-creds + uses: aws-actions/configure-aws-credentials@v4.3.1 with: - push-to-cache: ${{ github.secret_source == 'Actions' && 'true' || 'false' }} - env: - DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} - NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - - name: Aggressive disk cleanup for DuckDB build - if: matrix.runner == 'macos-latest-xlarge' + disable-retry: true + aws-region: us-east-2 + role-to-assume: arn:aws:iam::436098097459:role/nix-artifacts-deploy-role # supabase-dev + role-session-name: gha-oidc-${{ github.run_id }} + role-chaining: true + role-skip-session-tagging: true + role-duration-seconds: 900 # TODO: switch to 18000 (5 hours) + - name: Write creds files run: | - nix --version - echo "=== BEFORE CLEANUP ===" - df -h - # Remove major space consumers - sudo rm -rf /usr/share/dotnet || true - sudo rm -rf /usr/local/lib/android || true - sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform || true - sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/watchOS.platform || true - sudo rm -rf /Applications/Xcode.app/Contents/Developer/Platforms/tvOS.platform || true - # Clean everything possible - sudo rm -rf /opt/ghc || true - sudo rm -rf /usr/local/share/boost || true - sudo rm -rf /opt/homebrew || true - sudo xcrun simctl delete all 2>/dev/null || true - # Aggressive cache cleanup - sudo rm -rf /System/Library/Caches/* 2>/dev/null || true - sudo rm -rf /Library/Caches/* 2>/dev/null || true - sudo rm -rf ~/Library/Caches/* 2>/dev/null || true - sudo rm -rf /private/var/log/* 2>/dev/null || true - sudo rm -rf /tmp/* 2>/dev/null || true - echo "=== AFTER CLEANUP ===" - df -h + umask 006 + cat > /etc/nix/aws/nix-aws-credentials < nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - - run: nix build -L '.#${{ matrix.attr }}' - env: - AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} - AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} - + - name: nix-fast-build + run: nix build -L run-tests: needs: build-run-image if: ${{ success() }} diff --git a/nix/github-actions.nix b/nix/github-actions.nix index 752f0b41a..ad43f2a6f 100644 --- a/nix/github-actions.nix +++ b/nix/github-actions.nix @@ -1,9 +1,9 @@ { inputs, ... }: let githubPlatforms = { - "x86_64-linux" = "large-linux-x86"; - "aarch64-linux" = "large-linux-arm"; - "aarch64-darwin" = "macos-latest-xlarge"; + "aarch64-linux" = "aarch64-linux"; + "aarch64-darwin" = "aarch64-darwin"; + "x86_64-linux" = "blacksmith-32vcpu-ubuntu-2404"; }; in { From 61d34f48189cd6dc00126efd122713a67c4822ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 29 Sep 2025 22:15:14 +0200 Subject: [PATCH 06/58] feat(ci): do not build if already cached --- .github/workflows/nix-build.yml | 21 +++-- scripts/github-matrix.py | 151 ++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 11 deletions(-) create mode 100644 scripts/github-matrix.py diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index d4b5d1a49..0c953b98f 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -14,10 +14,6 @@ permissions: contents: write packages: write -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - jobs: nix-matrix: runs-on: @@ -33,15 +29,12 @@ jobs: name: Generate Nix Matrix run: | set -Eeu - echo matrix="$(nix eval --json '.#githubActions.matrix')" >> "$GITHUB_OUTPUT" + echo matrix="$(python scripts/github-matrix.py)" >> "$GITHUB_OUTPUT" build-run-image: name: ${{ matrix.name }} (${{ matrix.system }}) needs: nix-matrix - runs-on: - group: ${{ contains(matrix.os, 'blacksmith-32vcpu-ubuntu-2404') && '' || 'self-hosted-runners-nix' }} - labels: - - ${{ matrix.os }} + runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} strategy: fail-fast: false matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} @@ -76,8 +69,14 @@ jobs: - name: Build psql bundle run: > nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - - name: nix-fast-build - run: nix build -L + - name: nix build + run: | + if ${{ matrix.already_cached }}; then + echo "${{ matrix.attr }} already cached, skipping build" + exit 0 + fi + nix build -L .#${{ matrix.attr }} + run-tests: needs: build-run-image if: ${{ success() }} diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py new file mode 100644 index 000000000..53deb4f96 --- /dev/null +++ b/scripts/github-matrix.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 + +import json +import os +import subprocess +import sys +from typing import ( + Any, + Dict, + Generator, + List, + Literal, + NotRequired, + Optional, + Set, + TypedDict, +) + + +class NixEvalJobsOutput(TypedDict): + """Raw output from nix-eval-jobs command.""" + + attr: str + attrPath: List[str] + cacheStatus: Literal["notBuilt", "cached", "local"] + drvPath: str + isCached: bool + name: str + system: str + neededBuilds: NotRequired[List[Any]] + neededSubstitutes: NotRequired[List[Any]] + outputs: NotRequired[Dict[str, str]] + + +class RunsOnConfig(TypedDict): + """GitHub Actions runs-on configuration.""" + + group: NotRequired[str] + labels: List[str] + + +class GitHubActionPackage(TypedDict): + """Processed package for GitHub Actions matrix.""" + + attr: str + name: str + system: str + already_cached: bool + runs_on: RunsOnConfig + + +BUILD_RUNNER_MAP: Dict[str, RunsOnConfig] = { + "aarch64-linux": { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-linux"], + }, + "aarch64-darwin": { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-darwin"], + }, + "x86_64-linux": { + "labels": ["blacksmith-32vcpu-ubuntu-2404"], + }, +} + + +def get_worker_count() -> int: + """Get optimal worker count based on CPU cores.""" + try: + return max(1, int(os.cpu_count())) + except (OSError, AttributeError): + print( + "Warning: Unable to get CPU count, using default max_workers=1", + file=sys.stderr, + ) + return 1 + + +def build_nix_eval_command(max_workers: int) -> List[str]: + """Build the nix-eval-jobs command with appropriate flags.""" + return [ + "nix-eval-jobs", + "--flake", + ".#checks", + "--check-cache-status", + "--force-recurse", + "--quiet", + "--workers", + str(max_workers), + ] + + +def parse_nix_eval_line( + line: str, drv_paths: Set[str] +) -> Optional[GitHubActionPackage]: + """Parse a single line of nix-eval-jobs output""" + if not line.strip(): + return None + + try: + data: NixEvalJobsOutput = json.loads(line) + if data["drvPath"] in drv_paths: + return None + drv_paths.add(data["drvPath"]) + print(f"Processing package: {data}", file=sys.stderr) + + runs_on_config = BUILD_RUNNER_MAP[data["system"]] + + return { + "attr": "checks." + data["attr"], + "name": data["name"], + "system": data["system"], + "already_cached": data.get("cacheStatus") != "notBuilt", + "runs_on": runs_on_config, + } + except json.JSONDecodeError: + print(f"Skipping invalid JSON line: {line}", file=sys.stderr) + return None + + +def run_nix_eval_jobs(cmd: List[str]) -> Generator[GitHubActionPackage, None, None]: + """Run nix-eval-jobs and yield parsed package data.""" + print(f"Running command: {' '.join(cmd)}", file=sys.stderr) + + with subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) as process: + drv_paths = set() + + for line in process.stdout: + package = parse_nix_eval_line(line, drv_paths) + if package: + yield package + + if process.returncode and process.returncode != 0: + print("Error: Evaluation failed", file=sys.stderr) + sys.stderr.write(process.stderr.read()) + sys.exit(process.returncode) + + +def main() -> None: + max_workers = get_worker_count() + cmd = build_nix_eval_command(max_workers) + + gh_action_packages = list(run_nix_eval_jobs(cmd)) + gh_output = {"include": gh_action_packages} + print(json.dumps(gh_output)) + + +if __name__ == "__main__": + main() From 81c03b8c1ac33af1430e7b8892f48c0ec9eb6fcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 29 Sep 2025 23:45:46 +0200 Subject: [PATCH 07/58] chore: improve reproducibility of postgresql builds Do not create postgresql package that depends on the runtime evaluation context as that harms reproducibility. --- nix/postgresql/generic.nix | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/nix/postgresql/generic.nix b/nix/postgresql/generic.nix index 76904ced7..ea67cbc91 100644 --- a/nix/postgresql/generic.nix +++ b/nix/postgresql/generic.nix @@ -21,7 +21,6 @@ let libxml2, tzdata, libkrb5, - substituteAll, darwin, linux-pam, #orioledb specific @@ -193,11 +192,7 @@ let ./patches/paths-for-split-outputs.patch ./patches/specify_pkglibdir_at_runtime.patch ./patches/paths-with-postgresql-suffix.patch - - (substituteAll { - src = ./patches/locale-binary-path.patch; - locale = "${if stdenv.isDarwin then darwin.adv_cmds else lib.getBin stdenv.cc.libc}/bin/locale"; - }) + ./patches/locale-binary-path.patch ] ++ lib.optionals stdenv'.hostPlatform.isMusl ( # Using fetchurl instead of fetchpatch on purpose: https://github.com/NixOS/nixpkgs/issues/240141 @@ -213,6 +208,9 @@ let '' # Hardcode the path to pgxs so pg_config returns the path in $out substituteInPlace "src/common/config_info.c" --subst-var out + substituteInPlace "src/backend/commands/collationcmds.c" --replace-fail '@locale@' '${ + if stdenv.isDarwin then darwin.adv_cmds else lib.getBin stdenv.cc.libc + }/bin/locale' '' + lib.optionalString jitSupport '' # Force lookup of jit stuff in $out instead of $lib From 8ff188a99a231d7fd4f88dd746eee3bfbba01c4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 29 Sep 2025 23:55:01 +0200 Subject: [PATCH 08/58] chore: remove nix-github-actions dependency --- flake.lock | 21 --------------------- flake.nix | 3 --- nix/github-actions.nix | 18 ------------------ 3 files changed, 42 deletions(-) delete mode 100644 nix/github-actions.nix diff --git a/flake.lock b/flake.lock index 1d3909126..ad09ae311 100644 --- a/flake.lock +++ b/flake.lock @@ -144,26 +144,6 @@ "type": "github" } }, - "nix-github-actions": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1737420293, - "narHash": "sha256-F1G5ifvqTpJq7fdkT34e/Jy9VCyzd5XfJ9TO8fHhJWE=", - "owner": "nix-community", - "repo": "nix-github-actions", - "rev": "f4158fa080ef4503c8f4c820967d946c2af31ec9", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nix-github-actions", - "type": "github" - } - }, "nix2container": { "inputs": { "flake-utils": [ @@ -257,7 +237,6 @@ "git-hooks": "git-hooks", "nix-editor": "nix-editor", "nix-fast-build": "nix-fast-build", - "nix-github-actions": "nix-github-actions", "nix2container": "nix2container", "nixpkgs": "nixpkgs", "nixpkgs-go124": "nixpkgs-go124", diff --git a/flake.nix b/flake.nix index 4b4f18230..8f83a0a25 100644 --- a/flake.nix +++ b/flake.nix @@ -28,8 +28,6 @@ git-hooks.inputs.nixpkgs.follows = "nixpkgs"; nixpkgs-go124.url = "github:Nixos/nixpkgs/d2ac4dfa61fba987a84a0a81555da57ae0b9a2b0"; nixpkgs-pgbackrest.url = "github:nixos/nixpkgs/nixos-unstable-small"; - nix-github-actions.url = "github:nix-community/nix-github-actions"; - nix-github-actions.inputs.nixpkgs.follows = "nixpkgs"; }; outputs = @@ -51,7 +49,6 @@ nix/nixpkgs.nix nix/packages nix/overlays - nix/github-actions.nix ]; }); } diff --git a/nix/github-actions.nix b/nix/github-actions.nix deleted file mode 100644 index ad43f2a6f..000000000 --- a/nix/github-actions.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ inputs, ... }: -let - githubPlatforms = { - "aarch64-linux" = "aarch64-linux"; - "aarch64-darwin" = "aarch64-darwin"; - "x86_64-linux" = "blacksmith-32vcpu-ubuntu-2404"; - }; -in -{ - flake.githubActions = inputs.nix-github-actions.lib.mkGithubMatrix { - checks = inputs.nixpkgs.lib.getAttrs [ - "x86_64-linux" - "aarch64-linux" - "aarch64-darwin" - ] inputs.self.checks; - platforms = githubPlatforms; - }; -} From 166cc457a2bbdbb231dbd7fe52e3c2c4092dd239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 11:23:47 +0200 Subject: [PATCH 09/58] feat(ci): split nix build workflow into separate extensions and checks jobs Split the monolithic nix-build job into two workflows: one for building PostgreSQL extensions and another for checks. Building extensions can be resource-intensive and time-consuming, so isolating them allows for better resource allocation and parallelism. Once they are built, the checks job can run tests and validations on the already built extensions. --- .github/workflows/nix-build.yml | 77 ++++++++++++++++++++++++++++----- nix/packages/postgres.nix | 6 +-- scripts/github-matrix.py | 56 +++++++++++++++++++----- 3 files changed, 114 insertions(+), 25 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 0c953b98f..a1a72dd92 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -15,7 +15,7 @@ permissions: packages: write jobs: - nix-matrix: + extensions-matrix: runs-on: group: self-hosted-runners-nix labels: @@ -29,15 +29,74 @@ jobs: name: Generate Nix Matrix run: | set -Eeu - echo matrix="$(python scripts/github-matrix.py)" >> "$GITHUB_OUTPUT" + echo matrix="$(python scripts/github-matrix.py extensions)" >> "$GITHUB_OUTPUT" - build-run-image: + build-extensions: + name: ${{matrix.postgresql_version}}.${{ matrix.name }} (${{ matrix.system }}) + needs: extensions-matrix + runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + strategy: + fail-fast: false + max-parallel: 3 + matrix: ${{fromJSON(needs.extensions-matrix.outputs.matrix)}} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: aws-oidc + uses: aws-actions/configure-aws-credentials@v4.3.1 + with: + aws-region: us-east-2 + role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role # Shared Services + role-session-name: gha-oidc-${{ github.run_id }} + - name: aws-creds + uses: aws-actions/configure-aws-credentials@v4.3.1 + with: + disable-retry: true + aws-region: us-east-2 + role-to-assume: arn:aws:iam::436098097459:role/nix-artifacts-deploy-role # supabase-dev + role-session-name: gha-oidc-${{ github.run_id }} + role-chaining: true + role-skip-session-tagging: true + role-duration-seconds: 3600 + - name: Write creds files + run: | + umask 006 + cat > /etc/nix/aws/nix-aws-credentials <> "$GITHUB_OUTPUT" + + + build-checks: name: ${{ matrix.name }} (${{ matrix.system }}) - needs: nix-matrix + needs: [checks-matrix, build-extensions] runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} strategy: fail-fast: false - matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} + matrix: ${{fromJSON(needs.checks-matrix.outputs.matrix)}} steps: - name: Checkout Repo uses: actions/checkout@v4 @@ -56,7 +115,7 @@ jobs: role-session-name: gha-oidc-${{ github.run_id }} role-chaining: true role-skip-session-tagging: true - role-duration-seconds: 900 # TODO: switch to 18000 (5 hours) + role-duration-seconds: 3600 - name: Write creds files run: | umask 006 @@ -71,13 +130,9 @@ jobs: nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - name: nix build run: | - if ${{ matrix.already_cached }}; then - echo "${{ matrix.attr }} already cached, skipping build" - exit 0 - fi nix build -L .#${{ matrix.attr }} run-tests: - needs: build-run-image + needs: build-checks if: ${{ success() }} uses: ./.github/workflows/test.yml diff --git a/nix/packages/postgres.nix b/nix/packages/postgres.nix index 2ce9410f6..1ff0f5a25 100644 --- a/nix/packages/postgres.nix +++ b/nix/packages/postgres.nix @@ -1,7 +1,7 @@ { inputs, ... }: { perSystem = - { pkgs, ... }: + { pkgs, lib, ... }: let # Custom extensions that exist in our repository. These aren't upstream # either because nobody has done the work, maintaining them here is @@ -153,10 +153,9 @@ # install. # - exts: an attrset containing all the extensions, mapped to their # package names. - makePostgres = version: { + makePostgres = version: lib.recurseIntoAttrs { bin = makePostgresBin version; exts = makeOurPostgresPkgsSet version; - recurseForDerivations = true; }; basePackages = { psql_15 = makePostgres "15"; @@ -166,5 +165,6 @@ in { packages = inputs.flake-utils.lib.flattenTree basePackages; + legacyPackages = basePackages; }; } diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 53deb4f96..7adef8cfa 100644 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import argparse import json import os import subprocess @@ -47,6 +48,7 @@ class GitHubActionPackage(TypedDict): system: str already_cached: bool runs_on: RunsOnConfig + postgresql_version: NotRequired[str] BUILD_RUNNER_MAP: Dict[str, RunsOnConfig] = { @@ -76,22 +78,23 @@ def get_worker_count() -> int: return 1 -def build_nix_eval_command(max_workers: int) -> List[str]: +def build_nix_eval_command(max_workers: int, target: str) -> List[str]: """Build the nix-eval-jobs command with appropriate flags.""" - return [ + nix_eval_cmd = [ "nix-eval-jobs", "--flake", - ".#checks", + f".#{target}", "--check-cache-status", "--force-recurse", "--quiet", "--workers", str(max_workers), ] + return nix_eval_cmd def parse_nix_eval_line( - line: str, drv_paths: Set[str] + line: str, drv_paths: Set[str], target: str ) -> Optional[GitHubActionPackage]: """Parse a single line of nix-eval-jobs output""" if not line.strip(): @@ -102,12 +105,11 @@ def parse_nix_eval_line( if data["drvPath"] in drv_paths: return None drv_paths.add(data["drvPath"]) - print(f"Processing package: {data}", file=sys.stderr) runs_on_config = BUILD_RUNNER_MAP[data["system"]] return { - "attr": "checks." + data["attr"], + "attr": f"{target}.{data['attr']}", "name": data["name"], "system": data["system"], "already_cached": data.get("cacheStatus") != "notBuilt", @@ -118,7 +120,9 @@ def parse_nix_eval_line( return None -def run_nix_eval_jobs(cmd: List[str]) -> Generator[GitHubActionPackage, None, None]: +def run_nix_eval_jobs( + cmd: List[str], target: str +) -> Generator[GitHubActionPackage, None, None]: """Run nix-eval-jobs and yield parsed package data.""" print(f"Running command: {' '.join(cmd)}", file=sys.stderr) @@ -128,8 +132,8 @@ def run_nix_eval_jobs(cmd: List[str]) -> Generator[GitHubActionPackage, None, No drv_paths = set() for line in process.stdout: - package = parse_nix_eval_line(line, drv_paths) - if package: + package = parse_nix_eval_line(line, drv_paths, target) + if package and not package["already_cached"]: yield package if process.returncode and process.returncode != 0: @@ -138,11 +142,41 @@ def run_nix_eval_jobs(cmd: List[str]) -> Generator[GitHubActionPackage, None, No sys.exit(process.returncode) +def is_extension_pkg(pkg: GitHubActionPackage) -> bool: + """Check if the package is a postgresql extension package.""" + attrs = pkg["attr"].split(".") + return attrs[-2] == "exts" + + def main() -> None: + parser = argparse.ArgumentParser( + description="Generate GitHub Actions matrix for Nix builds" + ) + parser.add_argument( + "target", choices=["checks", "extensions"], help="Type of matrix to generate" + ) + + args = parser.parse_args() + max_workers = get_worker_count() - cmd = build_nix_eval_command(max_workers) - gh_action_packages = list(run_nix_eval_jobs(cmd)) + if args.target == "checks": + flake_output = "checks" + else: + flake_output = "legacyPackages" + + cmd = build_nix_eval_command(max_workers, flake_output) + + gh_action_packages = list(run_nix_eval_jobs(cmd, flake_output)) + + if args.target == "extensions": + # filter to only include extension packages and add postgresql_version field + gh_action_packages = [ + {**pkg, "postgresql_version": pkg["attr"].split(".")[-3]} + for pkg in gh_action_packages + if is_extension_pkg(pkg) + ] + gh_output = {"include": gh_action_packages} print(json.dumps(gh_output)) From ed6deb49881d797aee92ad90fa605b4dccdb0770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 12:54:58 +0200 Subject: [PATCH 10/58] feat(ci): extract nix build setup into reusable action and split builds by architecture Extract AWS credential setup and nix build steps into a composite action to reduce duplication. Split extension builds into separate jobs per architecture (aarch64-linux, aarch64-darwin, x86_64-linux) and update matrix generation to group packages by system. --- .github/actions/nix-build-setup/action.yml | 46 +++++++++ .github/workflows/nix-build.yml | 108 +++++++++------------ scripts/github-matrix.py | 17 +++- 3 files changed, 109 insertions(+), 62 deletions(-) create mode 100644 .github/actions/nix-build-setup/action.yml diff --git a/.github/actions/nix-build-setup/action.yml b/.github/actions/nix-build-setup/action.yml new file mode 100644 index 000000000..f6a8c450a --- /dev/null +++ b/.github/actions/nix-build-setup/action.yml @@ -0,0 +1,46 @@ +name: 'Nix Build Setup' +description: 'Sets up AWS credentials and builds a Nix package' +inputs: + attr: + description: 'The Nix attribute to build' + required: true + aws-role-duration: + description: 'AWS role session duration in seconds' + required: false + default: '3600' + +runs: + using: 'composite' + steps: + - name: aws-oidc + uses: aws-actions/configure-aws-credentials@v4.3.1 + with: + aws-region: us-east-2 + role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role # Shared Services + role-session-name: gha-oidc-${{ github.run_id }} + - name: aws-creds + uses: aws-actions/configure-aws-credentials@v4.3.1 + with: + disable-retry: true + aws-region: us-east-2 + role-to-assume: arn:aws:iam::436098097459:role/nix-artifacts-deploy-role # supabase-dev + role-session-name: gha-oidc-${{ github.run_id }} + role-chaining: true + role-skip-session-tagging: true + role-duration-seconds: ${{ inputs.aws-role-duration }} + - name: Write creds files + shell: bash + run: | + umask 006 + cat > /etc/nix/aws/nix-aws-credentials < + nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" + - name: nix build + shell: bash + run: nix build -L .#${{ inputs.attr }} diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index a1a72dd92..6a4ca7aad 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -30,50 +30,63 @@ jobs: run: | set -Eeu echo matrix="$(python scripts/github-matrix.py extensions)" >> "$GITHUB_OUTPUT" + # XXX debugging + exit 1 - build-extensions: - name: ${{matrix.postgresql_version}}.${{ matrix.name }} (${{ matrix.system }}) + build-extensions-aarch64-linux: + name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-linux) needs: extensions-matrix runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_linux != null }} strategy: fail-fast: false max-parallel: 3 - matrix: ${{fromJSON(needs.extensions-matrix.outputs.matrix)}} + matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_linux }} steps: - name: Checkout Repo uses: actions/checkout@v4 - - name: aws-oidc - uses: aws-actions/configure-aws-credentials@v4.3.1 + - name: Build Nix Package + uses: ./.github/actions/nix-build-setup with: - aws-region: us-east-2 - role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role # Shared Services - role-session-name: gha-oidc-${{ github.run_id }} - - name: aws-creds - uses: aws-actions/configure-aws-credentials@v4.3.1 + attr: ${{ matrix.attr }} + + build-extensions-aarch64-darwin: + name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) + needs: extensions-matrix + runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_darwin != null }} + strategy: + fail-fast: false + max-parallel: 3 + matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_darwin }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Build Nix Package + uses: ./.github/actions/nix-build-setup with: - disable-retry: true - aws-region: us-east-2 - role-to-assume: arn:aws:iam::436098097459:role/nix-artifacts-deploy-role # supabase-dev - role-session-name: gha-oidc-${{ github.run_id }} - role-chaining: true - role-skip-session-tagging: true - role-duration-seconds: 3600 - - name: Write creds files - run: | - umask 006 - cat > /etc/nix/aws/nix-aws-credentials < /etc/nix/aws/nix-aws-credentials < - nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - - name: nix build - run: | - nix build -L .#${{ matrix.attr }} + attr: ${{ matrix.attr }} run-tests: needs: build-checks diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 7adef8cfa..562c6b9ec 100644 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -134,6 +134,7 @@ def run_nix_eval_jobs( for line in process.stdout: package = parse_nix_eval_line(line, drv_paths, target) if package and not package["already_cached"]: + print(f"Found package: {package['attr']}", file=sys.stderr) yield package if process.returncode and process.returncode != 0: @@ -177,7 +178,21 @@ def main() -> None: if is_extension_pkg(pkg) ] - gh_output = {"include": gh_action_packages} + # Group packages by system + grouped_by_system = {} + for pkg in gh_action_packages: + system = pkg["system"] + if system not in grouped_by_system: + grouped_by_system[system] = [] + grouped_by_system[system].append(pkg) + + # Create output with system-specific matrices + gh_output = {} + for system, packages in grouped_by_system.items(): + gh_output[system.replace("-", "_")] = {"include": packages} + else: + gh_output = {"include": gh_action_packages} + print(json.dumps(gh_output)) From b849901f16667e0815612105b5528eb67f9e39cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 15:30:23 +0200 Subject: [PATCH 11/58] fix(ci): do not hide cached builds --- .github/actions/nix-build-setup/action.yml | 12 ++++++++++++ .github/workflows/nix-build.yml | 6 ++++-- nix/packages/postgres.nix | 10 ++++++---- scripts/github-matrix.py | 18 ++++++++++++------ 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/.github/actions/nix-build-setup/action.yml b/.github/actions/nix-build-setup/action.yml index f6a8c450a..308bf492c 100644 --- a/.github/actions/nix-build-setup/action.yml +++ b/.github/actions/nix-build-setup/action.yml @@ -8,17 +8,23 @@ inputs: description: 'AWS role session duration in seconds' required: false default: '3600' + cached: + description: 'Whether to use cached builds' + required: false + default: 'false' runs: using: 'composite' steps: - name: aws-oidc + if: ${{ inputs.cached == 'false' }} uses: aws-actions/configure-aws-credentials@v4.3.1 with: aws-region: us-east-2 role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role # Shared Services role-session-name: gha-oidc-${{ github.run_id }} - name: aws-creds + if: ${{ inputs.cached == 'false' }} uses: aws-actions/configure-aws-credentials@v4.3.1 with: disable-retry: true @@ -29,6 +35,7 @@ runs: role-skip-session-tagging: true role-duration-seconds: ${{ inputs.aws-role-duration }} - name: Write creds files + if: ${{ inputs.cached == 'false' }} shell: bash run: | umask 006 @@ -42,5 +49,10 @@ runs: run: > nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - name: nix build + if: ${{ inputs.cached == 'false' }} shell: bash run: nix build -L .#${{ inputs.attr }} + - name: nix build + if: ${{ inputs.cached == 'true' }} + shell: bash + run: echo "Skipping build of ${{ inputs.attr }} as it's marked cached" diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 6a4ca7aad..73628319d 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -30,8 +30,6 @@ jobs: run: | set -Eeu echo matrix="$(python scripts/github-matrix.py extensions)" >> "$GITHUB_OUTPUT" - # XXX debugging - exit 1 build-extensions-aarch64-linux: name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-linux) @@ -49,6 +47,7 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} + cached: ${{ matrix.is_cached }} build-extensions-aarch64-darwin: name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) @@ -66,6 +65,7 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} + cached: ${{ matrix.is_cached }} build-extensions-x86_64-linux: name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) @@ -83,6 +83,7 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} + cached: ${{ matrix.is_cached }} checks-matrix: @@ -117,6 +118,7 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} + cached: ${{ matrix.is_cached }} run-tests: needs: build-checks diff --git a/nix/packages/postgres.nix b/nix/packages/postgres.nix index 1ff0f5a25..ec25d44e4 100644 --- a/nix/packages/postgres.nix +++ b/nix/packages/postgres.nix @@ -153,10 +153,12 @@ # install. # - exts: an attrset containing all the extensions, mapped to their # package names. - makePostgres = version: lib.recurseIntoAttrs { - bin = makePostgresBin version; - exts = makeOurPostgresPkgsSet version; - }; + makePostgres = + version: + lib.recurseIntoAttrs { + bin = makePostgresBin version; + exts = makeOurPostgresPkgsSet version; + }; basePackages = { psql_15 = makePostgres "15"; psql_17 = makePostgres "17"; diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 562c6b9ec..6c601fabd 100644 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -46,7 +46,7 @@ class GitHubActionPackage(TypedDict): attr: str name: str system: str - already_cached: bool + is_cached: bool runs_on: RunsOnConfig postgresql_version: NotRequired[str] @@ -103,8 +103,10 @@ def parse_nix_eval_line( try: data: NixEvalJobsOutput = json.loads(line) if data["drvPath"] in drv_paths: - return None - drv_paths.add(data["drvPath"]) + print(f"Skipping duplicate drvPath: {data['drvPath']}", file=sys.stderr) + data["cacheStatus"] = "cached" + else: + drv_paths.add(data["drvPath"]) runs_on_config = BUILD_RUNNER_MAP[data["system"]] @@ -112,8 +114,10 @@ def parse_nix_eval_line( "attr": f"{target}.{data['attr']}", "name": data["name"], "system": data["system"], - "already_cached": data.get("cacheStatus") != "notBuilt", + "is_cached": data.get("cacheStatus") != "notBuilt", "runs_on": runs_on_config, + "drvPath": data["drvPath"], # For debugging purposes + "outputs": data.get("outputs", {}), # For debugging purposes } except json.JSONDecodeError: print(f"Skipping invalid JSON line: {line}", file=sys.stderr) @@ -133,8 +137,7 @@ def run_nix_eval_jobs( for line in process.stdout: package = parse_nix_eval_line(line, drv_paths, target) - if package and not package["already_cached"]: - print(f"Found package: {package['attr']}", file=sys.stderr) + if package: yield package if process.returncode and process.returncode != 0: @@ -186,6 +189,9 @@ def main() -> None: grouped_by_system[system] = [] grouped_by_system[system].append(pkg) + print("Grouped packages by system:", file=sys.stderr) + print(json.dumps(grouped_by_system, indent=2), file=sys.stderr) + # Create output with system-specific matrices gh_output = {} for system, packages in grouped_by_system.items(): From d94e34487ffa0e0455bc83b4b8c7c669b017da02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 16:19:24 +0200 Subject: [PATCH 12/58] Revert "fix(ci): do not hide cached builds" This reverts commit 9d41b1e45fd92df646baaf078b9889cf308ac32b. --- .github/actions/nix-build-setup/action.yml | 12 ------------ .github/workflows/nix-build.yml | 6 ++---- nix/packages/postgres.nix | 10 ++++------ scripts/github-matrix.py | 18 ++++++------------ 4 files changed, 12 insertions(+), 34 deletions(-) diff --git a/.github/actions/nix-build-setup/action.yml b/.github/actions/nix-build-setup/action.yml index 308bf492c..f6a8c450a 100644 --- a/.github/actions/nix-build-setup/action.yml +++ b/.github/actions/nix-build-setup/action.yml @@ -8,23 +8,17 @@ inputs: description: 'AWS role session duration in seconds' required: false default: '3600' - cached: - description: 'Whether to use cached builds' - required: false - default: 'false' runs: using: 'composite' steps: - name: aws-oidc - if: ${{ inputs.cached == 'false' }} uses: aws-actions/configure-aws-credentials@v4.3.1 with: aws-region: us-east-2 role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role # Shared Services role-session-name: gha-oidc-${{ github.run_id }} - name: aws-creds - if: ${{ inputs.cached == 'false' }} uses: aws-actions/configure-aws-credentials@v4.3.1 with: disable-retry: true @@ -35,7 +29,6 @@ runs: role-skip-session-tagging: true role-duration-seconds: ${{ inputs.aws-role-duration }} - name: Write creds files - if: ${{ inputs.cached == 'false' }} shell: bash run: | umask 006 @@ -49,10 +42,5 @@ runs: run: > nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - name: nix build - if: ${{ inputs.cached == 'false' }} shell: bash run: nix build -L .#${{ inputs.attr }} - - name: nix build - if: ${{ inputs.cached == 'true' }} - shell: bash - run: echo "Skipping build of ${{ inputs.attr }} as it's marked cached" diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 73628319d..6a4ca7aad 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -30,6 +30,8 @@ jobs: run: | set -Eeu echo matrix="$(python scripts/github-matrix.py extensions)" >> "$GITHUB_OUTPUT" + # XXX debugging + exit 1 build-extensions-aarch64-linux: name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-linux) @@ -47,7 +49,6 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} - cached: ${{ matrix.is_cached }} build-extensions-aarch64-darwin: name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) @@ -65,7 +66,6 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} - cached: ${{ matrix.is_cached }} build-extensions-x86_64-linux: name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) @@ -83,7 +83,6 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} - cached: ${{ matrix.is_cached }} checks-matrix: @@ -118,7 +117,6 @@ jobs: uses: ./.github/actions/nix-build-setup with: attr: ${{ matrix.attr }} - cached: ${{ matrix.is_cached }} run-tests: needs: build-checks diff --git a/nix/packages/postgres.nix b/nix/packages/postgres.nix index ec25d44e4..1ff0f5a25 100644 --- a/nix/packages/postgres.nix +++ b/nix/packages/postgres.nix @@ -153,12 +153,10 @@ # install. # - exts: an attrset containing all the extensions, mapped to their # package names. - makePostgres = - version: - lib.recurseIntoAttrs { - bin = makePostgresBin version; - exts = makeOurPostgresPkgsSet version; - }; + makePostgres = version: lib.recurseIntoAttrs { + bin = makePostgresBin version; + exts = makeOurPostgresPkgsSet version; + }; basePackages = { psql_15 = makePostgres "15"; psql_17 = makePostgres "17"; diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 6c601fabd..562c6b9ec 100644 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -46,7 +46,7 @@ class GitHubActionPackage(TypedDict): attr: str name: str system: str - is_cached: bool + already_cached: bool runs_on: RunsOnConfig postgresql_version: NotRequired[str] @@ -103,10 +103,8 @@ def parse_nix_eval_line( try: data: NixEvalJobsOutput = json.loads(line) if data["drvPath"] in drv_paths: - print(f"Skipping duplicate drvPath: {data['drvPath']}", file=sys.stderr) - data["cacheStatus"] = "cached" - else: - drv_paths.add(data["drvPath"]) + return None + drv_paths.add(data["drvPath"]) runs_on_config = BUILD_RUNNER_MAP[data["system"]] @@ -114,10 +112,8 @@ def parse_nix_eval_line( "attr": f"{target}.{data['attr']}", "name": data["name"], "system": data["system"], - "is_cached": data.get("cacheStatus") != "notBuilt", + "already_cached": data.get("cacheStatus") != "notBuilt", "runs_on": runs_on_config, - "drvPath": data["drvPath"], # For debugging purposes - "outputs": data.get("outputs", {}), # For debugging purposes } except json.JSONDecodeError: print(f"Skipping invalid JSON line: {line}", file=sys.stderr) @@ -137,7 +133,8 @@ def run_nix_eval_jobs( for line in process.stdout: package = parse_nix_eval_line(line, drv_paths, target) - if package: + if package and not package["already_cached"]: + print(f"Found package: {package['attr']}", file=sys.stderr) yield package if process.returncode and process.returncode != 0: @@ -189,9 +186,6 @@ def main() -> None: grouped_by_system[system] = [] grouped_by_system[system].append(pkg) - print("Grouped packages by system:", file=sys.stderr) - print(json.dumps(grouped_by_system, indent=2), file=sys.stderr) - # Create output with system-specific matrices gh_output = {} for system, packages in grouped_by_system.items(): From 5a20c98b4dfaadd3f12710b74f053aeca11a5f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 16:38:29 +0200 Subject: [PATCH 13/58] chore: Temporarily disable x86_64-linux builds We don't have a self hosted runner for x86_64-linux at the moment --- .github/workflows/nix-build.yml | 35 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 6a4ca7aad..3157b3a1b 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -67,26 +67,27 @@ jobs: with: attr: ${{ matrix.attr }} - build-extensions-x86_64-linux: - name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) - needs: extensions-matrix - runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).x86_64_linux != null }} - strategy: - fail-fast: false - max-parallel: 3 - matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).x86_64_linux }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Build Nix Package - uses: ./.github/actions/nix-build-setup - with: - attr: ${{ matrix.attr }} + # TODO + # build-extensions-x86_64-linux: + # name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) + # needs: extensions-matrix + # runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + # if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).x86_64_linux != null }} + # strategy: + # fail-fast: false + # max-parallel: 3 + # matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).x86_64_linux }} + # steps: + # - name: Checkout Repo + # uses: actions/checkout@v4 + # - name: Build Nix Package + # uses: ./.github/actions/nix-build-setup + # with: + # attr: ${{ matrix.attr }} checks-matrix: - needs: [build-extensions-aarch64-linux, build-extensions-aarch64-darwin, build-extensions-x86_64-linux] + needs: [build-extensions-aarch64-linux, build-extensions-aarch64-darwin] #, build-extensions-x86_64-linux] runs-on: group: self-hosted-runners-nix labels: From 8b9cdd565a3f77527d99fd40ec4fe3989a8f6000 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 16:38:29 +0200 Subject: [PATCH 14/58] fix: sort packages and filter out cached ones --- .github/workflows/nix-build.yml | 2 -- nix/packages/postgres.nix | 10 ++++--- scripts/github-matrix.py | 52 +++++++++++++++++++++++++++------ 3 files changed, 49 insertions(+), 15 deletions(-) mode change 100644 => 100755 scripts/github-matrix.py diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 3157b3a1b..884196206 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -30,8 +30,6 @@ jobs: run: | set -Eeu echo matrix="$(python scripts/github-matrix.py extensions)" >> "$GITHUB_OUTPUT" - # XXX debugging - exit 1 build-extensions-aarch64-linux: name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-linux) diff --git a/nix/packages/postgres.nix b/nix/packages/postgres.nix index 1ff0f5a25..ec25d44e4 100644 --- a/nix/packages/postgres.nix +++ b/nix/packages/postgres.nix @@ -153,10 +153,12 @@ # install. # - exts: an attrset containing all the extensions, mapped to their # package names. - makePostgres = version: lib.recurseIntoAttrs { - bin = makePostgresBin version; - exts = makeOurPostgresPkgsSet version; - }; + makePostgres = + version: + lib.recurseIntoAttrs { + bin = makePostgresBin version; + exts = makeOurPostgresPkgsSet version; + }; basePackages = { psql_15 = makePostgres "15"; psql_17 = makePostgres "17"; diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py old mode 100644 new mode 100755 index 562c6b9ec..607755da9 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import argparse +from collections import defaultdict +import graphlib import json import os import subprocess @@ -48,6 +50,9 @@ class GitHubActionPackage(TypedDict): system: str already_cached: bool runs_on: RunsOnConfig + drvPath: str + neededSubstitutes: List[str] + neededBuilds: List[str] postgresql_version: NotRequired[str] @@ -114,6 +119,9 @@ def parse_nix_eval_line( "system": data["system"], "already_cached": data.get("cacheStatus") != "notBuilt", "runs_on": runs_on_config, + "drvPath": data["drvPath"], + "neededSubstitutes": data.get("neededSubstitutes", []), + "neededBuilds": data.get("neededBuilds", []), } except json.JSONDecodeError: print(f"Skipping invalid JSON line: {line}", file=sys.stderr) @@ -133,8 +141,7 @@ def run_nix_eval_jobs( for line in process.stdout: package = parse_nix_eval_line(line, drv_paths, target) - if package and not package["already_cached"]: - print(f"Found package: {package['attr']}", file=sys.stderr) + if package: yield package if process.returncode and process.returncode != 0: @@ -149,6 +156,34 @@ def is_extension_pkg(pkg: GitHubActionPackage) -> bool: return attrs[-2] == "exts" +# thank you buildbot-nix https://github.com/nix-community/buildbot-nix/blob/985d069a2a45cf4a571a4346107671adc2bd2a16/buildbot_nix/buildbot_nix/build_trigger.py#L297 +def sort_pkgs_by_closures(jobs: list[GitHubActionPackage]) -> list[GitHubActionPackage]: + sorted_jobs = [] + + # Prepare job dependencies + job_set = {job["drvPath"] for job in jobs} + job_closures = { + k["drvPath"]: set(k["neededSubstitutes"]) + .union(set(k["neededBuilds"])) + .intersection(job_set) + .difference({k["drvPath"]}) + for k in jobs + } + + sorter = graphlib.TopologicalSorter(job_closures) + + for item in sorter.static_order(): + i = 0 + while i < len(jobs): + if item == jobs[i]["drvPath"]: + sorted_jobs.append(jobs[i]) + del jobs[i] + else: + i += 1 + + return sorted_jobs + + def main() -> None: parser = argparse.ArgumentParser( description="Generate GitHub Actions matrix for Nix builds" @@ -168,23 +203,22 @@ def main() -> None: cmd = build_nix_eval_command(max_workers, flake_output) - gh_action_packages = list(run_nix_eval_jobs(cmd, flake_output)) + gh_action_packages = sort_pkgs_by_closures( + list(run_nix_eval_jobs(cmd, flake_output)) + ) if args.target == "extensions": # filter to only include extension packages and add postgresql_version field gh_action_packages = [ {**pkg, "postgresql_version": pkg["attr"].split(".")[-3]} for pkg in gh_action_packages - if is_extension_pkg(pkg) + if is_extension_pkg(pkg) and not pkg["already_cached"] ] # Group packages by system - grouped_by_system = {} + grouped_by_system = defaultdict(list) for pkg in gh_action_packages: - system = pkg["system"] - if system not in grouped_by_system: - grouped_by_system[system] = [] - grouped_by_system[system].append(pkg) + grouped_by_system[pkg["system"]].append(pkg) # Create output with system-specific matrices gh_output = {} From 845b3d710c7a04a88f33bf4e9d06e895802a5be5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 17:37:09 +0200 Subject: [PATCH 15/58] fix: do not skip checks-matrix if dependencies are skipped --- .github/workflows/nix-build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 884196206..c1bdf9334 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -86,6 +86,10 @@ jobs: checks-matrix: needs: [build-extensions-aarch64-linux, build-extensions-aarch64-darwin] #, build-extensions-x86_64-linux] + if: | + always() && + (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && + (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') runs-on: group: self-hosted-runners-nix labels: From 4a0d67814afb1d9871fe6ec6a38bf093e705d6ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 17:47:30 +0200 Subject: [PATCH 16/58] fix: do not return debug fields in GitHub Actions matrix output --- scripts/github-matrix.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 607755da9..b0b3d2a0f 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -207,6 +207,16 @@ def main() -> None: list(run_nix_eval_jobs(cmd, flake_output)) ) + def clean_package_for_output(pkg: GitHubActionPackage) -> dict: + """Remove debug fields from package for final output""" + return { + "attr": pkg["attr"], + "name": pkg["name"], + "system": pkg["system"], + "runs_on": pkg["runs_on"], + **({ "postgresql_version": pkg["postgresql_version"]} if "postgresql_version" in pkg else {}) + } + if args.target == "extensions": # filter to only include extension packages and add postgresql_version field gh_action_packages = [ @@ -218,14 +228,15 @@ def main() -> None: # Group packages by system grouped_by_system = defaultdict(list) for pkg in gh_action_packages: - grouped_by_system[pkg["system"]].append(pkg) + grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) # Create output with system-specific matrices gh_output = {} for system, packages in grouped_by_system.items(): gh_output[system.replace("-", "_")] = {"include": packages} else: - gh_output = {"include": gh_action_packages} + cleaned_packages = [clean_package_for_output(pkg) for pkg in gh_action_packages] + gh_output = {"include": cleaned_packages} print(json.dumps(gh_output)) From df468a48776e701c311d38f8337b84d957b315d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 30 Sep 2025 18:24:50 +0200 Subject: [PATCH 17/58] debugging --- .github/workflows/nix-build.yml | 4 ++-- scripts/github-matrix.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index c1bdf9334..1c7d0993c 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -38,7 +38,7 @@ jobs: if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_linux != null }} strategy: fail-fast: false - max-parallel: 3 + max-parallel: 5 matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_linux }} steps: - name: Checkout Repo @@ -55,7 +55,7 @@ jobs: if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_darwin != null }} strategy: fail-fast: false - max-parallel: 3 + max-parallel: 5 matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_darwin }} steps: - name: Checkout Repo diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index b0b3d2a0f..0829afab9 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -238,6 +238,7 @@ def clean_package_for_output(pkg: GitHubActionPackage) -> dict: cleaned_packages = [clean_package_for_output(pkg) for pkg in gh_action_packages] gh_output = {"include": cleaned_packages} + print(f"debug: Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}", file=sys.stderr) print(json.dumps(gh_output)) From e657056cc6e613603323d18260b6952fcc4a085e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 2 Oct 2025 17:00:31 +0200 Subject: [PATCH 18/58] fix(ci): use !cancelled() instead of always() for dependent job conditions Replace always() with !cancelled() to prevent jobs from running when the workflow is cancelled. Add explicit success/skip conditions for dependent jobs to ensure proper workflow execution flow. --- .github/workflows/nix-build.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 1c7d0993c..780dad7c4 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -87,7 +87,7 @@ jobs: checks-matrix: needs: [build-extensions-aarch64-linux, build-extensions-aarch64-darwin] #, build-extensions-x86_64-linux] if: | - always() && + !cancelled() && (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') runs-on: @@ -110,6 +110,9 @@ jobs: name: ${{ matrix.name }} (${{ matrix.system }}) needs: [checks-matrix] runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: | + !cancelled() && + (needs.checks-matrix.result == 'skipped' || needs.checks-matrix.result == 'success') strategy: fail-fast: false matrix: ${{fromJSON(needs.checks-matrix.outputs.matrix)}} @@ -123,5 +126,7 @@ jobs: run-tests: needs: build-checks - if: ${{ success() }} + if: | + !cancelled() && + (needs.build-checks.result == 'skipped' || needs.build-checks.result == 'success') uses: ./.github/workflows/test.yml From f414400cbf795e91d8c916dd955a77095cede01f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 2 Oct 2025 17:20:26 +0200 Subject: [PATCH 19/58] fix(ci): stop chaining aws roles We couldn't set duration to 18000 seconds when chaining roles. --- .github/actions/nix-build-setup/action.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/actions/nix-build-setup/action.yml b/.github/actions/nix-build-setup/action.yml index f6a8c450a..afa06499f 100644 --- a/.github/actions/nix-build-setup/action.yml +++ b/.github/actions/nix-build-setup/action.yml @@ -12,12 +12,6 @@ inputs: runs: using: 'composite' steps: - - name: aws-oidc - uses: aws-actions/configure-aws-credentials@v4.3.1 - with: - aws-region: us-east-2 - role-to-assume: arn:aws:iam::279559813984:role/supabase-github-oidc-role # Shared Services - role-session-name: gha-oidc-${{ github.run_id }} - name: aws-creds uses: aws-actions/configure-aws-credentials@v4.3.1 with: @@ -25,9 +19,8 @@ runs: aws-region: us-east-2 role-to-assume: arn:aws:iam::436098097459:role/nix-artifacts-deploy-role # supabase-dev role-session-name: gha-oidc-${{ github.run_id }} - role-chaining: true - role-skip-session-tagging: true - role-duration-seconds: ${{ inputs.aws-role-duration }} + role-duration-seconds: 18000 + - name: Write creds files shell: bash run: | From f69715d91e940bfa5264483028a24553b2d27246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 2 Oct 2025 21:18:13 +0200 Subject: [PATCH 20/58] ci: run nixos test on aarch64-linux --- nix/checks.nix | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/nix/checks.nix b/nix/checks.nix index 0a1ab9d8e..ee76cad3a 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -397,24 +397,22 @@ pg_regress ; } - // pkgs.lib.optionalAttrs (system == "aarch64-linux") { - inherit (self'.packages) - postgresql_15_debug - postgresql_15_src - postgresql_orioledb-17_debug - postgresql_orioledb-17_src - postgresql_17_debug - postgresql_17_src - ; - } - // pkgs.lib.optionalAttrs (system == "x86_64-linux") ( + // pkgs.lib.optionalAttrs (system == "aarch64-linux") ( { - devShell = self'.devShells.default; + inherit (self'.packages) + postgresql_15_debug + postgresql_15_src + postgresql_orioledb-17_debug + postgresql_orioledb-17_src + postgresql_17_debug + postgresql_17_src + ; } // (import ./ext/tests { inherit self; inherit pkgs; }) + // pkgs.lib.optionalAttrs (system == "x86_64-linux") { devShell = self'.devShells.default; } ); }; } From 47df10207ab74010ebf8dc6e2c846b3a9e861d50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 2 Oct 2025 21:18:13 +0200 Subject: [PATCH 21/58] fix(ci): disable eval-cache and accept-flake-config --- scripts/github-matrix.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 0829afab9..769ebb1c7 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -92,6 +92,12 @@ def build_nix_eval_command(max_workers: int, target: str) -> List[str]: "--check-cache-status", "--force-recurse", "--quiet", + "--option", + "eval-cache", + "false", + "--option", + "accept-flake-config", + "true", "--workers", str(max_workers), ] @@ -214,7 +220,11 @@ def clean_package_for_output(pkg: GitHubActionPackage) -> dict: "name": pkg["name"], "system": pkg["system"], "runs_on": pkg["runs_on"], - **({ "postgresql_version": pkg["postgresql_version"]} if "postgresql_version" in pkg else {}) + **( + {"postgresql_version": pkg["postgresql_version"]} + if "postgresql_version" in pkg + else {} + ), } if args.target == "extensions": @@ -238,7 +248,10 @@ def clean_package_for_output(pkg: GitHubActionPackage) -> dict: cleaned_packages = [clean_package_for_output(pkg) for pkg in gh_action_packages] gh_output = {"include": cleaned_packages} - print(f"debug: Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}", file=sys.stderr) + print( + f"debug: Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}", + file=sys.stderr, + ) print(json.dumps(gh_output)) From 578f83d046cb212c6f8b3b1205690114bab4c2b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 2 Oct 2025 21:18:13 +0200 Subject: [PATCH 22/58] ci: split checks build jobs by system architecture Refactor GitHub Actions workflow to run build checks in parallel across different architectures (aarch64-linux, aarch64-darwin) with separate job matrices. --- .github/workflows/nix-build.yml | 58 ++++++++++++++++++++++++++++----- scripts/github-matrix.py | 20 +++++------- 2 files changed, 58 insertions(+), 20 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 780dad7c4..70309d28c 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -87,7 +87,7 @@ jobs: checks-matrix: needs: [build-extensions-aarch64-linux, build-extensions-aarch64-darwin] #, build-extensions-x86_64-linux] if: | - !cancelled() && + !cancelled() && (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') runs-on: @@ -105,17 +105,40 @@ jobs: set -Eeu echo matrix="$(python scripts/github-matrix.py checks)" >> "$GITHUB_OUTPUT" + build-checks-aarch64-linux: + name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) + needs: checks-matrix + runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: | + !cancelled() && + ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_linux != null }} && + (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && + (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') + strategy: + fail-fast: false + max-parallel: 5 + matrix: ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_linux }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Build Nix Package + uses: ./.github/actions/nix-build-setup + with: + attr: ${{ matrix.attr }} - build-checks: - name: ${{ matrix.name }} (${{ matrix.system }}) - needs: [checks-matrix] + build-checks-aarch64-darwin: + name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) + needs: checks-matrix runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: | !cancelled() && - (needs.checks-matrix.result == 'skipped' || needs.checks-matrix.result == 'success') + ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_darwin != null }} && + (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && + (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') strategy: fail-fast: false - matrix: ${{fromJSON(needs.checks-matrix.outputs.matrix)}} + max-parallel: 5 + matrix: ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_darwin }} steps: - name: Checkout Repo uses: actions/checkout@v4 @@ -124,9 +147,28 @@ jobs: with: attr: ${{ matrix.attr }} + # TODO + # build-checks-x86_64-linux: + # name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) + # needs: checks-matrix + # runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + # if: ${{ fromJSON(needs.checks-matrix.outputs.matrix).x86_64_linux != null }} + # strategy: + # fail-fast: false + # max-parallel: 5 + # matrix: ${{ fromJSON(needs.checks-matrix.outputs.matrix).x86_64_linux }} + # steps: + # - name: Checkout Repo + # uses: actions/checkout@v4 + # - name: Build Nix Package + # uses: ./.github/actions/nix-build-setup + # with: + # attr: ${{ matrix.attr }} + run-tests: - needs: build-checks + needs: [build-checks-aarch64-linux, build-checks-aarch64-darwin] #, build-checks-x86_64-linux] if: | !cancelled() && - (needs.build-checks.result == 'skipped' || needs.build-checks.result == 'success') + (needs.build-checks-aarch64-linux.result == 'skipped' || needs.build-checks-aarch64-linux.result == 'success') && + (needs.build-checks-aarch64-darwin.result == 'skipped' || needs.build-checks-aarch64-darwin.result == 'success') uses: ./.github/workflows/test.yml diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 769ebb1c7..4e23bf7f4 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -144,7 +144,6 @@ def run_nix_eval_jobs( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) as process: drv_paths = set() - for line in process.stdout: package = parse_nix_eval_line(line, drv_paths, target) if package: @@ -235,18 +234,15 @@ def clean_package_for_output(pkg: GitHubActionPackage) -> dict: if is_extension_pkg(pkg) and not pkg["already_cached"] ] - # Group packages by system - grouped_by_system = defaultdict(list) - for pkg in gh_action_packages: - grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) + # Group packages by system + grouped_by_system = defaultdict(list) + for pkg in gh_action_packages: + grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) - # Create output with system-specific matrices - gh_output = {} - for system, packages in grouped_by_system.items(): - gh_output[system.replace("-", "_")] = {"include": packages} - else: - cleaned_packages = [clean_package_for_output(pkg) for pkg in gh_action_packages] - gh_output = {"include": cleaned_packages} + # Create output with system-specific matrices + gh_output = {} + for system, packages in grouped_by_system.items(): + gh_output[system.replace("-", "_")] = {"include": packages} print( f"debug: Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}", From bd89688bb813dd2a2a269a151e7e013e930689d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 3 Oct 2025 11:47:49 +0200 Subject: [PATCH 23/58] fix(ci): use correct architecture name in aarch64-linux builds --- .github/workflows/nix-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 70309d28c..9b1775bb7 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -106,7 +106,7 @@ jobs: echo matrix="$(python scripts/github-matrix.py checks)" >> "$GITHUB_OUTPUT" build-checks-aarch64-linux: - name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) + name: ${{ matrix.name }} (aarch64-linux) needs: checks-matrix runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: | @@ -127,7 +127,7 @@ jobs: attr: ${{ matrix.attr }} build-checks-aarch64-darwin: - name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) + name: ${{ matrix.name }} (aarch64-darwin) needs: checks-matrix runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: | From e1a0e288b852436f21d144775349865361e8159d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 3 Oct 2025 11:47:49 +0200 Subject: [PATCH 24/58] fix(ci): do not try to build already cached checks --- scripts/github-matrix.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index 4e23bf7f4..b3b003afc 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -231,13 +231,14 @@ def clean_package_for_output(pkg: GitHubActionPackage) -> dict: gh_action_packages = [ {**pkg, "postgresql_version": pkg["attr"].split(".")[-3]} for pkg in gh_action_packages - if is_extension_pkg(pkg) and not pkg["already_cached"] + if is_extension_pkg(pkg) ] # Group packages by system grouped_by_system = defaultdict(list) for pkg in gh_action_packages: - grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) + if not pkg["already_cached"]: + grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) # Create output with system-specific matrices gh_output = {} From bdcf0aae7c04358ba293d2ce8b97faa533831ffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 3 Oct 2025 14:53:50 +0200 Subject: [PATCH 25/58] fix(ci): simplify GitHub Actions workflow for Nix builds Create a single nix-eval job to determine packages to build, removing redundant extension and check matrices. --- .github/actionlint.yaml | 4 ++ .github/workflows/nix-build.yml | 118 +++++--------------------------- scripts/github-matrix.py | 60 +++++----------- 3 files changed, 40 insertions(+), 142 deletions(-) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..0a2631f7d --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,4 @@ +self-hosted-runner: + labels: + - aarch64-darwin + - aarch64-linux diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 9b1775bb7..56e1235a6 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -15,7 +15,7 @@ permissions: packages: write jobs: - extensions-matrix: + nix-eval: runs-on: group: self-hosted-runners-nix labels: @@ -29,95 +29,17 @@ jobs: name: Generate Nix Matrix run: | set -Eeu - echo matrix="$(python scripts/github-matrix.py extensions)" >> "$GITHUB_OUTPUT" + echo matrix="$(python scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" - build-extensions-aarch64-linux: - name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-linux) - needs: extensions-matrix - runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_linux != null }} - strategy: - fail-fast: false - max-parallel: 5 - matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_linux }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Build Nix Package - uses: ./.github/actions/nix-build-setup - with: - attr: ${{ matrix.attr }} - - build-extensions-aarch64-darwin: - name: ${{matrix.postgresql_version}}.${{ matrix.name }} (aarch64-darwin) - needs: extensions-matrix - runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_darwin != null }} - strategy: - fail-fast: false - max-parallel: 5 - matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).aarch64_darwin }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Build Nix Package - uses: ./.github/actions/nix-build-setup - with: - attr: ${{ matrix.attr }} - - # TODO - # build-extensions-x86_64-linux: - # name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) - # needs: extensions-matrix - # runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - # if: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).x86_64_linux != null }} - # strategy: - # fail-fast: false - # max-parallel: 3 - # matrix: ${{ fromJSON(needs.extensions-matrix.outputs.matrix).x86_64_linux }} - # steps: - # - name: Checkout Repo - # uses: actions/checkout@v4 - # - name: Build Nix Package - # uses: ./.github/actions/nix-build-setup - # with: - # attr: ${{ matrix.attr }} - - - checks-matrix: - needs: [build-extensions-aarch64-linux, build-extensions-aarch64-darwin] #, build-extensions-x86_64-linux] - if: | - !cancelled() && - (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && - (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') - runs-on: - group: self-hosted-runners-nix - labels: - - aarch64-darwin - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - id: set-matrix - name: Generate Nix Matrix - run: | - set -Eeu - echo matrix="$(python scripts/github-matrix.py checks)" >> "$GITHUB_OUTPUT" - - build-checks-aarch64-linux: + nix-build-aarch64-linux: name: ${{ matrix.name }} (aarch64-linux) - needs: checks-matrix + needs: nix-eval runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: | - !cancelled() && - ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_linux != null }} && - (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && - (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') + if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_linux != null }} strategy: fail-fast: false max-parallel: 5 - matrix: ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_linux }} + matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_linux }} steps: - name: Checkout Repo uses: actions/checkout@v4 @@ -126,19 +48,15 @@ jobs: with: attr: ${{ matrix.attr }} - build-checks-aarch64-darwin: + nix-build-aarch64-darwin: name: ${{ matrix.name }} (aarch64-darwin) - needs: checks-matrix + needs: nix-eval runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: | - !cancelled() && - ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_darwin != null }} && - (needs.build-extensions-aarch64-linux.result == 'skipped' || needs.build-extensions-aarch64-linux.result == 'success') && - (needs.build-extensions-aarch64-darwin.result == 'skipped' || needs.build-extensions-aarch64-darwin.result == 'success') + if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin != null }} strategy: fail-fast: false max-parallel: 5 - matrix: ${{ fromJSON(needs.checks-matrix.outputs.matrix).aarch64_darwin }} + matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin }} steps: - name: Checkout Repo uses: actions/checkout@v4 @@ -148,15 +66,15 @@ jobs: attr: ${{ matrix.attr }} # TODO - # build-checks-x86_64-linux: + # nix-build-x86_64-linux: # name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) - # needs: checks-matrix + # needs: nix-eval # runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - # if: ${{ fromJSON(needs.checks-matrix.outputs.matrix).x86_64_linux != null }} + # if: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux != null }} # strategy: # fail-fast: false - # max-parallel: 5 - # matrix: ${{ fromJSON(needs.checks-matrix.outputs.matrix).x86_64_linux }} + # max-parallel: 3 + # matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux }} # steps: # - name: Checkout Repo # uses: actions/checkout@v4 @@ -166,9 +84,9 @@ jobs: # attr: ${{ matrix.attr }} run-tests: - needs: [build-checks-aarch64-linux, build-checks-aarch64-darwin] #, build-checks-x86_64-linux] + needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] if: | !cancelled() && - (needs.build-checks-aarch64-linux.result == 'skipped' || needs.build-checks-aarch64-linux.result == 'success') && - (needs.build-checks-aarch64-darwin.result == 'skipped' || needs.build-checks-aarch64-darwin.result == 'success') + (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && + (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') uses: ./.github/workflows/test.yml diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index b3b003afc..d6aa80ef3 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -71,24 +71,12 @@ class GitHubActionPackage(TypedDict): } -def get_worker_count() -> int: - """Get optimal worker count based on CPU cores.""" - try: - return max(1, int(os.cpu_count())) - except (OSError, AttributeError): - print( - "Warning: Unable to get CPU count, using default max_workers=1", - file=sys.stderr, - ) - return 1 - - -def build_nix_eval_command(max_workers: int, target: str) -> List[str]: +def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[str]: """Build the nix-eval-jobs command with appropriate flags.""" nix_eval_cmd = [ "nix-eval-jobs", "--flake", - f".#{target}", + ".", "--check-cache-status", "--force-recurse", "--quiet", @@ -100,12 +88,14 @@ def build_nix_eval_command(max_workers: int, target: str) -> List[str]: "true", "--workers", str(max_workers), + "--select", + f"outputs: {{ inherit (outputs) {' '.join(flake_outputs)}; }}", ] return nix_eval_cmd def parse_nix_eval_line( - line: str, drv_paths: Set[str], target: str + line: str, drv_paths: Set[str] ) -> Optional[GitHubActionPackage]: """Parse a single line of nix-eval-jobs output""" if not line.strip(): @@ -120,7 +110,7 @@ def parse_nix_eval_line( runs_on_config = BUILD_RUNNER_MAP[data["system"]] return { - "attr": f"{target}.{data['attr']}", + "attr": f"{data['attr']}", "name": data["name"], "system": data["system"], "already_cached": data.get("cacheStatus") != "notBuilt", @@ -134,22 +124,23 @@ def parse_nix_eval_line( return None -def run_nix_eval_jobs( - cmd: List[str], target: str -) -> Generator[GitHubActionPackage, None, None]: +def run_nix_eval_jobs(cmd: List[str]) -> Generator[GitHubActionPackage, None, None]: """Run nix-eval-jobs and yield parsed package data.""" print(f"Running command: {' '.join(cmd)}", file=sys.stderr) with subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) as process: - drv_paths = set() + drv_paths: Set[str] = set() + assert process.stdout is not None # for mypy + assert process.stderr is not None # for mypy for line in process.stdout: - package = parse_nix_eval_line(line, drv_paths, target) + package = parse_nix_eval_line(line, drv_paths) if package: yield package - if process.returncode and process.returncode != 0: + process.wait() + if process.returncode != 0: print("Error: Evaluation failed", file=sys.stderr) sys.stderr.write(process.stderr.read()) sys.exit(process.returncode) @@ -162,7 +153,7 @@ def is_extension_pkg(pkg: GitHubActionPackage) -> bool: # thank you buildbot-nix https://github.com/nix-community/buildbot-nix/blob/985d069a2a45cf4a571a4346107671adc2bd2a16/buildbot_nix/buildbot_nix/build_trigger.py#L297 -def sort_pkgs_by_closures(jobs: list[GitHubActionPackage]) -> list[GitHubActionPackage]: +def sort_pkgs_by_closures(jobs: List[GitHubActionPackage]) -> List[GitHubActionPackage]: sorted_jobs = [] # Prepare job dependencies @@ -194,23 +185,16 @@ def main() -> None: description="Generate GitHub Actions matrix for Nix builds" ) parser.add_argument( - "target", choices=["checks", "extensions"], help="Type of matrix to generate" + "flake_outputs", nargs="+", help="Nix flake outputs to evaluate" ) args = parser.parse_args() - max_workers = get_worker_count() + max_workers: int = os.cpu_count() or 1 - if args.target == "checks": - flake_output = "checks" - else: - flake_output = "legacyPackages" + cmd = build_nix_eval_command(max_workers, args.flake_outputs) - cmd = build_nix_eval_command(max_workers, flake_output) - - gh_action_packages = sort_pkgs_by_closures( - list(run_nix_eval_jobs(cmd, flake_output)) - ) + gh_action_packages = sort_pkgs_by_closures(list(run_nix_eval_jobs(cmd))) def clean_package_for_output(pkg: GitHubActionPackage) -> dict: """Remove debug fields from package for final output""" @@ -226,14 +210,6 @@ def clean_package_for_output(pkg: GitHubActionPackage) -> dict: ), } - if args.target == "extensions": - # filter to only include extension packages and add postgresql_version field - gh_action_packages = [ - {**pkg, "postgresql_version": pkg["attr"].split(".")[-3]} - for pkg in gh_action_packages - if is_extension_pkg(pkg) - ] - # Group packages by system grouped_by_system = defaultdict(list) for pkg in gh_action_packages: From d6290b311dd76ac848f6bb6daecaaa19710b17e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 3 Oct 2025 16:04:41 +0200 Subject: [PATCH 26/58] Revert "chores(ci): comment out vestigal testinfra-ami-build" This reverts commit e2db36800b8a2a0ad9e132fdad688613aeb5aa37. --- .github/workflows/nix-build.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 56e1235a6..86f40c64d 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -10,7 +10,7 @@ on: permissions: id-token: write - # required by dependent workflows + # required by testinfra-ami-build dependent workflows contents: write packages: write @@ -83,6 +83,16 @@ jobs: # with: # attr: ${{ matrix.attr }} + run-testinfra: + needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] + if: | + !cancelled() && + (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && + (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') + uses: ./.github/workflows/testinfra-ami-build.yml + secrets: + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} + run-tests: needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] if: | From 2b86d37496aa96820246d278d398b1d2efbeec75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 3 Oct 2025 21:04:09 +0200 Subject: [PATCH 27/58] feat(ci): eval on blacksmith-32vcpu-ubuntu-2404 --- .github/actionlint.yaml | 1 + .../action.yml | 2 +- .github/workflows/nix-build.yml | 21 ++++++++++--------- 3 files changed, 13 insertions(+), 11 deletions(-) rename .github/actions/{nix-build-setup => nix-build-self-hosted}/action.yml (94%) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 0a2631f7d..6eea24e6a 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -2,3 +2,4 @@ self-hosted-runner: labels: - aarch64-darwin - aarch64-linux + - blacksmith-32vcpu-ubuntu-2404 diff --git a/.github/actions/nix-build-setup/action.yml b/.github/actions/nix-build-self-hosted/action.yml similarity index 94% rename from .github/actions/nix-build-setup/action.yml rename to .github/actions/nix-build-self-hosted/action.yml index afa06499f..f7b202f1c 100644 --- a/.github/actions/nix-build-setup/action.yml +++ b/.github/actions/nix-build-self-hosted/action.yml @@ -36,4 +36,4 @@ runs: nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - name: nix build shell: bash - run: nix build -L .#${{ inputs.attr }} + run: nix build --accept-flake-config -L .#${{ inputs.attr }} diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 86f40c64d..d78bb9e6f 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -16,20 +16,21 @@ permissions: jobs: nix-eval: - runs-on: - group: self-hosted-runners-nix - labels: - - aarch64-darwin + runs-on: blacksmith-32vcpu-ubuntu-2404 outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - name: Checkout Repo uses: actions/checkout@v4 + - name: Install nix + uses: cachix/install-nix-action@v31 + with: + install_url: https://releases.nixos.org/nix/nix-2.31.2/install - id: set-matrix name: Generate Nix Matrix run: | set -Eeu - echo matrix="$(python scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" + echo matrix="$(nix shell nixpkgs/405fc615369e0ea1b9c284c107ca4c3e1bc15774#nix-eval-jobs --command scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" nix-build-aarch64-linux: name: ${{ matrix.name }} (aarch64-linux) @@ -44,7 +45,7 @@ jobs: - name: Checkout Repo uses: actions/checkout@v4 - name: Build Nix Package - uses: ./.github/actions/nix-build-setup + uses: ./.github/actions/nix-build-self-hosted with: attr: ${{ matrix.attr }} @@ -61,7 +62,7 @@ jobs: - name: Checkout Repo uses: actions/checkout@v4 - name: Build Nix Package - uses: ./.github/actions/nix-build-setup + uses: ./.github/actions/nix-build-self-hosted with: attr: ${{ matrix.attr }} @@ -79,14 +80,14 @@ jobs: # - name: Checkout Repo # uses: actions/checkout@v4 # - name: Build Nix Package - # uses: ./.github/actions/nix-build-setup + # uses: ./.github/actions/nix-build-self-hosted # with: # attr: ${{ matrix.attr }} run-testinfra: needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] if: | - !cancelled() && + !cancelled() && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') uses: ./.github/workflows/testinfra-ami-build.yml @@ -96,7 +97,7 @@ jobs: run-tests: needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] if: | - !cancelled() && + !cancelled() && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') uses: ./.github/workflows/test.yml From 4f632d991a8e0f83bb261c6af2ca5a40162a6d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 3 Oct 2025 21:04:09 +0200 Subject: [PATCH 28/58] feat: add ephemeral Nix install action for GitHub runners --- .github/workflows/nix-build.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index d78bb9e6f..5f3d6b781 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -23,14 +23,12 @@ jobs: - name: Checkout Repo uses: actions/checkout@v4 - name: Install nix - uses: cachix/install-nix-action@v31 - with: - install_url: https://releases.nixos.org/nix/nix-2.31.2/install + uses: ./.github/actions/nix-install-ephemeral - id: set-matrix name: Generate Nix Matrix run: | set -Eeu - echo matrix="$(nix shell nixpkgs/405fc615369e0ea1b9c284c107ca4c3e1bc15774#nix-eval-jobs --command scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" + echo matrix="$(nix shell github:nix-community/nix-eval-jobs --command scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" nix-build-aarch64-linux: name: ${{ matrix.name }} (aarch64-linux) @@ -97,7 +95,7 @@ jobs: run-tests: needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] if: | - !cancelled() && + !cancelled() && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') uses: ./.github/workflows/test.yml From ed29214af8900c9a85379e30f1c98b5568444f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Sat, 4 Oct 2025 01:57:06 +0200 Subject: [PATCH 29/58] refactor(ci): extract nix eval into reusable workflow --- .github/workflows/nix-build.yml | 17 ++++------------- .github/workflows/nix-eval.yml | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/nix-eval.yml diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 5f3d6b781..f15920795 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -16,19 +16,10 @@ permissions: jobs: nix-eval: - runs-on: blacksmith-32vcpu-ubuntu-2404 - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Install nix - uses: ./.github/actions/nix-install-ephemeral - - id: set-matrix - name: Generate Nix Matrix - run: | - set -Eeu - echo matrix="$(nix shell github:nix-community/nix-eval-jobs --command scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" + uses: ./.github/workflows/nix-eval.yml + secrets: + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} + NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} nix-build-aarch64-linux: name: ${{ matrix.name }} (aarch64-linux) diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml new file mode 100644 index 000000000..f5436ce46 --- /dev/null +++ b/.github/workflows/nix-eval.yml @@ -0,0 +1,32 @@ +name: Nix Eval + +on: + workflow_call: + outputs: + matrix: + description: 'Generated build matrix' + value: ${{ jobs.eval.outputs.matrix }} + secrets: + DEV_AWS_ROLE: + required: false + NIX_SIGN_SECRET_KEY: + required: false + +jobs: + eval: + runs-on: blacksmith-32vcpu-ubuntu-2404 + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Install nix + uses: ./.github/actions/nix-install-ephemeral + env: + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} + NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} + - id: set-matrix + name: Generate Nix Matrix + run: | + set -Eeu + echo matrix="$(nix shell github:nix-community/nix-eval-jobs/v2.31.0 --command scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" From 90b1b52c37b825d11646c40c29541c8f9c2c1c9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 13:15:02 +0200 Subject: [PATCH 30/58] feat: enable x86_64-linux builds in CI --- .../action.yml | 17 ++---- .github/workflows/nix-build.yml | 58 +++++++++++-------- 2 files changed, 37 insertions(+), 38 deletions(-) rename .github/actions/{nix-build-self-hosted => nix-install-self-hosted}/action.yml (62%) diff --git a/.github/actions/nix-build-self-hosted/action.yml b/.github/actions/nix-install-self-hosted/action.yml similarity index 62% rename from .github/actions/nix-build-self-hosted/action.yml rename to .github/actions/nix-install-self-hosted/action.yml index f7b202f1c..52e49db9d 100644 --- a/.github/actions/nix-build-self-hosted/action.yml +++ b/.github/actions/nix-install-self-hosted/action.yml @@ -1,13 +1,10 @@ -name: 'Nix Build Setup' -description: 'Sets up AWS credentials and builds a Nix package' +name: 'Configure Nix on self hosted runners' +description: 'Sets up AWS credentials to push to the Nix binary cache' inputs: - attr: - description: 'The Nix attribute to build' - required: true aws-role-duration: description: 'AWS role session duration in seconds' required: false - default: '3600' + default: '18000' runs: using: 'composite' @@ -19,7 +16,7 @@ runs: aws-region: us-east-2 role-to-assume: arn:aws:iam::436098097459:role/nix-artifacts-deploy-role # supabase-dev role-session-name: gha-oidc-${{ github.run_id }} - role-duration-seconds: 18000 + role-duration-seconds: ${{ inputs.aws-role-duration }} - name: Write creds files shell: bash @@ -31,9 +28,3 @@ runs: aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY} aws_session_token = ${AWS_SESSION_TOKEN} EOF - - name: Build psql bundle - run: > - nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - - name: nix build - shell: bash - run: nix build --accept-flake-config -L .#${{ inputs.attr }} diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index f15920795..8d4b43c17 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -34,9 +34,10 @@ jobs: - name: Checkout Repo uses: actions/checkout@v4 - name: Build Nix Package - uses: ./.github/actions/nix-build-self-hosted - with: - attr: ${{ matrix.attr }} + uses: ./.github/actions/nix-install-self-hosted + - name: nix build + shell: bash + run: nix build --accept-flake-config -L .#${{ matrix.attr }} nix-build-aarch64-darwin: name: ${{ matrix.name }} (aarch64-darwin) @@ -51,30 +52,37 @@ jobs: - name: Checkout Repo uses: actions/checkout@v4 - name: Build Nix Package - uses: ./.github/actions/nix-build-self-hosted - with: - attr: ${{ matrix.attr }} + uses: ./.github/actions/nix-install-self-hosted + - name: nix build + shell: bash + run: nix build --accept-flake-config -L .#${{ matrix.attr }} - # TODO - # nix-build-x86_64-linux: - # name: ${{matrix.postgresql_version}}.${{ matrix.name }} (x86_64-linux) - # needs: nix-eval - # runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - # if: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux != null }} - # strategy: - # fail-fast: false - # max-parallel: 3 - # matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux }} - # steps: - # - name: Checkout Repo - # uses: actions/checkout@v4 - # - name: Build Nix Package - # uses: ./.github/actions/nix-build-self-hosted - # with: - # attr: ${{ matrix.attr }} + nix-build-x86_64-linux: + name: ${{ matrix.name }} (x86_64-linux) + needs: nix-eval + runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + if: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux != null }} + strategy: + fail-fast: false + max-parallel: 5 + matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Install nix + uses: ./.github/actions/nix-install-ephemeral + env: + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} + NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} + - name: Build psql bundle + run: > + nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" + - name: nix build + shell: bash + run: nix build --accept-flake-config -L .#${{ matrix.attr }} run-testinfra: - needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] + needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] if: | !cancelled() && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && @@ -84,7 +92,7 @@ jobs: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} run-tests: - needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin] #, nix-build-x86_64-linux] + needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] if: | !cancelled() && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && From d81a96a87f818f4a9fe99b79d803f9b604608eca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 14:32:56 +0200 Subject: [PATCH 31/58] feat: add PostgreSQL version to GitHub Actions job names When building a postgres extension, the build matrix may include multiple time the same extension for different PostgreSQL versions. This change makes it easier to identify which job corresponds to which PostgreSQL version in the workflow runs. --- .github/workflows/nix-build.yml | 12 +++++++++--- scripts/github-matrix.py | 12 ++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 8d4b43c17..0c3e78bc2 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -22,7 +22,9 @@ jobs: NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} nix-build-aarch64-linux: - name: ${{ matrix.name }} (aarch64-linux) + name: >- + ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} + (aarch64-linux) needs: nix-eval runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_linux != null }} @@ -40,7 +42,9 @@ jobs: run: nix build --accept-flake-config -L .#${{ matrix.attr }} nix-build-aarch64-darwin: - name: ${{ matrix.name }} (aarch64-darwin) + name: >- + ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} + (aarch64-darwin) needs: nix-eval runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin != null }} @@ -58,7 +62,9 @@ jobs: run: nix build --accept-flake-config -L .#${{ matrix.attr }} nix-build-x86_64-linux: - name: ${{ matrix.name }} (x86_64-linux) + name: >- + ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} + (x86_64-linux) needs: nix-eval runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux != null }} diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index d6aa80ef3..b80a9f6c0 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -198,17 +198,17 @@ def main() -> None: def clean_package_for_output(pkg: GitHubActionPackage) -> dict: """Remove debug fields from package for final output""" - return { + returned_pkg = { "attr": pkg["attr"], "name": pkg["name"], "system": pkg["system"], "runs_on": pkg["runs_on"], - **( - {"postgresql_version": pkg["postgresql_version"]} - if "postgresql_version" in pkg - else {} - ), } + if is_extension_pkg(pkg): + # Extract PostgreSQL version from attribute path + attrs = pkg["attr"].split(".") + returned_pkg["postgresql_version"] = attrs[-3].split("_")[-1] + return returned_pkg # Group packages by system grouped_by_system = defaultdict(list) From bdac8a330be7e82ae06b73b288e83226256da299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 15:05:41 +0200 Subject: [PATCH 32/58] fix: disable treefmt flake check treefmt is already included in the pre-commit hooks check. --- nix/fmt.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/fmt.nix b/nix/fmt.nix index 562c3b3c5..08763e5b8 100644 --- a/nix/fmt.nix +++ b/nix/fmt.nix @@ -4,6 +4,7 @@ perSystem = { pkgs, ... }: { + treefmt.flakeCheck = false; treefmt.programs = { deadnix.enable = true; nixfmt = { From f71a3f2c7f20833bac39224ed55b26148b4c69ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 15:05:41 +0200 Subject: [PATCH 33/58] feat: run actionlint on new GitHub Actions workflows --- nix/hooks.nix | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/nix/hooks.nix b/nix/hooks.nix index 896c262ba..b659635f8 100644 --- a/nix/hooks.nix +++ b/nix/hooks.nix @@ -1,4 +1,11 @@ { inputs, ... }: +let + ghWorkflows = builtins.attrNames (builtins.readDir ../.github/workflows); + lintedWorkflows = [ + "nix-eval.yml" + "nix-build.yml" + ]; +in { imports = [ inputs.git-hooks.flakeModule ]; perSystem = @@ -8,9 +15,17 @@ check.enable = true; settings = { hooks = { + actionlint = { + enable = true; + excludes = builtins.filter (name: !builtins.elem name lintedWorkflows) ghWorkflows; + verbose = true; + }; + treefmt = { enable = true; package = config.treefmt.build.wrapper; + pass_filenames = false; + verbose = true; }; }; }; From bc3f10e218f5c70d0480f22b739688b082ba3758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 16:08:25 +0200 Subject: [PATCH 34/58] chore: improve github matrix script type annotations --- scripts/github-matrix.py | 43 +++++++++++----------------------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index b80a9f6c0..cc072f33e 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -43,16 +43,12 @@ class RunsOnConfig(TypedDict): class GitHubActionPackage(TypedDict): - """Processed package for GitHub Actions matrix.""" + """Final package output for GitHub Actions matrix.""" attr: str name: str system: str - already_cached: bool runs_on: RunsOnConfig - drvPath: str - neededSubstitutes: List[str] - neededBuilds: List[str] postgresql_version: NotRequired[str] @@ -94,9 +90,7 @@ def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[s return nix_eval_cmd -def parse_nix_eval_line( - line: str, drv_paths: Set[str] -) -> Optional[GitHubActionPackage]: +def parse_nix_eval_line(line: str, drv_paths: Set[str]) -> Optional[NixEvalJobsOutput]: """Parse a single line of nix-eval-jobs output""" if not line.strip(): return None @@ -106,25 +100,13 @@ def parse_nix_eval_line( if data["drvPath"] in drv_paths: return None drv_paths.add(data["drvPath"]) - - runs_on_config = BUILD_RUNNER_MAP[data["system"]] - - return { - "attr": f"{data['attr']}", - "name": data["name"], - "system": data["system"], - "already_cached": data.get("cacheStatus") != "notBuilt", - "runs_on": runs_on_config, - "drvPath": data["drvPath"], - "neededSubstitutes": data.get("neededSubstitutes", []), - "neededBuilds": data.get("neededBuilds", []), - } + return data except json.JSONDecodeError: print(f"Skipping invalid JSON line: {line}", file=sys.stderr) return None -def run_nix_eval_jobs(cmd: List[str]) -> Generator[GitHubActionPackage, None, None]: +def run_nix_eval_jobs(cmd: List[str]) -> Generator[NixEvalJobsOutput, None, None]: """Run nix-eval-jobs and yield parsed package data.""" print(f"Running command: {' '.join(cmd)}", file=sys.stderr) @@ -146,21 +128,21 @@ def run_nix_eval_jobs(cmd: List[str]) -> Generator[GitHubActionPackage, None, No sys.exit(process.returncode) -def is_extension_pkg(pkg: GitHubActionPackage) -> bool: +def is_extension_pkg(pkg: NixEvalJobsOutput) -> bool: """Check if the package is a postgresql extension package.""" attrs = pkg["attr"].split(".") return attrs[-2] == "exts" # thank you buildbot-nix https://github.com/nix-community/buildbot-nix/blob/985d069a2a45cf4a571a4346107671adc2bd2a16/buildbot_nix/buildbot_nix/build_trigger.py#L297 -def sort_pkgs_by_closures(jobs: List[GitHubActionPackage]) -> List[GitHubActionPackage]: +def sort_pkgs_by_closures(jobs: List[NixEvalJobsOutput]) -> List[NixEvalJobsOutput]: sorted_jobs = [] # Prepare job dependencies job_set = {job["drvPath"] for job in jobs} job_closures = { - k["drvPath"]: set(k["neededSubstitutes"]) - .union(set(k["neededBuilds"])) + k["drvPath"]: set(k.get("neededSubstitutes", [])) + .union(set(k.get("neededBuilds", []))) .intersection(job_set) .difference({k["drvPath"]}) for k in jobs @@ -196,13 +178,12 @@ def main() -> None: gh_action_packages = sort_pkgs_by_closures(list(run_nix_eval_jobs(cmd))) - def clean_package_for_output(pkg: GitHubActionPackage) -> dict: - """Remove debug fields from package for final output""" - returned_pkg = { + def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: + """Convert nix-eval-jobs output to GitHub Actions matrix package""" + returned_pkg: GitHubActionPackage = { "attr": pkg["attr"], "name": pkg["name"], "system": pkg["system"], - "runs_on": pkg["runs_on"], } if is_extension_pkg(pkg): # Extract PostgreSQL version from attribute path @@ -213,7 +194,7 @@ def clean_package_for_output(pkg: GitHubActionPackage) -> dict: # Group packages by system grouped_by_system = defaultdict(list) for pkg in gh_action_packages: - if not pkg["already_cached"]: + if pkg.get("cacheStatus") == "notBuilt": grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) # Create output with system-specific matrices From 0fa7e7b64bd390d805e4828ae893732eef24b217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 16:31:07 +0200 Subject: [PATCH 35/58] feat: optimize CI runner selection based on package size Dynamically assign larger runners (32vcpu) for Rust and PostGIS extensions while using smaller runners (8vcpu) for standard packages. --- scripts/github-matrix.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index cc072f33e..f43280552 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -54,15 +54,14 @@ class GitHubActionPackage(TypedDict): BUILD_RUNNER_MAP: Dict[str, RunsOnConfig] = { "aarch64-linux": { - "group": "self-hosted-runners-nix", - "labels": ["aarch64-linux"], + "labels": ["blacksmith-8vcpu-ubuntu-2404-arm"], }, "aarch64-darwin": { "group": "self-hosted-runners-nix", "labels": ["aarch64-darwin"], }, "x86_64-linux": { - "labels": ["blacksmith-32vcpu-ubuntu-2404"], + "labels": ["blacksmith-8vcpu-ubuntu-2404"], }, } @@ -162,6 +161,30 @@ def sort_pkgs_by_closures(jobs: List[NixEvalJobsOutput]) -> List[NixEvalJobsOutp return sorted_jobs +def is_large_pkg(pkg: NixEvalJobsOutput) -> bool: + """Determine if a package is considered large based on its attribute path.""" + RUST_EXTENSIONS = ["exts.wrappers", "exts.pg_jsonschema", "exts.pg_graphql"] + LARGE_C_EXTENSION = ["exts.postgis"] + return any( + indicator in pkg["attr"] for indicator in RUST_EXTENSIONS + LARGE_C_EXTENSION + ) + + +def get_runner_for_package(pkg: NixEvalJobsOutput) -> RunsOnConfig: + """Determine the appropriate GitHub Actions runner for a package.""" + system = pkg["system"] + if is_large_pkg(pkg): + # Use larger runners for large packages for x86_64-linux and aarch64-linux + if system == "x86_64-linux": + return {"labels": ["blacksmith-32vcpu-ubuntu-2404"]} + elif system == "aarch64-linux": + return {"labels": ["blacksmith-32vcpu-ubuntu-2404-arm"]} + if system in BUILD_RUNNER_MAP: + return BUILD_RUNNER_MAP[system] + else: + raise ValueError(f"No runner configuration for system: {system}") + + def main() -> None: parser = argparse.ArgumentParser( description="Generate GitHub Actions matrix for Nix builds" @@ -184,6 +207,7 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: "attr": pkg["attr"], "name": pkg["name"], "system": pkg["system"], + "runs_on": get_runner_for_package(pkg), } if is_extension_pkg(pkg): # Extract PostgreSQL version from attribute path From 958d75c7fb593664954916779a053b77a4aae282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 20:12:21 +0200 Subject: [PATCH 36/58] chore: fix package meta maintainers format --- nix/docs/adding-new-package.md | 2 +- nix/ext/pgvector.nix | 2 +- scripts/github-matrix.py | 15 ++++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/nix/docs/adding-new-package.md b/nix/docs/adding-new-package.md index 7d9fbdae9..d5620de09 100644 --- a/nix/docs/adding-new-package.md +++ b/nix/docs/adding-new-package.md @@ -42,7 +42,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Open-source vector similarity search for Postgres"; homepage = "https://github.com/${src.owner}/${src.repo}"; - maintainers = with maintainers; [ olirice ]; + maintainers = [ "olirice" ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index fa6dba77d..74571890e 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -69,7 +69,7 @@ let meta = with lib; { description = "Open-source vector similarity search for Postgres"; homepage = "https://github.com/${src.owner}/${src.repo}"; - maintainers = with maintainers; [ olirice ]; + maintainers = [ "olirice" ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/scripts/github-matrix.py b/scripts/github-matrix.py index f43280552..cf0256968 100755 --- a/scripts/github-matrix.py +++ b/scripts/github-matrix.py @@ -33,6 +33,7 @@ class NixEvalJobsOutput(TypedDict): neededBuilds: NotRequired[List[Any]] neededSubstitutes: NotRequired[List[Any]] outputs: NotRequired[Dict[str, str]] + error: NotRequired[str] class RunsOnConfig(TypedDict): @@ -96,6 +97,10 @@ def parse_nix_eval_line(line: str, drv_paths: Set[str]) -> Optional[NixEvalJobsO try: data: NixEvalJobsOutput = json.loads(line) + if "error" in data: + raise ValueError( + f"Error in nix-eval-jobs output for {data['attr']}: {data['error']}" + ) if data["drvPath"] in drv_paths: return None drv_paths.add(data["drvPath"]) @@ -149,14 +154,10 @@ def sort_pkgs_by_closures(jobs: List[NixEvalJobsOutput]) -> List[NixEvalJobsOutp sorter = graphlib.TopologicalSorter(job_closures) + job_by_drv = {job["drvPath"]: job for job in jobs} for item in sorter.static_order(): - i = 0 - while i < len(jobs): - if item == jobs[i]["drvPath"]: - sorted_jobs.append(jobs[i]) - del jobs[i] - else: - i += 1 + if item in job_by_drv: + sorted_jobs.append(job_by_drv[item]) return sorted_jobs From ba33531964754c7ff173db0010fd7109b82eae18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 21:32:36 +0200 Subject: [PATCH 37/58] chore: create a nix package for generating GitHub Actions matrix Add pytest tests for the package Add nix-eval-jobs in path for the package --- .github/workflows/nix-eval.yml | 2 +- flake.lock | 143 ++++++++++- flake.nix | 1 + nix/packages/default.nix | 3 + nix/packages/github-matrix/default.nix | 27 +++ .../packages/github-matrix/github_matrix.py | 78 ++++-- .../github-matrix/tests/test_github_matrix.py | 226 ++++++++++++++++++ 7 files changed, 453 insertions(+), 27 deletions(-) create mode 100644 nix/packages/github-matrix/default.nix rename scripts/github-matrix.py => nix/packages/github-matrix/github_matrix.py (77%) create mode 100644 nix/packages/github-matrix/tests/test_github_matrix.py diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml index f5436ce46..e97fe9171 100644 --- a/.github/workflows/nix-eval.yml +++ b/.github/workflows/nix-eval.yml @@ -29,4 +29,4 @@ jobs: name: Generate Nix Matrix run: | set -Eeu - echo matrix="$(nix shell github:nix-community/nix-eval-jobs/v2.31.0 --command scripts/github-matrix.py checks legacyPackages)" >> "$GITHUB_OUTPUT" + echo matrix="$(nix run .\#github-matrix checks legacyPackages)" >> "$GITHUB_OUTPUT" diff --git a/flake.lock b/flake.lock index ad09ae311..27409b771 100644 --- a/flake.lock +++ b/flake.lock @@ -34,6 +34,27 @@ "type": "github" } }, + "flake-parts_2": { + "inputs": { + "nixpkgs-lib": [ + "nix-eval-jobs", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1741352980, + "narHash": "sha256-+u2UunDA4Cl5Fci3m7S643HzKmIDAe+fiXrLqYsR2fs=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "f4330d22f1c5d2ba72d3d22df5597d123fdb60a9", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -95,6 +116,22 @@ "type": "github" } }, + "nix": { + "flake": false, + "locked": { + "lastModified": 1759793966, + "narHash": "sha256-h95Lz2j19/Sb9nuBM3BXUqlJgBkIZd15/QTkufajaLY=", + "owner": "NixOS", + "repo": "nix", + "rev": "eea6d75783e10e6057a097af29741c9a47cbb7e0", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nix", + "type": "github" + } + }, "nix-editor": { "inputs": { "nixpkgs": [ @@ -144,6 +181,28 @@ "type": "github" } }, + "nix-eval-jobs": { + "inputs": { + "flake-parts": "flake-parts_2", + "nix": "nix", + "nixpkgs": "nixpkgs_2", + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1760477819, + "narHash": "sha256-/hfZp5kjN2ratoMtv5JTvekLBUcaXu2i7JgqbNOsctw=", + "owner": "jfroche", + "repo": "nix-eval-jobs", + "rev": "1fe1983db41fbb3e148e8eb26b7ed4cb6f4cf49e", + "type": "github" + }, + "original": { + "owner": "jfroche", + "ref": "show-required-system-features", + "repo": "nix-eval-jobs", + "type": "github" + } + }, "nix2container": { "inputs": { "flake-utils": [ @@ -230,6 +289,66 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 315532800, + "narHash": "sha256-yDxtm0PESdgNetiJN5+MFxgubBcLDTiuSjjrJiyvsvM=", + "rev": "d7f52a7a640bc54c7bb414cca603835bf8dd4b10", + "type": "tarball", + "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre871443.d7f52a7a640b/nixexprs.tar.xz" + }, + "original": { + "type": "tarball", + "url": "https://nixos.org/channels/nixpkgs-unstable/nixexprs.tar.xz" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1697269602, + "narHash": "sha256-dSzV7Ud+JH4DPVD9od53EgDrxUVQOcSj4KGjggCDVJI=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9cb540e9c1910d74a7e10736277f6eb9dff51c81", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1712666087, + "narHash": "sha256-WwjUkWsjlU8iUImbivlYxNyMB1L5YVqE8QotQdL9jWc=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "a76c4553d7e741e17f289224eda135423de0491d", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_5": { + "locked": { + "lastModified": 1744536153, + "narHash": "sha256-awS2zRgF4uTwrOKwwiJcByDzDOdo3Q1rPZbiHQg/N38=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "18dd725c29603f582cf1900e0d25f9f1063dbf11", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { "flake-parts": "flake-parts", @@ -239,10 +358,11 @@ "nix-fast-build": "nix-fast-build", "nix2container": "nix2container", "nixpkgs": "nixpkgs", + "nix-eval-jobs": "nix-eval-jobs", "nixpkgs-go124": "nixpkgs-go124", "nixpkgs-pgbackrest": "nixpkgs-pgbackrest", "rust-overlay": "rust-overlay", - "treefmt-nix": "treefmt-nix" + "treefmt-nix": "treefmt-nix_2" } }, "rust-overlay": { @@ -281,6 +401,27 @@ } }, "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "nix-eval-jobs", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1752055615, + "narHash": "sha256-19m7P4O/Aw/6+CzncWMAJu89JaKeMh3aMle1CNQSIwM=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "c9d477b5d5bd7f26adddd3f96cfd6a904768d4f9", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + }, + "treefmt-nix_2": { "inputs": { "nixpkgs": [ "nixpkgs" diff --git a/flake.nix b/flake.nix index 8f83a0a25..aa1539553 100644 --- a/flake.nix +++ b/flake.nix @@ -28,6 +28,7 @@ git-hooks.inputs.nixpkgs.follows = "nixpkgs"; nixpkgs-go124.url = "github:Nixos/nixpkgs/d2ac4dfa61fba987a84a0a81555da57ae0b9a2b0"; nixpkgs-pgbackrest.url = "github:nixos/nixpkgs/nixos-unstable-small"; + nix-eval-jobs.url = "github:jfroche/nix-eval-jobs/show-required-system-features"; }; outputs = diff --git a/nix/packages/default.nix b/nix/packages/default.nix index c8eb02ef0..fa6ab10a4 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -33,6 +33,9 @@ cleanup-ami = pkgs.callPackage ./cleanup-ami.nix { }; dbmate-tool = pkgs.callPackage ./dbmate-tool.nix { inherit (self.supabase) defaults; }; docs = pkgs.callPackage ./docs.nix { }; + github-matrix = pkgs.callPackage ./github-matrix { + nix-eval-jobs = inputs'.nix-eval-jobs.packages.default; + }; supabase-groonga = pkgs.callPackage ./groonga { }; http-mock-server = pkgs.callPackage ./http-mock-server.nix { }; local-infra-bootstrap = pkgs.callPackage ./local-infra-bootstrap.nix { }; diff --git a/nix/packages/github-matrix/default.nix b/nix/packages/github-matrix/default.nix new file mode 100644 index 000000000..0d755c977 --- /dev/null +++ b/nix/packages/github-matrix/default.nix @@ -0,0 +1,27 @@ +{ + lib, + nix-eval-jobs, + python3Packages, +}: +let + pname = "github-matrix"; +in + +python3Packages.buildPythonApplication { + inherit pname; + version = "0.1.0"; + pyproject = false; + + src = ./.; + + makeWrapperArgs = [ "--suffix PATH : ${lib.makeBinPath [ nix-eval-jobs ]}" ]; + + nativeCheckInputs = with python3Packages; [ + pytestCheckHook + pytest-mypy + ]; + + installPhase = '' + install -Dm755 github_matrix.py "$out/bin/${pname}" + ''; +} diff --git a/scripts/github-matrix.py b/nix/packages/github-matrix/github_matrix.py similarity index 77% rename from scripts/github-matrix.py rename to nix/packages/github-matrix/github_matrix.py index cf0256968..2c4681670 100755 --- a/scripts/github-matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -19,6 +19,9 @@ TypedDict, ) +System = Literal["x86_64-linux", "aarch64-linux", "aarch64-darwin"] +RunnerType = Literal["ephemeral", "self-hosted"] + class NixEvalJobsOutput(TypedDict): """Raw output from nix-eval-jobs command.""" @@ -27,9 +30,8 @@ class NixEvalJobsOutput(TypedDict): attrPath: List[str] cacheStatus: Literal["notBuilt", "cached", "local"] drvPath: str - isCached: bool name: str - system: str + system: System neededBuilds: NotRequired[List[Any]] neededSubstitutes: NotRequired[List[Any]] outputs: NotRequired[Dict[str, str]] @@ -48,21 +50,29 @@ class GitHubActionPackage(TypedDict): attr: str name: str - system: str + system: System runs_on: RunsOnConfig postgresql_version: NotRequired[str] -BUILD_RUNNER_MAP: Dict[str, RunsOnConfig] = { - "aarch64-linux": { - "labels": ["blacksmith-8vcpu-ubuntu-2404-arm"], - }, - "aarch64-darwin": { - "group": "self-hosted-runners-nix", - "labels": ["aarch64-darwin"], +BUILD_RUNNER_MAP: Dict[RunnerType, Dict[System, RunsOnConfig]] = { + "ephemeral": { + "aarch64-linux": { + "labels": ["blacksmith-8vcpu-ubuntu-2404-arm"], + }, + "x86_64-linux": { + "labels": ["blacksmith-8vcpu-ubuntu-2404"], + }, }, - "x86_64-linux": { - "labels": ["blacksmith-8vcpu-ubuntu-2404"], + "self-hosted": { + "aarch64-darwin": { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-darwin"], + }, + "aarch64-linux": { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-linux"], + }, }, } @@ -76,6 +86,7 @@ def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[s "--check-cache-status", "--force-recurse", "--quiet", + "--show-required-system-features", "--option", "eval-cache", "false", @@ -171,19 +182,33 @@ def is_large_pkg(pkg: NixEvalJobsOutput) -> bool: ) -def get_runner_for_package(pkg: NixEvalJobsOutput) -> RunsOnConfig: - """Determine the appropriate GitHub Actions runner for a package.""" +def is_kvm_pkg(pkg: NixEvalJobsOutput) -> bool: + """Determine if a package requires KVM""" + return "kvm" in pkg.get("requiredSystemFeatures", []) + + +def get_runner_for_package(pkg: NixEvalJobsOutput) -> RunsOnConfig | None: + """Determine the appropriate GitHub Actions runner for a package. + + Priority order: + 1. KVM packages → self-hosted runners + 2. Large packages on Linux → 32vcpu ephemeral runners + 3. Darwin packages → self-hosted runners + 4. Default → ephemeral runners + """ system = pkg["system"] - if is_large_pkg(pkg): - # Use larger runners for large packages for x86_64-linux and aarch64-linux - if system == "x86_64-linux": - return {"labels": ["blacksmith-32vcpu-ubuntu-2404"]} - elif system == "aarch64-linux": - return {"labels": ["blacksmith-32vcpu-ubuntu-2404-arm"]} - if system in BUILD_RUNNER_MAP: - return BUILD_RUNNER_MAP[system] - else: - raise ValueError(f"No runner configuration for system: {system}") + + if is_kvm_pkg(pkg): + return BUILD_RUNNER_MAP["self-hosted"].get(system) + + if is_large_pkg(pkg) and system in ("x86_64-linux", "aarch64-linux"): + suffix = "-arm" if system == "aarch64-linux" else "" + return {"labels": [f"blacksmith-32vcpu-ubuntu-2404{suffix}"]} + + if system == "aarch64-darwin": + return BUILD_RUNNER_MAP["self-hosted"]["aarch64-darwin"] + + return BUILD_RUNNER_MAP["ephemeral"].get(system) def main() -> None: @@ -204,11 +229,14 @@ def main() -> None: def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: """Convert nix-eval-jobs output to GitHub Actions matrix package""" + runner = get_runner_for_package(pkg) + if runner is None: + raise ValueError(f"No runner configuration for system: {pkg['system']}") returned_pkg: GitHubActionPackage = { "attr": pkg["attr"], "name": pkg["name"], "system": pkg["system"], - "runs_on": get_runner_for_package(pkg), + "runs_on": runner, } if is_extension_pkg(pkg): # Extract PostgreSQL version from attribute path diff --git a/nix/packages/github-matrix/tests/test_github_matrix.py b/nix/packages/github-matrix/tests/test_github_matrix.py new file mode 100644 index 000000000..0e5069fdf --- /dev/null +++ b/nix/packages/github-matrix/tests/test_github_matrix.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 + +import pytest + +from github_matrix import ( + NixEvalJobsOutput, + get_runner_for_package, + is_extension_pkg, + is_kvm_pkg, + is_large_pkg, + sort_pkgs_by_closures, +) + + +class TestIsExtensionPkg: + def test_extension_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15.exts.pg_cron", + "attrPath": ["packages", "x86_64-linux", "psql_15", "exts", "pg_cron"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "pg_cron", + "system": "x86_64-linux", + } + assert is_extension_pkg(pkg) is True + + def test_non_extension_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15", + "attrPath": ["packages", "x86_64-linux", "psql_15"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgresql-16.0", + "system": "x86_64-linux", + } + assert is_extension_pkg(pkg) is False + + +class TestIsLargePkg: + @pytest.mark.parametrize( + "attr,expected", + [ + ("packages.x86_64-linux.psql_15.exts.wrappers", True), + ("packages.x86_64-linux.psql_15.exts.pg_jsonschema", True), + ("packages.x86_64-linux.psql_15.exts.pg_graphql", True), + ("packages.x86_64-linux.psql_15.exts.postgis", True), + ("packages.x86_64-linux.psql_15.exts.pg_cron", False), + ("packages.x86_64-linux.psql_15", False), + ], + ) + def test_large_package_detection(self, attr: str, expected: bool): + pkg: NixEvalJobsOutput = { + "attr": attr, + "attrPath": attr.split("."), + "cacheStatus": "notBuilt", + "drvPath": f"/nix/store/{attr}.drv", + "name": attr.split(".")[-1], + "system": "x86_64-linux", + } + assert is_large_pkg(pkg) is expected + + +class TestIsKvmPkg: + def test_kvm_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.vm-test", + "attrPath": ["packages", "x86_64-linux", "vm-test"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "vm-test", + "system": "x86_64-linux", + "requiredSystemFeatures": ["kvm"], + } + assert is_kvm_pkg(pkg) is True + + def test_non_kvm_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15", + "attrPath": ["packages", "x86_64-linux", "psql_15"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgresql-16.0", + "system": "x86_64-linux", + } + assert is_kvm_pkg(pkg) is False + + +class TestGetRunnerForPackage: + def test_kvm_package_x86_64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.vm-test", + "attrPath": ["packages", "x86_64-linux", "vm-test"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "vm-test", + "system": "x86_64-linux", + "requiredSystemFeatures": ["kvm"], + } + result = get_runner_for_package(pkg) + assert result is None # x86_64-linux not in self-hosted map + + def test_kvm_package_aarch64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.aarch64-linux.vm-test", + "attrPath": ["packages", "aarch64-linux", "vm-test"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "vm-test", + "system": "aarch64-linux", + "requiredSystemFeatures": ["kvm"], + } + result = get_runner_for_package(pkg) + assert result == { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-linux"], + } + + def test_large_package_x86_64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15.exts.postgis", + "attrPath": ["packages", "x86_64-linux", "psql_15", "exts", "postgis"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgis", + "system": "x86_64-linux", + } + result = get_runner_for_package(pkg) + assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404"]} + + def test_large_package_aarch64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.aarch64-linux.psql_15.exts.pg_graphql", + "attrPath": ["packages", "aarch64-linux", "psql_15", "exts", "pg_graphql"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "pg_graphql", + "system": "aarch64-linux", + } + result = get_runner_for_package(pkg) + assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404-arm"]} + + def test_darwin_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.aarch64-darwin.psql_15", + "attrPath": ["packages", "aarch64-darwin", "psql_15"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgresql-16.0", + "system": "aarch64-darwin", + } + result = get_runner_for_package(pkg) + assert result == { + "group": "self-hosted-runners-nix", + "labels": ["aarch64-darwin"], + } + + def test_default_x86_64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15.exts.pg_cron", + "attrPath": ["packages", "x86_64-linux", "psql_15", "exts", "pg_cron"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "pg_cron", + "system": "x86_64-linux", + } + result = get_runner_for_package(pkg) + assert result == {"labels": ["blacksmith-8vcpu-ubuntu-2404"]} + + def test_default_aarch64_linux(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.aarch64-linux.psql_15.exts.pg_cron", + "attrPath": ["packages", "aarch64-linux", "psql_15", "exts", "pg_cron"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "pg_cron", + "system": "aarch64-linux", + } + result = get_runner_for_package(pkg) + assert result == {"labels": ["blacksmith-8vcpu-ubuntu-2404-arm"]} + + +class TestSortPkgsByClosures: + def test_empty_list(self): + result = sort_pkgs_by_closures([]) + assert result == [] + + def test_single_package(self): + pkg: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.psql_15", + "attrPath": ["packages", "x86_64-linux", "psql_15"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/test.drv", + "name": "postgresql-16.0", + "system": "x86_64-linux", + } + result = sort_pkgs_by_closures([pkg]) + assert result == [pkg] + + def test_dependency_order(self): + pkg1: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.lib", + "attrPath": ["packages", "x86_64-linux", "lib"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/lib.drv", + "name": "lib", + "system": "x86_64-linux", + "neededBuilds": [], + "neededSubstitutes": [], + } + pkg2: NixEvalJobsOutput = { + "attr": "packages.x86_64-linux.app", + "attrPath": ["packages", "x86_64-linux", "app"], + "cacheStatus": "notBuilt", + "drvPath": "/nix/store/app.drv", + "name": "app", + "system": "x86_64-linux", + "neededBuilds": ["/nix/store/lib.drv"], + "neededSubstitutes": [], + } + + # Regardless of input order, lib should come before app + result = sort_pkgs_by_closures([pkg2, pkg1]) + assert result == [pkg1, pkg2] + + result = sort_pkgs_by_closures([pkg1, pkg2]) + assert result == [pkg1, pkg2] From 47fdc1d0aaf2fb446e5efe3f972d7f3c73d6c9b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 6 Oct 2025 21:32:36 +0200 Subject: [PATCH 38/58] fix: configure runner according to the matrix job The matrix job returns the type of runner, so we can configure the nix installation step accordingly. --- .github/workflows/nix-build.yml | 11 +++++++++-- .github/workflows/nix-eval.yml | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 0c3e78bc2..f326a29d3 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -35,7 +35,14 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@v4 - - name: Build Nix Package + - name: Install nix (ephemeral) + if: ${{ matrix.runs_on.group != 'self-hosted-runners-nix' }} + uses: ./.github/actions/nix-install-ephemeral + env: + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} + NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} + - name: Install nix (self-hosted) + if: ${{ matrix.runs_on.group == 'self-hosted-runners-nix' }} uses: ./.github/actions/nix-install-self-hosted - name: nix build shell: bash @@ -55,7 +62,7 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@v4 - - name: Build Nix Package + - name: Install nix uses: ./.github/actions/nix-install-self-hosted - name: nix build shell: bash diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml index e97fe9171..ada3ec844 100644 --- a/.github/workflows/nix-eval.yml +++ b/.github/workflows/nix-eval.yml @@ -29,4 +29,4 @@ jobs: name: Generate Nix Matrix run: | set -Eeu - echo matrix="$(nix run .\#github-matrix checks legacyPackages)" >> "$GITHUB_OUTPUT" + echo matrix="$(nix run --accept-flake-config .\#github-matrix checks legacyPackages)" >> "$GITHUB_OUTPUT" From 55828f7a74a3c1bfafdb1b701cf1fb99460392b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Wed, 15 Oct 2025 00:57:43 +0200 Subject: [PATCH 39/58] Update nix-eval-jobs Our changes were merged upstream, so we can now track the original repository again. --- flake.lock | 35 ++++++++++----------- flake.nix | 2 +- nix/packages/github-matrix/github_matrix.py | 1 - 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/flake.lock b/flake.lock index 27409b771..78fb1a714 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ ] }, "locked": { - "lastModified": 1741352980, - "narHash": "sha256-+u2UunDA4Cl5Fci3m7S643HzKmIDAe+fiXrLqYsR2fs=", + "lastModified": 1759362264, + "narHash": "sha256-wfG0S7pltlYyZTM+qqlhJ7GMw2fTF4mLKCIVhLii/4M=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "f4330d22f1c5d2ba72d3d22df5597d123fdb60a9", + "rev": "758cf7296bee11f1706a574c77d072b8a7baa881", "type": "github" }, "original": { @@ -119,11 +119,11 @@ "nix": { "flake": false, "locked": { - "lastModified": 1759793966, - "narHash": "sha256-h95Lz2j19/Sb9nuBM3BXUqlJgBkIZd15/QTkufajaLY=", + "lastModified": 1760472641, + "narHash": "sha256-BuKtM7Vr5EcxBXxUENBQPlOBwmNd5mkTRkSmlJi/iQ4=", "owner": "NixOS", "repo": "nix", - "rev": "eea6d75783e10e6057a097af29741c9a47cbb7e0", + "rev": "4041bfdb401ad6d1c31a292fab90392254be506a", "type": "github" }, "original": { @@ -189,16 +189,15 @@ "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1760477819, - "narHash": "sha256-/hfZp5kjN2ratoMtv5JTvekLBUcaXu2i7JgqbNOsctw=", - "owner": "jfroche", + "lastModified": 1760478325, + "narHash": "sha256-hA+NOH8KDcsuvH7vJqSwk74PyZP3MtvI/l+CggZcnTc=", + "owner": "nix-community", "repo": "nix-eval-jobs", - "rev": "1fe1983db41fbb3e148e8eb26b7ed4cb6f4cf49e", + "rev": "daa42f9e9c84aeff1e325dd50fda321f53dfd02c", "type": "github" }, "original": { - "owner": "jfroche", - "ref": "show-required-system-features", + "owner": "nix-community", "repo": "nix-eval-jobs", "type": "github" } @@ -292,10 +291,10 @@ "nixpkgs_2": { "locked": { "lastModified": 315532800, - "narHash": "sha256-yDxtm0PESdgNetiJN5+MFxgubBcLDTiuSjjrJiyvsvM=", - "rev": "d7f52a7a640bc54c7bb414cca603835bf8dd4b10", + "narHash": "sha256-vhAtaRMIQiEghARviANBmSnhGz9Qf2IQJ+nQgsDXnVs=", + "rev": "c12c63cd6c5eb34c7b4c3076c6a99e00fcab86ec", "type": "tarball", - "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre871443.d7f52a7a640b/nixexprs.tar.xz" + "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre877036.c12c63cd6c5e/nixexprs.tar.xz" }, "original": { "type": "tarball", @@ -408,11 +407,11 @@ ] }, "locked": { - "lastModified": 1752055615, - "narHash": "sha256-19m7P4O/Aw/6+CzncWMAJu89JaKeMh3aMle1CNQSIwM=", + "lastModified": 1760120816, + "narHash": "sha256-gq9rdocpmRZCwLS5vsHozwB6b5nrOBDNc2kkEaTXHfg=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "c9d477b5d5bd7f26adddd3f96cfd6a904768d4f9", + "rev": "761ae7aff00907b607125b2f57338b74177697ed", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index aa1539553..efd0b7e9e 100644 --- a/flake.nix +++ b/flake.nix @@ -28,7 +28,7 @@ git-hooks.inputs.nixpkgs.follows = "nixpkgs"; nixpkgs-go124.url = "github:Nixos/nixpkgs/d2ac4dfa61fba987a84a0a81555da57ae0b9a2b0"; nixpkgs-pgbackrest.url = "github:nixos/nixpkgs/nixos-unstable-small"; - nix-eval-jobs.url = "github:jfroche/nix-eval-jobs/show-required-system-features"; + nix-eval-jobs.url = "github:nix-community/nix-eval-jobs"; }; outputs = diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 2c4681670..22caa7d1c 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -86,7 +86,6 @@ def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[s "--check-cache-status", "--force-recurse", "--quiet", - "--show-required-system-features", "--option", "eval-cache", "false", From 059280de4de265b7d100122ea860ca2c165a365b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 27 Oct 2025 16:23:37 +0100 Subject: [PATCH 40/58] refactor(ci): standardize nix installation and disable cache push by default - Replace DeterminateSystems/nix-installer-action with custom nix-install-ephemeral action across all workflows - Change default push-to-cache from 'true' to 'false' to prevent unnecessary nix/aws configurations - Explicitly enable push-to-cache only for nix-build and nix-eval workflows where caching is beneficial --- .github/workflows/nix-build.yml | 4 ++++ .github/workflows/nix-eval.yml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index f326a29d3..c5eb06d84 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -38,6 +38,8 @@ jobs: - name: Install nix (ephemeral) if: ${{ matrix.runs_on.group != 'self-hosted-runners-nix' }} uses: ./.github/actions/nix-install-ephemeral + with: + push-to-cache: 'true' env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} @@ -84,6 +86,8 @@ jobs: uses: actions/checkout@v4 - name: Install nix uses: ./.github/actions/nix-install-ephemeral + with: + push-to-cache: 'true' env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml index ada3ec844..a899e2371 100644 --- a/.github/workflows/nix-eval.yml +++ b/.github/workflows/nix-eval.yml @@ -22,6 +22,8 @@ jobs: uses: actions/checkout@v4 - name: Install nix uses: ./.github/actions/nix-install-ephemeral + with: + push-to-cache: 'true' env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} From 374c5636bdaabbeec877b1b7d990be65e5febb78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 27 Oct 2025 18:02:20 +0100 Subject: [PATCH 41/58] feat: use big-parallel to identify large packages --- nix/ext/pg_graphql/default.nix | 7 +++++-- nix/ext/pg_jsonschema/default.nix | 7 +++++-- nix/ext/postgis.nix | 7 +++++-- nix/ext/wrappers/default.nix | 7 +++++-- nix/packages/github-matrix/github_matrix.py | 14 ++++++++------ .../github-matrix/tests/test_github_matrix.py | 10 ++++++++-- 6 files changed, 36 insertions(+), 16 deletions(-) diff --git a/nix/ext/pg_graphql/default.nix b/nix/ext/pg_graphql/default.nix index d944d5ede..a63fcd84e 100644 --- a/nix/ext/pg_graphql/default.nix +++ b/nix/ext/pg_graphql/default.nix @@ -129,7 +129,7 @@ let lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions ); in -buildEnv { +(buildEnv { name = pname; paths = packages; pathsToLink = [ @@ -177,4 +177,7 @@ buildEnv { version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; -} +}).overrideAttrs + (_: { + requiredSystemFeatures = [ "big-parallel" ]; + }) diff --git a/nix/ext/pg_jsonschema/default.nix b/nix/ext/pg_jsonschema/default.nix index 74742bed5..9939b733a 100644 --- a/nix/ext/pg_jsonschema/default.nix +++ b/nix/ext/pg_jsonschema/default.nix @@ -130,7 +130,7 @@ let lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions ); in -pkgs.buildEnv { +(pkgs.buildEnv { name = pname; paths = packages; pathsToLink = [ @@ -170,4 +170,7 @@ pkgs.buildEnv { version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; -} +}).overrideAttrs + (_: { + requiredSystemFeatures = [ "big-parallel" ]; + }) diff --git a/nix/ext/postgis.nix b/nix/ext/postgis.nix index ed1b738e4..e5b19ccfb 100644 --- a/nix/ext/postgis.nix +++ b/nix/ext/postgis.nix @@ -179,7 +179,7 @@ let }; }; in -buildEnv { +(buildEnv { name = pname; paths = packages; @@ -209,4 +209,7 @@ buildEnv { version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; -} +}).overrideAttrs + (_: { + requiredSystemFeatures = [ "big-parallel" ]; + }) diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 89effcfb3..4a7f03653 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -218,7 +218,7 @@ let lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions ); in -buildEnv { +(buildEnv { name = pname; paths = packages; pathsToLink = [ @@ -303,4 +303,7 @@ buildEnv { version = "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; -} +}).overrideAttrs + (_: { + requiredSystemFeatures = [ "big-parallel" ]; + }) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 22caa7d1c..324459706 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -36,6 +36,7 @@ class NixEvalJobsOutput(TypedDict): neededSubstitutes: NotRequired[List[Any]] outputs: NotRequired[Dict[str, str]] error: NotRequired[str] + requiredSystemFeatures: NotRequired[List[str]] class RunsOnConfig(TypedDict): @@ -174,11 +175,7 @@ def sort_pkgs_by_closures(jobs: List[NixEvalJobsOutput]) -> List[NixEvalJobsOutp def is_large_pkg(pkg: NixEvalJobsOutput) -> bool: """Determine if a package is considered large based on its attribute path.""" - RUST_EXTENSIONS = ["exts.wrappers", "exts.pg_jsonschema", "exts.pg_graphql"] - LARGE_C_EXTENSION = ["exts.postgis"] - return any( - indicator in pkg["attr"] for indicator in RUST_EXTENSIONS + LARGE_C_EXTENSION - ) + return "big-parallel" in pkg.get("requiredSystemFeatures", []) def is_kvm_pkg(pkg: NixEvalJobsOutput) -> bool: @@ -198,7 +195,12 @@ def get_runner_for_package(pkg: NixEvalJobsOutput) -> RunsOnConfig | None: system = pkg["system"] if is_kvm_pkg(pkg): - return BUILD_RUNNER_MAP["self-hosted"].get(system) + runConfig = BUILD_RUNNER_MAP["self-hosted"].get(system) + if runConfig is None: + raise ValueError( + f"No self-hosted with kvm support available for system: {system}" + ) + return runConfig if is_large_pkg(pkg) and system in ("x86_64-linux", "aarch64-linux"): suffix = "-arm" if system == "aarch64-linux" else "" diff --git a/nix/packages/github-matrix/tests/test_github_matrix.py b/nix/packages/github-matrix/tests/test_github_matrix.py index 0e5069fdf..3c8c9188d 100644 --- a/nix/packages/github-matrix/tests/test_github_matrix.py +++ b/nix/packages/github-matrix/tests/test_github_matrix.py @@ -56,6 +56,7 @@ def test_large_package_detection(self, attr: str, expected: bool): "drvPath": f"/nix/store/{attr}.drv", "name": attr.split(".")[-1], "system": "x86_64-linux", + "requiredSystemFeatures": ["big-parallel"] if expected else [], } assert is_large_pkg(pkg) is expected @@ -96,8 +97,11 @@ def test_kvm_package_x86_64_linux(self): "system": "x86_64-linux", "requiredSystemFeatures": ["kvm"], } - result = get_runner_for_package(pkg) - assert result is None # x86_64-linux not in self-hosted map + with pytest.raises( + ValueError, + match=r"No self-hosted with kvm support available for system: x86_64-linux", + ): + get_runner_for_package(pkg) def test_kvm_package_aarch64_linux(self): pkg: NixEvalJobsOutput = { @@ -123,6 +127,7 @@ def test_large_package_x86_64_linux(self): "drvPath": "/nix/store/test.drv", "name": "postgis", "system": "x86_64-linux", + "requiredSystemFeatures": ["big-parallel"], } result = get_runner_for_package(pkg) assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404"]} @@ -135,6 +140,7 @@ def test_large_package_aarch64_linux(self): "drvPath": "/nix/store/test.drv", "name": "pg_graphql", "system": "aarch64-linux", + "requiredSystemFeatures": ["big-parallel"], } result = get_runner_for_package(pkg) assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404-arm"]} From e2ffcc4fe86c41e0b0f1b3efc713df338a996640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 27 Oct 2025 18:02:20 +0100 Subject: [PATCH 42/58] fix(ci): ensure x86_64-linux build is considered in testinfra and test workflows --- .github/workflows/nix-build.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index c5eb06d84..da96d336a 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -103,7 +103,8 @@ jobs: if: | !cancelled() && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && - (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') + (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') && + (needs.nix-build-x86_64-linux.result == 'skipped' || needs.nix-build-x86_64-linux.result == 'success') uses: ./.github/workflows/testinfra-ami-build.yml secrets: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} @@ -113,5 +114,6 @@ jobs: if: | !cancelled() && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && - (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') + (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') && + (needs.nix-build-x86_64-linux.result == 'skipped' || needs.nix-build-x86_64-linux.result == 'success') uses: ./.github/workflows/test.yml From 607d7ada3bf8b1090f3f03b0550e970cf41fcc31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 27 Oct 2025 18:02:20 +0100 Subject: [PATCH 43/58] fix: nix devShell inclusion condition --- nix/checks.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/checks.nix b/nix/checks.nix index ee76cad3a..553d9b24e 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -412,7 +412,7 @@ inherit self; inherit pkgs; }) - // pkgs.lib.optionalAttrs (system == "x86_64-linux") { devShell = self'.devShells.default; } - ); + ) + // pkgs.lib.optionalAttrs (system == "x86_64-linux") ({ devShell = self'.devShells.default; }); }; } From 4a4a21bf82b6c3028ee4f5d891f31603f12e2fd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Wed, 29 Oct 2025 23:53:08 +0100 Subject: [PATCH 44/58] fix(ci): eval should fail if github-matrix run fails --- .github/workflows/nix-eval.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml index a899e2371..8f5776ebd 100644 --- a/.github/workflows/nix-eval.yml +++ b/.github/workflows/nix-eval.yml @@ -30,5 +30,5 @@ jobs: - id: set-matrix name: Generate Nix Matrix run: | - set -Eeu - echo matrix="$(nix run --accept-flake-config .\#github-matrix checks legacyPackages)" >> "$GITHUB_OUTPUT" + set -Eeu -o pipefail + echo matrix="$(nix run --accept-flake-config .\#github-matrix -- checks legacyPackages)" >> "$GITHUB_OUTPUT" From 4c30d3481790c48b7c45df1e8e7f014fee5fd468 Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Fri, 14 Nov 2025 10:54:26 +0100 Subject: [PATCH 45/58] fix(ci): remove redundant build psql bundle step --- .github/workflows/nix-build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index da96d336a..0f8331f7c 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -91,9 +91,6 @@ jobs: env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - - name: Build psql bundle - run: > - nix build ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" - name: nix build shell: bash run: nix build --accept-flake-config -L .#${{ matrix.attr }} From 7e2aae63314ee34c57b7f6fa9f28e78a1a6bf0e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 14 Nov 2025 15:39:15 +0100 Subject: [PATCH 46/58] fix: reduce ARM runner size from 8vcpu to 4vcpu for ephemeral builds We might not need the full 8vcpu for aarch64-linux builds, so this change reduces the runner size to 4vcpu to wait less for available blacksmith runners. --- nix/packages/github-matrix/github_matrix.py | 2 +- nix/packages/github-matrix/tests/test_github_matrix.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 324459706..68daa0edf 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -59,7 +59,7 @@ class GitHubActionPackage(TypedDict): BUILD_RUNNER_MAP: Dict[RunnerType, Dict[System, RunsOnConfig]] = { "ephemeral": { "aarch64-linux": { - "labels": ["blacksmith-8vcpu-ubuntu-2404-arm"], + "labels": ["blacksmith-4vcpu-ubuntu-2404-arm"], }, "x86_64-linux": { "labels": ["blacksmith-8vcpu-ubuntu-2404"], diff --git a/nix/packages/github-matrix/tests/test_github_matrix.py b/nix/packages/github-matrix/tests/test_github_matrix.py index 3c8c9188d..d5fed5732 100644 --- a/nix/packages/github-matrix/tests/test_github_matrix.py +++ b/nix/packages/github-matrix/tests/test_github_matrix.py @@ -182,7 +182,7 @@ def test_default_aarch64_linux(self): "system": "aarch64-linux", } result = get_runner_for_package(pkg) - assert result == {"labels": ["blacksmith-8vcpu-ubuntu-2404-arm"]} + assert result == {"labels": ["blacksmith-4vcpu-ubuntu-2404-arm"]} class TestSortPkgsByClosures: From d49a0f595ed7437434fc35dfe7ec5426a777aacd Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Fri, 21 Nov 2025 15:39:58 +0100 Subject: [PATCH 47/58] Revert "fix(ci): limit max-jobs of nix to 8 to prevent OOM while running nix flake check (#1933)" This reverts commit 593595291fe26a5cf8270fea6b7d4f6834d70d55. --- .github/actions/nix-install-ephemeral/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml index 82eaa32bd..caa9a051d 100644 --- a/.github/actions/nix-install-ephemeral/action.yml +++ b/.github/actions/nix-install-ephemeral/action.yml @@ -44,4 +44,3 @@ runs: substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= ${{ inputs.push-to-cache == 'true' && 'post-build-hook = /etc/nix/upload-to-cache.sh' || '' }} - max-jobs = 8 From 3d156305b94616fb6129ccbaa6d61d853ae42aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 24 Nov 2025 16:24:48 +0100 Subject: [PATCH 48/58] feat: do not return empty matrices if no package has to be built --- .github/workflows/nix-build.yml | 16 ++++++++++++---- nix/packages/github-matrix/github_matrix.py | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 0f8331f7c..379488878 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -34,9 +34,10 @@ jobs: matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_linux }} steps: - name: Checkout Repo + if: ${{ matrix.attr != '' }} uses: actions/checkout@v4 - name: Install nix (ephemeral) - if: ${{ matrix.runs_on.group != 'self-hosted-runners-nix' }} + if: ${{ matrix.attr != '' && matrix.runs_on.group != 'self-hosted-runners-nix' }} uses: ./.github/actions/nix-install-ephemeral with: push-to-cache: 'true' @@ -44,9 +45,10 @@ jobs: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - name: Install nix (self-hosted) - if: ${{ matrix.runs_on.group == 'self-hosted-runners-nix' }} + if: ${{ matrix.attr != '' && matrix.runs_on.group == 'self-hosted-runners-nix' }} uses: ./.github/actions/nix-install-self-hosted - name: nix build + if: ${{ matrix.attr != '' }} shell: bash run: nix build --accept-flake-config -L .#${{ matrix.attr }} @@ -55,7 +57,7 @@ jobs: ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} (aarch64-darwin) needs: nix-eval - runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin != null }} strategy: fail-fast: false @@ -63,10 +65,13 @@ jobs: matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).aarch64_darwin }} steps: - name: Checkout Repo + if: ${{ matrix.attr != '' }} uses: actions/checkout@v4 - name: Install nix + if: ${{ matrix.attr != '' }} uses: ./.github/actions/nix-install-self-hosted - name: nix build + if: ${{ matrix.attr != '' }} shell: bash run: nix build --accept-flake-config -L .#${{ matrix.attr }} @@ -75,7 +80,7 @@ jobs: ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} (x86_64-linux) needs: nix-eval - runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} + runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} if: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux != null }} strategy: fail-fast: false @@ -83,8 +88,10 @@ jobs: matrix: ${{ fromJSON(needs.nix-eval.outputs.matrix).x86_64_linux }} steps: - name: Checkout Repo + if: ${{ matrix.attr != '' }} uses: actions/checkout@v4 - name: Install nix + if: ${{ matrix.attr != '' }} uses: ./.github/actions/nix-install-ephemeral with: push-to-cache: 'true' @@ -92,6 +99,7 @@ jobs: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - name: nix build + if: ${{ matrix.attr != '' }} shell: bash run: nix build --accept-flake-config -L .#${{ matrix.attr }} diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 68daa0edf..e691dc2be 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -17,6 +17,7 @@ Optional, Set, TypedDict, + get_args, ) System = Literal["x86_64-linux", "aarch64-linux", "aarch64-darwin"] @@ -252,10 +253,23 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: grouped_by_system[pkg["system"]].append(clean_package_for_output(pkg)) # Create output with system-specific matrices + # Ensure that we have at least one entry per system gh_output = {} for system, packages in grouped_by_system.items(): gh_output[system.replace("-", "_")] = {"include": packages} + for system in get_args(System): + if system not in gh_output: + gh_output[system.replace("-", "_")] = { + "include": [ + { + "attr": "", + "name": "skipped", + "system": system, + "runs_on": {"labels": "ubuntu-latest"}, + } + ] + } print( f"debug: Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}", file=sys.stderr, From 679ac53b2bd2d6e8c7b7821a49df807da55012c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 24 Nov 2025 16:24:48 +0100 Subject: [PATCH 49/58] feat: fail pipeline if nix evaluation fails --- .github/workflows/nix-eval.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml index 8f5776ebd..ca3f98a5e 100644 --- a/.github/workflows/nix-eval.yml +++ b/.github/workflows/nix-eval.yml @@ -31,4 +31,5 @@ jobs: name: Generate Nix Matrix run: | set -Eeu -o pipefail - echo matrix="$(nix run --accept-flake-config .\#github-matrix -- checks legacyPackages)" >> "$GITHUB_OUTPUT" + result=$(nix run --accept-flake-config .\#github-matrix -- checks legacyPackages) + echo matrix="$result" >> "$GITHUB_OUTPUT" From e302372410b287e291cc652234839fe3b0f189be Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Mon, 24 Nov 2025 23:01:17 +0100 Subject: [PATCH 50/58] Update nix/ext/pgvector.nix Co-authored-by: samrose --- nix/ext/pgvector.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index 74571890e..b8f8347a3 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -69,7 +69,6 @@ let meta = with lib; { description = "Open-source vector similarity search for Postgres"; homepage = "https://github.com/${src.owner}/${src.repo}"; - maintainers = [ "olirice" ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; From 20fa875a8de1b138593d69856b726f5cd7dd40ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 25 Nov 2025 18:44:09 +0100 Subject: [PATCH 51/58] fix: add skip job only for systems that don't have any job --- nix/packages/github-matrix/github_matrix.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index e691dc2be..24c97ab74 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -259,8 +259,9 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: gh_output[system.replace("-", "_")] = {"include": packages} for system in get_args(System): - if system not in gh_output: - gh_output[system.replace("-", "_")] = { + s = system.replace("-", "_") + if s not in gh_output: + gh_output[s] = { "include": [ { "attr": "", From 555a4af48ceba3e676f286b810fb1d3f1f534361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 25 Nov 2025 20:47:29 +0100 Subject: [PATCH 52/58] fix(github-matrix): handle evaluation errors without deadlock Fix github-matrix that would hang when nix-eval-jobs encountered errors due to subprocess pipe deadlock - stderr buffer would fill while reading stdout. This change ensure that evaluation errors are visible and the workflow fails properly while still showing which packages succeeded. --- .github/workflows/nix-eval.yml | 4 ++ nix/packages/github-matrix/github_matrix.py | 44 +++++++++++++++------ 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml index ca3f98a5e..adf74eaa1 100644 --- a/.github/workflows/nix-eval.yml +++ b/.github/workflows/nix-eval.yml @@ -32,4 +32,8 @@ jobs: run: | set -Eeu -o pipefail result=$(nix run --accept-flake-config .\#github-matrix -- checks legacyPackages) + if [ -z "$result" ]; then + echo "Error: github-matrix returned empty output" >&2 + exit 1 + fi echo matrix="$result" >> "$GITHUB_OUTPUT" diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 24c97ab74..ba3c35410 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -102,7 +102,9 @@ def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[s return nix_eval_cmd -def parse_nix_eval_line(line: str, drv_paths: Set[str]) -> Optional[NixEvalJobsOutput]: +def parse_nix_eval_line( + line: str, drv_paths: Set[str], errors: List[str] +) -> Optional[NixEvalJobsOutput]: """Parse a single line of nix-eval-jobs output""" if not line.strip(): return None @@ -110,38 +112,44 @@ def parse_nix_eval_line(line: str, drv_paths: Set[str]) -> Optional[NixEvalJobsO try: data: NixEvalJobsOutput = json.loads(line) if "error" in data: - raise ValueError( + error_msg = ( f"Error in nix-eval-jobs output for {data['attr']}: {data['error']}" ) + errors.append(error_msg) + return None if data["drvPath"] in drv_paths: return None drv_paths.add(data["drvPath"]) return data except json.JSONDecodeError: - print(f"Skipping invalid JSON line: {line}", file=sys.stderr) + error_msg = f"Skipping invalid JSON line: {line}" + print(error_msg, file=sys.stderr) + errors.append(error_msg) return None -def run_nix_eval_jobs(cmd: List[str]) -> Generator[NixEvalJobsOutput, None, None]: +def run_nix_eval_jobs( + cmd: List[str], errors: List[str] +) -> Generator[NixEvalJobsOutput, None, None]: """Run nix-eval-jobs and yield parsed package data.""" print(f"Running command: {' '.join(cmd)}", file=sys.stderr) with subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + cmd, stdout=subprocess.PIPE, stderr=None, text=True ) as process: drv_paths: Set[str] = set() assert process.stdout is not None # for mypy - assert process.stderr is not None # for mypy for line in process.stdout: - package = parse_nix_eval_line(line, drv_paths) + package = parse_nix_eval_line(line, drv_paths, errors) if package: yield package process.wait() if process.returncode != 0: - print("Error: Evaluation failed", file=sys.stderr) - sys.stderr.write(process.stderr.read()) - sys.exit(process.returncode) + error_msg = "Error: nix-eval-jobs process failed with non-zero exit code" + print(error_msg, file=sys.stderr) + errors.append(error_msg) + # Don't exit here - let main() handle it after reporting all errors def is_extension_pkg(pkg: NixEvalJobsOutput) -> bool: @@ -227,7 +235,9 @@ def main() -> None: cmd = build_nix_eval_command(max_workers, args.flake_outputs) - gh_action_packages = sort_pkgs_by_closures(list(run_nix_eval_jobs(cmd))) + # Collect all evaluation errors + errors: List[str] = [] + gh_action_packages = sort_pkgs_by_closures(list(run_nix_eval_jobs(cmd, errors))) def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: """Convert nix-eval-jobs output to GitHub Actions matrix package""" @@ -277,6 +287,18 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: ) print(json.dumps(gh_output)) + # Check if any errors occurred during evaluation + if errors: + print("\n=== Evaluation Errors ===", file=sys.stderr) + for i, error in enumerate(errors, 1): + print(f"\nError {i}:", file=sys.stderr) + print(error, file=sys.stderr) + print( + f"\n=== Total: {len(errors)} error(s) occurred during evaluation ===", + file=sys.stderr, + ) + sys.exit(1) + if __name__ == "__main__": main() From 6b979ac0f3c60ea2e385d22a6236058e2dc9cc35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 25 Nov 2025 21:47:43 +0100 Subject: [PATCH 53/58] feat(github-matrix): integrate github-action-utils for better error visibility Integrates github-action-utils library to improve error and warning visibility in GitHub Actions UI through workflow command annotations. --- .github/workflows/nix-eval.yml | 7 +- nix/packages/github-matrix/default.nix | 24 ++++ nix/packages/github-matrix/github_matrix.py | 137 ++++++++++++-------- 3 files changed, 107 insertions(+), 61 deletions(-) diff --git a/.github/workflows/nix-eval.yml b/.github/workflows/nix-eval.yml index adf74eaa1..cc092b355 100644 --- a/.github/workflows/nix-eval.yml +++ b/.github/workflows/nix-eval.yml @@ -31,9 +31,4 @@ jobs: name: Generate Nix Matrix run: | set -Eeu -o pipefail - result=$(nix run --accept-flake-config .\#github-matrix -- checks legacyPackages) - if [ -z "$result" ]; then - echo "Error: github-matrix returned empty output" >&2 - exit 1 - fi - echo matrix="$result" >> "$GITHUB_OUTPUT" + nix run --accept-flake-config .\#github-matrix -- checks legacyPackages diff --git a/nix/packages/github-matrix/default.nix b/nix/packages/github-matrix/default.nix index 0d755c977..3e1be2df3 100644 --- a/nix/packages/github-matrix/default.nix +++ b/nix/packages/github-matrix/default.nix @@ -5,6 +5,25 @@ }: let pname = "github-matrix"; + + github-action-utils = python3Packages.buildPythonPackage rec { + pname = "github-action-utils"; + version = "1.1.0"; + pyproject = true; + + src = python3Packages.fetchPypi { + inherit pname version; + sha256 = "0q9xrb4jcvbn6954lvpn85gva1yc885ykdqb2q2410cxp280v94a"; + }; + + build-system = with python3Packages; [ setuptools ]; + + meta = with lib; { + description = "Collection of Python functions for GitHub Action Workflow Commands"; + homepage = "https://github.com/saadmk11/github-action-utils"; + license = licenses.mit; + }; + }; in python3Packages.buildPythonApplication { @@ -14,6 +33,11 @@ python3Packages.buildPythonApplication { src = ./.; + propagatedBuildInputs = [ + github-action-utils + python3Packages.result + ]; + makeWrapperArgs = [ "--suffix PATH : ${lib.makeBinPath [ nix-eval-jobs ]}" ]; nativeCheckInputs = with python3Packages; [ diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index ba3c35410..deac5bc1d 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import argparse -from collections import defaultdict +from collections import Counter, defaultdict import graphlib import json import os @@ -10,16 +10,19 @@ from typing import ( Any, Dict, - Generator, List, Literal, NotRequired, Optional, Set, + Tuple, TypedDict, get_args, ) +from github_action_utils import debug, error, set_output, warning +from result import Err, Ok, Result + System = Literal["x86_64-linux", "aarch64-linux", "aarch64-darwin"] RunnerType = Literal["ephemeral", "self-hosted"] @@ -103,11 +106,16 @@ def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[s def parse_nix_eval_line( - line: str, drv_paths: Set[str], errors: List[str] -) -> Optional[NixEvalJobsOutput]: - """Parse a single line of nix-eval-jobs output""" + line: str, drv_paths: Set[str] +) -> Result[Optional[NixEvalJobsOutput], str]: + """Parse a single line of nix-eval-jobs output. + + Returns: + Ok(package_data) if successful (None for empty/duplicate lines) + Err(error_message) if a nix evaluation error occurred + """ if not line.strip(): - return None + return Ok(None) try: data: NixEvalJobsOutput = json.loads(line) @@ -115,41 +123,58 @@ def parse_nix_eval_line( error_msg = ( f"Error in nix-eval-jobs output for {data['attr']}: {data['error']}" ) - errors.append(error_msg) - return None + error(error_msg, title="Nix Evaluation Error") + return Err(error_msg) if data["drvPath"] in drv_paths: - return None + return Ok(None) drv_paths.add(data["drvPath"]) - return data - except json.JSONDecodeError: - error_msg = f"Skipping invalid JSON line: {line}" - print(error_msg, file=sys.stderr) - errors.append(error_msg) - return None + return Ok(data) + except json.JSONDecodeError as e: + warning(f"Skipping invalid JSON line: {line}", title="JSON Parse Warning") + return Ok(None) def run_nix_eval_jobs( - cmd: List[str], errors: List[str] -) -> Generator[NixEvalJobsOutput, None, None]: - """Run nix-eval-jobs and yield parsed package data.""" - print(f"Running command: {' '.join(cmd)}", file=sys.stderr) - - with subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=None, text=True - ) as process: - drv_paths: Set[str] = set() - assert process.stdout is not None # for mypy - for line in process.stdout: - package = parse_nix_eval_line(line, drv_paths, errors) - if package: - yield package - - process.wait() - if process.returncode != 0: - error_msg = "Error: nix-eval-jobs process failed with non-zero exit code" - print(error_msg, file=sys.stderr) - errors.append(error_msg) - # Don't exit here - let main() handle it after reporting all errors + cmd: List[str], +) -> Tuple[List[NixEvalJobsOutput], List[str], bool]: + """Run nix-eval-jobs and return parsed package data, warnings, and error status. + + Returns: + Tuple of (packages, warnings_list, had_errors) + """ + debug(f"Running command: {' '.join(cmd)}") + + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + stdout_data, stderr_data = process.communicate() + + # Parse stdout for packages + packages: List[NixEvalJobsOutput] = [] + drv_paths: Set[str] = set() + had_errors = False + for line in stdout_data.splitlines(): + result = parse_nix_eval_line(line, drv_paths) + if result.is_err(): + had_errors = True + elif result.ok_value is not None: + packages.append(result.ok_value) + + # Parse stderr for warnings (lines starting with "warning:") + warnings_list: List[str] = [] + for line in stderr_data.splitlines(): + line = line.strip() + if line.startswith("warning:") or line.startswith("evaluation warning:"): + # Remove "warning:" prefix for cleaner messages + warnings_list.append(line[8:].strip()) + + if process.returncode != 0: + error( + "nix-eval-jobs process failed with non-zero exit code", + title="Process Failure", + ) + + return packages, warnings_list, had_errors def is_extension_pkg(pkg: NixEvalJobsOutput) -> bool: @@ -235,9 +260,9 @@ def main() -> None: cmd = build_nix_eval_command(max_workers, args.flake_outputs) - # Collect all evaluation errors - errors: List[str] = [] - gh_action_packages = sort_pkgs_by_closures(list(run_nix_eval_jobs(cmd, errors))) + # Run evaluation and collect packages and warnings + packages, warnings_list, had_errors = run_nix_eval_jobs(cmd) + gh_action_packages = sort_pkgs_by_closures(packages) def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: """Convert nix-eval-jobs output to GitHub Actions matrix package""" @@ -281,22 +306,24 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: } ] } - print( - f"debug: Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}", - file=sys.stderr, - ) - print(json.dumps(gh_output)) - - # Check if any errors occurred during evaluation - if errors: - print("\n=== Evaluation Errors ===", file=sys.stderr) - for i, error in enumerate(errors, 1): - print(f"\nError {i}:", file=sys.stderr) - print(error, file=sys.stderr) - print( - f"\n=== Total: {len(errors)} error(s) occurred during evaluation ===", - file=sys.stderr, - ) + + if warnings_list: + warning_counts = Counter(warnings_list) + for warn_msg, count in warning_counts.items(): + if count > 1: + warning( + f"{warn_msg} (occurred {count} times)", + title="Nix Evaluation Warning", + ) + else: + warning(warn_msg, title="Nix Evaluation Warning") + + # Output matrix to GitHub Actions + debug(f"Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}") + set_output("matrix", json.dumps(gh_output)) + + # Exit with error code if any evaluation errors occurred + if had_errors: sys.exit(1) From 88d24049853bb059d447727201e7bf6b135e3eed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 25 Nov 2025 22:50:05 +0100 Subject: [PATCH 54/58] feat(github-matrix): group evaluation errors by message Refactor error handling to collect and group evaluation errors similar to warnings. Errors with the same message are now displayed together with a list of affected attributes. --- nix/packages/github-matrix/github_matrix.py | 66 ++++++++++++++------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index deac5bc1d..478935769 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -20,7 +20,7 @@ get_args, ) -from github_action_utils import debug, error, set_output, warning +from github_action_utils import debug, notice, error, set_output, warning from result import Err, Ok, Result System = Literal["x86_64-linux", "aarch64-linux", "aarch64-darwin"] @@ -60,6 +60,13 @@ class GitHubActionPackage(TypedDict): postgresql_version: NotRequired[str] +class NixEvalError(TypedDict): + """Error information from nix evaluation.""" + + attr: str + error: str + + BUILD_RUNNER_MAP: Dict[RunnerType, Dict[System, RunsOnConfig]] = { "ephemeral": { "aarch64-linux": { @@ -107,12 +114,12 @@ def build_nix_eval_command(max_workers: int, flake_outputs: List[str]) -> List[s def parse_nix_eval_line( line: str, drv_paths: Set[str] -) -> Result[Optional[NixEvalJobsOutput], str]: +) -> Result[Optional[NixEvalJobsOutput], NixEvalError]: """Parse a single line of nix-eval-jobs output. Returns: Ok(package_data) if successful (None for empty/duplicate lines) - Err(error_message) if a nix evaluation error occurred + Err(NixEvalError) if a nix evaluation error occurred """ if not line.strip(): return Ok(None) @@ -120,11 +127,12 @@ def parse_nix_eval_line( try: data: NixEvalJobsOutput = json.loads(line) if "error" in data: - error_msg = ( - f"Error in nix-eval-jobs output for {data['attr']}: {data['error']}" - ) - error(error_msg, title="Nix Evaluation Error") - return Err(error_msg) + error_msg = data["error"] + # Strip the redundant first line if it contains "does not have valid outputs" + error_lines = error_msg.split("\n") + if len(error_lines) > 1 and "does not have valid outputs" in error_lines[0]: + error_msg = "\n".join(error_lines[1:]).strip() + return Err({"attr": data["attr"], "error": error_msg}) if data["drvPath"] in drv_paths: return Ok(None) drv_paths.add(data["drvPath"]) @@ -136,11 +144,11 @@ def parse_nix_eval_line( def run_nix_eval_jobs( cmd: List[str], -) -> Tuple[List[NixEvalJobsOutput], List[str], bool]: - """Run nix-eval-jobs and return parsed package data, warnings, and error status. +) -> Tuple[List[NixEvalJobsOutput], List[str], List[NixEvalError]]: + """Run nix-eval-jobs and return parsed package data, warnings, and errors. Returns: - Tuple of (packages, warnings_list, had_errors) + Tuple of (packages, warnings_list, errors_list) """ debug(f"Running command: {' '.join(cmd)}") @@ -152,11 +160,11 @@ def run_nix_eval_jobs( # Parse stdout for packages packages: List[NixEvalJobsOutput] = [] drv_paths: Set[str] = set() - had_errors = False + errors_list: List[NixEvalError] = [] for line in stdout_data.splitlines(): result = parse_nix_eval_line(line, drv_paths) if result.is_err(): - had_errors = True + errors_list.append(result.err_value) elif result.ok_value is not None: packages.append(result.ok_value) @@ -174,7 +182,7 @@ def run_nix_eval_jobs( title="Process Failure", ) - return packages, warnings_list, had_errors + return packages, warnings_list, errors_list def is_extension_pkg(pkg: NixEvalJobsOutput) -> bool: @@ -260,8 +268,8 @@ def main() -> None: cmd = build_nix_eval_command(max_workers, args.flake_outputs) - # Run evaluation and collect packages and warnings - packages, warnings_list, had_errors = run_nix_eval_jobs(cmd) + # Run evaluation and collect packages, warnings, and errors + packages, warnings_list, errors_list = run_nix_eval_jobs(cmd) gh_action_packages = sort_pkgs_by_closures(packages) def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: @@ -318,13 +326,31 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: else: warning(warn_msg, title="Nix Evaluation Warning") - # Output matrix to GitHub Actions - debug(f"Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}") - set_output("matrix", json.dumps(gh_output)) + if errors_list: + # Group errors by error message + errors_by_message: Dict[str, List[str]] = defaultdict(list) + for err in errors_list: + errors_by_message[err["error"]].append(err["attr"]) + + for error_msg, attrs in errors_by_message.items(): + if len(attrs) > 1: + error( + f"{error_msg}\nAffected attributes: {', '.join(attrs)}", + title="Nix Evaluation Error", + ) + else: + error( + f"{error_msg}\nAttribute: {attrs[0]}", + title="Nix Evaluation Error", + ) # Exit with error code if any evaluation errors occurred - if had_errors: + if errors_list: sys.exit(1) + else: + # Output matrix to GitHub Actions + notice(f"Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}") + set_output("matrix", json.dumps(gh_output)) if __name__ == "__main__": From f9729d5185c328760a19a75aeabe7d02098cbf5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 25 Nov 2025 23:18:10 +0100 Subject: [PATCH 55/58] fix(github-matrix): improve multiline error display in GitHub Actions Extract core error messages and format them better for GitHub Actions annotations. --- nix/packages/github-matrix/github_matrix.py | 45 ++++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 478935769..d0b696ad0 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -128,10 +128,23 @@ def parse_nix_eval_line( data: NixEvalJobsOutput = json.loads(line) if "error" in data: error_msg = data["error"] - # Strip the redundant first line if it contains "does not have valid outputs" + + # Extract the core error message (last "error:" line and following context) error_lines = error_msg.split("\n") - if len(error_lines) > 1 and "does not have valid outputs" in error_lines[0]: - error_msg = "\n".join(error_lines[1:]).strip() + core_error_idx = -1 + for i in range(len(error_lines) - 1, -1, -1): + if error_lines[i].strip().startswith("error:"): + core_error_idx = i + break + + if core_error_idx >= 0: + # Take the last error line and up to 3 lines of context after it + error_msg = "\n".join( + error_lines[ + core_error_idx : min(core_error_idx + 4, len(error_lines)) + ] + ).strip() + return Err({"attr": data["attr"], "error": error_msg}) if data["drvPath"] in drv_paths: return Ok(None) @@ -152,8 +165,12 @@ def run_nix_eval_jobs( """ debug(f"Running command: {' '.join(cmd)}") + # Disable colors in nix output + env = os.environ.copy() + env["NO_COLOR"] = "1" + process = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env ) stdout_data, stderr_data = process.communicate() @@ -333,23 +350,21 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: errors_by_message[err["error"]].append(err["attr"]) for error_msg, attrs in errors_by_message.items(): + # Format message with attributes on first line, then error details if len(attrs) > 1: - error( - f"{error_msg}\nAffected attributes: {', '.join(attrs)}", - title="Nix Evaluation Error", - ) + formatted_msg = f"Affected attributes ({len(attrs)}): {', '.join(attrs)}\n\n{error_msg}" else: - error( - f"{error_msg}\nAttribute: {attrs[0]}", - title="Nix Evaluation Error", - ) + formatted_msg = f"Attribute: {attrs[0]}\n\n{error_msg}" + formatted_msg = formatted_msg.replace("\n", "%0A") + error(formatted_msg, title="Nix Evaluation Error") - # Exit with error code if any evaluation errors occurred if errors_list: sys.exit(1) else: - # Output matrix to GitHub Actions - notice(f"Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}") + formatted_msg = f"Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}".replace( + "\n", "%0A" + ) + notice(formatted_msg, title="GitHub Actions Matrix") set_output("matrix", json.dumps(gh_output)) From b0d60882fb3b7bea24fa654e9e0a5540b603f5d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 25 Nov 2025 23:20:07 +0100 Subject: [PATCH 56/58] fix(ci): skip run-testinfra and run-tests when nix-eval fails Add nix-eval to needs dependencies and check its result in conditional expressions to prevent downstream test jobs from running when evaluation fails. --- .github/workflows/nix-build.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 379488878..3a163bd6f 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -104,9 +104,10 @@ jobs: run: nix build --accept-flake-config -L .#${{ matrix.attr }} run-testinfra: - needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] + needs: [nix-eval, nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] if: | !cancelled() && + needs.nix-eval.result == 'success' && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') && (needs.nix-build-x86_64-linux.result == 'skipped' || needs.nix-build-x86_64-linux.result == 'success') @@ -115,9 +116,10 @@ jobs: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} run-tests: - needs: [nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] + needs: [nix-eval, nix-build-aarch64-linux, nix-build-aarch64-darwin, nix-build-x86_64-linux] if: | - !cancelled() && + !cancelled() && + needs.nix-eval.result == 'success' && (needs.nix-build-aarch64-linux.result == 'skipped' || needs.nix-build-aarch64-linux.result == 'success') && (needs.nix-build-aarch64-darwin.result == 'skipped' || needs.nix-build-aarch64-darwin.result == 'success') && (needs.nix-build-x86_64-linux.result == 'skipped' || needs.nix-build-x86_64-linux.result == 'success') From 7992f435a4cdb9c6473137b0cd46120a5aa4b68e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 25 Nov 2025 23:37:54 +0100 Subject: [PATCH 57/58] chore(github-matrix): update message when there are no build for a system --- nix/packages/github-matrix/github_matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index d0b696ad0..68fdc0411 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -325,7 +325,7 @@ def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: "include": [ { "attr": "", - "name": "skipped", + "name": "no packages to build", "system": system, "runs_on": {"labels": "ubuntu-latest"}, } From a5fc47295143ac324797619f6b47864b4a85733f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Wed, 26 Nov 2025 00:22:56 +0100 Subject: [PATCH 58/58] fix(github-matrix): backward compatibility for Result access We are running an older version of the 'result' library that uses '_value' instead of 'ok_value' to access the successful result of a computation. --- nix/packages/github-matrix/github_matrix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py index 68fdc0411..7743753e5 100755 --- a/nix/packages/github-matrix/github_matrix.py +++ b/nix/packages/github-matrix/github_matrix.py @@ -182,8 +182,8 @@ def run_nix_eval_jobs( result = parse_nix_eval_line(line, drv_paths) if result.is_err(): errors_list.append(result.err_value) - elif result.ok_value is not None: - packages.append(result.ok_value) + elif result._value is not None: + packages.append(result._value) # Parse stderr for warnings (lines starting with "warning:") warnings_list: List[str] = []