chore: merge develop into douglashunley/indata-520-create-a-supabase-nix-pkg-for-pgbackrest

* origin/develop: (26 commits)
  chore(nix): remove LLVM JIT support from postgresql build (#2296)
  fix: cover pg_cron legacy extversion strings with alignment migration and alias scripts (#2300)
  docs: commit supabase/postgres's CLAUDE.md (#2395)
  ansible: Setup swap in stage2 not stage1 (#2381)
  ci: install AMI postgres profile via nix-env --set (#2309)
  images: clean up nix at end of process (#2393)
  ansible: Stop compiling binaries (#2382)
  Report final AMI disk usage to workflow job (#2384)
  Re-organize build scripts, add to shellcheck (#2380)
  feat: Do not hardcode `orioledb.main_buffers` in `postgresql.conf` (#2365)
  feat: add orioledb debug symbols (#2379)
  chore: Make postgresql.env optional in database-optimizations.service (#2372)
  feat: bump to PostgREST v14.17 (#2374)
  ansible: Bump versions to get new AMIs built (#2376)
  Revert PostgREST bump to v14.17 (#2371)
  fix: disable APT timers during Supabase upgrades (#2358)
  gha/nix-install-ephemeral: Actually copy to s3, not just echo it (#2369)
  feat: bump to PostgREST v14.17 (#2363)
  feat: add guest-to-host time synchronisation (#2364)
  ci: Add final job that depends on the important jobs (#2361)
  ...

Author: hunleyd

.github/actions/build-ami/action.yml

@@ -37,6 +37,9 @@ outputs:
   execution_id:
     description: 'The execution ID for this build'
     value: ${{ steps.set-execution-id.outputs.execution_id }}
+  disk_usage_json:
+    description: 'The stage 2 AMI root disk usage as json'
+    value: ${{ steps.build-stage2.outputs.disk_usage_json }}
 
 runs:
   using: "composite"

.github/actions/nix-install-ephemeral/setup-nix.sh

@@ -12,7 +12,7 @@ if systemctl whoami &>/dev/null || [[ $(uname) == Darwin ]]; then
 	tmpdir=$(mktemp -d)
 	trap 'cd; rm -rf $tmpdir' EXIT
 
-	function maybesudo { sudo -E env "$@"; }
+	function maybesudo { sudo -EH env "$@"; }
 else
 	# systemd is *not* running so we must install in single-user mode
 	daemon=--no-daemon

.github/actions/nix-install-ephemeral/setup-push.sh

@@ -28,7 +28,7 @@ cat >"$NIXCONFDIR/upload-to-cache.sh" <<-EOF
 	fi
 
 	export IFS=' '
-	echo $NIXBINDIR/nix copy --max-jobs 5 --to 's3://nix-postgres-artifacts?secret-key=$NIXCONFDIR/nix-secret-key' \$OUT_PATHS
+	$NIXBINDIR/nix copy --max-jobs 5 --to 's3://nix-postgres-artifacts?secret-key=$NIXCONFDIR/nix-secret-key' \$OUT_PATHS
 EOF
 chmod 755 "$NIXCONFDIR/upload-to-cache.sh"
 

.github/workflows/check-shellscripts.yml

@@ -20,10 +20,31 @@ concurrency:
 
 jobs:
   nix-eval:
-    uses: ./.github/workflows/nix-eval.yml
-    secrets:
-      DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
-      NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
+    runs-on: blacksmith-32vcpu-ubuntu-2404
+    outputs:
+      packages_matrix: ${{ steps.gen-matrix.outputs.packages_matrix }}
+      checks_matrix: ${{ steps.gen-matrix.outputs.checks_matrix }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+
+      - name: Install nix
+        uses: ./.github/actions/nix-install-ephemeral
+        with:
+          push-to-cache: true
+          nix-signing-key: ${{ secrets.NIX_SIGN_SECRET_KEY }}
+          role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
+
+      - name: Generate Nix Matrix
+        id: gen-matrix
+        run: |
+          set -Eeu -o pipefail
+          nix run --accept-flake-config .\#github-matrix -- checks legacyPackages
+
+          sudo systemctl stop nix-daemon.socket || true
+          sudo systemctl stop nix-daemon.service || true
+          sudo pkill -9 nix-daemon || true
+          sleep 2
 
   nix-build-packages-aarch64-linux:
     name: >-
@@ -202,6 +223,20 @@ jobs:
         with:
           attr: .#${{ matrix.attr }}
 
+  nix-build-checks:
+    name: Nix Checks ✅
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    needs:
+      - nix-build-checks-aarch64-darwin
+      - nix-build-checks-aarch64-linux
+      - nix-build-checks-x86_64-linux
+    steps:
+      - name: Check dependent job results
+        if: contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') || contains(needs.*.result, 'skipped')
+        run: |
+          jq -rn --argjson needs '${{ toJSON(needs) }}' '$needs|to_entries[]|select(.value.result!="success")|"::error::"+.key+": "+.value.result'
+          exit 1
+
   run-testinfra:
     needs:
       - nix-eval
@@ -227,3 +262,19 @@ jobs:
     secrets:
       DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
       NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
+
+  nix-ci-check:
+    name: Nix CI ✅
+    if: always()
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    needs:
+      - docker-image-test
+      - nix-build-checks
+      - run-testinfra
+      - run-tests
+    steps:
+      - name: Check dependent job results
+        if: contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') || contains(needs.*.result, 'skipped')
+        run: |
+          jq -rn --argjson needs '${{ toJSON(needs) }}' '$needs|to_entries[]|select(.value.result!="success")|"::error::"+.key+": "+.value.result'
+          exit 1

.github/workflows/nix-build.yml

@@ -8,6 +8,7 @@ on:
     paths:
       - '.github/workflows/publish-nix-pgupgrade-scripts.yml'
       - 'ansible/vars.yml'
+      - 'ansible/files/admin_api_scripts/pg_upgrade_scripts/**'
   workflow_dispatch:
     inputs:
       postgresVersion:

.github/workflows/nix-eval.yml

@@ -19,7 +19,7 @@ env:
   AWS_REGION: ap-southeast-1
 
 jobs:
-  prepare:
+  gen-matrix:
     runs-on: blacksmith-2vcpu-ubuntu-2404
     outputs:
       postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }}
@@ -37,11 +37,11 @@ jobs:
           echo "postgres_versions=$VERSIONS" >> "$GITHUB_OUTPUT"
 
   test-ami-nix:
-    needs: prepare
+    needs: gen-matrix
     strategy:
       fail-fast: false
       matrix:
-        postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
+        postgres_version: ${{ fromJson(needs.gen-matrix.outputs.postgres_versions) }}
         target:
           - arch: amd64
             instance_type: c6i.4xlarge
@@ -100,20 +100,35 @@ jobs:
           postgres_version: ${{ matrix.postgres_version }}
           region: ${{ env.AWS_REGION }}
 
-      - run: docker context create builders
-
-      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
+      - name: Save AMI disk usage
+        env:
+          ARCH: ${{ matrix.target.arch }}
+          DISK_USAGE_JSON: ${{ steps.build-ami.outputs.disk_usage_json }}
+          POSTGRES_VERSION: ${{ matrix.postgres_version }}
+        run: >-
+          jq -cnr
+          --arg arch "$ARCH"
+          --arg version "$POSTGRES_VERSION"
+          --argjson disk_usage "$DISK_USAGE_JSON"
+          '{$version,$arch,bytes:$disk_usage.bytes,human:$disk_usage.human}'
+          >"ami-disk-usage-$POSTGRES_VERSION-$ARCH.json"
+
+      - name: Upload AMI disk usage
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
-          endpoint: builders
+          name: ami-disk-usage-${{ matrix.postgres_version }}-${{ matrix.target.arch }}
+          path: ami-disk-usage-*.json
+          overwrite: true
+          retention-days: 1
 
       - name: Run tests
         timeout-minutes: 10
         env:
           AMI_ID: ${{ steps.build-ami.outputs.stage2_ami_id }}
           EXECUTION_ID: ${{ steps.build-ami.outputs.execution_id }}
         run: |
-          # TODO: use poetry for pkg mgmt
-          pip3 install boto3 "boto3-stubs[essential]" docker ec2instanceconnectcli pytest "pytest-testinfra[paramiko,docker]" requests
+          # TODO: use uv for pkg mgmt
+          pip3 install boto3 'boto3-stubs[essential]' ec2instanceconnectcli pytest 'pytest-testinfra[paramiko]' requests
           pytest -vv -s testinfra/test_ami_nix.py
 
       - name: Cleanup resources on build cancellation
@@ -173,3 +188,26 @@ jobs:
           else
             echo "No stage 2 AMI to clean up"
           fi
+
+  report-disk-usage:
+    needs: test-ami-nix
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    steps:
+      - name: Download AMI disk usage
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          pattern: ami-disk-usage-*
+          path: ami-disk-usage
+          merge-multiple: true
+
+      - name: Report AMI disk usage
+        run: |
+          cat <<EOF >>"$GITHUB_STEP_SUMMARY"
+          ### AMI Root Disk Usage
+
+          | Version | Arch | Human | Bytes |
+          | ------- | ---- | ----- | ----- |
+          EOF
+          cat ami-disk-usage/ami-disk-usage-*.json |
+          sort -V |
+          jq -rs '.[]|{version,arch,human,bytes}|"| \(join("|")) |"' >>"$GITHUB_STEP_SUMMARY"

.github/workflows/publish-nix-pgupgrade-scripts.yml

@@ -0,0 +1,114 @@
+# CLAUDE.md
+
+Guidance for Claude Code sessions working in this repo.
+
+## Repo identity — do not confuse with the upstream mirror
+
+This is **`supabase/postgres`**: Supabase's own build/packaging repo for its
+PostgreSQL distribution. It does **not** contain a fork of PostgreSQL's C
+source.
+
+## What this repo is
+
+A batteries-included PostgreSQL distribution: unmodified upstream PostgreSQL
+(15, 17, and an `orioledb-17` fork) plus a large curated set of pre-built
+extensions (see the extension tables in `README.md`). It produces three
+release artifacts from one source tree:
+
+1. **Docker images** (`Dockerfile-15`, `Dockerfile-17`, `Dockerfile-supabase`,
+   `Dockerfile-multigres`) — used by local dev / the Supabase CLI.
+2. **AWS AMIs** — the production Supabase-hosted Postgres image, built via
+   Nix + Packer + Ansible.
+3. **Nix packages** — `nix build .#psql_15.bin` etc., the underlying build
+   system both other artifacts consume.
+
+The project is mid-migration from a pure Dockerfile-based extension build
+(what `CONTRIBUTING.md` still documents) to a Nix-based one (what `README.md`
+and `nix/docs/` describe as current). When the two disagree, trust `nix/`
+and `nix/docs/` — `CONTRIBUTING.md`'s Dockerfile-stage workflow is legacy for
+extensions not yet ported.
+
+## Directory structure
+
+| Path | Purpose |
+|---|---|
+| `nix/` | Core build system (flake-parts modules). `nix/postgresql/` = PG version configs/patches; `nix/ext/` = one file per extension package; `nix/config.nix` = pinned PG versions/hashes per major (source of truth for "what version are we on"); `nix/tests/` = pg_regress + smoke + migration NixOS tests; `nix/docs/` = the real developer docs (start here, not README) |
+| `ansible/` | Config management for the production AMI. `ansible/playbook.yml` = main playbook (Postgres/PostgREST/pgbouncer/Auth); `ansible/vars.yml` = **source of truth for AMI version tracking** (`postgres_release`, Docker release matrix); `ansible/files/postgresql_config/postgresql.conf.j2` = the actual GUC defaults shipped to customers |
+| `migrations/db/` | SQL migrations and `init-scripts/` (what every new project's schema starts with — default-enabled extensions, roles) |
+| `migrations/tests/extensions/` | pgTAP tests, one per extension, run via `pg_prove` against a Nix-built Postgres as part of `nix flake check` (`nix/checks.nix`) — not against a Docker image |
+| `docker/`, `Dockerfile-*` | Container image definitions (`Dockerfile-supabase` is the version-parameterized base; `Dockerfile-multigres` layers `pgctld` + `pgbackrest` on top) |
+| `ebssurrogate/`, `*.pkr.hcl` | Packer/EBS-surrogate AMI build pipeline |
+| `testinfra/` | pytest suite (`test_ami_nix.py`) that runs against a live AMI/instance |
+| `audit-specs/` | CIS-benchmark-style compliance specs run against built images |
+| `docs/plans/` | Design docs for in-flight features (profiler, pgbackrest, etc.) |
+| `.claude/skills/pg-security-release-analysis/` | Skill for triaging upstream PG quarterly security releases into a CVE/impact catalog — see below |
+
+## Building and testing
+
+Exact commands verified against `nix/docs/build-postgres.md`,
+`nix/docs/development-workflow.md`, and `CONTRIBUTING.md`. Read `nix/docs/README.md`
+first — it's the doc index and points to the specific runbook you need
+(adding a package, updating an extension, testing migrations, new major PG
+version, etc.) rather than duplicating them here.
+
+```bash
+# Build a full PG install (binaries + all extensions) locally via Nix
+nix build .#psql_15.bin      # or psql_17, psql_orioledb-17
+
+# Full flake check (nix eval + pg_regress + migration tests)
+nix flake check -L
+nix build .#checks.aarch64-darwin.psql_17 -L   # one target/platform only
+
+# Docker images
+docker build -f Dockerfile-15 -t supabase-postgres:15 .
+docker build -f Dockerfile-supabase --build-arg PG_VERSION=17 -t supabase-postgres:17 .
+docker build -f Dockerfile-multigres --build-arg SUPABASE_IMAGE=supabase-postgres:17 -t multigres:17 .
+
+# Remote build/cache + AMI build + testinfra (needs aws-vault; see
+# nix/docs/development-workflow.md for the full loop)
+nix run .#trigger-nix-build
+aws-vault exec <profile> -- nix run .#build-test-ami 17
+nix run .#run-testinfra -- --aws-vault-profile <profile> --ami-name <ami-name>
+```
+
+Formatting/lint is `treefmt` (nixfmt + deadnix), wired as a git-hooks.nix
+pre-commit hook when using `nix develop`/direnv; also enforced in CI, so
+`nix flake check` catches format drift even without the hook installed.
+
+## Non-obvious conventions and gotchas
+
+- **Version pinning**: `nix/config.nix` pins the exact upstream PG version +
+  source hash per major (`supabase.supportedPostgresVersions`). `ansible/vars.yml`
+  (`postgres_release`) separately pins the *packaged* version string used for
+  Docker/AMI tags — the two numbers look similar but serve different layers
+  (Nix build input vs. release artifact tag); update both when bumping a PG
+  minor.
+- **Adding a new extension**: two documented paths depending on era —
+  `nix/docs/adding-new-package.md` (current, Nix-based: one file in `nix/ext/`)
+  vs. `CONTRIBUTING.md` (legacy, Dockerfile multi-stage build + `checkinstall`
+  into a `.deb`, then copied into the `extensions` stage). Check whether the
+  extension already has sibling packages under `nix/ext/` before following
+  the Dockerfile path.
+- **orioledb-17 is a separate PG fork**, not just another extension —
+  security/behavior analysis needs to check whether a finding applies to
+  vanilla 17, orioledb-17, or both.
+- **Multi-version extensions**: some extensions (e.g. `pg_repack`) ship
+  multiple versions side-by-side in the same install (`pg_repack-1.4.8`,
+  `pg_repack-1.5.2`, ...) with `switch_<ext>_version` wrapper scripts to pick
+  one at runtime — don't assume one binary name per extension.
+- **`shared_preload_libraries`** (every project's postmaster preloads these)
+  and **default-enabled extensions** (every project's initial schema) are two
+  different, smaller lists than "everything in `nix/ext/`" — see
+  `ansible/files/postgresql_config/postgresql.conf.j2` and
+  `migrations/db/init-scripts/00000000000000-initial-schema.sql` respectively
+  before assuming a change affects every customer project.
+- **`.claude/skills/pg-security-release-analysis/`**: run this when triaging a
+  new upstream PG quarterly security release. It has its own gotchas baked
+  in (e.g. `git log --grep` doesn't tell you what version a fix first landed
+  in — use `git tag --contains`; CVE severity in commit messages can diverge
+  from postgresql.org's canonical security page) and a Supabase-specific
+  "surface map" (which roles, extensions, and services — wal-g, PostgREST,
+  Realtime, pgbouncer — are affected by which class of upstream change).
+- Docs inside `nix/docs/` have previously gone stale relative to actual repo
+  structure (see commit `2678fc8c docs: fix stale docs`) — if a doc and the
+  actual `nix/` layout disagree, trust the code.