Skip to content

Commit 5a3ae34

Browse files
hunleydhunleyd
committed
chore: merge develop into douglashunley/indata-520-create-a-supabase-nix-pkg-for-pgbackrest
* origin/develop: (36 commits) Lots of bootstrap script clean ups (#2326) ci: Fix nix-build PUSH_TO_CACHE (#2345) ci/nix-install-ephemeral: Drop sticky disk config (#2346) chore: bump postgres_release to cut fresh AMIs (includes #2334) (#2349) fix(multigres): stop base config data_directory from overriding pooler data dir (#2344) fix(cron): remove unnecessary TRIGGER grant on cron.job_run_details from postgres (#2334) chore: enable extension version restriction (warn) and cut AMIs (#2315) feat(nix): add site-env packages for rolling instance updates (#2283) ci: resolve package install sha from a flake input override (#2327) fix(multigres): remove dangling wal-g include from postgresql.conf (#2338) fix: Allow test CI pipelines to use old Int CA access. For staging only (#2330) chore(nix): remove maintainers field from package definitions (#2280) chore: bump multigres to b713432 (#2323) ci: Use arm-native-runner for kvm builds on aarch64-linux (#2319) Whole lot of ansible clean up (#2272) fix(ansible): drop no-op zpool loop item from zswap task (#2322) ci: Fix dockerhub-release-matrix matrix generation (#2320) chore: bump pgctld (#2318) fix(ansible): skip zswap params the kernel does not expose (#2321) feat(docker): add Dockerfile-supabase base image and rewrite Dockerfile-multigres as layered image (#2160) ...
2 parents dc6bf9b + c1f34c6 commit 5a3ae34

126 files changed

Lines changed: 2951 additions & 3035 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/build-ami/action.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ inputs:
1414
git_sha:
1515
description: 'Git SHA for this build'
1616
required: true
17+
packages_git_sha:
18+
description: 'Git SHA to install packages from'
19+
required: true
1720
instance_type:
1821
description: 'EC2 instance type for the build'
1922
required: true
@@ -101,7 +104,7 @@ runs:
101104
nix run .#build-ami -- stage2 ${{ inputs.arch }} \
102105
-var "ami_name=${{ inputs.ami_name_prefix }}" \
103106
-var "git-head-version=${{ inputs.git_sha }}" \
104-
-var "git_sha=${{ inputs.git_sha }}" \
107+
-var "git_sha=${{ inputs.packages_git_sha }}" \
105108
-var "instance_type=${{ inputs.instance_type }}" \
106109
-var "packer-execution-id=$PACKER_EXECUTION_ID" \
107110
-var "postgres_major_version=${{ inputs.postgres_version }}" \

‎.github/actions/nix-install-ephemeral/action.yml‎

Lines changed: 36 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1,87 +1,52 @@
1-
name: 'Install Nix on ephemeral runners'
2-
description: 'Installs Nix and sets up AWS credentials to push to the Nix binary cache'
1+
name: Install Nix on ephemeral runners
2+
description: Installs Nix and sets up AWS credentials to push to the Nix binary cache
33
inputs:
4+
aws-region:
5+
description: AWS region for the Nix binary cache S3 bucket
6+
required: false
7+
default: us-east-1
8+
nix-signing-key:
9+
description: Key used to sign uploads to binary cache, required if push-to-cache is true
10+
required: false
411
push-to-cache:
5-
description: 'Whether to push build outputs to the Nix binary cache'
12+
description: Whether to push build outputs to the Nix binary cache
613
required: false
7-
default: 'false'
8-
aws-region:
9-
description: 'AWS region for the Nix binary cache S3 bucket'
14+
default: false
15+
role-to-assume:
16+
description: AWS Role to assume when configuring aws credentials, required if push-to-cache is true
1017
required: false
11-
default: 'us-east-1'
1218
runs:
13-
using: 'composite'
19+
using: composite
1420
steps:
15-
- name: aws-creds
21+
- name: validate push-to-cache required inputs
22+
if: inputs.push-to-cache == 'true'
23+
shell: bash
24+
run: |
25+
abort() { echo "$@" >&2; exit 1; }
26+
[[ -n ${NIX_SIGNING_KEY:+ok} ]] || abort "nix-signing-key is required when push-to-cache is true"
27+
[[ -n ${ROLE_TO_ASSUME:+ok} ]] || abort "role-to-assume is required when push-to-cache is true"
28+
env:
29+
NIX_SIGNING_KEY: ${{ inputs.nix-signing-key }}
30+
ROLE_TO_ASSUME: ${{ inputs.role-to-assume }}
31+
32+
- name: configure aws credentials
1633
uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
17-
if: ${{ inputs.push-to-cache == 'true' }}
34+
if: inputs.push-to-cache == 'true'
1835
with:
19-
role-to-assume: ${{ env.DEV_AWS_ROLE }}
36+
role-to-assume: ${{ inputs.role-to-assume }}
2037
aws-region: ${{ inputs.aws-region }}
2138
output-credentials: true
2239
role-duration-seconds: 7200
23-
- name: Setup AWS credentials for Nix
24-
if: ${{ inputs.push-to-cache == 'true' }}
25-
shell: bash
26-
run: |
27-
sudo -H aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
28-
sudo -H aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
29-
sudo -H aws configure set aws_session_token $AWS_SESSION_TOKEN
30-
sudo -H aws configure set region ${{ inputs.aws-region }}
31-
sudo mkdir -p /etc/nix
32-
sudo -E python -c "import os; file = open('/etc/nix/nix-secret-key', 'w'); file.write(os.environ['NIX_SIGN_SECRET_KEY']); file.close()"
33-
cat << 'EOF' | sudo tee /etc/nix/upload-to-cache.sh > /dev/null
34-
#!/usr/bin/env bash
35-
set -euo pipefail
36-
set -f
3740

38-
export IFS=' '
39-
/nix/var/nix/profiles/default/bin/nix copy --max-jobs 5 --to 's3://nix-postgres-artifacts?secret-key=/etc/nix/nix-secret-key' $OUT_PATHS
40-
EOF
41-
sudo chmod +x /etc/nix/upload-to-cache.sh
42-
env:
43-
NIX_SIGN_SECRET_KEY: ${{ env.NIX_SIGN_SECRET_KEY }}
44-
- name: Install Nix
45-
shell: bash
46-
run: |
47-
sudo tee /tmp/nix-extra.conf > /dev/null <<'NIXCONF'
48-
always-allow-substitutes = true
49-
extra-experimental-features = nix-command flakes
50-
extra-system-features = kvm
51-
max-jobs = 4
52-
substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
53-
trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
54-
NIXCONF
55-
56-
if [ "${{ inputs.push-to-cache }}" = "true" ]; then
57-
echo "post-build-hook = /etc/nix/upload-to-cache.sh" | sudo tee -a /tmp/nix-extra.conf > /dev/null
58-
fi
59-
60-
curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- --daemon --yes --nix-extra-conf-file /tmp/nix-extra.conf
61-
62-
cat /etc/nix/nix.conf
63-
64-
# Add nix to PATH for subsequent steps
65-
echo "/nix/var/nix/profiles/default/bin" >> "$GITHUB_PATH"
66-
# Source the daemon profile so nix works in this step too
67-
. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh
68-
- name: Configure Nix with GitHub token
41+
- name: setup nix
6942
shell: bash
43+
run: cd ${{ github.action_path }} && ./setup-nix.sh
7044
env:
71-
GH_TOKEN: ${{ github.token }}
72-
run: |
73-
echo "access-tokens = github.com=$GH_TOKEN" | sudo tee -a /etc/nix/nix.conf > /dev/null
74-
sudo systemctl restart nix-daemon || true
75-
- name: Print Nix version
45+
AWS_REGION: ${{ inputs.aws-region }}
46+
GITHUB_TOKEN: ${{ github.token }}
47+
NIX_SIGN_SECRET_KEY: ${{ inputs.nix-signing-key }}
48+
PUSH_TO_CACHE: ${{ inputs.push-to-cache == 'true' }}
49+
50+
- name: verify nix in path
7651
shell: bash
7752
run: nix --version
78-
- name: Setup KVM permissions
79-
shell: bash
80-
run: |
81-
if [ -e /dev/kvm ]; then
82-
sudo chown runner /dev/kvm
83-
sudo chmod 666 /dev/kvm
84-
echo "KVM configured: $(ls -l /dev/kvm)"
85-
else
86-
echo "KVM device not available"
87-
fi

‎.github/actions/nix-install-ephemeral/setup-github-access.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/usr/bin/env bash
2+
3+
umask 0277
4+
echo "access-tokens = github.com=${GITHUB_TOKEN:?}" >"$NIXCONFDIR/github.nix.conf"
5+
echo "!include $NIXCONFDIR/github.nix.conf"

‎.github/actions/nix-install-ephemeral/setup-nix.sh‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
if systemctl whoami &>/dev/null || [[ $(uname) == Darwin ]]; then
6+
# systemd is running so we can install in multi-user mode
7+
# or running on macos
8+
daemon=--daemon
9+
nixconfdir=/etc/nix
10+
path=/nix/var/nix/profiles/default/bin
11+
12+
tmpdir=$(mktemp -d)
13+
trap 'cd; rm -rf $tmpdir' EXIT
14+
15+
function maybesudo { sudo -E env "$@"; }
16+
else
17+
# systemd is *not* running so we must install in single-user mode
18+
daemon=--no-daemon
19+
nixconfdir=$HOME/.config/nix
20+
path=$HOME/.nix-profile/bin
21+
22+
tmpdir=$nixconfdir
23+
24+
function maybesudo { env "$@"; }
25+
fi
26+
27+
maybesudo mkdir -p "$nixconfdir"
28+
cat >"$tmpdir/nix.conf" <<-EOF
29+
always-allow-substitutes = true
30+
extra-experimental-features = flakes nix-command
31+
extra-substituters = https://nix-postgres-artifacts.s3.amazonaws.com
32+
extra-trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=
33+
max-jobs = 5
34+
EOF
35+
36+
if [[ -e /dev/kvm ]]; then
37+
sudo chown runner /dev/kvm
38+
sudo chmod 666 /dev/kvm
39+
echo 'extra-system-features = kvm' >>"$tmpdir/nix.conf"
40+
fi
41+
42+
if [[ ${PUSH_TO_CACHE:?} == true ]]; then
43+
set -x
44+
maybesudo NIXCONFDIR="$nixconfdir" NIXBINDIR="$path" ./setup-push.sh >>"$tmpdir/nix.conf"
45+
fi
46+
47+
maybesudo NIXCONFDIR="$nixconfdir" ./setup-github-access.sh >>"$tmpdir/nix.conf"
48+
49+
curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- $daemon --yes --nix-extra-conf-file "$tmpdir/nix.conf"
50+
cat "$nixconfdir/nix.conf"
51+
52+
# Add nix to PATH for subsequent steps
53+
echo "$path" >>"${GITHUB_PATH:?}"

‎.github/actions/nix-install-ephemeral/setup-push.sh‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
mkdir -p ~/.aws
6+
umask 0277
7+
cat >~/.aws/config <<-EOF
8+
[default]
9+
region = ${AWS_REGION:?}
10+
EOF
11+
cat >~/.aws/credentials <<-EOF
12+
[default]
13+
aws_access_key_id = ${AWS_ACCESS_KEY_ID:?}
14+
aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY:?}
15+
aws_session_token = ${AWS_SESSION_TOKEN:?}
16+
EOF
17+
18+
printenv NIX_SIGN_SECRET_KEY >"${NIXCONFDIR:?}/nix-secret-key"
19+
20+
cat >"$NIXCONFDIR/upload-to-cache.sh" <<-EOF
21+
#!/usr/bin/env bash
22+
set -euo pipefail
23+
set -f
24+
25+
if [[ ! -e ${NIXBINDIR:?}/nix ]]; then
26+
# called during nix install, but nix isn't available yet
27+
exit
28+
fi
29+
30+
export IFS=' '
31+
echo $NIXBINDIR/nix copy --max-jobs 5 --to 's3://nix-postgres-artifacts?secret-key=$NIXCONFDIR/nix-secret-key' \$OUT_PATHS
32+
EOF
33+
chmod 755 "$NIXCONFDIR/upload-to-cache.sh"
34+
35+
echo "post-build-hook = $NIXCONFDIR/upload-to-cache.sh"

‎.github/actions/resolve-git-sha/action.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: Resolve git sha
2+
description: Resolves the git sha to install packages from, honoring a flake input override
3+
4+
inputs:
5+
flake_input:
6+
description: 'Flake input name to resolve from flake.lock, if set'
7+
required: false
8+
default: ''
9+
10+
outputs:
11+
sha:
12+
description: 'Resolved git sha'
13+
value: ${{ steps.resolve.outputs.sha }}
14+
15+
runs:
16+
using: composite
17+
steps:
18+
- id: resolve
19+
shell: bash
20+
run: |
21+
SHA=$(jq -r --arg k "${{ inputs.flake_input }}" 'if $k == "" then empty else (.nodes[$k].locked.rev // empty) end' flake.lock 2>/dev/null || true)
22+
echo "sha=${SHA:-${{ github.sha }}}" >> "$GITHUB_OUTPUT"

‎.github/workflows/ami-release-nix.yml‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -81,11 +81,16 @@ jobs:
8181
- name: Install nix
8282
uses: ./.github/actions/nix-install-ephemeral
8383
with:
84-
push-to-cache: 'true'
84+
push-to-cache: true
8585
aws-region: ${{ env.AWS_REGION }}
86-
env:
87-
DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
88-
NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
86+
nix-signing-key: ${{ secrets.NIX_SIGN_SECRET_KEY }}
87+
role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
88+
89+
- name: Resolve git sha
90+
id: resolve-git-sha
91+
uses: ./.github/actions/resolve-git-sha
92+
with:
93+
flake_input: ${{ vars.GIT_SHA_FROM_FLAKE_INPUT }}
8994

9095
- name: Build AMI
9196
id: build-ami
@@ -95,6 +100,7 @@ jobs:
95100
ami_regions: '["${{ env.AWS_REGION }}"]'
96101
arch: ${{ matrix.target.arch }}
97102
git_sha: ${{ github.sha }}
103+
packages_git_sha: ${{ steps.resolve-git-sha.outputs.sha }}
98104
instance_type: ${{ matrix.target.instance_type }}
99105
postgres_version: ${{ matrix.postgres_version }}
100106
region: ${{ env.AWS_REGION }}
@@ -123,7 +129,7 @@ jobs:
123129
124130
- name: Create nix flake revision tarball
125131
run: |
126-
GIT_SHA=${{github.sha}}
132+
GIT_SHA=${{ steps.resolve-git-sha.outputs.sha }}
127133
128134
mkdir -p "/tmp/pg_upgrade_bin/${POSTGRES_MAJOR_VERSION}"
129135
echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${POSTGRES_MAJOR_VERSION}/nix_flake_version"
@@ -171,7 +177,7 @@ jobs:
171177

172178
- name: Update nix store path catalog
173179
run: |
174-
GIT_SHA="${{ github.sha }}"
180+
GIT_SHA="${{ steps.resolve-git-sha.outputs.sha }}"
175181
SYSTEM=$(nix eval --impure --raw --expr 'builtins.currentSystem')
176182
177183
# Get store path for this build
@@ -204,7 +210,7 @@ jobs:
204210
tag_name: ${{ env.RELEASE_TAG }}
205211
target_commitish: ${{ github.sha }}
206212

207-
- name: Create CLI tag for PG 17
213+
- name: Create CLI tag for PG 17 & Trigger Release
208214
if: matrix.postgres_version == '17' && matrix.target.arch == 'arm64' && github.event_name != 'workflow_dispatch'
209215
env:
210216
GH_TOKEN: ${{ github.token }}
@@ -214,12 +220,7 @@ jobs:
214220
git tag "${CLI_TAG}" "${{ github.sha }}"
215221
git push origin "${CLI_TAG}"
216222
217-
- name: Trigger CLI release workflow for PG 17
218-
if: matrix.postgres_version == '17' && github.event_name != 'workflow_dispatch'
219-
env:
220-
GH_TOKEN: ${{ github.token }}
221-
run: |
222-
CLI_TAG="v${POSTGRES_SUPABASE_VERSION}-cli"
223+
echo "Triggering Release Workflow: ${CLI_TAG}"
223224
gh workflow run cli-release.yml \
224225
--ref "${CLI_TAG}" \
225226
-f version="${CLI_TAG}"

‎.github/workflows/cli-smoke-test.yml‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,6 @@ jobs:
2727

2828
- name: Install nix
2929
uses: ./.github/actions/nix-install-ephemeral
30-
with:
31-
push-to-cache: 'false'
32-
env:
33-
DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
34-
NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
3530

3631
- name: Check Docker image changes
3732
id: check
@@ -56,11 +51,6 @@ jobs:
5651

5752
- name: Install nix
5853
uses: ./.github/actions/nix-install-ephemeral
59-
with:
60-
push-to-cache: 'false'
61-
env:
62-
DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
63-
NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
6454

6555
- name: Create Docker context
6656
run: docker context create builders

0 commit comments

Comments
 (0)