Skip to content

Commit 68ed0e7

Browse files
jchancojrjchancojr
authored
Merge branch 'develop' into INDATA-211
2 parents 264ab4d + 336f49e commit 68ed0e7

File tree

14 files changed

+585
-161
lines changed

14 files changed

+585
-161
lines changed

‎.github/workflows/publish-migrations-staging.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
name: Release Migrations - Staging
22

33
on:
4-
merge_group:
4+
push:
5+
branches:
6+
- develop
57
workflow_dispatch:
68

79
jobs:

‎ansible/files/postgresql_config/supautils.conf.j2‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ supautils.privileged_extensions = 'address_standardizer, address_standardizer_da
1010
supautils.extension_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts'
1111
supautils.privileged_extensions_superuser = 'supabase_admin'
1212
supautils.privileged_role = 'postgres'
13-
supautils.privileged_role_allowed_configs = 'auto_explain.*, log_lock_waits, log_min_duration_statement, log_min_messages, log_replication_commands, log_statement, log_temp_files, pg_net.batch_size, pg_net.ttl, pg_stat_statements.*, pgaudit.log, pgaudit.log_catalog, pgaudit.log_client, pgaudit.log_level, pgaudit.log_relation, pgaudit.log_rows, pgaudit.log_statement, pgaudit.log_statement_once, pgaudit.role, pgrst.*, plan_filter.*, safeupdate.enabled, session_replication_role, track_io_timing, wal_compression'
13+
supautils.privileged_role_allowed_configs = 'auto_explain.*, log_lock_waits, log_min_duration_statement, log_min_messages, log_parameter_max_length, log_replication_commands, log_statement, log_temp_files, pg_net.batch_size, pg_net.ttl, pg_stat_statements.*, pgaudit.log, pgaudit.log_catalog, pgaudit.log_client, pgaudit.log_level, pgaudit.log_relation, pgaudit.log_rows, pgaudit.log_statement, pgaudit.log_statement_once, pgaudit.role, pgrst.*, plan_filter.*, safeupdate.enabled, session_replication_role, track_io_timing, wal_compression'
1414
supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program, supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_realtime_admin, supabase_replication_admin, supabase_etl_admin, dashboard_user, pgbouncer, authenticator'
1515
supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_realtime_admin, supabase_replication_admin, supabase_etl_admin, dashboard_user, pgbouncer, service_role*, authenticator*, authenticated*, anon*'
1616
supautils.disable_program = 'true'

‎ansible/tasks/test-image.yml‎

Lines changed: 94 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -1,87 +1,104 @@
1-
# - name: Temporarily disable PG Sodium references in config
2-
# become: yes
3-
# become_user: postgres
4-
# shell:
5-
# cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf
6-
# when: debpkg_mode or stage2_nix
1+
- name: Execute tasks when (debpkg_mode or stage2_nix)
2+
when:
3+
- (debpkg_mode or stage2_nix)
4+
block:
5+
- name: Make a backup of the /etc/postgresql/postgresql.conf file
6+
ansible.builtin.copy:
7+
dest: '/etc/postgresql/postgresql.conf.bak'
8+
src: '/etc/postgresql/postgresql.conf'
9+
become: true
710

8-
- name: Temporarily disable PG Sodium and Supabase Vault references in config
9-
become: yes
10-
become_user: postgres
11-
shell:
12-
cmd: >
13-
sed -i.bak
14-
-e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/'
15-
-e 's/\(shared_preload_libraries = '\''.*\)supabase_vault,\(.*'\''\)/\1\2/'
16-
-e 's/\(shared_preload_libraries = '\''.*\), *supabase_vault'\''/\1'\''/'
17-
-e 's/pgsodium.getkey_script=/#pgsodium.getkey_script=/'
18-
/etc/postgresql/postgresql.conf
19-
when: debpkg_mode or stage2_nix
11+
- name: Temporarily disable PG Sodium and Supabase Vault references in /etc/postgresql/postgresql.conf
12+
ansible.builtin.replace:
13+
path: '/etc/postgresql/postgresql.conf'
14+
regexp: "{{ regx['in'] }}"
15+
replace: "{{ regx['out'] }}"
16+
become: true
17+
become_user: 'postgres'
18+
loop:
19+
- { in: "^(shared_preload_libraries = '.*)pgsodium(.*')", out: '\1\2' }
20+
- { in: "^(shared_preload_libraries = '.*)supabase_vault(.*')", out: '\1\2' }
21+
- { in: "^(shared_preload_libraries = '.*)*supabase_vault(.*')", out: '\1\2' }
22+
- { in: '^(pgsodium\.getkey_script=)', out: '#\1' }
23+
loop_control:
24+
loop_var: 'regx'
2025

21-
- name: Verify pgsodium and vault removal from config
22-
become: yes
23-
become_user: postgres
24-
shell:
25-
cmd: |
26-
FOUND=$(grep -E "shared_preload_libraries.*pgsodium|shared_preload_libraries.*supabase_vault|^pgsodium\.getkey_script" /etc/postgresql/postgresql.conf)
27-
if [ ! -z "$FOUND" ]; then
28-
echo "Found unremoved references:"
29-
echo "$FOUND"
30-
exit 1
31-
fi
32-
register: verify_result
33-
failed_when: verify_result.rc != 0
34-
when: debpkg_mode or stage2_nix
26+
- name: Make sure we disabled all the things
27+
ansible.builtin.lineinfile:
28+
path: '/etc/postgresql/postgresql.conf'
29+
regexp: "{{ regx }}"
30+
state: 'absent'
31+
check_mode: true
32+
failed_when:
33+
- (pgconf is changed) or (pgconf is failed)
34+
loop:
35+
- "^shared_preload_libraries = '.*pgsodium.*'"
36+
- "^shared_preload_libraries = '.*supabase_vault.*'"
37+
- '^pgsodium\.getkey_script='
38+
loop_control:
39+
loop_var: 'regx'
40+
register: 'pgconf'
3541

3642
- name: Start Postgres Database to load all extensions.
37-
become: yes
38-
become_user: postgres
39-
shell:
43+
ansible.builtin.command:
4044
cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start "-o -c config_file=/etc/postgresql/postgresql.conf"
41-
when: debpkg_mode
45+
become: true
46+
become_user: 'postgres'
47+
when:
48+
- debpkg_mode
4249

43-
- name: Stop Postgres Database in stage 2
44-
become: yes
45-
become_user: postgres
46-
shell: source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data stop
47-
args:
48-
executable: /bin/bash
49-
environment:
50-
LANG: en_US.UTF-8
51-
LANGUAGE: en_US.UTF-8
52-
LC_ALL: en_US.UTF-8
53-
LC_CTYPE: en_US.UTF-8
54-
LOCALE_ARCHIVE: /usr/lib/locale/locale-archive
55-
when: stage2_nix
50+
- name: Execute tasks when stage2_nix
51+
when:
52+
- stage2_nix
53+
block:
54+
- name: Restart Postgres Database in stage 2 to load all extensions
55+
ansible.builtin.command:
56+
cmd: "/usr/lib/postgresql/bin/pg_ctl --pgdata /var/lib/postgresql/data --mode fast --options '-c config_file=/etc/postgresql/postgresql.conf' {{ ctlcmd }}"
57+
become: true
58+
become_user: 'postgres'
59+
environment:
60+
LANG: 'en_US.UTF-8'
61+
LANGUAGE: 'en_US.UTF-8'
62+
LC_ALL: 'en_US.UTF-8'
63+
LC_CTYPE: 'en_US.UTF-8'
64+
LOCALE_ARCHIVE: '/usr/lib/locale/locale-archive'
65+
loop:
66+
- stop
67+
- start
68+
loop_control:
69+
loop_var: 'ctlcmd'
5670

57-
- name: Start Postgres Database to load all extensions.
58-
become: yes
59-
become_user: postgres
60-
shell: source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start "-o -c config_file=/etc/postgresql/postgresql.conf"
61-
args:
62-
executable: /bin/bash
63-
environment:
64-
LANG: en_US.UTF-8
65-
LANGUAGE: en_US.UTF-8
66-
LC_ALL: en_US.UTF-8
67-
LC_CTYPE: en_US.UTF-8
68-
LOCALE_ARCHIVE: /usr/lib/locale/locale-archive
69-
when: stage2_nix
71+
- name: Execute tasks when (debpkg_mode or stage2_nix)
72+
when:
73+
- (debpkg_mode or stage2_nix)
74+
block:
75+
- name: Re-enable PG Sodium references in /etc/postgresql/postgresql.conf
76+
ansible.builtin.command:
77+
cmd: mv /etc/postgresql/postgresql.conf.bak /etc/postgresql/postgresql.conf
78+
become: true
79+
become_user: 'postgres'
7080

71-
- name: Re-enable PG Sodium references in config
72-
become: yes
73-
become_user: postgres
74-
shell:
75-
cmd: mv /etc/postgresql/postgresql.conf.bak /etc/postgresql/postgresql.conf
76-
when: debpkg_mode or stage2_nix
81+
- name: Install psycopg2
82+
ansible.builtin.apt:
83+
name: 'python3-psycopg2'
84+
state: 'present'
85+
update_cache: true
86+
become: true
7787

78-
- name: Reset db stats
79-
shell: /usr/lib/postgresql/bin/psql --no-password --no-psqlrc -d postgres -h localhost -U supabase_admin -c 'SELECT pg_stat_statements_reset(); SELECT pg_stat_reset();'
80-
when: debpkg_mode or stage2_nix
88+
- name: Reset db stats
89+
community.postgresql.postgresql_query:
90+
login_db: 'postgres'
91+
login_host: 'localhost'
92+
login_user: 'supabase_admin'
93+
query: "{{ stat_item }}"
94+
loop:
95+
# - 'SELECT pg_stat_statements_reset()'
96+
- 'SELECT pg_stat_reset()'
97+
loop_control:
98+
loop_var: 'stat_item'
8199

82-
- name: Stop Postgres Database
83-
become: yes
84-
become_user: postgres
85-
shell:
86-
cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data stop
87-
when: debpkg_mode or stage2_nix
100+
- name: Restart Postgres Database in stage 2 to load all extensions
101+
ansible.builtin.command:
102+
cmd: /usr/lib/postgresql/bin/pg_ctl --pgdata /var/lib/postgresql/data --mode fast --options '-c config_file=/etc/postgresql/postgresql.conf' stop
103+
become: true
104+
become_user: 'postgres'

‎ansible/vars.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ postgres_major:
1010

1111
# Full version strings for each major version
1212
postgres_release:
13-
postgresorioledb-17: "17.5.1.059-orioledb"
14-
postgres17: "17.6.1.038"
15-
postgres15: "15.14.1.038"
13+
postgresorioledb-17: "17.5.1.064-orioledb"
14+
postgres17: "17.6.1.043"
15+
postgres15: "15.14.1.043"
1616

1717
# Non Postgres Extensions
1818
pgbouncer_release: 1.19.0
@@ -25,8 +25,8 @@ postgrest_release: 13.0.5
2525
postgrest_arm_release_checksum: sha256:7b4eafdaf76bc43b57f603109d460a838f89f949adccd02f452ca339f9a0a0d4
2626
postgrest_x86_release_checksum: sha256:05be2bd48abee6c1691fc7c5d005023466c6989e41a4fc7d1302b8212adb88b5
2727

28-
gotrue_release: 2.180.0
29-
gotrue_release_checksum: sha1:386c1fb6be075004091b2fbd8662dc9dcdc7af04
28+
gotrue_release: 2.182.1
29+
gotrue_release_checksum: sha1:38a12109ad62df32460d88e4c7b2a475b88e7865
3030

3131
aws_cli_release: 2.23.11
3232

@@ -53,7 +53,7 @@ postgres_exporter_release_checksum:
5353
arm64: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0
5454
amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84
5555

56-
adminapi_release: "0.92.2"
56+
adminapi_release: "0.93.0"
5757
adminmgr_release: "0.32.1"
5858
supabase_admin_agent_release: 1.4.38
5959
supabase_admin_agent_splay: 30s

‎migrations/README.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,12 @@ dbmate --migrations-dir="migrations/db/migrations" new '<some message>'
102102
Then, execute the migration at `./migrations/db/xxxxxxxxx_<some_message>` and make sure it runs successfully with:
103103

104104
```shell
105-
dbmate --no-dump-schema --migrations-dir"migrations/db/migrations" up
105+
# Make sure DATABASE_URL is set, or use the -u flag to specify the database connection
106+
# Example with DATABASE_URL:
107+
dbmate --no-dump-schema --migrations-dir="migrations/db/migrations" up
108+
109+
# Or with -u flag:
110+
dbmate --no-dump-schema --migrations-dir="migrations/db/migrations" -u "postgres://supabase_admin:postgres@localhost:5435/postgres?sslmode=disable" up
106111
```
107112

108113
Note: Migrations are applied using the `supabase_admin` superuser role, as specified in the "How it was Created" section above.

‎migrations/db/migrations/20251105172723_grant_pg_reload_conf_to_postgres.sql‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
-- migrate:up
2+
grant execute on function pg_catalog.pg_reload_conf() to postgres with grant option;
3+
4+
-- migrate:down
5+

‎nix/ext/pg_tle.nix‎

Lines changed: 92 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,45 +1,111 @@
11
{
22
lib,
33
stdenv,
4+
buildEnv,
45
fetchFromGitHub,
56
postgresql,
67
flex,
78
openssl,
89
libkrb5,
910
}:
10-
11-
stdenv.mkDerivation rec {
11+
let
1212
pname = "pg_tle";
13-
version = "1.4.0";
13+
build =
14+
version: hash:
15+
stdenv.mkDerivation rec {
16+
inherit pname version;
1417

15-
nativeBuildInputs = [ flex ];
16-
buildInputs = [
17-
openssl
18-
postgresql
19-
libkrb5
20-
];
18+
nativeBuildInputs = [ flex ];
19+
buildInputs = [
20+
openssl
21+
postgresql
22+
libkrb5
23+
];
2124

22-
src = fetchFromGitHub {
23-
owner = "aws";
24-
repo = pname;
25-
rev = "refs/tags/v${version}";
26-
hash = "sha256-crxj5R9jblIv0h8lpqddAoYe2UqgUlnvbOajKTzVces=";
27-
};
25+
src = fetchFromGitHub {
26+
owner = "aws";
27+
repo = pname;
28+
rev = "refs/tags/v${version}";
29+
inherit hash;
30+
};
31+
32+
makeFlags = [ "FLEX=flex" ];
33+
34+
installPhase = ''
2835
29-
makeFlags = [ "FLEX=flex" ];
36+
mkdir -p $out/{lib,share/postgresql/extension}
3037
31-
installPhase = ''
32-
mkdir -p $out/{lib,share/postgresql/extension}
38+
mv ${pname}${postgresql.dlSuffix} $out/lib/${pname}-${version}${postgresql.dlSuffix}
3339
34-
cp *${postgresql.dlSuffix} $out/lib
35-
cp *.sql $out/share/postgresql/extension
36-
cp *.control $out/share/postgresql/extension
40+
create_sql_files() {
41+
if test -f ${pname}--${version}.sql; then
42+
cp ${pname}--${version}.sql $out/share/postgresql/extension
43+
fi
44+
echo "Creating SQL files for previous versions..."
45+
if [[ "${version}" == "${latestVersion}" ]]; then
46+
cp *.sql $out/share/postgresql/extension
47+
fi
48+
}
49+
50+
create_control_files() {
51+
sed -e "/^default_version =/d" \
52+
-e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}'|" \
53+
${pname}.control > $out/share/postgresql/extension/${pname}--${version}.control
54+
55+
if [[ "${version}" == "${latestVersion}" ]]; then
56+
{
57+
echo "default_version = '${latestVersion}'"
58+
cat $out/share/postgresql/extension/${pname}--${latestVersion}.control
59+
} > $out/share/postgresql/extension/${pname}.control
60+
ln -sfn ${pname}-${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix}
61+
fi
62+
}
63+
64+
create_sql_files
65+
create_control_files
66+
'';
67+
68+
meta = with lib; {
69+
description = "Framework for 'Trusted Language Extensions' in PostgreSQL";
70+
homepage = "https://github.com/aws/${pname}";
71+
license = licenses.postgresql;
72+
inherit (postgresql.meta) platforms;
73+
};
74+
};
75+
allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).${pname};
76+
supportedVersions = lib.filterAttrs (
77+
_: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
78+
) allVersions;
79+
versions = lib.naturalSort (lib.attrNames supportedVersions);
80+
latestVersion = lib.last versions;
81+
numberOfVersions = builtins.length versions;
82+
packages = builtins.attrValues (
83+
lib.mapAttrs (name: value: build name value.hash) supportedVersions
84+
);
85+
in
86+
buildEnv {
87+
name = pname;
88+
paths = packages;
89+
pathsToLink = [
90+
"/lib"
91+
"/share/postgresql/extension"
92+
];
93+
postBuild = ''
94+
# checks
95+
(set -x
96+
test "$(ls -A $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)" = "${
97+
toString (numberOfVersions + 1)
98+
}"
99+
)
37100
'';
38101

39-
meta = with lib; {
40-
description = "Framework for 'Trusted Language Extensions' in PostgreSQL";
41-
homepage = "https://github.com/aws/${pname}";
42-
platforms = postgresql.meta.platforms;
43-
license = licenses.postgresql;
102+
passthru = {
103+
inherit versions numberOfVersions;
104+
pname = "${pname}-all";
105+
defaultSettings = {
106+
shared_preload_libraries = [ "pg_tle" ];
107+
};
108+
version =
109+
"multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
44110
};
45111
}

0 commit comments

Comments
 (0)