From f7c07cb466e63fb466b013c9cdd80fd4db8822ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 20 Jun 2025 19:32:10 +0200 Subject: [PATCH 1/5] feat: multiple versions for the vector extension Build multiple versions of the vector extension on different PostgreSQL versions. Add test for the extensions and their upgrade on PostgreSQL 15 and 17. --- nix/ext/pgvector.nix | 100 +++++++++++++++++------ nix/ext/tests/pgvector.nix | 157 +++++++++++++++++++++++++++++++++++++ nix/ext/versions.json | 48 ++++++++++++ 3 files changed, 283 insertions(+), 22 deletions(-) create mode 100644 nix/ext/tests/pgvector.nix diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index 60a8efea3..c49a02e71 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -1,36 +1,92 @@ { + pkgs, lib, stdenv, fetchFromGitHub, postgresql, }: +let + pname = "vector"; -stdenv.mkDerivation rec { - pname = "pgvector"; - version = "0.8.0"; + # Load version configuration from external file + allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).${pname}; - buildInputs = [ postgresql ]; + # Filter versions compatible with current PostgreSQL version + supportedVersions = lib.filterAttrs ( + _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + ) allVersions; - src = fetchFromGitHub { - owner = "pgvector"; - repo = pname; - rev = "refs/tags/v${version}"; - hash = "sha256-JsZV+I4eRMypXTjGmjCtMBXDVpqTIPHQa28ogXncE/Q="; - }; + # Derived version information + versions = lib.naturalSort (lib.attrNames supportedVersions); + latestVersion = lib.last versions; + numberOfVersions = builtins.length versions; + packages = builtins.attrValues ( + lib.mapAttrs (name: value: build name value.hash) supportedVersions + ); + + # Build function for individual versions + build = + version: hash: + stdenv.mkDerivation rec { + inherit pname version; + + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "pgvector"; + repo = "pgvector"; + rev = "refs/tags/v${version}"; + inherit hash; + }; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + # Install shared library with version suffix + mv ${pname}${postgresql.dlSuffix} $out/lib/${pname}-${version}${postgresql.dlSuffix} + + # Create version-specific control file + sed -e "/^default_version =/d" \ + -e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}'|" \ + ${pname}.control > $out/share/postgresql/extension/${pname}--${version}.control + + # Copy SQL file to install the specific version + cp sql/${pname}.sql $out/share/postgresql/extension/${pname}--${version}.sql + + # For the latest version, copy sql upgrade script, default control file and symlink + if [[ "${version}" == "${latestVersion}" ]]; then + cp sql/*.sql $out/share/postgresql/extension + { + echo "default_version = '${latestVersion}'" + cat $out/share/postgresql/extension/${pname}--${latestVersion}.control + } > $out/share/postgresql/extension/${pname}.control + ln -sfn ${pname}-${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix} + fi - installPhase = '' - mkdir -p $out/{lib,share/postgresql/extension} + runHook postInstall + ''; - cp *${postgresql.dlSuffix} $out/lib - cp sql/*.sql $out/share/postgresql/extension - cp *.control $out/share/postgresql/extension - ''; + meta = with lib; { + description = "Open-source vector similarity search for Postgres"; + homepage = "https://github.com/${src.owner}/${src.repo}"; + maintainers = with maintainers; [ olirice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; + }; +in +pkgs.buildEnv { + name = pname; + paths = packages; + pathsToLink = [ + "/lib" + "/share/postgresql/extension" + ]; - meta = with lib; { - description = "Open-source vector similarity search for Postgres"; - homepage = "https://github.com/${src.owner}/${src.repo}"; - maintainers = with maintainers; [ olirice ]; - platforms = postgresql.meta.platforms; - license = licenses.postgresql; + passthru = { + inherit versions numberOfVersions; + pname = "${pname}-all"; + version = + "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; } diff --git a/nix/ext/tests/pgvector.nix b/nix/ext/tests/pgvector.nix new file mode 100644 index 000000000..c41dc0d81 --- /dev/null +++ b/nix/ext/tests/pgvector.nix @@ -0,0 +1,157 @@ +{ self, pkgs }: +let + pname = "vector"; + inherit (pkgs) lib; + installedExtension = + postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; + postgresqlWithExtension = + postgresql: + let + majorVersion = lib.versions.major postgresql.version; + pkg = pkgs.buildEnv { + name = "postgresql-${majorVersion}-${pname}"; + paths = [ + postgresql + postgresql.lib + (installedExtension majorVersion) + ]; + passthru = { + inherit (postgresql) version psqlSchema; + lib = pkg; + withPackages = _: pkg; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + pathsToLink = [ + "/" + "/bin" + "/lib" + ]; + postBuild = '' + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib + ''; + }; + in + pkg; + pgsodiumGetKey = lib.getExe ( + pkgs.writeShellScriptBin "pgsodium-getkey" '' + echo 0000000000000000000000000000000000000000000000000000000000000000 + '' + ); +in +self.inputs.nixpkgs.lib.nixos.runTest { + name = pname; + hostPkgs = pkgs; + nodes.server = + { config, ... }: + { + virtualisation = { + forwardPorts = [ + { + from = "host"; + host.port = 13022; + guest.port = 22; + } + ]; + }; + + services.postgresql = { + enable = true; + package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + settings = { + "shared_preload_libraries" = pname; + "pgsodium.getkey_script" = pgsodiumGetKey; + }; + }; + + specialisation.postgresql17.configuration = { + services.postgresql = { + package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); + }; + + systemd.services.postgresql-migrate = { + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = "postgres"; + Group = "postgres"; + StateDirectory = "postgresql"; + WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; + }; + script = + let + oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; + oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; + newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; + in + '' + if [[ ! -d ${newDataDir} ]]; then + install -d -m 0700 -o postgres -g postgres "${newDataDir}" + ${newPostgresql}/bin/initdb -D "${newDataDir}" + echo "shared_preload_libraries = '${pname}'" >> "${newDataDir}/postgresql.conf" + echo "pgsodium.getkey_script = '${pgsodiumGetKey}'" >> "${newDataDir}/postgresql.conf"; + ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ + --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" + else + echo "${newDataDir} already exists" + fi + ''; + }; + + systemd.services.postgresql = { + after = [ "postgresql-migrate.service" ]; + requires = [ "postgresql-migrate.service" ]; + }; + }; + }; + testScript = + { nodes, ... }: + let + pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; + in + '' + versions = { + "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], + "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], + } + + def run_sql(query): + return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() + + def check_upgrade_path(pg_version): + with subtest("Check ${pname} upgrade path"): + firstVersion = versions[pg_version][0] + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'") + run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}' CASCADE;""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") + assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}" + for version in versions[pg_version][1:]: + run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") + assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}" + + start_all() + + server.wait_for_unit("multi-user.target") + server.wait_for_unit("postgresql.service") + + check_upgrade_path("15") + + with subtest("Check ${pname} latest extension version"): + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'") + server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname} CASCADE;'") + installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") + latestVersion = versions["15"][-1] + assert f"${pname},{latestVersion}" in installed_extensions + + with subtest("switch to postgresql 17"): + server.succeed( + "${pg17-configuration}/bin/switch-to-configuration test >&2" + ) + + check_upgrade_path("17") + ''; +} diff --git a/nix/ext/versions.json b/nix/ext/versions.json index 6d9c5a2db..31db1741f 100644 --- a/nix/ext/versions.json +++ b/nix/ext/versions.json @@ -126,6 +126,54 @@ "17" ], "hash": "sha256-Cpi2iASi1QJoED0Qs1dANqg/BNZTsz5S+pw8iYyW03Y=" + "vector": { + "0.4.0": { + "postgresql": [ + "15" + ], + "hash": "sha256-bOckX7zvHhgJDDhoAm+VZVIeVIf2hG/3oWZWuTtnZPo=" + }, + "0.5.1": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-ZNzq+dATZn9LUgeOczsaadr5hwdbt9y/+sAOPIdr77U=" + }, + "0.6.0": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-hXm+k0BZ9xZP1Tnek14jPoKCPQkA5ovscu9IX2mW7Kc=" + }, + "0.6.2": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-r+TpFJg6WrMn0L2B7RpmSRvw3XxpHzMRtpFWDCzLvgs=" + }, + "0.7.0": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-vFn7sNphOYyig6Jl1HILMaC2t9strFQBQ8ywL8Ibx1M=" + }, + "0.7.4": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-qwPaguQUdDHV8q6GDneLq5MuhVroPizpbqt7f08gKJI=" + }, + "0.8.0": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-JsZV+I4eRMypXTjGmjCtMBXDVpqTIPHQa28ogXncE/Q=" } } } From 1ac8ca92a55669774673a65f5584732b42dbb703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 12 Aug 2025 14:02:51 +0200 Subject: [PATCH 2/5] chore: add release suffix for testing --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 011045f8a..74b5df521 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.5.1.024-orioledb" - postgres17: "17.6.1.003" - postgres15: "15.14.1.003" + postgresorioledb-17: "17.5.1.024-orioledb-pgvector-1" + postgres17: "17.6.1.003-pgvector-1" + postgres15: "15.14.1.003-pgvector-1" # Non Postgres Extensions pgbouncer_release: "1.19.0" From c903fd076b795db69e56c9f4cad9580f0c5f80e9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 11 Sep 2025 13:24:07 -0400 Subject: [PATCH 3/5] fix: rebase to develop --- nix/ext/versions.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nix/ext/versions.json b/nix/ext/versions.json index 31db1741f..4b81c60cd 100644 --- a/nix/ext/versions.json +++ b/nix/ext/versions.json @@ -126,6 +126,8 @@ "17" ], "hash": "sha256-Cpi2iASi1QJoED0Qs1dANqg/BNZTsz5S+pw8iYyW03Y=" + } + }, "vector": { "0.4.0": { "postgresql": [ From 71d9787ae6e4489f43adc0f5be472ddef408965b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 12 Sep 2025 13:59:06 -0400 Subject: [PATCH 4/5] chore: bump version for release --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 74b5df521..23d447c15 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.5.1.024-orioledb-pgvector-1" - postgres17: "17.6.1.003-pgvector-1" - postgres15: "15.14.1.003-pgvector-1" + postgresorioledb-17: "17.5.1.025-orioledb" + postgres17: "17.6.1.004" + postgres15: "15.14.1.004" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 4de36807b6171560321a6387ea48ec1fbdc98387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 12 Sep 2025 20:21:59 +0200 Subject: [PATCH 5/5] chore: refactor to run default nixos test --- nix/ext/tests/default.nix | 1 + nix/ext/tests/pgvector.nix | 157 ------------------------------------- 2 files changed, 1 insertion(+), 157 deletions(-) delete mode 100644 nix/ext/tests/pgvector.nix diff --git a/nix/ext/tests/default.nix b/nix/ext/tests/default.nix index a346fac62..8032851f7 100644 --- a/nix/ext/tests/default.nix +++ b/nix/ext/tests/default.nix @@ -185,6 +185,7 @@ builtins.listToAttrs ( "index_advisor" "pg_cron" "pg_net" + "vector" "wrappers" ] ) diff --git a/nix/ext/tests/pgvector.nix b/nix/ext/tests/pgvector.nix deleted file mode 100644 index c41dc0d81..000000000 --- a/nix/ext/tests/pgvector.nix +++ /dev/null @@ -1,157 +0,0 @@ -{ self, pkgs }: -let - pname = "vector"; - inherit (pkgs) lib; - installedExtension = - postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; - versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; - postgresqlWithExtension = - postgresql: - let - majorVersion = lib.versions.major postgresql.version; - pkg = pkgs.buildEnv { - name = "postgresql-${majorVersion}-${pname}"; - paths = [ - postgresql - postgresql.lib - (installedExtension majorVersion) - ]; - passthru = { - inherit (postgresql) version psqlSchema; - lib = pkg; - withPackages = _: pkg; - }; - nativeBuildInputs = [ pkgs.makeWrapper ]; - pathsToLink = [ - "/" - "/bin" - "/lib" - ]; - postBuild = '' - wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib - wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib - wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib - ''; - }; - in - pkg; - pgsodiumGetKey = lib.getExe ( - pkgs.writeShellScriptBin "pgsodium-getkey" '' - echo 0000000000000000000000000000000000000000000000000000000000000000 - '' - ); -in -self.inputs.nixpkgs.lib.nixos.runTest { - name = pname; - hostPkgs = pkgs; - nodes.server = - { config, ... }: - { - virtualisation = { - forwardPorts = [ - { - from = "host"; - host.port = 13022; - guest.port = 22; - } - ]; - }; - - services.postgresql = { - enable = true; - package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; - settings = { - "shared_preload_libraries" = pname; - "pgsodium.getkey_script" = pgsodiumGetKey; - }; - }; - - specialisation.postgresql17.configuration = { - services.postgresql = { - package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); - }; - - systemd.services.postgresql-migrate = { - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - User = "postgres"; - Group = "postgres"; - StateDirectory = "postgresql"; - WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; - }; - script = - let - oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; - newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; - oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; - newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; - in - '' - if [[ ! -d ${newDataDir} ]]; then - install -d -m 0700 -o postgres -g postgres "${newDataDir}" - ${newPostgresql}/bin/initdb -D "${newDataDir}" - echo "shared_preload_libraries = '${pname}'" >> "${newDataDir}/postgresql.conf" - echo "pgsodium.getkey_script = '${pgsodiumGetKey}'" >> "${newDataDir}/postgresql.conf"; - ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ - --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" - else - echo "${newDataDir} already exists" - fi - ''; - }; - - systemd.services.postgresql = { - after = [ "postgresql-migrate.service" ]; - requires = [ "postgresql-migrate.service" ]; - }; - }; - }; - testScript = - { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in - '' - versions = { - "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], - "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], - } - - def run_sql(query): - return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() - - def check_upgrade_path(pg_version): - with subtest("Check ${pname} upgrade path"): - firstVersion = versions[pg_version][0] - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'") - run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}' CASCADE;""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}" - for version in versions[pg_version][1:]: - run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}" - - start_all() - - server.wait_for_unit("multi-user.target") - server.wait_for_unit("postgresql.service") - - check_upgrade_path("15") - - with subtest("Check ${pname} latest extension version"): - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'") - server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname} CASCADE;'") - installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") - latestVersion = versions["15"][-1] - assert f"${pname},{latestVersion}" in installed_extensions - - with subtest("switch to postgresql 17"): - server.succeed( - "${pg17-configuration}/bin/switch-to-configuration test >&2" - ) - - check_upgrade_path("17") - ''; -}