Multi version ext/pg partman (#1854)

* feat: pg_partman all major versions

* wip: stage pg_partman multi-version changes

* add tests for pg_partman

* add tests for pg_partman

* add partman to local tests. remove from initial schema

* add hypertable to partman test

* feat: multiversion pg_partman

* update v15 schema

* Add nixos test for pg_partman extension

Run upgrade and pg_regress tests for pg_partman extension in NixOS tests.

* add 5.3.1 version for partman

* fix: remove old pg_partman version and fix pg_regress outputs

* fix(pg_partman): use correct library name for background worker

The pg_partman extension uses a different library name (pg_partman_bgw) than its extension name (pg_partman). This change separates the library name from the extension name to properly handle the background worker library installation and version switching.

* chore: bump again to release

---------

Co-authored-by: Chris Gwilliams <517923+encima@users.noreply.github.com>
Co-authored-by: Jean-François Roche <jfroche@pyxel.be>

Author: 3 people

ansible/vars.yml

@@ -10,9 +10,9 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.6.0.017-orioledb"
-  postgres17: "17.6.1.060"
-  postgres15: "15.14.1.060"
+  postgresorioledb-17: "17.6.0.018-orioledb"
+  postgres17: "17.6.1.061"
+  postgres15: "15.14.1.061"
 
 # Non Postgres Extensions
 pgbouncer_release: 1.19.0

migrations/tests/extensions/30-pg_partman.sql

@@ -0,0 +1,4 @@
+BEGIN;
+create schema if not exists "partman";
+create extension if not exists pg_partman with schema "partman";
+ROLLBACK;

migrations/tests/extensions/test.sql

@@ -28,3 +28,4 @@
 \ir 27-pg_repack.sql
 \ir 28-pgvector.sql
 \ir 29-pg_tle.sql
+\ir 30-pg_partman.sql

nix/ext/pg_partman.nix

@@ -1,38 +1,113 @@
 {
+  pkgs,
   lib,
   stdenv,
   fetchFromGitHub,
   postgresql,
+  makeWrapper,
+  switch-ext-version,
 }:
 
-stdenv.mkDerivation rec {
+let
   pname = "pg_partman";
-  version = "5.1.0";
+  libName = "pg_partman_bgw";
+  build =
+    version: hash:
+    stdenv.mkDerivation rec {
+      inherit pname version;
 
-  buildInputs = [ postgresql ];
+      buildInputs = [ postgresql ];
+
+      src = fetchFromGitHub {
+        owner = "pgpartman";
+        repo = pname;
+        rev = "refs/tags/v${version}";
+        inherit hash;
+      };
+
+      installPhase = ''
+        mkdir -p $out/{lib,share/postgresql/extension}
+
+        # Install versioned library
+        install -Dm755 src/${libName}${postgresql.dlSuffix} $out/lib/${libName}-${version}${postgresql.dlSuffix}
+
+        # Only install SQL files for the latest version
+        if [[ "${version}" == "${latestVersion}" ]]; then
+          # Install all SQL files from sql/ directory
+          cp -r sql/* $out/share/postgresql/extension/
+
+          # Install upgrade scripts
+          cp updates/* $out/share/postgresql/extension/
+        fi
+
+        # Create versioned control file with modified module path
+        sed -e "/^default_version =/d" \
+            -e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}'|" \
+          ${pname}.control > $out/share/postgresql/extension/${pname}--${version}.control
+      '';
+
+      meta = with lib; {
+        description = "Partition management extension for PostgreSQL";
+        homepage = "https://github.com/pgpartman/pg_partman";
+        changelog = "https://github.com/pgpartman/pg_partman/blob/v${version}/CHANGELOG.md";
+        platforms = postgresql.meta.platforms;
+        license = licenses.postgresql;
+      };
+    };
+  allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).pg_partman;
+  supportedVersions = lib.filterAttrs (
+    _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
+  ) allVersions;
+  versions = lib.naturalSort (lib.attrNames supportedVersions);
+  latestVersion = lib.last versions;
+  numberOfVersions = builtins.length versions;
+  packages = builtins.attrValues (
+    lib.mapAttrs (name: value: build name value.hash) supportedVersions
+  );
+in
+pkgs.buildEnv {
+  name = pname;
+  paths = packages;
+  nativeBuildInputs = [ makeWrapper ];
+
+  pathsToLink = [
+    "/lib"
+    "/share/postgresql/extension"
+  ];
+
+  postBuild = ''
+    {
+      echo "default_version = '${latestVersion}'"
+      cat $out/share/postgresql/extension/${pname}--${latestVersion}.control
+    } > $out/share/postgresql/extension/${pname}.control
+    ln -sfn ${libName}-${latestVersion}${postgresql.dlSuffix} $out/lib/${libName}${postgresql.dlSuffix}
 
-  src = fetchFromGitHub {
-    owner = "pgpartman";
-    repo = pname;
-    rev = "refs/tags/v${version}";
-    sha256 = "sha256-GrVOJ5ywZMyqyDroYDLdKkXDdIJSDGhDfveO/ZvrmYs=";
-  };
 
-  installPhase = ''
-    mkdir -p $out/{lib,share/postgresql/extension}
+    # checks
+    (set -x
+       test "$(ls -A $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)" = "${
+         toString (numberOfVersions + 1)
+       }"
+    )
 
-    cp src/*${postgresql.dlSuffix} $out/lib
-    cp updates/*     $out/share/postgresql/extension
-    cp -r sql/*      $out/share/postgresql/extension
-    cp *.control     $out/share/postgresql/extension
+    makeWrapper ${lib.getExe switch-ext-version} $out/bin/switch_pg_partman_version \
+      --prefix EXT_WRAPPER : "$out" --prefix EXT_NAME : "${pname}" --prefix LIB_NAME : "${libName}"
   '';
 
-  meta = with lib; {
-    description = "Partition management extension for PostgreSQL";
-    homepage = "https://github.com/pgpartman/pg_partman";
-    changelog = "https://github.com/pgpartman/pg_partman/blob/v${version}/CHANGELOG.md";
-    platforms = postgresql.meta.platforms;
-    license = licenses.postgresql;
-    broken = versionOlder postgresql.version "14";
+  passthru = {
+    inherit
+      versions
+      numberOfVersions
+      switch-ext-version
+      libName
+      ;
+    pname = "${pname}-all";
+    hasBackgroundWorker = true;
+    defaultSchema = "partman";
+    defaultSettings = {
+      shared_preload_libraries = [ libName ];
+    };
+    version =
+      "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
   };
 }

nix/ext/tests/default.nix

@@ -146,12 +146,16 @@ let
             "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}],
           }
           extension_name = "${pname}"
+          support_upgrade = True
           pg17_configuration = "${pg17-configuration}"
           ext_has_background_worker = ${
             if (installedExtension "15") ? hasBackgroundWorker then "True" else "False"
           }
           sql_test_directory = Path("${../../tests}")
           pg_regress_test_name = "${(installedExtension "15").pgRegressTestName or pname}"
+          ext_schema = "${(installedExtension "15").defaultSchema or "public"}"
+          lib_name = "${(installedExtension "15").libName or pname}"
+          print(f"Running tests for extension: {lib_name}")
 
           ${builtins.readFile ./lib.py}
 
@@ -160,7 +164,8 @@ let
           server.wait_for_unit("multi-user.target")
           server.wait_for_unit("postgresql.service")
 
-          test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory)
+          test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory, support_upgrade, ext_schema)
+          test.create_schema()
 
           with subtest("Check upgrade path with postgresql 15"):
             test.check_upgrade_path("15")
@@ -174,7 +179,7 @@ let
 
           if ext_has_background_worker:
             with subtest("Test switch_${pname}_version"):
-              test.check_switch_extension_with_background_worker(Path("${psql_15}/lib/${pname}.so"), "15")
+              test.check_switch_extension_with_background_worker(Path(f"${psql_15}/lib/{lib_name}.so"), "15")
 
           with subtest("Check pg_regress with postgresql 15 after installing the last version"):
             test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name)
@@ -224,6 +229,7 @@ builtins.listToAttrs (
       "pg_stat_monitor"
       "pg_tle"
       "pgaudit"
+      "pg_partman"
       "vector"
       "wal2json"
       "wrappers"

nix/ext/tests/lib.py

@@ -20,6 +20,7 @@ def __init__(
         versions: Versions,
         sql_test_dir: Path,
         support_upgrade: bool = True,
+        schema: str = "public",
     ):
         """Initialize the PostgreSQL extension test framework.
 
@@ -35,6 +36,10 @@ def __init__(
         self.versions = versions
         self.support_upgrade = support_upgrade
         self.sql_test_dir = sql_test_dir
+        self.schema = schema
+
+    def create_schema(self):
+        self.run_sql(f"CREATE SCHEMA IF NOT EXISTS {self.schema};")
 
     def run_sql(self, query: str) -> str:
         return self.vm.succeed(
@@ -50,8 +55,12 @@ def drop_extension(self):
         self.run_sql(f"DROP EXTENSION IF EXISTS {self.extension_name};")
 
     def install_extension(self, version: str):
+        if self.schema != "public":
+            ext_schema = f"SCHEMA {self.schema} "
+        else:
+            ext_schema = ""
         self.run_sql(
-            f"""CREATE EXTENSION {self.extension_name} WITH VERSION '{version}' CASCADE;"""
+            f"""CREATE EXTENSION {self.extension_name} WITH {ext_schema}VERSION '{version}' CASCADE;"""
         )
         # Verify version was installed correctly
         self.assert_version_matches(version)

nix/ext/versions.json

@@ -353,21 +353,31 @@
       "hash": "sha256-9St/ESPiFq2NiPKqbwHLwkIyATKUkOGxFcUrWgT+Iqo="
    }
   },
-  "pgmq": {
-    "1.4.4": {
-      "postgresql": [
-        "15"
-      ],
-      "hash": "sha256-z+8/BqIlHwlMnuIzMz6eylmYbSmhtsNt7TJf/CxbdVw="
-    },
-    "1.5.1": {
+  "pg_partman": {
+    "5.3.1": {
       "postgresql": [
         "15",
-        "17"
+        "17",
+        "orioledb-17"
       ],
-      "hash": "sha256-IU+i6ONPwtgsFKdzya6E+222ualR66gkbb0lDr+7Rb8="
+      "hash": "sha256-LK+z/0puUuNLz2ah/SDaxxyLS/On/cmCBhKNwTLa7Pk="
     }
   },
+    "pgmq": {
+      "1.4.4": {
+        "postgresql": [
+          "15"
+        ],
+        "hash": "sha256-z+8/BqIlHwlMnuIzMz6eylmYbSmhtsNt7TJf/CxbdVw="
+      },
+      "1.5.1": {
+        "postgresql": [
+          "15",
+          "17"
+        ],
+        "hash": "sha256-IU+i6ONPwtgsFKdzya6E+222ualR66gkbb0lDr+7Rb8="
+      }
+    },
   "pgroonga": {
     "3.0.7": {
       "postgresql": [

nix/packages/postgres.nix

@@ -36,6 +36,7 @@
         ../ext/pg_graphql
         ../ext/pg_stat_monitor.nix
         ../ext/pg_jsonschema
+        ../ext/pg_partman.nix
         ../ext/pgvector.nix
         ../ext/vault.nix
         ../ext/hypopg.nix

nix/packages/switch-ext-version.nix

@@ -31,28 +31,32 @@ writeShellApplication {
       echo "  NIX_PROFILE - Path to nix profile (default: /var/lib/postgresql/.nix-profile)"
       echo "  LIB_DIR - Override library directory"
       echo "  EXTENSION_DIR - Override extension directory"
+      echo "  LIB_NAME - Override library name"
       exit 1
     fi
 
     VERSION="$1"
-    echo "$VERSION"
+
+    if [ -z "''${LIB_NAME:-}" ]; then
+      LIB_NAME="$EXT_NAME"
+    fi
 
     # Enable overlay on the wrapper package to be able to switch version
     ${lib.getExe overlayfs-on-package} "$EXT_WRAPPER"
 
     # Check if version exists
     EXT_WRAPPER_LIB="$EXT_WRAPPER/lib"
-    EXT_LIB_TO_USE="$EXT_WRAPPER_LIB/$EXT_NAME-$VERSION${postgresql.dlSuffix}"
+    EXT_LIB_TO_USE="$EXT_WRAPPER_LIB/$LIB_NAME-$VERSION${postgresql.dlSuffix}"
     if [ ! -f "$EXT_LIB_TO_USE" ]; then
       echo "Error: Version $VERSION not found in $EXT_WRAPPER_LIB"
       echo "Available versions:"
       #shellcheck disable=SC2012
-      ls "$EXT_WRAPPER_LIB/$EXT_NAME"-*${postgresql.dlSuffix} 2>/dev/null | sed "s/.*$EXT_NAME-/  /" | sed 's/${postgresql.dlSuffix}$//' || echo "  No versions found"
+      ls "$EXT_WRAPPER_LIB/$LIB_NAME"-*${postgresql.dlSuffix} 2>/dev/null | sed "s/.*$LIB_NAME-/  /" | sed 's/${postgresql.dlSuffix}$//' || echo "  No versions found"
       exit 1
     fi
 
     # Update library symlink
-    ln -sfnv "$EXT_LIB_TO_USE" "$EXT_WRAPPER_LIB/$EXT_NAME${postgresql.dlSuffix}"
+    ln -sfnv "$EXT_LIB_TO_USE" "$EXT_WRAPPER_LIB/$LIB_NAME${postgresql.dlSuffix}"
 
     # Handle extension specific steps
     if [ -x "''${EXTRA_STEPS:-}" ]; then

nix/tests/expected/pg_partman.out

@@ -0,0 +1,24 @@
+CREATE SCHEMA IF NOT EXISTS partman_test;
+CREATE TABLE partman_test.time_taptest_table
+    (col1 int,
+    col2 text default 'stuff',
+    col3 timestamptz NOT NULL DEFAULT now())
+PARTITION BY RANGE (col3);
+CREATE INDEX ON partman_test.time_taptest_table (col3);
+CREATE TABLE partman_test.time_taptest_table_template (LIKE partman_test.time_taptest_table);
+ALTER TABLE partman_test.time_taptest_table_template ADD PRIMARY KEY (col1);
+SELECT partman.create_parent(
+    p_parent_table := 'partman_test.time_taptest_table'
+    , p_control := 'col3'
+    , p_interval := '1 day'
+    , p_template_table := 'partman_test.time_taptest_table_template'
+);
+ create_parent 
+---------------
+ t
+(1 row)
+
+DROP SCHEMA partman_test CASCADE;
+NOTICE:  drop cascades to 2 other objects
+DETAIL:  drop cascades to table partman_test.time_taptest_table
+drop cascades to table partman_test.time_taptest_table_template