Orioledb (#51)

* feat: overlay with orioledb psql16 and all current supabase extensions
oriole extension will be in a future commit

* feat: reworking tools and tests

* chore: nixpkgs-fmt

* chore: deactivate migration test  on experimental orioledb for now

* feat: build psql, ext and wrappers with oriole patched psql 16 and orioledb extension

* chore: cleanup ci files

* feat: build, test, cache, docker images for psql 15, 16, oriole patched 16
this commit refactors building up package sets on 2 tracks
one for upstream psql 15 and 16, and one for psql 16 patched from
oriole/postgresql sources. There is an issue with oriole patched
postgresql where timescaledb extension cannot be built against the
patched orioledb postgresql sources. That issue will be addressed
in a follow up PR

.github/workflows/docker.yml

@@ -26,8 +26,11 @@ jobs:
 
       - name: Build images
         run: |
-          nix build .#psql_14/docker -o result-docker-14
           nix build .#psql_15/docker -o result-docker-15
+          nix build .#psql_16/docker -o result-docker-16
+          nix build .#psql_orioledb_16/docker -o result-docker-orioledb-16
+          
+
 
       - name: Tag images
         run: |

.gitignore

@@ -16,3 +16,6 @@ sbom.spdx.json
 sbom.cdx.json
 sbom.csv
 graph*.png
+init.sh
+postgres*
+build.log

ext/orioledb.nix

@@ -0,0 +1,32 @@
+{lib, stdenv, fetchFromGitHub, curl, libkrb5, postgresql, python3, openssl}:
+
+stdenv.mkDerivation rec {
+  pname = "orioledb";
+  name = pname;
+  src = fetchFromGitHub {
+    owner = "orioledb";
+    repo = "orioledb";
+    rev = "main";
+    sha256 = "sha256-QbDp9S8JXO66sfaHZIQ3wFCVRxsAaaNSRgC6hvL3EKY=";
+  };
+  version = "patches16_23";
+  buildInputs = [ curl libkrb5 postgresql python3 openssl ];
+  buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=23";
+  installPhase = ''
+    runHook preInstall
+    mkdir -p $out/{lib,share/postgresql/extension}
+
+    cp *.so      $out/lib
+    cp *.sql     $out/share/postgresql/extension
+    cp *.control $out/share/postgresql/extension
+        
+    runHook postInstall
+  '';
+  doCheck = true;
+  meta = with lib; {
+    description = "orioledb";
+    maintainers = with maintainers; [ samrose ];
+    platforms = postgresql.meta.platforms;
+    license = licenses.postgresql;
+  };
+}

flake.nix

@@ -32,9 +32,26 @@
         pgsqlSuperuser = "postgres";
         nix2img = nix2container.packages.${system}.nix2container;
 
-        # The 'pkgs' variable holds all the upstream packages in nixpkgs, which
+        # The 'oriole_pkgs' variable holds all the upstream packages in nixpkgs, which
         # we can use to build our own images; it is the common name to refer to
         # a copy of nixpkgs which contains all its packages.
+        # it also serves as a base for importing the orioldb/postgres overlay to 
+        #build the orioledb postgres patched version of postgresql16
+        oriole_pkgs = import nixpkgs {
+          inherit system;
+          overlays = [
+            # NOTE (aseipp): add any needed overlays here. in theory we could
+            # pull them from the overlays/ directory automatically, but we don't
+            # want to have an arbitrary order, since it might matter. being
+            # explicit is better.
+            (import ./overlays/cargo-pgrx.nix)
+            (import ./overlays/gdal-small.nix)
+            (import ./overlays/psql_16-oriole.nix)
+
+          ];
+        };
+        #This variable works the same as 'oriole_pkgs' but builds using the upstream
+        #nixpkgs builds of postgresql 15 and 16 + the overlays listed below
         pkgs = import nixpkgs {
           inherit system;
           overlays = [
@@ -44,9 +61,11 @@
             # explicit is better.
             (import ./overlays/cargo-pgrx.nix)
             (import ./overlays/gdal-small.nix)
+
           ];
         };
 
+
         # FIXME (aseipp): pg_prove is yet another perl program that needs
         # LOCALE_ARCHIVE set in non-NixOS environments. upstream this. once that's done, we
         # can remove this wrapper.
@@ -60,6 +79,7 @@
               --set LOCALE_ARCHIVE "${pkgs.glibcLocales}/lib/locale/locale-archive"
           done
         '';
+
 
         # Our list of PostgreSQL extensions which come from upstream Nixpkgs.
         # These are maintained upstream and can easily be used here just by
@@ -74,12 +94,31 @@
           "pgjwt"
           "plpgsql_check"
           "pg_safeupdate"
+          "wal2json"
+          /* pljava */
+          "rum"
+          "pg_repack"
+          "pgroonga"
           "timescaledb"
+        ];
+
+        #FIXME for now, timescaledb is not included in the orioledb version of supabase extensions, as there is an issue
+        # with building timescaledb with the orioledb patched version of postgresql
+        orioledbPsqlExtensions = [
+          "postgis"
+          "pgrouting"
+          "pgtap"
+          "pg_cron"
+          "pgaudit"
+          "pgjwt"
+          "plpgsql_check"
+          "pg_safeupdate"
           "wal2json"
           /* pljava */
           "rum"
           "pg_repack"
           "pgroonga"
+          /*"timescaledb"*/
         ];
 
         # Custom extensions that exist in our repository. These aren't upstream
@@ -112,6 +151,13 @@
           ./ext/plv8.nix
         ];
 
+        #Where we import and build the orioledb extension, we add on our custom extensions
+        # plus the orioledb option
+        orioledbExtension = ourExtensions ++ [./ext/orioledb.nix ];
+
+        #this var is a convenience setting to import the orioledb patched version of postgresql
+        postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb_16;
+
         # Create a 'receipt' file for a given postgresql package. This is a way
         # of adding a bit of metadata to the package, which can be used by other
         # tools to inspect what the contents of the install are: the PSQL
@@ -148,10 +194,23 @@
           };
         };
 
+        makeOurOrioleDbPostgresPkgs = version: patchedPostgres:
+          let postgresql = patchedPostgres;
+          in map (path: pkgs.callPackage path { inherit postgresql; }) orioledbExtension;
+
         makeOurPostgresPkgs = version:
           let postgresql = pkgs."postgresql_${version}";
           in map (path: pkgs.callPackage path { inherit postgresql; }) ourExtensions;
 
+        # Create an attrset that contains all the extensions included in a server for the orioledb version of postgresql + extension.
+        makeOurOrioleDbPostgresPkgsSet = version: patchedPostgres:
+          (builtins.listToAttrs (map
+            (drv:
+              { name = drv.pname; value = drv; }
+            )
+            (makeOurOrioleDbPostgresPkgs version patchedPostgres)))
+          // { recurseForDerivations = true; };
+
         # Create an attrset that contains all the extensions included in a server.
         makeOurPostgresPkgsSet = version:
           (builtins.listToAttrs (map
@@ -161,6 +220,7 @@
             (makeOurPostgresPkgs version)))
           // { recurseForDerivations = true; };
 
+
         # Create a binary distribution of PostgreSQL, given a version.
         #
         # NOTE: The version here does NOT refer to the exact PostgreSQL version;
@@ -189,6 +249,26 @@
             paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ];
           };
 
+        makeOrioleDbPostgresBin = version: patchedPostgres:
+          let
+            postgresql = patchedPostgres;
+            upstreamExts = map
+              (ext: {
+                name = postgresql.pkgs."${ext}".pname;
+                version = postgresql.pkgs."${ext}".version;
+              })
+              orioledbPsqlExtensions;
+            ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version postgresql);
+
+            pgbin = postgresql.withPackages (ps:
+              (map (ext: ps."${ext}") orioledbPsqlExtensions) ++ (makeOurOrioleDbPostgresPkgs version postgresql)
+            );
+          in
+          pkgs.symlinkJoin {
+            inherit (pgbin) name version;
+            paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ];
+          };
+
         # Make a Docker Image from a given PostgreSQL version and binary package.
         # updated to use https://github.com/nlewo/nix2container (samrose)
         makePostgresDocker = version: binPackage:
@@ -322,6 +402,12 @@
           docker = makePostgresDocker version bin;
           recurseForDerivations = true;
         };
+        makeOrioleDbPostgres = version: patchedPostgres: rec {
+          bin = makeOrioleDbPostgresBin version patchedPostgres;
+          exts = makeOurOrioleDbPostgresPkgsSet version patchedPostgres;
+          docker = makePostgresDocker version bin;
+          recurseForDerivations = true;
+        };
 
         # The base set of packages that we export from this Nix Flake, that can
         # be used with 'nix build'. Don't use the names listed below; check the
@@ -331,6 +417,7 @@
           # PostgreSQL versions.
           psql_15 = makePostgres "15";
           psql_16 = makePostgres "16";
+          psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16;
 
           # Start a version of the server.
           start-server =
@@ -345,6 +432,7 @@
                 --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \
                 --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \
                 --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \
+                --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_orioledb_16.bin}' \
                 --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \
                 --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}'
 
@@ -358,7 +446,8 @@
               --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \
               --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \
               --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}'\
-              --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}'
+              --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \
+              --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_orioledb_16.bin}'
             chmod +x $out/bin/start-postgres-client
           '';
 
@@ -388,7 +477,7 @@
             substitute ${./tools/run-replica.sh.in} $out/bin/start-postgres-replica \
               --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \
               --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}'\
-              --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}'
+              --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' 
             chmod +x $out/bin/start-postgres-replica
           '';
         };
@@ -420,6 +509,7 @@
 
             pkill postgres
             mv logfile $out
+            echo ${pgpkg}
           '';
 
       in
@@ -432,13 +522,15 @@
           inherit (pkgs)
             # NOTE: comes from our cargo-pgrx.nix overlay
             cargo-pgrx_0_11_2;
+
         };
 
         # The list of exported 'checks' that are run with every run of 'nix
         # flake check'. This is run in the CI system, as well.
         checks = {
           psql_15 = makeCheckHarness basePackages.psql_15.bin;
           psql_16 = makeCheckHarness basePackages.psql_16.bin;
+          psql_orioledb_16 = makeCheckHarness basePackages.psql_orioledb_16.bin;
         };
 
         # Apps is a list of names of things that can be executed with 'nix run';

justfile

@@ -11,6 +11,7 @@ alias c := check
 build-all:
     nix build .#psql_15/bin .#psql_15/docker
     nix build .#psql_16/bin .#psql_16/docker
+    nix build .#psql_orioledb_16/bin .#psql_orioledb_16/docker
 
 check:
     nix flake check -L

overlays/psql_16-oriole.nix

@@ -0,0 +1,21 @@
+final: prev: {
+  postgresql_16 = prev.postgresql_16.overrideAttrs (old: {
+    pname = "postgresql_16";
+    version = "16_23";
+    src = prev.fetchurl {
+      url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_23.tar.gz";
+      sha256 = "sha256-xWmcqn3DYyBG0FsBNqPWTFzUidSJZgoPWI6Rt0N9oJ4=";
+    };
+    buildInputs = old.buildInputs ++ [
+      prev.bison
+      prev.docbook5
+      prev.docbook_xsl
+      prev.docbook_xsl_ns
+      prev.docbook_xml_dtd_45
+      prev.flex
+      prev.libxslt
+      prev.perl
+    ];
+  });
+  postgresql_orioledb_16 = final.postgresql_16;
+}

tools/run-client.sh.in

@@ -5,15 +5,19 @@
 
 # first argument should be '15' or '16' for the version
 if [ "$1" == "15" ]; then
-    echo "Starting server for PSQL 15"
+    echo "Starting client for PSQL 15"
     PSQL15=@PSQL15_BINDIR@
     BINDIR="$PSQL15"
 elif [ "$1" == "16" ]; then
-    echo "Starting server for PSQL 16"
+    echo "Starting client for PSQL 16"
     PSQL16=@PSQL16_BINDIR@
     BINDIR="$PSQL16"
+elif [ "$1" == "orioledb-16" ]; then
+    echo "Starting client for PSQL ORIOLEDB 16"
+    PSQLORIOLEDB16=@PSQLORIOLEDB16_BINDIR@
+    BINDIR="$PSQLORIOLEDB16"
 else
-    echo "Please provide a valid Postgres version (15 or 16)"
+    echo "Please provide a valid Postgres version (15, 16, or orioledb-16)"
     exit 1
 fi
 

tools/run-replica.sh.in

@@ -12,8 +12,12 @@ elif [ "$1" == "16" ]; then
     echo "Starting server for PSQL 16"
     PSQL16=@PSQL16_BINDIR@
     BINDIR="$PSQL16"
+elif [ "$1" == "orioledb-16" ]; then
+    echo "Starting server for PSQL ORIOLEDB 16"
+    PSQLORIOLEDB16=@PSQLORIOLEDB16_BINDIR@
+    BINDIR="$PSQLORIOLEDB16"
 else
-    echo "Please provide a valid Postgres version (15 or 16)"
+    echo "Please provide a valid Postgres version (15, 16 or orioledb-16)"
     exit 1
 fi
 

tools/run-server.sh.in

@@ -12,8 +12,12 @@ elif [ "$1" == "16" ]; then
     echo "Starting server for PSQL 16"
     PSQL16=@PSQL16_BINDIR@
     BINDIR="$PSQL16"
+elif [ "$1" == "orioledb-16" ]; then
+    echo "Starting server for PSQL ORIOLEDB 16"
+    PSQLORIOLEDB16=@PSQLORIOLEDB16_BINDIR@
+    BINDIR="$PSQLORIOLEDB16"
 else
-    echo "Please provide a valid Postgres version (15 or 16)"
+    echo "Please provide a valid Postgres version (15, 16 or orioledb-16)"
     exit 1
 fi