From 1bc138dc537a8f983af8ff75663d89c8a2d0ab42 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 6 Mar 2024 19:16:51 -0500 Subject: [PATCH] Orioledb (#51) * feat: overlay with orioledb psql16 and all current supabase extensions oriole extension will be in a future commit * feat: reworking tools and tests * chore: nixpkgs-fmt * chore: deactivate migration test on experimental orioledb for now * feat: build psql, ext and wrappers with oriole patched psql 16 and orioledb extension * chore: cleanup ci files * feat: build, test, cache, docker images for psql 15, 16, oriole patched 16 this commit refactors building up package sets on 2 tracks one for upstream psql 15 and 16, and one for psql 16 patched from oriole/postgresql sources. There is an issue with oriole patched postgresql where timescaledb extension cannot be built against the patched orioledb postgresql sources. That issue will be addressed in a follow up PR --- .github/workflows/docker.yml | 5 +- .gitignore | 3 ++ ext/orioledb.nix | 32 ++++++++++++ flake.lock | 6 +-- flake.nix | 98 ++++++++++++++++++++++++++++++++++-- justfile | 1 + overlays/psql_16-oriole.nix | 21 ++++++++ tools/run-client.sh.in | 10 ++-- tools/run-replica.sh.in | 6 ++- tools/run-server.sh.in | 6 ++- 10 files changed, 176 insertions(+), 12 deletions(-) create mode 100644 ext/orioledb.nix create mode 100644 overlays/psql_16-oriole.nix diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d1e32f2..32464bb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -26,8 +26,11 @@ jobs: - name: Build images run: | - nix build .#psql_14/docker -o result-docker-14 nix build .#psql_15/docker -o result-docker-15 + nix build .#psql_16/docker -o result-docker-16 + nix build .#psql_orioledb_16/docker -o result-docker-orioledb-16 + + - name: Tag images run: | diff --git a/.gitignore b/.gitignore index 4de7e20..ff2f95d 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ sbom.spdx.json sbom.cdx.json sbom.csv graph*.png +init.sh +postgres* +build.log diff --git a/ext/orioledb.nix b/ext/orioledb.nix new file mode 100644 index 0000000..734d5c8 --- /dev/null +++ b/ext/orioledb.nix @@ -0,0 +1,32 @@ +{lib, stdenv, fetchFromGitHub, curl, libkrb5, postgresql, python3, openssl}: + +stdenv.mkDerivation rec { + pname = "orioledb"; + name = pname; + src = fetchFromGitHub { + owner = "orioledb"; + repo = "orioledb"; + rev = "main"; + sha256 = "sha256-QbDp9S8JXO66sfaHZIQ3wFCVRxsAaaNSRgC6hvL3EKY="; + }; + version = "patches16_23"; + buildInputs = [ curl libkrb5 postgresql python3 openssl ]; + buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=23"; + installPhase = '' + runHook preInstall + mkdir -p $out/{lib,share/postgresql/extension} + + cp *.so $out/lib + cp *.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + + runHook postInstall + ''; + doCheck = true; + meta = with lib; { + description = "orioledb"; + maintainers = with maintainers; [ samrose ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/flake.lock b/flake.lock index fc201b1..59e9002 100644 --- a/flake.lock +++ b/flake.lock @@ -72,11 +72,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1708847675, - "narHash": "sha256-RUZ7KEs/a4EzRELYDGnRB6i7M1Izii3JD/LyzH0c6Tg=", + "lastModified": 1709386671, + "narHash": "sha256-VPqfBnIJ+cfa78pd4Y5Cr6sOWVW8GYHRVucxJGmRf8Q=", "owner": "nixos", "repo": "nixpkgs", - "rev": "2a34566b67bef34c551f204063faeecc444ae9da", + "rev": "fa9a51752f1b5de583ad5213eb621be071806663", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 93861ce..857152f 100644 --- a/flake.nix +++ b/flake.nix @@ -32,9 +32,26 @@ pgsqlSuperuser = "postgres"; nix2img = nix2container.packages.${system}.nix2container; - # The 'pkgs' variable holds all the upstream packages in nixpkgs, which + # The 'oriole_pkgs' variable holds all the upstream packages in nixpkgs, which # we can use to build our own images; it is the common name to refer to # a copy of nixpkgs which contains all its packages. + # it also serves as a base for importing the orioldb/postgres overlay to + #build the orioledb postgres patched version of postgresql16 + oriole_pkgs = import nixpkgs { + inherit system; + overlays = [ + # NOTE (aseipp): add any needed overlays here. in theory we could + # pull them from the overlays/ directory automatically, but we don't + # want to have an arbitrary order, since it might matter. being + # explicit is better. + (import ./overlays/cargo-pgrx.nix) + (import ./overlays/gdal-small.nix) + (import ./overlays/psql_16-oriole.nix) + + ]; + }; + #This variable works the same as 'oriole_pkgs' but builds using the upstream + #nixpkgs builds of postgresql 15 and 16 + the overlays listed below pkgs = import nixpkgs { inherit system; overlays = [ @@ -44,9 +61,11 @@ # explicit is better. (import ./overlays/cargo-pgrx.nix) (import ./overlays/gdal-small.nix) + ]; }; + # FIXME (aseipp): pg_prove is yet another perl program that needs # LOCALE_ARCHIVE set in non-NixOS environments. upstream this. once that's done, we # can remove this wrapper. @@ -60,6 +79,7 @@ --set LOCALE_ARCHIVE "${pkgs.glibcLocales}/lib/locale/locale-archive" done ''; + # Our list of PostgreSQL extensions which come from upstream Nixpkgs. # These are maintained upstream and can easily be used here just by @@ -74,12 +94,31 @@ "pgjwt" "plpgsql_check" "pg_safeupdate" + "wal2json" + /* pljava */ + "rum" + "pg_repack" + "pgroonga" "timescaledb" + ]; + + #FIXME for now, timescaledb is not included in the orioledb version of supabase extensions, as there is an issue + # with building timescaledb with the orioledb patched version of postgresql + orioledbPsqlExtensions = [ + "postgis" + "pgrouting" + "pgtap" + "pg_cron" + "pgaudit" + "pgjwt" + "plpgsql_check" + "pg_safeupdate" "wal2json" /* pljava */ "rum" "pg_repack" "pgroonga" + /*"timescaledb"*/ ]; # Custom extensions that exist in our repository. These aren't upstream @@ -112,6 +151,13 @@ ./ext/plv8.nix ]; + #Where we import and build the orioledb extension, we add on our custom extensions + # plus the orioledb option + orioledbExtension = ourExtensions ++ [./ext/orioledb.nix ]; + + #this var is a convenience setting to import the orioledb patched version of postgresql + postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb_16; + # Create a 'receipt' file for a given postgresql package. This is a way # of adding a bit of metadata to the package, which can be used by other # tools to inspect what the contents of the install are: the PSQL @@ -148,10 +194,23 @@ }; }; + makeOurOrioleDbPostgresPkgs = version: patchedPostgres: + let postgresql = patchedPostgres; + in map (path: pkgs.callPackage path { inherit postgresql; }) orioledbExtension; + makeOurPostgresPkgs = version: let postgresql = pkgs."postgresql_${version}"; in map (path: pkgs.callPackage path { inherit postgresql; }) ourExtensions; + # Create an attrset that contains all the extensions included in a server for the orioledb version of postgresql + extension. + makeOurOrioleDbPostgresPkgsSet = version: patchedPostgres: + (builtins.listToAttrs (map + (drv: + { name = drv.pname; value = drv; } + ) + (makeOurOrioleDbPostgresPkgs version patchedPostgres))) + // { recurseForDerivations = true; }; + # Create an attrset that contains all the extensions included in a server. makeOurPostgresPkgsSet = version: (builtins.listToAttrs (map @@ -161,6 +220,7 @@ (makeOurPostgresPkgs version))) // { recurseForDerivations = true; }; + # Create a binary distribution of PostgreSQL, given a version. # # NOTE: The version here does NOT refer to the exact PostgreSQL version; @@ -189,6 +249,26 @@ paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; }; + makeOrioleDbPostgresBin = version: patchedPostgres: + let + postgresql = patchedPostgres; + upstreamExts = map + (ext: { + name = postgresql.pkgs."${ext}".pname; + version = postgresql.pkgs."${ext}".version; + }) + orioledbPsqlExtensions; + ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version postgresql); + + pgbin = postgresql.withPackages (ps: + (map (ext: ps."${ext}") orioledbPsqlExtensions) ++ (makeOurOrioleDbPostgresPkgs version postgresql) + ); + in + pkgs.symlinkJoin { + inherit (pgbin) name version; + paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; + }; + # Make a Docker Image from a given PostgreSQL version and binary package. # updated to use https://github.com/nlewo/nix2container (samrose) makePostgresDocker = version: binPackage: @@ -322,6 +402,12 @@ docker = makePostgresDocker version bin; recurseForDerivations = true; }; + makeOrioleDbPostgres = version: patchedPostgres: rec { + bin = makeOrioleDbPostgresBin version patchedPostgres; + exts = makeOurOrioleDbPostgresPkgsSet version patchedPostgres; + docker = makePostgresDocker version bin; + recurseForDerivations = true; + }; # The base set of packages that we export from this Nix Flake, that can # be used with 'nix build'. Don't use the names listed below; check the @@ -331,6 +417,7 @@ # PostgreSQL versions. psql_15 = makePostgres "15"; psql_16 = makePostgres "16"; + psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; # Start a version of the server. start-server = @@ -345,6 +432,7 @@ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ + --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_orioledb_16.bin}' \ --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \ --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' @@ -358,7 +446,8 @@ --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}'\ - --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' + --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ + --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_orioledb_16.bin}' chmod +x $out/bin/start-postgres-client ''; @@ -388,7 +477,7 @@ substitute ${./tools/run-replica.sh.in} $out/bin/start-postgres-replica \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}'\ - --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' + --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' chmod +x $out/bin/start-postgres-replica ''; }; @@ -420,6 +509,7 @@ pkill postgres mv logfile $out + echo ${pgpkg} ''; in @@ -432,6 +522,7 @@ inherit (pkgs) # NOTE: comes from our cargo-pgrx.nix overlay cargo-pgrx_0_11_2; + }; # The list of exported 'checks' that are run with every run of 'nix @@ -439,6 +530,7 @@ checks = { psql_15 = makeCheckHarness basePackages.psql_15.bin; psql_16 = makeCheckHarness basePackages.psql_16.bin; + psql_orioledb_16 = makeCheckHarness basePackages.psql_orioledb_16.bin; }; # Apps is a list of names of things that can be executed with 'nix run'; diff --git a/justfile b/justfile index bd6dd95..e8360fc 100755 --- a/justfile +++ b/justfile @@ -11,6 +11,7 @@ alias c := check build-all: nix build .#psql_15/bin .#psql_15/docker nix build .#psql_16/bin .#psql_16/docker + nix build .#psql_orioledb_16/bin .#psql_orioledb_16/docker check: nix flake check -L diff --git a/overlays/psql_16-oriole.nix b/overlays/psql_16-oriole.nix new file mode 100644 index 0000000..d55af10 --- /dev/null +++ b/overlays/psql_16-oriole.nix @@ -0,0 +1,21 @@ +final: prev: { + postgresql_16 = prev.postgresql_16.overrideAttrs (old: { + pname = "postgresql_16"; + version = "16_23"; + src = prev.fetchurl { + url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_23.tar.gz"; + sha256 = "sha256-xWmcqn3DYyBG0FsBNqPWTFzUidSJZgoPWI6Rt0N9oJ4="; + }; + buildInputs = old.buildInputs ++ [ + prev.bison + prev.docbook5 + prev.docbook_xsl + prev.docbook_xsl_ns + prev.docbook_xml_dtd_45 + prev.flex + prev.libxslt + prev.perl + ]; + }); + postgresql_orioledb_16 = final.postgresql_16; +} diff --git a/tools/run-client.sh.in b/tools/run-client.sh.in index 14850eb..e16198e 100644 --- a/tools/run-client.sh.in +++ b/tools/run-client.sh.in @@ -5,15 +5,19 @@ # first argument should be '15' or '16' for the version if [ "$1" == "15" ]; then - echo "Starting server for PSQL 15" + echo "Starting client for PSQL 15" PSQL15=@PSQL15_BINDIR@ BINDIR="$PSQL15" elif [ "$1" == "16" ]; then - echo "Starting server for PSQL 16" + echo "Starting client for PSQL 16" PSQL16=@PSQL16_BINDIR@ BINDIR="$PSQL16" +elif [ "$1" == "orioledb-16" ]; then + echo "Starting client for PSQL ORIOLEDB 16" + PSQLORIOLEDB16=@PSQLORIOLEDB16_BINDIR@ + BINDIR="$PSQLORIOLEDB16" else - echo "Please provide a valid Postgres version (15 or 16)" + echo "Please provide a valid Postgres version (15, 16, or orioledb-16)" exit 1 fi diff --git a/tools/run-replica.sh.in b/tools/run-replica.sh.in index fc6d666..e2096b1 100644 --- a/tools/run-replica.sh.in +++ b/tools/run-replica.sh.in @@ -12,8 +12,12 @@ elif [ "$1" == "16" ]; then echo "Starting server for PSQL 16" PSQL16=@PSQL16_BINDIR@ BINDIR="$PSQL16" +elif [ "$1" == "orioledb-16" ]; then + echo "Starting server for PSQL ORIOLEDB 16" + PSQLORIOLEDB16=@PSQLORIOLEDB16_BINDIR@ + BINDIR="$PSQLORIOLEDB16" else - echo "Please provide a valid Postgres version (15 or 16)" + echo "Please provide a valid Postgres version (15, 16 or orioledb-16)" exit 1 fi diff --git a/tools/run-server.sh.in b/tools/run-server.sh.in index 126ebd0..c5b6eb0 100644 --- a/tools/run-server.sh.in +++ b/tools/run-server.sh.in @@ -12,8 +12,12 @@ elif [ "$1" == "16" ]; then echo "Starting server for PSQL 16" PSQL16=@PSQL16_BINDIR@ BINDIR="$PSQL16" +elif [ "$1" == "orioledb-16" ]; then + echo "Starting server for PSQL ORIOLEDB 16" + PSQLORIOLEDB16=@PSQLORIOLEDB16_BINDIR@ + BINDIR="$PSQLORIOLEDB16" else - echo "Please provide a valid Postgres version (15 or 16)" + echo "Please provide a valid Postgres version (15, 16 or orioledb-16)" exit 1 fi