From 89f55bf3fbba186dcce850a8330e6618567dc897 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 10 Feb 2024 17:51:38 +0100 Subject: [PATCH] nix: Make postgrest-with-postgresql-xxx postgrest-test-io work better The upside is that postgrest-with-postgresql-xxx postgrest-test-io works as expected now. The downside is, that postgrest-with-postgresql-xxx psql now starts without any schema. This now needs an explicit postgrest-with-postgresql-xxx -f path/to.sql to do anything useful. Resolves #2864 --- .github/workflows/ci.yaml | 2 +- nix/tools/loadtest.nix | 2 +- nix/tools/memory.nix | 2 +- nix/tools/tests.nix | 20 ++++---- nix/tools/withTools.nix | 102 +++++++++++++++++++------------------- 5 files changed, 65 insertions(+), 63 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 89365611a0..1696540445 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -92,7 +92,7 @@ jobs: - name: Run IO tests if: always() - run: postgrest-with-postgresql-${{ matrix.pgVersion }} -f test/io/fixtures.sql postgrest-test-io -vv + run: postgrest-with-postgresql-${{ matrix.pgVersion }} postgrest-test-io -vv Test-Memory-Nix: diff --git a/nix/tools/loadtest.nix b/nix/tools/loadtest.nix index 36635654b6..6c2988b336 100644 --- a/nix/tools/loadtest.nix +++ b/nix/tools/loadtest.nix @@ -60,7 +60,7 @@ let abs_output="$(realpath "$_arg_output")" # shellcheck disable=SC2145 - ${withTools.withPg} --fixtures "$_arg_testdir"/fixtures.sql \ + ${withTools.withPg} -f "$_arg_testdir"/fixtures.sql \ ${withTools.withSlowPg} \ ${withTools.withPgrst} \ ${withTools.withSlowPgrst} \ diff --git a/nix/tools/memory.nix b/nix/tools/memory.nix index bdfa3c5da3..a233e33971 100644 --- a/nix/tools/memory.nix +++ b/nix/tools/memory.nix @@ -17,7 +17,7 @@ let withPath = [ postgrestProfiled curl ]; } '' - ${withTools.withPg} test/memory/memory-tests.sh + ${withTools.withPg} -f test/spec/fixtures/load.sql test/memory/memory-tests.sh ''; in diff --git a/nix/tools/tests.nix b/nix/tools/tests.nix index 469208e9f6..d661b398a5 100644 --- a/nix/tools/tests.nix +++ b/nix/tools/tests.nix @@ -28,8 +28,8 @@ let withEnv = postgrest.env; } '' - ${withTools.withPg} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} \ - test:spec -- "''${_arg_leftovers[@]}" + ${withTools.withPg} -f test/spec/fixtures/load.sql \ + ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec -- "''${_arg_leftovers[@]}" ''; testDoctests = @@ -59,9 +59,10 @@ let withEnv = postgrest.env; } '' - ${withTools.withPg} ${runtimeShell} -c " \ - ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec && \ - ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec" + ${withTools.withPg} -f test/spec/fixtures/load.sql \ + ${runtimeShell} -c " \ + ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec && \ + ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec" ''; ioTestPython = @@ -99,7 +100,7 @@ let withPath = [ jq ]; } '' - ${withTools.withPg} \ + ${withTools.withPg} -f test/spec/fixtures.sql \ ${cabal-install}/bin/cabal v2-run ${devCabalOptions} --verbose=0 -- \ postgrest --dump-schema \ | ${yq}/bin/yq -y . @@ -137,11 +138,12 @@ let # collect all tests HPCTIXFILE="$tmpdir"/io.tix \ - ${withTools.withPg} -f test/io/fixtures.sql ${cabal-install}/bin/cabal v2-exec ${devCabalOptions} -- \ - ${ioTestPython}/bin/pytest -v test/io + ${withTools.withPg} -f test/io/fixtures.sql \ + ${cabal-install}/bin/cabal v2-exec ${devCabalOptions} -- ${ioTestPython}/bin/pytest -v test/io HPCTIXFILE="$tmpdir"/spec.tix \ - ${withTools.withPg} ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec + ${withTools.withPg} -f test/spec/fixtures/load.sql \ + ${cabal-install}/bin/cabal v2-run ${devCabalOptions} test:spec # Note: No coverage for doctests, as doctests leverage GHCi and GHCi does not support hpc diff --git a/nix/tools/withTools.nix b/nix/tools/withTools.nix index 79c9802cac..51764a064b 100644 --- a/nix/tools/withTools.nix +++ b/nix/tools/withTools.nix @@ -17,15 +17,14 @@ let { name, postgresql }: let commandName = "postgrest-with-${name}"; - superuserRole = "postgres"; in checkedShellScript { name = commandName; - docs = "Run the given command in a temporary database with ${name}. If you wish to mutate the database, login with the '${superuserRole}' role."; + docs = "Run the given command in a temporary database with ${name}. If you wish to mutate the database, login with the postgres role."; args = [ - "ARG_OPTIONAL_SINGLE([fixtures], [f], [SQL file to load fixtures from], [test/spec/fixtures/load.sql])" + "ARG_OPTIONAL_SINGLE([fixtures], [f], [SQL file to load fixtures from])" "ARG_POSITIONAL_SINGLE([command], [Command to run])" "ARG_LEFTOVERS([command arguments])" "ARG_USE_ENV([PGUSER], [postgrest_test_authenticator], [Authenticator PG role])" @@ -41,63 +40,64 @@ let withTmpDir = true; } '' - # avoid starting multiple layers of withTmpDb - if test -v PGHOST; then - exec "$_arg_command" "''${_arg_leftovers[@]}" - fi - setuplog="$tmpdir/setup.log" log () { echo "$1" >> "$setuplog" } - mkdir -p "$tmpdir"/{db,socket} - # remove data dir, even if we keep tmpdir - no need to upload it to artifacts - trap 'rm -rf $tmpdir/db' EXIT - - export PGDATA="$tmpdir/db" - export PGHOST="$tmpdir/socket" - export PGUSER - export PGDATABASE - export PGRST_DB_SCHEMAS - export PGTZ - export PGOPTIONS - - HBA_FILE="$tmpdir/pg_hba.conf" - echo "local $PGDATABASE some_protected_user password" > "$HBA_FILE" - echo "local $PGDATABASE all trust" >> "$HBA_FILE" - - log "Initializing database cluster..." - # We try to make the database cluster as independent as possible from the host - # by specifying the timezone, locale and encoding. - # initdb -U creates a superuser(man initdb) - TZ=$PGTZ initdb --no-locale --encoding=UTF8 --nosync -U "${superuserRole}" --auth=trust \ - >> "$setuplog" - - log "Starting the database cluster..." - # Instead of listening on a local port, we will listen on a unix domain socket. - pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $PGHOST -c log_statement=\"all\" " \ - >> "$setuplog" - - # shellcheck disable=SC2317 - stop () { - log "Stopping the database cluster..." - pg_ctl stop -m i >> "$setuplog" - rm -rf "$tmpdir/db" - } - trap stop EXIT - - log "Creating a minimally privileged $PGUSER connection role..." - createuser "$PGUSER" -U "${superuserRole}" --host="$tmpdir/socket" --no-createdb --no-inherit --no-superuser --no-createrole --no-replication --login + # Avoid starting multiple layers of withTmpDb, but make sure to have the last invocation + # load fixtures. Otherwise postgrest-with-postgresql-xx postgrest-test-io would not be possible. + if ! test -v PGHOST; then + + mkdir -p "$tmpdir"/{db,socket} + # remove data dir, even if we keep tmpdir - no need to upload it to artifacts + trap 'rm -rf $tmpdir/db' EXIT + + export PGDATA="$tmpdir/db" + export PGHOST="$tmpdir/socket" + export PGUSER + export PGDATABASE + export PGRST_DB_SCHEMAS + export PGTZ + export PGOPTIONS + + HBA_FILE="$tmpdir/pg_hba.conf" + echo "local $PGDATABASE some_protected_user password" > "$HBA_FILE" + echo "local $PGDATABASE all trust" >> "$HBA_FILE" + + log "Initializing database cluster..." + # We try to make the database cluster as independent as possible from the host + # by specifying the timezone, locale and encoding. + # initdb -U creates a superuser(man initdb) + TZ=$PGTZ initdb --no-locale --encoding=UTF8 --nosync -U postgres --auth=trust \ + >> "$setuplog" + + log "Starting the database cluster..." + # Instead of listening on a local port, we will listen on a unix domain socket. + pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $PGHOST -c log_statement=\"all\" " \ + >> "$setuplog" + + # shellcheck disable=SC2317 + stop () { + log "Stopping the database cluster..." + pg_ctl stop -m i >> "$setuplog" + rm -rf "$tmpdir/db" + } + trap stop EXIT - log "Loading fixtures under the ${superuserRole} role..." - psql -U "${superuserRole}" -v PGUSER="$PGUSER" -v ON_ERROR_STOP=1 -f "$_arg_fixtures" >> "$setuplog" + log "Creating a minimally privileged $PGUSER connection role..." + createuser "$PGUSER" -U postgres --host="$tmpdir/socket" --no-createdb --no-inherit --no-superuser --no-createrole --no-replication --login - log "Done. Running command..." + echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$tmpdir/socket' -U postgres" + echo "${commandName}: You can tail the logs with: tail -f $tmpdir/db.log" + fi - echo "${commandName}: You can connect with: psql 'postgres:///$PGDATABASE?host=$tmpdir/socket' -U ${superuserRole}" - echo "${commandName}: You can tail the logs with: tail -f $tmpdir/db.log" + if test "$_arg_fixtures"; then + log "Loading fixtures under the postgres role..." + psql -U postgres -v PGUSER="$PGUSER" -v ON_ERROR_STOP=1 -f "$_arg_fixtures" >> "$setuplog" + log "Done. Running command..." + fi ("$_arg_command" "''${_arg_leftovers[@]}") '';