data migration approach to PG upgrade

compose.build.yaml

@@ -9,6 +9,13 @@ services:
         - netdisco/netdisco:latest-postgresql
         - localhost:5000/netdisco:latest-postgresql
 
+  netdisco-postgresql-13:
+    image: netdisco/netdisco:latest-postgresql-13
+    build:
+      context: ./netdisco-postgresql
+      args:
+        PGVER: 13.4
+
   netdisco-base:
     build:
       context: ./netdisco-base
@@ -42,3 +49,9 @@ services:
         - localhost:5000/netdisco:latest-do
       additional_contexts:
         "localhost:5000/netdisco:latest-backend": "service:netdisco-backend"
+
+  netdisco-db-init:
+    image: netdisco/netdisco:latest-do
+    build:
+      context: ./netdisco-do
+

docker-compose.yml

@@ -13,6 +13,7 @@ x-common-environment: &common-environment
   PGPORT:
   PGUSER:
   PGPASSWORD:
+  NETDISCO_CURRENT_PG_VERSION: '${NETDISCO_CURRENT_PG_VERSION:-18}'
 
 services:
 
@@ -21,11 +22,20 @@ services:
     shm_size: 128mb
     hostname: netdisco-postgresql
     # !! healthcheck is defined in the Dockerfile
+    volumes:
+      - "./netdisco/postgresql:/var/lib/postgresql"
+    environment:
+      <<: *common-environment
+      NETDISCO_DB_SUPERUSER:
+
+  netdisco-postgresql-13:
+    image: netdisco/netdisco:latest-postgresql-13
+    shm_size: 128mb
+    hostname: netdisco-postgresql-13
     volumes:
       - "./netdisco/pgdata:/var/lib/postgresql/data"
     environment:
       <<: *common-environment
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-rwD62cAVG4eboyM9}
       NETDISCO_DB_SUPERUSER:
 
   netdisco-do:
@@ -53,7 +63,8 @@ services:
       netdisco-postgresql:
         condition: service_healthy
     volumes:
-      - "./netdisco/config:/home/netdisco/environments"
+      - "./netdisco/pgdata:/var/lib/pgversions/pg13"
+      - "./netdisco/postgresql:/var/lib/pgversions/new"
     environment:
       <<: *common-environment
       DEPLOY_ADMIN_USER: '${DEPLOY_ADMIN_USER:-YES}' # or set to NO

netdisco-do/netdisco-updatedb.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -uxo pipefail
+set -euo pipefail
 
 # User needs to either set these in docker-compose environment
 # or else run wrapped in netdisco-env
@@ -33,6 +33,47 @@ if [ -z $("${psql[@]}" -A -t -c "SELECT 1 FROM dbix_class_schema_versions WHERE
   STAMP=$(date '+v%Y%m%d_%H%M%S.000')
   "${psql[@]}" -c "CREATE TABLE dbix_class_schema_versions (version varchar(10) PRIMARY KEY, installed varchar(20) NOT NULL)"
   "${psql[@]}" -c "INSERT INTO dbix_class_schema_versions VALUES ('${MAXSCHEMA}', '${STAMP}')"
+
+  echo >&2 -e "${COL}netdisco-updatedb: finding upgrade candidate${NC}"
+  TEST_TO=$(($NETDISCO_CURRENT_PG_VERSION - 1))
+
+  if [ 13 -le $TEST_TO ]; then
+    for ((VER=$TEST_TO;VER>=13;VER--)); do
+      if [ $VER -eq 13 ]; then
+        ROOT="/var/lib/pgversions/pg13"
+      else
+        ROOT="/var/lib/pgversions/new/${VER}/docker"
+      fi
+      echo >&2 -e "${COL}netdisco-updatedb: checking pg ${VER} datadir${NC}"
+
+      if [ -f "${ROOT}/NETDISCO_UPGRADED" ]; then
+        echo >&2 -e "${COL}netdisco-updatedb: pg ${VER} already migrated${NC}"
+        break
+
+      else
+        if [ -f "${ROOT}/PG_VERSION" ]; then
+          echo >&2 -e "${COL}netdisco-updatedb: found candidate pg version ${VER} to upgrade${NC}"
+
+          echo >&2 -e "${COL}netdisco-updatedb: bringing old pg version up to date${NC}"
+          ls -1 /home/netdisco/perl5/lib/perl5/auto/share/dist/App-Netdisco/schema_versions/App-Netdisco-DB-* | \
+            xargs -n1 basename | sort -n -t '-' -k4 | \
+            while read file; do
+              PGHOST= PGPORT= "${psql[@]}" --port=50432 --host=netdisco-postgresql-${VER} \
+                -f "/home/netdisco/perl5/lib/perl5/auto/share/dist/App-Netdisco/schema_versions/$file"
+            done
+
+          echo >&2 -e "${COL}netdisco-updatedb: copying data${NC}"
+          NEWDBHOST=$PGHOST
+          PGHOST= PGPORT= "pg_dump" --port=50432 --host=netdisco-postgresql-${VER} -a -x ${PGDATABASE} \
+            | "${psql[@]}" --port=5432 --host=${NEWDBHOST}
+
+          echo >&2 -e "${COL}netdisco-updatedb: signalling old pg version ${VER} to shutdown${NC}"
+          touch "${ROOT}/NETDISCO_UPGRADED"
+          break
+        fi
+      fi
+    done
+  fi
 fi
 
 echo >&2 -e "${COL}netdisco-updatedb: importing OUI${NC}"

netdisco-postgresql/Dockerfile

@@ -1,5 +1,6 @@
 # vim: ft=Dockerfile
-FROM docker.io/postgres:13.4-alpine
+ARG PGVER=18
+FROM docker.io/postgres:${PGVER}-alpine
 
 ARG BUILD_DATE
 ARG COMMITTISH=HEAD
@@ -17,11 +18,13 @@ LABEL org.label-schema.docker.schema-version="1.0" \
       org.netdisco.maintainer="The Netdisco Project" \
       org.netdisco.version=${COMMITTISH}
 
-VOLUME ["/var/lib/postgresql/data"]
-
 COPY netdisco-initdb.sh /docker-entrypoint-initdb.d/
+COPY netdisco-db-entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/netdisco-db-entrypoint.sh
 
 HEALTHCHECK --interval=3s --timeout=3s --retries=10 --start-period=10s \
   CMD ["pg_isready", "-h", "netdisco-postgresql", "-U", "postgres"]
 
 WORKDIR /
+ENTRYPOINT ["/usr/local/bin/netdisco-db-entrypoint.sh"]
+CMD ["postgres"]

netdisco-postgresql/netdisco-db-entrypoint.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -efuo pipefail
+
+export COL='\033[0;35m'
+export NC='\033[0m'
+
+# Explicit assign; we don't want to pick up PGUSER from environment
+# because that many be the same as NETDISCO_DB_USER.
+PGUSER="${NETDISCO_DB_SUPERUSER:=postgres}"
+su=( su-exec "${PGUSER}" )
+
+# fast quit if we've been upgraded already
+if [ -f "${PGDATA}/NETDISCO_UPGRADED" ]; then exit 0; fi
+
+# pass through if we're the current latest version
+if [ "$PG_MAJOR" = "$NETDISCO_CURRENT_PG_VERSION" ]; then
+  echo >&2 -e "${COL}netdisco-db-entrypoint: starting latest pg version ${PG_MAJOR}${NC}"
+  if [ ! -s "${PGDATA}/PG_VERSION" ]; then
+      echo >&2 -e "${COL}netdisco-db-entrypoint: generating random POSTGRES_PASSWORD${NC}"
+      export POSTGRES_PASSWORD=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+  fi
+  echo >&2 -e "${COL}netdisco-db-entrypoint: passing control to postgresql docker-entrypoint${NC}"
+  exec /usr/local/bin/docker-entrypoint.sh "$@"
+fi
+
+# fast quit if we were never used and have no data
+if [ ! -s "${PGDATA}/PG_VERSION" ]; then exit 0; fi
+
+# so we have a database, but we are not the current.
+# if there's any newer data, silently quit
+
+TEST_FROM=$(($PG_MAJOR + 1))
+TEST_TO=$(($NETDISCO_CURRENT_PG_VERSION - 1))
+
+if [ $TEST_FROM -le $TEST_TO ]; then
+  for ((VER=$TEST_FROM;VER<=TEST_TO;VER++)); do
+    if [ -s "/var/lib/postgresql/$VER/docker/PG_VERSION" ]; then
+      exit 0
+    fi
+  done
+fi
+
+# otherwise run on special port for data migration, wait until data deployed, then quit
+echo >&2 -e "${COL}netdisco-db-entrypoint: ready for upgrade; starting on port 50432${NC}"
+
+NOTIFY_SOCKET= "${su[@]}" \
+  pg_ctl -D "$PGDATA" \
+  -o "-c listen_addresses='*' -p 50432 -c log_min_error_statement=LOG -c log_min_messages=LOG" \
+  -w start
+
+while [ ! -f "${PGDATA}/NETDISCO_UPGRADED" ]; do
+   sleep 2
+done
+echo >&2 -e "${COL}netdisco-db-entrypoint: data migration complete; stopping${NC}"
+
+sleep 2
+"${su[@]}" pg_ctl -D "$PGDATA" -m fast -w stop
+
+exit 0

netdisco-postgresql/netdisco-initdb.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -efuxo pipefail
+set -efuo pipefail
 
 # If running in docker-compose, user gets an initialised database
 # using default username/etc as below.