bird-house · mishaschwartz · Sep 27, 2023 · Jul 13, 2023 · Jul 25, 2023 · Jul 25, 2023
@@ -15,7 +15,32 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+## Changes
+- Make bind-mount locations more flexible
+
+  Previously, most bind mount locations on the host machine were subdirectories of the folder specified by the 
+  `DATA_PERSIST_ROOT` environment variable (`/data` by default). This change allows the user to set custom locations
+  for the following additional variables, so that they don't need to be all under the same common directory.
+
+  - `LOGROTATE_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/logrotate`)
+  - `MONGODB_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/mongodb_persist`)
+  - `COWBIRD_MONGODB_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/mongodb_cowbird_persist`)
+  - `POSTGRES_DATA_DIR` (default `${DATA_PERSIST_ROOT}/frontend_persist`)
+  - `WEAVER_MONGODB_DATA_DIR` (default `${DATA_PERSIST_ROOT}/mongodb_weaver_persist`)
+  - `WEAVER_WPS_PRIVATE_DIR` (default `${DATA_PERSIST_ROOT}/wps_private`)
+
+  The following variable is also added which is another location on disk where files that may contain links
+  are placed. Because the links need to be mounted together in order to resolve properly, the subdirectories
+  of this directory are not configurable:
+
+  - `DATA_PERSIST_SHARED_ROOT` (default: same as `DATA_PERSIST_ROOT`)
+
+  The following variables now create subdirectories under `DATA_PERSIST_SHARED_ROOT` (previously they were
+  created under `DATA_PERSIST_ROOT` by default):
+
+  - `USER_WORKSPACES` (default `user_workspaces`)
+  - `WEAVER_WPS_OUTPUTS_DIR` (default `wps_outputs/weaver`)
+
 
 [1.33.0](https://github.com/bird-house/birdhouse-deploy/tree/1.33.0) (2023-09-25)
 ------------------------------------------------------------------------------------------------------------------

@@ -13,7 +13,7 @@ services:
       - ${JUPYTERHUB_USER_DATA_DIR}:${JUPYTERHUB_USER_DATA_DIR}
   jupyterhub:
     environment:
-      WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES}
+      WORKSPACE_DIR: ${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}
       PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR: ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR}
     volumes:
-      - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_ROOT}/${USER_WORKSPACES}"
+      - "${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}"
@@ -39,7 +39,7 @@ export COWBIRD_MONGODB_VERSION=5.0
 export COWBIRD_MONGODB_HOST=cowbird-mongodb
 export COWBIRD_MONGODB_PORT=27017
 
-# Subdirectory containing the user workspaces used by Cowbird
+# Subdirectory of DATA_PERSIST_SHARED_ROOT containing the user workspaces used by Cowbird
 export USER_WORKSPACES="user_workspaces"
 
 # Subdirectory containing the hardlinks to the public WPS outputs data
@@ -50,6 +50,13 @@ export PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR="public/wps_outputs"
 # Default name for the secure-data-proxy service from Magpie.
 export SECURE_DATA_PROXY_NAME="secure-data-proxy"
 
+COWBIRD_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_cowbird_persist'
+
+DELAYED_EVAL="
+  $DELAYED_EVAL
+  COWBIRD_MONGODB_DATA_DIR
+"
+
 # this dependency is only required if the mongo instance is the one provided in config/mongodb.
 # (include this for Cowbird<2.0.0)
 #COMPONENT_DEPENDENCIES="

@@ -27,8 +27,9 @@ services:
       SECURE_DATA_PROXY_NAME: ${SECURE_DATA_PROXY_NAME}
       # Note that WPS_OUTPUTS_DIR and WORKSPACE_DIR must both point to paths from the same volume.
       # This is to allow the creation of hardlinks between the wpsoutputs and the user workspace.
+      # By default, both are mounted under the "${DATA_PERSIST_SHARED_ROOT}" volume.
       WPS_OUTPUTS_DIR: ${WPS_OUTPUTS_DIR}
-      WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES}
+      WORKSPACE_DIR: ${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}
     links:
       - cowbird-mongodb
     networks:
@@ -41,7 +42,7 @@ services:
       - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini
       # even if not running tasks here, they must be registered to send them off to the right place!
       - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py
-      - "${DATA_PERSIST_ROOT}:${DATA_PERSIST_ROOT}"
+      - "${DATA_PERSIST_SHARED_ROOT}:${DATA_PERSIST_SHARED_ROOT}"
     restart: always
     logging: *default-logging
 
@@ -57,7 +58,7 @@ services:
       # root user
       COWBIRD_FILESYSTEM_ADMIN_UID: 0
       COWBIRD_FILESYSTEM_ADMIN_GID: 0
-      WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES}
+      WORKSPACE_DIR: ${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}
     links:
       - cowbird-mongodb
     networks:
@@ -70,7 +71,7 @@ services:
       - ./components/cowbird/config/cowbird/config.yml:/opt/local/src/cowbird/config/cowbird.yml
       - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini
       - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py
-      - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_ROOT}/${USER_WORKSPACES}"
+      - "${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}"
     restart: always
     logging: *default-logging
 
@@ -81,7 +82,7 @@ services:
     networks:
       - cowbird-mongodb
     volumes:
-      - ${DATA_PERSIST_ROOT}/mongodb_cowbird_persist:/data/db
+      - ${COWBIRD_MONGODB_DATA_DIR}:/data/db
     restart: always
     logging: *default-logging
 

@@ -6,7 +6,7 @@
   dockerargs: >-
     --rm --name logrotate
     --volume /var/log/PAVICS:/var/log/PAVICS:rw
-    --volume ${DATA_PERSIST_ROOT}/logrotate:/var/lib:rw
+    --volume ${LOGROTATE_DATA_DIR}:/var/lib:rw
     --volume ${COMPOSE_DIR}/deployment/PAVICS-deploy.logrotate:/etc/logrotate.conf.orig:ro
   image: 'stakater/logrotate:3.13.0'
 

@@ -17,6 +17,13 @@ export AUTODEPLOY_PLATFORM_FREQUENCY="7 5 * * *"
 # Hourly
 export AUTODEPLOY_NOTEBOOK_FREQUENCY="@hourly"
 
+export LOGROTATE_DATA_DIR='${DATA_PERSIST_ROOT}/logrotate'
+
+export DELAYED_EVAL="
+  $DELAYED_EVAL
+  LOGROTATE_DATA_DIR
+"
+
 if [ -L "$BIRDHOUSE_LOCAL_ENV" ]; then
     # Add volume-mount if env.local is a symlink and not a real file,
     # so the symlink can resolve.
@@ -28,6 +35,11 @@ if [ -L "$BIRDHOUSE_LOCAL_ENV" ]; then
 fi
 
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here
+VARS="
+  $VARS
+  \$LOGROTATE_DATA_DIR
+"
+
 OPTIONAL_VARS="
   $OPTIONAL_VARS
   \$AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES

@@ -96,7 +96,13 @@ export WEAVER_WPS_PROVIDERS_RETRY_COUNT=5
 # control interval time between retries (duration in seconds, counts toward maximum timeout)
 export WEAVER_WPS_PROVIDERS_RETRY_AFTER=5
 
+export WEAVER_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_weaver_persist'
+
+export WEAVER_WPS_PRIVATE_DIR='${DATA_PERSIST_ROOT}/wps_private'
+
 export DELAYED_EVAL="
   $DELAYED_EVAL
   WEAVER_WPS_OUTPUTS_DIR
+  WEAVER_MONGODB_DATA_DIR
+  WEAVER_WPS_PRIVATE_DATA_DIR
 "
@@ -63,7 +63,7 @@ services:
       - weaver-mongodb
     volumes:
       #- wps_private:/data/wps_private
-      - ${DATA_PERSIST_ROOT}/wps_private:/data/wps_private
+      - ${WEAVER_WPS_PRIVATE_DIR}:/data/wps_private
       - ./components/weaver/config/weaver/weaver.ini:/opt/local/src/weaver/config/weaver.ini:ro
       - ./components/weaver/config/weaver/request_options.yml:/opt/local/src/weaver/config/request_options.yml:ro
       - ./components/weaver/celery-healthcheck:/opt/local/bin/weaver/celery-healthcheck:ro
@@ -97,7 +97,7 @@ services:
     networks:
       - weaver-mongodb
     volumes:
-      - ${DATA_PERSIST_ROOT}/mongodb_weaver_persist:/data/db
+      - ${WEAVER_MONGODB_DATA_DIR}/mongodb_weaver_persist:/data/db
     restart: always
     logging: *default-logging
 

@@ -0,0 +1,12 @@
+# All env in this default.env can be overridden by env.local.
+
+# All env in this default.env must NOT depend on any other env.  If they do, they
+# must use single quotes to avoid early expansion before overrides in env.local
+# are applied and must be added to the list of DELAYED_EVAL.
+
+export MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_persist'
+
+export DELAYED_EVAL="
+  $DELAYED_EVAL
+  MONGODB_DATA_DIR
+"
@@ -14,7 +14,7 @@ services:
     networks:
       - mongodb
     volumes:
-      - ${DATA_PERSIST_ROOT}/mongodb_persist:/data/db
+      - ${MONGODB_DATA_DIR}:/data/db
     # Mongodb crash with permission denied errors if the command is not overridden like this
     command: bash -c 'chown -R mongodb:mongodb /data && chmod -R 755 /data && mongod'
     restart: always

@@ -4,6 +4,13 @@
 # must use single quotes to avoid early expansion before overrides in env.local
 # are applied and must be added to the list of DELAYED_EVAL.
 
+export POSTGRES_DATA_DIR='${DATA_PERSIST_ROOT}/frontend_persist'
+
+export DELAYED_EVAL="
+  $DELAYED_EVAL
+  POSTGRES_DATA_DIR
+"
+
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here
 VARS="
   $VARS

@@ -13,7 +13,7 @@ services:
     container_name: postgres
     # ports:  # should not be exposed directly, enable 'optional-components/database-external-ports' as needed
     volumes:
-      - ${DATA_PERSIST_ROOT}/frontend_persist:/var/lib/postgresql/data/pgdata
+      - ${POSTGRES_DATA_DIR}:/var/lib/postgresql/data/pgdata
       - ./config/postgres/postgres-setup.sh:/postgres-setup.sh
     environment:
       PGDATA: /var/lib/postgresql/data/pgdata

@@ -11,6 +11,11 @@ export BASH_IMAGE="bash:5.1.4"
 # Root directory under which all data persistence should be nested under
 export DATA_PERSIST_ROOT="/data"
 
+# shellcheck disable=SC2016
+# Root directory for all files that are persisted on disk and may contain links (ie. the files
+# are "shared" between subdirectories).
+export DATA_PERSIST_SHARED_ROOT='${DATA_PERSIST_ROOT}'
+
 # Log directory used for the various scheduler tasks
 # TODO: use this variable for other references of the log path (only used in the pavics-jupyter-base's .env file for now)
 export PAVICS_LOG_DIR=/var/log/PAVICS
@@ -24,6 +29,7 @@ export PAVICS_FQDN_PUBLIC='${PAVICS_FQDN}'
 export DELAYED_EVAL="
   $DELAYED_EVAL
   PAVICS_FQDN_PUBLIC
+  DATA_PERSIST_SHARED_ROOT
 "
 
 
@@ -53,4 +59,4 @@ export DEFAULT_CONF_DIRS='
 export USER_WORKSPACE_UID=1000
 export USER_WORKSPACE_GID=1000
 
-export WPS_OUTPUTS_DIR="${DATA_PERSIST_ROOT}/wps_outputs"
+export WPS_OUTPUTS_DIR="${DATA_PERSIST_SHARED_ROOT}/wps_outputs"
@@ -9,6 +9,10 @@
 # Override data persistence root directory
 # export DATA_PERSIST_ROOT="/data/custom/path"  # otherwise use value of 'default.env', directory must exist
 
+# Root directory for all files that are persisted on disk and may contain links (ie. the files
+# are "shared" between subdirectories). This means that the subdirectory structure is fixed.
+#export DATA_PERSIST_SHARED_ROOT='${DATA_PERSIST_ROOT}' # otherwise use the value from 'default.env', must exist
+
 export SSL_CERTIFICATE="/path/to/ssl/cert.pem"  # *absolute* path to the nginx ssl certificate, path and key bundle
 export PAVICS_FQDN="hostname.domainname" # Fully qualified domain name of this Pavics installation
 export DOC_URL="https://www.example.com/" # URL where /doc gets redirected
@@ -213,6 +217,10 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass
 #  . $COMPOSE_DIR/components/scheduler/deploy_data_job.env
 #fi
 
+# Mount point on host machine for the scheduler to write data from log rotations
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#export LOGROTATE_DATA_DIR='${DATA_PERSIST_ROOT}/logrotate'
+
 # Content of "location /" in file config/proxy/conf.d/all-services.include.template
 # Useful to have a custom homepage.
 # Default:
@@ -459,6 +467,26 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass
 # See raven/default.env for more info.
 #export RAVEN_GEO_URL="https://${PAVICS_FQDN}/geoserver/"
 
+# Mount point on host machine to store mongodb server data
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#export MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_persist'
+
+# Mount point on host machine for cowbird to store data from its mongodb server
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#export COWBIRD_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_cowbird_persist'
+
+# Mount point on host machine to store postgres server data
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#export POSTGRES_DATA_DIR='${DATA_PERSIST_ROOT}/frontend_persist'
+
+# Mount point on host machine for weaver to store data from its mongodb server
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#export WEAVER_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_weaver_persist'
+
+# Mount point on host machine for weaver to store private wps data
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#WEAVER_WPS_PRIVATE_DIR='${DATA_PERSIST_ROOT}/wps_private'
+
 
 #############################################################################
 # Monitoring components configs

@@ -22,6 +22,7 @@ VARS='
   $DOC_URL
   $SUPPORT_EMAIL
   $DATA_PERSIST_ROOT
+  $DATA_PERSIST_SHARED_ROOT
 '
 
 # list of vars to be substituted in template but they do not have to be set in env.local