Make bind-mount locations more flexible (#356)

## Overview Previously, most bind mount locations on the host machine were subdirectories of the folder specified by the `DATA_PERSIST_ROOT` environment variable (`/data` by default). This change allows the user to set custom locations for the following additional variables, so that they don't need to be all under the same common directory. - `LOGROTATE_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/logrotate`) - `MONGODB_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/mongodb_persist`) - `COWBIRD_MONGODB_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/mongodb_cowbird_persist`) - `POSTGRES_DATA_DIR` (default `${DATA_PERSIST_ROOT}/frontend_persist`) - `WEAVER_MONGODB_DATA_DIR` (default `${DATA_PERSIST_ROOT}/mongodb_weaver_persist`) The following variable is also added which is another location on disk where files that may contain links are placed. Because the links need to be mounted together in order to resolve properly, the subdirectories of this directory are not configurable: - `DATA_PERSIST_SHARED_ROOT` (default: same as `DATA_PERSIST_ROOT`) The following variables now create subdirectories under `DATA_PERSIST_SHARED_ROOT` (previously they were created under `DATA_PERSIST_ROOT` by default): - `USER_WORKSPACES` (default `user_workspaces`) - `WEAVER_WPS_OUTPUTS_DIR` (default `wps_outputs/weaver`) ## Changes **Non-breaking changes** - Adds additional configuration options but all defaults remain the same as before **Breaking changes** None ## Related Issue / Discussion - Resolves #355 ## Additional Information Links to other issues or sources. - [ ] Things to do...
bird-house · Sep 27, 2023 · 547cc08 · 547cc08
2 parents 4884f82 + 59a7cdb
commit 547cc08
Show file tree

Hide file tree

Showing 21 changed files with 133 additions and 29 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.33.1
+current_version = 1.33.2
 commit = True
 tag = False
 tag_name = {new_version}
@@ -30,11 +30,11 @@ search = {current_version}
 replace = {new_version}
 
 [bumpversion:file:RELEASE.txt]
-search = {current_version} 2023-09-25T15:46:44Z
+search = {current_version} 2023-09-27T13:25:37Z
 replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ}
 
 [bumpversion:part:releaseTime]
-values = 2023-09-25T15:46:44Z
+values = 2023-09-27T13:25:37Z
 
 [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template]
 search = 'version': '{current_version}'

diff --git a/CHANGES.md b/CHANGES.md
@@ -17,6 +17,35 @@
 
 [//]: # (list changes here, using '-' for each new entry, remove this when items are added)
 
+[1.33.2](https://github.com/bird-house/birdhouse-deploy/tree/1.33.2) (2023-09-27)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+- Make bind-mount locations more flexible
+
+  Previously, most bind mount locations on the host machine were subdirectories of the folder specified by the 
+  `DATA_PERSIST_ROOT` environment variable (`/data` by default). This change allows the user to set custom locations
+  for the following additional variables, so that they don't need to be all under the same common directory.
+
+  - `LOGROTATE_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/logrotate`)
+  - `MONGODB_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/mongodb_persist`)
+  - `COWBIRD_MONGODB_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/mongodb_cowbird_persist`)
+  - `POSTGRES_DATA_DIR` (default `${DATA_PERSIST_ROOT}/frontend_persist`)
+  - `WEAVER_MONGODB_DATA_DIR` (default `${DATA_PERSIST_ROOT}/mongodb_weaver_persist`)
+
+  The following variable is also added which is another location on disk where files that may contain links
+  are placed. Because the links need to be mounted together in order to resolve properly, the subdirectories
+  of this directory are not configurable:
+
+  - `DATA_PERSIST_SHARED_ROOT` (default: same as `DATA_PERSIST_ROOT`)
+
+  The following variables now create subdirectories under `DATA_PERSIST_SHARED_ROOT` (previously they were
+  created under `DATA_PERSIST_ROOT` by default):
+
+  - `USER_WORKSPACES` (default `user_workspaces`)
+  - `WEAVER_WPS_OUTPUTS_DIR` (default `wps_outputs/weaver`)
+
+
 [1.33.1](https://github.com/bird-house/birdhouse-deploy/tree/1.33.1) (2023-09-25)
 ------------------------------------------------------------------------------------------------------------------
 

diff --git a/Makefile b/Makefile
@@ -1,7 +1,7 @@
 # Generic variables
 override SHELL       := bash
 override APP_NAME    := birdhouse-deploy
-override APP_VERSION := 1.33.1
+override APP_VERSION := 1.33.2
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")

diff --git a/README.rst b/README.rst
@@ -14,13 +14,13 @@ for a full-fledged production platform.
     * - releases
       - | |latest-version| |commits-since|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.33.1.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.33.2.svg
     :alt: Commits since latest release
-    :target: https://github.com/bird-house/birdhouse-deploy/compare/1.33.1...master
+    :target: https://github.com/bird-house/birdhouse-deploy/compare/1.33.2...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-1.33.1-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-1.33.2-blue.svg?style=flat
     :alt: Latest Tag
-    :target: https://github.com/bird-house/birdhouse-deploy/tree/1.33.1
+    :target: https://github.com/bird-house/birdhouse-deploy/tree/1.33.2
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
     :alt: ReadTheDocs Build Status (latest version)

diff --git a/RELEASE.txt b/RELEASE.txt
@@ -1 +1 @@
-1.33.1 2023-09-25T15:46:44Z
+1.33.2 2023-09-27T13:25:37Z
diff --git a/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml b/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml
@@ -13,7 +13,7 @@ services:
       - ${JUPYTERHUB_USER_DATA_DIR}:${JUPYTERHUB_USER_DATA_DIR}
   jupyterhub:
     environment:
-      WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES}
+      WORKSPACE_DIR: ${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}
       PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR: ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR}
     volumes:
-      - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_ROOT}/${USER_WORKSPACES}"
+      - "${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}"
diff --git a/birdhouse/components/cowbird/default.env b/birdhouse/components/cowbird/default.env
@@ -39,7 +39,7 @@ export COWBIRD_MONGODB_VERSION=5.0
 export COWBIRD_MONGODB_HOST=cowbird-mongodb
 export COWBIRD_MONGODB_PORT=27017
 
-# Subdirectory containing the user workspaces used by Cowbird
+# Subdirectory of DATA_PERSIST_SHARED_ROOT containing the user workspaces used by Cowbird
 export USER_WORKSPACES="user_workspaces"
 
 # Subdirectory containing the hardlinks to the public WPS outputs data
@@ -50,6 +50,13 @@ export PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR="public/wps_outputs"
 # Default name for the secure-data-proxy service from Magpie.
 export SECURE_DATA_PROXY_NAME="secure-data-proxy"
 
+COWBIRD_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_cowbird_persist'
+
+DELAYED_EVAL="
+  $DELAYED_EVAL
+  COWBIRD_MONGODB_DATA_DIR
+"
+
 # this dependency is only required if the mongo instance is the one provided in config/mongodb.
 # (include this for Cowbird<2.0.0)
 #COMPONENT_DEPENDENCIES="

diff --git a/birdhouse/components/cowbird/docker-compose-extra.yml b/birdhouse/components/cowbird/docker-compose-extra.yml
@@ -27,8 +27,9 @@ services:
       SECURE_DATA_PROXY_NAME: ${SECURE_DATA_PROXY_NAME}
       # Note that WPS_OUTPUTS_DIR and WORKSPACE_DIR must both point to paths from the same volume.
       # This is to allow the creation of hardlinks between the wpsoutputs and the user workspace.
+      # By default, both are mounted under the "${DATA_PERSIST_SHARED_ROOT}" volume.
       WPS_OUTPUTS_DIR: ${WPS_OUTPUTS_DIR}
-      WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES}
+      WORKSPACE_DIR: ${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}
     links:
       - cowbird-mongodb
     networks:
@@ -41,7 +42,7 @@ services:
       - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini
       # even if not running tasks here, they must be registered to send them off to the right place!
       - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py
-      - "${DATA_PERSIST_ROOT}:${DATA_PERSIST_ROOT}"
+      - "${DATA_PERSIST_SHARED_ROOT}:${DATA_PERSIST_SHARED_ROOT}"
     restart: always
     logging: *default-logging
 
@@ -57,7 +58,7 @@ services:
       # root user
       COWBIRD_FILESYSTEM_ADMIN_UID: 0
       COWBIRD_FILESYSTEM_ADMIN_GID: 0
-      WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES}
+      WORKSPACE_DIR: ${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}
     links:
       - cowbird-mongodb
     networks:
@@ -70,7 +71,7 @@ services:
       - ./components/cowbird/config/cowbird/config.yml:/opt/local/src/cowbird/config/cowbird.yml
       - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini
       - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py
-      - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_ROOT}/${USER_WORKSPACES}"
+      - "${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}"
     restart: always
     logging: *default-logging
 
@@ -81,7 +82,7 @@ services:
     networks:
       - cowbird-mongodb
     volumes:
-      - ${DATA_PERSIST_ROOT}/mongodb_cowbird_persist:/data/db
+      - ${COWBIRD_MONGODB_DATA_DIR}:/data/db
     restart: always
     logging: *default-logging
 

diff --git a/birdhouse/components/scheduler/config.yml.template b/birdhouse/components/scheduler/config.yml.template
@@ -6,7 +6,7 @@
   dockerargs: >-
     --rm --name logrotate
     --volume /var/log/PAVICS:/var/log/PAVICS:rw
-    --volume ${DATA_PERSIST_ROOT}/logrotate:/var/lib:rw
+    --volume ${LOGROTATE_DATA_DIR}:/var/lib:rw
     --volume ${COMPOSE_DIR}/deployment/PAVICS-deploy.logrotate:/etc/logrotate.conf.orig:ro
   image: 'stakater/logrotate:3.13.0'
 

diff --git a/birdhouse/components/scheduler/default.env b/birdhouse/components/scheduler/default.env
@@ -17,6 +17,13 @@ export AUTODEPLOY_PLATFORM_FREQUENCY="7 5 * * *"
 # Hourly
 export AUTODEPLOY_NOTEBOOK_FREQUENCY="@hourly"
 
+export LOGROTATE_DATA_DIR='${DATA_PERSIST_ROOT}/logrotate'
+
+export DELAYED_EVAL="
+  $DELAYED_EVAL
+  LOGROTATE_DATA_DIR
+"
+
 if [ -L "$BIRDHOUSE_LOCAL_ENV" ]; then
     # Add volume-mount if env.local is a symlink and not a real file,
     # so the symlink can resolve.
@@ -28,6 +35,11 @@ if [ -L "$BIRDHOUSE_LOCAL_ENV" ]; then
 fi
 
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here
+VARS="
+  $VARS
+  \$LOGROTATE_DATA_DIR
+"
+
 OPTIONAL_VARS="
   $OPTIONAL_VARS
   \$AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES

diff --git a/birdhouse/components/weaver/default.env b/birdhouse/components/weaver/default.env
@@ -96,7 +96,12 @@ export WEAVER_WPS_PROVIDERS_RETRY_COUNT=5
 # control interval time between retries (duration in seconds, counts toward maximum timeout)
 export WEAVER_WPS_PROVIDERS_RETRY_AFTER=5
 
+export WEAVER_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_weaver_persist'
+
+
 export DELAYED_EVAL="
   $DELAYED_EVAL
   WEAVER_WPS_OUTPUTS_DIR
+  WEAVER_MONGODB_DATA_DIR
+  WEAVER_WPS_PRIVATE_DATA_DIR
 "
diff --git a/birdhouse/components/weaver/docker-compose-extra.yml b/birdhouse/components/weaver/docker-compose-extra.yml
@@ -63,7 +63,6 @@ services:
       - weaver-mongodb
     volumes:
       #- wps_private:/data/wps_private
-      - ${DATA_PERSIST_ROOT}/wps_private:/data/wps_private
       - ./components/weaver/config/weaver/weaver.ini:/opt/local/src/weaver/config/weaver.ini:ro
       - ./components/weaver/config/weaver/request_options.yml:/opt/local/src/weaver/config/request_options.yml:ro
       - ./components/weaver/celery-healthcheck:/opt/local/bin/weaver/celery-healthcheck:ro
@@ -97,7 +96,7 @@ services:
     networks:
       - weaver-mongodb
     volumes:
-      - ${DATA_PERSIST_ROOT}/mongodb_weaver_persist:/data/db
+      - ${WEAVER_MONGODB_DATA_DIR}/mongodb_weaver_persist:/data/db
     restart: always
     logging: *default-logging
 

diff --git a/birdhouse/config/canarie-api/docker_configuration.py.template b/birdhouse/config/canarie-api/docker_configuration.py.template
@@ -109,8 +109,8 @@ SERVICES = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '1.33.1',
-            'releaseTime': '2023-09-25T15:46:44Z',
+            'version': '1.33.2',
+            'releaseTime': '2023-09-27T13:25:37Z',
             'institution': 'Ouranos',
             'researchSubject': 'Climatology',
             'supportEmail': '${SUPPORT_EMAIL}',
@@ -142,8 +142,8 @@ PLATFORMS = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '1.33.1',
-            'releaseTime': '2023-09-25T15:46:44Z',
+            'version': '1.33.2',
+            'releaseTime': '2023-09-27T13:25:37Z',
             'institution': 'Ouranos',
             'researchSubject': 'Climatology',
             'supportEmail': '${SUPPORT_EMAIL}',

diff --git a/birdhouse/config/mongodb/default.env b/birdhouse/config/mongodb/default.env
@@ -0,0 +1,12 @@
+# All env in this default.env can be overridden by env.local.
+
+# All env in this default.env must NOT depend on any other env.  If they do, they
+# must use single quotes to avoid early expansion before overrides in env.local
+# are applied and must be added to the list of DELAYED_EVAL.
+
+export MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_persist'
+
+export DELAYED_EVAL="
+  $DELAYED_EVAL
+  MONGODB_DATA_DIR
+"
diff --git a/birdhouse/config/mongodb/docker-compose-extra.yml b/birdhouse/config/mongodb/docker-compose-extra.yml
@@ -14,7 +14,7 @@ services:
     networks:
       - mongodb
     volumes:
-      - ${DATA_PERSIST_ROOT}/mongodb_persist:/data/db
+      - ${MONGODB_DATA_DIR}:/data/db
     # Mongodb crash with permission denied errors if the command is not overridden like this
     command: bash -c 'chown -R mongodb:mongodb /data && chmod -R 755 /data && mongod'
     restart: always

diff --git a/birdhouse/config/postgres/default.env b/birdhouse/config/postgres/default.env
@@ -4,6 +4,13 @@
 # must use single quotes to avoid early expansion before overrides in env.local
 # are applied and must be added to the list of DELAYED_EVAL.
 
+export POSTGRES_DATA_DIR='${DATA_PERSIST_ROOT}/frontend_persist'
+
+export DELAYED_EVAL="
+  $DELAYED_EVAL
+  POSTGRES_DATA_DIR
+"
+
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here
 VARS="
   $VARS

diff --git a/birdhouse/config/postgres/docker-compose-extra.yml b/birdhouse/config/postgres/docker-compose-extra.yml
@@ -13,7 +13,7 @@ services:
     container_name: postgres
     # ports:  # should not be exposed directly, enable 'optional-components/database-external-ports' as needed
     volumes:
-      - ${DATA_PERSIST_ROOT}/frontend_persist:/var/lib/postgresql/data/pgdata
+      - ${POSTGRES_DATA_DIR}:/var/lib/postgresql/data/pgdata
       - ./config/postgres/postgres-setup.sh:/postgres-setup.sh
     environment:
       PGDATA: /var/lib/postgresql/data/pgdata

diff --git a/birdhouse/default.env b/birdhouse/default.env
@@ -11,6 +11,11 @@ export BASH_IMAGE="bash:5.1.4"
 # Root directory under which all data persistence should be nested under
 export DATA_PERSIST_ROOT="/data"
 
+# shellcheck disable=SC2016
+# Root directory for all files that are persisted on disk and may contain links (ie. the files
+# are "shared" between subdirectories).
+export DATA_PERSIST_SHARED_ROOT='${DATA_PERSIST_ROOT}'
+
 # Log directory used for the various scheduler tasks
 # TODO: use this variable for other references of the log path (only used in the pavics-jupyter-base's .env file for now)
 export PAVICS_LOG_DIR=/var/log/PAVICS
@@ -24,6 +29,8 @@ export PAVICS_FQDN_PUBLIC='${PAVICS_FQDN}'
 export DELAYED_EVAL="
   $DELAYED_EVAL
   PAVICS_FQDN_PUBLIC
+  DATA_PERSIST_SHARED_ROOT
+  WPS_OUTPUTS_DIR
 "
 
 
@@ -53,4 +60,4 @@ export DEFAULT_CONF_DIRS='
 export USER_WORKSPACE_UID=1000
 export USER_WORKSPACE_GID=1000
 
-export WPS_OUTPUTS_DIR="${DATA_PERSIST_ROOT}/wps_outputs"
+export WPS_OUTPUTS_DIR='${DATA_PERSIST_SHARED_ROOT}/wps_outputs'
diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example
@@ -9,6 +9,10 @@
 # Override data persistence root directory
 # export DATA_PERSIST_ROOT="/data/custom/path"  # otherwise use value of 'default.env', directory must exist
 
+# Root directory for all files that are persisted on disk and may contain links (ie. the files
+# are "shared" between subdirectories). This means that the subdirectory structure is fixed.
+#export DATA_PERSIST_SHARED_ROOT='${DATA_PERSIST_ROOT}' # otherwise use the value from 'default.env', must exist
+
 export SSL_CERTIFICATE="/path/to/ssl/cert.pem"  # *absolute* path to the nginx ssl certificate, path and key bundle
 export PAVICS_FQDN="hostname.domainname" # Fully qualified domain name of this Pavics installation
 export DOC_URL="https://www.example.com/" # URL where /doc gets redirected
@@ -213,6 +217,10 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass
 #  . $COMPOSE_DIR/components/scheduler/deploy_data_job.env
 #fi
 
+# Mount point on host machine for the scheduler to write data from log rotations
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#export LOGROTATE_DATA_DIR='${DATA_PERSIST_ROOT}/logrotate'
+
 # Content of "location /" in file config/proxy/conf.d/all-services.include.template
 # Useful to have a custom homepage.
 # Default:
@@ -459,6 +467,22 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass
 # See raven/default.env for more info.
 #export RAVEN_GEO_URL="https://${PAVICS_FQDN}/geoserver/"
 
+# Mount point on host machine to store mongodb server data
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#export MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_persist'
+
+# Mount point on host machine for cowbird to store data from its mongodb server
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#export COWBIRD_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_cowbird_persist'
+
+# Mount point on host machine to store postgres server data
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#export POSTGRES_DATA_DIR='${DATA_PERSIST_ROOT}/frontend_persist'
+
+# Mount point on host machine for weaver to store data from its mongodb server
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#export WEAVER_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_weaver_persist'
+
 
 #############################################################################
 # Monitoring components configs

diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh
@@ -22,6 +22,7 @@ VARS='
   $DOC_URL
   $SUPPORT_EMAIL
   $DATA_PERSIST_ROOT
+  $DATA_PERSIST_SHARED_ROOT
 '
 
 # list of vars to be substituted in template but they do not have to be set in env.local

diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -69,9 +69,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.33.1'
+version = '1.33.2'
 # The full version, including alpha/beta/rc tags.
-release = '1.33.1'
+release = '1.33.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.