From 0c1ab2fcca8bdb54bb1af21f31b6a889230f268a Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Thu, 13 Jul 2023 16:21:01 -0400
Subject: [PATCH 1/8] allow flexible bind mount locations for more variables

---
 CHANGES.md                                    | 14 ++++++++++-
 .../jupyterhub/docker-compose-extra.yml       |  4 ++--
 birdhouse/components/cowbird/default.env      |  9 +++++--
 .../cowbird/docker-compose-extra.yml          |  8 +++----
 .../components/scheduler/config.yml.template  |  2 +-
 birdhouse/components/scheduler/default.env    | 12 ++++++++++
 birdhouse/components/weaver/default.env       |  6 +++++
 .../weaver/docker-compose-extra.yml           |  4 ++--
 birdhouse/config/mongodb/default.env          | 12 ++++++++++
 .../config/mongodb/docker-compose-extra.yml   |  2 +-
 birdhouse/config/postgres/default.env         |  7 ++++++
 .../config/postgres/docker-compose-extra.yml  |  2 +-
 birdhouse/env.local.example                   | 24 +++++++++++++++++++
 13 files changed, 92 insertions(+), 14 deletions(-)
 create mode 100644 birdhouse/config/mongodb/default.env

diff --git a/CHANGES.md b/CHANGES.md
index 6bb3c7265..0ae8ef734 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -15,7 +15,19 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+## Changes
+- Make bind-mount locations more flexible
+
+  Previously, most bind mount locations on the host machine were subdirectories of the folder specified by the 
+  `DATA_PERSIST_ROOT` environment variable (`/data` by default). This change allows the user to set custom locations
+  for the following additional variables, so that they don't need to be all under the same common directory.
+
+  - `LOGROTATE_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/logrotate`)
+  - `MONGODB_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/mongodb_persist`)
+  - `POSTGRES_DATA_DIR` (default `${DATA_PERSIST_ROOT}/frontend_persist`)
+  - `USER_WORKSPACES` (default `${DATA_PERSIST_ROOT}/user_workspaces`)
+  - `WEAVER_MONGODB_DATA_DIR` (default `${DATA_PERSIST_ROOT}/mongodb_weaver_persist`)
+  - `WEAVER_WPS_PRIVATE_DIR` (default `${DATA_PERSIST_ROOT}/wps_private`)
 
 [1.27.1](https://github.com/bird-house/birdhouse-deploy/tree/1.27.1) (2023-07-10)
 ------------------------------------------------------------------------------------------------------------------
diff --git a/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml b/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml
index 04942fdec..ac405f33c 100644
--- a/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml
+++ b/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml
@@ -13,6 +13,6 @@ services:
       - ${JUPYTERHUB_USER_DATA_DIR}:${JUPYTERHUB_USER_DATA_DIR}
   jupyterhub:
     environment:
-      WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES}
+      WORKSPACE_DIR: ${USER_WORKSPACES}
     volumes:
-      - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_ROOT}/${USER_WORKSPACES}"
+      - "${USER_WORKSPACES}:${USER_WORKSPACES}"
diff --git a/birdhouse/components/cowbird/default.env b/birdhouse/components/cowbird/default.env
index 3fb07dd6a..2368c6cbb 100644
--- a/birdhouse/components/cowbird/default.env
+++ b/birdhouse/components/cowbird/default.env
@@ -34,8 +34,13 @@ export COWBIRD_MONGODB_SERVICE=mongodb
 export COWBIRD_MONGODB_HOST=${COWBIRD_MONGODB_SERVICE}
 export COWBIRD_MONGODB_PORT=27017
 
-# Subdirectory containing the user workspaces used by Cowbird
-export USER_WORKSPACES="user_workspaces"
+# Directory on host containing the user workspaces used by Cowbird
+export USER_WORKSPACES='${DATA_PERSIST_ROOT}/user_workspaces'
+
+export DELAYED_EVAL="
+  $DELAYED_EVAL
+  USER_WORKSPACES
+"
 
 # this dependency is only required if COWBIRD_MONGODB_SERVICE is the one provided in config/mongodb.
 # (include this by default to support backwards compatibility for now)
diff --git a/birdhouse/components/cowbird/docker-compose-extra.yml b/birdhouse/components/cowbird/docker-compose-extra.yml
index 0f1d837f4..0fe8e22e5 100644
--- a/birdhouse/components/cowbird/docker-compose-extra.yml
+++ b/birdhouse/components/cowbird/docker-compose-extra.yml
@@ -23,7 +23,7 @@ services:
       # root user
       COWBIRD_FILESYSTEM_ADMIN_UID: 0
       COWBIRD_FILESYSTEM_ADMIN_GID: 0
-      WORKSPACE_DIR: /${USER_WORKSPACES}
+      WORKSPACE_DIR: /user_workspaces
     links:
       - "${COWBIRD_MONGODB_SERVICE}"
     networks:
@@ -36,7 +36,7 @@ services:
       - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini
       # even if not running tasks here, they must be registered to send them off to the right place!
       - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py
-      - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:/${USER_WORKSPACES}"
+      - "${USER_WORKSPACES}:/user_workspaces"
     restart: always
     logging: *default-logging
 
@@ -52,7 +52,7 @@ services:
       # root user
       COWBIRD_FILESYSTEM_ADMIN_UID: 0
       COWBIRD_FILESYSTEM_ADMIN_GID: 0
-      WORKSPACE_DIR: /${USER_WORKSPACES}
+      WORKSPACE_DIR: /user_workspaces
     links:
       - "${COWBIRD_MONGODB_SERVICE}"
     networks:
@@ -65,6 +65,6 @@ services:
       - ./components/cowbird/config/cowbird/config.yml:/opt/local/src/cowbird/config/cowbird.yml
       - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini
       - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py
-      - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:/${USER_WORKSPACES}"
+      - "${USER_WORKSPACES}:/user_workspaces"
     restart: always
     logging: *default-logging
diff --git a/birdhouse/components/scheduler/config.yml.template b/birdhouse/components/scheduler/config.yml.template
index 81d863c76..31a056593 100644
--- a/birdhouse/components/scheduler/config.yml.template
+++ b/birdhouse/components/scheduler/config.yml.template
@@ -6,7 +6,7 @@
   dockerargs: >-
     --rm --name logrotate
     --volume /var/log/PAVICS:/var/log/PAVICS:rw
-    --volume ${DATA_PERSIST_ROOT}/logrotate:/var/lib:rw
+    --volume ${LOGROTATE_DATA_DIR}:/var/lib:rw
     --volume ${COMPOSE_DIR}/deployment/PAVICS-deploy.logrotate:/etc/logrotate.conf.orig:ro
   image: 'stakater/logrotate:3.13.0'
 
diff --git a/birdhouse/components/scheduler/default.env b/birdhouse/components/scheduler/default.env
index bc6339829..1261f3c08 100644
--- a/birdhouse/components/scheduler/default.env
+++ b/birdhouse/components/scheduler/default.env
@@ -17,6 +17,13 @@ export AUTODEPLOY_PLATFORM_FREQUENCY="7 5 * * *"
 # Hourly
 export AUTODEPLOY_NOTEBOOK_FREQUENCY="@hourly"
 
+export LOGROTATE_DATA_DIR='${DATA_PERSIST_ROOT}/logrotate'
+
+export DELAYED_EVAL="
+  $DELAYED_EVAL
+  LOGROTATE_DATA_DIR
+"
+
 if [ -L "$BIRDHOUSE_LOCAL_ENV" ]; then
     # Add volume-mount if env.local is a symlink and not a real file,
     # so the symlink can resolve.
@@ -28,6 +35,11 @@ if [ -L "$BIRDHOUSE_LOCAL_ENV" ]; then
 fi
 
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here
+VARS="
+  $VARS
+  \$LOGROTATE_DATA_DIR
+"
+
 OPTIONAL_VARS="
   $OPTIONAL_VARS
   \$AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES
diff --git a/birdhouse/components/weaver/default.env b/birdhouse/components/weaver/default.env
index bb3e66835..159d7b536 100644
--- a/birdhouse/components/weaver/default.env
+++ b/birdhouse/components/weaver/default.env
@@ -96,7 +96,13 @@ export WEAVER_WPS_PROVIDERS_RETRY_COUNT=5
 # control interval time between retries (duration in seconds, counts toward maximum timeout)
 export WEAVER_WPS_PROVIDERS_RETRY_AFTER=5
 
+export WEAVER_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_weaver_persist'
+
+export WEAVER_WPS_PRIVATE_DIR='${DATA_PERSIST_ROOT}/wps_private'
+
 export DELAYED_EVAL="
   $DELAYED_EVAL
   WEAVER_WPS_OUTPUTS_DIR
+  WEAVER_MONGODB_DATA_DIR
+  WEAVER_WPS_PRIVATE_DATA_DIR
 "
diff --git a/birdhouse/components/weaver/docker-compose-extra.yml b/birdhouse/components/weaver/docker-compose-extra.yml
index 5800443c2..d1b92efd5 100644
--- a/birdhouse/components/weaver/docker-compose-extra.yml
+++ b/birdhouse/components/weaver/docker-compose-extra.yml
@@ -65,7 +65,7 @@ services:
       - weaver-mongodb
     volumes:
       #- wps_private:/data/wps_private
-      - ${DATA_PERSIST_ROOT}/wps_private:/data/wps_private
+      - ${WEAVER_WPS_PRIVATE_DIR}:/data/wps_private
       - ./components/weaver/config/weaver/weaver.ini:/opt/local/src/weaver/config/weaver.ini:ro
       - ./components/weaver/config/weaver/request_options.yml:/opt/local/src/weaver/config/request_options.yml:ro
       - ./components/weaver/celery-healthcheck:/opt/local/bin/weaver/celery-healthcheck:ro
@@ -99,7 +99,7 @@ services:
     networks:
       - weaver-mongodb
     volumes:
-      - ${DATA_PERSIST_ROOT}/mongodb_weaver_persist:/data/db
+      - ${WEAVER_MONGODB_DATA_DIR}/mongodb_weaver_persist:/data/db
     restart: always
     logging: *default-logging
 
diff --git a/birdhouse/config/mongodb/default.env b/birdhouse/config/mongodb/default.env
new file mode 100644
index 000000000..e031e852c
--- /dev/null
+++ b/birdhouse/config/mongodb/default.env
@@ -0,0 +1,12 @@
+# All env in this default.env can be overridden by env.local.
+
+# All env in this default.env must NOT depend on any other env.  If they do, they
+# must use single quotes to avoid early expansion before overrides in env.local
+# are applied and must be added to the list of DELAYED_EVAL.
+
+export MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_persist'
+
+export DELAYED_EVAL="
+  $DELAYED_EVAL
+  MONGODB_DATA_DIR
+"
diff --git a/birdhouse/config/mongodb/docker-compose-extra.yml b/birdhouse/config/mongodb/docker-compose-extra.yml
index fedf54a47..c7f96a198 100644
--- a/birdhouse/config/mongodb/docker-compose-extra.yml
+++ b/birdhouse/config/mongodb/docker-compose-extra.yml
@@ -14,7 +14,7 @@ services:
     networks:
       - mongodb
     volumes:
-      - ${DATA_PERSIST_ROOT}/mongodb_persist:/data/db
+      - ${MONGODB_DATA_DIR}:/data/db
     # Mongodb crash with permission denied errors if the command is not overridden like this
     command: bash -c 'chown -R mongodb:mongodb /data && chmod -R 755 /data && mongod'
     restart: always
diff --git a/birdhouse/config/postgres/default.env b/birdhouse/config/postgres/default.env
index e5f8d44d6..38496f27c 100644
--- a/birdhouse/config/postgres/default.env
+++ b/birdhouse/config/postgres/default.env
@@ -4,6 +4,13 @@
 # must use single quotes to avoid early expansion before overrides in env.local
 # are applied and must be added to the list of DELAYED_EVAL.
 
+export POSTGRES_DATA_DIR='${DATA_PERSIST_ROOT}/frontend_persist'
+
+export DELAYED_EVAL="
+  $DELAYED_EVAL
+  POSTGRES_DATA_DIR
+"
+
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here
 VARS="
   $VARS
diff --git a/birdhouse/config/postgres/docker-compose-extra.yml b/birdhouse/config/postgres/docker-compose-extra.yml
index d65bf977f..db004bf7f 100644
--- a/birdhouse/config/postgres/docker-compose-extra.yml
+++ b/birdhouse/config/postgres/docker-compose-extra.yml
@@ -13,7 +13,7 @@ services:
     container_name: postgres
     # ports:  # should not be exposed directly, enable 'optional-components/database-external-ports' as needed
     volumes:
-      - ${DATA_PERSIST_ROOT}/frontend_persist:/var/lib/postgresql/data/pgdata
+      - ${POSTGRES_DATA_DIR}:/var/lib/postgresql/data/pgdata
       - ./config/postgres/postgres-setup.sh:/postgres-setup.sh
     environment:
       PGDATA: /var/lib/postgresql/data/pgdata
diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example
index 87e1a53c3..23694e901 100644
--- a/birdhouse/env.local.example
+++ b/birdhouse/env.local.example
@@ -213,6 +213,10 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass
 #  . $COMPOSE_DIR/components/scheduler/deploy_data_job.env
 #fi
 
+# Mount point on host machine for the scheduler to write data from log rotations
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#export LOGROTATE_DATA_DIR='${DATA_PERSIST_ROOT}/logrotate'
+
 # Content of "location /" in file config/proxy/conf.d/all-services.include.template
 # Useful to have a custom homepage.
 # Default:
@@ -456,6 +460,26 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass
 # See raven/default.env for more info.
 #export RAVEN_GEO_URL="https://${PAVICS_FQDN}/geoserver/"
 
+# Mount point on host machine to store mongodb server data
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#export MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_persist'
+
+# Mount point on host machine to store postgres server data
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#export POSTGRES_DATA_DIR='${DATA_PERSIST_ROOT}/frontend_persist'
+
+# Mount point on host machine for the user workspace shared by the cowbird and jupyterhub components
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#export USER_WORKSPACES='${DATA_PERSIST_ROOT}/user_workspaces'
+
+# Mount point on host machine for weaver to store data from its mongodb server
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#export WEAVER_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_weaver_persist'
+
+# Mount point on host machine for weaver to store private wps data
+# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
+#WEAVER_WPS_PRIVATE_DIR='${DATA_PERSIST_ROOT}/wps_private'
+
 
 #############################################################################
 # Monitoring components configs

From 9a50dcbd0466cab3d82c1592764aa12979a86d19 Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Tue, 25 Jul 2023 14:19:48 -0400
Subject: [PATCH 2/8] create persist shared directory

---
 CHANGES.md                                          | 13 ++++++++++++-
 .../config/jupyterhub/docker-compose-extra.yml      |  4 ++--
 birdhouse/components/cowbird/default.env            |  7 +------
 .../components/cowbird/docker-compose-extra.yml     |  8 ++++----
 birdhouse/components/weaver/default.env             |  2 +-
 birdhouse/default.env                               |  6 ++++++
 birdhouse/env.local.example                         |  4 ++++
 birdhouse/pavics-compose.sh                         |  1 +
 8 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 0ae8ef734..1dbc18a1e 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -25,10 +25,21 @@
   - `LOGROTATE_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/logrotate`)
   - `MONGODB_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/mongodb_persist`)
   - `POSTGRES_DATA_DIR` (default `${DATA_PERSIST_ROOT}/frontend_persist`)
-  - `USER_WORKSPACES` (default `${DATA_PERSIST_ROOT}/user_workspaces`)
   - `WEAVER_MONGODB_DATA_DIR` (default `${DATA_PERSIST_ROOT}/mongodb_weaver_persist`)
   - `WEAVER_WPS_PRIVATE_DIR` (default `${DATA_PERSIST_ROOT}/wps_private`)
 
+  The following variable is also added which is another location on disk where files that may contain links
+  are placed. Because the links need to be mounted together in order to resolve properly, the subdirectories
+  of this directory are not configurable:
+
+  - `DATA_PERSIST_SHARED_ROOT` (default: same as `DATA_PERSIST_ROOT`)
+
+  The following variables now create subdirectories under `DATA_PERSIST_SHARED_ROOT` (previously they were
+  created under `DATA_PERSIST_ROOT` by default):
+
+  - `USER_WORKSPACES` (default `user_workspaces`)
+
+
 [1.27.1](https://github.com/bird-house/birdhouse-deploy/tree/1.27.1) (2023-07-10)
 ------------------------------------------------------------------------------------------------------------------
 
diff --git a/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml b/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml
index ac405f33c..853c83ddf 100644
--- a/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml
+++ b/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml
@@ -13,6 +13,6 @@ services:
       - ${JUPYTERHUB_USER_DATA_DIR}:${JUPYTERHUB_USER_DATA_DIR}
   jupyterhub:
     environment:
-      WORKSPACE_DIR: ${USER_WORKSPACES}
+      WORKSPACE_DIR: ${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}
     volumes:
-      - "${USER_WORKSPACES}:${USER_WORKSPACES}"
+      - "${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}"
diff --git a/birdhouse/components/cowbird/default.env b/birdhouse/components/cowbird/default.env
index 2368c6cbb..65e08fee4 100644
--- a/birdhouse/components/cowbird/default.env
+++ b/birdhouse/components/cowbird/default.env
@@ -35,12 +35,7 @@ export COWBIRD_MONGODB_HOST=${COWBIRD_MONGODB_SERVICE}
 export COWBIRD_MONGODB_PORT=27017
 
 # Directory on host containing the user workspaces used by Cowbird
-export USER_WORKSPACES='${DATA_PERSIST_ROOT}/user_workspaces'
-
-export DELAYED_EVAL="
-  $DELAYED_EVAL
-  USER_WORKSPACES
-"
+export USER_WORKSPACES="user_workspaces"
 
 # this dependency is only required if COWBIRD_MONGODB_SERVICE is the one provided in config/mongodb.
 # (include this by default to support backwards compatibility for now)
diff --git a/birdhouse/components/cowbird/docker-compose-extra.yml b/birdhouse/components/cowbird/docker-compose-extra.yml
index 0fe8e22e5..d4f9fc614 100644
--- a/birdhouse/components/cowbird/docker-compose-extra.yml
+++ b/birdhouse/components/cowbird/docker-compose-extra.yml
@@ -23,7 +23,7 @@ services:
       # root user
       COWBIRD_FILESYSTEM_ADMIN_UID: 0
       COWBIRD_FILESYSTEM_ADMIN_GID: 0
-      WORKSPACE_DIR: /user_workspaces
+      WORKSPACE_DIR: /${USER_WORKSPACES}
     links:
       - "${COWBIRD_MONGODB_SERVICE}"
     networks:
@@ -36,7 +36,7 @@ services:
       - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini
       # even if not running tasks here, they must be registered to send them off to the right place!
       - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py
-      - "${USER_WORKSPACES}:/user_workspaces"
+      - "${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}:/${USER_WORKSPACES}"
     restart: always
     logging: *default-logging
 
@@ -52,7 +52,7 @@ services:
       # root user
       COWBIRD_FILESYSTEM_ADMIN_UID: 0
       COWBIRD_FILESYSTEM_ADMIN_GID: 0
-      WORKSPACE_DIR: /user_workspaces
+      WORKSPACE_DIR: /${USER_WORKSPACES}
     links:
       - "${COWBIRD_MONGODB_SERVICE}"
     networks:
@@ -65,6 +65,6 @@ services:
       - ./components/cowbird/config/cowbird/config.yml:/opt/local/src/cowbird/config/cowbird.yml
       - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini
       - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py
-      - "${USER_WORKSPACES}:/user_workspaces"
+      - "${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}:/${USER_WORKSPACES}"
     restart: always
     logging: *default-logging
diff --git a/birdhouse/components/weaver/default.env b/birdhouse/components/weaver/default.env
index 159d7b536..7797c2f68 100644
--- a/birdhouse/components/weaver/default.env
+++ b/birdhouse/components/weaver/default.env
@@ -82,7 +82,7 @@ export WEAVER_WPS_OUTPUTS_PATH="/wpsoutputs/weaver"
 # data persistence of wps outputs (directory path)
 # NOTE:
 #   follow the same convension of 'wps_outputs/<bird>' as the rest of the stact
-export WEAVER_WPS_OUTPUTS_DIR='${DATA_PERSIST_ROOT}/wps_outputs/weaver'
+export WEAVER_WPS_OUTPUTS_DIR='${DATA_PERSIST_SHARED_ROOT}/wps_outputs/weaver'
 export WEAVER_WPS_WORKDIR="/tmp/wps_workdir/weaver"
 
 # logging
diff --git a/birdhouse/default.env b/birdhouse/default.env
index 6fd3456ae..4dab367be 100644
--- a/birdhouse/default.env
+++ b/birdhouse/default.env
@@ -11,6 +11,11 @@ export BASH_IMAGE="bash:5.1.4"
 # Root directory under which all data persistence should be nested under
 export DATA_PERSIST_ROOT="/data"
 
+# shellcheck disable=SC2016
+# Root directory for all files that are persisted on disk and may contain links (ie. the files
+# are "shared" between subdirectories). This means that the subdirectory structure is fixed.
+export DATA_PERSIST_SHARED_ROOT='${DATA_PERSIST_ROOT}'
+
 # Log directory used for the various scheduler tasks
 # TODO: use this variable for other references of the log path (only used in the pavics-jupyter-base's .env file for now)
 export PAVICS_LOG_DIR=/var/log/PAVICS
@@ -24,6 +29,7 @@ export PAVICS_FQDN_PUBLIC='${PAVICS_FQDN}'
 export DELAYED_EVAL="
   $DELAYED_EVAL
   PAVICS_FQDN_PUBLIC
+  DATA_PERSIST_SHARED_ROOT
 "
 
 
diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example
index 23694e901..c9ba7b957 100644
--- a/birdhouse/env.local.example
+++ b/birdhouse/env.local.example
@@ -9,6 +9,10 @@
 # Override data persistence root directory
 # export DATA_PERSIST_ROOT="/data/custom/path"  # otherwise use value of 'default.env', directory must exist
 
+# Root directory for all files that are persisted on disk and may contain links (ie. the files
+# are "shared" between subdirectories). This means that the subdirectory structure is fixed.
+#export DATA_PERSIST_SHARED_ROOT='${DATA_PERSIST_ROOT}' # otherwise use the value from 'default.env', must exist
+
 export SSL_CERTIFICATE="/path/to/ssl/cert.pem"  # *absolute* path to the nginx ssl certificate, path and key bundle
 export PAVICS_FQDN="hostname.domainname" # Fully qualified domain name of this Pavics installation
 export DOC_URL="https://www.example.com/" # URL where /doc gets redirected
diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh
index 8e8644b4a..1858de2f3 100755
--- a/birdhouse/pavics-compose.sh
+++ b/birdhouse/pavics-compose.sh
@@ -22,6 +22,7 @@ VARS='
   $DOC_URL
   $SUPPORT_EMAIL
   $DATA_PERSIST_ROOT
+  $DATA_PERSIST_SHARED_ROOT
 '
 
 # list of vars to be substituted in template but they do not have to be set in env.local

From f6a3dc0705499ddf1967595112891839076ef1fd Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Tue, 25 Jul 2023 14:21:22 -0400
Subject: [PATCH 3/8] update changes file

---
 CHANGES.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES.md b/CHANGES.md
index 1dbc18a1e..cff9b9e1f 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -38,6 +38,7 @@
   created under `DATA_PERSIST_ROOT` by default):
 
   - `USER_WORKSPACES` (default `user_workspaces`)
+  - `WEAVER_WPS_OUTPUTS_DIR` (default `wps_outputs/weaver`)
 
 
 [1.27.1](https://github.com/bird-house/birdhouse-deploy/tree/1.27.1) (2023-07-10)

From 586ea3d4b4af40e53017687acbc734aa061409e4 Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Tue, 25 Jul 2023 14:23:44 -0400
Subject: [PATCH 4/8] clarify documentation for cowbird

---
 birdhouse/components/cowbird/default.env | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/birdhouse/components/cowbird/default.env b/birdhouse/components/cowbird/default.env
index 65e08fee4..e5d805c24 100644
--- a/birdhouse/components/cowbird/default.env
+++ b/birdhouse/components/cowbird/default.env
@@ -34,7 +34,7 @@ export COWBIRD_MONGODB_SERVICE=mongodb
 export COWBIRD_MONGODB_HOST=${COWBIRD_MONGODB_SERVICE}
 export COWBIRD_MONGODB_PORT=27017
 
-# Directory on host containing the user workspaces used by Cowbird
+# Subdirectory of DATA_PERSIST_SHARED_ROOT containing the user workspaces used by Cowbird
 export USER_WORKSPACES="user_workspaces"
 
 # this dependency is only required if COWBIRD_MONGODB_SERVICE is the one provided in config/mongodb.

From a9ef0df62e2fc2158f3cec62fc32f25df7f9d329 Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Wed, 26 Jul 2023 09:17:40 -0400
Subject: [PATCH 5/8] config and documentation updates

---
 birdhouse/default.env       | 2 +-
 birdhouse/env.local.example | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/birdhouse/default.env b/birdhouse/default.env
index 4dab367be..44af06828 100644
--- a/birdhouse/default.env
+++ b/birdhouse/default.env
@@ -13,7 +13,7 @@ export DATA_PERSIST_ROOT="/data"
 
 # shellcheck disable=SC2016
 # Root directory for all files that are persisted on disk and may contain links (ie. the files
-# are "shared" between subdirectories). This means that the subdirectory structure is fixed.
+# are "shared" between subdirectories).
 export DATA_PERSIST_SHARED_ROOT='${DATA_PERSIST_ROOT}'
 
 # Log directory used for the various scheduler tasks
diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example
index c9ba7b957..58c4c964e 100644
--- a/birdhouse/env.local.example
+++ b/birdhouse/env.local.example
@@ -472,10 +472,6 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass
 # (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
 #export POSTGRES_DATA_DIR='${DATA_PERSIST_ROOT}/frontend_persist'
 
-# Mount point on host machine for the user workspace shared by the cowbird and jupyterhub components
-# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
-#export USER_WORKSPACES='${DATA_PERSIST_ROOT}/user_workspaces'
-
 # Mount point on host machine for weaver to store data from its mongodb server
 # (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
 #export WEAVER_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_weaver_persist'

From efa146cd81c153e0ecc9e6524c61b26c1682884d Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Mon, 25 Sep 2023 13:40:46 -0400
Subject: [PATCH 6/8] remove WEAVER_WPS_PRIVATE_DIR

---
 CHANGES.md                                           | 1 -
 birdhouse/components/weaver/default.env              | 1 -
 birdhouse/components/weaver/docker-compose-extra.yml | 1 -
 birdhouse/env.local.example                          | 4 ----
 4 files changed, 7 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index d6402c5e4..581f28c81 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -27,7 +27,6 @@
   - `COWBIRD_MONGODB_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/mongodb_cowbird_persist`)
   - `POSTGRES_DATA_DIR` (default `${DATA_PERSIST_ROOT}/frontend_persist`)
   - `WEAVER_MONGODB_DATA_DIR` (default `${DATA_PERSIST_ROOT}/mongodb_weaver_persist`)
-  - `WEAVER_WPS_PRIVATE_DIR` (default `${DATA_PERSIST_ROOT}/wps_private`)
 
   The following variable is also added which is another location on disk where files that may contain links
   are placed. Because the links need to be mounted together in order to resolve properly, the subdirectories
diff --git a/birdhouse/components/weaver/default.env b/birdhouse/components/weaver/default.env
index b1e65faa0..3c6b72bec 100644
--- a/birdhouse/components/weaver/default.env
+++ b/birdhouse/components/weaver/default.env
@@ -98,7 +98,6 @@ export WEAVER_WPS_PROVIDERS_RETRY_AFTER=5
 
 export WEAVER_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_weaver_persist'
 
-export WEAVER_WPS_PRIVATE_DIR='${DATA_PERSIST_ROOT}/wps_private'
 
 export DELAYED_EVAL="
   $DELAYED_EVAL
diff --git a/birdhouse/components/weaver/docker-compose-extra.yml b/birdhouse/components/weaver/docker-compose-extra.yml
index f56454af7..179be7c65 100644
--- a/birdhouse/components/weaver/docker-compose-extra.yml
+++ b/birdhouse/components/weaver/docker-compose-extra.yml
@@ -63,7 +63,6 @@ services:
       - weaver-mongodb
     volumes:
       #- wps_private:/data/wps_private
-      - ${WEAVER_WPS_PRIVATE_DIR}:/data/wps_private
       - ./components/weaver/config/weaver/weaver.ini:/opt/local/src/weaver/config/weaver.ini:ro
       - ./components/weaver/config/weaver/request_options.yml:/opt/local/src/weaver/config/request_options.yml:ro
       - ./components/weaver/celery-healthcheck:/opt/local/bin/weaver/celery-healthcheck:ro
diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example
index 4f253e4ce..df68cb4b7 100644
--- a/birdhouse/env.local.example
+++ b/birdhouse/env.local.example
@@ -483,10 +483,6 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass
 # (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
 #export WEAVER_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_weaver_persist'
 
-# Mount point on host machine for weaver to store private wps data
-# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
-#WEAVER_WPS_PRIVATE_DIR='${DATA_PERSIST_ROOT}/wps_private'
-
 
 #############################################################################
 # Monitoring components configs

From 601749ea29663e69038542903f48f919a0ba48ba Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Mon, 25 Sep 2023 15:32:17 -0400
Subject: [PATCH 7/8] fix WPS_OUTPUTS_DIR delayed eval

---
 birdhouse/default.env | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/birdhouse/default.env b/birdhouse/default.env
index 2207f1c17..086885ca3 100644
--- a/birdhouse/default.env
+++ b/birdhouse/default.env
@@ -30,6 +30,7 @@ export DELAYED_EVAL="
   $DELAYED_EVAL
   PAVICS_FQDN_PUBLIC
   DATA_PERSIST_SHARED_ROOT
+  WPS_OUTPUTS_DIR
 "
 
 
@@ -59,4 +60,4 @@ export DEFAULT_CONF_DIRS='
 export USER_WORKSPACE_UID=1000
 export USER_WORKSPACE_GID=1000
 
-export WPS_OUTPUTS_DIR="${DATA_PERSIST_SHARED_ROOT}/wps_outputs"
+export WPS_OUTPUTS_DIR='${DATA_PERSIST_SHARED_ROOT}/wps_outputs'

From 59a7cdba87034f1a36d1bcb687462345daca4d27 Mon Sep 17 00:00:00 2001
From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com>
Date: Wed, 27 Sep 2023 09:25:38 -0400
Subject: [PATCH 8/8] =?UTF-8?q?Bump=20version:=201.33.1=20=E2=86=92=201.33?=
 =?UTF-8?q?.2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg                                          | 6 +++---
 CHANGES.md                                                | 5 +++++
 Makefile                                                  | 2 +-
 README.rst                                                | 8 ++++----
 RELEASE.txt                                               | 2 +-
 .../config/canarie-api/docker_configuration.py.template   | 8 ++++----
 docs/source/conf.py                                       | 4 ++--
 7 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 1bc457f47..5f01eac65 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.33.1
+current_version = 1.33.2
 commit = True
 tag = False
 tag_name = {new_version}
@@ -30,11 +30,11 @@ search = {current_version}
 replace = {new_version}
 
 [bumpversion:file:RELEASE.txt]
-search = {current_version} 2023-09-25T15:46:44Z
+search = {current_version} 2023-09-27T13:25:37Z
 replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ}
 
 [bumpversion:part:releaseTime]
-values = 2023-09-25T15:46:44Z
+values = 2023-09-27T13:25:37Z
 
 [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template]
 search = 'version': '{current_version}'
diff --git a/CHANGES.md b/CHANGES.md
index 581f28c81..46d5d37e7 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -15,6 +15,11 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
+[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+
+[1.33.2](https://github.com/bird-house/birdhouse-deploy/tree/1.33.2) (2023-09-27)
+------------------------------------------------------------------------------------------------------------------
+
 ## Changes
 - Make bind-mount locations more flexible
 
diff --git a/Makefile b/Makefile
index 654070853..4a6129200 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 # Generic variables
 override SHELL       := bash
 override APP_NAME    := birdhouse-deploy
-override APP_VERSION := 1.33.1
+override APP_VERSION := 1.33.2
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")
diff --git a/README.rst b/README.rst
index 61aa8d8fb..fd82a64da 100644
--- a/README.rst
+++ b/README.rst
@@ -14,13 +14,13 @@ for a full-fledged production platform.
     * - releases
       - | |latest-version| |commits-since|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.33.1.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.33.2.svg
     :alt: Commits since latest release
-    :target: https://github.com/bird-house/birdhouse-deploy/compare/1.33.1...master
+    :target: https://github.com/bird-house/birdhouse-deploy/compare/1.33.2...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-1.33.1-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-1.33.2-blue.svg?style=flat
     :alt: Latest Tag
-    :target: https://github.com/bird-house/birdhouse-deploy/tree/1.33.1
+    :target: https://github.com/bird-house/birdhouse-deploy/tree/1.33.2
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
     :alt: ReadTheDocs Build Status (latest version)
diff --git a/RELEASE.txt b/RELEASE.txt
index eec449d9b..bd810f2e0 100644
--- a/RELEASE.txt
+++ b/RELEASE.txt
@@ -1 +1 @@
-1.33.1 2023-09-25T15:46:44Z
+1.33.2 2023-09-27T13:25:37Z
diff --git a/birdhouse/config/canarie-api/docker_configuration.py.template b/birdhouse/config/canarie-api/docker_configuration.py.template
index e0398de62..3bd42324b 100644
--- a/birdhouse/config/canarie-api/docker_configuration.py.template
+++ b/birdhouse/config/canarie-api/docker_configuration.py.template
@@ -109,8 +109,8 @@ SERVICES = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '1.33.1',
-            'releaseTime': '2023-09-25T15:46:44Z',
+            'version': '1.33.2',
+            'releaseTime': '2023-09-27T13:25:37Z',
             'institution': 'Ouranos',
             'researchSubject': 'Climatology',
             'supportEmail': '${SUPPORT_EMAIL}',
@@ -142,8 +142,8 @@ PLATFORMS = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '1.33.1',
-            'releaseTime': '2023-09-25T15:46:44Z',
+            'version': '1.33.2',
+            'releaseTime': '2023-09-27T13:25:37Z',
             'institution': 'Ouranos',
             'researchSubject': 'Climatology',
             'supportEmail': '${SUPPORT_EMAIL}',
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 82b3516a3..12f8c62bc 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -69,9 +69,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.33.1'
+version = '1.33.2'
 # The full version, including alpha/beta/rc tags.
-release = '1.33.1'
+release = '1.33.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.