diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1bc457f47..5f01eac65 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.33.1 +current_version = 1.33.2 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2023-09-25T15:46:44Z +search = {current_version} 2023-09-27T13:25:37Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2023-09-25T15:46:44Z +values = 2023-09-27T13:25:37Z [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/CHANGES.md b/CHANGES.md index 18687b67c..46d5d37e7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,35 @@ [//]: # (list changes here, using '-' for each new entry, remove this when items are added) +[1.33.2](https://github.com/bird-house/birdhouse-deploy/tree/1.33.2) (2023-09-27) +------------------------------------------------------------------------------------------------------------------ + +## Changes +- Make bind-mount locations more flexible + + Previously, most bind mount locations on the host machine were subdirectories of the folder specified by the + `DATA_PERSIST_ROOT` environment variable (`/data` by default). This change allows the user to set custom locations + for the following additional variables, so that they don't need to be all under the same common directory. + + - `LOGROTATE_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/logrotate`) + - `MONGODB_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/mongodb_persist`) + - `COWBIRD_MONGODB_DATA_DIR` (default: `${DATA_PERSIST_ROOT}/mongodb_cowbird_persist`) + - `POSTGRES_DATA_DIR` (default `${DATA_PERSIST_ROOT}/frontend_persist`) + - `WEAVER_MONGODB_DATA_DIR` (default `${DATA_PERSIST_ROOT}/mongodb_weaver_persist`) + + The following variable is also added which is another location on disk where files that may contain links + are placed. Because the links need to be mounted together in order to resolve properly, the subdirectories + of this directory are not configurable: + + - `DATA_PERSIST_SHARED_ROOT` (default: same as `DATA_PERSIST_ROOT`) + + The following variables now create subdirectories under `DATA_PERSIST_SHARED_ROOT` (previously they were + created under `DATA_PERSIST_ROOT` by default): + + - `USER_WORKSPACES` (default `user_workspaces`) + - `WEAVER_WPS_OUTPUTS_DIR` (default `wps_outputs/weaver`) + + [1.33.1](https://github.com/bird-house/birdhouse-deploy/tree/1.33.1) (2023-09-25) ------------------------------------------------------------------------------------------------------------------ diff --git a/Makefile b/Makefile index 654070853..4a6129200 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 1.33.1 +override APP_VERSION := 1.33.2 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index 61aa8d8fb..fd82a64da 100644 --- a/README.rst +++ b/README.rst @@ -14,13 +14,13 @@ for a full-fledged production platform. * - releases - | |latest-version| |commits-since| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.33.1.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.33.2.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/1.33.1...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/1.33.2...master -.. |latest-version| image:: https://img.shields.io/badge/tag-1.33.1-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-1.33.2-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/1.33.1 + :target: https://github.com/bird-house/birdhouse-deploy/tree/1.33.2 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index eec449d9b..bd810f2e0 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -1.33.1 2023-09-25T15:46:44Z +1.33.2 2023-09-27T13:25:37Z diff --git a/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml b/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml index af5307d01..6c98a641b 100644 --- a/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml +++ b/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml @@ -13,7 +13,7 @@ services: - ${JUPYTERHUB_USER_DATA_DIR}:${JUPYTERHUB_USER_DATA_DIR} jupyterhub: environment: - WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES} + WORKSPACE_DIR: ${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES} PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR: ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR} volumes: - - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_ROOT}/${USER_WORKSPACES}" + - "${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}" diff --git a/birdhouse/components/cowbird/default.env b/birdhouse/components/cowbird/default.env index 70bd64714..af633a900 100644 --- a/birdhouse/components/cowbird/default.env +++ b/birdhouse/components/cowbird/default.env @@ -39,7 +39,7 @@ export COWBIRD_MONGODB_VERSION=5.0 export COWBIRD_MONGODB_HOST=cowbird-mongodb export COWBIRD_MONGODB_PORT=27017 -# Subdirectory containing the user workspaces used by Cowbird +# Subdirectory of DATA_PERSIST_SHARED_ROOT containing the user workspaces used by Cowbird export USER_WORKSPACES="user_workspaces" # Subdirectory containing the hardlinks to the public WPS outputs data @@ -50,6 +50,13 @@ export PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR="public/wps_outputs" # Default name for the secure-data-proxy service from Magpie. export SECURE_DATA_PROXY_NAME="secure-data-proxy" +COWBIRD_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_cowbird_persist' + +DELAYED_EVAL=" + $DELAYED_EVAL + COWBIRD_MONGODB_DATA_DIR +" + # this dependency is only required if the mongo instance is the one provided in config/mongodb. # (include this for Cowbird<2.0.0) #COMPONENT_DEPENDENCIES=" diff --git a/birdhouse/components/cowbird/docker-compose-extra.yml b/birdhouse/components/cowbird/docker-compose-extra.yml index 4a905b878..a6b28bd77 100644 --- a/birdhouse/components/cowbird/docker-compose-extra.yml +++ b/birdhouse/components/cowbird/docker-compose-extra.yml @@ -27,8 +27,9 @@ services: SECURE_DATA_PROXY_NAME: ${SECURE_DATA_PROXY_NAME} # Note that WPS_OUTPUTS_DIR and WORKSPACE_DIR must both point to paths from the same volume. # This is to allow the creation of hardlinks between the wpsoutputs and the user workspace. + # By default, both are mounted under the "${DATA_PERSIST_SHARED_ROOT}" volume. WPS_OUTPUTS_DIR: ${WPS_OUTPUTS_DIR} - WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES} + WORKSPACE_DIR: ${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES} links: - cowbird-mongodb networks: @@ -41,7 +42,7 @@ services: - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini # even if not running tasks here, they must be registered to send them off to the right place! - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py - - "${DATA_PERSIST_ROOT}:${DATA_PERSIST_ROOT}" + - "${DATA_PERSIST_SHARED_ROOT}:${DATA_PERSIST_SHARED_ROOT}" restart: always logging: *default-logging @@ -57,7 +58,7 @@ services: # root user COWBIRD_FILESYSTEM_ADMIN_UID: 0 COWBIRD_FILESYSTEM_ADMIN_GID: 0 - WORKSPACE_DIR: ${DATA_PERSIST_ROOT}/${USER_WORKSPACES} + WORKSPACE_DIR: ${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES} links: - cowbird-mongodb networks: @@ -70,7 +71,7 @@ services: - ./components/cowbird/config/cowbird/config.yml:/opt/local/src/cowbird/config/cowbird.yml - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py - - "${DATA_PERSIST_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_ROOT}/${USER_WORKSPACES}" + - "${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}" restart: always logging: *default-logging @@ -81,7 +82,7 @@ services: networks: - cowbird-mongodb volumes: - - ${DATA_PERSIST_ROOT}/mongodb_cowbird_persist:/data/db + - ${COWBIRD_MONGODB_DATA_DIR}:/data/db restart: always logging: *default-logging diff --git a/birdhouse/components/scheduler/config.yml.template b/birdhouse/components/scheduler/config.yml.template index 81d863c76..31a056593 100644 --- a/birdhouse/components/scheduler/config.yml.template +++ b/birdhouse/components/scheduler/config.yml.template @@ -6,7 +6,7 @@ dockerargs: >- --rm --name logrotate --volume /var/log/PAVICS:/var/log/PAVICS:rw - --volume ${DATA_PERSIST_ROOT}/logrotate:/var/lib:rw + --volume ${LOGROTATE_DATA_DIR}:/var/lib:rw --volume ${COMPOSE_DIR}/deployment/PAVICS-deploy.logrotate:/etc/logrotate.conf.orig:ro image: 'stakater/logrotate:3.13.0' diff --git a/birdhouse/components/scheduler/default.env b/birdhouse/components/scheduler/default.env index bc6339829..1261f3c08 100644 --- a/birdhouse/components/scheduler/default.env +++ b/birdhouse/components/scheduler/default.env @@ -17,6 +17,13 @@ export AUTODEPLOY_PLATFORM_FREQUENCY="7 5 * * *" # Hourly export AUTODEPLOY_NOTEBOOK_FREQUENCY="@hourly" +export LOGROTATE_DATA_DIR='${DATA_PERSIST_ROOT}/logrotate' + +export DELAYED_EVAL=" + $DELAYED_EVAL + LOGROTATE_DATA_DIR +" + if [ -L "$BIRDHOUSE_LOCAL_ENV" ]; then # Add volume-mount if env.local is a symlink and not a real file, # so the symlink can resolve. @@ -28,6 +35,11 @@ if [ -L "$BIRDHOUSE_LOCAL_ENV" ]; then fi # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here +VARS=" + $VARS + \$LOGROTATE_DATA_DIR +" + OPTIONAL_VARS=" $OPTIONAL_VARS \$AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES diff --git a/birdhouse/components/weaver/default.env b/birdhouse/components/weaver/default.env index fd58cf85b..3c6b72bec 100644 --- a/birdhouse/components/weaver/default.env +++ b/birdhouse/components/weaver/default.env @@ -96,7 +96,12 @@ export WEAVER_WPS_PROVIDERS_RETRY_COUNT=5 # control interval time between retries (duration in seconds, counts toward maximum timeout) export WEAVER_WPS_PROVIDERS_RETRY_AFTER=5 +export WEAVER_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_weaver_persist' + + export DELAYED_EVAL=" $DELAYED_EVAL WEAVER_WPS_OUTPUTS_DIR + WEAVER_MONGODB_DATA_DIR + WEAVER_WPS_PRIVATE_DATA_DIR " diff --git a/birdhouse/components/weaver/docker-compose-extra.yml b/birdhouse/components/weaver/docker-compose-extra.yml index 7038c4e97..179be7c65 100644 --- a/birdhouse/components/weaver/docker-compose-extra.yml +++ b/birdhouse/components/weaver/docker-compose-extra.yml @@ -63,7 +63,6 @@ services: - weaver-mongodb volumes: #- wps_private:/data/wps_private - - ${DATA_PERSIST_ROOT}/wps_private:/data/wps_private - ./components/weaver/config/weaver/weaver.ini:/opt/local/src/weaver/config/weaver.ini:ro - ./components/weaver/config/weaver/request_options.yml:/opt/local/src/weaver/config/request_options.yml:ro - ./components/weaver/celery-healthcheck:/opt/local/bin/weaver/celery-healthcheck:ro @@ -97,7 +96,7 @@ services: networks: - weaver-mongodb volumes: - - ${DATA_PERSIST_ROOT}/mongodb_weaver_persist:/data/db + - ${WEAVER_MONGODB_DATA_DIR}/mongodb_weaver_persist:/data/db restart: always logging: *default-logging diff --git a/birdhouse/config/canarie-api/docker_configuration.py.template b/birdhouse/config/canarie-api/docker_configuration.py.template index e0398de62..3bd42324b 100644 --- a/birdhouse/config/canarie-api/docker_configuration.py.template +++ b/birdhouse/config/canarie-api/docker_configuration.py.template @@ -109,8 +109,8 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.33.1', - 'releaseTime': '2023-09-25T15:46:44Z', + 'version': '1.33.2', + 'releaseTime': '2023-09-27T13:25:37Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', @@ -142,8 +142,8 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.33.1', - 'releaseTime': '2023-09-25T15:46:44Z', + 'version': '1.33.2', + 'releaseTime': '2023-09-27T13:25:37Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', diff --git a/birdhouse/config/mongodb/default.env b/birdhouse/config/mongodb/default.env new file mode 100644 index 000000000..e031e852c --- /dev/null +++ b/birdhouse/config/mongodb/default.env @@ -0,0 +1,12 @@ +# All env in this default.env can be overridden by env.local. + +# All env in this default.env must NOT depend on any other env. If they do, they +# must use single quotes to avoid early expansion before overrides in env.local +# are applied and must be added to the list of DELAYED_EVAL. + +export MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_persist' + +export DELAYED_EVAL=" + $DELAYED_EVAL + MONGODB_DATA_DIR +" diff --git a/birdhouse/config/mongodb/docker-compose-extra.yml b/birdhouse/config/mongodb/docker-compose-extra.yml index fedf54a47..c7f96a198 100644 --- a/birdhouse/config/mongodb/docker-compose-extra.yml +++ b/birdhouse/config/mongodb/docker-compose-extra.yml @@ -14,7 +14,7 @@ services: networks: - mongodb volumes: - - ${DATA_PERSIST_ROOT}/mongodb_persist:/data/db + - ${MONGODB_DATA_DIR}:/data/db # Mongodb crash with permission denied errors if the command is not overridden like this command: bash -c 'chown -R mongodb:mongodb /data && chmod -R 755 /data && mongod' restart: always diff --git a/birdhouse/config/postgres/default.env b/birdhouse/config/postgres/default.env index e5f8d44d6..38496f27c 100644 --- a/birdhouse/config/postgres/default.env +++ b/birdhouse/config/postgres/default.env @@ -4,6 +4,13 @@ # must use single quotes to avoid early expansion before overrides in env.local # are applied and must be added to the list of DELAYED_EVAL. +export POSTGRES_DATA_DIR='${DATA_PERSIST_ROOT}/frontend_persist' + +export DELAYED_EVAL=" + $DELAYED_EVAL + POSTGRES_DATA_DIR +" + # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here VARS=" $VARS diff --git a/birdhouse/config/postgres/docker-compose-extra.yml b/birdhouse/config/postgres/docker-compose-extra.yml index d65bf977f..db004bf7f 100644 --- a/birdhouse/config/postgres/docker-compose-extra.yml +++ b/birdhouse/config/postgres/docker-compose-extra.yml @@ -13,7 +13,7 @@ services: container_name: postgres # ports: # should not be exposed directly, enable 'optional-components/database-external-ports' as needed volumes: - - ${DATA_PERSIST_ROOT}/frontend_persist:/var/lib/postgresql/data/pgdata + - ${POSTGRES_DATA_DIR}:/var/lib/postgresql/data/pgdata - ./config/postgres/postgres-setup.sh:/postgres-setup.sh environment: PGDATA: /var/lib/postgresql/data/pgdata diff --git a/birdhouse/default.env b/birdhouse/default.env index 3504d84c0..086885ca3 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -11,6 +11,11 @@ export BASH_IMAGE="bash:5.1.4" # Root directory under which all data persistence should be nested under export DATA_PERSIST_ROOT="/data" +# shellcheck disable=SC2016 +# Root directory for all files that are persisted on disk and may contain links (ie. the files +# are "shared" between subdirectories). +export DATA_PERSIST_SHARED_ROOT='${DATA_PERSIST_ROOT}' + # Log directory used for the various scheduler tasks # TODO: use this variable for other references of the log path (only used in the pavics-jupyter-base's .env file for now) export PAVICS_LOG_DIR=/var/log/PAVICS @@ -24,6 +29,8 @@ export PAVICS_FQDN_PUBLIC='${PAVICS_FQDN}' export DELAYED_EVAL=" $DELAYED_EVAL PAVICS_FQDN_PUBLIC + DATA_PERSIST_SHARED_ROOT + WPS_OUTPUTS_DIR " @@ -53,4 +60,4 @@ export DEFAULT_CONF_DIRS=' export USER_WORKSPACE_UID=1000 export USER_WORKSPACE_GID=1000 -export WPS_OUTPUTS_DIR="${DATA_PERSIST_ROOT}/wps_outputs" +export WPS_OUTPUTS_DIR='${DATA_PERSIST_SHARED_ROOT}/wps_outputs' diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index bbef17ae7..df68cb4b7 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -9,6 +9,10 @@ # Override data persistence root directory # export DATA_PERSIST_ROOT="/data/custom/path" # otherwise use value of 'default.env', directory must exist +# Root directory for all files that are persisted on disk and may contain links (ie. the files +# are "shared" between subdirectories). This means that the subdirectory structure is fixed. +#export DATA_PERSIST_SHARED_ROOT='${DATA_PERSIST_ROOT}' # otherwise use the value from 'default.env', must exist + export SSL_CERTIFICATE="/path/to/ssl/cert.pem" # *absolute* path to the nginx ssl certificate, path and key bundle export PAVICS_FQDN="hostname.domainname" # Fully qualified domain name of this Pavics installation export DOC_URL="https://www.example.com/" # URL where /doc gets redirected @@ -213,6 +217,10 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass # . $COMPOSE_DIR/components/scheduler/deploy_data_job.env #fi +# Mount point on host machine for the scheduler to write data from log rotations +# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env') +#export LOGROTATE_DATA_DIR='${DATA_PERSIST_ROOT}/logrotate' + # Content of "location /" in file config/proxy/conf.d/all-services.include.template # Useful to have a custom homepage. # Default: @@ -459,6 +467,22 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass # See raven/default.env for more info. #export RAVEN_GEO_URL="https://${PAVICS_FQDN}/geoserver/" +# Mount point on host machine to store mongodb server data +# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env') +#export MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_persist' + +# Mount point on host machine for cowbird to store data from its mongodb server +# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env') +#export COWBIRD_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_cowbird_persist' + +# Mount point on host machine to store postgres server data +# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env') +#export POSTGRES_DATA_DIR='${DATA_PERSIST_ROOT}/frontend_persist' + +# Mount point on host machine for weaver to store data from its mongodb server +# (note: if using 'DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env') +#export WEAVER_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_weaver_persist' + ############################################################################# # Monitoring components configs diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index 8e8644b4a..1858de2f3 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -22,6 +22,7 @@ VARS=' $DOC_URL $SUPPORT_EMAIL $DATA_PERSIST_ROOT + $DATA_PERSIST_SHARED_ROOT ' # list of vars to be substituted in template but they do not have to be set in env.local diff --git a/docs/source/conf.py b/docs/source/conf.py index 82b3516a3..12f8c62bc 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,9 +69,9 @@ # built documents. # # The short X.Y version. -version = '1.33.1' +version = '1.33.2' # The full version, including alpha/beta/rc tags. -release = '1.33.1' +release = '1.33.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages.