From 3c0a7c301aa654666556ed8ece22e498a19ce9b6 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Wed, 19 Jul 2023 12:21:12 -0400 Subject: [PATCH 01/20] Change locations for data served by thredds --- birdhouse/config/thredds/catalog.xml.template | 4 ++-- birdhouse/config/thredds/default.env | 9 +++++++++ birdhouse/config/thredds/docker-compose-extra.yml | 9 ++++----- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/birdhouse/config/thredds/catalog.xml.template b/birdhouse/config/thredds/catalog.xml.template index 974842164..0f2379da5 100644 --- a/birdhouse/config/thredds/catalog.xml.template +++ b/birdhouse/config/thredds/catalog.xml.template @@ -14,7 +14,7 @@ - + all @@ -31,7 +31,7 @@ - + all diff --git a/birdhouse/config/thredds/default.env b/birdhouse/config/thredds/default.env index 6e8e6de35..b63d62eee 100644 --- a/birdhouse/config/thredds/default.env +++ b/birdhouse/config/thredds/default.env @@ -6,6 +6,15 @@ export THREDDS_DOCKER=pavics/thredds-docker export THREDDS_IMAGE="${THREDDS_DOCKER}:${THREDDS_VERSION}" export THREDDS_ORGANIZATION="Birdhouse" +# All default values for data locations are set to ensure backward compatibility +export STAC_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER="/pavics-ncml" +export MISC_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER="/pavics-data" +export STAC_DATA_LOCATION_NAME="Datasets" +export MISC_DATA_LOCATION_NAME="Birdhouse" +export STAC_DATA_LOCATION_NAME_lower=$(echo "$STAC_DATA_LOCATION_NAME" | tr '[:upper:]' '[:lower:]') +export MISC_DATA_LOCATION_NAME_lower=$(echo "$MISC_DATA_LOCATION_NAME" | tr '[:upper:]' '[:lower:]') + + # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here VARS=" $VARS diff --git a/birdhouse/config/thredds/docker-compose-extra.yml b/birdhouse/config/thredds/docker-compose-extra.yml index 8ae7735e9..1b05e3917 100644 --- a/birdhouse/config/thredds/docker-compose-extra.yml +++ b/birdhouse/config/thredds/docker-compose-extra.yml @@ -1,7 +1,6 @@ version: "3.4" -x-logging: - &default-logging +x-logging: &default-logging driver: "json-file" options: max-size: "50m" @@ -19,8 +18,8 @@ services: PAVICS_FQDN_PUBLIC: $PAVICS_FQDN_PUBLIC volumes: - thredds_persistence:/usr/local/tomcat/content/thredds - - ${DATA_PERSIST_ROOT}/datasets:/pavics-data - - ${DATA_PERSIST_ROOT}/ncml:/pavics-ncml + - ${MISC_DATA_LOCATION_FOR_THREDDS_ON_HOST}:${MISC_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER} + - ${STAC_DATA_LOCATION_FOR_THREDDS_ON_HOST}:${STAC_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER} - wps_outputs:/pavics-data/wps_outputs - ./config/thredds/catalog.xml:/usr/local/tomcat/content/thredds/catalog.xml:ro - ./config/thredds/threddsConfig.xml:/usr/local/tomcat/content/thredds/threddsConfig.xml:ro @@ -35,7 +34,7 @@ services: "CMD", "curl", "--fail", - "http://localhost:8080${TWITCHER_PROTECTED_PATH}/thredds/catalog.html", + "http://localhost:8080${TWITCHER_PROTECTED_PATH}/thredds/catalog.html" ] volumes: From 9ac853fa5f92d8b5160900e2da61de83a9f8b817 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Wed, 19 Jul 2023 12:27:33 -0400 Subject: [PATCH 02/20] forgot to add this file in last commit --- birdhouse/default.env | 3 +++ 1 file changed, 3 insertions(+) diff --git a/birdhouse/default.env b/birdhouse/default.env index 6fd3456ae..06e081ec5 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -10,6 +10,9 @@ export BASH_IMAGE="bash:5.1.4" # Root directory under which all data persistence should be nested under export DATA_PERSIST_ROOT="/data" +export STAC_DATA_LOCATION_FOR_THREDDS_ON_HOST=${DATA_PERSIST_ROOT}/ncml # this default is for backward compatibility +export MISC_DATA_LOCATION_FOR_THREDDS_ON_HOST=${DATA_PERSIST_ROOT}/datasets # this default is for backward compatibility + # Log directory used for the various scheduler tasks # TODO: use this variable for other references of the log path (only used in the pavics-jupyter-base's .env file for now) From f809ba6a94c993a6e10d0b5f70efebede766e5bc Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Tue, 8 Aug 2023 14:55:48 -0400 Subject: [PATCH 03/20] Renaming variables to those agreed upon in PR --- birdhouse/config/thredds/catalog.xml.template | 4 ++-- birdhouse/config/thredds/default.env | 12 ++++++------ birdhouse/config/thredds/docker-compose-extra.yml | 4 ++-- birdhouse/default.env | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/birdhouse/config/thredds/catalog.xml.template b/birdhouse/config/thredds/catalog.xml.template index 0f2379da5..e4e27259f 100644 --- a/birdhouse/config/thredds/catalog.xml.template +++ b/birdhouse/config/thredds/catalog.xml.template @@ -14,7 +14,7 @@ - + all @@ -31,7 +31,7 @@ - + all diff --git a/birdhouse/config/thredds/default.env b/birdhouse/config/thredds/default.env index b63d62eee..c081ef5ca 100644 --- a/birdhouse/config/thredds/default.env +++ b/birdhouse/config/thredds/default.env @@ -7,12 +7,12 @@ export THREDDS_IMAGE="${THREDDS_DOCKER}:${THREDDS_VERSION}" export THREDDS_ORGANIZATION="Birdhouse" # All default values for data locations are set to ensure backward compatibility -export STAC_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER="/pavics-ncml" -export MISC_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER="/pavics-data" -export STAC_DATA_LOCATION_NAME="Datasets" -export MISC_DATA_LOCATION_NAME="Birdhouse" -export STAC_DATA_LOCATION_NAME_lower=$(echo "$STAC_DATA_LOCATION_NAME" | tr '[:upper:]' '[:lower:]') -export MISC_DATA_LOCATION_NAME_lower=$(echo "$MISC_DATA_LOCATION_NAME" | tr '[:upper:]' '[:lower:]') +export DATA_LOCATION_FOR_THREDDS_ON_CONTAINER="/pavics-ncml" +export RAW_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER="/pavics-data" +export DATA_LOCATION_NAME="Datasets" +export RAW_DATA_LOCATION_NAME="Birdhouse" +export DATA_LOCATION_NAME_lower=$(echo "$DATA_LOCATION_NAME" | tr '[:upper:]' '[:lower:]') +export RAW_DATA_LOCATION_NAME_lower=$(echo "$RAW_DATA_LOCATION_NAME" | tr '[:upper:]' '[:lower:]') # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here diff --git a/birdhouse/config/thredds/docker-compose-extra.yml b/birdhouse/config/thredds/docker-compose-extra.yml index 1b05e3917..fcceb6631 100644 --- a/birdhouse/config/thredds/docker-compose-extra.yml +++ b/birdhouse/config/thredds/docker-compose-extra.yml @@ -18,8 +18,8 @@ services: PAVICS_FQDN_PUBLIC: $PAVICS_FQDN_PUBLIC volumes: - thredds_persistence:/usr/local/tomcat/content/thredds - - ${MISC_DATA_LOCATION_FOR_THREDDS_ON_HOST}:${MISC_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER} - - ${STAC_DATA_LOCATION_FOR_THREDDS_ON_HOST}:${STAC_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER} + - ${RAW_DATA_LOCATION_FOR_THREDDS_ON_HOST}:${RAW_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER} + - ${DATA_LOCATION_FOR_THREDDS_ON_HOST}:${DATA_LOCATION_FOR_THREDDS_ON_CONTAINER} - wps_outputs:/pavics-data/wps_outputs - ./config/thredds/catalog.xml:/usr/local/tomcat/content/thredds/catalog.xml:ro - ./config/thredds/threddsConfig.xml:/usr/local/tomcat/content/thredds/threddsConfig.xml:ro diff --git a/birdhouse/default.env b/birdhouse/default.env index 06e081ec5..34df1b1fa 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -10,8 +10,8 @@ export BASH_IMAGE="bash:5.1.4" # Root directory under which all data persistence should be nested under export DATA_PERSIST_ROOT="/data" -export STAC_DATA_LOCATION_FOR_THREDDS_ON_HOST=${DATA_PERSIST_ROOT}/ncml # this default is for backward compatibility -export MISC_DATA_LOCATION_FOR_THREDDS_ON_HOST=${DATA_PERSIST_ROOT}/datasets # this default is for backward compatibility +export DATA_LOCATION_FOR_THREDDS_ON_HOST=${DATA_PERSIST_ROOT}/ncml # this default is for backward compatibility +export RAW_DATA_LOCATION_FOR_THREDDS_ON_HOST=${DATA_PERSIST_ROOT}/datasets # this default is for backward compatibility # Log directory used for the various scheduler tasks From 317d96c39db7a6d79d1568a7094441ccdedc55ae Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Tue, 8 Aug 2023 15:53:03 -0400 Subject: [PATCH 04/20] Change "birdhouse" in URLs to RAW_DATA_LOCATION_NAME_lower --- .../cowbird/config/cowbird/config.yml.template | 12 ++++++------ .../canarie-api/canarie_api_monitoring.py.template | 2 +- birdhouse/config/thredds/entrypointwrapper | 2 +- .../catalog/catalog.cfg.template | 2 +- .../canarie-api/canarie_api_monitoring.py.template | 2 +- .../secure-thredds/config/magpie/permissions.cfg | 12 ++++++------ .../testthredds/entrypointwrapper | 2 +- birdhouse/scripts/deprecated/trigger-pavicscrawler | 2 +- birdhouse/scripts/get-components-json.include.sh | 2 +- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/birdhouse/components/cowbird/config/cowbird/config.yml.template b/birdhouse/components/cowbird/config/cowbird/config.yml.template index 9e32de893..cde4ba622 100644 --- a/birdhouse/components/cowbird/config/cowbird/config.yml.template +++ b/birdhouse/components/cowbird/config/cowbird/config.yml.template @@ -181,10 +181,10 @@ sync_permissions: - name: "{outputID}" type: route thredds: - # /twitcher/ows/proxy/thredds/catalog/birdhouse/wps_outputs/weaver/catalog.html - # /twitcher/ows/proxy/thredds/catalog/birdhouse/wps_outputs/weaver/{public|}/catalog.html - # /twitcher/ows/proxy/thredds/catalog/birdhouse/wps_outputs/weaver/{public|}/{job-id}/catalog.html - # /twitcher/ows/proxy/thredds/catalog/birdhouse/wps_outputs/weaver/{public|}/{job-id}/{output-file} + # /twitcher/ows/proxy/thredds/catalog/${RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/catalog.html + # /twitcher/ows/proxy/thredds/catalog/${RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/catalog.html + # /twitcher/ows/proxy/thredds/catalog/${RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/{job-id}/catalog.html + # /twitcher/ows/proxy/thredds/catalog/${RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/{job-id}/{output-file} # note: paths start after ows-proxy portion extracted when Twitcher/Magpie resolve between each other thredds_wps_outputs: - name: thredds @@ -193,7 +193,7 @@ sync_permissions: # 'catalog' is the file/view format specifier for the rest of the path # - name: catalog # type: directory - - name: birdhouse + - name: ${RAW_DATA_LOCATION_NAME_lower} type: directory - name: wps_outputs type: directory @@ -210,7 +210,7 @@ sync_permissions: # 'catalog' is the file/view format specifier for the rest of the path # - name: catalog # type: directory - - name: birdhouse + - name: ${RAW_DATA_LOCATION_NAME_lower} type: directory - name: wps_outputs type: directory diff --git a/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template index c0ebc61f4..dd29af7f7 100644 --- a/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template @@ -29,7 +29,7 @@ SERVICES['renderer'] = { 'monitoring': { 'ncWMS': { 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/wms/birdhouse/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' + 'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/wms/${RAW_DATA_LOCATION_NAME_lower}/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' } }, } diff --git a/birdhouse/config/thredds/entrypointwrapper b/birdhouse/config/thredds/entrypointwrapper index a1d617823..d37e3afe6 100755 --- a/birdhouse/config/thredds/entrypointwrapper +++ b/birdhouse/config/thredds/entrypointwrapper @@ -4,7 +4,7 @@ CONF_FILE="/usr/local/tomcat/conf/server.xml" if ! grep ' relaxedQueryChars=' $CONF_FILE; then # allow angle bracket in query params, ex: - # https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/testdata/tasmax_day_BCCAQv2+ANUSPLIN300_BNU-ESM_historical+rcp85_r1i1p1_19500101-21001231.nc.ascii?tasmax[0:1:0][0:1:0][0:1:0] + # https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/${RAW_DATA_LOCATION_NAME_lower}/testdata/tasmax_day_BCCAQv2+ANUSPLIN300_BNU-ESM_historical+rcp85_r1i1p1_19500101-21001231.nc.ascii?tasmax[0:1:0][0:1:0][0:1:0] cp -v $CONF_FILE ${CONF_FILE}.bak.relaxedQueryChars.$$ sed -i 's//dev/null)" -eq 1 ]; then +if [ "$(echo "${BIRDHOUSE_DEPLOY_COMPONENTS_ROOT}" | grep -cE "/${RAW_DATA_LOCATION_NAME_lower}/?\$" 2>/dev/null)" -eq 1 ]; then BIRDHOUSE_DEPLOY_COMPONENTS_ROOT=. else BIRDHOUSE_DEPLOY_COMPONENTS_ROOT="${BIRDHOUSE_DEPLOY_COMPONENTS_ROOT}/.." From d1a82477772054e32f5feec122442c464e604c2d Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Tue, 8 Aug 2023 16:27:45 -0400 Subject: [PATCH 05/20] Changes to how newly defined variable are declared --- birdhouse/config/thredds/default.env | 27 +++++++++++++++++++++++---- birdhouse/default.env | 3 --- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/birdhouse/config/thredds/default.env b/birdhouse/config/thredds/default.env index c081ef5ca..4d66ad44d 100644 --- a/birdhouse/config/thredds/default.env +++ b/birdhouse/config/thredds/default.env @@ -7,18 +7,27 @@ export THREDDS_IMAGE="${THREDDS_DOCKER}:${THREDDS_VERSION}" export THREDDS_ORGANIZATION="Birdhouse" # All default values for data locations are set to ensure backward compatibility -export DATA_LOCATION_FOR_THREDDS_ON_CONTAINER="/pavics-ncml" -export RAW_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER="/pavics-data" -export DATA_LOCATION_NAME="Datasets" -export RAW_DATA_LOCATION_NAME="Birdhouse" +export DATA_LOCATION_FOR_THREDDS_ON_CONTAINER='/pavics-ncml' +export RAW_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER='/pavics-data' +export DATA_LOCATION_FOR_THREDDS_ON_HOST='${DATA_PERSIST_ROOT}/ncml' # this default is for backward compatibility +export RAW_DATA_LOCATION_FOR_THREDDS_ON_HOST='${DATA_PERSIST_ROOT}/datasets' # this default is for backward compatibility +export DATA_LOCATION_NAME='Datasets' # this default is for backward compatibility +export RAW_DATA_LOCATION_NAME='Birdhouse' # this default is for backward compatibility export DATA_LOCATION_NAME_lower=$(echo "$DATA_LOCATION_NAME" | tr '[:upper:]' '[:lower:]') export RAW_DATA_LOCATION_NAME_lower=$(echo "$RAW_DATA_LOCATION_NAME" | tr '[:upper:]' '[:lower:]') + # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here VARS=" $VARS \$CMIP5_THREDDS_ROOT + \$RAW_DATA_LOCATION_NAME + \$RAW_DATA_LOCATION_NAME_lower + \$RAW_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER + \$DATA_LOCATION_NAME + \$DATA_LOCATION_NAME_lower + \$DATA_LOCATION_FOR_THREDDS_ON_CONTAINER " OPTIONAL_VARS=" @@ -28,3 +37,13 @@ OPTIONAL_VARS=" \$THREDDS_DOCKER \$THREDDS_VERSION " + +export DELAYED_EVAL=" + $DELAYED_EVAL + DATA_LOCATION_NAME + DATA_LOCATION_FOR_THREDDS_ON_CONTAINER + DATA_LOCATION_FOR_THREDDS_ON_HOST + RAW_DATA_LOCATION_NAME + RAW_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER + RAW_DATA_LOCATION_FOR_THREDDS_ON_HOST +" diff --git a/birdhouse/default.env b/birdhouse/default.env index 34df1b1fa..6fd3456ae 100644 --- a/birdhouse/default.env +++ b/birdhouse/default.env @@ -10,9 +10,6 @@ export BASH_IMAGE="bash:5.1.4" # Root directory under which all data persistence should be nested under export DATA_PERSIST_ROOT="/data" -export DATA_LOCATION_FOR_THREDDS_ON_HOST=${DATA_PERSIST_ROOT}/ncml # this default is for backward compatibility -export RAW_DATA_LOCATION_FOR_THREDDS_ON_HOST=${DATA_PERSIST_ROOT}/datasets # this default is for backward compatibility - # Log directory used for the various scheduler tasks # TODO: use this variable for other references of the log path (only used in the pavics-jupyter-base's .env file for now) From c33b8ba73bdc4e28fffe72effd6eeedc98cec36c Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Tue, 15 Aug 2023 14:22:47 -0400 Subject: [PATCH 06/20] Adding changes to CHANGES.md --- CHANGES.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d8b8dc6f2..469f257fd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,10 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ -[//]: # (list changes here, using '-' for each new entry, remove this when items are added) +## Changes +- Flexible locations for data served by THREDDS + - This PR add the flexibility to configure the locations of THREDDS data on the host and the contained + - It also allows to configure the names of the top level THREDDS directories serving 'catalog' and 'raw' data [1.29.1](https://github.com/bird-house/birdhouse-deploy/tree/1.29.1) (2023-08-15) ------------------------------------------------------------------------------------------------------------------ From cf4e0af4ef7b296ea570df3d46fb4746566018c2 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Wed, 16 Aug 2023 11:39:52 -0400 Subject: [PATCH 07/20] modifying variables list in DELAYED_EVAL --- birdhouse/config/thredds/default.env | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/birdhouse/config/thredds/default.env b/birdhouse/config/thredds/default.env index 4d66ad44d..b56c67512 100644 --- a/birdhouse/config/thredds/default.env +++ b/birdhouse/config/thredds/default.env @@ -40,10 +40,8 @@ OPTIONAL_VARS=" export DELAYED_EVAL=" $DELAYED_EVAL - DATA_LOCATION_NAME - DATA_LOCATION_FOR_THREDDS_ON_CONTAINER + DATA_LOCATION_NAME_lower DATA_LOCATION_FOR_THREDDS_ON_HOST - RAW_DATA_LOCATION_NAME - RAW_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER + RAW_DATA_LOCATION_NAME_lower RAW_DATA_LOCATION_FOR_THREDDS_ON_HOST " From 3703362ada790db0f502729ef3d0775e2517877e Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Mon, 2 Oct 2023 16:25:01 -0400 Subject: [PATCH 08/20] changes requested by misha --- .../config/cowbird/config.yml.template | 12 +++---- birdhouse/config/thredds/catalog.xml.template | 4 +-- .../canarie_api_monitoring.py.template | 2 +- birdhouse/config/thredds/default.env | 36 +++++++++---------- .../config/thredds/docker-compose-extra.yml | 4 +-- birdhouse/config/thredds/entrypointwrapper | 2 +- .../catalog/catalog.cfg.template | 2 +- .../canarie_api_monitoring.py.template | 2 +- .../config/magpie/permissions.cfg | 12 +++---- .../testthredds/entrypointwrapper | 2 +- .../scripts/deprecated/trigger-pavicscrawler | 2 +- .../scripts/get-components-json.include.sh | 2 +- 12 files changed, 41 insertions(+), 41 deletions(-) diff --git a/birdhouse/components/cowbird/config/cowbird/config.yml.template b/birdhouse/components/cowbird/config/cowbird/config.yml.template index 8f18bc2f2..f572be8e2 100644 --- a/birdhouse/components/cowbird/config/cowbird/config.yml.template +++ b/birdhouse/components/cowbird/config/cowbird/config.yml.template @@ -187,10 +187,10 @@ sync_permissions: - name: "{outputID}" type: route thredds: - # /twitcher/ows/proxy/thredds/catalog/${RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/catalog.html - # /twitcher/ows/proxy/thredds/catalog/${RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/catalog.html - # /twitcher/ows/proxy/thredds/catalog/${RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/{job-id}/catalog.html - # /twitcher/ows/proxy/thredds/catalog/${RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/{job-id}/{output-file} + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/catalog.html + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/catalog.html + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/{job-id}/catalog.html + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/{job-id}/{output-file} # note: paths start after ows-proxy portion extracted when Twitcher/Magpie resolve between each other thredds_wps_outputs: - name: thredds @@ -199,7 +199,7 @@ sync_permissions: # 'catalog' is the file/view format specifier for the rest of the path # - name: catalog # type: directory - - name: ${RAW_DATA_LOCATION_NAME_lower} + - name: ${THREDDS_RAW_DATA_LOCATION_NAME_lower} type: directory - name: wps_outputs type: directory @@ -216,7 +216,7 @@ sync_permissions: # 'catalog' is the file/view format specifier for the rest of the path # - name: catalog # type: directory - - name: ${RAW_DATA_LOCATION_NAME_lower} + - name: ${THREDDS_RAW_DATA_LOCATION_NAME_lower} type: directory - name: wps_outputs type: directory diff --git a/birdhouse/config/thredds/catalog.xml.template b/birdhouse/config/thredds/catalog.xml.template index e4e27259f..f1d4f6886 100644 --- a/birdhouse/config/thredds/catalog.xml.template +++ b/birdhouse/config/thredds/catalog.xml.template @@ -14,7 +14,7 @@ - + all @@ -31,7 +31,7 @@ - + all diff --git a/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template index dd29af7f7..0ce359912 100644 --- a/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template @@ -29,7 +29,7 @@ SERVICES['renderer'] = { 'monitoring': { 'ncWMS': { 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/wms/${RAW_DATA_LOCATION_NAME_lower}/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' + 'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/wms/${THREDDS_RAW_DATA_LOCATION_NAME_lower}/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' } }, } diff --git a/birdhouse/config/thredds/default.env b/birdhouse/config/thredds/default.env index b56c67512..546e42a11 100644 --- a/birdhouse/config/thredds/default.env +++ b/birdhouse/config/thredds/default.env @@ -7,14 +7,14 @@ export THREDDS_IMAGE="${THREDDS_DOCKER}:${THREDDS_VERSION}" export THREDDS_ORGANIZATION="Birdhouse" # All default values for data locations are set to ensure backward compatibility -export DATA_LOCATION_FOR_THREDDS_ON_CONTAINER='/pavics-ncml' -export RAW_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER='/pavics-data' -export DATA_LOCATION_FOR_THREDDS_ON_HOST='${DATA_PERSIST_ROOT}/ncml' # this default is for backward compatibility -export RAW_DATA_LOCATION_FOR_THREDDS_ON_HOST='${DATA_PERSIST_ROOT}/datasets' # this default is for backward compatibility -export DATA_LOCATION_NAME='Datasets' # this default is for backward compatibility -export RAW_DATA_LOCATION_NAME='Birdhouse' # this default is for backward compatibility -export DATA_LOCATION_NAME_lower=$(echo "$DATA_LOCATION_NAME" | tr '[:upper:]' '[:lower:]') -export RAW_DATA_LOCATION_NAME_lower=$(echo "$RAW_DATA_LOCATION_NAME" | tr '[:upper:]' '[:lower:]') +export THREDDS_DATA_LOCATION_ON_CONTAINER='/pavics-ncml' +export THREDDS_RAW_DATA_LOCATION_ON_CONTAINER='/pavics-data' +export THREDDS_DATA_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/ncml' # this default is for backward compatibility +export THREDDS_RAW_DATA_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/datasets' # this default is for backward compatibility +export THREDDS_DATA_LOCATION_NAME='Datasets' # this default is for backward compatibility +export THREDDS_RAW_DATA_LOCATION_NAME='Birdhouse' # this default is for backward compatibility +export THREDDS_DATA_LOCATION_NAME_lower='$(echo "$THREDDS_DATA_LOCATION_NAME" | tr "[:upper:]" "[:lower:]")' +export THREDDS_RAW_DATA_LOCATION_NAME_lower='$(echo "$THREDDS_RAW_DATA_LOCATION_NAME" | tr '[:upper:]' '[:lower:]')' @@ -22,12 +22,12 @@ export RAW_DATA_LOCATION_NAME_lower=$(echo "$RAW_DATA_LOCATION_NAME" | tr '[:upp VARS=" $VARS \$CMIP5_THREDDS_ROOT - \$RAW_DATA_LOCATION_NAME - \$RAW_DATA_LOCATION_NAME_lower - \$RAW_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER - \$DATA_LOCATION_NAME - \$DATA_LOCATION_NAME_lower - \$DATA_LOCATION_FOR_THREDDS_ON_CONTAINER + \$THREDDS_RAW_DATA_LOCATION_NAME + \$THREDDS_RAW_DATA_LOCATION_NAME_lower + \$THREDDS_RAW_DATA_LOCATION_ON_CONTAINER + \$THREDDS_DATA_LOCATION_NAME + \$THREDDS_DATA_LOCATION_NAME_lower + \$THREDDS_DATA_LOCATION_ON_CONTAINER " OPTIONAL_VARS=" @@ -40,8 +40,8 @@ OPTIONAL_VARS=" export DELAYED_EVAL=" $DELAYED_EVAL - DATA_LOCATION_NAME_lower - DATA_LOCATION_FOR_THREDDS_ON_HOST - RAW_DATA_LOCATION_NAME_lower - RAW_DATA_LOCATION_FOR_THREDDS_ON_HOST + THREDDS_DATA_LOCATION_NAME_lower + THREDDS_DATA_LOCATION_ON_HOST + THREDDS_RAW_DATA_LOCATION_NAME_lower + THREDDS_RAW_DATA_LOCATION_ON_HOST " diff --git a/birdhouse/config/thredds/docker-compose-extra.yml b/birdhouse/config/thredds/docker-compose-extra.yml index fcceb6631..b7a0fb5d3 100644 --- a/birdhouse/config/thredds/docker-compose-extra.yml +++ b/birdhouse/config/thredds/docker-compose-extra.yml @@ -18,8 +18,8 @@ services: PAVICS_FQDN_PUBLIC: $PAVICS_FQDN_PUBLIC volumes: - thredds_persistence:/usr/local/tomcat/content/thredds - - ${RAW_DATA_LOCATION_FOR_THREDDS_ON_HOST}:${RAW_DATA_LOCATION_FOR_THREDDS_ON_CONTAINER} - - ${DATA_LOCATION_FOR_THREDDS_ON_HOST}:${DATA_LOCATION_FOR_THREDDS_ON_CONTAINER} + - ${THREDDS_RAW_DATA_LOCATION_ON_HOST}:${THREDDS_RAW_DATA_LOCATION_ON_CONTAINER} + - ${THREDDS_DATA_LOCATION_ON_HOST}:${THREDDS_DATA_LOCATION_ON_CONTAINER} - wps_outputs:/pavics-data/wps_outputs - ./config/thredds/catalog.xml:/usr/local/tomcat/content/thredds/catalog.xml:ro - ./config/thredds/threddsConfig.xml:/usr/local/tomcat/content/thredds/threddsConfig.xml:ro diff --git a/birdhouse/config/thredds/entrypointwrapper b/birdhouse/config/thredds/entrypointwrapper index d37e3afe6..56a8a5a2e 100755 --- a/birdhouse/config/thredds/entrypointwrapper +++ b/birdhouse/config/thredds/entrypointwrapper @@ -4,7 +4,7 @@ CONF_FILE="/usr/local/tomcat/conf/server.xml" if ! grep ' relaxedQueryChars=' $CONF_FILE; then # allow angle bracket in query params, ex: - # https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/${RAW_DATA_LOCATION_NAME_lower}/testdata/tasmax_day_BCCAQv2+ANUSPLIN300_BNU-ESM_historical+rcp85_r1i1p1_19500101-21001231.nc.ascii?tasmax[0:1:0][0:1:0][0:1:0] + # https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/${THREDDS_RAW_DATA_LOCATION_NAME_lower}/testdata/tasmax_day_BCCAQv2+ANUSPLIN300_BNU-ESM_historical+rcp85_r1i1p1_19500101-21001231.nc.ascii?tasmax[0:1:0][0:1:0][0:1:0] cp -v $CONF_FILE ${CONF_FILE}.bak.relaxedQueryChars.$$ sed -i 's//dev/null)" -eq 1 ]; then +if [ "$(echo "${BIRDHOUSE_DEPLOY_COMPONENTS_ROOT}" | grep -cE "/${THREDDS_RAW_DATA_LOCATION_NAME_lower}/?\$" 2>/dev/null)" -eq 1 ]; then BIRDHOUSE_DEPLOY_COMPONENTS_ROOT=. else BIRDHOUSE_DEPLOY_COMPONENTS_ROOT="${BIRDHOUSE_DEPLOY_COMPONENTS_ROOT}/.." From 745f3ca4fc916e625cdf9cba6bc12d4b48dbdf08 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Thu, 12 Oct 2023 13:23:25 -0400 Subject: [PATCH 09/20] replace RAW_DATA with SERVICE_DATA --- .../cowbird/config/cowbird/config.yml.template | 12 ++++++------ birdhouse/config/thredds/catalog.xml.template | 2 +- .../canarie_api_monitoring.py.template | 2 +- birdhouse/config/thredds/default.env | 18 +++++++++--------- .../config/thredds/docker-compose-extra.yml | 2 +- birdhouse/config/thredds/entrypointwrapper | 2 +- .../catalog/catalog.cfg.template | 2 +- .../canarie_api_monitoring.py.template | 2 +- .../config/magpie/permissions.cfg | 12 ++++++------ .../testthredds/entrypointwrapper | 2 +- .../scripts/deprecated/trigger-pavicscrawler | 2 +- .../scripts/get-components-json.include.sh | 2 +- 12 files changed, 30 insertions(+), 30 deletions(-) diff --git a/birdhouse/components/cowbird/config/cowbird/config.yml.template b/birdhouse/components/cowbird/config/cowbird/config.yml.template index f572be8e2..f31abcfb7 100644 --- a/birdhouse/components/cowbird/config/cowbird/config.yml.template +++ b/birdhouse/components/cowbird/config/cowbird/config.yml.template @@ -187,10 +187,10 @@ sync_permissions: - name: "{outputID}" type: route thredds: - # /twitcher/ows/proxy/thredds/catalog/${THREDDS_RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/catalog.html - # /twitcher/ows/proxy/thredds/catalog/${THREDDS_RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/catalog.html - # /twitcher/ows/proxy/thredds/catalog/${THREDDS_RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/{job-id}/catalog.html - # /twitcher/ows/proxy/thredds/catalog/${THREDDS_RAW_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/{job-id}/{output-file} + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/catalog.html + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/catalog.html + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/{job-id}/catalog.html + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/{job-id}/{output-file} # note: paths start after ows-proxy portion extracted when Twitcher/Magpie resolve between each other thredds_wps_outputs: - name: thredds @@ -199,7 +199,7 @@ sync_permissions: # 'catalog' is the file/view format specifier for the rest of the path # - name: catalog # type: directory - - name: ${THREDDS_RAW_DATA_LOCATION_NAME_lower} + - name: ${THREDDS_SERVICE_DATA_LOCATION_NAME_lower} type: directory - name: wps_outputs type: directory @@ -216,7 +216,7 @@ sync_permissions: # 'catalog' is the file/view format specifier for the rest of the path # - name: catalog # type: directory - - name: ${THREDDS_RAW_DATA_LOCATION_NAME_lower} + - name: ${THREDDS_SERVICE_DATA_LOCATION_NAME_lower} type: directory - name: wps_outputs type: directory diff --git a/birdhouse/config/thredds/catalog.xml.template b/birdhouse/config/thredds/catalog.xml.template index f1d4f6886..770ad5d08 100644 --- a/birdhouse/config/thredds/catalog.xml.template +++ b/birdhouse/config/thredds/catalog.xml.template @@ -14,7 +14,7 @@ - + all diff --git a/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template index 0ce359912..2a5f12534 100644 --- a/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template @@ -29,7 +29,7 @@ SERVICES['renderer'] = { 'monitoring': { 'ncWMS': { 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/wms/${THREDDS_RAW_DATA_LOCATION_NAME_lower}/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' + 'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/wms/${THREDDS_SERVICE_DATA_LOCATION_NAME_lower}/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' } }, } diff --git a/birdhouse/config/thredds/default.env b/birdhouse/config/thredds/default.env index 546e42a11..21241ef87 100644 --- a/birdhouse/config/thredds/default.env +++ b/birdhouse/config/thredds/default.env @@ -8,13 +8,13 @@ export THREDDS_ORGANIZATION="Birdhouse" # All default values for data locations are set to ensure backward compatibility export THREDDS_DATA_LOCATION_ON_CONTAINER='/pavics-ncml' -export THREDDS_RAW_DATA_LOCATION_ON_CONTAINER='/pavics-data' +export THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER='/pavics-data' export THREDDS_DATA_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/ncml' # this default is for backward compatibility -export THREDDS_RAW_DATA_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/datasets' # this default is for backward compatibility +export THREDDS_SERVICE_DATA_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/datasets' # this default is for backward compatibility export THREDDS_DATA_LOCATION_NAME='Datasets' # this default is for backward compatibility -export THREDDS_RAW_DATA_LOCATION_NAME='Birdhouse' # this default is for backward compatibility +export THREDDS_SERVICE_DATA_LOCATION_NAME='Birdhouse' # this default is for backward compatibility export THREDDS_DATA_LOCATION_NAME_lower='$(echo "$THREDDS_DATA_LOCATION_NAME" | tr "[:upper:]" "[:lower:]")' -export THREDDS_RAW_DATA_LOCATION_NAME_lower='$(echo "$THREDDS_RAW_DATA_LOCATION_NAME" | tr '[:upper:]' '[:lower:]')' +export THREDDS_SERVICE_DATA_LOCATION_NAME_lower='$(echo "$THREDDS_SERVICE_DATA_LOCATION_NAME" | tr '[:upper:]' '[:lower:]')' @@ -22,9 +22,9 @@ export THREDDS_RAW_DATA_LOCATION_NAME_lower='$(echo "$THREDDS_RAW_DATA_LOCATION_ VARS=" $VARS \$CMIP5_THREDDS_ROOT - \$THREDDS_RAW_DATA_LOCATION_NAME - \$THREDDS_RAW_DATA_LOCATION_NAME_lower - \$THREDDS_RAW_DATA_LOCATION_ON_CONTAINER + \$THREDDS_SERVICE_DATA_LOCATION_NAME + \$THREDDS_SERVICE_DATA_LOCATION_NAME_lower + \$THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER \$THREDDS_DATA_LOCATION_NAME \$THREDDS_DATA_LOCATION_NAME_lower \$THREDDS_DATA_LOCATION_ON_CONTAINER @@ -42,6 +42,6 @@ export DELAYED_EVAL=" $DELAYED_EVAL THREDDS_DATA_LOCATION_NAME_lower THREDDS_DATA_LOCATION_ON_HOST - THREDDS_RAW_DATA_LOCATION_NAME_lower - THREDDS_RAW_DATA_LOCATION_ON_HOST + THREDDS_SERVICE_DATA_LOCATION_NAME_lower + THREDDS_SERVICE_DATA_LOCATION_ON_HOST " diff --git a/birdhouse/config/thredds/docker-compose-extra.yml b/birdhouse/config/thredds/docker-compose-extra.yml index b7a0fb5d3..5ccd24eef 100644 --- a/birdhouse/config/thredds/docker-compose-extra.yml +++ b/birdhouse/config/thredds/docker-compose-extra.yml @@ -18,7 +18,7 @@ services: PAVICS_FQDN_PUBLIC: $PAVICS_FQDN_PUBLIC volumes: - thredds_persistence:/usr/local/tomcat/content/thredds - - ${THREDDS_RAW_DATA_LOCATION_ON_HOST}:${THREDDS_RAW_DATA_LOCATION_ON_CONTAINER} + - ${THREDDS_SERVICE_DATA_LOCATION_ON_HOST}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} - ${THREDDS_DATA_LOCATION_ON_HOST}:${THREDDS_DATA_LOCATION_ON_CONTAINER} - wps_outputs:/pavics-data/wps_outputs - ./config/thredds/catalog.xml:/usr/local/tomcat/content/thredds/catalog.xml:ro diff --git a/birdhouse/config/thredds/entrypointwrapper b/birdhouse/config/thredds/entrypointwrapper index 56a8a5a2e..22e77b66e 100755 --- a/birdhouse/config/thredds/entrypointwrapper +++ b/birdhouse/config/thredds/entrypointwrapper @@ -4,7 +4,7 @@ CONF_FILE="/usr/local/tomcat/conf/server.xml" if ! grep ' relaxedQueryChars=' $CONF_FILE; then # allow angle bracket in query params, ex: - # https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/${THREDDS_RAW_DATA_LOCATION_NAME_lower}/testdata/tasmax_day_BCCAQv2+ANUSPLIN300_BNU-ESM_historical+rcp85_r1i1p1_19500101-21001231.nc.ascii?tasmax[0:1:0][0:1:0][0:1:0] + # https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/${THREDDS_SERVICE_DATA_LOCATION_NAME_lower}/testdata/tasmax_day_BCCAQv2+ANUSPLIN300_BNU-ESM_historical+rcp85_r1i1p1_19500101-21001231.nc.ascii?tasmax[0:1:0][0:1:0][0:1:0] cp -v $CONF_FILE ${CONF_FILE}.bak.relaxedQueryChars.$$ sed -i 's//dev/null)" -eq 1 ]; then +if [ "$(echo "${BIRDHOUSE_DEPLOY_COMPONENTS_ROOT}" | grep -cE "/${THREDDS_SERVICE_DATA_LOCATION_NAME_lower}/?\$" 2>/dev/null)" -eq 1 ]; then BIRDHOUSE_DEPLOY_COMPONENTS_ROOT=. else BIRDHOUSE_DEPLOY_COMPONENTS_ROOT="${BIRDHOUSE_DEPLOY_COMPONENTS_ROOT}/.." From 1b9d8bfd1e7ee7a86bc8f6029f9e471acaee832d Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Thu, 12 Oct 2023 13:24:41 -0400 Subject: [PATCH 10/20] replace DATA with DATASET --- birdhouse/config/thredds/catalog.xml.template | 2 +- birdhouse/config/thredds/default.env | 18 +++++++++--------- .../config/thredds/docker-compose-extra.yml | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/birdhouse/config/thredds/catalog.xml.template b/birdhouse/config/thredds/catalog.xml.template index 770ad5d08..f10a91062 100644 --- a/birdhouse/config/thredds/catalog.xml.template +++ b/birdhouse/config/thredds/catalog.xml.template @@ -31,7 +31,7 @@ - + all diff --git a/birdhouse/config/thredds/default.env b/birdhouse/config/thredds/default.env index 21241ef87..55ad0862a 100644 --- a/birdhouse/config/thredds/default.env +++ b/birdhouse/config/thredds/default.env @@ -7,13 +7,13 @@ export THREDDS_IMAGE="${THREDDS_DOCKER}:${THREDDS_VERSION}" export THREDDS_ORGANIZATION="Birdhouse" # All default values for data locations are set to ensure backward compatibility -export THREDDS_DATA_LOCATION_ON_CONTAINER='/pavics-ncml' +export THREDDS_DATASET_LOCATION_ON_CONTAINER='/pavics-ncml' export THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER='/pavics-data' -export THREDDS_DATA_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/ncml' # this default is for backward compatibility +export THREDDS_DATASET_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/ncml' # this default is for backward compatibility export THREDDS_SERVICE_DATA_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/datasets' # this default is for backward compatibility -export THREDDS_DATA_LOCATION_NAME='Datasets' # this default is for backward compatibility +export THREDDS_DATASET_LOCATION_NAME='Datasets' # this default is for backward compatibility export THREDDS_SERVICE_DATA_LOCATION_NAME='Birdhouse' # this default is for backward compatibility -export THREDDS_DATA_LOCATION_NAME_lower='$(echo "$THREDDS_DATA_LOCATION_NAME" | tr "[:upper:]" "[:lower:]")' +export THREDDS_DATASET_LOCATION_NAME_lower='$(echo "$THREDDS_DATASET_LOCATION_NAME" | tr "[:upper:]" "[:lower:]")' export THREDDS_SERVICE_DATA_LOCATION_NAME_lower='$(echo "$THREDDS_SERVICE_DATA_LOCATION_NAME" | tr '[:upper:]' '[:lower:]')' @@ -25,9 +25,9 @@ VARS=" \$THREDDS_SERVICE_DATA_LOCATION_NAME \$THREDDS_SERVICE_DATA_LOCATION_NAME_lower \$THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER - \$THREDDS_DATA_LOCATION_NAME - \$THREDDS_DATA_LOCATION_NAME_lower - \$THREDDS_DATA_LOCATION_ON_CONTAINER + \$THREDDS_DATASET_LOCATION_NAME + \$THREDDS_DATASET_LOCATION_NAME_lower + \$THREDDS_DATASET_LOCATION_ON_CONTAINER " OPTIONAL_VARS=" @@ -40,8 +40,8 @@ OPTIONAL_VARS=" export DELAYED_EVAL=" $DELAYED_EVAL - THREDDS_DATA_LOCATION_NAME_lower - THREDDS_DATA_LOCATION_ON_HOST + THREDDS_DATASET_LOCATION_NAME_lower + THREDDS_DATASET_LOCATION_ON_HOST THREDDS_SERVICE_DATA_LOCATION_NAME_lower THREDDS_SERVICE_DATA_LOCATION_ON_HOST " diff --git a/birdhouse/config/thredds/docker-compose-extra.yml b/birdhouse/config/thredds/docker-compose-extra.yml index 5ccd24eef..27239dd54 100644 --- a/birdhouse/config/thredds/docker-compose-extra.yml +++ b/birdhouse/config/thredds/docker-compose-extra.yml @@ -19,7 +19,7 @@ services: volumes: - thredds_persistence:/usr/local/tomcat/content/thredds - ${THREDDS_SERVICE_DATA_LOCATION_ON_HOST}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} - - ${THREDDS_DATA_LOCATION_ON_HOST}:${THREDDS_DATA_LOCATION_ON_CONTAINER} + - ${THREDDS_DATASET_LOCATION_ON_HOST}:${THREDDS_DATASET_LOCATION_ON_CONTAINER} - wps_outputs:/pavics-data/wps_outputs - ./config/thredds/catalog.xml:/usr/local/tomcat/content/thredds/catalog.xml:ro - ./config/thredds/threddsConfig.xml:/usr/local/tomcat/content/thredds/threddsConfig.xml:ro From ccfb41625f38660af1746957ef1ed1a27d4f138b Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Thu, 12 Oct 2023 13:29:13 -0400 Subject: [PATCH 11/20] changing to URL_PATH variables --- .../cowbird/config/cowbird/config.yml.template | 12 ++++++------ birdhouse/config/thredds/catalog.xml.template | 4 ++-- .../canarie-api/canarie_api_monitoring.py.template | 2 +- birdhouse/config/thredds/default.env | 10 ++++------ birdhouse/config/thredds/entrypointwrapper | 2 +- .../catalog/catalog.cfg.template | 2 +- .../canarie-api/canarie_api_monitoring.py.template | 2 +- .../secure-thredds/config/magpie/permissions.cfg | 12 ++++++------ .../testthredds/entrypointwrapper | 2 +- birdhouse/scripts/deprecated/trigger-pavicscrawler | 2 +- birdhouse/scripts/get-components-json.include.sh | 2 +- 11 files changed, 25 insertions(+), 27 deletions(-) diff --git a/birdhouse/components/cowbird/config/cowbird/config.yml.template b/birdhouse/components/cowbird/config/cowbird/config.yml.template index f31abcfb7..b264f096b 100644 --- a/birdhouse/components/cowbird/config/cowbird/config.yml.template +++ b/birdhouse/components/cowbird/config/cowbird/config.yml.template @@ -187,10 +187,10 @@ sync_permissions: - name: "{outputID}" type: route thredds: - # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/catalog.html - # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/catalog.html - # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/{job-id}/catalog.html - # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_LOCATION_NAME_lower}/wps_outputs/weaver/{public|}/{job-id}/{output-file} + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_URL_PATH}/wps_outputs/weaver/catalog.html + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_URL_PATH}/wps_outputs/weaver/{public|}/catalog.html + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_URL_PATH}/wps_outputs/weaver/{public|}/{job-id}/catalog.html + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_URL_PATH}/wps_outputs/weaver/{public|}/{job-id}/{output-file} # note: paths start after ows-proxy portion extracted when Twitcher/Magpie resolve between each other thredds_wps_outputs: - name: thredds @@ -199,7 +199,7 @@ sync_permissions: # 'catalog' is the file/view format specifier for the rest of the path # - name: catalog # type: directory - - name: ${THREDDS_SERVICE_DATA_LOCATION_NAME_lower} + - name: ${THREDDS_SERVICE_DATA_URL_PATH} type: directory - name: wps_outputs type: directory @@ -216,7 +216,7 @@ sync_permissions: # 'catalog' is the file/view format specifier for the rest of the path # - name: catalog # type: directory - - name: ${THREDDS_SERVICE_DATA_LOCATION_NAME_lower} + - name: ${THREDDS_SERVICE_DATA_URL_PATH} type: directory - name: wps_outputs type: directory diff --git a/birdhouse/config/thredds/catalog.xml.template b/birdhouse/config/thredds/catalog.xml.template index f10a91062..c798d9d5c 100644 --- a/birdhouse/config/thredds/catalog.xml.template +++ b/birdhouse/config/thredds/catalog.xml.template @@ -14,7 +14,7 @@ - + all @@ -31,7 +31,7 @@ - + all diff --git a/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template index 2a5f12534..910202d60 100644 --- a/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template @@ -29,7 +29,7 @@ SERVICES['renderer'] = { 'monitoring': { 'ncWMS': { 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/wms/${THREDDS_SERVICE_DATA_LOCATION_NAME_lower}/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' + 'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/wms/${THREDDS_SERVICE_DATA_URL_PATH}/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' } }, } diff --git a/birdhouse/config/thredds/default.env b/birdhouse/config/thredds/default.env index 55ad0862a..68ce9c038 100644 --- a/birdhouse/config/thredds/default.env +++ b/birdhouse/config/thredds/default.env @@ -13,8 +13,8 @@ export THREDDS_DATASET_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/ncml' # this defa export THREDDS_SERVICE_DATA_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/datasets' # this default is for backward compatibility export THREDDS_DATASET_LOCATION_NAME='Datasets' # this default is for backward compatibility export THREDDS_SERVICE_DATA_LOCATION_NAME='Birdhouse' # this default is for backward compatibility -export THREDDS_DATASET_LOCATION_NAME_lower='$(echo "$THREDDS_DATASET_LOCATION_NAME" | tr "[:upper:]" "[:lower:]")' -export THREDDS_SERVICE_DATA_LOCATION_NAME_lower='$(echo "$THREDDS_SERVICE_DATA_LOCATION_NAME" | tr '[:upper:]' '[:lower:]')' +export THREDDS_DATASET_URL_PATH='datasets' # this default is for backward compatibility +export THREDDS_SERVICE_DATA_URL_PATH='birdhouse' # this default is for backward compatibility @@ -23,10 +23,10 @@ VARS=" $VARS \$CMIP5_THREDDS_ROOT \$THREDDS_SERVICE_DATA_LOCATION_NAME - \$THREDDS_SERVICE_DATA_LOCATION_NAME_lower + \$THREDDS_SERVICE_DATA_URL_PATH \$THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER \$THREDDS_DATASET_LOCATION_NAME - \$THREDDS_DATASET_LOCATION_NAME_lower + \$THREDDS_DATASET_URL_PATH \$THREDDS_DATASET_LOCATION_ON_CONTAINER " @@ -40,8 +40,6 @@ OPTIONAL_VARS=" export DELAYED_EVAL=" $DELAYED_EVAL - THREDDS_DATASET_LOCATION_NAME_lower THREDDS_DATASET_LOCATION_ON_HOST - THREDDS_SERVICE_DATA_LOCATION_NAME_lower THREDDS_SERVICE_DATA_LOCATION_ON_HOST " diff --git a/birdhouse/config/thredds/entrypointwrapper b/birdhouse/config/thredds/entrypointwrapper index 22e77b66e..1572c441b 100755 --- a/birdhouse/config/thredds/entrypointwrapper +++ b/birdhouse/config/thredds/entrypointwrapper @@ -4,7 +4,7 @@ CONF_FILE="/usr/local/tomcat/conf/server.xml" if ! grep ' relaxedQueryChars=' $CONF_FILE; then # allow angle bracket in query params, ex: - # https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/${THREDDS_SERVICE_DATA_LOCATION_NAME_lower}/testdata/tasmax_day_BCCAQv2+ANUSPLIN300_BNU-ESM_historical+rcp85_r1i1p1_19500101-21001231.nc.ascii?tasmax[0:1:0][0:1:0][0:1:0] + # https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/${THREDDS_SERVICE_DATA_URL_PATH}/testdata/tasmax_day_BCCAQv2+ANUSPLIN300_BNU-ESM_historical+rcp85_r1i1p1_19500101-21001231.nc.ascii?tasmax[0:1:0][0:1:0][0:1:0] cp -v $CONF_FILE ${CONF_FILE}.bak.relaxedQueryChars.$$ sed -i 's//dev/null)" -eq 1 ]; then +if [ "$(echo "${BIRDHOUSE_DEPLOY_COMPONENTS_ROOT}" | grep -cE "/${THREDDS_SERVICE_DATA_URL_PATH}/?\$" 2>/dev/null)" -eq 1 ]; then BIRDHOUSE_DEPLOY_COMPONENTS_ROOT=. else BIRDHOUSE_DEPLOY_COMPONENTS_ROOT="${BIRDHOUSE_DEPLOY_COMPONENTS_ROOT}/.." From 8c5f78c27dfba47c18ba7b34918cb1df2e013923 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Thu, 12 Oct 2023 13:43:28 -0400 Subject: [PATCH 12/20] adding ability to add additional catalogs --- birdhouse/config/thredds/catalog.xml.template | 2 ++ birdhouse/env.local.example | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/birdhouse/config/thredds/catalog.xml.template b/birdhouse/config/thredds/catalog.xml.template index c798d9d5c..231b266c8 100644 --- a/birdhouse/config/thredds/catalog.xml.template +++ b/birdhouse/config/thredds/catalog.xml.template @@ -48,4 +48,6 @@ + ${THREDDS_ADDITIONAL_CATALOG} + diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index df68cb4b7..ba4d82c1f 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -409,6 +409,28 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass # Name of organization hosting the Thredds server #export THREDDS_ORGANIZATION="Birdhouse" +# Additional catalogs for THREDDS. Add as many datasetScan XML blocks as needed to THREDDS_ADDITIONAL_CATALOG. +# Each block defines a new top-level catalog. See birdhouse/config/thredds/catalog.xml.template for more information. +export THREDDS_ADDITIONAL_CATALOG="" +#export THREDDS_ADDITIONAL_CATALOG=" +# +# +# +# all +# +# +# +# +# +# +# +# +# +# +# +# +#" + # Allow using Github as external AuthN/AuthZ provider with Magpie # To setup Github as login, goto under section [OAuth Apps] # and create a new Magpie application with configurations: From fe1c190a940f4c83642aec57a4518f11d98fab11 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Thu, 12 Oct 2023 17:02:15 -0400 Subject: [PATCH 13/20] Fix to additional catalog feature --- birdhouse/config/thredds/default.env | 2 ++ birdhouse/env.local.example | 16 ++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/birdhouse/config/thredds/default.env b/birdhouse/config/thredds/default.env index 68ce9c038..80cbaa625 100644 --- a/birdhouse/config/thredds/default.env +++ b/birdhouse/config/thredds/default.env @@ -5,6 +5,7 @@ export THREDDS_VERSION=4.6.18-unidata-2022-01 export THREDDS_DOCKER=pavics/thredds-docker export THREDDS_IMAGE="${THREDDS_DOCKER}:${THREDDS_VERSION}" export THREDDS_ORGANIZATION="Birdhouse" +export THREDDS_ADDITIONAL_CATALOG="" # All default values for data locations are set to ensure backward compatibility export THREDDS_DATASET_LOCATION_ON_CONTAINER='/pavics-ncml' @@ -36,6 +37,7 @@ OPTIONAL_VARS=" \$TWITCHER_PROTECTED_PATH \$THREDDS_DOCKER \$THREDDS_VERSION + \$THREDDS_ADDITIONAL_CATALOG " export DELAYED_EVAL=" diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index ba4d82c1f..2d6dec67a 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -413,19 +413,19 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass # Each block defines a new top-level catalog. See birdhouse/config/thredds/catalog.xml.template for more information. export THREDDS_ADDITIONAL_CATALOG="" #export THREDDS_ADDITIONAL_CATALOG=" -# +# # -# +# # all # # # -# -# -# -# -# -# +# +# +# +# +# +# # # # From 91af29eff88393f5a7d4c1778c8e5d90a8e8615d Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Fri, 13 Oct 2023 16:17:02 -0400 Subject: [PATCH 14/20] fixing previously missed instances of variable renaming --- .../weaver/config/proxy/docker-compose-extra.yml | 2 +- birdhouse/config/thredds/docker-compose-extra.yml | 2 +- .../conf.extra-service.d/wps_outputs-volume.conf.template | 2 +- .../config/proxy/docker-compose-extra.yml | 4 ++-- .../deprecated-components/malleefowl/custom.cfg.template | 2 +- .../malleefowl/docker-compose-extra.yml | 5 ++--- .../config/wps_outputs-volume/docker-compose-extra.yml | 2 +- .../deprecated-components/ncwms2/custom.cfg.template | 2 +- .../deprecated-components/ncwms2/docker-compose-extra.yml | 5 ++--- .../testthredds/docker-compose-extra.yml | 8 ++++---- .../docker-compose.override.local_https_thredds.yml | 2 +- .../templates/docker-compose.override.local_https_wps.yml | 2 +- .../templates/docker-compose.override.public_thredds.yml | 2 +- 13 files changed, 19 insertions(+), 21 deletions(-) diff --git a/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml b/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml index afb7fae36..11d971b66 100644 --- a/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml +++ b/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml @@ -7,6 +7,6 @@ services: - ./components/weaver/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/weaver:ro # because of mounting path naming restrictions (see note in 'worker' definition), # we must add the custom path on top of named 'wps_outputs' volume of other birds for the proxy to expose results - - ${WEAVER_WPS_OUTPUTS_DIR}:/pavics-data/wps_outputs/weaver:ro + - ${WEAVER_WPS_OUTPUTS_DIR}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER}/wps_outputs/weaver:ro links: - weaver diff --git a/birdhouse/config/thredds/docker-compose-extra.yml b/birdhouse/config/thredds/docker-compose-extra.yml index 27239dd54..26c4609af 100644 --- a/birdhouse/config/thredds/docker-compose-extra.yml +++ b/birdhouse/config/thredds/docker-compose-extra.yml @@ -20,7 +20,7 @@ services: - thredds_persistence:/usr/local/tomcat/content/thredds - ${THREDDS_SERVICE_DATA_LOCATION_ON_HOST}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} - ${THREDDS_DATASET_LOCATION_ON_HOST}:${THREDDS_DATASET_LOCATION_ON_CONTAINER} - - wps_outputs:/pavics-data/wps_outputs + - wps_outputs:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER}/wps_outputs - ./config/thredds/catalog.xml:/usr/local/tomcat/content/thredds/catalog.xml:ro - ./config/thredds/threddsConfig.xml:/usr/local/tomcat/content/thredds/threddsConfig.xml:ro - ./config/thredds/wmsConfig.xml:/usr/local/tomcat/content/thredds/wmsConfig.xml:ro diff --git a/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf.template b/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf.template index 10f2599f5..bbe87989e 100644 --- a/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf.template +++ b/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf.template @@ -1,5 +1,5 @@ location /wpsoutputs/ { ${SECURE_DATA_PROXY_AUTH_INCLUDE} - alias /pavics-data/wps_outputs/; + alias ${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER}/wps_outputs/; } diff --git a/birdhouse/config/wps_outputs-volume/config/proxy/docker-compose-extra.yml b/birdhouse/config/wps_outputs-volume/config/proxy/docker-compose-extra.yml index 5d2378963..fb92c530b 100644 --- a/birdhouse/config/wps_outputs-volume/config/proxy/docker-compose-extra.yml +++ b/birdhouse/config/wps_outputs-volume/config/proxy/docker-compose-extra.yml @@ -2,5 +2,5 @@ version: "3.4" services: proxy: volumes: - - ./config/wps_outputs-volume/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/wps_outputs-volume:ro - - wps_outputs:/pavics-data/wps_outputs + - ./config/wps_outputs-volume/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/wps_outputs-volume:ro + - wps_outputs:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER}/wps_outputs diff --git a/birdhouse/deprecated-components/malleefowl/custom.cfg.template b/birdhouse/deprecated-components/malleefowl/custom.cfg.template index 48010d931..16e6472ce 100644 --- a/birdhouse/deprecated-components/malleefowl/custom.cfg.template +++ b/birdhouse/deprecated-components/malleefowl/custom.cfg.template @@ -2,7 +2,7 @@ extends=profiles/docker.cfg [settings] -persist-path = /pavics-data +persist-path = ${environment:THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} archive-root = / thredds_url = https://${environment:HOSTNAME}/twitcher/ows/proxy/thredds/fileServer/birdhouse wps_url = https://${environment:PAVICS_FQDN_PUBLIC}/wpsoutputs diff --git a/birdhouse/deprecated-components/malleefowl/docker-compose-extra.yml b/birdhouse/deprecated-components/malleefowl/docker-compose-extra.yml index 8babd5f43..9874a9535 100644 --- a/birdhouse/deprecated-components/malleefowl/docker-compose-extra.yml +++ b/birdhouse/deprecated-components/malleefowl/docker-compose-extra.yml @@ -1,7 +1,6 @@ version: "3.4" -x-logging: - &default-logging +x-logging: &default-logging driver: "json-file" options: max-size: "50m" @@ -23,7 +22,7 @@ services: - "38091:38091" - "48091:9001" volumes: - - ${DATA_PERSIST_ROOT}/datasets:/pavics-data + - ${THREDDS_SERVICE_DATA_LOCATION_ON_HOST}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} - ./deprecated-components/malleefowl/custom.cfg:/opt/birdhouse/src/malleefowl/custom.cfg depends_on: - postgres diff --git a/birdhouse/deprecated-components/ncwms2/config/wps_outputs-volume/docker-compose-extra.yml b/birdhouse/deprecated-components/ncwms2/config/wps_outputs-volume/docker-compose-extra.yml index 5ba92b768..b6c8f8c5e 100644 --- a/birdhouse/deprecated-components/ncwms2/config/wps_outputs-volume/docker-compose-extra.yml +++ b/birdhouse/deprecated-components/ncwms2/config/wps_outputs-volume/docker-compose-extra.yml @@ -3,4 +3,4 @@ version: "3.4" services: ncwms2: volumes: - - wps_outputs:/pavics-data/wps_outputs + - wps_outputs:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER}/wps_outputs diff --git a/birdhouse/deprecated-components/ncwms2/custom.cfg.template b/birdhouse/deprecated-components/ncwms2/custom.cfg.template index aae16a885..e8562ff77 100644 --- a/birdhouse/deprecated-components/ncwms2/custom.cfg.template +++ b/birdhouse/deprecated-components/ncwms2/custom.cfg.template @@ -8,4 +8,4 @@ supervisor-port = 9001 tomcat-ncwms-password = ${TOMCAT_NCWMS_PASSWORD} [ncwms] -data_dir = /pavics-data +data_dir = ${environment:THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} diff --git a/birdhouse/deprecated-components/ncwms2/docker-compose-extra.yml b/birdhouse/deprecated-components/ncwms2/docker-compose-extra.yml index 46c797fc1..40d7de8c5 100644 --- a/birdhouse/deprecated-components/ncwms2/docker-compose-extra.yml +++ b/birdhouse/deprecated-components/ncwms2/docker-compose-extra.yml @@ -1,7 +1,6 @@ version: "3.4" -x-logging: - &default-logging +x-logging: &default-logging driver: "json-file" options: max-size: "50m" @@ -12,7 +11,7 @@ services: image: pavics/ncwms2:2.0.4 container_name: ncwms2 volumes: - - ${DATA_PERSIST_ROOT}/datasets:/pavics-data + - ${THREDDS_SERVICE_DATA_LOCATION_ON_HOST}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} - ./deprecated-components/ncwms2/custom.cfg:/opt/birdhouse/custom.cfg - ./deprecated-components/ncwms2/server.xml:/opt/birdhouse/eggs/birdhousebuilder.recipe.tomcat-0.2.9-py2.7.egg/birdhousebuilder/recipe/tomcat/server.xml restart: always diff --git a/birdhouse/optional-components/testthredds/docker-compose-extra.yml b/birdhouse/optional-components/testthredds/docker-compose-extra.yml index 4daa4138e..4e7546ec5 100644 --- a/birdhouse/optional-components/testthredds/docker-compose-extra.yml +++ b/birdhouse/optional-components/testthredds/docker-compose-extra.yml @@ -14,10 +14,10 @@ services: volumes: - testthredds_persistence:/usr/local/tomcat/content/thredds - ${DATA_PERSIST_ROOT}/testdatasets:/pavics-testdata:ro - - ${DATA_PERSIST_ROOT}/datasets:/pavics-data:ro - - ${DATA_PERSIST_ROOT}/ncml:/pavics-ncml:ro + - ${THREDDS_SERVICE_DATA_LOCATION_ON_HOST}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER}:ro + - ${THREDDS_DATASET_LOCATION_ON_HOST}:${THREDDS_DATASET_LOCATION_ON_CONTAINER}:ro - testwps_outputs:/testwps_outputs:ro - - wps_outputs:/pavics-data/wps_outputs:ro + - wps_outputs:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER}/wps_outputs:ro - ./optional-components/testthredds/catalog.xml:/usr/local/tomcat/content/thredds/catalog.xml:ro - ./optional-components/testthredds/threddsConfig.xml:/usr/local/tomcat/content/thredds/threddsConfig.xml:ro - ./optional-components/testthredds/wmsConfig.xml:/usr/local/tomcat/content/thredds/wmsConfig.xml:ro @@ -30,7 +30,7 @@ services: "CMD", "curl", "--fail", - "http://localhost:${TESTTHREDDS_INTERNAL_PORT}/${TESTTHREDDS_CONTEXT_ROOT}/catalog.html", + "http://localhost:${TESTTHREDDS_INTERNAL_PORT}/${TESTTHREDDS_CONTEXT_ROOT}/catalog.html" ] volumes: diff --git a/birdhouse/templates/docker-compose.override.local_https_thredds.yml b/birdhouse/templates/docker-compose.override.local_https_thredds.yml index 450c669ab..0f469e07a 100644 --- a/birdhouse/templates/docker-compose.override.local_https_thredds.yml +++ b/birdhouse/templates/docker-compose.override.local_https_thredds.yml @@ -4,7 +4,7 @@ services: extra_hosts: - "dummy.crim.ca:${LOCALHOST_IP}" volumes: - - ${PATH_TO_LOCAL_NETCDF_FILES}:/pavics-data + - ${PATH_TO_LOCAL_NETCDF_FILES}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} proxy: extra_hosts: diff --git a/birdhouse/templates/docker-compose.override.local_https_wps.yml b/birdhouse/templates/docker-compose.override.local_https_wps.yml index 82ea27426..68dfd3f39 100644 --- a/birdhouse/templates/docker-compose.override.local_https_wps.yml +++ b/birdhouse/templates/docker-compose.override.local_https_wps.yml @@ -4,7 +4,7 @@ services: extra_hosts: - "dummy.crim.ca:${LOCALHOST_IP}" volumes: - - ${PATH_TO_LOCAL_NETCDF_FILES}:/pavics-data + - ${PATH_TO_LOCAL_NETCDF_FILES}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} proxy: extra_hosts: diff --git a/birdhouse/templates/docker-compose.override.public_thredds.yml b/birdhouse/templates/docker-compose.override.public_thredds.yml index 4adc95217..12477c95d 100644 --- a/birdhouse/templates/docker-compose.override.public_thredds.yml +++ b/birdhouse/templates/docker-compose.override.public_thredds.yml @@ -2,5 +2,5 @@ version: "3.4" services: thredds: volumes: - - ${PATH_TO_LOCAL_NETCDF_FILES}:/pavics-data + - ${PATH_TO_LOCAL_NETCDF_FILES}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} - ./config/thredds/catalog-public.xml:/opt/birdhouse/eggs/birdhousebuilder.recipe.thredds-0.3.0-py2.7.egg/birdhousebuilder/recipe/thredds/catalog.xml From 555974ec26ddb5ca9cc5264b232437f81d3cb178 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Fri, 13 Oct 2023 18:05:51 -0400 Subject: [PATCH 15/20] adding more THREDDS variables to env.local.example --- birdhouse/env.local.example | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index 2d6dec67a..a6265a63a 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -409,6 +409,15 @@ export GEOSERVER_ADMIN_PASSWORD=geoserverpass # Name of organization hosting the Thredds server #export THREDDS_ORGANIZATION="Birdhouse" +#export THREDDS_DATASET_LOCATION_ON_CONTAINER='/pavics-ncml' # this default is for backward compatibility +#export THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER='/pavics-data' # this default is for backward compatibility +#export THREDDS_DATASET_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/ncml' # this default is for backward compatibility +#export THREDDS_SERVICE_DATA_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/datasets' # this default is for backward compatibility +#export THREDDS_DATASET_LOCATION_NAME='Datasets' # this default is for backward compatibility +#export THREDDS_SERVICE_DATA_LOCATION_NAME='Birdhouse' # this default is for backward compatibility +#export THREDDS_DATASET_URL_PATH='datasets' # this default is for backward compatibility +#export THREDDS_SERVICE_DATA_URL_PATH='birdhouse' # this default is for backward compatibility + # Additional catalogs for THREDDS. Add as many datasetScan XML blocks as needed to THREDDS_ADDITIONAL_CATALOG. # Each block defines a new top-level catalog. See birdhouse/config/thredds/catalog.xml.template for more information. export THREDDS_ADDITIONAL_CATALOG="" From ec0ffe98834704d64e00a4fbe3c8994d36900fb3 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Tue, 17 Oct 2023 11:40:27 -0400 Subject: [PATCH 16/20] revert some changes to variables --- .../components/weaver/config/proxy/docker-compose-extra.yml | 2 +- birdhouse/config/thredds/default.env | 4 ++-- .../conf.extra-service.d/wps_outputs-volume.conf.template | 2 +- .../wps_outputs-volume/config/proxy/docker-compose-extra.yml | 2 +- .../deprecated-components/malleefowl/custom.cfg.template | 2 +- .../deprecated-components/malleefowl/docker-compose-extra.yml | 2 +- .../ncwms2/config/wps_outputs-volume/docker-compose-extra.yml | 2 +- birdhouse/deprecated-components/ncwms2/custom.cfg.template | 2 +- .../deprecated-components/ncwms2/docker-compose-extra.yml | 2 +- .../templates/docker-compose.override.local_https_wps.yml | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml b/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml index 11d971b66..afb7fae36 100644 --- a/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml +++ b/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml @@ -7,6 +7,6 @@ services: - ./components/weaver/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/weaver:ro # because of mounting path naming restrictions (see note in 'worker' definition), # we must add the custom path on top of named 'wps_outputs' volume of other birds for the proxy to expose results - - ${WEAVER_WPS_OUTPUTS_DIR}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER}/wps_outputs/weaver:ro + - ${WEAVER_WPS_OUTPUTS_DIR}:/pavics-data/wps_outputs/weaver:ro links: - weaver diff --git a/birdhouse/config/thredds/default.env b/birdhouse/config/thredds/default.env index 80cbaa625..07b70a8aa 100644 --- a/birdhouse/config/thredds/default.env +++ b/birdhouse/config/thredds/default.env @@ -8,8 +8,8 @@ export THREDDS_ORGANIZATION="Birdhouse" export THREDDS_ADDITIONAL_CATALOG="" # All default values for data locations are set to ensure backward compatibility -export THREDDS_DATASET_LOCATION_ON_CONTAINER='/pavics-ncml' -export THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER='/pavics-data' +export THREDDS_DATASET_LOCATION_ON_CONTAINER='/pavics-ncml' # this default is for backward compatibility +export THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER='/pavics-data' # this default is for backward compatibility export THREDDS_DATASET_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/ncml' # this default is for backward compatibility export THREDDS_SERVICE_DATA_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/datasets' # this default is for backward compatibility export THREDDS_DATASET_LOCATION_NAME='Datasets' # this default is for backward compatibility diff --git a/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf.template b/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf.template index bbe87989e..10f2599f5 100644 --- a/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf.template +++ b/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf.template @@ -1,5 +1,5 @@ location /wpsoutputs/ { ${SECURE_DATA_PROXY_AUTH_INCLUDE} - alias ${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER}/wps_outputs/; + alias /pavics-data/wps_outputs/; } diff --git a/birdhouse/config/wps_outputs-volume/config/proxy/docker-compose-extra.yml b/birdhouse/config/wps_outputs-volume/config/proxy/docker-compose-extra.yml index fb92c530b..eeb5cfcc2 100644 --- a/birdhouse/config/wps_outputs-volume/config/proxy/docker-compose-extra.yml +++ b/birdhouse/config/wps_outputs-volume/config/proxy/docker-compose-extra.yml @@ -3,4 +3,4 @@ services: proxy: volumes: - ./config/wps_outputs-volume/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/wps_outputs-volume:ro - - wps_outputs:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER}/wps_outputs + - wps_outputs:/pavics-data/wps_outputs diff --git a/birdhouse/deprecated-components/malleefowl/custom.cfg.template b/birdhouse/deprecated-components/malleefowl/custom.cfg.template index 16e6472ce..48010d931 100644 --- a/birdhouse/deprecated-components/malleefowl/custom.cfg.template +++ b/birdhouse/deprecated-components/malleefowl/custom.cfg.template @@ -2,7 +2,7 @@ extends=profiles/docker.cfg [settings] -persist-path = ${environment:THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} +persist-path = /pavics-data archive-root = / thredds_url = https://${environment:HOSTNAME}/twitcher/ows/proxy/thredds/fileServer/birdhouse wps_url = https://${environment:PAVICS_FQDN_PUBLIC}/wpsoutputs diff --git a/birdhouse/deprecated-components/malleefowl/docker-compose-extra.yml b/birdhouse/deprecated-components/malleefowl/docker-compose-extra.yml index 9874a9535..64aed87db 100644 --- a/birdhouse/deprecated-components/malleefowl/docker-compose-extra.yml +++ b/birdhouse/deprecated-components/malleefowl/docker-compose-extra.yml @@ -22,7 +22,7 @@ services: - "38091:38091" - "48091:9001" volumes: - - ${THREDDS_SERVICE_DATA_LOCATION_ON_HOST}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} + - ${DATA_PERSIST_ROOT}/datasets:/pavics-data - ./deprecated-components/malleefowl/custom.cfg:/opt/birdhouse/src/malleefowl/custom.cfg depends_on: - postgres diff --git a/birdhouse/deprecated-components/ncwms2/config/wps_outputs-volume/docker-compose-extra.yml b/birdhouse/deprecated-components/ncwms2/config/wps_outputs-volume/docker-compose-extra.yml index b6c8f8c5e..5ba92b768 100644 --- a/birdhouse/deprecated-components/ncwms2/config/wps_outputs-volume/docker-compose-extra.yml +++ b/birdhouse/deprecated-components/ncwms2/config/wps_outputs-volume/docker-compose-extra.yml @@ -3,4 +3,4 @@ version: "3.4" services: ncwms2: volumes: - - wps_outputs:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER}/wps_outputs + - wps_outputs:/pavics-data/wps_outputs diff --git a/birdhouse/deprecated-components/ncwms2/custom.cfg.template b/birdhouse/deprecated-components/ncwms2/custom.cfg.template index e8562ff77..aae16a885 100644 --- a/birdhouse/deprecated-components/ncwms2/custom.cfg.template +++ b/birdhouse/deprecated-components/ncwms2/custom.cfg.template @@ -8,4 +8,4 @@ supervisor-port = 9001 tomcat-ncwms-password = ${TOMCAT_NCWMS_PASSWORD} [ncwms] -data_dir = ${environment:THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} +data_dir = /pavics-data diff --git a/birdhouse/deprecated-components/ncwms2/docker-compose-extra.yml b/birdhouse/deprecated-components/ncwms2/docker-compose-extra.yml index 40d7de8c5..6d684d51b 100644 --- a/birdhouse/deprecated-components/ncwms2/docker-compose-extra.yml +++ b/birdhouse/deprecated-components/ncwms2/docker-compose-extra.yml @@ -11,7 +11,7 @@ services: image: pavics/ncwms2:2.0.4 container_name: ncwms2 volumes: - - ${THREDDS_SERVICE_DATA_LOCATION_ON_HOST}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} + - ${DATA_PERSIST_ROOT}/datasets:/pavics-data - ./deprecated-components/ncwms2/custom.cfg:/opt/birdhouse/custom.cfg - ./deprecated-components/ncwms2/server.xml:/opt/birdhouse/eggs/birdhousebuilder.recipe.tomcat-0.2.9-py2.7.egg/birdhousebuilder/recipe/tomcat/server.xml restart: always diff --git a/birdhouse/templates/docker-compose.override.local_https_wps.yml b/birdhouse/templates/docker-compose.override.local_https_wps.yml index 68dfd3f39..82ea27426 100644 --- a/birdhouse/templates/docker-compose.override.local_https_wps.yml +++ b/birdhouse/templates/docker-compose.override.local_https_wps.yml @@ -4,7 +4,7 @@ services: extra_hosts: - "dummy.crim.ca:${LOCALHOST_IP}" volumes: - - ${PATH_TO_LOCAL_NETCDF_FILES}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} + - ${PATH_TO_LOCAL_NETCDF_FILES}:/pavics-data proxy: extra_hosts: From f892fa008c14b94f9e0d8a8c1fabeef7d71d5e4e Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Tue, 17 Oct 2023 22:53:04 -0400 Subject: [PATCH 17/20] replacing /pavics-data mount point with /data in containers --- .../components/weaver/config/proxy/docker-compose-extra.yml | 2 +- .../proxy/conf.extra-service.d/wps_outputs-volume.conf.template | 2 +- .../wps_outputs-volume/config/proxy/docker-compose-extra.yml | 2 +- birdhouse/deprecated-components/malleefowl/custom.cfg.template | 2 +- .../deprecated-components/malleefowl/docker-compose-extra.yml | 2 +- .../ncwms2/config/wps_outputs-volume/docker-compose-extra.yml | 2 +- birdhouse/deprecated-components/ncwms2/custom.cfg.template | 2 +- birdhouse/deprecated-components/ncwms2/docker-compose-extra.yml | 2 +- birdhouse/templates/docker-compose.override.local_https_wps.yml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml b/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml index afb7fae36..949218b01 100644 --- a/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml +++ b/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml @@ -7,6 +7,6 @@ services: - ./components/weaver/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/weaver:ro # because of mounting path naming restrictions (see note in 'worker' definition), # we must add the custom path on top of named 'wps_outputs' volume of other birds for the proxy to expose results - - ${WEAVER_WPS_OUTPUTS_DIR}:/pavics-data/wps_outputs/weaver:ro + - ${WEAVER_WPS_OUTPUTS_DIR}:/data/wps_outputs/weaver:ro links: - weaver diff --git a/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf.template b/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf.template index 10f2599f5..ddae8ee1e 100644 --- a/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf.template +++ b/birdhouse/config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf.template @@ -1,5 +1,5 @@ location /wpsoutputs/ { ${SECURE_DATA_PROXY_AUTH_INCLUDE} - alias /pavics-data/wps_outputs/; + alias /data/wps_outputs/; } diff --git a/birdhouse/config/wps_outputs-volume/config/proxy/docker-compose-extra.yml b/birdhouse/config/wps_outputs-volume/config/proxy/docker-compose-extra.yml index eeb5cfcc2..faf961162 100644 --- a/birdhouse/config/wps_outputs-volume/config/proxy/docker-compose-extra.yml +++ b/birdhouse/config/wps_outputs-volume/config/proxy/docker-compose-extra.yml @@ -3,4 +3,4 @@ services: proxy: volumes: - ./config/wps_outputs-volume/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/wps_outputs-volume:ro - - wps_outputs:/pavics-data/wps_outputs + - wps_outputs:/data/wps_outputs diff --git a/birdhouse/deprecated-components/malleefowl/custom.cfg.template b/birdhouse/deprecated-components/malleefowl/custom.cfg.template index 48010d931..9d435a691 100644 --- a/birdhouse/deprecated-components/malleefowl/custom.cfg.template +++ b/birdhouse/deprecated-components/malleefowl/custom.cfg.template @@ -2,7 +2,7 @@ extends=profiles/docker.cfg [settings] -persist-path = /pavics-data +persist-path = /data archive-root = / thredds_url = https://${environment:HOSTNAME}/twitcher/ows/proxy/thredds/fileServer/birdhouse wps_url = https://${environment:PAVICS_FQDN_PUBLIC}/wpsoutputs diff --git a/birdhouse/deprecated-components/malleefowl/docker-compose-extra.yml b/birdhouse/deprecated-components/malleefowl/docker-compose-extra.yml index 64aed87db..d528ce45f 100644 --- a/birdhouse/deprecated-components/malleefowl/docker-compose-extra.yml +++ b/birdhouse/deprecated-components/malleefowl/docker-compose-extra.yml @@ -22,7 +22,7 @@ services: - "38091:38091" - "48091:9001" volumes: - - ${DATA_PERSIST_ROOT}/datasets:/pavics-data + - ${DATA_PERSIST_ROOT}/datasets:/data - ./deprecated-components/malleefowl/custom.cfg:/opt/birdhouse/src/malleefowl/custom.cfg depends_on: - postgres diff --git a/birdhouse/deprecated-components/ncwms2/config/wps_outputs-volume/docker-compose-extra.yml b/birdhouse/deprecated-components/ncwms2/config/wps_outputs-volume/docker-compose-extra.yml index 5ba92b768..fe9a8ba18 100644 --- a/birdhouse/deprecated-components/ncwms2/config/wps_outputs-volume/docker-compose-extra.yml +++ b/birdhouse/deprecated-components/ncwms2/config/wps_outputs-volume/docker-compose-extra.yml @@ -3,4 +3,4 @@ version: "3.4" services: ncwms2: volumes: - - wps_outputs:/pavics-data/wps_outputs + - wps_outputs:/data/wps_outputs diff --git a/birdhouse/deprecated-components/ncwms2/custom.cfg.template b/birdhouse/deprecated-components/ncwms2/custom.cfg.template index aae16a885..f252d7f57 100644 --- a/birdhouse/deprecated-components/ncwms2/custom.cfg.template +++ b/birdhouse/deprecated-components/ncwms2/custom.cfg.template @@ -8,4 +8,4 @@ supervisor-port = 9001 tomcat-ncwms-password = ${TOMCAT_NCWMS_PASSWORD} [ncwms] -data_dir = /pavics-data +data_dir = /data diff --git a/birdhouse/deprecated-components/ncwms2/docker-compose-extra.yml b/birdhouse/deprecated-components/ncwms2/docker-compose-extra.yml index 6d684d51b..974deb484 100644 --- a/birdhouse/deprecated-components/ncwms2/docker-compose-extra.yml +++ b/birdhouse/deprecated-components/ncwms2/docker-compose-extra.yml @@ -11,7 +11,7 @@ services: image: pavics/ncwms2:2.0.4 container_name: ncwms2 volumes: - - ${DATA_PERSIST_ROOT}/datasets:/pavics-data + - ${DATA_PERSIST_ROOT}/datasets:/data - ./deprecated-components/ncwms2/custom.cfg:/opt/birdhouse/custom.cfg - ./deprecated-components/ncwms2/server.xml:/opt/birdhouse/eggs/birdhousebuilder.recipe.tomcat-0.2.9-py2.7.egg/birdhousebuilder/recipe/tomcat/server.xml restart: always diff --git a/birdhouse/templates/docker-compose.override.local_https_wps.yml b/birdhouse/templates/docker-compose.override.local_https_wps.yml index 82ea27426..c7fc25a3f 100644 --- a/birdhouse/templates/docker-compose.override.local_https_wps.yml +++ b/birdhouse/templates/docker-compose.override.local_https_wps.yml @@ -4,7 +4,7 @@ services: extra_hosts: - "dummy.crim.ca:${LOCALHOST_IP}" volumes: - - ${PATH_TO_LOCAL_NETCDF_FILES}:/pavics-data + - ${PATH_TO_LOCAL_NETCDF_FILES}:/data proxy: extra_hosts: From 4ab0fc74cb8fa601d75ecfc2a94749b23f60109c Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Thu, 16 Nov 2023 11:11:06 -0500 Subject: [PATCH 18/20] last minor fixes --- birdhouse/scripts/bootstrap-testdata | 2 +- birdhouse/scripts/get-components-json.include.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/birdhouse/scripts/bootstrap-testdata b/birdhouse/scripts/bootstrap-testdata index 47cf951b5..4a10fb437 100755 --- a/birdhouse/scripts/bootstrap-testdata +++ b/birdhouse/scripts/bootstrap-testdata @@ -11,7 +11,7 @@ if [ -z "$DATASET_ROOT" ]; then - DATASET_ROOT="${DATA_PERSIST_ROOT:-/data}/datasets" + DATASET_ROOT="${DATA_PERSIST_ROOT}/${THREDDS_SERVICE_DATA_LOCATION_ON_HOST}" fi FROM_SERVER="https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/fileServer/birdhouse" diff --git a/birdhouse/scripts/get-components-json.include.sh b/birdhouse/scripts/get-components-json.include.sh index 112337823..c3cc13d11 100755 --- a/birdhouse/scripts/get-components-json.include.sh +++ b/birdhouse/scripts/get-components-json.include.sh @@ -26,7 +26,7 @@ fi # resolve path considering if sourced or executed, and whether from current dir, pavics-compose include or another dir BIRDHOUSE_DEPLOY_COMPONENTS_ROOT=$(dirname -- "$(realpath "$0")") -if [ "$(echo "${BIRDHOUSE_DEPLOY_COMPONENTS_ROOT}" | grep -cE "/${THREDDS_SERVICE_DATA_URL_PATH}/?\$" 2>/dev/null)" -eq 1 ]; then +if [ "$(echo "${BIRDHOUSE_DEPLOY_COMPONENTS_ROOT}" | grep -cE "/birdhouse/?\$" 2>/dev/null)" -eq 1 ]; then BIRDHOUSE_DEPLOY_COMPONENTS_ROOT=. else BIRDHOUSE_DEPLOY_COMPONENTS_ROOT="${BIRDHOUSE_DEPLOY_COMPONENTS_ROOT}/.." From 190ff60c2cb25291ccc478348ed1a95c15f186b6 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Tue, 21 Nov 2023 11:50:12 -0500 Subject: [PATCH 19/20] Updating CHANGES.md --- CHANGES.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c286d1a41..b7dad9579 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,9 +16,25 @@ ------------------------------------------------------------------------------------------------------------------ ## Changes -- Flexible locations for data served by THREDDS - - This PR add the flexibility to configure the locations of THREDDS data on the host and the contained - - It also allows to configure the names of the top level THREDDS directories serving 'catalog' and 'raw' data +Flexible locations for data served by THREDDS. This PR adds two capabilities: + +- Makes it possible to configure all aspects of the two default top-level THREDDS catalogs that has been available on Birdhouse (conventionally referred to as `Birdhouse` and `Datasets` on PAIVCS). This is done by defining the following two sets of new environment variables. The `THREDDS_DATASET_` set of variables are meant to control properties of the `Datasets` catalog: + + * THREDDS_DATASET_LOCATION_ON_CONTAINER + * THREDDS_DATASET_LOCATION_ON_HOST + * THREDDS_DATASET_LOCATION_NAME + * THREDDS_DATASET_URL_PATH + + The `THREDDS_SERVICE_DATA_` set of variables control properties of the `Birdhouse` catalog. + + * THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER + * THREDDS_SERVICE_DATA_LOCATION_ON_HOST + * THREDDS_SERVICE_DATA_LOCATION_NAME + * THREDDS_SERVICE_DATA_URL_PATH + + These new variables are defined in [`thredds/default.env`](./birdhouse/config/thredds/default.env) and included in [`env.local.example`](./birdhouse/env.local.example). Their default values have been chosen to ensure the behaviours of the two catalogs remain unchanged (for reasons of backward compatibility). + +- Adds the ability to define additional top-level THREDDS catalogs. This is achieved by introducing the `THREDDS_ADDITIONAL_CATALOG` variable in [`thredds/default.env`](./birdhouse/config/thredds/default.env) that can be used to inject custom XML configuration for a new catalog. This information is picked up by the THREDDS server. An example is provided in [`env.local.example`](./birdhouse/env.local.example). [1.37.2](https://github.com/bird-house/birdhouse-deploy/tree/1.37.2) (2023-11-10) ------------------------------------------------------------------------------------------------------------------ From 95a47719d26c7f64e22dbde84952bb4ab8fbc860 Mon Sep 17 00:00:00 2001 From: Deepak Chandan Date: Tue, 21 Nov 2023 11:50:24 -0500 Subject: [PATCH 20/20] =?UTF-8?q?Bump=20version:=201.37.2=20=E2=86=92=201.?= =?UTF-8?q?38.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 6 +++--- CHANGES.md | 5 +++++ Makefile | 2 +- README.rst | 8 ++++---- RELEASE.txt | 2 +- .../config/canarie-api/docker_configuration.py.template | 8 ++++---- docs/source/conf.py | 4 ++-- 7 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1cdaf7299..bf3a149c9 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.37.2 +current_version = 1.38.0 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2023-11-10T20:16:33Z +search = {current_version} 2023-11-21T16:50:24Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2023-11-10T20:16:33Z +values = 2023-11-21T16:50:24Z [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/CHANGES.md b/CHANGES.md index b7dad9579..be484d243 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,11 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ +[//]: # (list changes here, using '-' for each new entry, remove this when items are added) + +[1.38.0](https://github.com/bird-house/birdhouse-deploy/tree/1.38.0) (2023-11-21) +------------------------------------------------------------------------------------------------------------------ + ## Changes Flexible locations for data served by THREDDS. This PR adds two capabilities: diff --git a/Makefile b/Makefile index f4ee81b00..63c4f86a4 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 1.37.2 +override APP_VERSION := 1.38.0 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index f6f57e9db..3c67559cd 100644 --- a/README.rst +++ b/README.rst @@ -14,13 +14,13 @@ for a full-fledged production platform. * - releases - | |latest-version| |commits-since| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.37.2.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.38.0.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/1.37.2...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/1.38.0...master -.. |latest-version| image:: https://img.shields.io/badge/tag-1.37.2-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-1.38.0-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/1.37.2 + :target: https://github.com/bird-house/birdhouse-deploy/tree/1.38.0 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index f42af8c46..e00b91d6c 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -1.37.2 2023-11-10T20:16:33Z +1.38.0 2023-11-21T16:50:24Z diff --git a/birdhouse/config/canarie-api/docker_configuration.py.template b/birdhouse/config/canarie-api/docker_configuration.py.template index 773fe82e1..c8de1a026 100644 --- a/birdhouse/config/canarie-api/docker_configuration.py.template +++ b/birdhouse/config/canarie-api/docker_configuration.py.template @@ -109,8 +109,8 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.37.2', - 'releaseTime': '2023-11-10T20:16:33Z', + 'version': '1.38.0', + 'releaseTime': '2023-11-21T16:50:24Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', @@ -142,8 +142,8 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.37.2', - 'releaseTime': '2023-11-10T20:16:33Z', + 'version': '1.38.0', + 'releaseTime': '2023-11-21T16:50:24Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', diff --git a/docs/source/conf.py b/docs/source/conf.py index c237710c6..dbdf8be04 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,9 +69,9 @@ # built documents. # # The short X.Y version. -version = '1.37.2' +version = '1.38.0' # The full version, including alpha/beta/rc tags. -release = '1.37.2' +release = '1.38.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages.