diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1cdaf7299..bf3a149c9 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.37.2 +current_version = 1.38.0 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2023-11-10T20:16:33Z +search = {current_version} 2023-11-21T16:50:24Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2023-11-10T20:16:33Z +values = 2023-11-21T16:50:24Z [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/CHANGES.md b/CHANGES.md index 0f35cd69c..2f454af20 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -23,6 +23,30 @@ the POST'ed body employs the convenient JSON format to provide search criteria, it is safe to set this permission when the STAC service was configured to be publicly searchable. +[1.38.0](https://github.com/bird-house/birdhouse-deploy/tree/1.38.0) (2023-11-21) +------------------------------------------------------------------------------------------------------------------ + +## Changes +Flexible locations for data served by THREDDS. This PR adds two capabilities: + +- Makes it possible to configure all aspects of the two default top-level THREDDS catalogs that has been available on Birdhouse (conventionally referred to as `Birdhouse` and `Datasets` on PAIVCS). This is done by defining the following two sets of new environment variables. The `THREDDS_DATASET_` set of variables are meant to control properties of the `Datasets` catalog: + + * THREDDS_DATASET_LOCATION_ON_CONTAINER + * THREDDS_DATASET_LOCATION_ON_HOST + * THREDDS_DATASET_LOCATION_NAME + * THREDDS_DATASET_URL_PATH + + The `THREDDS_SERVICE_DATA_` set of variables control properties of the `Birdhouse` catalog. + + * THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER + * THREDDS_SERVICE_DATA_LOCATION_ON_HOST + * THREDDS_SERVICE_DATA_LOCATION_NAME + * THREDDS_SERVICE_DATA_URL_PATH + + These new variables are defined in [`thredds/default.env`](./birdhouse/config/thredds/default.env) and included in [`env.local.example`](./birdhouse/env.local.example). Their default values have been chosen to ensure the behaviours of the two catalogs remain unchanged (for reasons of backward compatibility). + +- Adds the ability to define additional top-level THREDDS catalogs. This is achieved by introducing the `THREDDS_ADDITIONAL_CATALOG` variable in [`thredds/default.env`](./birdhouse/config/thredds/default.env) that can be used to inject custom XML configuration for a new catalog. This information is picked up by the THREDDS server. An example is provided in [`env.local.example`](./birdhouse/env.local.example). + [1.37.2](https://github.com/bird-house/birdhouse-deploy/tree/1.37.2) (2023-11-10) ------------------------------------------------------------------------------------------------------------------ diff --git a/Makefile b/Makefile index f4ee81b00..63c4f86a4 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 1.37.2 +override APP_VERSION := 1.38.0 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index f6f57e9db..3c67559cd 100644 --- a/README.rst +++ b/README.rst @@ -14,13 +14,13 @@ for a full-fledged production platform. * - releases - | |latest-version| |commits-since| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.37.2.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.38.0.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/1.37.2...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/1.38.0...master -.. |latest-version| image:: https://img.shields.io/badge/tag-1.37.2-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-1.38.0-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/1.37.2 + :target: https://github.com/bird-house/birdhouse-deploy/tree/1.38.0 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index f42af8c46..e00b91d6c 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -1.37.2 2023-11-10T20:16:33Z +1.38.0 2023-11-21T16:50:24Z diff --git a/birdhouse/components/cowbird/config/cowbird/config.yml.template b/birdhouse/components/cowbird/config/cowbird/config.yml.template index 2b26195cc..b290064ba 100644 --- a/birdhouse/components/cowbird/config/cowbird/config.yml.template +++ b/birdhouse/components/cowbird/config/cowbird/config.yml.template @@ -187,10 +187,10 @@ sync_permissions: - name: "{outputID}" type: route thredds: - # /twitcher/ows/proxy/thredds/catalog/birdhouse/wps_outputs/weaver/catalog.html - # /twitcher/ows/proxy/thredds/catalog/birdhouse/wps_outputs/weaver/{public|}/catalog.html - # /twitcher/ows/proxy/thredds/catalog/birdhouse/wps_outputs/weaver/{public|}/{job-id}/catalog.html - # /twitcher/ows/proxy/thredds/catalog/birdhouse/wps_outputs/weaver/{public|}/{job-id}/{output-file} + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_URL_PATH}/wps_outputs/weaver/catalog.html + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_URL_PATH}/wps_outputs/weaver/{public|}/catalog.html + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_URL_PATH}/wps_outputs/weaver/{public|}/{job-id}/catalog.html + # /twitcher/ows/proxy/thredds/catalog/${THREDDS_SERVICE_DATA_URL_PATH}/wps_outputs/weaver/{public|}/{job-id}/{output-file} # note: paths start after ows-proxy portion extracted when Twitcher/Magpie resolve between each other thredds_wps_outputs: - name: thredds @@ -199,7 +199,7 @@ sync_permissions: # 'catalog' is the file/view format specifier for the rest of the path # - name: catalog # type: directory - - name: birdhouse + - name: ${THREDDS_SERVICE_DATA_URL_PATH} type: directory - name: wps_outputs type: directory @@ -216,7 +216,7 @@ sync_permissions: # 'catalog' is the file/view format specifier for the rest of the path # - name: catalog # type: directory - - name: birdhouse + - name: ${THREDDS_SERVICE_DATA_URL_PATH} type: directory - name: wps_outputs type: directory diff --git a/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml b/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml index afb7fae36..949218b01 100644 --- a/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml +++ b/birdhouse/components/weaver/config/proxy/docker-compose-extra.yml @@ -7,6 +7,6 @@ services: - ./components/weaver/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/weaver:ro # because of mounting path naming restrictions (see note in 'worker' definition), # we must add the custom path on top of named 'wps_outputs' volume of other birds for the proxy to expose results - - ${WEAVER_WPS_OUTPUTS_DIR}:/pavics-data/wps_outputs/weaver:ro + - ${WEAVER_WPS_OUTPUTS_DIR}:/data/wps_outputs/weaver:ro links: - weaver diff --git a/birdhouse/config/canarie-api/docker_configuration.py.template b/birdhouse/config/canarie-api/docker_configuration.py.template index 773fe82e1..c8de1a026 100644 --- a/birdhouse/config/canarie-api/docker_configuration.py.template +++ b/birdhouse/config/canarie-api/docker_configuration.py.template @@ -109,8 +109,8 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.37.2', - 'releaseTime': '2023-11-10T20:16:33Z', + 'version': '1.38.0', + 'releaseTime': '2023-11-21T16:50:24Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', @@ -142,8 +142,8 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.37.2', - 'releaseTime': '2023-11-10T20:16:33Z', + 'version': '1.38.0', + 'releaseTime': '2023-11-21T16:50:24Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', diff --git a/birdhouse/config/thredds/catalog.xml.template b/birdhouse/config/thredds/catalog.xml.template index 974842164..231b266c8 100644 --- a/birdhouse/config/thredds/catalog.xml.template +++ b/birdhouse/config/thredds/catalog.xml.template @@ -14,7 +14,7 @@ - + all @@ -31,7 +31,7 @@ - + all @@ -48,4 +48,6 @@ + ${THREDDS_ADDITIONAL_CATALOG} + diff --git a/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template index c0ebc61f4..910202d60 100644 --- a/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/config/thredds/config/canarie-api/canarie_api_monitoring.py.template @@ -29,7 +29,7 @@ SERVICES['renderer'] = { 'monitoring': { 'ncWMS': { 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/wms/birdhouse/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' + 'url': 'https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/thredds/wms/${THREDDS_SERVICE_DATA_URL_PATH}/testdata/ta_Amon_MRI-CGCM3_decadal1980_r1i1p1_199101-200012.nc?service=WMS&version=1.3.0&request=GetCapabilities' } }, } diff --git a/birdhouse/config/thredds/default.env b/birdhouse/config/thredds/default.env index 6e8e6de35..07b70a8aa 100644 --- a/birdhouse/config/thredds/default.env +++ b/birdhouse/config/thredds/default.env @@ -5,11 +5,30 @@ export THREDDS_VERSION=4.6.18-unidata-2022-01 export THREDDS_DOCKER=pavics/thredds-docker export THREDDS_IMAGE="${THREDDS_DOCKER}:${THREDDS_VERSION}" export THREDDS_ORGANIZATION="Birdhouse" +export THREDDS_ADDITIONAL_CATALOG="" + +# All default values for data locations are set to ensure backward compatibility +export THREDDS_DATASET_LOCATION_ON_CONTAINER='/pavics-ncml' # this default is for backward compatibility +export THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER='/pavics-data' # this default is for backward compatibility +export THREDDS_DATASET_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/ncml' # this default is for backward compatibility +export THREDDS_SERVICE_DATA_LOCATION_ON_HOST='${DATA_PERSIST_ROOT}/datasets' # this default is for backward compatibility +export THREDDS_DATASET_LOCATION_NAME='Datasets' # this default is for backward compatibility +export THREDDS_SERVICE_DATA_LOCATION_NAME='Birdhouse' # this default is for backward compatibility +export THREDDS_DATASET_URL_PATH='datasets' # this default is for backward compatibility +export THREDDS_SERVICE_DATA_URL_PATH='birdhouse' # this default is for backward compatibility + + # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here VARS=" $VARS \$CMIP5_THREDDS_ROOT + \$THREDDS_SERVICE_DATA_LOCATION_NAME + \$THREDDS_SERVICE_DATA_URL_PATH + \$THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER + \$THREDDS_DATASET_LOCATION_NAME + \$THREDDS_DATASET_URL_PATH + \$THREDDS_DATASET_LOCATION_ON_CONTAINER " OPTIONAL_VARS=" @@ -18,4 +37,11 @@ OPTIONAL_VARS=" \$TWITCHER_PROTECTED_PATH \$THREDDS_DOCKER \$THREDDS_VERSION + \$THREDDS_ADDITIONAL_CATALOG +" + +export DELAYED_EVAL=" + $DELAYED_EVAL + THREDDS_DATASET_LOCATION_ON_HOST + THREDDS_SERVICE_DATA_LOCATION_ON_HOST " diff --git a/birdhouse/config/thredds/docker-compose-extra.yml b/birdhouse/config/thredds/docker-compose-extra.yml index 8ae7735e9..26c4609af 100644 --- a/birdhouse/config/thredds/docker-compose-extra.yml +++ b/birdhouse/config/thredds/docker-compose-extra.yml @@ -1,7 +1,6 @@ version: "3.4" -x-logging: - &default-logging +x-logging: &default-logging driver: "json-file" options: max-size: "50m" @@ -19,9 +18,9 @@ services: PAVICS_FQDN_PUBLIC: $PAVICS_FQDN_PUBLIC volumes: - thredds_persistence:/usr/local/tomcat/content/thredds - - ${DATA_PERSIST_ROOT}/datasets:/pavics-data - - ${DATA_PERSIST_ROOT}/ncml:/pavics-ncml - - wps_outputs:/pavics-data/wps_outputs + - ${THREDDS_SERVICE_DATA_LOCATION_ON_HOST}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER} + - ${THREDDS_DATASET_LOCATION_ON_HOST}:${THREDDS_DATASET_LOCATION_ON_CONTAINER} + - wps_outputs:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER}/wps_outputs - ./config/thredds/catalog.xml:/usr/local/tomcat/content/thredds/catalog.xml:ro - ./config/thredds/threddsConfig.xml:/usr/local/tomcat/content/thredds/threddsConfig.xml:ro - ./config/thredds/wmsConfig.xml:/usr/local/tomcat/content/thredds/wmsConfig.xml:ro @@ -35,7 +34,7 @@ services: "CMD", "curl", "--fail", - "http://localhost:8080${TWITCHER_PROTECTED_PATH}/thredds/catalog.html", + "http://localhost:8080${TWITCHER_PROTECTED_PATH}/thredds/catalog.html" ] volumes: diff --git a/birdhouse/config/thredds/entrypointwrapper b/birdhouse/config/thredds/entrypointwrapper index a1d617823..1572c441b 100755 --- a/birdhouse/config/thredds/entrypointwrapper +++ b/birdhouse/config/thredds/entrypointwrapper @@ -4,7 +4,7 @@ CONF_FILE="/usr/local/tomcat/conf/server.xml" if ! grep ' relaxedQueryChars=' $CONF_FILE; then # allow angle bracket in query params, ex: - # https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/testdata/tasmax_day_BCCAQv2+ANUSPLIN300_BNU-ESM_historical+rcp85_r1i1p1_19500101-21001231.nc.ascii?tasmax[0:1:0][0:1:0][0:1:0] + # https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/${THREDDS_SERVICE_DATA_URL_PATH}/testdata/tasmax_day_BCCAQv2+ANUSPLIN300_BNU-ESM_historical+rcp85_r1i1p1_19500101-21001231.nc.ascii?tasmax[0:1:0][0:1:0][0:1:0] cp -v $CONF_FILE ${CONF_FILE}.bak.relaxedQueryChars.$$ sed -i 's/ under section [OAuth Apps] # and create a new Magpie application with configurations: diff --git a/birdhouse/optional-components/secure-thredds/config/magpie/permissions.cfg b/birdhouse/optional-components/secure-thredds/config/magpie/permissions.cfg index 337017228..7fdc2e3d6 100644 --- a/birdhouse/optional-components/secure-thredds/config/magpie/permissions.cfg +++ b/birdhouse/optional-components/secure-thredds/config/magpie/permissions.cfg @@ -7,7 +7,7 @@ permissions: # these can be reverted or combined with other set of permissions on resources 'above' or 'under' in the hierarchy # users or groups will need explicit permissions under following resources for them to access sub-directories/files - service: thredds - resource: /birdhouse/testdata/secure + resource: /${THREDDS_SERVICE_DATA_URL_PATH}/testdata/secure type: directory permission: name: browse @@ -17,7 +17,7 @@ permissions: action: create - service: thredds - resource: /birdhouse/testdata/secure + resource: /${THREDDS_SERVICE_DATA_URL_PATH}/testdata/secure type: directory permission: name: read @@ -27,7 +27,7 @@ permissions: action: create - service: thredds - resource: /birdhouse/testdata/secure + resource: /${THREDDS_SERVICE_DATA_URL_PATH}/testdata/secure type: directory permission: name: write @@ -44,7 +44,7 @@ permissions: # script when bootstrapping the instance for testing purpose. - service: thredds - resource: /birdhouse/testdata/secure + resource: /${THREDDS_SERVICE_DATA_URL_PATH}/testdata/secure type: directory permission: name: browse @@ -54,7 +54,7 @@ permissions: action: create - service: thredds - resource: /birdhouse/testdata/secure + resource: /${THREDDS_SERVICE_DATA_URL_PATH}/testdata/secure type: directory permission: name: read @@ -64,7 +64,7 @@ permissions: action: create - service: thredds - resource: /birdhouse/testdata/secure + resource: /${THREDDS_SERVICE_DATA_URL_PATH}/testdata/secure type: directory permission: name: write diff --git a/birdhouse/optional-components/testthredds/docker-compose-extra.yml b/birdhouse/optional-components/testthredds/docker-compose-extra.yml index 4daa4138e..4e7546ec5 100644 --- a/birdhouse/optional-components/testthredds/docker-compose-extra.yml +++ b/birdhouse/optional-components/testthredds/docker-compose-extra.yml @@ -14,10 +14,10 @@ services: volumes: - testthredds_persistence:/usr/local/tomcat/content/thredds - ${DATA_PERSIST_ROOT}/testdatasets:/pavics-testdata:ro - - ${DATA_PERSIST_ROOT}/datasets:/pavics-data:ro - - ${DATA_PERSIST_ROOT}/ncml:/pavics-ncml:ro + - ${THREDDS_SERVICE_DATA_LOCATION_ON_HOST}:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER}:ro + - ${THREDDS_DATASET_LOCATION_ON_HOST}:${THREDDS_DATASET_LOCATION_ON_CONTAINER}:ro - testwps_outputs:/testwps_outputs:ro - - wps_outputs:/pavics-data/wps_outputs:ro + - wps_outputs:${THREDDS_SERVICE_DATA_LOCATION_ON_CONTAINER}/wps_outputs:ro - ./optional-components/testthredds/catalog.xml:/usr/local/tomcat/content/thredds/catalog.xml:ro - ./optional-components/testthredds/threddsConfig.xml:/usr/local/tomcat/content/thredds/threddsConfig.xml:ro - ./optional-components/testthredds/wmsConfig.xml:/usr/local/tomcat/content/thredds/wmsConfig.xml:ro @@ -30,7 +30,7 @@ services: "CMD", "curl", "--fail", - "http://localhost:${TESTTHREDDS_INTERNAL_PORT}/${TESTTHREDDS_CONTEXT_ROOT}/catalog.html", + "http://localhost:${TESTTHREDDS_INTERNAL_PORT}/${TESTTHREDDS_CONTEXT_ROOT}/catalog.html" ] volumes: diff --git a/birdhouse/optional-components/testthredds/entrypointwrapper b/birdhouse/optional-components/testthredds/entrypointwrapper index a1d617823..1572c441b 100755 --- a/birdhouse/optional-components/testthredds/entrypointwrapper +++ b/birdhouse/optional-components/testthredds/entrypointwrapper @@ -4,7 +4,7 @@ CONF_FILE="/usr/local/tomcat/conf/server.xml" if ! grep ' relaxedQueryChars=' $CONF_FILE; then # allow angle bracket in query params, ex: - # https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/testdata/tasmax_day_BCCAQv2+ANUSPLIN300_BNU-ESM_historical+rcp85_r1i1p1_19500101-21001231.nc.ascii?tasmax[0:1:0][0:1:0][0:1:0] + # https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/${THREDDS_SERVICE_DATA_URL_PATH}/testdata/tasmax_day_BCCAQv2+ANUSPLIN300_BNU-ESM_historical+rcp85_r1i1p1_19500101-21001231.nc.ascii?tasmax[0:1:0][0:1:0][0:1:0] cp -v $CONF_FILE ${CONF_FILE}.bak.relaxedQueryChars.$$ sed -i 's/