diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 89de7c4fe..09fb6fe6b 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.1.3 +current_version = 2.2.0 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2024-04-09T00:40:41Z +search = {current_version} 2024-04-18T15:04:09Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2024-04-09T00:40:41Z +values = 2024-04-18T15:04:09Z [bumpversion:file(version):birdhouse/components/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/CHANGES.md b/CHANGES.md index bc9043c5b..5236130a0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,23 @@ [//]: # (list changes here, using '-' for each new entry, remove this when items are added) +[2.2.0](https://github.com/bird-house/birdhouse-deploy/tree/2.2.0) (2024-04-18) +------------------------------------------------------------------------------------------------------------------ + +## Changes + +- Node Services: Add definitions and variables for every service represented by + the [DACCS-Climate/Marble-node-registry](https://github.com/DACCS-Climate/Marble-node-registry). + + - Add `version` field using the corresponding `_VERSION` variables. + - Add `types` field restricted by specific values instead of previous `keywords` expected to be extendable. + - Add `_IMAGE_URI` variables to provide `rel: service-meta` link for every service. + - Update all `$schema` references of service node registry + to [1.2.0](https://github.com/DACCS-Climate/Marble-node-registry/releases/tag/1.2.0) instead of `main`. + During unit tests, specific `$schema` reference in the respective service configuration will be used for validation. + + See [bird-house/birdhouse-deploy#441](https://github.com/bird-house/birdhouse-deploy/issues/441) for more details. + [2.1.3](https://github.com/bird-house/birdhouse-deploy/tree/2.1.3) (2024-04-09) ------------------------------------------------------------------------------------------------------------------ diff --git a/Makefile b/Makefile index f8dd4be70..568358ef8 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 2.1.3 +override APP_VERSION := 2.2.0 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index fe4e51ad7..5bf2cd4ff 100644 --- a/README.rst +++ b/README.rst @@ -18,13 +18,13 @@ for a full-fledged production platform. * - citation - | |citation| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.1.3.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.2.0.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/2.1.3...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/2.2.0...master -.. |latest-version| image:: https://img.shields.io/badge/tag-2.1.3-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-2.2.0-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/2.1.3 + :target: https://github.com/bird-house/birdhouse-deploy/tree/2.2.0 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index f1943de27..28007fe42 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -2.1.3 2024-04-09T00:40:41Z +2.2.0 2024-04-18T15:04:09Z diff --git a/birdhouse/components/canarie-api/docker_configuration.py.template b/birdhouse/components/canarie-api/docker_configuration.py.template index 35bb8aae5..ce38ed64f 100644 --- a/birdhouse/components/canarie-api/docker_configuration.py.template +++ b/birdhouse/components/canarie-api/docker_configuration.py.template @@ -109,8 +109,8 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '2.1.3', - 'releaseTime': '2024-04-09T00:40:41Z', + 'version': '2.2.0', + 'releaseTime': '2024-04-18T15:04:09Z', 'institution': '${SERVER_INSTITUTION}', 'researchSubject': '${SERVER_SUBJECT}', 'supportEmail': '${SUPPORT_EMAIL}', @@ -142,8 +142,8 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '2.1.3', - 'releaseTime': '2024-04-09T00:40:41Z', + 'version': '2.2.0', + 'releaseTime': '2024-04-18T15:04:09Z', 'institution': '${SERVER_INSTITUTION}', 'researchSubject': '${SERVER_SUBJECT}', 'supportEmail': '${SUPPORT_EMAIL}', diff --git a/birdhouse/components/finch/default.env b/birdhouse/components/finch/default.env index b11b0f701..eb0174473 100644 --- a/birdhouse/components/finch/default.env +++ b/birdhouse/components/finch/default.env @@ -5,7 +5,9 @@ # are applied and must be added to the list of DELAYED_EVAL. export FINCH_VERSION=0.9.2 -export FINCH_IMAGE="birdhouse/finch:version-${FINCH_VERSION}" +export FINCH_DOCKER=birdhouse/finch +export FINCH_IMAGE='${FINCH_DOCKER}:version-${FINCH_VERSION}' +export FINCH_IMAGE_URI='registry.hub.docker.com/${FINCH_IMAGE}' # This tells the postgres service to create a database on startup export POSTGRES_DATABASES_TO_CREATE=" @@ -25,7 +27,16 @@ COMPONENT_DEPENDENCIES=" ./components/postgres " +export DELAYED_EVAL=" + $DELAYED_EVAL + FINCH_IMAGE + FINCH_IMAGE_URI +" + OPTIONAL_VARS=" $OPTIONAL_VARS \$FINCH_VERSION + \$FINCH_DOCKER + \$FINCH_IMAGE + \$FINCH_IMAGE_URI " diff --git a/birdhouse/components/finch/service-config.json.template b/birdhouse/components/finch/service-config.json.template index f29c0db7f..bceffe0fa 100644 --- a/birdhouse/components/finch/service-config.json.template +++ b/birdhouse/components/finch/service-config.json.template @@ -1,25 +1,36 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "finch", - "keywords": [ - "service-wps" - ], - "description": "A Web Processing Service for Climate Indicators.", - "links": [ - { - "rel": "service", - "type": "text/xml", - "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&request=GetCapabilities" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://finch.readthedocs.io" - }, - { - "rel": "service-desc", - "type": "text/xml", - "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&request=GetCapabilities" - } - ] -} +[ + { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/1.2.0/node_registry.schema.json#service", + "name": "finch", + "version": "${FINCH_VERSION}", + "types": [ + "wps" + ], + "keywords": [ + "service-wps" + ], + "description": "A Web Processing Service for Climate Indicators.", + "links": [ + { + "rel": "service", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://finch.readthedocs.io" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/finch?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-meta", + "type": "application/vnd.oci.image.index.v1+json", + "href": "${FINCH_IMAGE_URI}" + } + ] + } +] diff --git a/birdhouse/components/geoserver/default.env b/birdhouse/components/geoserver/default.env index 84235d307..9db03a085 100644 --- a/birdhouse/components/geoserver/default.env +++ b/birdhouse/components/geoserver/default.env @@ -11,7 +11,8 @@ export GEOSERVER_DOCKER=pavics/geoserver export GEOSERVER_VERSION=2.22.2 export GEOSERVER_TAGGED=2.22.2-kartoza-build20230226-r7-allow-change-context-root-and-fix-missing-stable-plugins-and-avoid-chown-datadir -export GEOSERVER_IMAGE="${GEOSERVER_DOCKER}:${GEOSERVER_TAGGED}" +export GEOSERVER_IMAGE='${GEOSERVER_DOCKER}:${GEOSERVER_TAGGED}' +export GEOSERVER_IMAGE_URI='registry.hub.docker.com/${GEOSERVER_IMAGE}' export GEOSERVER_ADMIN_USER="admin" @@ -34,6 +35,8 @@ export DELAYED_EVAL=" $DELAYED_EVAL GEOSERVER_DATA_DIR GEOSERVER_SKIP_AUTH_PROXY_INCLUDE + GEOSERVER_IMAGE + GEOSERVER_IMAGE_URI " # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here @@ -49,5 +52,6 @@ OPTIONAL_VARS=" \$GEOSERVER_VERSION \$GEOSERVER_TAGGED \$GEOSERVER_IMAGE + \$GEOSERVER_IMAGE_URI \$GEOSERVER_SKIP_AUTH_PROXY_INCLUDE " diff --git a/birdhouse/components/geoserver/service-config.json.template b/birdhouse/components/geoserver/service-config.json.template index f8838008b..b30a5bfdb 100644 --- a/birdhouse/components/geoserver/service-config.json.template +++ b/birdhouse/components/geoserver/service-config.json.template @@ -1,23 +1,37 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "geoserver", - "keywords": [ - "data", - "service-wms", - "service-wfs", - "service-wps" - ], - "description": "GeoServer is a server that allows users to view and edit geospatial data.", - "links": [ - { - "rel": "service", - "type": "text/html", - "href": "https://${PAVICS_FQDN_PUBLIC}/geoserver/" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://docs.geoserver.org/" - } - ] -} +[ + { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/1.2.0/node_registry.schema.json#service", + "name": "geoserver", + "version": "${GEOSERVER_VERSION}", + "types": [ + "data", + "wms", + "wfs", + "wps" + ], + "keywords": [ + "data", + "service-wms", + "service-wfs", + "service-wps" + ], + "description": "GeoServer is a server that allows users to view and edit geospatial data.", + "links": [ + { + "rel": "service", + "type": "text/html", + "href": "https://${PAVICS_FQDN_PUBLIC}/geoserver/" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://docs.geoserver.org/" + }, + { + "rel": "service-meta", + "type": "application/vnd.oci.image.index.v1+json", + "href": "${GEOSERVER_IMAGE_URI}" + } + ] + } +] diff --git a/birdhouse/components/hummingbird/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/hummingbird/config/canarie-api/canarie_api_monitoring.py.template index 33fa047c0..935399dbb 100644 --- a/birdhouse/components/hummingbird/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/hummingbird/config/canarie-api/canarie_api_monitoring.py.template @@ -1,5 +1,6 @@ +HUMMINGBIRD_TAGGED = "${HUMMINGBIRD_TAGGED}" HUMMINGBIRD_VERSION = "${HUMMINGBIRD_VERSION}" -HUMMINGBIRD_RELEASE = get_release_time_from_repo_tag("docker", "pavics/hummingbird", HUMMINGBIRD_VERSION) +HUMMINGBIRD_RELEASE = get_release_time_from_repo_tag("docker", "pavics/hummingbird", HUMMINGBIRD_TAGGED) SERVICES['hummingbird'] = { 'info': { diff --git a/birdhouse/components/hummingbird/default.env b/birdhouse/components/hummingbird/default.env index 3a3ed4c94..b2856493c 100644 --- a/birdhouse/components/hummingbird/default.env +++ b/birdhouse/components/hummingbird/default.env @@ -1,4 +1,11 @@ -export HUMMINGBIRD_VERSION=0.5_dev +# note: minor adjustments for 'version' field of service +# - version must include patch (https://semver.org/#spec-item-2) +# - pattern does not allow '_' (https://semver.org/#spec-item-9) +export HUMMINGBIRD_VERSION=0.5.0-dev +export HUMMINGBIRD_TAGGED=0.5_dev +export HUMMINGBIRD_DOCKER=pavics/hummingbird +export HUMMINGBIRD_IMAGE='${HUMMINGBIRD_DOCKER}:${HUMMINGBIRD_TAGGED}' +export HUMMINGBIRD_IMAGE_URI='registry.hub.docker.com/${HUMMINGBIRD_IMAGE}' # This tells the postgres service to create a database on startup export POSTGRES_DATABASES_TO_CREATE=" @@ -19,7 +26,17 @@ COMPONENT_DEPENDENCIES=" ./components/data-volume " +export DELAYED_EVAL=" + $DELAYED_EVAL + HUMMINGBIRD_IMAGE + HUMMINGBIRD_IMAGE_URI +" + OPTIONAL_VARS=" $OPTIONAL_VARS \$HUMMINGBIRD_VERSION + \$HUMMINGBIRD_DOCKER + \$HUMMINGBIRD_TAGGED + \$HUMMINGBIRD_IMAGE + \$HUMMINGBIRD_IMAGE_URI " diff --git a/birdhouse/components/hummingbird/docker-compose-extra.yml b/birdhouse/components/hummingbird/docker-compose-extra.yml index 28c8b485d..b31a98727 100644 --- a/birdhouse/components/hummingbird/docker-compose-extra.yml +++ b/birdhouse/components/hummingbird/docker-compose-extra.yml @@ -9,7 +9,7 @@ x-logging: services: hummingbird: - image: pavics/hummingbird:${HUMMINGBIRD_VERSION} + image: ${HUMMINGBIRD_IMAGE} container_name: hummingbird environment: HOSTNAME: $HOSTNAME diff --git a/birdhouse/components/hummingbird/service-config.json.template b/birdhouse/components/hummingbird/service-config.json.template index c68ad3f9b..0d3890849 100644 --- a/birdhouse/components/hummingbird/service-config.json.template +++ b/birdhouse/components/hummingbird/service-config.json.template @@ -1,25 +1,36 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "hummingbird", - "keywords": [ - "service-wps" - ], - "description": "A WPS for metadata compliance checks used in the climate science community.", - "links": [ - { - "rel": "service", - "type": "text/xml", - "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&request=GetCapabilities" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://birdhouse-hummingbird.readthedocs.io" - }, - { - "rel": "service-desc", - "type": "text/xml", - "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&request=GetCapabilities" - } - ] -} +[ + { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/1.2.0/node_registry.schema.json#service", + "name": "hummingbird", + "version": "${HUMMINGBIRD_VERSION}", + "types": [ + "wps" + ], + "keywords": [ + "service-wps" + ], + "description": "A WPS for metadata compliance checks used in the climate science community.", + "links": [ + { + "rel": "service", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://birdhouse-hummingbird.readthedocs.io" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/hummingbird?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-meta", + "type": "application/vnd.oci.image.index.v1+json", + "href": "${HUMMINGBIRD_IMAGE_URI}" + } + ] + } +] diff --git a/birdhouse/components/jupyterhub/default.env b/birdhouse/components/jupyterhub/default.env index e5177ec30..e59a11a94 100644 --- a/birdhouse/components/jupyterhub/default.env +++ b/birdhouse/components/jupyterhub/default.env @@ -6,6 +6,8 @@ export JUPYTERHUB_DOCKER=pavics/jupyterhub export JUPYTERHUB_VERSION=4.1.4-20240408 +export JUPYTERHUB_IMAGE='${JUPYTERHUB_DOCKER}:${JUPYTERHUB_VERSION}' +export JUPYTERHUB_IMAGE_URI='registry.hub.docker.com/${JUPYTERHUB_IMAGE}' # Jupyter single-user server images, can be overriden in env.local to have a space separated list of multiple images export DOCKER_NOTEBOOK_IMAGES="pavics/workflow-tests:py39-230601-1-update240116" @@ -75,6 +77,8 @@ export DELAYED_EVAL=" $DELAYED_EVAL JUPYTERHUB_USER_DATA_DIR JUPYTERHUB_ADMIN_USERS + JUPYTERHUB_IMAGE + JUPYTERHUB_IMAGE_URI " # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here @@ -93,6 +97,8 @@ OPTIONAL_VARS=" \$JUPYTERHUB_CONFIG_OVERRIDE \$JUPYTERHUB_DOCKER \$JUPYTERHUB_VERSION + \$JUPYTERHUB_IMAGE + \$JUPYTERHUB_IMAGE_URI \$JUPYTERHUB_AUTHENTICATOR_AUTHORIZATION_URL \$JUPYTERHUB_AUTHENTICATOR_REFRESH_AGE \$JUPYTER_IDLE_SERVER_CULL_TIMEOUT diff --git a/birdhouse/components/jupyterhub/docker-compose-extra.yml b/birdhouse/components/jupyterhub/docker-compose-extra.yml index 15df6f921..d7722d2ee 100644 --- a/birdhouse/components/jupyterhub/docker-compose-extra.yml +++ b/birdhouse/components/jupyterhub/docker-compose-extra.yml @@ -9,7 +9,7 @@ x-logging: services: jupyterhub: - image: ${JUPYTERHUB_DOCKER}:${JUPYTERHUB_VERSION} + image: ${JUPYTERHUB_IMAGE} container_name: jupyterhub hostname: jupyterhub environment: diff --git a/birdhouse/components/jupyterhub/service-config.json.template b/birdhouse/components/jupyterhub/service-config.json.template index 5b0346394..cbf7c8f72 100644 --- a/birdhouse/components/jupyterhub/service-config.json.template +++ b/birdhouse/components/jupyterhub/service-config.json.template @@ -1,20 +1,31 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "jupyterhub", - "keywords": [ - "jupyterhub" - ], - "description": "An interactive development environment use to create and share documents that contain live code.", - "links": [ - { - "rel": "service", - "type": "text/html", - "href": "https://${PAVICS_FQDN_PUBLIC}/jupyter" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://jupyter.org/hub" - } - ] -} +[ + { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/1.2.0/node_registry.schema.json#service", + "name": "jupyterhub", + "version": "${JUPYTERHUB_VERSION}", + "types": [ + "jupyterhub" + ], + "keywords": [ + "jupyterhub" + ], + "description": "An interactive development environment use to create and share documents that contain live code.", + "links": [ + { + "rel": "service", + "type": "text/html", + "href": "https://${PAVICS_FQDN_PUBLIC}/jupyter" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://jupyter.org/hub" + }, + { + "rel": "service-meta", + "type": "application/vnd.oci.image.index.v1+json", + "href": "${JUPYTERHUB_IMAGE_URI}" + } + ] + } +] diff --git a/birdhouse/components/raven/default.env b/birdhouse/components/raven/default.env index 86d1e7073..a97f6a281 100644 --- a/birdhouse/components/raven/default.env +++ b/birdhouse/components/raven/default.env @@ -8,6 +8,9 @@ export RAVEN_GEO_URL="https://pavics.ouranos.ca/geoserver/" export RAVEN_VERSION="0.18.1" +export RAVEN_DOCKER=pavics/raven +export RAVEN_IMAGE='${RAVEN_DOCKER}:${RAVEN_VERSION}' +export RAVEN_IMAGE_URI='registry.hub.docker.com/${RAVEN_IMAGE}' # This tells the postgres service to create a database on startup export POSTGRES_DATABASES_TO_CREATE=" @@ -27,7 +30,16 @@ COMPONENT_DEPENDENCIES=" ./components/wps_outputs-volume " +export DELAYED_EVAL=" + $DELAYED_EVAL + RAVEN_IMAGE + RAVEN_IMAGE_URI +" + OPTIONAL_VARS=" $OPTIONAL_VARS \$RAVEN_VERSION + \$RAVEN_DOCKER + \$RAVEN_IMAGE + \$RAVEN_IMAGE_URI " diff --git a/birdhouse/components/raven/docker-compose-extra.yml b/birdhouse/components/raven/docker-compose-extra.yml index 25d0d3a79..9c8a36e3f 100644 --- a/birdhouse/components/raven/docker-compose-extra.yml +++ b/birdhouse/components/raven/docker-compose-extra.yml @@ -9,7 +9,7 @@ x-logging: services: raven: - image: pavics/raven:${RAVEN_VERSION} + image: ${RAVEN_IMAGE} container_name: raven environment: PYWPS_CFG: /wps.cfg diff --git a/birdhouse/components/raven/service-config.json.template b/birdhouse/components/raven/service-config.json.template index dd9d1e49f..dfaa0ba4a 100644 --- a/birdhouse/components/raven/service-config.json.template +++ b/birdhouse/components/raven/service-config.json.template @@ -1,25 +1,37 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "raven", - "keywords": [ - "service-wps" - ], - "description": "A WPS providing hydrological modeling and analysis services.", - "links": [ - { - "rel": "service", - "type": "text/xml", - "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&request=GetCapabilities" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://pavics-raven.readthedocs.io" - }, - { - "rel": "service-desc", - "type": "text/xml", - "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&request=GetCapabilities" - } - ] -} +[ + { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/1.2.0/node_registry.schema.json#service", + "name": "raven", + "version": "${RAVEN_VERSION}", + "types": [ + "wps" + ], + "keywords": [ + "service-wps" + ], + "description": "A WPS providing hydrological modeling and analysis services.", + "links": [ + { + "rel": "service", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://pavics-raven.readthedocs.io" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/raven?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-meta", + "type": "application/vnd.oci.image.index.v1+json", + "href": "${RAVEN_IMAGE_URI}" + } + ] + } + +] diff --git a/birdhouse/components/stac/config/canarie-api/canarie_api_monitoring.py.template b/birdhouse/components/stac/config/canarie-api/canarie_api_monitoring.py.template index 3fba3b4a8..3c728409f 100644 --- a/birdhouse/components/stac/config/canarie-api/canarie_api_monitoring.py.template +++ b/birdhouse/components/stac/config/canarie-api/canarie_api_monitoring.py.template @@ -2,7 +2,7 @@ SERVICES['STAC'] = { 'info': { 'name': 'STAC', 'synopsis': 'STAC is the common name of the REST API that implements the STAC specification, common representation of geospatial information.', - 'version': "", + 'version': '${STAC_VERSION}', 'institution': 'CRIM', 'releaseTime': "2023-06-16T00:00:00Z", 'researchSubject': 'Any', diff --git a/birdhouse/components/stac/default.env b/birdhouse/components/stac/default.env index 25b7916be..85460bd40 100644 --- a/birdhouse/components/stac/default.env +++ b/birdhouse/components/stac/default.env @@ -3,6 +3,20 @@ export STAC_POSTGRES_PASSWORD=${POSTGRES_PAVICS_PASSWORD} export STAC_PGUSER=${POSTGRES_PAVICS_USERNAME} export STAC_PGPASSWORD=${POSTGRES_PAVICS_PASSWORD} +# 'main' branch points at https://github.com/stac-utils/stac-fastapi/commit/d53e792 +# (see: https://github.com/crim-ca/stac-app/blob/40cad1aa7a094d58fca2d3184182761e248f781d/Dockerfile#L15-L20) +# which corresponds to 2.4.3 release +export STAC_VERSION=2.4.3-crim-main +export STAC_IMAGE='ghcr.io/crim-ca/stac-app:main' +export STAC_IMAGE_URI='${STAC_IMAGE}' + +# 'docker_image_push' branch points at https://github.com/crim-ca/stac-browser/tree/docker_image_push +# which is some commits ahead of 'v3.0.0-beta.5' (and multiple behind latest official releases) +# version name is slightly tweaked to fulfill schema while leaving an obvious trace +export STAC_BROWSER_VERSION=3.0.0-beta.5-crim-docker-image-push +export STAC_BROWSER_IMAGE='ghcr.io/crim-ca/stac-browser:docker_image_push' +export STAC_BROWSER_IMAGE_URI='${STAC_BROWSER_IMAGE}' + # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here # single quotes are important in below list to keep variable names intact until 'pavics-compose' parses them EXTRA_VARS=' @@ -14,3 +28,21 @@ EXTRA_VARS=' # extend the original 'VARS' from 'birdhouse/pavics-compose.sh' to employ them for template substitution # adding them to 'VARS', they will also be validated in case of override of 'default.env' using 'env.local' VARS="$VARS $EXTRA_VARS" + +export DELAYED_EVAL=" + $DELAYED_EVAL + STAC_IMAGE + STAC_IMAGE_URI + STAC_BROWSER_IMAGE + STAC_BROWSER_IMAGE_URI +" + +OPTIONAL_VARS=" + $OPTIONAL_VARS + \$STAC_VERSION + \$STAC_IMAGE + \$STAC_IMAGE_URI + \$STAC_BROWSER_VERSION + \$STAC_BROWSER_IMAGE + \$STAC_BROWSER_IMAGE_URI +" diff --git a/birdhouse/components/stac/docker-compose-extra.yml b/birdhouse/components/stac/docker-compose-extra.yml index 721a3dae8..6b7143ab7 100644 --- a/birdhouse/components/stac/docker-compose-extra.yml +++ b/birdhouse/components/stac/docker-compose-extra.yml @@ -10,7 +10,7 @@ x-logging: services: stac: container_name: stac - image: ghcr.io/crim-ca/stac-app:main + image: ${STAC_IMAGE} depends_on: - stac-db environment: @@ -28,7 +28,7 @@ services: stac-browser: container_name: stac-browser - image: ghcr.io/crim-ca/stac-browser:docker_image_push + image: ${STAC_BROWSER_IMAGE} environment: - CATALOG_URL=https://${PAVICS_FQDN_PUBLIC}/stac/ - ROOT_PATH=/stac-browser/ diff --git a/birdhouse/components/stac/service-config.json.template b/birdhouse/components/stac/service-config.json.template index 45297ea3a..f81be9a05 100644 --- a/birdhouse/components/stac/service-config.json.template +++ b/birdhouse/components/stac/service-config.json.template @@ -1,54 +1,77 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "stac", - "keywords": [ - "catalog" - ], - "description": "Searchable spatiotemporal metadata describing climate and Earth observation datasets.", - "links": [ - { - "rel": "service", - "type": "application/json", - "href": "https://${PAVICS_FQDN_PUBLIC}/stac/" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://github.com/radiantearth/stac-api-spec/tree/v1.0.0" - }, - { - "rel": "service-desc", - "type": "application/yaml", - "href": "https://raw.githubusercontent.com/radiantearth/stac-api-spec/main/core/openapi.yaml" - }, - { - "rel": "alternate", - "type": "text/html", - "href": "https://${PAVICS_FQDN_PUBLIC}/stac-browser/" - } - ] -}, { - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "stac-browser", - "keywords": [ - "catalog" - ], - "description": "A web interface to browse the STAC catalogs.", - "links": [ - { - "rel": "service", - "type": "text/html", - "href": "https://${PAVICS_FQDN_PUBLIC}/stac-browser/" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://github.com/radiantearth/stac-browser" - }, - { - "rel": "alternate", - "type": "application/json", - "href": "https://${PAVICS_FQDN_PUBLIC}/stac/" - } - ] -} +[ + { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/1.2.0/node_registry.schema.json#service", + "name": "stac", + "version": "${STAC_VERSION}", + "types": [ + "catalog", + "data" + ], + "keywords": [ + "catalog" + ], + "description": "Searchable spatiotemporal metadata describing climate and Earth observation datasets.", + "links": [ + { + "rel": "service", + "type": "application/json", + "href": "https://${PAVICS_FQDN_PUBLIC}/stac/" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://github.com/radiantearth/stac-api-spec/tree/v1.0.0" + }, + { + "rel": "service-desc", + "type": "application/yaml", + "href": "https://raw.githubusercontent.com/radiantearth/stac-api-spec/main/core/openapi.yaml" + }, + { + "rel": "alternate", + "type": "text/html", + "href": "https://${PAVICS_FQDN_PUBLIC}/stac-browser/" + }, + { + "rel": "service-meta", + "type": "application/vnd.oci.image.index.v1+json", + "href": "${STAC_IMAGE_URI}" + } + ] + }, + { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/1.2.0/node_registry.schema.json#service", + "name": "stac-browser", + "version": "${STAC_BROWSER_VERSION}", + "types": [ + "catalog", + "data" + ], + "keywords": [ + "catalog" + ], + "description": "A web interface to browse the STAC catalogs.", + "links": [ + { + "rel": "service", + "type": "text/html", + "href": "https://${PAVICS_FQDN_PUBLIC}/stac-browser/" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://github.com/radiantearth/stac-browser" + }, + { + "rel": "alternate", + "type": "application/json", + "href": "https://${PAVICS_FQDN_PUBLIC}/stac/" + }, + { + "rel": "service-meta", + "type": "application/vnd.oci.image.index.v1+json", + "href": "${STAC_BROWSER_IMAGE_URI}" + } + ] + } +] diff --git a/birdhouse/components/thredds/default.env b/birdhouse/components/thredds/default.env index 07b70a8aa..e97a1a41c 100644 --- a/birdhouse/components/thredds/default.env +++ b/birdhouse/components/thredds/default.env @@ -3,7 +3,8 @@ # thredds-docker >= 4.6.18 or >= 5.2 strongly recommended to avoid Log4J CVE-2021-44228. export THREDDS_VERSION=4.6.18-unidata-2022-01 export THREDDS_DOCKER=pavics/thredds-docker -export THREDDS_IMAGE="${THREDDS_DOCKER}:${THREDDS_VERSION}" +export THREDDS_IMAGE='${THREDDS_DOCKER}:${THREDDS_VERSION}' +export THREDDS_IMAGE_URI='registry.hub.docker.com/${THREDDS_IMAGE}' export THREDDS_ORGANIZATION="Birdhouse" export THREDDS_ADDITIONAL_CATALOG="" @@ -37,6 +38,8 @@ OPTIONAL_VARS=" \$TWITCHER_PROTECTED_PATH \$THREDDS_DOCKER \$THREDDS_VERSION + \$THREDDS_IMAGE + \$THREDDS_IMAGE_URI \$THREDDS_ADDITIONAL_CATALOG " @@ -44,4 +47,6 @@ export DELAYED_EVAL=" $DELAYED_EVAL THREDDS_DATASET_LOCATION_ON_HOST THREDDS_SERVICE_DATA_LOCATION_ON_HOST + THREDDS_IMAGE + THREDDS_IMAGE_URI " diff --git a/birdhouse/components/thredds/service-config.json.template b/birdhouse/components/thredds/service-config.json.template index 7441e9a91..d03703f6e 100644 --- a/birdhouse/components/thredds/service-config.json.template +++ b/birdhouse/components/thredds/service-config.json.template @@ -1,26 +1,38 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "thredds", - "keywords": [ - "data", - "catalog" - ], - "description": "A server that provides catalog, metadata, and data access services for scientific data.", - "links": [ - { - "rel": "service", - "type": "text/html", - "href": "https://${PAVICS_FQDN_PUBLIC}/thredds/" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://www.unidata.ucar.edu/software/tds/" - }, - { - "rel": "service-desc", - "type": "text/xml", - "href": "https://${PAVICS_FQDN_PUBLIC}/thredds/catalog.xml" - } - ] -} +[ + { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/1.2.0/node_registry.schema.json#service", + "name": "thredds", + "version": "${THREDDS_VERSION}", + "types": [ + "data", + "catalog" + ], + "keywords": [ + "data", + "catalog" + ], + "description": "A server that provides catalog, metadata, and data access services for scientific data.", + "links": [ + { + "rel": "service", + "type": "text/html", + "href": "https://${PAVICS_FQDN_PUBLIC}/thredds/" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://www.unidata.ucar.edu/software/tds/" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}/thredds/catalog.xml" + }, + { + "rel": "service-meta", + "type": "application/vnd.oci.image.index.v1+json", + "href": "${THREDDS_IMAGE_URI}" + } + ] + } +] diff --git a/birdhouse/components/weaver/default.env b/birdhouse/components/weaver/default.env index 6dccc8370..ca94f178f 100644 --- a/birdhouse/components/weaver/default.env +++ b/birdhouse/components/weaver/default.env @@ -36,6 +36,12 @@ VARS="$VARS $EXTRA_VARS" OPTIONAL_VARS=" $OPTIONAL_VARS + \$WEAVER_DOCKER + \$WEAVER_VERSION + \$WEAVER_WORKER_IMAGE + \$WEAVER_MANAGER_IMAGE + \$WEAVER_IMAGE + \$WEAVER_IMAGE_URI \$SECURE_DATA_PROXY_AUTH_INCLUDE " @@ -48,6 +54,11 @@ export WEAVER_CONFIG=HYBRID # default release version that will be used to fetch docker images (API mananger & celery workers services) export WEAVER_VERSION=4.32.0 +export WEAVER_DOCKER=pavics/weaver +export WEAVER_IMAGE='${WEAVER_DOCKER}:${WEAVER_VERSION}' +export WEAVER_MANAGER_IMAGE='${WEAVER_IMAGE}-manager' +export WEAVER_WORKER_IMAGE='${WEAVER_IMAGE}-worker' +export WEAVER_IMAGE_URI='registry.hub.docker.com/${WEAVER_IMAGE}' # default release of the MongoDB version employed by Weaver # NOTE: @@ -58,7 +69,7 @@ export WEAVER_MONGODB_VERSION=5.0.4 # if credentials are desired, they can be defined with the override of the URL variable export WEAVER_MONGODB_HOST=weaver-mongodb export WEAVER_MONGODB_PORT=27017 -export WEAVER_MONGODB_URL=mongodb://${WEAVER_MONGODB_HOST}:${WEAVER_MONGODB_PORT} +export WEAVER_MONGODB_URL='mongodb://${WEAVER_MONGODB_HOST}:${WEAVER_MONGODB_PORT}' # real names of the weaver/worker services # 'WEAVER_MANAGER_NAME' value will generate "/" URI to access its API behind secured proxy @@ -107,4 +118,9 @@ export DELAYED_EVAL=" $DELAYED_EVAL WEAVER_WPS_OUTPUTS_DIR WEAVER_MONGODB_DATA_DIR + WEAVER_MONGODB_URL + WEAVER_IMAGE + WEAVER_IMAGE_URI + WEAVER_MANAGER_IMAGE + WEAVER_WORKER_IMAGE " diff --git a/birdhouse/components/weaver/docker-compose-extra.yml b/birdhouse/components/weaver/docker-compose-extra.yml index ec3afc4fb..6091cd4a4 100644 --- a/birdhouse/components/weaver/docker-compose-extra.yml +++ b/birdhouse/components/weaver/docker-compose-extra.yml @@ -16,7 +16,7 @@ services: # Weaver is started, they will all be able to request them and register the providers correctly. # This is needed because simply adding 'depends_on' only ensures that containers are 'running', but startup of # WPS applications themselves are not necessarily completed. Successful HTTP responses ensure they are 'ready'. - image: pavics/weaver:${WEAVER_VERSION}-manager + image: ${WEAVER_MANAGER_IMAGE} environment: HOSTNAME: ${PAVICS_FQDN} FORWARDED_ALLOW_IPS: "*" @@ -42,7 +42,7 @@ services: # Image 'worker' is the job runner of Weaver, possibly spawning Docker applications according to process definition weaver-worker: - image: pavics/weaver:${WEAVER_VERSION}-worker + image: ${WEAVER_WORKER_IMAGE} container_name: ${WEAVER_WORKER_NAME} environment: DOCKER_HOST: "tcp://docker-proxy:2375" diff --git a/birdhouse/components/weaver/service-config.json.template b/birdhouse/components/weaver/service-config.json.template index 584ae4060..19f5a818a 100644 --- a/birdhouse/components/weaver/service-config.json.template +++ b/birdhouse/components/weaver/service-config.json.template @@ -1,30 +1,42 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "weaver", - "keywords": [ - "service-ogcapi_processes" - ], - "description": "An OGC-API flavored Execution Management Service", - "links": [ - { - "rel": "service", - "type": "application/json", - "href": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://pavics-weaver.readthedocs.io/" - }, - { - "rel": "service-desc", - "type": "application/json", - "href": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/" - }, - { - "rel": "conformance", - "type": "application/json", - "href": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/conformance/" - } - ] -} +[ + { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/1.2.0/node_registry.schema.json#service", + "name": "weaver", + "version": "${WEAVER_VERSION}", + "types": [ + "ogcapi_processes", + "wps" + ], + "keywords": [ + "service-ogcapi_processes" + ], + "description": "An OGC-API flavored Execution Management Service", + "links": [ + { + "rel": "service", + "type": "application/json", + "href": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://pavics-weaver.readthedocs.io/" + }, + { + "rel": "service-desc", + "type": "application/json", + "href": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/" + }, + { + "rel": "conformance", + "type": "application/json", + "href": "https://${PAVICS_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}/conformance/" + }, + { + "rel": "service-meta", + "type": "application/vnd.oci.image.index.v1+json", + "href": "${WEAVER_IMAGE_URI}" + } + ] + } +] diff --git a/birdhouse/deprecated-components/flyingpigeon/default.env b/birdhouse/deprecated-components/flyingpigeon/default.env index 7d6e9cc4c..25abdb974 100644 --- a/birdhouse/deprecated-components/flyingpigeon/default.env +++ b/birdhouse/deprecated-components/flyingpigeon/default.env @@ -1,4 +1,7 @@ export FLYINGPIGEON_VERSION=1.6 +export FLYINGPIGEON_DOCKER=birdhouse/flyingpigeon +export FLYINGPIGEON_IMAGE='${FLYINGPIGEON_DOCKER}:${FLYINGPIGEON_VERSION}' +export FLYINGPIGEON_IMAGE_URI='registry.hub.docker.com/${FLYINGPIGEON_IMAGE}' # This tells the postgres service to create a database on startup export POSTGRES_DATABASES_TO_CREATE=" @@ -21,4 +24,13 @@ COMPONENT_DEPENDENCIES=" OPTIONAL_VARS=" $OPTIONAL_VARS \$FLYINGPIGEON_VERSION + \$FLYINGPIGEON_DOCKER + \$FLYINGPIGEON_IMAGE + \$FLYINGPIGEON_IMAGE_URI +" + +export DELAYED_EVAL=" + $DELAYED_EVAL + FLYINGPIGEON_IMAGE + FLYINGPIGEON_IMAGE_URI " diff --git a/birdhouse/deprecated-components/flyingpigeon/docker-compose-extra.yml b/birdhouse/deprecated-components/flyingpigeon/docker-compose-extra.yml index 099f4952f..096bf326e 100644 --- a/birdhouse/deprecated-components/flyingpigeon/docker-compose-extra.yml +++ b/birdhouse/deprecated-components/flyingpigeon/docker-compose-extra.yml @@ -9,7 +9,7 @@ x-logging: services: flyingpigeon: - image: birdhouse/flyingpigeon:${FLYINGPIGEON_VERSION} + image: ${FLYINGPIGEON_IMAGE} container_name: flyingpigeon environment: - PYWPS_CFG=/wps.cfg diff --git a/birdhouse/deprecated-components/flyingpigeon/service-config.json.template b/birdhouse/deprecated-components/flyingpigeon/service-config.json.template index cb6e0c56e..23dfc61f0 100644 --- a/birdhouse/deprecated-components/flyingpigeon/service-config.json.template +++ b/birdhouse/deprecated-components/flyingpigeon/service-config.json.template @@ -1,25 +1,36 @@ -{ - "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/main/node_registry.schema.json#service", - "name": "flyingpigeon", - "keywords": [ - "service-wps" - ], - "description": "WPS for climate model data, indices and extreme events.", - "links": [ - { - "rel": "service", - "type": "text/xml", - "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon?service=WPS&request=GetCapabilities" - }, - { - "rel": "service-doc", - "type": "text/html", - "href": "https://flyingpigeon.readthedocs.io" - }, - { - "rel": "service-desc", - "type": "text/xml", - "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon?service=WPS&request=GetCapabilities" - } - ] -} +[ + { + "$schema": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry/1.2.0/node_registry.schema.json#service", + "name": "flyingpigeon", + "version": "${FLYINGPIGEON_VERSION}", + "types": [ + "wps" + ], + "keywords": [ + "service-wps" + ], + "description": "WPS for climate model data, indices and extreme events.", + "links": [ + { + "rel": "service", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-doc", + "type": "text/html", + "href": "https://flyingpigeon.readthedocs.io" + }, + { + "rel": "service-desc", + "type": "text/xml", + "href": "https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/flyingpigeon?service=WPS&request=GetCapabilities" + }, + { + "rel": "service-meta", + "type": "application/vnd.oci.image.index.v1+json", + "href": "${FLYINGPIGEON_IMAGE_URI}" + } + ] + } +] diff --git a/birdhouse/optional-components/testthredds/default.env b/birdhouse/optional-components/testthredds/default.env index 25dea9e9b..ed1ca81b9 100644 --- a/birdhouse/optional-components/testthredds/default.env +++ b/birdhouse/optional-components/testthredds/default.env @@ -4,7 +4,7 @@ # must use single quotes to avoid early expansion before overrides in env.local # are applied and must be added to the list of DELAYED_EVAL. -export TESTTHREDDS_IMAGE="$THREDDS_IMAGE" +export TESTTHREDDS_IMAGE='${THREDDS_IMAGE}' export TESTTHREDDS_INTERNAL_PORT="8080" # context root for Nginx proxy and Thredds catalog export TESTTHREDDS_CONTEXT_ROOT="testthredds" @@ -28,3 +28,8 @@ OPTIONAL_VARS=" COMPONENT_DEPENDENCIES=" ./components/thredds " + +export DELAYED_EVAL=" + $DELAYED_EVAL + TESTTHREDDS_IMAGE +" diff --git a/birdhouse/scripts/get-services-json.include.sh b/birdhouse/scripts/get-services-json.include.sh index b365ea4a5..dad1b4f97 100755 --- a/birdhouse/scripts/get-services-json.include.sh +++ b/birdhouse/scripts/get-services-json.include.sh @@ -12,7 +12,11 @@ fi for adir in ${ALL_CONF_DIRS}; do [ -f "${adir}/service-config.json" ] || continue - SERVICES="${SERVICES}$([ -n "${SERVICES}" ] && echo ',') $(cat "${adir}/service-config.json")" + # read and strip leading/trailing whitespaces + SERVICE_CONF="$(cat "${adir}/service-config.json" | sed -z 's/^\s*//;s/\s*$//')" + # remove the leading/trailing [] to get a pseudo-json of nested objects to extend the list + SERVICE_CONF="$(echo "${SERVICE_CONF}" | sed -z 's/^\s*\[\s*//;s/\s*\]\s*$//')" + SERVICES="${SERVICES}$([ -n "${SERVICES}" ] && echo ',') ${SERVICE_CONF}" done if [ -z "${SERVICES}" ]; then diff --git a/docs/source/conf.py b/docs/source/conf.py index 2c2f1d399..5e1abe514 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,9 +69,9 @@ # built documents. # # The short X.Y version. -version = '2.1.3' +version = '2.2.0' # The full version, including alpha/beta/rc tags. -release = '2.1.3' +release = '2.2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/tests/requirements.txt b/tests/requirements.txt index 212d9bfa3..83e4f5bf5 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,5 @@ pytest==7.2.2 +pytest-lazy-fixture==0.6.3 +python-dotenv==1.0.1 jsonschema==4.17.3 requests==2.31.0 diff --git a/tests/test_deployment.py b/tests/test_deployment.py index 926fc6ef2..c6d4f4e68 100644 --- a/tests/test_deployment.py +++ b/tests/test_deployment.py @@ -2,7 +2,9 @@ import json import os from string import Template +from typing import Any, Dict, List +import dotenv import jsonschema import pytest @@ -19,56 +21,105 @@ def root_dir(request): yield os.path.dirname(os.path.dirname(request.fspath)) -@pytest.fixture +@pytest.fixture(scope="module") def component_paths(root_dir): + # type: (str) -> List[str] yield [path for loc in COMPONENT_LOCATIONS for path in glob.glob(os.path.join(root_dir, "birdhouse", loc, "*"))] @pytest.fixture(scope="module") -def services_config_schema(): - branch = os.environ.get("DACCS_NODE_REGISTRY_BRANCH", "main") - schema = { +def component_service_configs(component_paths): + # type: (List[str]) -> List[str] + yield [ + os.path.join(component_path, "service-config.json.template") + for component_path in component_paths + if os.path.isfile(os.path.join(component_path, "service-config.json.template")) + ] + + +@pytest.fixture(scope="module") +def template_substitutions(component_paths): + # type: (List[str]) -> Dict[str, Any] + templates = {} + for component in component_paths: + component_default = os.path.join(component, "default.env") + if os.path.isfile(component_default): + templates.update(dotenv.dotenv_values(component_default)) + templates.update(TEMPLATE_SUBSTITUTIONS) + return templates + + +@pytest.fixture(scope="module", params=[pytest.lazy_fixture("component_service_configs")]) +def resolved_services_config_schema(request): + """ + For each of the services provided by ``component_paths`` fixture, obtain the referenced ``$schema``. + + If variable ``DACCS_NODE_REGISTRY_BRANCH`` is defined, the referenced ``$schema`` is ignored in favor of it. + """ + service_config_paths = request.param + assert service_config_paths, "Invalid service configuration. No service config found." + + # test override + branch = os.environ.get("DACCS_NODE_REGISTRY_BRANCH", None) + default_schema = { "$ref": "https://raw.githubusercontent.com/DACCS-Climate/Marble-node-registry" - f"/{branch}/node_registry.schema.json#service" + f"/{branch or 'main'}/node_registry.schema.json#service" } - return schema + if branch: + return [(default_schema, path) for path in service_config_paths] + + else: + service_config_schemas = [] + for config_path in service_config_paths: + with open(config_path, mode="r", encoding="utf-8") as config_file: + config_data = json.load(config_file) + if isinstance(config_data, dict): + config_data = [config_data] + for config_item in config_data: + config_schema = config_item.get("$schema", None) + config_schema = config_schema or default_schema + config_schema = {"$ref": config_schema} + service_config_schemas.append((config_schema, config_path)) + return service_config_schemas + + +def load_templated_service_config(service_config_path, template_variables): + # type: (str, Dict[str, Any]) -> List[Dict[str, Any]] + """ + Each service configuration file is expected to be an array of 'service' to allow multiple entries. + """ + with open(service_config_path) as service_config_file: + service_config_json = Template(service_config_file.read()).safe_substitute(template_variables) + service_configs = json.loads(service_config_json) + return service_configs class TestDockerCompose: - def test_service_config_name_same_as_dirname(self, component_paths): + def test_service_config_name_same_as_dirname(self, component_service_configs, template_substitutions): invalid_names = [] - for path in component_paths: - service_config_file = os.path.join(path, "service-config.json.template") - if os.path.isfile(service_config_file): - with open(service_config_file) as f: - config_info = Template(f.read()).safe_substitute(TEMPLATE_SUBSTITUTIONS) - service_configs = json.loads("[{}]".format(config_info)) - invalid_config_names = [] - for service_config in service_configs: - config_name = service_config.get("name") - path_name = os.path.basename(path) - if config_name == path_name: - # If at least one service_config in the file contains a matching name, that's ok - break - invalid_config_names.append((config_name, path_name)) - else: - invalid_names.extend(invalid_config_names) + for service_config_path in component_service_configs: + service_configs = load_templated_service_config(service_config_path, template_substitutions) + invalid_config_names = [] + for service_config in service_configs: + config_name = service_config.get("name") + path_name = os.path.basename(os.path.dirname(service_config_path)) + if config_name == path_name: + # If at least one service_config in the file contains a matching name, that's ok + break + invalid_config_names.append((config_name, path_name)) + else: + invalid_names.extend(invalid_config_names) assert not invalid_names, "service names in service-config.json.template should match the directory name" @pytest.mark.online - def test_service_config_valid(self, component_paths, services_config_schema): + def test_service_config_valid(self, resolved_services_config_schema, template_substitutions): invalid_schemas = [] - - for path in component_paths: - service_config_file = os.path.join(path, "service-config.json.template") - if os.path.isfile(service_config_file): - with open(service_config_file) as f: - config_info = Template(f.read()).safe_substitute(TEMPLATE_SUBSTITUTIONS) - service_configs = json.loads("[{}]".format(config_info)) - for service_config in service_configs: - try: - jsonschema.validate(instance=service_config, schema=services_config_schema) - except jsonschema.exceptions.ValidationError as e: - invalid_schemas.append(f"{os.path.basename(path)} contains invalid service configuration: {e}") + for service_config_schema, service_config_path in resolved_services_config_schema: + service_configs = load_templated_service_config(service_config_path, template_substitutions) + for service_config in service_configs: + try: + jsonschema.validate(instance=service_config, schema=service_config_schema) + except jsonschema.exceptions.ValidationError as e: + invalid_schemas.append(f"{service_config_path} contains invalid service configuration: {e}") assert not invalid_schemas, "\n".join(invalid_schemas)