node details overrides + logging utils

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.0.6
+current_version = 2.1.0
 commit = True
 tag = False
 tag_name = {new_version}
@@ -30,11 +30,11 @@ search = {current_version}
 replace = {new_version}
 
 [bumpversion:file:RELEASE.txt]
-search = {current_version} 2024-02-15T16:29:01Z
+search = {current_version} 2024-02-23T20:32:41Z
 replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ}
 
 [bumpversion:part:releaseTime]
-values = 2024-02-15T16:29:01Z
+values = 2024-02-23T20:32:41Z
 
 [bumpversion:file(version):birdhouse/components/canarie-api/docker_configuration.py.template]
 search = 'version': '{current_version}'

.gitignore

@@ -29,3 +29,4 @@ venv/
 
 ## Testing
 .pytest_cache/
+*.log

CHANGES.md

@@ -15,13 +15,49 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-## Fixes
+[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+
+[2.1.0](https://github.com/bird-house/birdhouse-deploy/tree/2.1.0) (2024-02-23)
+------------------------------------------------------------------------------------------------------------------
 
+## Changes
+- Compose script utilities:
+  * Add `BIRDHOUSE_COLOR` option and various logging/messaging definitions in `birdhouse/scripts/logging.include.sh`.
+  * Replace all explicit color "logging" related `echo` in scripts by a utility `log {LEVEL} {message}` function
+    that employs variables `LOG_DEBUG`, `LOG_INFO`, `LOG_WARN`, `LOG_ERROR` and `LOG_CRITICAL` as applicable per
+    respective messages to report logging messages in a standard approach.
+    Colors can be disabled with `BIRDHOUSE_COLOR=0` and logging level can be set with `BIRDHOUSE_LOG_LEVEL={LEVEL}`
+    where all levels above or equal to the configured one will be displayed (default logging level is `INFO`).
+  * Unify all `birdhouse/scripts` utilities to employ the same `COMPOSE_DIR` variable (auto-resolved or explicitly set)
+    in order to include or source any relevant dependencies they might have within the `birdhouse-deploy` repository.
+  * Add `info` option (ie: `pavics-compose.sh info`) that will stop processing just before `docker-compose` call.
+    This can be used to perform a "dry-run" of the command and validate that was is loaded is as expected, by inspecting
+    provided log messages.
+  * Replace older backtick (``` ` ```) executions by `$(...)` representation except for `eval` calls that require
+    them for backward compatibility of `sh` on some server instances.
+  * Modify the `sh -x` calls to scripts listed in `COMPONENT_PRE_COMPOSE_UP` and `COMPONENT_POST_COMPOSE_UP` to employ
+    the `-x` flag (showing commands) only when `BIRDHOUSE_LOG_LEVEL=DEBUG`.
+
+- Defaults:
+  * Add multiple `SERVER_[...]` variables with defaults using previously hard coded values referring to PAVICS.
+    These variables use a special combination of `DELAYED_EVAL` and `OPTIONAL_VARS` definitions that can make use
+    of a variable formatted as `<ANY_NAME>='${__DEFAULT__<ANY_NAME>}'` that will print a warning messages indicating
+    that the default is employed, although *STRONGLY* recommended to be overridden. This allows a middle ground between
+    backward-compatible `env.local` while flagging potentially misused configurations.
+
+## Fixes
+- Canarie-API: updated references
+  * Use the new `SERVER_[...]` variables.
+  * Replace the LICENSE URL of the server node pointing
+    at [Ouranosinc/pavics-sdi](https://github.com/Ouranosinc/pavics-sdi) instead
+    of intended [bird-house/birdhouse-deploy](https://github.com/bird-house/birdhouse-deploy).
+- Magpie: ensure that the `MAGPIE_ADMIN_USERNAME` variable is respected
+  * When determining the `JUPYTERHUB_ADMIN_USERS` variable
+  * Double check that it is being respected everywhere else
 - env.local.example: fix `JUPYTERHUB_CONFIG_OVERRIDE` comment section
 
   `JUPYTERHUB_CONFIG_OVERRIDE` was disconnected from its sample code.
 
-
 [2.0.6](https://github.com/bird-house/birdhouse-deploy/tree/2.0.6) (2024-02-15)
 ------------------------------------------------------------------------------------------------------------------
 

Makefile

@@ -1,7 +1,7 @@
 # Generic variables
 override SHELL       := bash
 override APP_NAME    := birdhouse-deploy
-override APP_VERSION := 2.0.6
+override APP_VERSION := 2.1.0
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")

README.rst

@@ -14,13 +14,13 @@ for a full-fledged production platform.
     * - releases
       - | |latest-version| |commits-since|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.0.6.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.1.0.svg
     :alt: Commits since latest release
-    :target: https://github.com/bird-house/birdhouse-deploy/compare/2.0.6...master
+    :target: https://github.com/bird-house/birdhouse-deploy/compare/2.1.0...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-2.0.6-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-2.1.0-blue.svg?style=flat
     :alt: Latest Tag
-    :target: https://github.com/bird-house/birdhouse-deploy/tree/2.0.6
+    :target: https://github.com/bird-house/birdhouse-deploy/tree/2.1.0
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
     :alt: ReadTheDocs Build Status (latest version)

RELEASE.txt

@@ -1 +1 @@
-2.0.6 2024-02-15T16:29:01Z
+2.1.0 2024-02-23T20:32:41Z

birdhouse/README.rst

@@ -148,7 +148,7 @@ instructions below.
 Manual instructions:
 
 * Go to
-  ``https://<PAVICS_FQDN>/magpie/ui/login`` and login with the ``admin`` user. The password should be in ``env.local``.
+  ``https://<PAVICS_FQDN>/magpie/ui/login`` and login with the ``MAGPIE_ADMIN_USERNAME`` user. The password should be in ``env.local``.
 
 * Then go to ``https://<PAVICS_FQDN>/magpie/ui/users/add``.
 

birdhouse/components/README.rst

@@ -304,7 +304,7 @@ birdhouse-deploy software stack and the machine that it is running on. It is hig
 make these routes available to anyone who does not have proper access permissions.
 
 Add existing users to the ``monitoring`` group to allow them access to the various monitoring WebUI.
-This way, we do not need to share the ``admin`` user account and do not have to add them to the
+This way, we do not need to share the ``MAGPIE_ADMIN_USERNAME`` user account and do not have to add them to the
 ``administrators`` group, which would give them too much permissions.
 
 

birdhouse/components/canarie-api/docker_configuration.py.template

@@ -109,25 +109,25 @@ SERVICES = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '2.0.6',
-            'releaseTime': '2024-02-15T16:29:01Z',
-            'institution': 'Ouranos',
-            'researchSubject': 'Climatology',
+            'version': '2.1.0',
+            'releaseTime': '2024-02-23T20:32:41Z',
+            'institution': '${SERVER_INSTITUTION}',
+            'researchSubject': '${SERVER_SUBJECT}',
             'supportEmail': '${SUPPORT_EMAIL}',
             'category': 'Resource/Cloud Management',
-            'tags': ['Climatology']
+            'tags': [tag.strip() for tag in "${SERVER_TAGS}".split(",") if tag.strip()],
         },
         'stats': {
             'method': '.*',
             'route': '(?!)'  # this will be set by CANARIE_STATS_ROUTES (see below)
         },
         'redirect': {
-            'doc': 'https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html',
-            'releasenotes': 'https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md',
-            'support': 'https://github.com/bird-house/birdhouse-deploy/issues',
+            'doc': '${SERVER_DOCUMENTATION_URL}',
+            'releasenotes': '${SERVER_RELEASE_NOTES_URL}',
+            'support': '${SERVER_SUPPORT_URL}',
             'source': 'https://github.com/bird-house/birdhouse-deploy',
             'tryme': 'https://${PAVICS_FQDN_PUBLIC}',
-            'licence': 'https://pavics-sdi.readthedocs.io/en/latest/license.html',
+            'licence': '${SERVER_LICENSE_URL}',
             'provenance': 'https://pavics-sdi.readthedocs.io/en/latest/provenance/index.html'
         },
         'monitoring': {}  # filled in after processing everything, see end of script
@@ -142,12 +142,12 @@ PLATFORMS = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '2.0.6',
-            'releaseTime': '2024-02-15T16:29:01Z',
-            'institution': 'Ouranos',
-            'researchSubject': 'Climatology',
+            'version': '2.1.0',
+            'releaseTime': '2024-02-23T20:32:41Z',
+            'institution': '${SERVER_INSTITUTION}',
+            'researchSubject': '${SERVER_SUBJECT}',
             'supportEmail': '${SUPPORT_EMAIL}',
-            'tags': ['Climatology', 'Cloud']
+            'tags': [tag.strip() for tag in "${SERVER_TAGS}".split(",") if tag.strip()],
         },
         'stats': {
             'method': '.*',

birdhouse/components/geoserver/pre-docker-compose-up

@@ -1,10 +1,17 @@
 #!/bin/sh
 
-THIS_FILE="`realpath "$0"`"
-THIS_DIR="`dirname "$THIS_FILE"`"
-COMPOSE_DIR="$THIS_DIR/../.."
+THIS_FILE="$(readlink -f "$0" || realpath "$0")"
+THIS_DIR="$(dirname "${THIS_FILE}")"
+COMPOSE_DIR="${COMPOSE_DIR:-$(dirname "${THIS_DIR}/..")}"
+
+if [ -f "${COMPOSE_DIR}/read-configs.include.sh" ]; then
+  . "${COMPOSE_DIR}/read-configs.include.sh"
+
+  # resolve GEOSERVER_DATA_DIR
+  read_configs
+fi
 
 if [ ! -f "${GEOSERVER_DATA_DIR}/global.xml" ]; then
-  echo "fix GeoServer data dir permission on first run only, when data dir do not exist yet."
-  FIRST_RUN_ONLY=1 "$COMPOSE_DIR"/deployment/fix-geoserver-data-dir-perm
+  log INFO "fix GeoServer data dir permission on first run only, when data dir do not exist yet."
+  FIRST_RUN_ONLY=1 "${COMPOSE_DIR}/deployment/fix-geoserver-data-dir-perm"
 fi

birdhouse/components/jupyterhub/default.env

@@ -68,9 +68,13 @@ export JUPYTERHUB_CRYPT_KEY=
 # JUPYTERHUB_CRYPT_KEY is set.
 export JUPYTERHUB_AUTHENTICATOR_REFRESH_AGE=60
 
+# Usernames that should be given admin access in jupyterhub
+export JUPYTERHUB_ADMIN_USERS='{\"${MAGPIE_ADMIN_USERNAME}\"}'  # python set syntax
+
 export DELAYED_EVAL="
   $DELAYED_EVAL
   JUPYTERHUB_USER_DATA_DIR
+  JUPYTERHUB_ADMIN_USERS
 "
 
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here

birdhouse/components/proxy/pre-docker-compose-up

@@ -3,12 +3,11 @@
 # Create JSON files containing the version information, available services, and
 # enabled components. These files will be served by the nginx proxy as static files.
 
-THIS_FILE="$(realpath "$0")"
-THIS_DIR="$(dirname "$THIS_FILE")"
+THIS_FILE="$(readlink -f "$0" || realpath "$0")"
+THIS_DIR="$(dirname "${THIS_FILE}")"
 
 mkdir -p "${THIS_DIR}/static"
 
 echo "${BIRDHOUSE_VERSION_JSON}" > "${THIS_DIR}/static/version.json"
 echo "${BIRDHOUSE_DEPLOY_SERVICES_JSON}" > "${THIS_DIR}/static/services.json"
 echo "${BIRDHOUSE_DEPLOY_COMPONENTS_JSON}" > "${THIS_DIR}/static/components.json"
-

birdhouse/default.env

@@ -6,6 +6,13 @@
 # must use single quotes to avoid early expansion before overrides in env.local
 # are applied and must be added to the list of DELAYED_EVAL.
 
+# Any default value that should be marked for security concern or recommended modificiation should
+# use the '${__DEFAULT__{var}}' naming format. These can then be referenced in 'env.local.example' to
+# avoid literal value duplication, and ensure they remain in sync. Also, those '${__DEFAULT__{var}}'
+# definitions should *NOT* be exported to avoid unnecessary polution of the environment variables.
+# Variables with format '${__DEFAULT__{var}}' will be flagged accordingly to their required/optional status
+# (see also: 'check_default_vars' in 'birdhouse/read-configs.include.sh').
+
 export BASH_IMAGE="bash:5.1.4"
 
 # Root directory under which all data persistence should be nested under
@@ -29,19 +36,86 @@ export PAVICS_FQDN_PUBLIC='${PAVICS_FQDN}'
 export DELAYED_EVAL="
   $DELAYED_EVAL
   PAVICS_FQDN_PUBLIC
+  DOC_URL
+  SUPPORT_EMAIL
+  SSL_CERTIFICATE
   DATA_PERSIST_SHARED_ROOT
   WPS_OUTPUTS_DIR
+  SERVER_NAME
+  SERVER_DESCRIPTION
+  SERVER_INSTITUTION
+  SERVER_SUBJECT
+  SERVER_TAGS
+  SERVER_DOCUMENTATION_URL
+  SERVER_RELEASE_NOTES_URL
+  SERVER_SUPPORT_URL
+  SERVER_LICENSE_URL
 "
 
-
-export SERVER_NAME=PAVICS
-export SERVER_DESCRIPTION="
+# Server Identification Details
+#   Following definitions should definitenly be updated.
+#   Previous defaults are defined for backward-compatibility.
+#   If not overridden explicitly by their non '__' prefixed variant,
+#   a WARN message will be displayed by pavics-compose.
+__DEFAULT__SERVER_NAME="PAVICS"
+__DEFAULT__SERVER_DESCRIPTION="
 The PAVICS (Power Analytics for Visualization of Climate Science) platform is a collection of
 climate analysis services served through Open Geospatial Consortium (OGC) protocols.
 These services include data access, processing and visualization. Both data and algorithms
 can be accessed either programmatically, through OGC-compliant clients such as QGIS or ArcGIS,
 or a custom web interface.
 "
+__DEFAULT__SERVER_INSTITUTION="Ouranos"
+__DEFAULT__SERVER_SUBJECT="Climatology"
+# below can be a CSV list of tags
+__DEFAULT__SERVER_TAGS="Climatology"
+__DEFAULT__SERVER_DOCUMENTATION_URL="https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html"
+__DEFAULT__SERVER_RELEASE_NOTES_URL="https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md"
+__DEFAULT__SERVER_SUPPORT_URL="https://github.com/bird-house/birdhouse-deploy/issues"
+# NOTE:
+#   This value does not use the previously hard coded default.
+#   Previous default pointed at the wrong repository with a mismatching LICENSE file.
+__DEFAULT__SERVER_LICENSE_URL="https://github.com/bird-house/birdhouse-deploy/blob/master/LICENSE"
+__DEFAULT__SUPPORT_EMAIL="[email protected]"
+__DEFAULT__DOC_URL="https://www.example.com/"
+__DEFAULT__PAVICS_FQDN="hostname.domainname"
+__DEFAULT__SSL_CERTIFICATE="/path/to/ssl/cert.pem"
+
+# apply overrides or fallback above defaults with delayed evaluation
+# exceptions for 'SUPPORT_EMAIL' and 'DOC_URL' using the old name for backward compatibility.
+export SUPPORT_EMAIL='${__DEFAULT__SUPPORT_EMAIL}'
+export DOC_URL='${__DEFAULT__DOC_URL}'
+export SSL_CERTIFICATE='${__DEFAULT__SSL_CERTIFICATE}'
+export SERVER_NAME='${__DEFAULT__SERVER_NAME}'
+export SERVER_DESCRIPTION='${__DEFAULT__SERVER_DESCRIPTION}'
+export SERVER_INSTITUTION='${__DEFAULT__SERVER_INSTITUTION}'
+export SERVER_SUBJECT='${__DEFAULT__SERVER_SUBJECT}'
+export SERVER_TAGS='${__DEFAULT__SERVER_TAGS}'
+export SERVER_DOCUMENTATION_URL='${__DEFAULT__SERVER_DOCUMENTATION_URL}'
+export SERVER_RELEASE_NOTES_URL='${__DEFAULT__SERVER_RELEASE_NOTES_URL}'
+export SERVER_SUPPORT_URL='${__DEFAULT__SERVER_SUPPORT_URL}'
+export SERVER_LICENSE_URL='${__DEFAULT__SERVER_LICENSE_URL}'
+
+# Defaults for required variables recommended for override for security reasons.
+# Those will not be set explicitly as defaults to ensure they are overridden explicitly by the instance.
+# These values would be detected only if the instance was configured using a copy of 'env.local.example'.
+__DEFAULT__MAGPIE_SECRET="itzaseekrit"
+__DEFAULT__MAGPIE_ADMIN_USERNAME="admin"
+__DEFAULT__MAGPIE_ADMIN_PASSWORD="qwertyqwerty!"
+__DEFAULT__POSTGRES_PAVICS_USERNAME="postgres-pavics"
+__DEFAULT__POSTGRES_PAVICS_PASSWORD="postgres-qwerty"
+__DEFAULT__POSTGRES_MAGPIE_USERNAME="postgres-magpie"
+__DEFAULT__POSTGRES_MAGPIE_PASSWORD="postgres-qwerty"
+__DEFAULT__GEOSERVER_ADMIN_USER="admingeo"
+__DEFAULT__GEOSERVER_ADMIN_PASSWORD="geoserverpass"
+#############################################################################
+# Deprecated vars (for components in the ./deprecated-components directory)
+#############################################################################
+__DEFAULT__TOMCAT_NCWMS_PASSWORD="ncwmspass"
+__DEFAULT__CATALOG_USERNAME="admin-catalog"
+__DEFAULT__CATALOG_PASSWORD="qwerty"
+__DEFAULT__PHOENIX_PASSWORD="phoenix_pass"
+__DEFAULT__PHOENIX_PASSWORD_HASH="sha256:123456789012:1234567890123456789012345678901234567890123456789012345678901234"
 
 export DEFAULT_CONF_DIRS='
   ./components/proxy

birdhouse/deployment/certbotwrapper

@@ -46,8 +46,8 @@ certbotwrapper START_TIME=$START_TIME"
 
 set -x
 
-THIS_FILE="`realpath "$0"`"
-THIS_DIR="`dirname "$THIS_FILE"`"
+THIS_FILE="$(readlink -f "$0" || realpath "$0")"
+THIS_DIR="$(dirname "${THIS_FILE}")"
 SAVED_PWD="`pwd`"
 
 . "$THIS_DIR/../read-configs.include.sh"