node details overrides + logging utils

.gitignore

@@ -29,3 +29,4 @@ venv/
 
 ## Testing
 .pytest_cache/
+*.log

CHANGES.md

@@ -15,7 +15,29 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+## Changes
+- Compose script utilities:
+  * Add `BIRDHOUSE_COLOR` option and various logging/messaging definitions in `birdhouse/scripts/logging.include.sh`.
+  * Replace all explicit color "logging" related `echo` in scripts by utility variables
+    `MSG_DEBUG`, `MSG_INFO`,  `MSG_WARN` and `MSG_ERROR` as applicable per respective messages.
+  * Move `read-configs.include.sh` into `birdhouse/scripts` along other include scripts.
+  * Add `info` option (ie: `pavics-compose.sh info`) that will stop processing just before `docker-compose` call.
+    This can be used to run a "dry-run" of the command and validate that was is loaded is as expected, by inspecting
+    provided log messages.
+
+- Defaults:
+  * Add multiple `SERVER_[...]` variables with defaults using previously hard coded values referring to PAVICS.
+    These variables use a special combination of `DELAYED_EVAL` and `OPTIONAL_VARS` definitions that can make use
+    of a variable formatted as `<ANY_NAME>='${__DEFAULT__<ANY_NAME>}'` that will print a warning messages indicating
+    that the default is employed, although *STRONGLY* recommended to be overridden. This allows a middle ground between
+    backward-compatible `env.local` while flagging potentially misused configurations.
+
+## Fixes
+- Canarie-API: updated references
+  * Use the new `SERVER_[...]` variables.
+  * Replace the LICENSE URL of the server node pointing
+    at [Ouranosinc/pavics-sdi](https://github.com/Ouranosinc/pavics-sdi) instead
+    of intended [bird-house/birdhouse-deploy](https://github.com/bird-house/birdhouse-deploy).
 
 [2.0.1](https://github.com/bird-house/birdhouse-deploy/tree/2.0.1) (2023-12-11)
 ------------------------------------------------------------------------------------------------------------------

birdhouse/components/canarie-api/docker_configuration.py.template

@@ -111,23 +111,23 @@ SERVICES = {
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
             'version': '2.0.1',
             'releaseTime': '2023-12-11T22:30:20Z',
-            'institution': 'Ouranos',
-            'researchSubject': 'Climatology',
+            'institution': '${SERVER_INSTITUTION}',
+            'researchSubject': '${SERVER_SUBJECT}',
             'supportEmail': '${SUPPORT_EMAIL}',
             'category': 'Resource/Cloud Management',
-            'tags': ['Climatology']
+            'tags': [tag.strip() for tag in "${SERVER_TAGS}".split(",") if tag.strip()],
         },
         'stats': {
             'method': '.*',
             'route': '(?!)'  # this will be set by CANARIE_STATS_ROUTES (see below)
         },
         'redirect': {
-            'doc': 'https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html',
-            'releasenotes': 'https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md',
-            'support': 'https://github.com/bird-house/birdhouse-deploy/issues',
+            'doc': '${SERVER_DOCUMENTATION_URL}',
+            'releasenotes': '${SERVER_RELEASE_NOTES_URL}',
+            'support': '${SERVER_SUPPORT_URL}',
             'source': 'https://github.com/bird-house/birdhouse-deploy',
             'tryme': 'https://${PAVICS_FQDN_PUBLIC}',
-            'licence': 'https://pavics-sdi.readthedocs.io/en/latest/license.html',
+            'licence': '${SERVER_LICENSE_URL}',
             'provenance': 'https://pavics-sdi.readthedocs.io/en/latest/provenance/index.html'
         },
         'monitoring': {}  # filled in after processing everything, see end of script

birdhouse/components/geoserver/pre-docker-compose-up

@@ -1,10 +1,14 @@
 #!/bin/sh
 
-THIS_FILE="`realpath "$0"`"
+THIS_FILE="`readlink -f "$0" || realpath "$0"`"
 THIS_DIR="`dirname "$THIS_FILE"`"
 COMPOSE_DIR="$THIS_DIR/../.."
 
+if [ -f "${COMPOSE_DIR}/scripts/logging.include.sh" ]; then
+  . "${COMPOSE_DIR}/scripts/logging.include.sh"
+fi
+
 if [ ! -f "${GEOSERVER_DATA_DIR}/global.xml" ]; then
-  echo "fix GeoServer data dir permission on first run only, when data dir do not exist yet."
+  echo "${MSG_INFO}fix GeoServer data dir permission on first run only, when data dir do not exist yet."
   FIRST_RUN_ONLY=1 "$COMPOSE_DIR"/deployment/fix-geoserver-data-dir-perm
 fi

birdhouse/components/proxy/pre-docker-compose-up

@@ -3,12 +3,11 @@
 # Create JSON files containing the version information, available services, and
 # enabled components. These files will be served by the nginx proxy as static files.
 
-THIS_FILE="$(realpath "$0")"
-THIS_DIR="$(dirname "$THIS_FILE")"
+THIS_FILE="`readlink -f "$0" || realpath "$0"`"
+THIS_DIR="`dirname "$THIS_FILE"`"
 
 mkdir -p "${THIS_DIR}/static"
 
 echo "${BIRDHOUSE_VERSION_JSON}" > "${THIS_DIR}/static/version.json"
 echo "${BIRDHOUSE_DEPLOY_SERVICES_JSON}" > "${THIS_DIR}/static/services.json"
 echo "${BIRDHOUSE_DEPLOY_COMPONENTS_JSON}" > "${THIS_DIR}/static/components.json"
-

birdhouse/default.env

@@ -31,17 +31,54 @@ export DELAYED_EVAL="
   PAVICS_FQDN_PUBLIC
   DATA_PERSIST_SHARED_ROOT
   WPS_OUTPUTS_DIR
+  SERVER_NAME
+  SERVER_DESCRIPTION
+  SERVER_INSTITUTION
+  SERVER_SUBJECT
+  SERVER_TAGS
+  SERVER_DOCUMENTATION_URL
+  SERVER_RELEASE_NOTES_URL
+  SERVER_SUPPORT_URL
+  SERVER_LICENSE_URL
 "
 
-
-export SERVER_NAME=PAVICS
-export SERVER_DESCRIPTION="
+# Server Identification Details
+#   Following definitions should definitenly be updated.
+#   Previous defaults are defined for backward-compatibility.
+#   If not overridden explicitly by their non '__' prefixed variant,
+#   a WARN message will be displayed by pavics-compose.
+export __DEFAULT__SERVER_NAME=PAVICS
+export __DEFAULT__SERVER_DESCRIPTION="
 The PAVICS (Power Analytics for Visualization of Climate Science) platform is a collection of
 climate analysis services served through Open Geospatial Consortium (OGC) protocols.
 These services include data access, processing and visualization. Both data and algorithms
 can be accessed either programmatically, through OGC-compliant clients such as QGIS or ArcGIS,
 or a custom web interface.
 "
+export __DEFAULT__SERVER_INSTITUTION=Ouranos
+export __DEFAULT__SERVER_SUBJECT=Climatology
+# below can be a CSV list of tags
+export __DEFAULT__SERVER_TAGS=Climatology
+export __DEFAULT__SERVER_DOCUMENTATION_URL=https://pavics-sdi.readthedocs.io/en/latest/arch/backend.html
+export __DEFAULT__SERVER_RELEASE_NOTES_URL=https://github.com/bird-house/birdhouse-deploy/blob/master/CHANGES.md
+export __DEFAULT__SERVER_SUPPORT_URL=https://github.com/bird-house/birdhouse-deploy/issues
+# NOTE:
+#   This value does not use the previously hard coded default.
+#   Previous default pointed at the wrong repository with a mismatching LICENSE file.
+export __DEFAULT__SERVER_LICENSE_URL=https://github.com/bird-house/birdhouse-deploy/blob/master/LICENSE
+
+# apply overrides or fallback above defaults with delayed evaluation
+# see also: 'SUPPORT_EMAIL'
+# This parameter is relevant to server details, but uses the old name for backward compatibility.
+export SERVER_NAME='${__DEFAULT__SERVER_NAME}'
+export SERVER_DESCRIPTION='${__DEFAULT__SERVER_DESCRIPTION}'
+export SERVER_INSTITUTION='${__DEFAULT__SERVER_INSTITUTION}'
+export SERVER_SUBJECT='${__DEFAULT__SERVER_SUBJECT}'
+export SERVER_TAGS='${__DEFAULT__SERVER_TAGS}'
+export SERVER_DOCUMENTATION_URL='${__DEFAULT__SERVER_DOCUMENTATION_URL}'
+export SERVER_RELEASE_NOTES_URL='${__DEFAULT__SERVER_RELEASE_NOTES_URL}'
+export SERVER_SUPPORT_URL='${__DEFAULT__SERVER_SUPPORT_URL}'
+export SERVER_LICENSE_URL='${__DEFAULT__SERVER_LICENSE_URL}'
 
 export DEFAULT_CONF_DIRS='
   ./components/proxy

birdhouse/deployment/certbotwrapper

@@ -46,7 +46,7 @@ certbotwrapper START_TIME=$START_TIME"
 
 set -x
 
-THIS_FILE="`realpath "$0"`"
+THIS_FILE="`readlink -f "$0" || realpath "$0"`"
 THIS_DIR="`dirname "$THIS_FILE"`"
 SAVED_PWD="`pwd`"
 

birdhouse/deployment/deploy.sh

@@ -105,7 +105,7 @@ cd $COMPOSE_DIR
 START_TIME="`date -Isecond`"
 echo "deploy START_TIME=$START_TIME"
 
-. "$COMPOSE_DIR/read-configs.include.sh"
+. "${COMPOSE_DIR}/scripts/read-configs.include.sh"
 
 # Read AUTODEPLOY_EXTRA_REPOS
 read_basic_configs_only
@@ -171,7 +171,7 @@ done
 cd $COMPOSE_DIR
 
 # reload again after git pull because this file could be changed by the pull
-. "$COMPOSE_DIR/read-configs.include.sh"
+. "${COMPOSE_DIR}/scripts/read-configs.include.sh"
 
 # reload again after default.env since env.local can override default.env
 # (ex: JUPYTERHUB_USER_DATA_DIR)

birdhouse/deployment/fix-geoserver-data-dir-perm

@@ -8,13 +8,13 @@
 # global.xml will exist and this script will not execute.  Without
 # FIRST_RUN_ONLY, this script will always execute.
 
-THIS_FILE="`realpath "$0"`"
+THIS_FILE="`readlink -f "$0" || realpath "$0"`"
 THIS_DIR="`dirname "$THIS_FILE"`"
 
 # Go to repo root.
 cd $THIS_DIR/../..
 
-. birdhouse/read-configs.include.sh
+. birdhouse/scripts/read-configs.include.sh
 
 # Get BASH_IMAGE
 # Get GEOSERVER_DATA_DIR

birdhouse/deployment/fix-write-perm

@@ -37,13 +37,13 @@
 # So the setfacl solution is the simplest, most portable/generic and most
 # localized (only the directories we need) solution.
 
-THIS_FILE="`realpath "$0"`"
+THIS_FILE="`readlink -f "$0" || realpath "$0"`"
 THIS_DIR="`dirname "$THIS_FILE"`"
 
 # Go to repo root.
 cd $THIS_DIR/../..
 
-. birdhouse/read-configs.include.sh
+. birdhouse/scripts/read-configs.include.sh
 
 # Get GEOSERVER_DATA_DIR, JUPYTERHUB_USER_DATA_DIR, MAGPIE_PERSIST_DIR
 read_configs

birdhouse/deployment/install-deploy-notebook

@@ -29,7 +29,7 @@ if [ ! -e "$REPO_ROOT/birdhouse/deployment/trigger-deploy-notebook" ]; then
     exit 2
 fi
 
-. "$REPO_ROOT/birdhouse/read-configs.include.sh"
+. "$REPO_ROOT/birdhouse/scripts/read-configs.include.sh"
 
 # Get JUPYTERHUB_USER_DATA_DIR
 read_configs

birdhouse/deployment/trigger-deploy-notebook

@@ -34,8 +34,8 @@ if [ -z "$COMPOSE_DIR" ]; then
     COMPOSE_DIR="$(dirname -- "$(dirname -- "$(realpath "$0")")")"
 fi
 
-if [ -f "$COMPOSE_DIR/read-configs.include.sh" ]; then
-    . "$COMPOSE_DIR/read-configs.include.sh"
+if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then
+    . "${COMPOSE_DIR}/scripts/read-configs.include.sh"
 
     # Get JUPYTERHUB_USER_DATA_DIR
     read_configs

birdhouse/deployment/triggerdeploy.sh

@@ -173,7 +173,7 @@ START_TIME="`date -Isecond`"
 echo "==========
 triggerdeploy START_TIME=$START_TIME"
 
-. "$COMPOSE_DIR/read-configs.include.sh"
+. "${COMPOSE_DIR}/scripts/read-configs.include.sh"
 
 # Read AUTODEPLOY_EXTRA_REPOS
 read_basic_configs_only

birdhouse/pavics-compose.sh

@@ -10,10 +10,6 @@
 # * Try to keep the same behavior/code, inside and outside of the
 #   autodeploy container to catch error early with the autodeploy.
 
-YELLOW=$(tput setaf 3)
-RED=$(tput setaf 1)
-NORMAL=$(tput sgr0)
-
 # list of all variables to be substituted in templates
 #   some of these variables *could* employ provided values in 'default.env',
 #   but they must ultimately be defined one way or another for the server to work
@@ -34,31 +30,38 @@ OPTIONAL_VARS='
   $EXTRA_PYWPS_CONFIG
   $SERVER_NAME
   $SERVER_DESCRIPTION
+  $SERVER_INSTITUTION
+  $SERVER_SUBJECT
+  $SERVER_TAGS
+  $SERVER_DOCUMENTATION_URL
+  $SERVER_RELEASE_NOTES_URL
+  $SERVER_SUPPORT_URL
+  $SERVER_LICENSE_URL
 '
 
 # we switch to the real directory of the script, so it still works when used from $PATH
 # tip: ln -s /path/to/pavics-compose.sh ~/bin/
 # Setup PWD for sourcing env.local.
-cd $(dirname $(readlink -f $0 || realpath $0))
+cd "$(dirname "$(readlink -f "$0" || realpath "$0")")" || exit 1
 
 # Setup COMPOSE_DIR for sourcing env.local.
 # Prevent un-expected difference when this script is run inside autodeploy
 # container and manually from the host.
 COMPOSE_DIR="`pwd`"
 
-. "$COMPOSE_DIR/read-configs.include.sh"
+. "${COMPOSE_DIR}/scripts/read-configs.include.sh"
 read_configs # this sets ALL_CONF_DIRS
 
-. ./scripts/get-components-json.include.sh
-. ./scripts/get-services-json.include.sh
-. ./scripts/get-version-json.include.sh
+. "${COMPOSE_DIR}/scripts/get-components-json.include.sh"
+. "${COMPOSE_DIR}/scripts/get-services-json.include.sh"
+. "${COMPOSE_DIR}/scripts/get-version-json.include.sh"
 
 for i in ${VARS}
 do
   v="${i}"
   if [ -z "`eval "echo ${v}"`" ]
   then
-    echo "${RED}Error${NORMAL}: Required variable $v is not set. Check env.local file."
+    echo "${MSG_ERROR}Required variable $v is not set. Check env.local file."
     exit 1
   fi
 done
@@ -67,15 +70,15 @@ done
 # will add delay
 # if [ ! -f $SSL_CERTIFICATE ]
 # then
-#   echo "Error, SSL certificate file $SSL_CERTIFICATE is missing"
+#   echo "${MSG_ERROR}SSL certificate file $SSL_CERTIFICATE is missing"
 #   exit 1
 # fi
 
 TIMEWAIT_REUSE=$(/sbin/sysctl -n  net.ipv4.tcp_tw_reuse)
-if [ ${TIMEWAIT_REUSE} -eq 0 ]
+if [ "${TIMEWAIT_REUSE}" -eq 0 ]
 then
-  echo "${YELLOW}Warning:${NORMAL} the sysctl net.ipv4.tcp_tw_reuse is not enabled"
-  echo "         It it suggested to set it to 1, otherwise the pavicscrawler may fail"
+  echo "${MSG_WARN}the sysctl net.ipv4.tcp_tw_reuse is not enabled. " \
+       "It it suggested to set it to 1, otherwise the pavicscrawler may fail."
 fi
 
 export AUTODEPLOY_EXTRA_REPOS_AS_DOCKER_VOLUMES=""
@@ -91,14 +94,14 @@ find $ALL_CONF_DIRS -name '*.template' |
   while read FILE
   do
     DEST=${FILE%.template}
-    cat ${FILE} | envsubst "$VARS" | envsubst "$OPTIONAL_VARS" > ${DEST}
+    cat "${FILE}" | envsubst "$VARS" | envsubst "$OPTIONAL_VARS" > "${DEST}"
   done
 
 if [ x"$1" = x"up" ]; then
   for adir in $ALL_CONF_DIRS; do
     COMPONENT_PRE_COMPOSE_UP="$adir/pre-docker-compose-up"
     if [ -x "$COMPONENT_PRE_COMPOSE_UP" ]; then
-      echo "executing '$COMPONENT_PRE_COMPOSE_UP'"
+      echo "${MSG_INFO}executing '$COMPONENT_PRE_COMPOSE_UP'"
       sh -x "$COMPONENT_PRE_COMPOSE_UP"
     fi
   done
@@ -108,6 +111,11 @@ create_compose_conf_list # this sets COMPOSE_CONF_LIST
 echo "COMPOSE_CONF_LIST="
 echo ${COMPOSE_CONF_LIST} | tr ' ' '\n' | grep -v '^-f'
 
+if [ x"$1" = x"info" ]; then
+  echo "${MSG_INFO}Stopping before execution of docker-compose command."
+  exit 0
+fi
+
 # the PROXY_SECURE_PORT is a little trick to make the compose file invalid without the usage of this wrapper script
 PROXY_SECURE_PORT=443 HOSTNAME=${PAVICS_FQDN} docker-compose ${COMPOSE_CONF_LIST} $*
 ERR=$?
@@ -116,7 +124,7 @@ ERR=$?
 type post-compose 2>&1 | grep 'post-compose is a function' > /dev/null
 if [ $? -eq 0 ]
 then
-  [ ${ERR} -gt 0 ] && { echo "Error occurred with docker-compose, not running post-compose"; exit $?; }
+  [ ${ERR} -gt 0 ] && { echo "${MSG_ERROR}Error occurred with docker-compose, not running post-compose"; exit $?; }
   post-compose $*
 fi
 
@@ -137,7 +145,7 @@ do
     for adir in $ALL_CONF_DIRS; do
       COMPONENT_POST_COMPOSE_UP="$adir/post-docker-compose-up"
       if [ -x "$COMPONENT_POST_COMPOSE_UP" ]; then
-        echo "executing '$COMPONENT_POST_COMPOSE_UP'"
+        echo "${MSG_INFO}executing '$COMPONENT_POST_COMPOSE_UP'"
         sh -x "$COMPONENT_POST_COMPOSE_UP"
       fi
     done

birdhouse/scripts/bootstrap-instance-for-testsuite

@@ -9,7 +9,7 @@
 # been called).
 #
 
-THIS_FILE="`realpath "$0"`"
+THIS_FILE="`readlink -f "$0" || realpath "$0"`"
 THIS_DIR="`dirname "$THIS_FILE"`"
 
 set -x

birdhouse/scripts/check-autodeploy-repos

@@ -2,12 +2,12 @@
 # Check Autodeploy repos status.
 # If there are changes or uncommitted, autodeploy will not trigger.
 
-THIS_FILE="`realpath "$0"`"
+THIS_FILE="`readlink -f "$0" || realpath "$0"`"
 THIS_DIR="`dirname "$THIS_FILE"`"
 COMPOSE_DIR="`dirname "$THIS_DIR"`"
 
-if [ -f "$COMPOSE_DIR/read-configs.include.sh" ]; then
-    . "$COMPOSE_DIR/read-configs.include.sh"
+if [ -f "${COMPOSE_DIR}/scripts/read-configs.include.sh" ]; then
+    . "${COMPOSE_DIR}/scripts/read-configs.include.sh"
 
     # Get AUTODEPLOY_EXTRA_REPOS
     read_configs