Merge branch 'master' into jupyter-behind-twitcher-keep-magpie-login

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.27.1
+current_version = 1.29.1
 commit = True
 tag = False
 tag_name = {new_version}
@@ -30,11 +30,11 @@ search = {current_version}
 replace = {new_version}
 
 [bumpversion:file:RELEASE.txt]
-search = {current_version} 2023-07-10T19:20:25Z
+search = {current_version} 2023-08-15T17:18:59Z
 replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ}
 
 [bumpversion:part:releaseTime]
-values = 2023-07-10T19:20:25Z
+values = 2023-08-15T17:18:59Z
 
 [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template]
 search = 'version': '{current_version}'

CHANGES.md

@@ -21,6 +21,62 @@
   - Sets magpie cookies whenever a user logs in or out through jupyterhub so that they are automatically logged in 
     or out through magpie as well.
 
+[1.29.1](https://github.com/bird-house/birdhouse-deploy/tree/1.29.1) (2023-08-15)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+
+- Small STAC changes
+  - This PR includes some changes that were suggested in a review for #297. But because the PR was already merged,
+    further updates are included here:
+    - removes extra block to include in docker compose files (no longer needed)
+    - moves docker compose file in `stac-public-access` component to the correct location
+    - uses `PAVICS_FQDN_PUBLIC` for public facing URLs in all places
+
+[1.29.0](https://github.com/bird-house/birdhouse-deploy/tree/1.29.0) (2023-08-10)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+- Do not expose additional ports:
+  - Docker compose no longer exposes any container ports outside the default network except for ports 80 and 443 from 
+    the proxy container. This ensures that ports that are not intended for external access are not exposed to the wider 
+    internet even if firewall rules are not set correctly.
+  - Note that if the `monitoring` component is used then port 9100 will be exposed from the `node-exporter` container.
+    This is because this container must be run on the host machine's network and unfortunately there is no known
+    workaround that would not require this port to be exposed on the host machine.
+  - Fixes https://github.com/bird-house/birdhouse-deploy/issues/222
+
+
+[1.28.0](https://github.com/bird-house/birdhouse-deploy/tree/1.28.0) (2023-08-10)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+- Adds [STAC](https://github.com/crim-ca/stac-app) to the stack (optional) when ``./components/stac`` 
+  is added to ``EXTRA_CONF_DIRS``. For more details, refer to 
+  [STAC Component](https://github.com/bird-house/birdhouse-deploy/blob/master/birdhouse/components/README.rst#STAC)
+  Following happens when enabled:
+
+  * Service ``stac`` (API) gets added with endpoints ``/twitcher/ows/proxy/stac`` and ``/stac``.
+
+  * STAC catalog can be explored via the ``stac-browser`` component, available under ``/stac-browser``.
+
+  * Image [crim-ca/stac-app](https://github.com/crim-ca/stac-app) is a STAC implementation based on 
+  [stac-utils/stac-fastapi](https://github.com/stac-utils/stac-fastapi).
+
+  * Image [crim-ca/stac-browser](https://github.com/crim-ca/stac-browser) is a fork of 
+  [radiantearth/stac-browser](https://github.com/radiantearth/stac-browser) in order to have the capacity to build 
+  the Docker container. The image reference will change when the 
+  [stac-browser PR related to Dockerfile](https://github.com/bird-house/birdhouse-deploy/issues/346) will have been 
+  merged.
+
+  * Adds `Magpie` permissions and service for `stac` endpoints.
+
+- Adds [stac-populator](https://github.com/crim-ca/stac-populator) to populate STAC catalog with sample collection 
+  items via [CEDA STAC Generator](https://github.com/cedadev/stac-generator), employed in sample 
+  [CMIP Dataset Ingestion Workflows](https://github.com/cedadev/stac-generator-example/tree/master/conf).
+
+- Adds ``optional-components/stac-public-access`` to give public access to the STAC catalog.
+
 [1.27.1](https://github.com/bird-house/birdhouse-deploy/tree/1.27.1) (2023-07-10)
 ------------------------------------------------------------------------------------------------------------------
 
@@ -186,6 +242,7 @@
 ------------------------------------------------------------------------------------------------------------------
 
 ## Changes
+
 - Update Zenodo config
   *  Add Misha to creators
   *  Add birdhouse community

Makefile

@@ -1,7 +1,7 @@
 # Generic variables
 override SHELL       := bash
 override APP_NAME    := birdhouse-deploy
-override APP_VERSION := 1.27.1
+override APP_VERSION := 1.29.1
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")

README.rst

@@ -14,13 +14,13 @@ for a full-fledged production platform.
     * - releases
       - | |latest-version| |commits-since|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.27.1.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.29.1.svg
     :alt: Commits since latest release
-    :target: https://github.com/bird-house/birdhouse-deploy/compare/1.27.1...master
+    :target: https://github.com/bird-house/birdhouse-deploy/compare/1.29.1...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-1.27.1-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-1.29.1-blue.svg?style=flat
     :alt: Latest Tag
-    :target: https://github.com/bird-house/birdhouse-deploy/tree/1.27.1
+    :target: https://github.com/bird-house/birdhouse-deploy/tree/1.29.1
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
     :alt: ReadTheDocs Build Status (latest version)

RELEASE.txt

@@ -1 +1 @@
-1.27.1 2023-07-10T19:20:25Z
+1.29.1 2023-08-15T17:18:59Z

birdhouse/components/README.rst

@@ -294,17 +294,14 @@ Prometheus stack is used:
 Usage
 -----
 
-- Grafana to view metric graphs: http://PAVICS_FQDN:3001/d/pf6xQMWGz/docker-and-system-monitoring
-- Prometheus alert rules: http://PAVICS_FQDN:9090/rules
-- AlertManager to manage alerts: http://PAVICS_FQDN:9093
+- Grafana to view metric graphs: https://PAVICS_FQDN/grafana/d/pf6xQMWGz/docker-and-system-monitoring
+- Prometheus alert rules: https://PAVICS_FQDN/prometheus/rules
+- AlertManager to manage alerts: https://PAVICS_FQDN/alertmanager
 
-The paths above are purposely not behind the proxy to not expose them publicly,
-assuming only ports 80 and 443 are publicly exposed on the internet.  All other
-ports are not exposed.
-
-Only Grafana has authentication, Prometheus alert rules and AlertManager have
-no authentication at all so had they been behind the proxy, anyone will be
-able to access them.
+The paths above are by default only accessible to a user logged in to magpie as an administrator.
+These routes provide sensitive information about the birdhouse-deploy software stack and the machine
+that it is running on. It is highly discouraged to make these routes available to anyone who is not
+an administrator.
 
 
 How to Enable the Component
@@ -427,7 +424,7 @@ How to Enable the Component
 
 - Edit ``env.local`` (a copy of `env.local.example`_)
 
-  - Add ``"./components/weaver"`` to ``EXTRA_CONF_DIRS``.
+  - Add ``./components/weaver`` to ``EXTRA_CONF_DIRS``.
 
   - Component ``birdhouse/optional-components/all-public-access`` should also be enabled to ensure that `Weaver`_
     can request ``GetCapabilities`` of every WPS provider to be registered. Publicly inaccessible services will not
@@ -505,7 +502,7 @@ How to Enable the Component
 ---------------------------
 
 - Edit ``env.local`` (a copy of `env.local.example`_)
-- Add ``"./components/cowbird"`` to ``EXTRA_CONF_DIRS``.
+- Add ``./components/cowbird`` to ``EXTRA_CONF_DIRS``.
 
 Customizing the Component
 -------------------------
@@ -519,3 +516,40 @@ define your custom values in ``env.local`` directly.
 
 .. |cowbird-default| replace:: cowbird/default.env
 .. _cowbird-default: ./cowbird/default.env
+
+
+STAC
+====
+
+`STAC`_ is the common name of the REST API that implements the STAC specification, common representation of geospatial 
+information.
+
+.. _STAC: https://stacspec.org/en
+
+Usage
+-----
+
+The STAC API can be browsed via the ``stac-browser`` component. By default, the browser will point to the STAC API 
+exposed by the current stack instance. Once this component is enabled, STAC API will be accessible at 
+``https://<PAVICS_FQDN_PUBLIC>/stac`` endpoint and the STAC browser will be available at 
+``https://<PAVICS_FQDN_PUBLIC>/stac-browser`` endpoint. In order to make the STAC browser the default entrypoint, 
+define the following in the ``env.local`` file::
+
+  export PROXY_ROOT_LOCATION="return 302 https://\$host/stac-browser;"
+
+Here is a sample search query using a CLI::
+
+.. code-block:: shell
+
+    pip install pystac-client
+    stac-client search $PAVIS_FQDN/stac -q "variable_id=txgt_32" "scenario=ssp585"
+
+Calls to the STAC API pass through Twitcher in order to validate authorization. Unauthenticated users will have 
+read-only access by default to STAC API resources while members of the `stac-admin` group can create and modify 
+resources. STAC Browser is not protected by any authorization mechanism.
+
+How to Enable the Component
+---------------------------
+
+- Edit ``env.local`` (a copy of `env.local.example`_)
+- Add ``./optional-components/stac`` to ``EXTRA_CONF_DIRS``.

birdhouse/components/monitoring/.gitignore

@@ -3,3 +3,5 @@ grafana_datasources.yml
 grafana_dashboards.yml
 alertmanager.yml
 prometheus.rules
+config/magpie/config.yml
+config/proxy/conf.extra-service.d/monitoring.conf

birdhouse/components/monitoring/config/magpie/config.yml.template

@@ -0,0 +1,54 @@
+providers:
+  grafana:
+    # below URL is only used to fill in the required location in Magpie
+    # actual auth validation is performed with Twitcher 'verify' endpoint without accessing this proxied URL
+    url: http://proxy:80
+    title: Grafana
+    public: true
+    c4i: false
+    type: api
+    sync_type: api
+  prometheus:
+    # below URL is only used to fill in the required location in Magpie
+    # actual auth validation is performed with Twitcher 'verify' endpoint without accessing this proxied URL
+    url: http://proxy:80
+    title: Prometheus
+    public: true
+    c4i: false
+    type: api
+    sync_type: api
+  alertmanager:
+    # below URL is only used to fill in the required location in Magpie
+    # actual auth validation is performed with Twitcher 'verify' endpoint without accessing this proxied URL
+    url: http://proxy:80
+    title: AlertManager
+    public: true
+    c4i: false
+    type: api
+    sync_type: api
+
+permissions:
+  - service: grafana
+    permission: read
+    group: administrators
+    action: create
+  - service: grafana
+    permission: write
+    group: administrators
+    action: create
+  - service: prometheus
+    permission: read
+    group: administrators
+    action: create
+  - service: prometheus
+    permission: write
+    group: administrators
+    action: create
+  - service: alertmanager
+    permission: read
+    group: administrators
+    action: create
+  - service: alertmanager
+    permission: write
+    group: administrators
+    action: create

birdhouse/components/monitoring/config/magpie/docker-compose-extra.yml

@@ -0,0 +1,7 @@
+version: "3.4"
+
+services:
+  magpie:
+    volumes:
+      - ./components/monitoring/config/magpie/config.yml:${MAGPIE_PERMISSIONS_CONFIG_PATH}/monitoring.yml:ro
+      - ./components/monitoring/config/magpie/config.yml:${MAGPIE_PROVIDERS_CONFIG_PATH}/monitoring.yml:ro

birdhouse/components/monitoring/config/proxy/conf.extra-service.d/monitoring.conf.template

@@ -0,0 +1,57 @@
+
+    location /grafana {
+        auth_request /secure-grafana-auth;
+        auth_request_set $auth_status $upstream_status;
+        proxy_pass http://grafana:3000;
+        proxy_set_header Host $host;
+    }
+
+    location /prometheus {
+        auth_request /secure-prometheus-auth;
+        auth_request_set $auth_status $upstream_status;
+        proxy_pass http://prometheus:9090;
+        proxy_set_header Host $host;
+    }
+
+    location /alertmanager {
+        auth_request /secure-alertmanager-auth;
+        auth_request_set $auth_status $upstream_status;
+        proxy_pass http://alertmanager:9093;
+        proxy_set_header Host $host;
+    }
+
+    location = /secure-grafana-auth {
+        internal;
+        proxy_pass https://${PAVICS_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/grafana$request_uri;
+        proxy_pass_request_body off;
+        proxy_set_header Host $host;
+        proxy_set_header Content-Length "";
+        proxy_set_header X-Original-URI $request_uri;
+        proxy_set_header X-Forwarded-Proto $real_scheme;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host $host:$server_port;
+    }
+
+    location = /secure-prometheus-auth {
+        internal;
+        proxy_pass https://${PAVICS_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/prometheus$request_uri;
+        proxy_pass_request_body off;
+        proxy_set_header Host $host;
+        proxy_set_header Content-Length "";
+        proxy_set_header X-Original-URI $request_uri;
+        proxy_set_header X-Forwarded-Proto $real_scheme;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host $host:$server_port;
+    }
+
+    location = /secure-alertmanager-auth {
+        internal;
+        proxy_pass https://${PAVICS_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/alertmanager$request_uri;
+        proxy_pass_request_body off;
+        proxy_set_header Host $host;
+        proxy_set_header Content-Length "";
+        proxy_set_header X-Original-URI $request_uri;
+        proxy_set_header X-Forwarded-Proto $real_scheme;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host $host:$server_port;
+    }

birdhouse/components/monitoring/config/proxy/docker-compose-extra.yml

@@ -0,0 +1,6 @@
+version: "3.4"
+
+services:
+  proxy:
+    volumes:
+      - ./components/monitoring/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/monitoring:ro

birdhouse/components/monitoring/docker-compose-extra.yml

@@ -11,8 +11,6 @@ services:
       - /var/run:/var/run:ro
       - /sys:/sys:ro
       - /var/lib/docker:/var/lib/docker:ro
-    ports:
-      - 9999:8080
     devices:
       - /dev/kmsg
     restart: always
@@ -38,8 +36,6 @@ services:
       - ./components/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
       - ./components/monitoring/prometheus.rules:/etc/prometheus/prometheus.rules:ro
       - prometheus_persistence:/prometheus:rw
-    ports:
-      - 9090:9090
     command:
       # restore original CMD from image
       - --config.file=/etc/prometheus/prometheus.yml
@@ -49,7 +45,7 @@ services:
       # https://prometheus.io/docs/prometheus/latest/storage/
       - --storage.tsdb.retention.time=90d
       # wrong default was http://container-hash:9090/
-      - --web.external-url=http://${PAVICS_FQDN}:9090/
+      - --web.external-url=https://${PAVICS_FQDN_PUBLIC}/prometheus/
     restart: always
 
   # https://grafana.com/docs/grafana/latest/installation/docker/
@@ -65,8 +61,9 @@ services:
       - grafana_persistence:/var/lib/grafana:rw
     environment:
       GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
-    ports:
-      - 3001:3000
+      GF_SERVER_ROOT_URL: https://${PAVICS_FQDN_PUBLIC}/grafana
+      GF_SERVER_SERVE_FROM_SUB_PATH: 'true'
+      GF_SERVER_DOMAIN: ${PAVICS_FQDN_PUBLIC}
     restart: always
 
   # https://github.com/prometheus/alertmanager
@@ -86,9 +83,7 @@ services:
       # enable debug logging
       - --log.level=debug
       # wrong default was http://container-hash:9093/
-      - --web.external-url=http://${PAVICS_FQDN}:9093/
-    ports:
-      - 9093:9093
+      - --web.external-url=https://${PAVICS_FQDN_PUBLIC}/alertmanager
     restart: always
 
 volumes:

birdhouse/components/monitoring/grafana_datasources.yml.template

@@ -6,6 +6,6 @@ datasources:
     type: prometheus
     access: proxy
     uid: local_pavics_prometheus
-    url: http://${PAVICS_FQDN}:9090
+    url: http://prometheus:9090/prometheus
     isDefault: true
     editable: false