Merge branch 'master' of github.com:bird-house/birdhouse-deploy into …

…jenkins-contribution

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.27.1
+current_version = 1.29.2
 commit = True
 tag = False
 tag_name = {new_version}
@@ -30,11 +30,11 @@ search = {current_version}
 replace = {new_version}
 
 [bumpversion:file:RELEASE.txt]
-search = {current_version} 2023-07-10T19:20:25Z
+search = {current_version} 2023-08-24T15:56:45Z
 replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ}
 
 [bumpversion:part:releaseTime]
-values = 2023-07-10T19:20:25Z
+values = 2023-08-24T15:56:45Z
 
 [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template]
 search = 'version': '{current_version}'

CHANGES.md

@@ -15,6 +15,52 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
+[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+
+[1.29.2](https://github.com/bird-house/birdhouse-deploy/tree/1.29.2) (2023-08-24)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+
+- Monitoring: allow access to magpie members of group `monitoring`
+
+  To allow accessing the various monitoring WebUI without having full blown
+  magpie admin priviledge to add and remove users.
+
+  Add existing users to this new `monitoring` group to allow them access to the
+  various monitoring WebUI.  This way, we do not need to share the `admin` user
+  account and do not have to add them to the `administrators` group.
+
+
+[1.29.1](https://github.com/bird-house/birdhouse-deploy/tree/1.29.1) (2023-08-15)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+
+- Small STAC changes
+  - This PR includes some changes that were suggested in a review for #297. But because the PR was already merged,
+    further updates are included here:
+    - removes extra block to include in docker compose files (no longer needed)
+    - moves docker compose file in `stac-public-access` component to the correct location
+    - uses `PAVICS_FQDN_PUBLIC` for public facing URLs in all places
+
+[1.29.0](https://github.com/bird-house/birdhouse-deploy/tree/1.29.0) (2023-08-10)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+- Do not expose additional ports:
+  - Docker compose no longer exposes any container ports outside the default network except for ports 80 and 443 from 
+    the proxy container. This ensures that ports that are not intended for external access are not exposed to the wider 
+    internet even if firewall rules are not set correctly.
+  - Note that if the `monitoring` component is used then port 9100 will be exposed from the `node-exporter` container.
+    This is because this container must be run on the host machine's network and unfortunately there is no known
+    workaround that would not require this port to be exposed on the host machine.
+  - Fixes https://github.com/bird-house/birdhouse-deploy/issues/222
+
+
+[1.28.0](https://github.com/bird-house/birdhouse-deploy/tree/1.28.0) (2023-08-10)
+------------------------------------------------------------------------------------------------------------------
+
 ## Changes
 - Adds [STAC](https://github.com/crim-ca/stac-app) to the stack (optional) when ``./components/stac`` 
   is added to ``EXTRA_CONF_DIRS``. For more details, refer to 
@@ -207,6 +253,7 @@
 ------------------------------------------------------------------------------------------------------------------
 
 ## Changes
+
 - Update Zenodo config
   *  Add Misha to creators
   *  Add birdhouse community

Makefile

@@ -1,7 +1,7 @@
 # Generic variables
 override SHELL       := bash
 override APP_NAME    := birdhouse-deploy
-override APP_VERSION := 1.27.1
+override APP_VERSION := 1.29.2
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")

README.rst

@@ -14,13 +14,13 @@ for a full-fledged production platform.
     * - releases
       - | |latest-version| |commits-since|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.27.1.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.29.2.svg
     :alt: Commits since latest release
-    :target: https://github.com/bird-house/birdhouse-deploy/compare/1.27.1...master
+    :target: https://github.com/bird-house/birdhouse-deploy/compare/1.29.2...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-1.27.1-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-1.29.2-blue.svg?style=flat
     :alt: Latest Tag
-    :target: https://github.com/bird-house/birdhouse-deploy/tree/1.27.1
+    :target: https://github.com/bird-house/birdhouse-deploy/tree/1.29.2
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
     :alt: ReadTheDocs Build Status (latest version)

RELEASE.txt

@@ -1 +1 @@
-1.27.1 2023-07-10T19:20:25Z
+1.29.2 2023-08-24T15:56:45Z

birdhouse/components/README.rst

@@ -294,17 +294,18 @@ Prometheus stack is used:
 Usage
 -----
 
-- Grafana to view metric graphs: http://PAVICS_FQDN:3001/d/pf6xQMWGz/docker-and-system-monitoring
-- Prometheus alert rules: http://PAVICS_FQDN:9090/rules
-- AlertManager to manage alerts: http://PAVICS_FQDN:9093
-
-The paths above are purposely not behind the proxy to not expose them publicly,
-assuming only ports 80 and 443 are publicly exposed on the internet.  All other
-ports are not exposed.
-
-Only Grafana has authentication, Prometheus alert rules and AlertManager have
-no authentication at all so had they been behind the proxy, anyone will be
-able to access them.
+- Grafana to view metric graphs: https://PAVICS_FQDN/grafana/d/pf6xQMWGz/docker-and-system-monitoring
+- Prometheus alert rules: https://PAVICS_FQDN/prometheus/rules
+- AlertManager to manage alerts: https://PAVICS_FQDN/alertmanager
+
+The paths above are by default only accessible to a user logged in to magpie as an administrator or
+as a member of group ``monitoring``.  These routes provide sensitive information about the
+birdhouse-deploy software stack and the machine that it is running on. It is highly discouraged to
+make these routes available to anyone who does not have proper access permissions.
+
+Add existing users to the ``monitoring`` group to allow them access to the various monitoring WebUI.
+This way, we do not need to share the ``admin`` user account and do not have to add them to the
+``administrators`` group, which would give them too much permissions.
 
 
 How to Enable the Component

birdhouse/components/monitoring/.gitignore

@@ -3,3 +3,5 @@ grafana_datasources.yml
 grafana_dashboards.yml
 alertmanager.yml
 prometheus.rules
+config/magpie/config.yml
+config/proxy/conf.extra-service.d/monitoring.conf

birdhouse/components/monitoring/config/magpie/config.yml.template

@@ -0,0 +1,78 @@
+providers:
+  grafana:
+    # below URL is only used to fill in the required location in Magpie
+    # actual auth validation is performed with Twitcher 'verify' endpoint without accessing this proxied URL
+    url: http://proxy:80
+    title: Grafana
+    public: true
+    c4i: false
+    type: api
+    sync_type: api
+  prometheus:
+    # below URL is only used to fill in the required location in Magpie
+    # actual auth validation is performed with Twitcher 'verify' endpoint without accessing this proxied URL
+    url: http://proxy:80
+    title: Prometheus
+    public: true
+    c4i: false
+    type: api
+    sync_type: api
+  alertmanager:
+    # below URL is only used to fill in the required location in Magpie
+    # actual auth validation is performed with Twitcher 'verify' endpoint without accessing this proxied URL
+    url: http://proxy:80
+    title: AlertManager
+    public: true
+    c4i: false
+    type: api
+    sync_type: api
+
+permissions:
+  - service: grafana
+    permission: read
+    group: administrators
+    action: create
+  - service: grafana
+    permission: write
+    group: administrators
+    action: create
+  - service: prometheus
+    permission: read
+    group: administrators
+    action: create
+  - service: prometheus
+    permission: write
+    group: administrators
+    action: create
+  - service: alertmanager
+    permission: read
+    group: administrators
+    action: create
+  - service: alertmanager
+    permission: write
+    group: administrators
+    action: create
+  - service: grafana
+    permission: read
+    group: monitoring
+    action: create
+  - service: grafana
+    permission: write
+    group: monitoring
+    action: create
+  - service: prometheus
+    permission: read
+    group: monitoring
+    action: create
+  - service: prometheus
+    permission: write
+    group: monitoring
+    action: create
+  - service: alertmanager
+    permission: read
+    group: monitoring
+    action: create
+  - service: alertmanager
+    permission: write
+    group: monitoring
+    action: create

birdhouse/components/monitoring/config/magpie/docker-compose-extra.yml

@@ -0,0 +1,7 @@
+version: "3.4"
+
+services:
+  magpie:
+    volumes:
+      - ./components/monitoring/config/magpie/config.yml:${MAGPIE_PERMISSIONS_CONFIG_PATH}/monitoring.yml:ro
+      - ./components/monitoring/config/magpie/config.yml:${MAGPIE_PROVIDERS_CONFIG_PATH}/monitoring.yml:ro

birdhouse/components/monitoring/config/proxy/conf.extra-service.d/monitoring.conf.template

@@ -0,0 +1,57 @@
+
+    location /grafana {
+        auth_request /secure-grafana-auth;
+        auth_request_set $auth_status $upstream_status;
+        proxy_pass http://grafana:3000;
+        proxy_set_header Host $host;
+    }
+
+    location /prometheus {
+        auth_request /secure-prometheus-auth;
+        auth_request_set $auth_status $upstream_status;
+        proxy_pass http://prometheus:9090;
+        proxy_set_header Host $host;
+    }
+
+    location /alertmanager {
+        auth_request /secure-alertmanager-auth;
+        auth_request_set $auth_status $upstream_status;
+        proxy_pass http://alertmanager:9093;
+        proxy_set_header Host $host;
+    }
+
+    location = /secure-grafana-auth {
+        internal;
+        proxy_pass https://${PAVICS_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/grafana$request_uri;
+        proxy_pass_request_body off;
+        proxy_set_header Host $host;
+        proxy_set_header Content-Length "";
+        proxy_set_header X-Original-URI $request_uri;
+        proxy_set_header X-Forwarded-Proto $real_scheme;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host $host:$server_port;
+    }
+
+    location = /secure-prometheus-auth {
+        internal;
+        proxy_pass https://${PAVICS_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/prometheus$request_uri;
+        proxy_pass_request_body off;
+        proxy_set_header Host $host;
+        proxy_set_header Content-Length "";
+        proxy_set_header X-Original-URI $request_uri;
+        proxy_set_header X-Forwarded-Proto $real_scheme;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host $host:$server_port;
+    }
+
+    location = /secure-alertmanager-auth {
+        internal;
+        proxy_pass https://${PAVICS_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}/alertmanager$request_uri;
+        proxy_pass_request_body off;
+        proxy_set_header Host $host;
+        proxy_set_header Content-Length "";
+        proxy_set_header X-Original-URI $request_uri;
+        proxy_set_header X-Forwarded-Proto $real_scheme;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host $host:$server_port;
+    }

birdhouse/components/monitoring/config/proxy/docker-compose-extra.yml

@@ -0,0 +1,6 @@
+version: "3.4"
+
+services:
+  proxy:
+    volumes:
+      - ./components/monitoring/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/monitoring:ro

birdhouse/components/monitoring/docker-compose-extra.yml

@@ -11,8 +11,6 @@ services:
       - /var/run:/var/run:ro
       - /sys:/sys:ro
       - /var/lib/docker:/var/lib/docker:ro
-    ports:
-      - 9999:8080
     devices:
       - /dev/kmsg
     restart: always
@@ -38,8 +36,6 @@ services:
       - ./components/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
       - ./components/monitoring/prometheus.rules:/etc/prometheus/prometheus.rules:ro
       - prometheus_persistence:/prometheus:rw
-    ports:
-      - 9090:9090
     command:
       # restore original CMD from image
       - --config.file=/etc/prometheus/prometheus.yml
@@ -49,7 +45,7 @@ services:
       # https://prometheus.io/docs/prometheus/latest/storage/
       - --storage.tsdb.retention.time=90d
       # wrong default was http://container-hash:9090/
-      - --web.external-url=http://${PAVICS_FQDN}:9090/
+      - --web.external-url=https://${PAVICS_FQDN_PUBLIC}/prometheus/
     restart: always
 
   # https://grafana.com/docs/grafana/latest/installation/docker/
@@ -65,8 +61,9 @@ services:
       - grafana_persistence:/var/lib/grafana:rw
     environment:
       GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
-    ports:
-      - 3001:3000
+      GF_SERVER_ROOT_URL: https://${PAVICS_FQDN_PUBLIC}/grafana
+      GF_SERVER_SERVE_FROM_SUB_PATH: 'true'
+      GF_SERVER_DOMAIN: ${PAVICS_FQDN_PUBLIC}
     restart: always
 
   # https://github.com/prometheus/alertmanager
@@ -86,9 +83,7 @@ services:
       # enable debug logging
       - --log.level=debug
       # wrong default was http://container-hash:9093/
-      - --web.external-url=http://${PAVICS_FQDN}:9093/
-    ports:
-      - 9093:9093
+      - --web.external-url=https://${PAVICS_FQDN_PUBLIC}/alertmanager
     restart: always
 
 volumes:

birdhouse/components/monitoring/grafana_datasources.yml.template

@@ -6,6 +6,6 @@ datasources:
     type: prometheus
     access: proxy
     uid: local_pavics_prometheus
-    url: http://${PAVICS_FQDN}:9090
+    url: http://prometheus:9090/prometheus
     isDefault: true
     editable: false

birdhouse/components/monitoring/prometheus.yml.template

@@ -1,5 +1,5 @@
 # https://prometheus.io/docs/prometheus/latest/configuration/configuration/
-# http://PAVICS_FQDN:9090/config
+# http://PAVICS_FQDN/prometheus/config
 global:
   scrape_interval:     60s
   evaluation_interval: 30s
@@ -10,8 +10,10 @@ scrape_configs:
   honor_labels: true
   static_configs:
   - targets:
-    - ${PAVICS_FQDN}:9999
+    - cadvisor:8080
 
+# Node exporter is required to run on the host network so it is not accessible through the docker network.
+# It is only accessible via the host network which can be accessed using the PAVICS_FQDN variable.
 - job_name: node-exporter
   honor_labels: true
   static_configs:
@@ -26,4 +28,4 @@ alerting:
   - scheme: http
     static_configs:
     - targets:
-      - "${PAVICS_FQDN}:9093"
+      - alertmanager:9093

birdhouse/components/stac/docker-compose-extra.yml

@@ -52,12 +52,5 @@ services:
       timeout: 5s
       retries: 5
 
-  # extend proxy with endpoint and config for STAC API access
-  proxy:
-    volumes:
-      - ./components/stac/conf.extra-service.d:/etc/nginx/conf.extra-service.d/stac:ro
-    links:
-      - stac
-
 volumes:
   stac-db: