Merge pull request #49 from scality/feature/RELENG-4942-add-monitoring

tcarmet · web-flow · commit a1c423504595 · 2021-11-05T10:21:49.000-07:00
First monitoring setup
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -33,13 +33,20 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-multi-buildx
 
-      - name: Login to Registry
+      - name: Login to ghcr Registry
         uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Login to Scality Registry
+        uses: docker/login-action@v1
+        with:
+          registry: registry.scality.com
+          username: ${{ secrets.REGISTRY_LOGIN }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+
       - name: Build and push
         uses: docker/build-push-action@v2
         with:
@@ -53,6 +60,16 @@ jobs:
           # And: https://github.com/docker/buildx#--cache-tonametypetypekeyvalue
           cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new
 
+      - name: Build and push manager
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./manager.Dockerfile
+          push: true
+          tags: "registry.scality.com/openstack-actions-runner-dev/openstack-actions-runner:${{ github.sha }}"
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new
+
       - name: Move cache
         run: |
           rm -rf /tmp/.buildx-cache
diff --git a/charts/openstack-actions-runner/templates/deployment.yaml b/charts/openstack-actions-runner/templates/deployment.yaml
@@ -3,10 +3,12 @@ kind: Deployment
 metadata:
   name: {{ include "openstack-actions-runner.fullname" . }}
   labels:
+    role: manager
     {{- include "openstack-actions-runner.labels" . | nindent 4 }}
 spec:
   selector:
     matchLabels:
+      role: manager
       {{- include "openstack-actions-runner.selectorLabels" . | nindent 6 }}
   template:
     metadata:
@@ -18,6 +20,7 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
+        role: manager
         {{- include "openstack-actions-runner.selectorLabels" . | nindent 8 }}
     spec:
       {{- with .Values.imagePullSecrets }}
@@ -37,6 +40,9 @@ spec:
         args: ["--settings-file", "/app/config/settings.yml"]
         resources:
           {{- toYaml .Values.resources | nindent 12 }}
+        ports:
+        - name: metrics
+          containerPort: {{ .Values.metricsPort }}
         env:
         - name: REDIS_PASSWORD
           valueFrom:
diff --git a/charts/openstack-actions-runner/templates/service-manager.yaml b/charts/openstack-actions-runner/templates/service-manager.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "openstack-actions-runner.fullname" . }}-manager
+  labels:
+    role: manager
+    {{- include "openstack-actions-runner.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.deployment.service.type }}
+  ports:
+    - name: metrics
+      port: {{ .Values.metricsPort }}
+      protocol: TCP
+  selector:
+    role: manager
+    {{- include "openstack-actions-runner.selectorLabels" . | nindent 4 }}
diff --git a/charts/openstack-actions-runner/templates/service-monitor.yaml b/charts/openstack-actions-runner/templates/service-monitor.yaml
@@ -0,0 +1,24 @@
+{{ if .Values.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "openstack-actions-runner.fullname" . }}-manager
+  labels:
+    {{- include "openstack-actions-runner.labels" . | nindent 4 }}
+    {{- if .Values.serviceMonitor.labels }}
+    {{- toYaml .Values.serviceMonitor.labels | nindent 4 }}
+    {{- end }}
+spec:
+  endpoints:
+  - interval: {{ .Values.serviceMonitor.interval }}
+    scrapeTimeout: {{ .Values.serviceMonitor.timeout }}
+    port: metrics
+    path: "/"
+  selector:
+    matchLabels:
+      role: manager
+      {{- include "openstack-actions-runner.labels" . | nindent 6 }}
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace | quote }}
+{{- end }}
diff --git a/charts/openstack-actions-runner/templates/settings.yml b/charts/openstack-actions-runner/templates/settings.yml
@@ -8,6 +8,7 @@ data:
     cloud_nine_region: {{ .Values.cloudNineRegion }}
     cloud_nine_tenant: {{ .Values.cloudNineTenant }}
     python_config: {{ .Values.pythonConfigModule }}
+    metrics_port: {{ .Values.metricsPort }}
     runner_pool:
 {{ .Values.runnerPool | toYaml | indent 6 }}
     extra_runner_timer:
diff --git a/charts/openstack-actions-runner/values.yaml b/charts/openstack-actions-runner/values.yaml
@@ -144,3 +144,10 @@ redis:
   networkPolicy:
     enabled: true
 
+metricsPort: 5000
+
+serviceMonitor:
+  enabled: false
+  labels: {}
+  interval: 30s
+  timeout: 30s
diff --git a/config_example.yml b/config_example.yml
@@ -1,4 +1,4 @@
-# Your github organization on witch you want to attache your self-hosted runners
+# Your github organization where you want to attach your self-hosted runners
 github_organization: ""
 
 # Cloud nine connection infos
@@ -42,3 +42,7 @@ timeout_runner_timer:
 redis:
   host: redis
   port: 6379
+
+
+# The Openmetrics port
+metrics_port: 5000
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,7 +1,9 @@
 version: "3.9"
 services:
   runner_manager:
-    build: .
+    build:
+      context: .
+      dockerfile: manager.Dockerfile
     command:
       - python3
       - /app/runners_manager/start.py
@@ -16,6 +18,8 @@ services:
     environment:
       - PYTHONUNBUFFERED=0
       - PYTHONIOENCODING=UTF-8
+    ports:
+      - 5000:5000
 
   web:
     build:
@@ -39,3 +43,5 @@ services:
       - "8080:8080"
   redis:
     image: "redis:alpine"
+    ports:
+      - 6379:6379
diff --git a/manager-requirements.txt b/manager-requirements.txt
@@ -9,4 +9,5 @@ coverage==5.5
 marshmallow==3.12.1
 wheel==0.36.2
 redis==3.5.3
-fakeredis==1.6.0
+fakeredis==1.6.0
+prometheus-client==0.12.0
diff --git a/manager.Dockerfile b/manager.Dockerfile
@@ -30,7 +30,13 @@ RUN apt-get --assume-yes update \
 
 
 WORKDIR /app
-COPY ./ /app/
+
+COPY manager-requirements.txt /app/
+
+RUN pip3 install -r manager-requirements.txt
+
+COPY . /app/
+
 RUN pip3 install . --use-feature=in-tree-build
 
 CMD runner-manager
diff --git a/runners_manager/main.py b/runners_manager/main.py
@@ -4,6 +4,10 @@
 import importlib
 import redis
 
+
+from prometheus_client import start_http_server
+
+
 from runners_manager.vm_creation.github_actions_api import GithubManager
 from runners_manager.vm_creation.openstack import OpenstackManager
 from runners_manager.runner.Manager import Manager
@@ -38,4 +42,5 @@ def main(settings: dict, args: argparse.Namespace):
                                  port=settings['redis']['port'],
                                  password=args.redis_password)
     runner_m = Manager(settings, openstack_manager, github_manager, redis_database)
+    start_http_server(settings['metrics_port'])
     maintain_number_of_runner(runner_m, github_manager)
diff --git a/runners_manager/monitoring/__init__.py b/runners_manager/monitoring/__init__.py
diff --git a/runners_manager/monitoring/prometheus.py b/runners_manager/monitoring/prometheus.py
@@ -0,0 +1,11 @@
+"""Global class to handle all Prometheus metrics."""
+
+from prometheus_client import Enum
+
+
+class Metrics(object):
+    runner_status = Enum(
+        'openstack_actions_runner_status', 'Metrics displaying the status of a runner',
+        states=['creating', 'deleting', 'respawning', 'online', 'running', 'offline'],
+        labelnames=['name', 'flavor', 'image'],
+    )
diff --git a/runners_manager/runner/Runner.py b/runners_manager/runner/Runner.py
@@ -2,8 +2,10 @@
 import logging
 
 from runners_manager.runner.VmType import VmType
+from runners_manager.monitoring.prometheus import Metrics
 
 logger = logging.getLogger("runner_manager")
+metrics = Metrics()
 
 
 class Runner(object):
@@ -27,7 +29,7 @@ def __init__(self, name: str, vm_id: str or None, vm_type: VmType):
         self.vm_type = vm_type
 
         self.created_at = datetime.datetime.now()
-        self.status = 'offline'
+        self.status = ''
         self.status_history = []
         self.action_id = None
         self.started_at = None
@@ -84,21 +86,41 @@ def toJson(self):
 
         return d
 
-    def update_status(self, elem):
-        if elem['status'] == 'online' and elem['busy'] is True:
-            status = 'running'
-        else:
-            status = elem['status']
-
+    def update_status(self, status: str):
         if self.status == status:
             return
 
-        if self.is_offline and status != 'offline':
+        if self.is_offline and status in ['online', 'running']:
             self.started_at = datetime.datetime.now()
 
         self.status_history.append(self.status)
+
+        logger.info(f'Runner {self.name} updating status from {self.status} to {status}')
         self.status = status
 
+        metrics.runner_status.labels(
+            name=self.name,
+            flavor=self.vm_type.flavor,
+            image=self.vm_type.image
+        ).state(self.status)
+
+        if self.status == 'deleting':
+            metrics.runner_status.remove(
+                self.name, self.vm_type.flavor, self.vm_type.image
+            )
+
+    def update_from_github(self, github_runner: dict):
+        """Take all information from github and update the runner state"""
+        # Update status
+        if github_runner['status'] == 'online' and github_runner['busy'] is True:
+            self.update_status('running')
+        else:
+            self.update_status(github_runner['status'])
+
+        # Set the action id
+        if self.action_id is None:
+            self.action_id = github_runner['id']
+
     @property
     def time_since_created(self):
         return datetime.datetime.now() - self.created_at
@@ -107,6 +129,11 @@ def time_since_created(self):
     def time_online(self):
         return datetime.datetime.now() - self.started_at
 
+    @property
+    def is_offline(self) -> bool:
+        """Return bool regarding runner status from GitHub point of view."""
+        return self.status not in ['online', 'running']
+
     @property
     def has_run(self) -> bool:
         return self.is_offline and \
@@ -119,7 +146,3 @@ def is_running(self) -> bool:
     @property
     def is_online(self) -> bool:
         return self.status == 'online'
-
-    @property
-    def is_offline(self) -> bool:
-        return self.status == 'offline'
diff --git a/runners_manager/runner/RunnerManager.py b/runners_manager/runner/RunnerManager.py
@@ -4,6 +4,7 @@
 import json
 from collections.abc import Callable
 
+
 from runners_manager.runner.Runner import Runner
 from runners_manager.runner.RunnerFactory import RunnerFactory
 from runners_manager.runner.VmType import VmType
@@ -66,23 +67,25 @@ def update(self, github_runners: list[dict]):
         for elem in github_runners:
             if elem['name'] in self.runners:
                 runner = self.runners[elem['name']]
-                runner.update_status(elem)
+                runner.update_from_github(elem)
 
-                if runner.action_id is None:
-                    runner.action_id = elem['id']
         self._save_runners()
 
     def create_runner(self):
         r = self.factory.create_runner(self.vm_type)
+        r.update_status('creating')
         self.runners[r.name] = r
         self._save_runners()
 
-    def delete_runner(self, runner):
+    def delete_runner(self, runner: Runner):
+        runner.update_status('deleting')
         self.factory.delete_runner(runner)
         del self.runners[runner.name]
+        del runner
         self._save_runners()
 
     def respawn_runner(self, runner: Runner):
+        runner.update_status('respawning')
         self.factory.respawn_replace(runner)
         self._save_runners()
 
diff --git a/runners_manager/runner/tests/test_runner_manager.py b/runners_manager/runner/tests/test_runner_manager.py
diff --git a/runners_manager/settings/yaml_config.py b/runners_manager/settings/yaml_config.py
diff --git a/runners_manager/vm_creation/openstack.py b/runners_manager/vm_creation/openstack.py
