Merge pull request #39 from scality/feature/RELENG-4568/patch-rebooting-errors

Erwan BERNARD · web-flow · commit 1857f18c5f61 · 2021-09-28T18:16:40.000+02:00
[RELENG-4568] patch-rebooting-errors
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,6 @@
+.env
+.gitignore
+config_example.yml
+docker-compose.yml
+Dockerfile
+README.md
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -23,8 +23,15 @@ jobs:
 
       - name: Set up Docker Buildk
         uses: docker/setup-buildx-action@v1
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
         with:
-          buildkitd-flags: --debug
+          path: /tmp/.buildx-cache
+          # Key is named differently to avoid collision
+          key: ${{ runner.os }}-multi-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-multi-buildx
 
       - name: Login to Registry
         uses: docker/login-action@v1
@@ -40,6 +47,16 @@ jobs:
           file: ./test.Dockerfile
           push: true
           tags: "ghcr.io/${{ github.repository }}:${{ github.sha }}"
+          cache-from: type=local,src=/tmp/.buildx-cache
+          # Note the mode=max here
+          # More: https://github.com/moby/buildkit#--export-cache-options
+          # And: https://github.com/docker/buildx#--cache-tonametypetypekeyvalue
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
   lint:
     runs-on: ubuntu-latest
diff --git a/charts/openstack-actions-runner/requirements.lock b/charts/openstack-actions-runner/requirements.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: redis
+  repository: https://charts.bitnami.com/bitnami
+  version: 11.3.4
+digest: sha256:59dd6d6aaa362dc8d29b59ca16413dfc304b0daa940b4aa06c8b01ba67cba927
+generated: "2021-09-20T14:16:25.327127092+02:00"
diff --git a/charts/openstack-actions-runner/requirements.yaml b/charts/openstack-actions-runner/requirements.yaml
@@ -0,0 +1,5 @@
+---
+dependencies:
+- name: redis
+  version: "11.3.4"
+  repository: "https://charts.bitnami.com/bitnami"
diff --git a/charts/openstack-actions-runner/templates/deployment.yaml b/charts/openstack-actions-runner/templates/deployment.yaml
@@ -37,6 +37,12 @@ spec:
         args: ["--settings-file", "/app/config/settings.yml"]
         resources:
           {{- toYaml .Values.resources | nindent 12 }}
+        env:
+        - name: REDIS_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.redis.fullnameOverride }}
+              key: redis-password
         envFrom:
         - configMapRef:
             name: {{ .Release.Name }}-config
diff --git a/charts/openstack-actions-runner/templates/settings.yml b/charts/openstack-actions-runner/templates/settings.yml
@@ -12,3 +12,6 @@ data:
 {{ .Values.runnerPool | toYaml | indent 6 }}
     extra_runner_timer:
 {{ .Values.extraRunnerTimer | toYaml | indent 6 }}
+    redis:
+      host: {{ .Values.redis.fullnameOverride  }}-master
+      port: {{ .Values.redis.redisPort }}
diff --git a/charts/openstack-actions-runner/values.yaml b/charts/openstack-actions-runner/values.yaml
@@ -71,7 +71,7 @@ cloudNineRegion: ""
 # The python module, used for configuration
 # example:
 #  'runners_manager.settings.settings_local'
-pythonConfigModule: "runners_manager.settings.settings_tests"
+pythonConfigModule: ""
 
 # Represent the infos about each group of runners
 # Define by:
@@ -93,3 +93,12 @@ runnerPool: []
 extraRunnerTimer:
   minutes: 10
   hours: 0
+
+
+# Redis database config
+redis:
+  fullnameOverride: redis
+  redisPort: 6379
+  networkPolicy:
+    enabled: true
+
diff --git a/config_example.yml b/config_example.yml
@@ -32,3 +32,8 @@ runner_pool: []
 extra_runner_timer:
   minutes: 10
   hours: 0
+
+# Define the credentials to connect your redis database
+redis:
+  host: redis
+  port: 6379
diff --git a/constraints.txt b/constraints.txt
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0 +1,20 @@
+version: "3.9"
+services:
+  web:
+    build: .
+    command:
+      - python3
+      - /app/runners_manager/start.py
+      - --settings-file=/app/settings.yaml
+    volumes:
+    - ./runners_manager:/app/runners_manager
+    - ./settings.yaml:/app/settings.yaml
+    depends_on:
+      - redis
+    env_file:
+      - .env
+    environment:
+       - PYTHONUNBUFFERED=0
+       - PYTHONIOENCODING=UTF-8
+  redis:
+    image: "redis:alpine"
diff --git a/docs/redis.md b/docs/redis.md
@@ -0,0 +1,26 @@
+# Redis integration
+
+For local environment easy development, we can start `docker-compose -f ./docker-composee.yml up`
+It start a redis database and the runner manager with a config file.
+
+## credentials 
+Defined in the settings file, and set the password with the script arg `--redis-password` or in the environment `REDIS_PASSWORD`
+```yaml
+redis:
+  host: redis
+  port: 6379
+```
+
+## Usage in the code base
+On creation each `RunnerManager` load his data from redis. 
+Then, when it creates or update a runner, it automatically updates the data in redis.
+
+
+## Helm template
+```yaml
+dependencies:
+- name: redis
+  version: "15.3.2"
+  repository: "https://charts.bitnami.com/bitnami"
+
+```
diff --git a/requirements.txt b/requirements.txt
@@ -9,3 +9,5 @@ jinja2==2.11.3
 coverage==5.5
 marshmallow==3.12.1
 wheel==0.36.2
+redis==3.5.3
+fakeredis==1.6.0
diff --git a/runners_manager/main.py b/runners_manager/main.py
@@ -2,6 +2,7 @@
 import time
 import logging
 import importlib
+import redis
 
 from runners_manager.vm_creation.github_actions_api import GithubManager
 from runners_manager.vm_creation.openstack import OpenstackManager
@@ -33,5 +34,8 @@ def main(settings: dict, args: argparse.Namespace):
                                          region=settings['cloud_nine_region'])
     github_manager = GithubManager(organization=settings['github_organization'],
                                    token=args.github_token)
-    runner_m = Manager(settings, openstack_manager, github_manager)
+    redis_database = redis.Redis(host=settings['redis']['host'],
+                                 port=settings['redis']['port'],
+                                 password=args.redis_password)
+    runner_m = Manager(settings, openstack_manager, github_manager, redis_database)
     maintain_number_of_runner(runner_m, github_manager)
diff --git a/runners_manager/runner/Manager.py b/runners_manager/runner/Manager.py
@@ -1,7 +1,6 @@
 import datetime
 import logging
-from collections.abc import Callable
-
+import redis
 
 from runners_manager.runner.Runner import Runner
 from runners_manager.vm_creation.github_actions_api import GithubManager
@@ -18,18 +17,21 @@ class Manager(object):
     This object contain the main logic to maintain the number of needed runners
     """
     factory: RunnerFactory
+    redis: redis.Redis
     runner_managers: list[RunnerManager]
     extra_runner_online_timer: dict
 
     def __init__(self, settings: dict,
                  openstack_manager: OpenstackManager,
-                 github_manager: GithubManager):
+                 github_manager: GithubManager,
+                 r: redis.Redis):
         self.factory = RunnerFactory(openstack_manager,
                                      github_manager,
                                      settings['github_organization'])
+        self.redis = r
         self.runner_managers = []
         for v_type in settings['runner_pool']:
-            self.runner_managers.append(RunnerManager(VmType(v_type), self.factory))
+            self.runner_managers.append(RunnerManager(VmType(v_type), self.factory, self.redis))
 
         self.extra_runner_online_timer = settings['extra_runner_timer']
 
@@ -55,13 +57,13 @@ def update(self, github_runners: list[dict]):
                 manager.respawn_runner(elem)
 
             # Delete if you have too many and they are not used for the last x hours / minutes
-            for runner in manager.filter_runners(self.need_less_runner(manager)):
-                manager.delete_runner(runner)
+            for runner in manager.filter_runners(self.need_less_runner):
+                if manager.too_much_runner():
+                    manager.delete_runner(runner)
 
-    def need_less_runner(self, manager: RunnerManager) -> Callable[Runner, bool]:
-        return lambda runner: runner.status == 'online' and not runner.has_run \
-            and runner.time_online > datetime.timedelta(**self.extra_runner_online_timer) \
-            and manager.too_much_runner()
+    def need_less_runner(self, runner: Runner) -> bool:
+        return runner.status == 'online' and not runner.has_run \
+            and runner.time_online > datetime.timedelta(**self.extra_runner_online_timer)
 
     @staticmethod
     def runner_should_never_spawn(runner: Runner) -> bool:
diff --git a/runners_manager/runner/Runner.py b/runners_manager/runner/Runner.py
@@ -37,6 +37,49 @@ def __unicode__(self):
     def __str__(self):
         return self.__unicode__()
 
+    @staticmethod
+    def fromJson(dict):
+        """
+        Build a Runner from json data
+        :param dict:
+        :return:
+        """
+        logger.info(dict)
+        runner = Runner(dict["name"], dict["vm_id"], VmType(dict["vm_type"]))
+
+        runner.status = dict["status"]
+        runner.status_history = dict["status_history"]
+        runner.action_id = dict["action_id"]
+        runner.created_at = datetime.datetime.strptime(dict["created_at"], "%Y-%m-%d %H:%M:%S.%f")
+
+        if dict["started_at"]:
+            runner.started_at = datetime.datetime.strptime(
+                dict["started_at"], "%Y-%m-%d %H:%M:%S.%f"
+            )
+        else:
+            runner.started_at = None
+        return runner
+
+    def toJson(self):
+        """
+        The fields_to_serialized, list the field to put in the dict
+        :return: dict object representative of Self
+        """
+        fields_to_serialized = ["name", "status", "status_history", "action_id", "vm_id"]
+        d = {
+            "vm_type": self.vm_type.toJson(),
+            "created_at": str(self.created_at)
+        }
+        if self.started_at:
+            d["started_at"] = str(self.started_at)
+        else:
+            d["started_at"] = None
+
+        for field in fields_to_serialized:
+            d[field] = self.__getattribute__(field)
+
+        return d
+
     def update_status(self, elem):
         if elem['status'] == 'online' and elem['busy'] is True:
             status = 'running'
diff --git a/runners_manager/runner/RunnerManager.py b/runners_manager/runner/RunnerManager.py
@@ -1,7 +1,8 @@
 import logging
+import redis
+import json
 from collections.abc import Callable
 
-
 from runners_manager.runner.Runner import Runner
 from runners_manager.runner.RunnerFactory import RunnerFactory
 from runners_manager.runner.VmType import VmType
@@ -12,20 +13,53 @@
 class RunnerManager(object):
     """
     This object create, delete and update runners infos bout VM and github runner
+    We save every infos updated on the redis database
     """
+    redis: redis.Redis
     vm_type: VmType
     runners: dict[str, Runner]
     factory: RunnerFactory
 
-    def __init__(self, vm_type: VmType, factory: RunnerFactory):
+    def __init__(self, vm_type: VmType, factory: RunnerFactory, redis: redis.Redis):
+        self.redis = redis
         self.vm_type = vm_type
         self.factory = factory
         self.runners = {}
+        self._get_runners()
+
+    def _redis_key_name(self):
+        """
+        Define the redis key name for this instance
+        :return:
+        """
+        return f'managers:{"-".join(self.vm_type.tags)}'
 
-        for index in range(0, vm_type.quantity['min']):
-            self.create_runner()
+    def _get_runners(self):
+        """
+        Build runners from redis
+        :return:
+        """
+        value = self.redis.get(self._redis_key_name())
+        if value is None:
+            self.runners = {}
+        else:
+            self.runners = dict([(k, Runner.fromJson(v)) for k, v in json.loads(value).items()])
+
+    def _save_runners(self):
+        """
+        Save runners json data in redis
+        :return:
+        """
+        d = dict([(k, v.toJson()) for k, v in self.runners.items()])
+        self.redis.set(self._redis_key_name(), json.dumps(d))
 
     def update(self, github_runners: list[dict]):
+        """
+        Update internal runners info from github.com
+        :param github_runners:
+        :return:
+        """
+        self._get_runners()
         # Update status of each runner
         for elem in github_runners:
             if elem['name'] in self.runners:
@@ -34,17 +68,21 @@ def update(self, github_runners: list[dict]):
 
                 if runner.action_id is None:
                     runner.action_id = elem['id']
+        self._save_runners()
 
     def create_runner(self):
         r = self.factory.create_runner(self.vm_type)
         self.runners[r.name] = r
+        self._save_runners()
 
     def delete_runner(self, runner):
         self.factory.delete_runner(runner)
         del self.runners[runner.name]
+        self._save_runners()
 
     def respawn_runner(self, runner: Runner):
         self.factory.respawn_replace(runner)
+        self._save_runners()
 
     def too_much_runner(self):
         current_online = len(
@@ -69,16 +107,12 @@ def need_new_runner(self):
             current_running + current_online_or_creating < self.vm_type.quantity['max']
 
     def filter_runners(self, cond: Callable[[Runner], bool]):
+        """
+        Return a list of runner matching the condition function
+        :param cond:
+        :return:
+        """
         return list(filter(
             lambda e: e.vm_type.tags == self.vm_type.tags and cond(e),
             self.runners.values()
         ))
-
-    def __del__(self):
-        """
-        Here we delete the runners currently in the manager
-        TODO Remove this function when we have a persistent database
-        :return:
-        """
-        for runner in self.runners.copy().values():
-            self.factory.delete_runner(runner)
diff --git a/runners_manager/runner/VmType.py b/runners_manager/runner/VmType.py
diff --git a/runners_manager/runner/tests/test_runner_manager.py b/runners_manager/runner/tests/test_runner_manager.py
diff --git a/runners_manager/runner/tests/tests_manager.py b/runners_manager/runner/tests/tests_manager.py
diff --git a/runners_manager/settings/settings_prod.py b/runners_manager/settings/settings_prod.py
diff --git a/runners_manager/settings/yaml_config.py b/runners_manager/settings/yaml_config.py
diff --git a/runners_manager/start.py b/runners_manager/start.py
