Merge pull request #31 from scality/bugfix/RELENG-4763/verify-vm-spawn-correclty

[RELENG-4763] verify VM spawn correctly

Author: Erwan BERNARD

runners_manager/runner/Runner.py

@@ -9,6 +9,7 @@
 class Runner(object):
     name: str
     started_at: datetime.datetime or None
+    created_at: datetime.datetime
     status: str
     status_history: list[str]
 
@@ -21,6 +22,7 @@ def __init__(self, name: str, vm_id: str or None, vm_type: VmType):
         self.vm_id = vm_id
         self.vm_type = vm_type
 
+        self.created_at = datetime.datetime.now()
         self.status = 'offline'
         self.status_history = []
         self.action_id = None

runners_manager/runner/RunnerManager.py

@@ -65,11 +65,19 @@ def update(self, github_runners: list[dict]):
             while self.need_new_runner(vm_type):
                 self.create_runner(vm_type)
 
+            # Respawn runner if they are offline for more then 15min after spawn
+            # TODO add this timer in the config file
+            for runner in self.filter_runners(vm_type, lambda r: r.status == 'offline'):
+                time_since_spawn = datetime.datetime.now() - runner.created_at
+                if runner.has_run is False \
+                        and time_since_spawn > datetime.timedelta(minutes=15):
+                    self.respawn_replace(runner)
+
             # Delete if you have too many and they are not used for the last x hours / minutes
-            for elem in self.filter_runners(vm_type, lambda r: r.status == 'online'):
-                if elem.time_online > datetime.timedelta(**self.extra_runner_online_timer) \
+            for runner in self.filter_runners(vm_type, lambda r: r.status == 'online'):
+                if runner.time_online > datetime.timedelta(**self.extra_runner_online_timer) \
                         and self.too_much_runner(vm_type):
-                    self.delete_runner(elem)
+                    self.delete_runner(runner)
 
     def too_much_runner(self, vm_type: VmType):
         current_online = len(
@@ -136,6 +144,7 @@ def respawn_replace(self, runner: Runner):
         )
         runner.status_history = []
         runner.vm_id = vm.id
+        runner.created_at = datetime.datetime.now()
 
     def delete_runner(self, runner: Runner):
         logger.info(f"Deleting {runner.name}: type {runner.vm_type}")

runners_manager/runner/tests/tests_runner.py

@@ -2,7 +2,6 @@
 from unittest.mock import patch, Mock, MagicMock
 
 from runners_manager.runner.RunnerManager import RunnerManager
-from runners_manager.runner.Runner import Runner
 from runners_manager.runner.VmType import VmType
 
 
@@ -95,9 +94,14 @@ def test_config_vm_type(self, name_generator: Mock,
         self.assertEqual(r.runner_management.__len__(), 1)
 
         self.assertEqual(r.runners.__len__(), 2)
-        self.assertEqual(r.runners['0'], Runner('0', '1', r.runner_management[0]))
-        self.assertEqual(r.runners['1'].__dict__,
-                         Runner('1', '2', r.runner_management[0]).__dict__)
+        self.assertEqual(r.runners['0'].name, '0')
+        self.assertEqual(r.runners['0'].vm_id, '1')
+        self.assertEqual(r.runners['0'].vm_type, r.runner_management[0])
+
+        self.assertEqual(r.runners['1'].name, '1')
+        self.assertEqual(r.runners['1'].vm_id, '2')
+        self.assertEqual(r.runners['1'].vm_type, r.runner_management[0])
+
         self.assertEqual(r.runner_counter, 2)
         self.github_manager.create_runner_token.assert_called()
         self.openstack_manager.create_vm.assert_called()

runners_manager/vm_creation/openstack.py

@@ -1,4 +1,5 @@
 import logging
+import time
 
 import keystoneauth1.session
 import keystoneclient.auth.identity.v3
@@ -74,9 +75,16 @@ def create_vm(self, runner: Runner, runner_token: int or None,
             userdata=self.script_init_runner(runner, runner_token, github_organization,
                                              installer)
         )
+        while instance.status not in ['ACTIVE', 'ERROR']:
+            instance = self.nova_client.servers.get(instance.id)
+            time.sleep(2)
+        if instance.status == 'ERROR':
+            logger.info('vm failed, creating a new one')
+            self.delete_vm(instance.id)
+            return self.create_vm(runner, runner_token, github_organization, installer)
 
         logger.info("vm is successfully created")
         return instance
 
-    def delete_vm(self, id):
+    def delete_vm(self, id: str):
         self.nova_client.servers.delete(id)