Remove RunJobs when wfs are stopped prematurely

rodan/models/project.py

@@ -1,6 +1,7 @@
 import os
 import logging
 import shutil
+import traceback
 import uuid
 from django.conf import settings
 from django.contrib.auth.models import User, Group
@@ -10,6 +11,7 @@
 
 logger = logging.getLogger("rodan")
 
+
 class Project(models.Model):
     """
     The top-level model. A `Project` is mostly administrative and organizational.
@@ -91,10 +93,16 @@ def save(self, *args, **kwargs):
             os.makedirs(self.project_path)
 
     def delete(self, *args, **kwargs):
+        from rodan.models import RunJob
+
+        # logger.debug(RunJob.objects.filter(workflow_run__project=self))
+        # RunJob.objects.filter(workflow_run__project=self).delete()
+        for i in RunJob.objects.filter(workflow_run__project=self):
+            i.delete()
+
         # remove protected links from input/output to resource by deleting all
         # workflowruns prior to resources
         from rodan.models import WorkflowRun
-
         WorkflowRun.objects.filter(project=self).delete()
 
         # delete project, project folder, and project groups
@@ -105,10 +113,13 @@ def delete(self, *args, **kwargs):
         ag.delete()
         wg.delete()
         logger.info("Deleting: {}".format(proj_path))
+
         try:
             shutil.rmtree(proj_path)
-        except:
+        except Exception as e:
             logger.warning("Deleting folder failed: {}".format(proj_path))
+            logger.warning(e)
+            traceback.print_exc()
 
     @property
     def workflow_count(self):

rodan/models/runjob.py

@@ -1,9 +1,16 @@
+import logging
 import uuid
+
+from celery.task.control import revoke
 from django.db import models
+from django.contrib.auth.models import User
 from jsonfield import JSONField
-from rodan.models.job import Job
+
 from rodan.constants import task_status
-from django.contrib.auth.models import User
+from rodan.models.job import Job
+
+
+logger = logging.getLogger("rodan")
 
 
 class RunJob(models.Model):
@@ -133,3 +140,10 @@ def job(self):
     @property
     def project(self):
         return self.workflow_run.project
+
+    def delete(self):
+        logger.info("Killing Celery task_id: {}".format(self.celery_task_id))
+
+        # https://docs.celeryproject.org/en/v4.3.0/reference/celery.app.control.html#celery.app.control.Control.revoke  # noqa
+        # https://www.gnu.org/software/libc/manual/html_node/Termination-Signals.html
+        revoke(self.celery_task_id, terminate=True, signal="SIGTERM")

rodan/models/workflowrun.py

@@ -1,10 +1,14 @@
 # import os
+import logging
 import uuid
 from django.db import models
 from rodan.constants import task_status
 # import shutil
 
 
+logger = logging.getLogger("rodan")
+
+
 class WorkflowRun(models.Model):
     """
     Represents the running of a workflow. Since Rodan is based on a RESTful design,
@@ -101,7 +105,15 @@ def delete(self, *args, **kwargs):
         # remove protected links from runjobs to workflowrun by deleting the jobruns
         from rodan.models import RunJob
 
-        RunJob.objects.filter(workflow_run=self).delete()
+        logger.info("Stopping workflow run: {}".format(self))
+
+        # Funny these don't work... django version? celery version?
+        # RunJob.objects.filter(workflow_run=self).delete()
+        # RunJob.objects.filter(workflow_run=self.uuid).delete()
+        for i in RunJob.objects.filter(workflow_run=self.uuid):
+            logger.info("Stopping RunJob: {}".format(i))
+            i.delete()
+
         # if os.path.exists(self.resource_path):
         #     shutil.rmtree(self.resource_path)
         super(WorkflowRun, self).delete(*args, **kwargs)