diff --git a/damnit/backend/listener.py b/damnit/backend/listener.py index 798d3ada..3a9b69a0 100644 --- a/damnit/backend/listener.py +++ b/damnit/backend/listener.py @@ -5,6 +5,7 @@ import platform from pathlib import Path from socket import gethostname +from threading import Thread from kafka import KafkaConsumer @@ -110,7 +111,11 @@ def handle_event(self, record, msg: dict, run_data: RunData): log.info(f"Added p%d r%d ({run_data.value} data) to database", proposal, run) req = ExtractionRequest(run, proposal, run_data) - self.submitter.submit(req) + try: + self.submitter.submit(req) + except Exception: + log.warning("Slurm job submission failed, starting process locally") + Thread(target=self.submitter.execute_direct, args=(req, )).start() def listen():