From 2f8d2a2f07291296f4d0a3516de8d4bbcfc694e3 Mon Sep 17 00:00:00 2001 From: tmichela Date: Fri, 30 Aug 2024 14:17:52 +0200 Subject: [PATCH] fallback on local execution if slurm fails --- damnit/backend/listener.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/damnit/backend/listener.py b/damnit/backend/listener.py index 798d3ada..3a9b69a0 100644 --- a/damnit/backend/listener.py +++ b/damnit/backend/listener.py @@ -5,6 +5,7 @@ import platform from pathlib import Path from socket import gethostname +from threading import Thread from kafka import KafkaConsumer @@ -110,7 +111,11 @@ def handle_event(self, record, msg: dict, run_data: RunData): log.info(f"Added p%d r%d ({run_data.value} data) to database", proposal, run) req = ExtractionRequest(run, proposal, run_data) - self.submitter.submit(req) + try: + self.submitter.submit(req) + except Exception: + log.warning("Slurm job submission failed, starting process locally") + Thread(target=self.submitter.execute_direct, args=(req, )).start() def listen():