fallback on local execution if slurm fails

European-XFEL · Aug 30, 2024 · 2f8d2a2 · 2f8d2a2
1 parent c295f8d
commit 2f8d2a2
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/damnit/backend/listener.py b/damnit/backend/listener.py
@@ -5,6 +5,7 @@
 import platform
 from pathlib import Path
 from socket import gethostname
+from threading import Thread
 
 from kafka import KafkaConsumer
 
@@ -110,7 +111,11 @@ def handle_event(self, record, msg: dict, run_data: RunData):
         log.info(f"Added p%d r%d ({run_data.value} data) to database", proposal, run)
 
         req = ExtractionRequest(run, proposal, run_data)
-        self.submitter.submit(req)
+        try:
+            self.submitter.submit(req)
+        except Exception:
+            log.warning("Slurm job submission failed, starting process locally")
+            Thread(target=self.submitter.execute_direct, args=(req, )).start()
 
 
 def listen():