Merge pull request #21 from European-XFEL/slurm

Slurm improvements

damnit/backend/extract_data.py

@@ -154,10 +154,15 @@ def proposal(self):
         return self._proposal
 
     def slurm_options(self):
+        opts = ["--time", self.db.metameta.get("slurm_time", "02:00:00")]
+
         if reservation := self.db.metameta.get('slurm_reservation', ''):
-            return ['--reservation', reservation]
-        partition = self.db.metameta.get('slurm_partition', '') or default_slurm_partition()
-        return ['--partition', partition]
+            opts.extend(['--reservation', reservation])
+        else:
+            partition = self.db.metameta.get('slurm_partition', '') or default_slurm_partition()
+            opts.extend(['--partition', partition])
+
+        return opts
 
     def extract_and_ingest(self, proposal, run, cluster=False,
                            run_data=RunData.ALL, match=(), mock=False):
@@ -193,11 +198,19 @@ def extract_and_ingest(self, proposal, run, cluster=False,
         ctx =       self.ctx_whole.filter(run_data=run_data, name_matches=match, cluster=cluster)
         ctx_slurm = self.ctx_whole.filter(run_data=run_data, name_matches=match, cluster=True)
         if set(ctx_slurm.vars) > set(ctx.vars):
+            slurm_logs_dir = Path.cwd() / "slurm_logs"
+            slurm_logs_dir.mkdir(exist_ok=True)
+            slurm_logs_dir.chmod(0o777)
+
             python_cmd = [sys.executable, '-m', 'damnit.backend.extract_data',
                           '--cluster-job', str(proposal), str(run), run_data.value]
             res = subprocess.run([
                 'sbatch', '--parsable',
                 *self.slurm_options(),
+                '-o', str(slurm_logs_dir / f"r{run}-p{proposal}-%j.out"),
+                # Note: we put the run number first so that it's visible in
+                # squeue's default 11-character column for the JobName.
+                '--job-name', f"r{run}-p{proposal}-damnit",
                 '--wrap', shlex.join(python_cmd)
             ], stdout=subprocess.PIPE, text=True)
             job_id = res.stdout.partition(';')[0]