From abf6e13446ff4a3d9db453a69ce1912b11d471d4 Mon Sep 17 00:00:00 2001 From: JamesWrigley Date: Wed, 29 Mar 2023 20:59:03 +0200 Subject: [PATCH] Make slurm jobs respect --match Previously the --match arguments weren't passed to the slurm job, so all variables would be reprocessed. --- damnit/backend/extract_data.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/damnit/backend/extract_data.py b/damnit/backend/extract_data.py index c8372384..c261e17b 100644 --- a/damnit/backend/extract_data.py +++ b/damnit/backend/extract_data.py @@ -204,6 +204,9 @@ def extract_and_ingest(self, proposal, run, cluster=False, python_cmd = [sys.executable, '-m', 'damnit.backend.extract_data', '--cluster-job', str(proposal), str(run), run_data.value] + for m in match: + python_cmd.extend(["--match", m]) + res = subprocess.run([ 'sbatch', '--parsable', *self.slurm_options(), @@ -222,9 +225,11 @@ def extract_and_ingest(self, proposal, run, cluster=False, ap.add_argument('run', type=int) ap.add_argument('run_data', choices=('raw', 'proc', 'all')) ap.add_argument('--cluster-job', action="store_true") + ap.add_argument('--match', action="append", default=[]) args = ap.parse_args() logging.basicConfig(level=logging.INFO) Extractor().extract_and_ingest(args.proposal, args.run, cluster=args.cluster_job, - run_data=RunData(args.run_data)) + run_data=RunData(args.run_data), + match=args.match)