advance the Post-Processing Agent version and add system test for pos…

…t-processing task time limit

Dockerfile.autoreducer

@@ -19,7 +19,7 @@ ARG CONFIG_FILE=tests/configuration/post_process_consumer.conf
 COPY ${CONFIG_FILE} /etc/autoreduce/post_processing.conf
 
 # install postprocessing
-RUN dnf install -y https://github.com/neutrons/post_processing_agent/releases/download/v3.3.4/postprocessing-3.3.4-1.el9.noarch.rpm
+RUN dnf install -y https://github.com/neutrons/post_processing_agent/releases/download/v3.4.0/postprocessing-3.4.0-1.el9.noarch.rpm
 
 # install the fake test data
 ARG DATA_TARBALL=/tmp/SNSdata.tar.gz

tests/configuration/post_process_consumer.conf

@@ -28,6 +28,7 @@
     "publish_url_template": "https://172.16.238.222/plots/$instrument/$run_number/upload_plot_data/",
     "publisher_username": "workflow",
     "publisher_password": "workflow",
-    "system_mem_limit_perc": 10,
-    "mem_check_interval_sec": 0.1
+    "system_mem_limit_perc": 5,
+    "mem_check_interval_sec": 0.01,
+    "task_time_limit_minutes": 0.1
 }

tests/data/NOM/shared/autoreduce/reduce_NOM.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+import sys
+import time
+from datetime import datetime
+
+print("Running reduction for " + sys.argv[1] + " at " + datetime.isoformat(datetime.now()))
+
+time.sleep(10)

tests/test_autoreducer_time_limit.py

@@ -0,0 +1,37 @@
+"""
+Test of the autoreducer memory management that sets a max limit
+on the memory used by reduction scripts.
+"""
+
+import time
+import tests.utils.db as db_utils
+
+
+class TestAutoreducerTimeLimit:
+    user = "InstrumentScientist"
+    pwd = "InstrumentScientist"
+    instrument = "nom"
+    IPTS = "IPTS-1001"
+    run_number = 10002
+
+    def test_reduction_script_exceeds_time_limit(self, db_connection, request_page):
+        """test that the reduction is terminated and an error is logged"""
+        run_id = db_utils.add_instrument_data_run(db_connection, self.instrument, self.IPTS, self.run_number)
+        db_utils.clear_previous_runstatus(db_connection, run_id)
+
+        # login and send reduction request
+        response = request_page("/report/nom/10002/reduce/", self.user, self.pwd)
+        assert response.status_code == 200
+        assert response.url.endswith("/report/nom/10002/")
+
+        # wait for reduction job to be terminated and database to be updated
+        # note: the post-processing task time limit is set to 6 seconds, since if it is too short,
+        # the memory limit test will hit the time limit before it hits the memory limit
+        time.sleep(7.0)
+
+        assert db_utils.check_run_status_exist(db_connection, run_id, "REDUCTION.REQUEST")
+        assert db_utils.check_run_status_exist(db_connection, run_id, "REDUCTION.STARTED")
+        assert db_utils.check_run_status_exist(db_connection, run_id, "REDUCTION.DATA_READY")
+        assert db_utils.check_run_status_exist(db_connection, run_id, "REDUCTION.ERROR")
+
+        assert db_utils.check_error_msg_contains(db_connection, run_id, "Time limit exceeded")