improvement(stress_thread): HDR file collection

From time to time we faced out with the problem that HDR file was not
collected. But it is not clear if the file was not created or collected.
This commit check if the file was not collected and try to collect it.
If the file does not exist on the loader - throw error

Fixes: #8949
(cherry picked from commit f1d5ba6)

defaults/severities.yaml

@@ -143,3 +143,4 @@ CommitLogCheckErrorEvent: ERROR
 ValidatorEvent: ERROR
 ScrubValidationErrorEvent: ERROR
 FailedResultEvent: ERROR
+HDRFileMissed: ERROR

sdcm/sct_events/loaders.py

@@ -66,6 +66,13 @@ def msgfmt(self):
         return fmt
 
 
+class HDRFileMissed(SctEvent):
+    def __init__(self, message: str, severity=Severity.NORMAL):
+        super().__init__(severity=severity)
+
+        self.message = message
+
+
 class CassandraStressEvent(StressEvent):
     ...
 

sdcm/stress_thread.py

@@ -30,7 +30,7 @@
 from sdcm.sct_events import Severity
 from sdcm.utils.common import FileFollowerThread, get_data_dir_path, time_period_str_to_seconds, SoftTimeoutContext
 from sdcm.utils.user_profile import get_profile_content, replace_scylla_qa_internal_path
-from sdcm.sct_events.loaders import CassandraStressEvent, CS_ERROR_EVENTS_PATTERNS, CS_NORMAL_EVENTS_PATTERNS
+from sdcm.sct_events.loaders import CassandraStressEvent, CS_ERROR_EVENTS_PATTERNS, CS_NORMAL_EVENTS_PATTERNS, HDRFileMissed
 from sdcm.stress.base import DockerBasedStressThread
 from sdcm.utils.docker_remote import RemoteDocker
 from sdcm.utils.remote_logger import SSHLoggerBase
@@ -82,11 +82,32 @@ def __init__(self, node: BaseNode, remote_log_file: str, target_log_file: str):
     def _logger_cmd_template(self) -> str:
         return f"tail -f {self._remote_log_file}"
 
+    def validate_and_collect_hdr_file(self):
+        """
+        Validate that HDR file exists on the SCT runner.
+        If it does not exist check if the file was created on the loader.
+        If the HDR file found on the loader, try to copy to the runner.
+        If the file is missed even on the loader - print error event.
+        """
+        if os.path.exists(self._target_log_file):
+            return
+
+        LOGGER.debug("'%s' file is not found on the runner. Try to find it on the loader %s",
+                     self._target_log_file, self._node.name)
+        result = self._node.remoter.run(f"test -f {self._remote_log_file}", ignore_status=True)
+        if not result.ok:
+            HDRFileMissed(message=f"'{self._remote_log_file}' HDR file was not created on the loader {self._node.name}",
+                          severity=Severity.ERROR).publish()
+
+        LOGGER.debug("The '%s' file found on the loader %s", self._remote_log_file, self._node.name)
+        self._node.remoter.receive_files(src=self._remote_log_file, dst=self._target_log_file)
+
     def __enter__(self):
         self.start()
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
+        self.validate_and_collect_hdr_file()
         self.stop()