Standartize sanity failures when using exit_pytest_execution

tests/conftest.py

@@ -463,10 +463,12 @@ def cnv_tests_utilities_namespace(admin_client, installing_cnv):
         name = "cnv-tests-utilities"
         if Namespace(client=admin_client, name=name).exists:
             exit_pytest_execution(
-                message=f"{name} namespace already exists."
+                log_message=f"{name} namespace already exists."
                 f"\nAfter verifying no one else is performing tests against the cluster, run:"
                 f"\n'oc delete namespace {name}'",
                 return_code=100,
+                message=f"{name} namespace already exists.",
+                filename="cnv_tests_utilities_ns_failure.txt",
             )
 
         else:
@@ -2582,7 +2584,10 @@ def updated_default_storage_class_ocs_virt(
             namespace=golden_images_namespace.name,
         )
         if not boot_source_imported_successfully:
-            exit_pytest_execution(message=f"Failed to set {ocs_storage_class.name} as default storage class")
+            exit_pytest_execution(
+                log_message=f"Failed to set {ocs_storage_class.name} as default storage class",
+                filename="default_storage_class_failure.txt",
+            )
     else:
         yield
 

tests/install_upgrade_operators/product_upgrade/conftest.py

@@ -322,7 +322,9 @@ def fired_alerts_during_upgrade(fired_alerts_before_upgrade, alert_dir, promethe
 def eus_cnv_upgrade_path(eus_target_cnv_version):
     if eus_target_cnv_version is None:
         exit_pytest_execution(
-            message="EUS upgrade can not be performed from non-eus version", return_code=EUS_ERROR_CODE
+            log_message="EUS upgrade can not be performed from non-eus version",
+            return_code=EUS_ERROR_CODE,
+            filename="eus_upgrade_failure.txt",
         )
     # Get the shortest path to the target (EUS) version
     upgrade_path_to_target_version = get_shortest_upgrade_path(target_version=eus_target_cnv_version)

tests/network/conftest.py

@@ -378,8 +378,9 @@ def _verify_nmstate_running_pods(_admin_client, namespace):
         err_msg = "\n".join(failure_msgs)
         LOGGER.error(f"Network cluster verification failed! Missing components:\n{err_msg}")
         exit_pytest_execution(
-            message=err_msg,
+            log_message=err_msg,
             return_code=91,
             filename="network_cluster_sanity_failure.txt",
             junitxml_property=junitxml_plugin,
+            message="Network cluster verification failed",
         )

tests/virt/conftest.py

@@ -169,10 +169,11 @@ def _verify_if_1tb_memory_or_more_node(_memory_per_node):
         err_msg = "\n".join(failed_verifications_list)
         LOGGER.error(f"Special_infra cluster verification failed! Missing components:\n{err_msg}")
         exit_pytest_execution(
-            message=err_msg,
+            log_message=err_msg,
             return_code=98,
             filename="virt_special_infra_sanity_failure.txt",
             junitxml_property=junitxml_plugin,
+            message="Virt special_infra cluster verification failed",
         )
 
 

utilities/data_collector.py

@@ -90,6 +90,7 @@ def write_to_file(file_name, content, base_directory, mode="w"):
     os.makedirs(base_directory, exist_ok=True)
     file_path = os.path.join(base_directory, file_name)
 
+    LOGGER.info(f"Writing data to file: {file_path}")
     try:
         with open(file_path, mode) as fd:
             fd.write(content)

utilities/pytest_utils.py

@@ -26,7 +26,7 @@
 )
 from utilities.data_collector import (
     collect_default_cnv_must_gather_with_vm_gather,
-    get_data_collector_dir,
+    get_data_collector_base_directory,
     write_to_file,
 )
 from utilities.exceptions import MissingEnvironmentVariableError
@@ -165,10 +165,12 @@ def stop_if_run_in_progress():
     run_in_progress = run_in_progress_config_map()
     if run_in_progress.exists:
         exit_pytest_execution(
-            message=f"openshift-virtualization-tests run already in progress: \n{run_in_progress.instance.data}"
+            log_message=f"openshift-virtualization-tests run already in progress: \n{run_in_progress.instance.data}"
             f"\nAfter verifying no one else is performing tests against the cluster, run:"
             f"\n'oc delete configmap -n {run_in_progress.namespace} {run_in_progress.name}'",
             return_code=100,
+            message="openshift-virtualization-tests run already in progress",
+            filename="cnv_tests_run_in_progress_failure.txt",
         )
 
 
@@ -277,19 +279,22 @@ def get_tests_cluster_markers(items, filepath=None) -> None:
             fd.write(json.dumps(tests_cluster_markers))
 
 
-def exit_pytest_execution(message, return_code=SANITY_TESTS_FAILURE, filename=None, junitxml_property=None):
+def exit_pytest_execution(
+    log_message, return_code=SANITY_TESTS_FAILURE, filename=None, junitxml_property=None, message=None
+):
     """Exit pytest execution
 
     Exit pytest execution; invokes pytest_sessionfinish.
     Optionally, log an error message to tests-collected-info/utilities/pytest_exit_errors/<filename>
 
     Args:
-        message (str):  Message to display upon exit and to log in errors file
+        log_message (str):  Message to display upon exit and to log in errors file
         return_code (int. Default: 99): Exit return code
         filename (str, optional. Default: None): filename where the given message will be saved
         junitxml_property (pytest plugin): record_testsuite_property
+        message (str): Message to log in error file. If not provided, `log_message` will be used.
     """
-    target_location = os.path.join(get_data_collector_dir(), "pytest_exit_errors")
+    target_location = os.path.join(get_data_collector_base_directory(), "utilities", "pytest_exit_errors")
     # collect must-gather for past 5 minutes:
     if return_code == SANITY_TESTS_FAILURE:
         try:
@@ -303,9 +308,9 @@ def exit_pytest_execution(message, return_code=SANITY_TESTS_FAILURE, filename=No
     if filename:
         write_to_file(
             file_name=filename,
-            content=message,
+            content=message or log_message,
             base_directory=target_location,
         )
     if junitxml_property:
         junitxml_property(name="exit_code", value=return_code)
-    pytest.exit(reason=message, returncode=return_code)
+    pytest.exit(reason=log_message, returncode=return_code)

utilities/sanity.py

@@ -143,6 +143,7 @@ def cluster_sanity(
     except (ClusterSanityError, NodeUnschedulableError, NodeNotReadyError, StorageSanityError) as ex:
         exit_pytest_execution(
             filename=exceptions_filename,
-            message=str(ex),
+            log_message=str(ex),
             junitxml_property=junitxml_property,
+            message="Cluster sanity checks failed.",
         )

utilities/unittests/test_pytest_utils.py

@@ -458,7 +458,7 @@ def test_stop_if_run_in_progress_exists(self, mock_exit, mock_config_map):
         stop_if_run_in_progress()
 
         mock_exit.assert_called_once()
-        assert "test_user" in mock_exit.call_args[1]["message"]
+        assert "test_user" in mock_exit.call_args[1]["log_message"]
         assert mock_exit.call_args[1]["return_code"] == 100
 
     @patch("utilities.pytest_utils.run_in_progress_config_map")
@@ -922,81 +922,81 @@ class TestExitPytestExecution:
     """Test cases for exit_pytest_execution function"""
 
     @patch("utilities.pytest_utils.pytest.exit")
-    @patch("utilities.pytest_utils.get_data_collector_dir")
-    def test_exit_pytest_execution_basic(self, mock_get_dir, mock_pytest_exit):
+    @patch("utilities.pytest_utils.get_data_collector_base_directory")
+    def test_exit_pytest_execution_basic(self, mock_get_base_dir, mock_pytest_exit):
         """Test basic exit with message"""
-        mock_get_dir.return_value = "/tmp/test"
-        message = "Test exit message"
+        mock_get_base_dir.return_value = "/tmp/test"
+        log_message = "Test exit message"
 
-        exit_pytest_execution(message, return_code=1)
+        exit_pytest_execution(log_message=log_message, return_code=1)
 
-        mock_pytest_exit.assert_called_once_with(reason=message, returncode=1)
+        mock_pytest_exit.assert_called_once_with(reason=log_message, returncode=1)
 
     @patch("utilities.pytest_utils.pytest.exit")
     @patch("utilities.pytest_utils.write_to_file")
-    @patch("utilities.pytest_utils.get_data_collector_dir")
-    def test_exit_pytest_execution_with_filename(self, mock_get_dir, mock_write, mock_pytest_exit):
+    @patch("utilities.pytest_utils.get_data_collector_base_directory")
+    def test_exit_pytest_execution_with_filename(self, mock_get_base_dir, mock_write, mock_pytest_exit):
         """Test exit with filename for logging"""
-        mock_get_dir.return_value = "/tmp/test"
-        message = "Test error"
+        mock_get_base_dir.return_value = "/tmp/test"
+        log_message = "Test error"
         filename = "test_error.log"
 
-        exit_pytest_execution(message, return_code=1, filename=filename)
+        exit_pytest_execution(log_message=log_message, return_code=1, filename=filename)
 
         mock_write.assert_called_once_with(
             file_name=filename,
-            content=message,
-            base_directory="/tmp/test/pytest_exit_errors",
+            content=log_message,
+            base_directory="/tmp/test/utilities/pytest_exit_errors",
         )
         mock_pytest_exit.assert_called_once()
 
     @patch("utilities.pytest_utils.pytest.exit")
-    @patch("utilities.pytest_utils.get_data_collector_dir")
-    def test_exit_pytest_execution_with_junitxml(self, mock_get_dir, mock_pytest_exit):
+    @patch("utilities.pytest_utils.get_data_collector_base_directory")
+    def test_exit_pytest_execution_with_junitxml(self, mock_get_base_dir, mock_pytest_exit):
         """Test exit with junitxml_property"""
-        mock_get_dir.return_value = "/tmp/test"
-        message = "Test exit"
+        mock_get_base_dir.return_value = "/tmp/test"
+        log_message = "Test exit"
         mock_junitxml = MagicMock()
 
-        exit_pytest_execution(message, return_code=5, junitxml_property=mock_junitxml)
+        exit_pytest_execution(log_message=log_message, return_code=5, junitxml_property=mock_junitxml)
 
         mock_junitxml.assert_called_once_with(name="exit_code", value=5)
         mock_pytest_exit.assert_called_once()
 
     @patch("utilities.pytest_utils.pytest.exit")
     @patch("utilities.pytest_utils.collect_default_cnv_must_gather_with_vm_gather")
-    @patch("utilities.pytest_utils.get_data_collector_dir")
+    @patch("utilities.pytest_utils.get_data_collector_base_directory")
     @patch("utilities.pytest_utils.SANITY_TESTS_FAILURE", 99)
     @patch("utilities.pytest_utils.TIMEOUT_5MIN", 300)
     def test_exit_pytest_execution_sanity_failure_collects_must_gather(
-        self, mock_get_dir, mock_collect, mock_pytest_exit
+        self, mock_get_base_dir, mock_collect, mock_pytest_exit
     ):
         """Test must-gather collection on SANITY_TESTS_FAILURE"""
-        mock_get_dir.return_value = "/tmp/test"
-        message = "Sanity test failure"
+        mock_get_base_dir.return_value = "/tmp/test"
+        log_message = "Sanity test failure"
 
-        exit_pytest_execution(message)
+        exit_pytest_execution(log_message=log_message)
 
         mock_collect.assert_called_once_with(
             since_time=300,
-            target_dir="/tmp/test/pytest_exit_errors",
+            target_dir="/tmp/test/utilities/pytest_exit_errors",
         )
         mock_pytest_exit.assert_called_once()
 
     @patch("utilities.pytest_utils.pytest.exit")
     @patch("utilities.pytest_utils.collect_default_cnv_must_gather_with_vm_gather")
-    @patch("utilities.pytest_utils.get_data_collector_dir")
+    @patch("utilities.pytest_utils.get_data_collector_base_directory")
     @patch("utilities.pytest_utils.LOGGER")
     @patch("utilities.pytest_utils.SANITY_TESTS_FAILURE", 99)
     def test_exit_pytest_execution_must_gather_fails_silently(
-        self, mock_logger, mock_get_dir, mock_collect, mock_pytest_exit
+        self, mock_logger, mock_get_base_dir, mock_collect, mock_pytest_exit
     ):
         """Test that must-gather failure doesn't prevent exit"""
-        mock_get_dir.return_value = "/tmp/test"
+        mock_get_base_dir.return_value = "/tmp/test"
         mock_collect.side_effect = Exception("Must-gather failed")
-        message = "Sanity test failure"
+        log_message = "Sanity test failure"
 
-        exit_pytest_execution(message)
+        exit_pytest_execution(log_message=log_message)
 
         # Should log warning but still exit
         mock_logger.warning.assert_called_once()
@@ -1005,46 +1005,46 @@ def test_exit_pytest_execution_must_gather_fails_silently(
 
     @patch("utilities.pytest_utils.pytest.exit")
     @patch("utilities.pytest_utils.collect_default_cnv_must_gather_with_vm_gather")
-    @patch("utilities.pytest_utils.get_data_collector_dir")
+    @patch("utilities.pytest_utils.get_data_collector_base_directory")
     @patch("utilities.pytest_utils.SANITY_TESTS_FAILURE", 99)
-    def test_exit_pytest_execution_custom_return_code(self, mock_get_dir, mock_collect, mock_pytest_exit):
+    def test_exit_pytest_execution_custom_return_code(self, mock_get_base_dir, mock_collect, mock_pytest_exit):
         """Test with non-SANITY_TESTS_FAILURE code (skips must-gather)"""
-        mock_get_dir.return_value = "/tmp/test"
-        message = "Regular exit"
+        mock_get_base_dir.return_value = "/tmp/test"
+        log_message = "Regular exit"
 
-        exit_pytest_execution(message, return_code=5)
+        exit_pytest_execution(log_message=log_message, return_code=5)
 
         # Should not collect must-gather
         mock_collect.assert_not_called()
-        mock_pytest_exit.assert_called_once_with(reason=message, returncode=5)
+        mock_pytest_exit.assert_called_once_with(reason=log_message, returncode=5)
 
     @patch("utilities.pytest_utils.pytest.exit")
     @patch("utilities.pytest_utils.write_to_file")
     @patch("utilities.pytest_utils.collect_default_cnv_must_gather_with_vm_gather")
-    @patch("utilities.pytest_utils.get_data_collector_dir")
+    @patch("utilities.pytest_utils.get_data_collector_base_directory")
     @patch("utilities.pytest_utils.SANITY_TESTS_FAILURE", 99)
     @patch("utilities.pytest_utils.TIMEOUT_5MIN", 300)
-    def test_exit_pytest_execution_all_options(self, mock_get_dir, mock_collect, mock_write, mock_pytest_exit):
+    def test_exit_pytest_execution_all_options(self, mock_get_base_dir, mock_collect, mock_write, mock_pytest_exit):
         """Test with all options provided"""
-        mock_get_dir.return_value = "/tmp/test"
-        message = "Complete failure"
+        mock_get_base_dir.return_value = "/tmp/test"
+        log_message = "Complete failure"
         filename = "error.log"
         mock_junitxml = MagicMock()
 
-        exit_pytest_execution(message, filename=filename, junitxml_property=mock_junitxml)
+        exit_pytest_execution(log_message=log_message, filename=filename, junitxml_property=mock_junitxml)
 
         # All components should be called
         mock_collect.assert_called_once_with(
             since_time=300,
-            target_dir="/tmp/test/pytest_exit_errors",
+            target_dir="/tmp/test/utilities/pytest_exit_errors",
         )
         mock_write.assert_called_once_with(
             file_name=filename,
-            content=message,
-            base_directory="/tmp/test/pytest_exit_errors",
+            content=log_message,
+            base_directory="/tmp/test/utilities/pytest_exit_errors",
         )
         mock_junitxml.assert_called_once_with(name="exit_code", value=99)
-        mock_pytest_exit.assert_called_once_with(reason=message, returncode=99)
+        mock_pytest_exit.assert_called_once_with(reason=log_message, returncode=99)
 
 
 class TestGetMatrixParamsAdditionalCoverage:

utilities/unittests/test_sanity.py

@@ -279,7 +279,7 @@ def test_cluster_sanity_storage_sanity_error(
         mock_exit_pytest.assert_called_once()
         call_args = mock_exit_pytest.call_args
         assert call_args[1]["filename"] == "cluster_sanity_failure.txt"
-        assert "Cluster is missing storage class" in call_args[1]["message"]
+        assert "Cluster is missing storage class" in call_args[1]["log_message"]
 
     @patch("utilities.sanity.storage_sanity_check")
     @patch("utilities.sanity.assert_nodes_in_healthy_condition")
@@ -316,7 +316,7 @@ def test_cluster_sanity_node_unschedulable_error(
         mock_exit_pytest.assert_called_once()
         call_args = mock_exit_pytest.call_args
         assert call_args[1]["filename"] == "cluster_sanity_failure.txt"
-        assert error_message in call_args[1]["message"]
+        assert error_message in call_args[1]["log_message"]
 
     @patch("utilities.sanity.storage_sanity_check")
     @patch("utilities.sanity.assert_nodes_in_healthy_condition")
@@ -353,7 +353,7 @@ def test_cluster_sanity_node_not_ready_error(
         mock_exit_pytest.assert_called_once()
         call_args = mock_exit_pytest.call_args
         assert call_args[1]["filename"] == "cluster_sanity_failure.txt"
-        assert error_message in call_args[1]["message"]
+        assert error_message in call_args[1]["log_message"]
 
     @patch("utilities.sanity.storage_sanity_check")
     @patch("utilities.sanity.assert_nodes_in_healthy_condition")
@@ -390,7 +390,7 @@ def test_cluster_sanity_cluster_sanity_error(
         mock_exit_pytest.assert_called_once()
         call_args = mock_exit_pytest.call_args
         assert call_args[1]["filename"] == "cluster_sanity_failure.txt"
-        assert error_message in call_args[1]["message"]
+        assert error_message in call_args[1]["log_message"]
 
     @patch("utilities.sanity.storage_sanity_check")
     @patch("utilities.sanity.assert_nodes_in_healthy_condition")
@@ -433,8 +433,8 @@ def test_cluster_sanity_timeout_expired_error_converted(
         mock_exit_pytest.assert_called_once()
         call_args = mock_exit_pytest.call_args
         assert call_args[1]["filename"] == "cluster_sanity_failure.txt"
-        assert "Timed out waiting for all pods" in call_args[1]["message"]
-        assert "test-namespace" in call_args[1]["message"]
+        assert "Timed out waiting for all pods" in call_args[1]["log_message"]
+        assert "test-namespace" in call_args[1]["log_message"]
 
     @patch("utilities.sanity.storage_sanity_check")
     @patch("utilities.sanity.assert_nodes_in_healthy_condition")