Fix: Local setup test cases fixed

README.md

@@ -14,6 +14,7 @@
 * [Run Test & Coverage](#run-tests-&-coverage-report)
 * [Documentation](Documentation.md#project-documentation)
 * [Reference](#reference)
+* [Common Errors](#common-errors)
 
 ## Objective
 
@@ -128,3 +129,6 @@ To run tests & coverage report, run the following commands in the root folder of
 
 [Glue Programming libraries](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-libraries.html)
 
+## Common Errors
+
+['sparkDriver' failed after 16 retries](https://stackoverflow.com/questions/52133731/how-to-solve-cant-assign-requested-address-service-sparkdriver-failed-after)

jobs/demo.py

@@ -2,8 +2,8 @@
 from dotenv import load_dotenv
 import app.environment as env
 
-load_dotenv("../app/.custom_env") # Loading env for databricks
-load_dotenv() # Loading env for glue
+load_dotenv("../app/.custom_env")  # Loading env for databricks
+load_dotenv()  # Loading env for glue
 
 # COMMAND ----------
 

jobs/main.py

@@ -9,8 +9,8 @@
 import app.environment as env
 import app.spark_wrapper as sw
 
-load_dotenv("../app/.custom_env") # Loading env for databricks
-load_dotenv() # Loading env for glue
+load_dotenv("../app/.custom_env")  # Loading env for databricks
+load_dotenv()  # Loading env for glue
 
 # COMMAND ----------
 

requirements.txt

@@ -4,4 +4,3 @@ coverage
 python-dotenv
 kaggle~=1.5.16
 pre-commit
-pyspark==3.3.0

tests/test_spark_wrapper_failure.py

@@ -29,10 +29,14 @@ def test_value_counts_invalid_column(self):
         with self.assertRaises(U.AnalysisException) as context:
             value_counts(self.df, "nonexistent_column")
 
-        expected_error_message = re.compile("Column '.+' does not exist")
+        expected_error_message_1 = re.compile("Column '.+' does not exist")
+        expected_error_message_2 = re.compile("cannot resolve '.+' given input columns")
         actual_error_message = str(context.exception)
 
-        self.assertTrue(expected_error_message.search(actual_error_message))
+        self.assertTrue(
+            expected_error_message_1.search(actual_error_message)
+            or expected_error_message_2.search(actual_error_message)
+        )
 
     def test_create_frame_invalid_path(self):
         with self.assertRaises(U.AnalysisException) as context:
@@ -48,19 +52,30 @@ def test_make_window_invalid_window_spec(self):
             window_spec = make_window("invalid_column", "date", -20, -1)
             self.df.withColumn("literal_1", F.lit(1).over(window_spec))
 
-        expected_error_message = re.compile("Column '.+' does not exist")
+        expected_error_message_1 = re.compile("Column '.+' does not exist")
+        expected_error_message_2 = re.compile("cannot resolve '.+' given input columns")
         actual_error_message = str(context.exception)
 
-        self.assertTrue(expected_error_message.search(actual_error_message))
+        self.assertTrue(
+            expected_error_message_1.search(actual_error_message)
+            or expected_error_message_2.search(actual_error_message)
+        )
 
     def test_make_window_invalid_range(self):
         with self.assertRaises(U.AnalysisException) as context:
             window_spec = make_window("market", "date", 5, 2)
             self.df.withColumn("literal_1", F.lit(1).over(window_spec))
 
-        expected_error_message = "The lower bound of a window frame must be less than or equal to the upper bound"
+        expected_error_message_1 = "The lower bound of a window frame must be less than or equal to the upper bound"
+        expected_error_message_2 = re.compile(
+            "The data type of the lower bound '.+' does not match the expected data type '.+'"
+        )
         actual_error_message = str(context.exception)
-        self.assertTrue(expected_error_message in actual_error_message)
+
+        self.assertTrue(
+            expected_error_message_1 in actual_error_message
+            or expected_error_message_2.search(actual_error_message)
+        )
 
     def test_rename_column_invalid_column(self):
         with self.assertRaises(ValueError) as context: