setup changes

automation/glue_setup.sh

@@ -29,7 +29,7 @@ export SPARK_HOME=$(pwd)/spark
 
 # Export Path
 export PATH=$PATH:$SPARK_HOME/bin:$MAVEN_HOME/bin:$AWS_GLUE_HOME/bin
-export PYTHONPATH=$PROJECT_ROOT
+export PYTHONPATH=$PROJECT_ROOT:$AWS_GLUE_HOME/PyGlue.zip:$SPARK_HOME/python/lib/py4j-0.10.9-src.zip:$SPARK_HOME/python/
 
 # Download Glue ETL .jar files
 cd $AWS_GLUE_HOME
@@ -39,11 +39,13 @@ mvn install dependency:copy-dependencies
 cp $AWS_GLUE_HOME/jarsv1/AWSGlue*.jar $SPARK_HOME/jars/
 cp $AWS_GLUE_HOME/jarsv1/aws*.jar $SPARK_HOME/jars/
 
-echo "export AWS_GLUE_HOME=$AWS_GLUE_HOME
+echo "
+export AWS_GLUE_HOME=$AWS_GLUE_HOME
 export MAVEN_HOME=$MAVEN_HOME
 export SPARK_HOME=$SPARK_HOME
 export PATH=$PATH:$SPARK_HOME/bin:$MAVEN_HOME/bin:$AWS_GLUE_HOME/bin
-export PYTHONPATH=$PROJECT_ROOT" >> $SOURCE_FILE
+export PYTHONPATH=$PROJECT_ROOT:$AWS_GLUE_HOME/PyGlue.zip:$SPARK_HOME/python/lib/py4j-0.10.9-src.zip:$SPARK_HOME/python/
+" >> $SOURCE_FILE
 
 
 cd $PROJECT_ROOT

jobs/demo.py

@@ -2,8 +2,8 @@
 from dotenv import load_dotenv
 import app.environment as env
 
-load_dotenv("../app/.custom_env") # Loading env for databricks
-load_dotenv() # Loading env for glue
+load_dotenv("../app/.custom_env")  # Loading env for databricks
+load_dotenv()  # Loading env for glue
 
 # COMMAND ----------
 

jobs/main.py

@@ -9,8 +9,8 @@
 import app.environment as env
 import app.spark_wrapper as sw
 
-load_dotenv("../app/.custom_env") # Loading env for databricks
-load_dotenv() # Loading env for glue
+load_dotenv("../app/.custom_env")  # Loading env for databricks
+load_dotenv()  # Loading env for glue
 
 # COMMAND ----------
 

requirements.txt

@@ -4,3 +4,4 @@ coverage
 python-dotenv
 kaggle~=1.5.16
 pre-commit
+pyspark==3.1.1

tests/test_spark_wrapper_failure.py

@@ -29,10 +29,14 @@ def test_value_counts_invalid_column(self):
         with self.assertRaises(U.AnalysisException) as context:
             value_counts(self.df, "nonexistent_column")
 
-        expected_error_message = re.compile("Column '.+' does not exist")
+        expected_error_message_1 = re.compile("Column '.+' does not exist")
+        expected_error_message_2 = re.compile("cannot resolve '.+' given input columns")
         actual_error_message = str(context.exception)
 
-        self.assertTrue(expected_error_message.search(actual_error_message))
+        self.assertTrue(
+            expected_error_message_1.search(actual_error_message)
+            or expected_error_message_2.search(actual_error_message)
+        )
 
     def test_create_frame_invalid_path(self):
         with self.assertRaises(U.AnalysisException) as context:
@@ -48,19 +52,30 @@ def test_make_window_invalid_window_spec(self):
             window_spec = make_window("invalid_column", "date", -20, -1)
             self.df.withColumn("literal_1", F.lit(1).over(window_spec))
 
-        expected_error_message = re.compile("Column '.+' does not exist")
+        expected_error_message_1 = re.compile("Column '.+' does not exist")
+        expected_error_message_2 = re.compile("cannot resolve '.+' given input columns")
         actual_error_message = str(context.exception)
 
-        self.assertTrue(expected_error_message.search(actual_error_message))
+        self.assertTrue(
+            expected_error_message_1.search(actual_error_message)
+            or expected_error_message_2.search(actual_error_message)
+        )
 
     def test_make_window_invalid_range(self):
         with self.assertRaises(U.AnalysisException) as context:
             window_spec = make_window("market", "date", 5, 2)
             self.df.withColumn("literal_1", F.lit(1).over(window_spec))
 
-        expected_error_message = "The lower bound of a window frame must be less than or equal to the upper bound"
+        expected_error_message_1 = "The lower bound of a window frame must be less than or equal to the upper bound"
+        expected_error_message_2 = re.compile(
+            "The data type of the lower bound '.+' does not match the expected data type '.+'"
+        )
         actual_error_message = str(context.exception)
-        self.assertTrue(expected_error_message in actual_error_message)
+
+        self.assertTrue(
+            expected_error_message_1 in actual_error_message
+            or expected_error_message_2.search(actual_error_message)
+        )
 
     def test_rename_column_invalid_column(self):
         with self.assertRaises(ValueError) as context: