From 7336da24aa466db0aaae79c4928540c10b9b7e2b Mon Sep 17 00:00:00 2001 From: Alessandro Bellina Date: Fri, 28 Jun 2024 07:01:01 -0700 Subject: [PATCH 1/3] Run ucx shuffle smoke tests under scala 2.13 Signed-off-by: Alessandro Bellina --- jenkins/spark-premerge-build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh index 697722c0138..67bf72053de 100755 --- a/jenkins/spark-premerge-build.sh +++ b/jenkins/spark-premerge-build.sh @@ -203,6 +203,9 @@ ci_scala213() { $MVN_CMD -U -B $MVN_URM_MIRROR clean package $MVN_BUILD_ARGS -DskipTests=true cd .. # Run integration tests in the project root dir to leverage test cases and resource files + + rapids_shuffle_smoke_test + export TEST_TAGS="not premerge_ci_1" export TEST_TYPE="pre-commit" # SPARK_HOME (and related) must be set to a Spark built with Scala 2.13 From 401bdb856e78c8a72ff48adb5ab0f0d3186c959e Mon Sep 17 00:00:00 2001 From: Alessandro Bellina Date: Fri, 28 Jun 2024 09:11:27 -0700 Subject: [PATCH 2/3] Force spark 3.3.0 for scala 2.13 --- jenkins/spark-premerge-build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh index 67bf72053de..56d9905041b 100755 --- a/jenkins/spark-premerge-build.sh +++ b/jenkins/spark-premerge-build.sh @@ -188,6 +188,7 @@ ci_scala213() { # Download a Scala 2.13 version of Spark prepare_spark 3.3.0 2.13 + export SHUFFLE_SPARK_SHIM=spark330 # build Scala 2.13 versions for version in "${SPARK_SHIM_VERSIONS_PREMERGE_SCALA213[@]}" From 5fa9758d7952f7ccf8525911ce386136189b6d99 Mon Sep 17 00:00:00 2001 From: Alessandro Bellina Date: Fri, 28 Jun 2024 14:53:14 -0700 Subject: [PATCH 3/3] Cleanup spark-premerge-build slightly --- jenkins/spark-premerge-build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh index 56d9905041b..9f990f2b996 100755 --- a/jenkins/spark-premerge-build.sh +++ b/jenkins/spark-premerge-build.sh @@ -188,6 +188,8 @@ ci_scala213() { # Download a Scala 2.13 version of Spark prepare_spark 3.3.0 2.13 + # Match the shim version to the Spark we are using for integration tests + # in order to pick the correct shuffle manager in our shuffle smoke tests export SHUFFLE_SPARK_SHIM=spark330 # build Scala 2.13 versions @@ -205,8 +207,6 @@ ci_scala213() { $MVN_CMD -U -B $MVN_URM_MIRROR clean package $MVN_BUILD_ARGS -DskipTests=true cd .. # Run integration tests in the project root dir to leverage test cases and resource files - rapids_shuffle_smoke_test - export TEST_TAGS="not premerge_ci_1" export TEST_TYPE="pre-commit" # SPARK_HOME (and related) must be set to a Spark built with Scala 2.13 @@ -218,6 +218,8 @@ ci_scala213() { # export 'LC_ALL' to set locale with UTF-8 so regular expressions are enabled SPARK_HOME=$SPARK_HOME PYTHONPATH=$PYTHONPATH \ LC_ALL="en_US.UTF-8" TEST="regexp_test.py" ./integration_tests/run_pyspark_from_build.sh + + rapids_shuffle_smoke_test } prepare_spark() {