From 1f42b57d931a69c01ec962835f9020d9980bd079 Mon Sep 17 00:00:00 2001
From: Umayr Hassan <umayrh@gmail.com>
Date: Sat, 28 Dec 2019 22:56:48 -0800
Subject: [PATCH] Update libs + fix Apache Spark mirror

---
 requirements.txt                    |  8 ++++----
 scripts/travis-setup.sh             | 16 ++++++++--------
 setup.py                            |  2 +-
 src/sparktuner.egg-info/SOURCES.txt |  1 -
 4 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 900b6f1..99cf639 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
-opentuner==0.8.0
-humanfriendly==4.17
-chainmap==1.0.2
+opentuner==0.8.2
+humanfriendly==4.18
+chainmap==1.0.3
 psutil==5.4.8
-requests==2.21.0
+requests==2.22.0
 requests-mock==1.5.2
\ No newline at end of file
diff --git a/scripts/travis-setup.sh b/scripts/travis-setup.sh
index 95167c6..f94a5fb 100755
--- a/scripts/travis-setup.sh
+++ b/scripts/travis-setup.sh
@@ -5,8 +5,9 @@ set -ex
 OS=$(uname -s)
 
 ## Service versions
-SPARK_VERSION=${SPARK_VERSION:-"2.4.0"}
+SPARK_VERSION=${SPARK_VERSION:-"2.4.4"}
 HADOOP_VERSION=${HADOOP_VERSION:-"2.7"}
+SPARK_MIRROR="http://ftp.wayne.edu/apache/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz"
 
 ## OS-specific package installation
 bootstrap() {
@@ -34,16 +35,15 @@ createGlobalEnvFile() {
 ## Installs a specific version of Spark
 setupSpark() {
     local SPARK_DIR_NAME=spark-${SPARK_VERSION}
-    SPARK_DIST_NAME=${SPARK_DIR_NAME}-bin-hadoop${HADOOP_VERSION}
-    if [[ ! -d "$HOME/.cache/${SPARK_DIST_NAME}" ]]; then
+    if [[ ! -d "$HOME/.cache/${SPARK_DIR_NAME}" ]]; then
         cd $HOME/.cache
+        SPARK_DIST_NAME=${SPARK_DIR_NAME}-bin-hadoop${HADOOP_VERSION}
         rm -fr ./${SPARK_DIST_NAME}.tgz*
-        # Use axel again when https://github.com/axel-download-accelerator/axel/issues/192
-        # has been fixed.
-        # axel --quiet http://www-us.apache.org/dist/spark/${SPARK_DIR_NAME}/${SPARK_DIST_NAME}.tgz
-        wget --quiet http://www-us.apache.org/dist/spark/${SPARK_DIR_NAME}/${SPARK_DIST_NAME}.tgz
-        ls -alh ${SPARK_DIST_NAME}.tgz
+        axel --quiet ${SPARK_MIRROR}
         tar -xf ./${SPARK_DIST_NAME}.tgz
+        export SPARK_HOME=`pwd`/${SPARK_DIST_NAME}
+        # TODO: need a more systematic method for setting up Spark properties
+        echo "spark.yarn.jars=${SPARK_HOME}/jars/*.jar" > ${SPARK_HOME}/conf/spark-defaults.conf
         cd ..
     fi
     export SPARK_HOME="${HOME}/.cache/${SPARK_DIST_NAME}"
diff --git a/setup.py b/setup.py
index 702de4c..6e286e6 100644
--- a/setup.py
+++ b/setup.py
@@ -152,7 +152,7 @@ def load_pinned_deps(self):
     description='Package for automating Spark application tuning',
     long_description=open('README.txt').read(),
     install_requires=[
-        "opentuner==0.8.0",
+        "opentuner==0.8.2",
         "humanfriendly==4.17",
         "chainmap==1.0.2",
         "psutil==5.4.8",
diff --git a/src/sparktuner.egg-info/SOURCES.txt b/src/sparktuner.egg-info/SOURCES.txt
index dc51751..d0b87fc 100644
--- a/src/sparktuner.egg-info/SOURCES.txt
+++ b/src/sparktuner.egg-info/SOURCES.txt
@@ -19,7 +19,6 @@ gradle/wrapper/gradle-wrapper.jar
 gradle/wrapper/gradle-wrapper.properties
 scripts/get_requirements.py
 scripts/pivy-importer-0.12.8-SNAPSHOT-all.jar
-scripts/pivy-importer-0.9.9-all.jar
 scripts/travis-setup.sh
 src/sparktuner/__init__.py
 src/sparktuner/args.py