Skip to content

Commit 1d9f521

Browse files
Subham-KRLXSubham-KRLX
committed
your commit message
1 parent 320d58d commit 1d9f521

File tree

10 files changed

+90
-421
lines changed

10 files changed

+90
-421
lines changed

‎.github/workflows/scripts/install_r_deps.sh‎

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -7,29 +7,11 @@
77
# (the "License"); you may not use this file except in compliance with
88
# the License. You may obtain a copy of the License at
99
#
10-
# http://www.apache.org/licenses/LICENSE-2.0
10+
# http://www.apache.org/licenses/LICENSE-2.0
1111
#
1212
# Unless required by applicable law or agreed to in writing, software
1313
# distributed under the License is distributed on an "AS IS" BASIS,
1414
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1515
# See the License for the specific language governing permissions and
1616
# limitations under the License.
1717
#
18-
19-
set -efux -o pipefail
20-
21-
TEST_DEPS="testthat"
22-
SEP='"\\s+"'
23-
24-
R_REMOTES_NO_ERRORS_FROM_WARNINGS=true Rscript - <<_RSCRIPT_EOF_
25-
if (!require(remotes))
26-
install.packages("remotes")
27-
remotes::install_deps(pkgdir = "./R", dependencies = c("Imports"), upgrade = "always")
28-
29-
test_deps <- strsplit("$TEST_DEPS", $SEP)[[1]]
30-
for (pkg in test_deps)
31-
if (!require(pkg, character.only = TRUE))
32-
install.packages(pkg)
33-
34-
remotes::install_github("sparklyr/sparklyr", ref = "main", upgrade = TRUE)
35-
_RSCRIPT_EOF_

‎.github/workflows/scripts/install_system_deps_for_r_build.sh‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,11 @@
77
# (the "License"); you may not use this file except in compliance with
88
# the License. You may obtain a copy of the License at
99
#
10-
# http://www.apache.org/licenses/LICENSE-2.0
10+
# http://www.apache.org/licenses/LICENSE-2.0
1111
#
1212
# Unless required by applicable law or agreed to in writing, software
1313
# distributed under the License is distributed on an "AS IS" BASIS,
1414
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1515
# See the License for the specific language governing permissions and
1616
# limitations under the License.
1717
#
18-
19-
set -efux -o pipefail
20-
21-
sudo DEBIAN_FRONTEND=noninteractive apt-get -y install libcurl4-openssl-dev

‎.github/workflows/scripts/prepare_sparklyr_sedona_test_env.sh‎

Lines changed: 14 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,17 @@
11
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
29
#
3-
# Licensed to the Apache Software Foundation (ASF) under one or more
4-
# contributor license agreements. See the NOTICE file distributed with
5-
# this work for additional information regarding copyright ownership.
6-
# The ASF licenses this file to You under the Apache License, Version 2.0
7-
# (the "License"); you may not use this file except in compliance with
8-
# the License. You may obtain a copy of the License at
10+
# http://www.apache.org/licenses/LICENSE-2.0
911
#
10-
# http://www.apache.org/licenses/LICENSE-2.0
11-
#
12-
# Unless required by applicable law or agreed to in writing, software
13-
# distributed under the License is distributed on an "AS IS" BASIS,
14-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15-
# See the License for the specific language governing permissions and
16-
# limitations under the License.
17-
#
18-
19-
sedona_jar_files () {
20-
local subdir
21-
for subdir in 'spark-shaded'; do
22-
echo "$(pwd)/$(find ${subdir}/target -name 'sedona-*.jar' \! -name '*javadoc*')"
23-
done
24-
}
25-
26-
export SEDONA_JAR_FILES="$(sedona_jar_files | paste -sd':')"
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.

‎docker/build.sh‎

Lines changed: 14 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -1,106 +1,17 @@
11
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
29
#
3-
# Licensed to the Apache Software Foundation (ASF) under one or more
4-
# contributor license agreements. See the NOTICE file distributed with
5-
# this work for additional information regarding copyright ownership.
6-
# The ASF licenses this file to You under the Apache License, Version 2.0
7-
# (the "License"); you may not use this file except in compliance with
8-
# the License. You may obtain a copy of the License at
10+
# http://www.apache.org/licenses/LICENSE-2.0
911
#
10-
# http://www.apache.org/licenses/LICENSE-2.0
11-
#
12-
# Unless required by applicable law or agreed to in writing, software
13-
# distributed under the License is distributed on an "AS IS" BASIS,
14-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15-
# See the License for the specific language governing permissions and
16-
# limitations under the License.
17-
#
18-
19-
SPARK_VERSION=$1
20-
SEDONA_VERSION=$2
21-
BUILD_MODE=$3
22-
GEOTOOLS_VERSION=${4:-auto}
23-
24-
SEDONA_SPARK_VERSION=${SPARK_VERSION:0:3}
25-
if [ "${SPARK_VERSION:0:1}" -eq "3" ] && [ "${SPARK_VERSION:2:1}" -le "3" ]; then
26-
# 3.0, 3.1, 3.2, 3.3
27-
SEDONA_SPARK_VERSION=3.0
28-
fi
29-
30-
# Function to compare two version numbers
31-
version_gt() {
32-
# Compare two version numbers
33-
# Returns 0 if the first version is greater, 1 otherwise
34-
[ "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1" ]
35-
}
36-
37-
# Function to get the latest version of a Maven package
38-
get_latest_version_with_suffix() {
39-
BASE_URL=$1
40-
SUFFIX=$2
41-
42-
# Fetch the maven-metadata.xml file
43-
METADATA_URL="${BASE_URL}maven-metadata.xml"
44-
METADATA_XML=$(curl -s "$METADATA_URL")
45-
46-
# Extract versions from the XML
47-
VERSIONS=$(echo "$METADATA_XML" | grep -o '<version>[^<]*</version>' | awk -F'[<>]' '{print $3}')
48-
49-
LATEST_VERSION=""
50-
51-
# Filter versions that end with the specified suffix and find the largest one
52-
for VERSION in $VERSIONS; do
53-
if [[ $VERSION == *$SUFFIX ]]; then
54-
if [[ -z $LATEST_VERSION ]] || version_gt "$VERSION" "$LATEST_VERSION"; then
55-
LATEST_VERSION=$VERSION
56-
fi
57-
fi
58-
done
59-
60-
if [[ -z $LATEST_VERSION ]]; then
61-
exit 1
62-
else
63-
echo "$LATEST_VERSION"
64-
fi
65-
}
66-
67-
if [ "$GEOTOOLS_VERSION" = "auto" ]; then
68-
GEOTOOLS_VERSION=$(mvn help:evaluate -Dexpression=geotools.version -q -DforceStdout)
69-
echo "GeoTools version inferred from pom.xml: $GEOTOOLS_VERSION"
70-
fi
71-
72-
GEOTOOLS_WRAPPER_VERSION="${SEDONA_VERSION}-${GEOTOOLS_VERSION}"
73-
if [ "$SEDONA_VERSION" = "latest" ]; then
74-
GEOTOOLS_WRAPPER_VERSION=$(get_latest_version_with_suffix "https://repo1.maven.org/maven2/org/datasyslab/geotools-wrapper/" "$GEOTOOLS_VERSION")
75-
if [ -z "$GEOTOOLS_WRAPPER_VERSION" ]; then
76-
echo "No geotools-wrapper version with suffix $GEOTOOLS_VERSION"
77-
exit 1
78-
fi
79-
echo "Using latest geotools-wrapper version: $GEOTOOLS_WRAPPER_VERSION"
80-
81-
# The compilation must take place outside Docker to avoid unnecessary maven packages
82-
mvn clean install -DskipTests -Dspark="${SEDONA_SPARK_VERSION}" -Dscala=2.12
83-
fi
84-
85-
# -- Building the image
86-
87-
if [ -z "$BUILD_MODE" ] || [ "$BUILD_MODE" = "local" ]; then
88-
# If local, build the image for the local environment
89-
docker buildx build --load \
90-
--build-arg spark_version="${SPARK_VERSION}" \
91-
--build-arg sedona_version="${SEDONA_VERSION}" \
92-
--build-arg geotools_wrapper_version="${GEOTOOLS_WRAPPER_VERSION}" \
93-
-f docker/sedona-docker.dockerfile \
94-
-t apache/sedona:"${SEDONA_VERSION}" .
95-
else
96-
# If release, build the image for cross-platform
97-
docker buildx build --platform linux/amd64,linux/arm64 \
98-
--progress=plain \
99-
--no-cache \
100-
--output type=registry \
101-
--build-arg spark_version="${SPARK_VERSION}" \
102-
--build-arg sedona_version="${SEDONA_VERSION}" \
103-
--build-arg geotools_wrapper_version="${GEOTOOLS_WRAPPER_VERSION}" \
104-
-f docker/sedona-docker.dockerfile \
105-
-t apache/sedona:"${SEDONA_VERSION}" .
106-
fi
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.

‎docker/install-sedona.sh‎

Lines changed: 1 addition & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -7,54 +7,11 @@
77
# (the "License"); you may not use this file except in compliance with
88
# the License. You may obtain a copy of the License at
99
#
10-
# http://www.apache.org/licenses/LICENSE-2.0
10+
# http://www.apache.org/licenses/LICENSE-2.0
1111
#
1212
# Unless required by applicable law or agreed to in writing, software
1313
# distributed under the License is distributed on an "AS IS" BASIS,
1414
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1515
# See the License for the specific language governing permissions and
1616
# limitations under the License.
1717
#
18-
19-
set -e
20-
21-
# Define variables
22-
sedona_version=$1
23-
geotools_wrapper_version=$2
24-
spark_version=$3
25-
spark_extension_version=$4
26-
27-
spark_compat_version=${spark_version:0:3}
28-
sedona_spark_version=${spark_compat_version}
29-
if [ "${spark_version:0:1}" -eq "3" ] && [ "${spark_version:2:1}" -le "3" ]; then
30-
# 3.0, 3.1, 3.2, 3.3
31-
sedona_spark_version=3.0
32-
fi
33-
34-
if [ "$sedona_version" = "latest" ]; then
35-
# Code to execute when SEDONA_VERSION is "latest"
36-
cp "${SEDONA_HOME}"/spark-shaded/target/sedona-spark-shaded-*.jar "${SPARK_HOME}"/jars/
37-
cd "${SEDONA_HOME}"/python; pip3 install .
38-
else
39-
# Code to execute when SEDONA_VERSION is not "latest"
40-
# Download Sedona
41-
curl --retry 5 --retry-delay 10 --retry-connrefused \
42-
https://repo1.maven.org/maven2/org/apache/sedona/sedona-spark-shaded-"${sedona_spark_version}"_2.12/"${sedona_version}"/sedona-spark-shaded-"${sedona_spark_version}"_2.12-"${sedona_version}".jar \
43-
-o "$SPARK_HOME"/jars/sedona-spark-shaded-"${sedona_spark_version}"_2.12-"${sedona_version}".jar
44-
45-
# Install Sedona Python
46-
pip3 install apache-sedona=="${sedona_version}"
47-
fi
48-
49-
# Download gresearch spark extension
50-
curl --retry 5 --retry-delay 10 --retry-connrefused \
51-
https://repo1.maven.org/maven2/uk/co/gresearch/spark/spark-extension_2.12/"${spark_extension_version}"-"${spark_compat_version}"/spark-extension_2.12-"${spark_extension_version}"-"${spark_compat_version}".jar \
52-
-o "$SPARK_HOME"/jars/spark-extension_2.12-"${spark_extension_version}"-"${spark_compat_version}".jar
53-
54-
# Install Spark extension Python
55-
pip3 install pyspark-extension=="${spark_extension_version}"."${spark_compat_version}"
56-
57-
# Download GeoTools jar
58-
curl --retry 5 --retry-delay 10 --retry-connrefused \
59-
https://repo1.maven.org/maven2/org/datasyslab/geotools-wrapper/"${geotools_wrapper_version}"/geotools-wrapper-"${geotools_wrapper_version}".jar \
60-
-o "$SPARK_HOME"/jars/geotools-wrapper-"${geotools_wrapper_version}".jar

‎docker/install-spark.sh‎

Lines changed: 2 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -7,61 +7,11 @@
77
# (the "License"); you may not use this file except in compliance with
88
# the License. You may obtain a copy of the License at
99
#
10-
# http://www.apache.org/licenses/LICENSE-2.0
10+
# http://www.apache.org/licenses/LICENSE-2.0
1111
#
1212
# Unless required by applicable law or agreed to in writing, software
1313
# distributed under the License is distributed on an "AS IS" BASIS,
1414
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1515
# See the License for the specific language governing permissions and
1616
# limitations under the License.
17-
#
18-
19-
set -e
20-
21-
# Define variables
22-
spark_version=$1
23-
hadoop_s3_version=$2
24-
aws_sdk_version=$3
25-
spark_xml_version=$4
26-
27-
# Validate inputs
28-
if [[ -z "$spark_version" || -z "$hadoop_s3_version" || -z "$aws_sdk_version" || -z "$spark_xml_version" ]]; then
29-
echo "Usage: $0 <spark_version> <hadoop_s3_version> <aws_sdk_version> <spark_xml_version>"
30-
exit 1
31-
fi
32-
33-
# Download Spark jar and set up PySpark
34-
curl --retry 5 --retry-delay 10 --retry-connrefused "https://archive.apache.org/dist/spark/spark-${spark_version}/spark-${spark_version}-bin-hadoop3.tgz" -o spark.tgz
35-
tar -xf spark.tgz && mv "spark-${spark_version}-bin-hadoop3"/* "${SPARK_HOME}/"
36-
rm spark.tgz
37-
rm -rf "spark-${spark_version}-bin-hadoop3"
38-
39-
# Add S3 jars
40-
curl --retry 5 --retry-delay 10 --retry-connrefused "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${hadoop_s3_version}/hadoop-aws-${hadoop_s3_version}.jar" -o "${SPARK_HOME}/jars/hadoop-aws-${hadoop_s3_version}.jar"
41-
curl --retry 5 --retry-delay 10 --retry-connrefused "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${aws_sdk_version}/aws-java-sdk-bundle-${aws_sdk_version}.jar" -o "${SPARK_HOME}/jars/aws-java-sdk-bundle-${aws_sdk_version}.jar"
42-
43-
# Add spark-xml jar
44-
curl --retry 5 --retry-delay 10 --retry-connrefused "https://repo1.maven.org/maven2/com/databricks/spark-xml_2.12/${spark_xml_version}/spark-xml_2.12-${spark_xml_version}.jar" -o "${SPARK_HOME}/jars/spark-xml_2.12-${spark_xml_version}.jar"
45-
46-
# Set up master IP address and executor memory
47-
cp "${SPARK_HOME}/conf/spark-defaults.conf.template" "${SPARK_HOME}/conf/spark-defaults.conf"
48-
49-
# Install required libraries for GeoPandas on Apple chip mac
50-
# Note: 'apt-get' is for Debian/Ubuntu, check if running on Linux
51-
if command -v apt-get &>/dev/null; then
52-
apt-get update
53-
apt-get install -y gdal-bin libgdal-dev openssh-client openssh-server
54-
systemctl enable ssh
55-
elif [[ "$(uname)" == "Darwin" ]]; then
56-
echo "Please install gdal and openssh with Homebrew:"
57-
echo " brew install gdal openssh"
58-
else
59-
echo "Unsupported OS for automatic package installation. Please install gdal and openssh manually."
60-
fi
61-
62-
# Enable passwordless SSH
63-
if [[ ! -f ~/.ssh/id_rsa ]]; then
64-
ssh-keygen -t rsa -f ~/.ssh/id_rsa -N ""
65-
fi
66-
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
67-
chmod 600 ~/.ssh/authorized_keys
17+
#

‎docker/install-zeppelin.sh‎

Lines changed: 14 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,17 @@
11
#!/bin/bash
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
29
#
3-
# Licensed to the Apache Software Foundation (ASF) under one or more
4-
# contributor license agreements. See the NOTICE file distributed with
5-
# this work for additional information regarding copyright ownership.
6-
# The ASF licenses this file to You under the Apache License, Version 2.0
7-
# (the "License"); you may not use this file except in compliance with
8-
# the License. You may obtain a copy of the License at
10+
# http://www.apache.org/licenses/LICENSE-2.0
911
#
10-
# http://www.apache.org/licenses/LICENSE-2.0
11-
#
12-
# Unless required by applicable law or agreed to in writing, software
13-
# distributed under the License is distributed on an "AS IS" BASIS,
14-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15-
# See the License for the specific language governing permissions and
16-
# limitations under the License.
17-
#
18-
19-
set -e
20-
21-
# Define Zeppelin version and target directory
22-
ZEPPELIN_VERSION=${1:-0.9.0}
23-
TARGET_DIR=${2:-/opt}
24-
25-
# Download and extract Zeppelin using curl
26-
curl -L --retry 5 --retry-delay 10 --retry-connrefused https://archive.apache.org/dist/zeppelin/zeppelin-"${ZEPPELIN_VERSION}"/zeppelin-"${ZEPPELIN_VERSION}"-bin-netinst.tgz \
27-
-o zeppelin-"${ZEPPELIN_VERSION}"-bin-netinst.tgz
28-
tar -xzf zeppelin-"${ZEPPELIN_VERSION}"-bin-netinst.tgz -C "${TARGET_DIR}"
29-
mv "${TARGET_DIR}"/zeppelin-"${ZEPPELIN_VERSION}"-bin-netinst "${ZEPPELIN_HOME}"
30-
rm zeppelin-"${ZEPPELIN_VERSION}"-bin-netinst.tgz
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.

0 commit comments

Comments
 (0)