testing kubernetes deployment for spark

Author: angelosnm

Diff for: .github/workflows/tithia-jupyter-docker-build.yml

@@ -0,0 +1,37 @@
+name: Build and Push Tithia Jupyter Docker Image to GHCR
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - '.github/workflows/tithia-jupyter-docker-build.yml'
+      - 'docker/tithia-jupyter/**'
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GHCR_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./docker
+          file: ./docker/Dockerfile
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/${{ github.repository }}-jupyter:latest
+            ghcr.io/${{ github.repository_owner }}/${{ github.repository }}-jupyter:0.${{ github.run_number }}

Diff for: .github/workflows/docker-build.yml renamed to .github/workflows/tithia-spark-docker-build

@@ -1,12 +1,12 @@
-name: Build and Push Docker Image to GHCR
+name: Build and Push Tithia Spark Docker Image to GHCR
 
 on:
   push:
     branches:
       - main
     paths:
-      - '.github/workflows/docker-build.yml'
-      - 'docker/**'
+      - '.github/workflows/tithia-spark-docker-build.yml'
+      - 'docker/tithia-spark/**'
 
 jobs:
   build-and-push:
@@ -33,5 +33,5 @@ jobs:
           file: ./docker/Dockerfile
           push: true
           tags: |
-            ghcr.io/${{ github.repository_owner }}/${{ github.repository }}:latest
-            ghcr.io/${{ github.repository_owner }}/${{ github.repository }}:0.${{ github.run_number }}
+            ghcr.io/${{ github.repository_owner }}/${{ github.repository }}-spark:latest
+            ghcr.io/${{ github.repository_owner }}/${{ github.repository }}-spark:3.5.2-0.${{ github.run_number }}

Diff for: README.md

@@ -259,4 +259,8 @@ helm upgrade jupyterhub jupyterhub/jupyterhub --install --cleanup-on-fail -n jup
 k apply -f jupyterhub/certificate.yaml
 
 k apply -f jupyterhub/ingress.yaml
-```
+```
+
+### Spark
+
+kubectl create configmap spark-config --from-file=spark-defaults.conf --from-file=spark-env.sh -n spark

Diff for: docker/docker-compose.yaml

@@ -5,5 +5,4 @@ keplergl==0.3.2
 rasterio==1.4.2
 pydeck==0.9.1
 apache-sedona==1.6.1
-jupyterlab-s3-browser==0.12.0
-sparkmagic==0.22.0
+jupyterlab-s3-browser==0.12.0

Diff for: docker/.dockerignore renamed to docker/tithia-jupyter/.dockerignore

@@ -0,0 +1,4 @@
+FROM apache/spark:3.5.2
+
+RUN wget -P /opt/spark/jars https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar
+RUN wget -P /opt/spark/jars https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.777/aws-java-sdk-bundle-1.12.777.jar

Diff for: docker/Dockerfile renamed to docker/tithia-jupyter/Dockerfile

@@ -366,7 +366,7 @@ singleuser:
     blockWithIptables: true
     ip: 169.254.169.254
   networkPolicy:
-    enabled: true
+    enabled: false
     ingress: []
     egress: []
     egressAllowRules:

Diff for: docker/requirements.txt renamed to docker/tithia-jupyter/requirements.txt

@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: spark-master
+  namespace: spark
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: spark
+      role: master
+  template:
+    metadata:
+      labels:
+        app: spark
+        role: master
+    spec:
+      containers:
+        - name: spark-master
+          image: apache/spark:3.5.2
+          command: ["/bin/bash", "-c", "/opt/spark/bin/spark-class org.apache.spark.deploy.master.Master"]
+          ports:
+            - containerPort: 7077 # Spark Master port
+            - containerPort: 8080 # Spark Web UI

Diff for: docker/tithia-spark/Dockerfile

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: spark-worker
+  namespace: spark
+spec:
+  replicas: 2 # Number of Spark Worker replicas
+  selector:
+    matchLabels:
+      app: spark
+      role: worker
+  template:
+    metadata:
+      labels:
+        app: spark
+        role: worker
+    spec:
+      containers:
+        - name: spark-worker
+          image: apache/spark:3.5.2
+          command: ["/bin/bash", "-c", "/opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077"]
+          ports:
+            - containerPort: 8081 # Spark Worker Web UI

Diff for: kube/jupyterhub/values.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: spark-master
+  namespace: spark
+spec:
+  selector:
+    app: spark
+    role: master
+  ports:
+    - protocol: TCP
+      port: 7077
+      targetPort: 7077
+      name: spark
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080
+      name: web
+  type: ClusterIP