chore: update jupyterhub-pyspark demo with newer image and dependencies (#112)

* chore: update jupyterhub-pyspark demo with newer image and dependencies

* linting

* linting II

* linting III

* linting again

* remove spaces in braces

* deactivate the arm runner

* renaming

Author: adwk67

.github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml .github/workflows/dev_spark-k8s-with-scikit-learn.yaml

@@ -1,9 +1,9 @@
 ---
-name: Build and publish pyspark-k8s-with-scikit-learn
+name: Build and publish spark-k8s-with-scikit-learn
 
 env:
-  IMAGE_NAME: pyspark-k8s-with-scikit-learn
-  IMAGE_VERSION: 3.4.0-stackable0.0.0-dev
+  IMAGE_NAME: spark-k8s-with-scikit-learn
+  IMAGE_VERSION: 3.5.0-stackable24.3.0
   REGISTRY_PATH: stackable
   DOCKERFILE_PATH: "demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile"
 
@@ -15,7 +15,7 @@ on:
     paths:
       - demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile
       - demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt
-      - .github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml
+      - .github/workflows/dev_spark-k8s-with-scikit-learn.yaml
 
 jobs:
   build:
@@ -27,7 +27,9 @@ jobs:
       matrix:
         runner:
           - {name: "ubuntu-latest", arch: "amd64"}
-          - {name: "ubicloud-standard-8-arm", arch: "arm64"}
+          # TODO: the image 3.5.0-stackable24.3.0 does not have an arm64 build.
+          # Re-activate the arm runner when the image is updated to one that does.
+          #- {name: "ubicloud-standard-8-arm", arch: "arm64"}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.stackable.tech/stackable/spark-k8s:3.5.1-stackable24.7.0
+FROM docker.stackable.tech/stackable/spark-k8s:3.5.0-stackable24.3.0
 
 COPY demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt .
 

demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt

@@ -1,2 +1,2 @@
-scikit-learn==1.1.3
-pandas==1.5.1
+scikit-learn==1.3.1
+pandas==2.0.3

stacks/_templates/jupyterhub.yaml

@@ -1,3 +1,4 @@
+---
 releaseName: jupyterhub
 name: jupyterhub
 repo:
@@ -11,7 +12,7 @@ options:
         allowed_users:
           - admin
       DummyAuthenticator:
-        password: {{ jupyterHubAdminPassword }}
+        password: {{jupyterHubAdminPassword}}
       JupyterHub:
         authenticator_class: dummy
     labels:
@@ -39,7 +40,7 @@ options:
       # Inspect the Dockerfile at:
       # https://github.com/jupyter/docker-stacks/tree/HEAD/datascience-notebook/Dockerfile
       name: jupyter/pyspark-notebook
-      tag: python-3.9
+      tag: python-3.11
     serviceAccountName: spark
     networkPolicy:
       enabled: false

stacks/jupyterhub-pyspark-hdfs/notebook.ipynb

@@ -34,7 +34,7 @@
     "spark = (SparkSession\n",
     "            .builder\n",
     "            .master(f'k8s://https://{os.environ[\"KUBERNETES_SERVICE_HOST\"]}:{os.environ[\"KUBERNETES_SERVICE_PORT\"]}')\n",
-    "            .config(\"spark.kubernetes.container.image\", \"docker.stackable.tech/demos/pyspark-k8s-with-scikit-learn:3.3.0-stackable23.4\")\n",
+    "            .config(\"spark.kubernetes.container.image\", \"docker.stackable.tech/demos/spark-k8s-with-scikit-learn:3.5.0-stackable24.3.0\")\n",
     "            .config(\"spark.driver.port\", \"2222\")\n",
     "            .config(\"spark.driver.blockManager.port\", \"7777\")\n",
     "            .config(\"spark.driver.host\", \"driver-service.default.svc.cluster.local\")\n",