Merge pull request #31 from kbase/dev_cdm_mcp

upgrade deps and add spark log

Author: Tianhao-Gu

Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/kbase/cdm-spark-standalone:pr-34
+FROM ghcr.io/kbase/cdm-spark-standalone:pr-36
 
 # Switch to root to install packages
 USER root

config/cdm-spark-job-logs-policy.json

@@ -0,0 +1,21 @@
+
+{
+    "Version": "2012-10-17",
+    "Statement": [
+      {
+        "Effect": "Allow",
+        "Action": [
+          "s3:GetBucketLocation",
+          "s3:ListBucket",
+          "s3:GetObject",
+          "s3:PutObject",
+          "s3:DeleteObject"
+        ],
+        "Resource": [
+          "arn:aws:s3:::cdm-spark-job-logs",
+          "arn:aws:s3:::cdm-spark-job-logs/*"
+        ]
+      }
+    ]
+  } 
+  

docker-compose.yaml

@@ -3,7 +3,7 @@
 services:
 
   spark-master:
-    image: ghcr.io/kbase/cdm-spark-standalone:pr-34
+    image: ghcr.io/kbase/cdm-spark-standalone:pr-36
     platform: linux/amd64
     ports:
       - "8090:8090"
@@ -27,7 +27,7 @@ services:
       - cdm-jupyterhub-network
 
   spark-worker-1:
-    image: ghcr.io/kbase/cdm-spark-standalone:pr-34
+    image: ghcr.io/kbase/cdm-spark-standalone:pr-36
     platform: linux/amd64
     ports:
       - "8081:8081"
@@ -51,7 +51,7 @@ services:
       - cdm-jupyterhub-network
 
   spark-worker-2:
-    image: ghcr.io/kbase/cdm-spark-standalone:pr-34
+    image: ghcr.io/kbase/cdm-spark-standalone:pr-36
     platform: linux/amd64
     ports:
       - "8082:8082"
@@ -103,12 +103,13 @@ services:
     volumes:
       - ./config/cdm-read-only-policy.json:/config/cdm-read-only-policy.json
       - ./config/cdm-read-write-policy.json:/config/cdm-read-write-policy.json
+      - ./config/cdm-spark-job-logs-policy.json:/config/cdm-spark-job-logs-policy.json
       - ./scripts/minio_create_bucket_entrypoint.sh:/scripts/minio_create_bucket_entrypoint.sh
     networks:
       - cdm-jupyterhub-network
 
   cdm_jupyterhub:
-    image: ghcr.io/kbase/cdm-jupyterhub:pr-177
+    image: ghcr.io/kbase/cdm-jupyterhub:pr-193
     platform: linux/amd64
     ports:
       - "4043:4043"
@@ -140,6 +141,11 @@ services:
       - USE_KBASE_AUTHENTICATOR=true
       - USE_KUBE_SPAWNER=false
       - REMOVE_STOPPED_CONTAINER_AND_POD=false
+      - SPARK_JOB_LOG_DIR=s3a://cdm-spark-job-logs/spark-job-logs
+      - SPARK_JOB_LOG_DIR_CATEGORY=jupyter-master
+      - MINIO_URL=http://minio:9002
+      - MINIO_LOG_USER_ACCESS_KEY=minio-log-access
+      - MINIO_LOG_USER_SECRET_KEY=minio123
     volumes:
       - ./cdr/cdm/jupyter/cdm_shared_workspace:/cdm_shared_workspace
       - ./cdr/cdm/jupyter/jupyterhub_secrets:/jupyterhub_secrets
@@ -198,6 +204,11 @@ services:
       - REDIS_HOST=redis
       - REDIS_PORT=6379
       - DELTALAKE_WAREHOUSE_DIR=s3a://cdm-lake/warehouse
+      - SPARK_JOB_LOG_DIR=s3a://cdm-spark-job-logs/spark-job-logs
+      - SPARK_JOB_LOG_DIR_CATEGORY=mcp-server
+      - MINIO_URL=http://minio:9002
+      - MINIO_LOG_USER_ACCESS_KEY=minio-log-access
+      - MINIO_LOG_USER_SECRET_KEY=minio123
     networks:
       - cdm-jupyterhub-network
 

pyproject.toml

@@ -6,12 +6,12 @@ readme = "README.md"
 requires-python = "==3.11.9"
 dependencies = [
     "fastapi==0.115.12",
-    "uvicorn==0.34.2",
+    "uvicorn==0.34.3",
     "fastapi-mcp==0.3.4",
-    "mcp==1.9.1",
+    "mcp==1.9.2",
     "pyspark==3.5.5",
-    "delta-spark==3.3.1",
-    "aiohttp==3.11.18",
+    "delta-spark==3.3.2",
+    "aiohttp==3.12.8",
     "cacheout==0.16.0",
     "minio==7.2.15",
     "psycopg==3.2.9",
@@ -20,7 +20,7 @@ dependencies = [
 
 [dependency-groups]
 dev = [
-    "ipython>=9.2.0",
-    "pytest==8.3.5",
+    "ipython==9.3.0",
+    "pytest==8.4.0",
     "pytest-cov==6.1.1",
 ]

scripts/minio_create_bucket_entrypoint.sh

@@ -9,16 +9,37 @@ else
   echo 'bucket cdm-lake already exists'
 fi
 
+# make spark job logs bucket
+if ! mc ls minio/cdm-spark-job-logs 2>/dev/null; then
+  mc mb minio/cdm-spark-job-logs && echo 'Bucket cdm-spark-job-logs created'
+else
+  echo 'bucket cdm-spark-job-logs already exists'
+fi
+
+# create spark-job-logs directory inside the bucket
+# Create a dummy file to ensure the directory exists, then remove it
+echo "dummy" | mc pipe minio/cdm-spark-job-logs/spark-job-logs/.dummy
+echo 'spark-job-logs directory created in cdm-spark-job-logs bucket'
+
+
 # create policies
 mc admin policy create minio cdm-read-only-policy /config/cdm-read-only-policy.json
 mc admin policy create minio cdm-read-write-policy /config/cdm-read-write-policy.json
+mc admin policy create minio cdm-spark-job-logs-policy /config/cdm-spark-job-logs-policy.json
 
 # make read only user for user notebook
 mc admin user add minio minio-readonly minio123
 mc admin policy attach minio cdm-read-only-policy --user=minio-readonly
+mc admin policy attach minio cdm-spark-job-logs-policy --user=minio-readonly
 echo 'CDM Read-only user and policy set'
 
 # make read/write user
 mc admin user add minio minio-readwrite minio123
 mc admin policy attach minio cdm-read-write-policy --user=minio-readwrite
-echo 'CDM read-write user and policy set'
+mc admin policy attach minio cdm-spark-job-logs-policy --user=minio-readwrite
+echo 'CDM read-write user and policy set'
+
+# make spark events log access user
+mc admin user add minio minio-log-access minio123
+mc admin policy attach minio cdm-spark-job-logs-policy --user=minio-log-access
+echo 'Spark events log access user and policy set'