MET-6552 Add first version of s3fs for openshift deployment (WIP)

Author: Ruufuus

docker/build-docker-images.sh

@@ -3,18 +3,18 @@ set -e
 echo building flink-poc docker image that can be deployed on the openshift cluster.
 
 #images
-echo "building :: flink java 21 base $(realpath flink-java21)"
-docker build --no-cache -t flink:2.0.0-java21_poc flink-java21
+#echo "building :: flink java 21 base $(realpath flink-java21)"
+#docker build --no-cache -t flink:2.0.0-java21_poc flink-java21
 
-echo "building :: flink-node $(realpath flink-with-tools)"
-docker build --no-cache -t flink-with-tools:2.0.0-java21 flink-with-tools
+#echo "building :: flink-node $(realpath flink-with-tools)"
+#docker build --no-cache -t flink-with-tools:2.0.0-java21 flink-with-tools
 
 echo "building :: shared libs: $(realpath metis-processing-engine-flink/shared-libs)"
 mvn -f metis-processing-engine-flink/shared-libs clean install
 echo "building :: flink-node $(realpath metis-processing-engine-flink)"
 docker build --no-cache -t metis-processing-engine-flink:2.0.0-java21 metis-processing-engine-flink
 
 #echo "Pushing to the repository: registry.paas.psnc.pl"
-#docker tag metis-processing-engine-flink:2.0.0-java21 registry.paas.psnc.pl/ecloud-poc/metis-processing-engine-flink:2.0.0-java21
-#docker push registry.paas.psnc.pl/ecloud-poc/metis-processing-engine-flink:2.0.0-java21
+docker tag metis-processing-engine-flink:2.0.0-java21 registry.paas.psnc.pl/ecloud-poc/metis-processing-engine-flink:2.0.0-java21-s3
+docker push registry.paas.psnc.pl/ecloud-poc/metis-processing-engine-flink:2.0.0-java21-s3
 

docker/flink-with-tools/Dockerfile

@@ -21,6 +21,8 @@ RUN apt-get update && apt-get install -y \
     net-tools \
     dnsutils \
     vim \
+    s3fs \
+    fuse \
     && rm -rf /var/lib/apt/lists/*
 
 COPY install.sh /

docker/metis-processing-engine-flink/Dockerfile

@@ -2,3 +2,12 @@ FROM flink-with-tools:2.0.0-java21
 
 LABEL maintainer="PSNC"
 COPY shared-libs/target/dependency/* /opt/flink/lib/
+
+RUN mkdir -p /mnt/s3
+RUN chmod 777 /mnt/s3
+
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/metis-processing-engine-flink/entrypoint.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -e
+if [ -z "${S3_ACCESS_KEY}" ] || [ -z "${S3_SECRET_KEY}" ] || [ -z "${S3_BUCKET}" ] || [ -z "${S3_ENDPOINT}" ] || [ -z "${S3_REGION}" ]; then
+  echo "Some S3 variables not found in environment"
+  if  [ -z "${S3_ACCESS_KEY}" ]; then
+    echo "S3_ACCESS_KEY not found in environment"
+  fi
+
+  if  [ -z "${S3_SECRET_KEY}" ]; then
+    echo "S3_SECRET_KEY not found in environment"
+  fi
+
+  if  [ -z "${S3_BUCKET}" ]; then
+    echo "S3_BUCKET not found in environment"
+  fi
+
+  if  [ -z "${S3_ENDPOINT}" ]; then
+    echo "S3_ENDPOINT not found in environment"
+  fi
+
+  if  [ -z "${S3_REGION}" ]; then
+    echo "S3_REGION not found in environment"
+  fi
+
+  echo "Proceeding without mounting s3 to filesystem"
+else
+  # Save credentials in a file for s3fs
+  echo "${S3_ACCESS_KEY}:${S3_SECRET_KEY}" > /tmp/passwd-s3fs
+  chmod 600 /tmp/passwd-s3fs
+
+  # Mount the S3 bucket using s3fs
+  s3fs "${S3_BUCKET}" /mnt/s3 \
+    -o passwd_file=/tmp/passwd-s3fs \
+    -o url="${S3_ENDPOINT}" \
+    -o endpoint="${S3_REGION}" \
+    -o use_path_request_style \
+    -o allow_other \
+    -o nonempty
+
+  mkdir -p /mnt/s3/http-jobs
+  mkdir -p /http-jobs
+
+  ln -s /mnt/s3/http-jobs /http-jobs
+
+fi
+
+exec "$@"