Merge Develop into main (#27)

* lock lab version to 3.6.3 and wait for extension updates

* allow homes other than /home/jovyan with gid=0

* git lfs. see #25

* add env support for other code-server userdata dirs

* change default config of python extension + user path changes.

* vscode update

* update base images

* fix prolog install

* add util image with extended cappabilities

* add opencv

base-images/vscode/Dockerfile

@@ -1,7 +1,7 @@
 ARG JUPYTER_VARIANT
 ARG CUDA_VARIANT
 
-FROM jupyter/${JUPYTER_VARIANT}-notebook:python-3.10 as nocuda
+FROM jupyter/${JUPYTER_VARIANT}-notebook:lab-3.6.3 as nocuda
 
 ARG JUPYTER_VARIANT
 ARG CUDA_VARIANT
@@ -202,18 +202,23 @@ RUN apt-get update && \
     htop \
     wget \
     curl \
+    jq \
+    moreutils \
     openssh-client \
     iputils-ping \
     && apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
 #  Code Server server-proxy/vscode-proxy from https://coder.com/docs/code-server/latest
-ENV CODE_VERSION=4.12.0
+ENV CODE_VERSION=4.13.0
 
 ENV CODE_BUILTIN_EXTENSIONS_DIR=/usr/lib/code-server/lib/vscode/extensions/
 ENV PATH=/opt/code-server/bin:$PATH \
     CS_DISABLE_GETTING_STARTED_OVERRIDE=1
 
+ENV CODE_EXTENSIONSDIR='/home/$NB_USER/.local/share/code-server-$BUILD_IMAGE_NAME/extensions/'
+ENV CODE_USERDATADIR='/home/$NB_USER/.local/share/code-server-$BUILD_IMAGE_NAME/'
+
 ENV SERVICE_URL=https://open-vsx.org/vscode/gallery 
 ENV ITEM_URL=https://open-vsx.org/vscode/item
 
@@ -223,7 +228,10 @@ RUN cd /tmp \
     && curl -sL https://upload.wikimedia.org/wikipedia/commons/9/9a/Visual_Studio_Code_1.35_icon.svg -o vscode.svg \
     && dpkg -i code-server_${CODE_VERSION}_amd64.deb \
     && rm -f code-server_${CODE_VERSION}_amd64.deb \
-    && code-server --extensions-dir ${CODE_BUILTIN_EXTENSIONS_DIR} --install-extension ms-python.python \
+    && rm -rf /tmp/* $HOME/.config $HOME/.local $HOME/.cache \
+    && fix-permissions $HOME && fix-permissions $CONDA_DIR && fix-permissions /home/$NB_USER
+
+RUN code-server --extensions-dir ${CODE_BUILTIN_EXTENSIONS_DIR} --install-extension ms-python.python \
     && code-server --extensions-dir ${CODE_BUILTIN_EXTENSIONS_DIR} --install-extension ms-toolsai.jupyter \
     && code-server --extensions-dir ${CODE_BUILTIN_EXTENSIONS_DIR} --install-extension ms-toolsai.vscode-jupyter-cell-tags \
     && code-server --extensions-dir ${CODE_BUILTIN_EXTENSIONS_DIR} --install-extension humao.rest-client \
@@ -235,26 +243,69 @@ RUN cd /tmp \
     && cd ${CODE_BUILTIN_EXTENSIONS_DIR}/ms-toolsai.jupyter-202* \
     && mkdir -m 1777 temp \
     && mkdir -m 1777 tmp \
+    # update defualt values 
+    && QUERY='.contributes.configuration.properties."python.condaPath".default="/opt/conda/bin/conda"' \
+    && jq $QUERY /usr/lib/code-server/lib/vscode/extensions/ms-python.python-*/package.json | sponge /usr/lib/code-server/lib/vscode/extensions/ms-python.python-*/package.json \
+    && QUERY='.contributes.configuration.properties."python.defaultInterpreterPath".default="/opt/conda/bin/python"' \
+    && jq $QUERY /usr/lib/code-server/lib/vscode/extensions/ms-python.python-*/package.json | sponge /usr/lib/code-server/lib/vscode/extensions/ms-python.python-*/package.json \
     && rm -rf /tmp/* $HOME/.config $HOME/.local $HOME/.cache \
-    && fix-permissions $HOME && fix-permissions $CONDA_DIR && fix-permissions /home/$NB_USER
-
-RUN pip install jupyter-server-proxy jupyter-vscode-proxy nbgitpuller \
+    && fix-permissions $HOME && fix-permissions $CONDA_DIR && fix-permissions /home/$NB_USER \
+    && cd /tmp 
+
+RUN pip install \
+    jupyterlab-lsp \
+    jupyter-server-proxy \
+    git+https://github.com/fhswf/Jupyterhub-K8s.git@main#subdirectory=packages/jupyter_vscodeserver_proxy  \
+    jedi-language-server \
+    nbgitpuller \
     && rm -rf /tmp/* $HOME/.cache
 
 # only install scilab-kernel for scipy notebooks
 RUN if [ "$JUPYTER_VARIANT" = "scipy" ]; \
     then pip install scilab-kernel && rm -rf /tmp/* $HOME/.cache; \
     fi 
 
-COPY setup_env.sh /usr/local/bin/start-notebook.d/  
-RUN  chmod +x /usr/local/bin/start-notebook.d/setup_env.sh
+# conda tectlive latex lsp
+# conda install --channel conda-forge tectonic texlab chktex
+# conda install -c conda-forge texlab chktex
+
+# make /home group writable in case we want to move /home/$NB_USER
+# fix for https://github.com/jupyter/docker-stacks/issues/1923
+RUN chmod g+rwx /home
+# copy start scripts
+COPY rebind_home.sh /usr/local/bin/before-notebook.d/001_rebind_home.sh
+RUN  chmod +x /usr/local/bin/before-notebook.d/001_rebind_home.sh
+COPY setup_env.sh /usr/local/bin/before-notebook.d/010_setup_env.sh  
+RUN  chmod +x /usr/local/bin/before-notebook.d/010_setup_env.sh
+
+# git LFS
+RUN apt-get update \
+    && curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash \
+    && apt-get install -y --no-install-recommends \
+    git-lfs \
+    && apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
+# user 
 USER $NB_UID
 
 RUN pip install bash_kernel \
     && python -m bash_kernel.install --prefix /opt/conda \
     && rm -rf /tmp/* $HOME/.cache
 
+## npm language servers:
+RUN npm install -g --save-dev \
+    bash-language-server \
+    pyright \
+    sql-language-server \
+    typescript-language-server \
+    unified-language-server \
+    vscode-css-languageserver-bin \
+    vscode-html-languageserver-bin \
+    yaml-language-server \
+    vscode-json-languageserver-bin \
+    && rm -rf /tmp/* $HOME/.cache
+
 RUN npm config set cache /opt/conda/npm-cache --global
 
 LABEL ki.fh-swf.de.jupyterhub.namelabel="vscode-${BUILD_JUPYTER_VARIANT}-${BUILD_CUDA_VARIANT}"

base-images/vscode/rebind_home.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -e
+
+# works for command: docker run --user 5000:100 --group-add root --group-add 1000 -e NB_USER="myuser" ...
+if [[ -w /etc/passwd ]] && [[ -w /home ]]; then
+    if [[ "${NB_USER}" != "jovyan" ]]; then
+         _log "creating enw home for user ${NB_USER}"
+        mkdir -p /home/${NB_USER};
+        head -n -1 /etc/passwd > /tmp/passwd;
+        echo "creating passwd entry ${NB_USER}:x:$(id -u):$(id -g):,,,:/home/${NB_USER}:/bin/bash";
+        echo "${NB_USER}:x:$(id -u):$(id -g):,,,:/home/${NB_USER}:/bin/bash" >> /tmp/passwd;
+        cat /tmp/passwd > /etc/passwd;
+        rm /tmp/passwd;
+        export HOME=/home/${NB_USER};
+        if [[ "${PWD}/" == "/home/jovyan/"* ]]; then
+            new_wd="/home/${NB_USER}/${PWD:13}"
+            _log "Changing working directory to ${new_wd}"
+            cd "${new_wd}"
+        fi
+    fi
+else
+    _log "WARNING: unable to fix missing /etc/passwd entry because we don't have write permission. Try setting gid=0 with \"--user=$(id -u):0\"."
+fi

images/deepml/Dockerfile

@@ -1,4 +1,7 @@
-FROM ghcr.io/fhswf/jupyterhub-k8s/vscode-scipy-cuda-devel:sha-d11cf9b
+ARG BASE_CONTAINER=ghcr.io/fhswf/jupyterhub-k8s/vscode-scipy-cuda-devel:sha-372d538
+FROM $BASE_CONTAINER
+ARG BASE_CONTAINER
+ENV BUILD_IMAGE_NAME deepml
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
@@ -30,7 +33,6 @@ RUN pip install multidict \
 # we have to upgrade cuda
 #ENV TRT_VERSION 8.4.3.1-1+cuda11.6
 #ENV LIBINFER_VERSION 8.4.3-1+cuda11.6
-
 #RUN apt-get update \
 #    && apt-get install -y --no-install-recommends \
 #    libnvinfer-plugin8=${LIBINFER_VERSION} \
@@ -40,19 +42,14 @@ RUN pip install multidict \
 #    libnvparsers-dev=${LIBINFER_VERSION} \
 #    libnvonnxparsers-dev=${LIBINFER_VERSION} \
 #    libcudnn8-dev=8.5.0.96-1+cuda11.7 \
-#    libnvinfer8=${LIBINFER_VERSION} \
+#    libnvinfer8=${LIBINFER_VERSION} \ 
 #    tensorrt-dev=${TRT_VERSION} \
 #    libnvinfer-dev=${LIBINFER_VERSION} \
 #    && apt-get clean && \
 #    rm -rf /var/lib/apt/lists/*
 
 USER $NB_UID
 
-## Install Tensorflow deps
-# this is not required for devel packages
-# RUN conda install -c conda-forge cudatoolkit=11.7.0
-# RUN conda install -c "nvidia/label/cuda-11.7.0" cuda-nvcc
-
 # Install PyTorch
 RUN pip install \
     torch==2.0.0+cu117 torchvision==0.15.1+cu117 torchaudio==2.0.1 --index-url https://download.pytorch.org/whl/cu117 \
@@ -63,19 +60,19 @@ RUN pip install \
     nvidia-cudnn-cu11==8.5.0.96 tensorflow==2.11.* mysql-connector-python keras-tuner pymysql \
     librosa gradio wandb Cython flair \ 
     transformers datasets metrics tokenizers evaluate adapter-transformers diffusers[torch] accelerate \
-    spacy nltk gensim jiwer audiofile \
+    spacy nltk gensim jiwer audiofile opencv-python\
     && rm -rf /tmp/* $HOME/.cache
 
 # optional stuff not included currently
 # nemo_toolkit[all] 
 
-
 # see here: https://github.com/tensorflow/tensorflow/issues/58681
 # bugfix for ${CUDA_DIR}/nvvm/libdevice.
 
 LABEL ki.fh-swf.de.jupyterhub.runtime="NVIDIA-GPU"
-LABEL ki.fh-swf.de.jupyterhub.namelabel="deepml-cuda"
+LABEL ki.fh-swf.de.jupyterhub.namelabel="$BUILD_IMAGE_NAME"
 LABEL ki.fh-swf.de.jupyterhub.description="Multipurpose Deep Learning image with cuda"
 LABEL org.opencontainers.image.description="Multipurpose Deep Learning image with cuda"
 LABEL org.opencontainers.image.authors="Fachhoschschule Südwestfalen"
-LABEL org.opencontainers.image.licenses="MIT"
+LABEL org.opencontainers.image.licenses="MIT"
+LABEL org.opencontainers.image.base.name=$BASE_CONTAINER

images/fulp/Dockerfile

@@ -1,7 +1,7 @@
-ARG BASE_CONTAINER=ghcr.io/fhswf/jupyterhub-k8s/vscode-minimal-nocuda:sha-d11cf9b
-
+ARG BASE_CONTAINER=ghcr.io/fhswf/jupyterhub-k8s/vscode-minimal-nocuda:sha-372d538  
 FROM $BASE_CONTAINER
-
+ARG BASE_CONTAINER
+ENV BUILD_IMAGE_NAME fulp
 # Extra arguments to `stack build`. Used to build --fast, see Makefile.
 ARG STACK_ARGS=
 
@@ -218,9 +218,6 @@ RUN mkdir -p /tmp/hls && cd /tmp/hls \
     && fix-permissions /opt/bin  \
     && cd /tmp && rm -rf /tmp/hls
 
-# vscode haskell extension
-# RUN code-server --extensions-dir ${CODE_BUILTIN_EXTENSIONS_DIR} --install-extension haskell.haskell && fix-permissions "/home/${NB_USER}"
-
 ############
 # swi prolog & jupyterextension
 ############
@@ -256,9 +253,12 @@ RUN set -eux \
           .. \
     && make \
     && make install \
-    && rm -rf /tmp/src \
-    # && apt-get purge -y --auto-remove $BUILD_DEPS \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /tmp/src 
+    # && apt-get purge -y --auto-remove $BUILD_DEPS 
+
+RUN set -eux \
+    && swipl -g "pack_install(log4p,[interactive(false)]) " -t halt \
+    && swipl -g "pack_install(prolog_lsp,[interactive(false)]) "  -t halt 
 
 RUN pip install prolog_kernel jupyterlab_prolog_codemirror_extension \
     && python -m prolog_kernel.install --prefix /usr/local \
@@ -267,7 +267,19 @@ RUN pip install prolog_kernel jupyterlab_prolog_codemirror_extension \
 # check kernels
 RUN jupyter kernelspec list
 
+# vscode extensions
+RUN code-server --extensions-dir /usr/lib/code-server/lib/vscode/extensions/ --install-extension haskell.haskell \
+    && code-server --extensions-dir /usr/lib/code-server/lib/vscode/extensions/ --install-extension hargettp.pure-prolog-language-support \
+    && rm -rf /tmp/* $HOME/.config $HOME/.local $HOME/.cache \
+    && fix-permissions $HOME && fix-permissions $CONDA_DIR && fix-permissions /home/$NB_USER
+
 # Switch back to jovyan user
 USER $NB_UID
 
-
+LABEL ki.fh-swf.de.jupyterhub.runtime="CPU"
+LABEL ki.fh-swf.de.jupyterhub.namelabel="$BUILD_IMAGE_NAME"
+LABEL ki.fh-swf.de.jupyterhub.description="Functional and logical programming"
+LABEL org.opencontainers.image.description="Functional and logical programming"
+LABEL org.opencontainers.image.authors="Fachhoschschule Südwestfalen"
+LABEL org.opencontainers.image.licenses="MIT"
+LABEL org.opencontainers.image.base.name=$BASE_CONTAINER

images/java/Dockerfile

@@ -1,7 +1,10 @@
-ARG BASE_CONTAINER=ghcr.io/fhswf/jupyterhub-k8s/vscode-minimal-nocuda:sha-d39be35 
+ARG BASE_CONTAINER=ghcr.io/fhswf/jupyterhub-k8s/vscode-minimal-nocuda:sha-372d538  
 
 FROM $BASE_CONTAINER
 
+ARG BASE_CONTAINER
+ENV BUILD_IMAGE_NAME java
+
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 USER root
@@ -26,11 +29,16 @@ RUN apt-get update \
     # openjdk-17-jdk \
     && rm -rf /var/lib/apt/lists/*
 
+RUN code-server --extensions-dir /usr/lib/code-server/lib/vscode/extensions/ --install-extension vscjava.vscode-java-pack \
+    && rm -rf /tmp/* $HOME/.config $HOME/.local $HOME/.cache \
+    && fix-permissions $HOME && fix-permissions $CONDA_DIR && fix-permissions /home/$NB_USER
+
 USER $NB_UID
 
 LABEL ki.fh-swf.de.jupyterhub.runtime="CPU"
-LABEL ki.fh-swf.de.jupyterhub.namelabel="java"
+LABEL ki.fh-swf.de.jupyterhub.namelabel="$BUILD_IMAGE_NAME"
 LABEL ki.fh-swf.de.jupyterhub.description="Java"
-LABEL org.opencontainers.image.description="Java" 
+LABEL org.opencontainers.image.description="Java JDK20" 
 LABEL org.opencontainers.image.authors="Fachhoschschule Südwestfalen"
-LABEL org.opencontainers.image.licenses="MIT"
+LABEL org.opencontainers.image.licenses="MIT"
+LABEL org.opencontainers.image.base.name=$BASE_CONTAINER

images/multitool/Dockerfile

@@ -0,0 +1,8 @@
+FROM wbitt/network-multitool:e21fa4b
+
+USER root
+
+RUN setcap cap_chown,cap_fowner,cap_dac_read_search,cap_dac_override,cap_fsetid,cap_setuid,cap_setgid+ep /bin/chown
+RUN setcap cap_chown,cap_fowner,cap_dac_read_search,cap_dac_override,cap_fsetid,cap_setuid,cap_setgid+ep /bin/chmod
+
+# docker run --user 1000:100 --cap-add=CHOWN --cap-add=FOWNER --cap-add=cap_dac_override --cap-add=cap_dac_read_search --rm -it mymultitool:latest sh

images/multitool/README.md

@@ -0,0 +1,3 @@
+# Multitool
+A container with added capabilities for some selected binaries to be used in init container.
+Extented from wbitt/network-multitool former pragma/network-multitool