Merge branch 'main' into feature/DM-65

dandi · Jan 17, 2025 · 7e4da46 · 7e4da46
2 parents f09597a + 090dd85
commit 7e4da46
Show file tree

Hide file tree

Showing 12 changed files with 241 additions and 80 deletions.
diff --git a/.github/workflows/docker-push.yaml b/.github/workflows/docker-push.yaml
@@ -5,7 +5,7 @@ on:
     branches:
       - main
     paths:
-      - 'Dockerfile*'
+      - 'images/Dockerfile*'
   workflow_dispatch:
 
 jobs:
@@ -14,15 +14,16 @@ jobs:
 
     strategy:
       matrix:
-        image:
-          - name: base
+        include:
+          - tag: latest
             dockerfile: Dockerfile
-          - name: matlab
-            dockerfile: Dockerfile.matlab
-          - name: gpu
+          - tag: latest-allensdk
+            dockerfile: Dockerfile.allensdk
+          - tag: latest-gpu
             dockerfile: Dockerfile.gpu
-          - name: gpu-matlab
-            dockerfile: Dockerfile.gpu.matlab
+          - tag: latest-gpu-allensdk
+            dockerfile: Dockerfile.gpu.allensdk
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -42,10 +43,10 @@ jobs:
       - name: Build and push Docker Image
         uses: docker/build-push-action@v6
         with:
-          context: images
-          file: ${{ matrix.dockerfile }}
+          context: ./images
+          file: ./images/${{ matrix.dockerfile }}
           push: true
           tags: |
-            ${{ secrets.DOCKERHUB_DANDIHUB_USERNAME }}/dandihub:latest-${{ matrix.image.name }}
-          cache-from: type=registry,ref=${{ secrets.DOCKERHUB_DANDIHUB_USERNAME }}/dandihub:latest-${{ matrix.image.name }}
+            ${{ secrets.DOCKERHUB_DANDIHUB_USERNAME }}/dandihub:${{ matrix.tag }}
+          cache-from: type=registry,ref=${{ secrets.DOCKERHUB_DANDIHUB_USERNAME }}/dandihub:${{ matrix.tag }}
           cache-to: type=inline
diff --git a/.github/workflows/docker-test.yaml b/.github/workflows/docker-test.yaml
@@ -3,7 +3,7 @@ name: Build Docker Images
 on:
   pull_request:
     paths:
-      - 'Dockerfile*'
+      - 'images/Dockerfile*'
   workflow_dispatch:
 
 jobs:
@@ -12,15 +12,16 @@ jobs:
 
     strategy:
       matrix:
-        image:
-          - name: base
+        include:
+          - tag: latest
             dockerfile: Dockerfile
-          - name: matlab
-            dockerfile: Dockerfile.matlab
-          - name: gpu
+          - tag: latest-allensdk
+            dockerfile: Dockerfile.allensdk
+          - tag: latest-gpu
             dockerfile: Dockerfile.gpu
-          - name: gpu-matlab
-            dockerfile: Dockerfile.gpu.matlab
+          - tag: latest-gpu-allensdk
+            dockerfile: Dockerfile.gpu.allensdk
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -34,8 +35,8 @@ jobs:
       - name: Build Docker Image
         uses: docker/build-push-action@v6
         with:
-          context: images
-          file: ${{ matrix.dockerfile }}
+          context: ./images
+          file: ./images/${{ matrix.dockerfile }}
           push: false
-          cache-from: type=registry,ref=${{ secrets.DOCKERHUB_DANDIHUB_USERNAME }}/dandihub:latest-${{ matrix.image.name }}
+          cache-from: type=registry,ref=${{ secrets.DOCKERHUB_DANDIHUB_USERNAME }}/dandihub:${{ matrix.tag }}
           cache-to: type=inline
diff --git a/addons.tf b/addons.tf
@@ -108,8 +108,11 @@ module "eks_blueprints_addons" {
     repository_username = data.aws_ecrpublic_authorization_token.token.user_name
     repository_password = data.aws_ecrpublic_authorization_token.token.password
     values = [<<EOT
-        controller:
-          image: public.ecr.aws/karpenter/controller:0.35.0@sha256:48d1246f6b2066404e300cbf3e26d0bcdc57a76531dcb634d571f4f0e050cb57
+      controller:
+        image:
+          repository: public.ecr.aws/karpenter/controller
+          tag: "0.35.0"
+          digest: "sha256:48d1246f6b2066404e300cbf3e26d0bcdc57a76531dcb634d571f4f0e050cb57"
     EOT
     ]
   }

diff --git a/envs/shared/jupyterhub.yaml b/envs/shared/jupyterhub.yaml
@@ -50,7 +50,12 @@ singleuser:
               display_name: "Standard"
               default: true
               kubespawner_override:
-                image: "${singleuser_image_repo}:${singleuser_image_tag}-base"
+                image: "${singleuser_image_repo}:${singleuser_image_tag}"
+            allensdk:
+              display_name: "AllenSDK"
+              default: true
+              kubespawner_override:
+                image: "${singleuser_image_repo}:${singleuser_image_tag}-allensdk"
       kubespawner_override:
         image_pull_policy: Always
         cpu_limit: 2
@@ -69,7 +74,12 @@ singleuser:
     #           display_name: "Standard"
     #           default: true
     #           kubespawner_override:
-    #             image: "${singleuser_image_repo}:${singleuser_image_tag}-base"
+    #             image: "${singleuser_image_repo}:${singleuser_image_tag}"
+    #         allensdk:
+    #           display_name: "AllenSDK"
+    #           default: true
+    #           kubespawner_override:
+    #             image: "${singleuser_image_repo}:${singleuser_image_tag}-allensdk"
     #         matlab:
     #           display_name: "MATLAB (must provide your own license)"
     #           kubespawner_override:
@@ -93,7 +103,12 @@ singleuser:
               display_name: "Standard"
               default: true
               kubespawner_override:
-                image: "${singleuser_image_repo}:${singleuser_image_tag}-base"
+                image: "${singleuser_image_repo}:${singleuser_image_tag}"
+            allensdk:
+              display_name: "AllenSDK"
+              default: true
+              kubespawner_override:
+                image: "${singleuser_image_repo}:${singleuser_image_tag}-allensdk"
             matlab:
               display_name: "MATLAB (must provide your own license)"
               kubespawner_override:
@@ -118,7 +133,12 @@ singleuser:
               display_name: "Standard"
               default: true
               kubespawner_override:
-                image: "${singleuser_image_repo}:${singleuser_image_tag}-base"
+                image: "${singleuser_image_repo}:${singleuser_image_tag}"
+            allensdk:
+              display_name: "AllenSDK"
+              default: true
+              kubespawner_override:
+                image: "${singleuser_image_repo}:${singleuser_image_tag}-allensdk"
             matlab:
               display_name: "MATLAB (must provide your own license)"
               kubespawner_override:
@@ -141,7 +161,12 @@ singleuser:
               display_name: "Standard"
               default: true
               kubespawner_override:
-                image: "${singleuser_image_repo}:${singleuser_image_tag}-base"
+                image: "${singleuser_image_repo}:${singleuser_image_tag}"
+            allensdk:
+              display_name: "AllenSDK"
+              default: true
+              kubespawner_override:
+                image: "${singleuser_image_repo}:${singleuser_image_tag}-allensdk"
             matlab:
               display_name: "MATLAB (must provide your own license)"
               kubespawner_override:
@@ -165,6 +190,11 @@ singleuser:
               default: true
               kubespawner_override:
                 image: "${singleuser_image_repo}:${singleuser_image_tag}-gpu"
+            allensdk:
+              display_name: "AllenSDK GPU"
+              default: true
+              kubespawner_override:
+                image: "${singleuser_image_repo}:${singleuser_image_tag}-gpu-allensdk"
             matlab:
               display_name: "MATLAB GPU (must provide your own license)"
               kubespawner_override:

diff --git a/images/Dockerfile b/images/Dockerfile
@@ -1,4 +1,4 @@
-FROM quay.io/jupyter/datascience-notebook:2024-02-06
+FROM quay.io/jupyter/datascience-notebook:python-3.11.10
 
 USER root
 
@@ -44,22 +44,16 @@ USER $NB_USER
 
 RUN pip install --no-cache-dir jupyter-remote-desktop-proxy
 
-# Install Allen SDK
-RUN mamba create --yes -n allen -c conda-forge python=3.8 pip ipykernel 'h5py>=3.4=mpi*' \
-  && /opt/conda/envs/allen/bin/pip install --no-cache-dir allensdk \
-  && conda clean --all -f -y && rm -rf /tmp/*
+ENV MAMBA_NO_LOW_SPEED_LIMIT=1
 
 RUN mamba install --yes 'datalad>=0.18' rclone 'h5py>3.3=mpi*' ipykernel zarr blosc eccodes websockify \
   && wget --quiet https://raw.githubusercontent.com/DanielDent/git-annex-remote-rclone/v0.8/git-annex-remote-rclone \
   && chmod +x git-annex-remote-rclone && mv git-annex-remote-rclone /opt/conda/bin \
   && conda clean --all -f -y && rm -rf /tmp/*
 
-RUN /opt/conda/envs/allen/bin/python -m ipykernel install --user --name allen \
-    --display-name="Allen SDK"
-
 RUN pip install --no-cache-dir -U plotly jupyter_bokeh jupytext nbgitpuller datalad_container \
     datalad-osf dandi nibabel nilearn pybids spikeinterface neo itkwidgets \
-    'pydra>=0.17' 'pynwb>=2.3.1' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]"  \
+    'pydra>=0.25' 'pynwb>=2.8.3' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]"  \
     aicsimageio kerchunk 'neuroglancer>=2.28' cloud-volume ipywidgets ome-zarr \
     webio_jupyter_extension https://github.com/balbasty/dandi-io/archive/refs/heads/main.zip \
     tensorstore anndata

diff --git a/images/Dockerfile.allensdk b/images/Dockerfile.allensdk
@@ -0,0 +1,70 @@
+FROM quay.io/jupyter/datascience-notebook:python-3.11.10
+
+USER root
+
+RUN apt update \
+ && apt install -y software-properties-common \
+ && add-apt-repository -y 'ppa:apptainer/ppa' \
+ && apt update \
+ && apt install -y apptainer-suid \
+ && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/*
+
+RUN apt-get update && apt-get install -y ca-certificates libseccomp2 \
+   s3fs netbase less parallel tmux screen vim emacs htop curl \
+   && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/*
+
+RUN curl --silent --show-error "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" \
+  -o "awscliv2.zip" && unzip awscliv2.zip \
+  && ./aws/install && rm -rf ./aws awscliv2.zip
+
+# Install jupyter server proxy and desktop
+RUN curl -fsSLo /usr/share/keyrings/brave-browser-archive-keyring.gpg https://brave-browser-apt-release.s3.brave.com/brave-browser-archive-keyring.gpg \
+   && echo "deb [signed-by=/usr/share/keyrings/brave-browser-archive-keyring.gpg] https://brave-browser-apt-release.s3.brave.com/ stable main"|sudo tee /etc/apt/sources.list.d/brave-browser-release.list \
+   && apt-get -y -qq update \
+   && apt-get -y -qq install \
+        dbus-x11 \
+        xfce4 \
+        xfce4-panel \
+        xfce4-session \
+        xfce4-settings \
+        xorg \
+        xubuntu-icon-theme \
+        brave-browser \
+        tigervnc-standalone-server \
+        tigervnc-xorg-extension \
+    # chown $HOME to workaround that the xorg installation creates a
+    # /home/jovyan/.cache directory owned by root
+   && chown -R $NB_UID:$NB_GID $HOME \
+   && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/*
+
+# apt-get may result in root-owned directories/files under $HOME
+RUN mkdir /opt/extras && chown -R $NB_UID:$NB_GID $HOME /opt/extras
+
+USER $NB_USER
+
+RUN pip install --no-cache-dir jupyter-remote-desktop-proxy
+
+ENV MAMBA_NO_LOW_SPEED_LIMIT=1
+
+# Install Allen SDK
+RUN mamba create --yes -n allen -c conda-forge python=3.8 pip ipykernel 'h5py>=3.4=mpi*' \
+  && /opt/conda/envs/allen/bin/pip install --no-cache-dir allensdk \
+  && conda clean --all -f -y && rm -rf /tmp/*
+
+RUN mamba install --yes 'datalad>=0.18' rclone 'h5py>3.3=mpi*' ipykernel zarr blosc eccodes websockify \
+  && wget --quiet https://raw.githubusercontent.com/DanielDent/git-annex-remote-rclone/v0.8/git-annex-remote-rclone \
+  && chmod +x git-annex-remote-rclone && mv git-annex-remote-rclone /opt/conda/bin \
+  && conda clean --all -f -y && rm -rf /tmp/*
+
+RUN /opt/conda/envs/allen/bin/python -m ipykernel install --user --name allen \
+    --display-name="Allen SDK"
+
+RUN pip install --no-cache-dir -U plotly jupyter_bokeh jupytext nbgitpuller datalad_container \
+    datalad-osf dandi pybids spikeinterface neo itkwidgets \
+    'pydra>=0.25' 'pynwb>=2.8.3' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]"  \
+    aicsimageio kerchunk cloud-volume ipywidgets ome-zarr \
+    webio_jupyter_extension https://github.com/balbasty/dandi-io/archive/refs/heads/main.zip \
+    tensorstore anndata
+
+RUN pip install --no-cache-dir multiscale_spatial_image jupyterlab-git \
+        datalad-fuse
diff --git a/images/Dockerfile.gpu b/images/Dockerfile.gpu
@@ -1,4 +1,4 @@
-FROM quay.io/jupyter/datascience-notebook:2024-02-06
+FROM quay.io/jupyter/datascience-notebook:python-3.11.10
 
 USER root
 
@@ -44,27 +44,21 @@ USER $NB_USER
 
 RUN pip install --no-cache-dir jupyter-remote-desktop-proxy
 
-# Install Allen SDK
-RUN mamba create --yes -n allen -c conda-forge python=3.11 pip ipykernel 'h5py>=3.4=mpi*' \
-  && /opt/conda/envs/allen/bin/pip install --no-cache-dir allensdk \
-  && conda clean --all -f -y && rm -rf /tmp/*
+ENV MAMBA_NO_LOW_SPEED_LIMIT=1
 
 RUN mamba install --yes datalad rclone 'h5py>3.3=mpi*' ipykernel zarr blosc eccodes websockify \
   && wget --quiet https://raw.githubusercontent.com/DanielDent/git-annex-remote-rclone/v0.8/git-annex-remote-rclone \
   && chmod +x git-annex-remote-rclone && mv git-annex-remote-rclone /opt/conda/bin \
   && conda clean --all -f -y && rm -rf /tmp/*
 
-RUN /opt/conda/envs/allen/bin/python -m ipykernel install --user --name allen \
-    --display-name="Allen SDK"
-
 RUN pip install --no-cache-dir -U plotly jupyter_bokeh jupytext nbgitpuller datalad_container \
     datalad-osf dandi nibabel nilearn pybids spikeinterface neo itkwidgets \
-    'pydra>=0.17' 'pynwb>=2.3.1' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]"  \
+    'pydra>=0.25' 'pynwb>=2.8.3' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]"  \
     aicsimageio kerchunk 'neuroglancer>=2.28' cloud-volume ipywidgets ome-zarr \
     webio_jupyter_extension https://github.com/balbasty/dandi-io/archive/refs/heads/main.zip \
     tensorstore anndata && rm -rf /tmp/*
 
-RUN mamba install --yes -c "nvidia/label/cuda-11.8.0" cuda-toolkit cudnn \
+RUN mamba install --yes -c "nvidia/label/cuda-12.6.3" cuda-toolkit cudnn \
   && conda clean --all -f -y && rm -rf /tmp/*
 
 RUN pip install --no-cache-dir multiscale_spatial_image jupyterlab-git \