From 868dac9c5b080896eef4f821033c6787331a5741 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Thu, 12 Dec 2024 15:34:09 -0600 Subject: [PATCH 01/30] Simplify docker image builds --- .github/workflows/docker-test.yaml | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docker-test.yaml b/.github/workflows/docker-test.yaml index 89bdf7b..94ee74a 100644 --- a/.github/workflows/docker-test.yaml +++ b/.github/workflows/docker-test.yaml @@ -12,16 +12,12 @@ jobs: strategy: matrix: - image: - - name: base - dockerfile: Dockerfile - - name: matlab - dockerfile: Dockerfile.matlab - - name: gpu - dockerfile: Dockerfile.gpu - - name: gpu-matlab - dockerfile: Dockerfile.gpu.matlab - steps: + dockerfile: + - { file: Dockerfile, tag: latest } + - { file: Dockerfile.matlab, tag: latest-matlab } + - { file: Dockerfile.gpu, tag: latest-gpu } + - { file: Dockerfile.gpu.matlab, tag: latest-gpu-matlab } + - name: Checkout repository uses: actions/checkout@v4 @@ -34,8 +30,8 @@ jobs: - name: Build Docker Image uses: docker/build-push-action@v6 with: - context: images - file: ${{ matrix.dockerfile }} + context: ./images + file: ./images/${{ matrix.dockerfile.file }} push: false - cache-from: type=registry,ref=${{ secrets.DOCKERHUB_DANDIHUB_USERNAME }}/dandihub:latest-${{ matrix.image.name }} + cache-from: type=registry,ref=${{ secrets.DOCKERHUB_DANDIHUB_USERNAME }}/dandihub:${{ matrix.dockerfile.tag }} cache-to: type=inline \ No newline at end of file From 98b7102fc8f4a1868bca314c9dec13b5d960c711 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Thu, 12 Dec 2024 15:37:47 -0600 Subject: [PATCH 02/30] Temporarily remove filter --- .github/workflows/docker-test.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/docker-test.yaml b/.github/workflows/docker-test.yaml index 94ee74a..cf4cd8e 100644 --- a/.github/workflows/docker-test.yaml +++ b/.github/workflows/docker-test.yaml @@ -2,8 +2,6 @@ name: Build Docker Images on: pull_request: - paths: - - 'Dockerfile*' workflow_dispatch: jobs: From 5029061b438b776a5163c8960502dc9669df79ac Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Thu, 12 Dec 2024 15:38:29 -0600 Subject: [PATCH 03/30] Revert change --- .github/workflows/docker-test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-test.yaml b/.github/workflows/docker-test.yaml index cf4cd8e..94ee74a 100644 --- a/.github/workflows/docker-test.yaml +++ b/.github/workflows/docker-test.yaml @@ -2,6 +2,8 @@ name: Build Docker Images on: pull_request: + paths: + - 'Dockerfile*' workflow_dispatch: jobs: From 856e5ad1b95d9af93af08f2ef80fafb31d737eba Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Thu, 12 Dec 2024 15:41:43 -0600 Subject: [PATCH 04/30] Update matrix --- .github/workflows/docker-test.yaml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-test.yaml b/.github/workflows/docker-test.yaml index 94ee74a..f830325 100644 --- a/.github/workflows/docker-test.yaml +++ b/.github/workflows/docker-test.yaml @@ -12,11 +12,16 @@ jobs: strategy: matrix: - dockerfile: - - { file: Dockerfile, tag: latest } - - { file: Dockerfile.matlab, tag: latest-matlab } - - { file: Dockerfile.gpu, tag: latest-gpu } - - { file: Dockerfile.gpu.matlab, tag: latest-gpu-matlab } + file: + - Dockerfile + - Dockerfile.matlab + - Dockerfile.gpu + - Dockerfile.gpu.matlab + tag: + - latest + - latest-matlab + - latest-gpu + - latest-gpu-matlab - name: Checkout repository uses: actions/checkout@v4 @@ -31,7 +36,7 @@ jobs: uses: docker/build-push-action@v6 with: context: ./images - file: ./images/${{ matrix.dockerfile.file }} + file: ./images/${{ matrix.file }} push: false - cache-from: type=registry,ref=${{ secrets.DOCKERHUB_DANDIHUB_USERNAME }}/dandihub:${{ matrix.dockerfile.tag }} + cache-from: type=registry,ref=${{ secrets.DOCKERHUB_DANDIHUB_USERNAME }}/dandihub:${{ matrix.tag }} cache-to: type=inline \ No newline at end of file From c21b501635aca9995d9e6a0879f13f31e3fc51be Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Thu, 12 Dec 2024 15:51:19 -0600 Subject: [PATCH 05/30] Update matrix --- .github/workflows/docker-test.yaml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker-test.yaml b/.github/workflows/docker-test.yaml index f830325..38ae3dc 100644 --- a/.github/workflows/docker-test.yaml +++ b/.github/workflows/docker-test.yaml @@ -12,16 +12,15 @@ jobs: strategy: matrix: - file: - - Dockerfile - - Dockerfile.matlab - - Dockerfile.gpu - - Dockerfile.gpu.matlab - tag: - - latest - - latest-matlab - - latest-gpu - - latest-gpu-matlab + include: + - tag: latest + dockerfile: Dockerfile + - tag: latest-matlab + dockerfile: Dockerfile.matlab + - tag: latest-gpu + dockerfile: Dockerfile.gpu + - tag: latest-gpu-matlab + dockerfile: Dockerfile.gpu.matlab - name: Checkout repository uses: actions/checkout@v4 @@ -36,7 +35,7 @@ jobs: uses: docker/build-push-action@v6 with: context: ./images - file: ./images/${{ matrix.file }} + file: ./images/${{ matrix.dockerfile }} push: false cache-from: type=registry,ref=${{ secrets.DOCKERHUB_DANDIHUB_USERNAME }}/dandihub:${{ matrix.tag }} cache-to: type=inline \ No newline at end of file From cd8c550c682fba865e89d1517e7a36aa80ddbbc5 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Thu, 12 Dec 2024 15:52:19 -0600 Subject: [PATCH 06/30] Fix syntax --- .github/workflows/docker-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-test.yaml b/.github/workflows/docker-test.yaml index 38ae3dc..8f1520e 100644 --- a/.github/workflows/docker-test.yaml +++ b/.github/workflows/docker-test.yaml @@ -22,6 +22,7 @@ jobs: - tag: latest-gpu-matlab dockerfile: Dockerfile.gpu.matlab + steps: - name: Checkout repository uses: actions/checkout@v4 From 1f59b2b2f030ff25a5e67b01823fccf7ab0dd426 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Thu, 12 Dec 2024 15:56:18 -0600 Subject: [PATCH 07/30] Comment out matlab image builds --- .github/workflows/docker-test.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-test.yaml b/.github/workflows/docker-test.yaml index 8f1520e..1356b0f 100644 --- a/.github/workflows/docker-test.yaml +++ b/.github/workflows/docker-test.yaml @@ -15,12 +15,12 @@ jobs: include: - tag: latest dockerfile: Dockerfile - - tag: latest-matlab - dockerfile: Dockerfile.matlab + # - tag: latest-matlab + # dockerfile: Dockerfile.matlab - tag: latest-gpu dockerfile: Dockerfile.gpu - - tag: latest-gpu-matlab - dockerfile: Dockerfile.gpu.matlab + # - tag: latest-gpu-matlab + # dockerfile: Dockerfile.gpu.matlab steps: - name: Checkout repository From 35ea4978c16c573094c54c92f045849cc2c1fac4 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Thu, 12 Dec 2024 16:06:34 -0600 Subject: [PATCH 08/30] Fix image builds --- .github/workflows/docker-push.yaml | 19 ++++++++++--------- .github/workflows/docker-test.yaml | 8 ++++---- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docker-push.yaml b/.github/workflows/docker-push.yaml index b69a572..40ed002 100644 --- a/.github/workflows/docker-push.yaml +++ b/.github/workflows/docker-push.yaml @@ -14,15 +14,16 @@ jobs: strategy: matrix: - image: - - name: base + include: + - tag: latest dockerfile: Dockerfile - - name: matlab + - tag: latest-matlab dockerfile: Dockerfile.matlab - - name: gpu + - tag: latest-gpu dockerfile: Dockerfile.gpu - - name: gpu-matlab + - tag: latest-gpu-matlab dockerfile: Dockerfile.gpu.matlab + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -42,10 +43,10 @@ jobs: - name: Build and push Docker Image uses: docker/build-push-action@v6 with: - context: images - file: ${{ matrix.dockerfile }} + context: ./images + file: ./images/${{ matrix.dockerfile }} push: true tags: | - ${{ secrets.DOCKERHUB_DANDIHUB_USERNAME }}/dandihub:latest-${{ matrix.image.name }} - cache-from: type=registry,ref=${{ secrets.DOCKERHUB_DANDIHUB_USERNAME }}/dandihub:latest-${{ matrix.image.name }} + ${{ secrets.DOCKERHUB_DANDIHUB_USERNAME }}/dandihub:${{ matrix.tag }} + cache-from: type=registry,ref=${{ secrets.DOCKERHUB_DANDIHUB_USERNAME }}/dandihub:${{ matrix.tag }} cache-to: type=inline \ No newline at end of file diff --git a/.github/workflows/docker-test.yaml b/.github/workflows/docker-test.yaml index 1356b0f..8f1520e 100644 --- a/.github/workflows/docker-test.yaml +++ b/.github/workflows/docker-test.yaml @@ -15,12 +15,12 @@ jobs: include: - tag: latest dockerfile: Dockerfile - # - tag: latest-matlab - # dockerfile: Dockerfile.matlab + - tag: latest-matlab + dockerfile: Dockerfile.matlab - tag: latest-gpu dockerfile: Dockerfile.gpu - # - tag: latest-gpu-matlab - # dockerfile: Dockerfile.gpu.matlab + - tag: latest-gpu-matlab + dockerfile: Dockerfile.gpu.matlab steps: - name: Checkout repository From 344664d6d21b78e9acf7a8132d2bf906f04e087b Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Thu, 12 Dec 2024 16:11:05 -0600 Subject: [PATCH 09/30] Temporarily comment out matlab --- .github/workflows/docker-push.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-push.yaml b/.github/workflows/docker-push.yaml index 40ed002..bf9f553 100644 --- a/.github/workflows/docker-push.yaml +++ b/.github/workflows/docker-push.yaml @@ -17,12 +17,12 @@ jobs: include: - tag: latest dockerfile: Dockerfile - - tag: latest-matlab - dockerfile: Dockerfile.matlab + # - tag: latest-matlab + # dockerfile: Dockerfile.matlab - tag: latest-gpu dockerfile: Dockerfile.gpu - - tag: latest-gpu-matlab - dockerfile: Dockerfile.gpu.matlab + # - tag: latest-gpu-matlab + # dockerfile: Dockerfile.gpu.matlab steps: - name: Checkout repository From edd808772dbbb2e5e8ee8e7f48f5726f490d26e4 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Thu, 12 Dec 2024 16:25:06 -0600 Subject: [PATCH 10/30] Revert tag rename --- envs/shared/jupyterhub.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/envs/shared/jupyterhub.yaml b/envs/shared/jupyterhub.yaml index a114f5c..5a3e3ed 100755 --- a/envs/shared/jupyterhub.yaml +++ b/envs/shared/jupyterhub.yaml @@ -50,7 +50,7 @@ singleuser: display_name: "Standard" default: true kubespawner_override: - image: "${singleuser_image_repo}:${singleuser_image_tag}-base" + image: "${singleuser_image_repo}:${singleuser_image_tag}" kubespawner_override: image_pull_policy: Always cpu_limit: 2 @@ -69,7 +69,7 @@ singleuser: # display_name: "Standard" # default: true # kubespawner_override: - # image: "${singleuser_image_repo}:${singleuser_image_tag}-base" + # image: "${singleuser_image_repo}:${singleuser_image_tag}" # matlab: # display_name: "MATLAB (must provide your own license)" # kubespawner_override: @@ -93,7 +93,7 @@ singleuser: display_name: "Standard" default: true kubespawner_override: - image: "${singleuser_image_repo}:${singleuser_image_tag}-base" + image: "${singleuser_image_repo}:${singleuser_image_tag}" matlab: display_name: "MATLAB (must provide your own license)" kubespawner_override: @@ -118,7 +118,7 @@ singleuser: display_name: "Standard" default: true kubespawner_override: - image: "${singleuser_image_repo}:${singleuser_image_tag}-base" + image: "${singleuser_image_repo}:${singleuser_image_tag}" matlab: display_name: "MATLAB (must provide your own license)" kubespawner_override: @@ -141,7 +141,7 @@ singleuser: display_name: "Standard" default: true kubespawner_override: - image: "${singleuser_image_repo}:${singleuser_image_tag}-base" + image: "${singleuser_image_repo}:${singleuser_image_tag}" matlab: display_name: "MATLAB (must provide your own license)" kubespawner_override: From f86a786a926a864cc8186bc5d41045d9ff5a926d Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Thu, 12 Dec 2024 16:45:44 -0600 Subject: [PATCH 11/30] Remove allensdk --- images/Dockerfile.gpu | 8 -------- 1 file changed, 8 deletions(-) diff --git a/images/Dockerfile.gpu b/images/Dockerfile.gpu index 45808eb..6d80774 100644 --- a/images/Dockerfile.gpu +++ b/images/Dockerfile.gpu @@ -44,19 +44,11 @@ USER $NB_USER RUN pip install --no-cache-dir jupyter-remote-desktop-proxy -# Install Allen SDK -RUN mamba create --yes -n allen -c conda-forge python=3.11 pip ipykernel 'h5py>=3.4=mpi*' \ - && /opt/conda/envs/allen/bin/pip install --no-cache-dir allensdk \ - && conda clean --all -f -y && rm -rf /tmp/* - RUN mamba install --yes datalad rclone 'h5py>3.3=mpi*' ipykernel zarr blosc eccodes websockify \ && wget --quiet https://raw.githubusercontent.com/DanielDent/git-annex-remote-rclone/v0.8/git-annex-remote-rclone \ && chmod +x git-annex-remote-rclone && mv git-annex-remote-rclone /opt/conda/bin \ && conda clean --all -f -y && rm -rf /tmp/* -RUN /opt/conda/envs/allen/bin/python -m ipykernel install --user --name allen \ - --display-name="Allen SDK" - RUN pip install --no-cache-dir -U plotly jupyter_bokeh jupytext nbgitpuller datalad_container \ datalad-osf dandi nibabel nilearn pybids spikeinterface neo itkwidgets \ 'pydra>=0.17' 'pynwb>=2.3.1' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ From 28fda89fade6a80541f350e704acbc67bb9b3558 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Thu, 12 Dec 2024 17:06:06 -0600 Subject: [PATCH 12/30] Uncomment matlab Dockerfiles --- .github/workflows/docker-push.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-push.yaml b/.github/workflows/docker-push.yaml index bf9f553..40ed002 100644 --- a/.github/workflows/docker-push.yaml +++ b/.github/workflows/docker-push.yaml @@ -17,12 +17,12 @@ jobs: include: - tag: latest dockerfile: Dockerfile - # - tag: latest-matlab - # dockerfile: Dockerfile.matlab + - tag: latest-matlab + dockerfile: Dockerfile.matlab - tag: latest-gpu dockerfile: Dockerfile.gpu - # - tag: latest-gpu-matlab - # dockerfile: Dockerfile.gpu.matlab + - tag: latest-gpu-matlab + dockerfile: Dockerfile.gpu.matlab steps: - name: Checkout repository From f4122d223538758e8e5b5fa670c120fdb7ffc56a Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 13 Dec 2024 13:31:58 -0600 Subject: [PATCH 13/30] Remove Matlab images from build matrix --- .github/workflows/docker-push.yaml | 4 ---- .github/workflows/docker-test.yaml | 4 ---- 2 files changed, 8 deletions(-) diff --git a/.github/workflows/docker-push.yaml b/.github/workflows/docker-push.yaml index 40ed002..941fea7 100644 --- a/.github/workflows/docker-push.yaml +++ b/.github/workflows/docker-push.yaml @@ -17,12 +17,8 @@ jobs: include: - tag: latest dockerfile: Dockerfile - - tag: latest-matlab - dockerfile: Dockerfile.matlab - tag: latest-gpu dockerfile: Dockerfile.gpu - - tag: latest-gpu-matlab - dockerfile: Dockerfile.gpu.matlab steps: - name: Checkout repository diff --git a/.github/workflows/docker-test.yaml b/.github/workflows/docker-test.yaml index 8f1520e..14e23d9 100644 --- a/.github/workflows/docker-test.yaml +++ b/.github/workflows/docker-test.yaml @@ -15,12 +15,8 @@ jobs: include: - tag: latest dockerfile: Dockerfile - - tag: latest-matlab - dockerfile: Dockerfile.matlab - tag: latest-gpu dockerfile: Dockerfile.gpu - - tag: latest-gpu-matlab - dockerfile: Dockerfile.gpu.matlab steps: - name: Checkout repository From 2c5273252d4664ff0ad30b1f0614da7838da4455 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 13 Dec 2024 14:03:53 -0600 Subject: [PATCH 14/30] Revert "Remove Matlab images from build matrix" This reverts commit f4122d223538758e8e5b5fa670c120fdb7ffc56a. --- .github/workflows/docker-push.yaml | 4 ++++ .github/workflows/docker-test.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/docker-push.yaml b/.github/workflows/docker-push.yaml index 941fea7..40ed002 100644 --- a/.github/workflows/docker-push.yaml +++ b/.github/workflows/docker-push.yaml @@ -17,8 +17,12 @@ jobs: include: - tag: latest dockerfile: Dockerfile + - tag: latest-matlab + dockerfile: Dockerfile.matlab - tag: latest-gpu dockerfile: Dockerfile.gpu + - tag: latest-gpu-matlab + dockerfile: Dockerfile.gpu.matlab steps: - name: Checkout repository diff --git a/.github/workflows/docker-test.yaml b/.github/workflows/docker-test.yaml index 14e23d9..8f1520e 100644 --- a/.github/workflows/docker-test.yaml +++ b/.github/workflows/docker-test.yaml @@ -15,8 +15,12 @@ jobs: include: - tag: latest dockerfile: Dockerfile + - tag: latest-matlab + dockerfile: Dockerfile.matlab - tag: latest-gpu dockerfile: Dockerfile.gpu + - tag: latest-gpu-matlab + dockerfile: Dockerfile.gpu.matlab steps: - name: Checkout repository From eac92492f10544747a66d83f90d65224c7a61a66 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 13 Dec 2024 14:05:33 -0600 Subject: [PATCH 15/30] [WIP] Update base image --- images/Dockerfile.gpu.matlab | 2 +- images/Dockerfile.matlab | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/images/Dockerfile.gpu.matlab b/images/Dockerfile.gpu.matlab index cd0b914..87da5fb 100644 --- a/images/Dockerfile.gpu.matlab +++ b/images/Dockerfile.gpu.matlab @@ -1,5 +1,5 @@ ARG MATLAB_RELEASE=r2023a -FROM --platform=linux/amd64 ghcr.io/mathworks-ref-arch/matlab-integration-for-jupyter/jupyter-byoi-matlab-notebook:${MATLAB_RELEASE} +FROM --platform=linux/amd64 ghcr.io/mathworks-ref-arch/matlab-integration-for-jupyter/jupyter-matlab-notebook:${MATLAB_RELEASE} USER root ARG VERSION="1.1.5" diff --git a/images/Dockerfile.matlab b/images/Dockerfile.matlab index bc3d791..166d4c8 100644 --- a/images/Dockerfile.matlab +++ b/images/Dockerfile.matlab @@ -1,5 +1,5 @@ ARG MATLAB_RELEASE=r2023a -FROM --platform=linux/amd64 ghcr.io/mathworks-ref-arch/matlab-integration-for-jupyter/jupyter-byoi-matlab-notebook:${MATLAB_RELEASE} +FROM --platform=linux/amd64 ghcr.io/mathworks-ref-arch/matlab-integration-for-jupyter/jupyter-matlab-notebook:${MATLAB_RELEASE} USER root ARG VERSION="1.1.5" From c0cc8bb33760fd3212e4a9d18881d479b755c37c Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 13 Dec 2024 14:15:43 -0600 Subject: [PATCH 16/30] Revert "[WIP] Update base image" This reverts commit eac92492f10544747a66d83f90d65224c7a61a66. --- images/Dockerfile.gpu.matlab | 2 +- images/Dockerfile.matlab | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/images/Dockerfile.gpu.matlab b/images/Dockerfile.gpu.matlab index 87da5fb..cd0b914 100644 --- a/images/Dockerfile.gpu.matlab +++ b/images/Dockerfile.gpu.matlab @@ -1,5 +1,5 @@ ARG MATLAB_RELEASE=r2023a -FROM --platform=linux/amd64 ghcr.io/mathworks-ref-arch/matlab-integration-for-jupyter/jupyter-matlab-notebook:${MATLAB_RELEASE} +FROM --platform=linux/amd64 ghcr.io/mathworks-ref-arch/matlab-integration-for-jupyter/jupyter-byoi-matlab-notebook:${MATLAB_RELEASE} USER root ARG VERSION="1.1.5" diff --git a/images/Dockerfile.matlab b/images/Dockerfile.matlab index 166d4c8..bc3d791 100644 --- a/images/Dockerfile.matlab +++ b/images/Dockerfile.matlab @@ -1,5 +1,5 @@ ARG MATLAB_RELEASE=r2023a -FROM --platform=linux/amd64 ghcr.io/mathworks-ref-arch/matlab-integration-for-jupyter/jupyter-matlab-notebook:${MATLAB_RELEASE} +FROM --platform=linux/amd64 ghcr.io/mathworks-ref-arch/matlab-integration-for-jupyter/jupyter-byoi-matlab-notebook:${MATLAB_RELEASE} USER root ARG VERSION="1.1.5" From f5b00aee5f2b376be72e0a063b8cc68936a7c2be Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 13 Dec 2024 13:31:58 -0600 Subject: [PATCH 17/30] Remove Matlab images from build matrix --- .github/workflows/docker-push.yaml | 4 ---- .github/workflows/docker-test.yaml | 4 ---- 2 files changed, 8 deletions(-) diff --git a/.github/workflows/docker-push.yaml b/.github/workflows/docker-push.yaml index 40ed002..941fea7 100644 --- a/.github/workflows/docker-push.yaml +++ b/.github/workflows/docker-push.yaml @@ -17,12 +17,8 @@ jobs: include: - tag: latest dockerfile: Dockerfile - - tag: latest-matlab - dockerfile: Dockerfile.matlab - tag: latest-gpu dockerfile: Dockerfile.gpu - - tag: latest-gpu-matlab - dockerfile: Dockerfile.gpu.matlab steps: - name: Checkout repository diff --git a/.github/workflows/docker-test.yaml b/.github/workflows/docker-test.yaml index 8f1520e..14e23d9 100644 --- a/.github/workflows/docker-test.yaml +++ b/.github/workflows/docker-test.yaml @@ -15,12 +15,8 @@ jobs: include: - tag: latest dockerfile: Dockerfile - - tag: latest-matlab - dockerfile: Dockerfile.matlab - tag: latest-gpu dockerfile: Dockerfile.gpu - - tag: latest-gpu-matlab - dockerfile: Dockerfile.gpu.matlab steps: - name: Checkout repository From f6c63ed7b54bb9509f969c320afc5991bf07b14f Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 13 Dec 2024 14:30:06 -0600 Subject: [PATCH 18/30] Add Dockerfile.gpu.allensdk --- images/Dockerfile.gpu.allensdk | 73 ++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 images/Dockerfile.gpu.allensdk diff --git a/images/Dockerfile.gpu.allensdk b/images/Dockerfile.gpu.allensdk new file mode 100644 index 0000000..45808eb --- /dev/null +++ b/images/Dockerfile.gpu.allensdk @@ -0,0 +1,73 @@ +FROM quay.io/jupyter/datascience-notebook:2024-02-06 + +USER root + +RUN apt update \ + && apt install -y software-properties-common \ + && add-apt-repository -y 'ppa:apptainer/ppa' \ + && apt update \ + && apt install -y apptainer-suid \ + && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* + +RUN apt-get update && apt-get install -y ca-certificates libseccomp2 \ + s3fs netbase less parallel tmux screen vim emacs htop curl \ + && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* + +RUN curl --silent --show-error "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" \ + -o "awscliv2.zip" && unzip awscliv2.zip \ + && ./aws/install && rm -rf ./aws awscliv2.zip + +# Install jupyter server proxy and desktop +RUN curl -fsSLo /usr/share/keyrings/brave-browser-archive-keyring.gpg https://brave-browser-apt-release.s3.brave.com/brave-browser-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/brave-browser-archive-keyring.gpg] https://brave-browser-apt-release.s3.brave.com/ stable main"|sudo tee /etc/apt/sources.list.d/brave-browser-release.list \ + && apt-get -y -qq update \ + && apt-get -y -qq install \ + dbus-x11 \ + xfce4 \ + xfce4-panel \ + xfce4-session \ + xfce4-settings \ + xorg \ + xubuntu-icon-theme \ + brave-browser \ + tigervnc-standalone-server \ + tigervnc-xorg-extension \ + # chown $HOME to workaround that the xorg installation creates a + # /home/jovyan/.cache directory owned by root + && chown -R $NB_UID:$NB_GID $HOME \ + && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* + +# apt-get may result in root-owned directories/files under $HOME +RUN mkdir /opt/extras && chown -R $NB_UID:$NB_GID $HOME /opt/extras + +USER $NB_USER + +RUN pip install --no-cache-dir jupyter-remote-desktop-proxy + +# Install Allen SDK +RUN mamba create --yes -n allen -c conda-forge python=3.11 pip ipykernel 'h5py>=3.4=mpi*' \ + && /opt/conda/envs/allen/bin/pip install --no-cache-dir allensdk \ + && conda clean --all -f -y && rm -rf /tmp/* + +RUN mamba install --yes datalad rclone 'h5py>3.3=mpi*' ipykernel zarr blosc eccodes websockify \ + && wget --quiet https://raw.githubusercontent.com/DanielDent/git-annex-remote-rclone/v0.8/git-annex-remote-rclone \ + && chmod +x git-annex-remote-rclone && mv git-annex-remote-rclone /opt/conda/bin \ + && conda clean --all -f -y && rm -rf /tmp/* + +RUN /opt/conda/envs/allen/bin/python -m ipykernel install --user --name allen \ + --display-name="Allen SDK" + +RUN pip install --no-cache-dir -U plotly jupyter_bokeh jupytext nbgitpuller datalad_container \ + datalad-osf dandi nibabel nilearn pybids spikeinterface neo itkwidgets \ + 'pydra>=0.17' 'pynwb>=2.3.1' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ + aicsimageio kerchunk 'neuroglancer>=2.28' cloud-volume ipywidgets ome-zarr \ + webio_jupyter_extension https://github.com/balbasty/dandi-io/archive/refs/heads/main.zip \ + tensorstore anndata && rm -rf /tmp/* + +RUN mamba install --yes -c "nvidia/label/cuda-11.8.0" cuda-toolkit cudnn \ + && conda clean --all -f -y && rm -rf /tmp/* + +RUN pip install --no-cache-dir multiscale_spatial_image jupyterlab-git \ + datalad-fuse jupyterlab_nvdashboard + +RUN pip install --no-cache-dir brainrender napari From 6af7eb24331bc53936d737bab9f6f8a9e21c1ee4 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 13 Dec 2024 14:30:37 -0600 Subject: [PATCH 19/30] Remove AllenSDK from `Dockerfile` and create `Dockerfile.allensdk` --- images/Dockerfile | 8 ----- images/Dockerfile.allensdk | 70 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 images/Dockerfile.allensdk diff --git a/images/Dockerfile b/images/Dockerfile index aa20736..f94a11c 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -44,19 +44,11 @@ USER $NB_USER RUN pip install --no-cache-dir jupyter-remote-desktop-proxy -# Install Allen SDK -RUN mamba create --yes -n allen -c conda-forge python=3.8 pip ipykernel 'h5py>=3.4=mpi*' \ - && /opt/conda/envs/allen/bin/pip install --no-cache-dir allensdk \ - && conda clean --all -f -y && rm -rf /tmp/* - RUN mamba install --yes 'datalad>=0.18' rclone 'h5py>3.3=mpi*' ipykernel zarr blosc eccodes websockify \ && wget --quiet https://raw.githubusercontent.com/DanielDent/git-annex-remote-rclone/v0.8/git-annex-remote-rclone \ && chmod +x git-annex-remote-rclone && mv git-annex-remote-rclone /opt/conda/bin \ && conda clean --all -f -y && rm -rf /tmp/* -RUN /opt/conda/envs/allen/bin/python -m ipykernel install --user --name allen \ - --display-name="Allen SDK" - RUN pip install --no-cache-dir -U plotly jupyter_bokeh jupytext nbgitpuller datalad_container \ datalad-osf dandi nibabel nilearn pybids spikeinterface neo itkwidgets \ 'pydra>=0.17' 'pynwb>=2.3.1' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ diff --git a/images/Dockerfile.allensdk b/images/Dockerfile.allensdk new file mode 100644 index 0000000..aa20736 --- /dev/null +++ b/images/Dockerfile.allensdk @@ -0,0 +1,70 @@ +FROM quay.io/jupyter/datascience-notebook:2024-02-06 + +USER root + +RUN apt update \ + && apt install -y software-properties-common \ + && add-apt-repository -y 'ppa:apptainer/ppa' \ + && apt update \ + && apt install -y apptainer-suid \ + && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* + +RUN apt-get update && apt-get install -y ca-certificates libseccomp2 \ + s3fs netbase less parallel tmux screen vim emacs htop curl \ + && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* + +RUN curl --silent --show-error "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" \ + -o "awscliv2.zip" && unzip awscliv2.zip \ + && ./aws/install && rm -rf ./aws awscliv2.zip + +# Install jupyter server proxy and desktop +RUN curl -fsSLo /usr/share/keyrings/brave-browser-archive-keyring.gpg https://brave-browser-apt-release.s3.brave.com/brave-browser-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/brave-browser-archive-keyring.gpg] https://brave-browser-apt-release.s3.brave.com/ stable main"|sudo tee /etc/apt/sources.list.d/brave-browser-release.list \ + && apt-get -y -qq update \ + && apt-get -y -qq install \ + dbus-x11 \ + xfce4 \ + xfce4-panel \ + xfce4-session \ + xfce4-settings \ + xorg \ + xubuntu-icon-theme \ + brave-browser \ + tigervnc-standalone-server \ + tigervnc-xorg-extension \ + # chown $HOME to workaround that the xorg installation creates a + # /home/jovyan/.cache directory owned by root + && chown -R $NB_UID:$NB_GID $HOME \ + && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* + +# apt-get may result in root-owned directories/files under $HOME +RUN mkdir /opt/extras && chown -R $NB_UID:$NB_GID $HOME /opt/extras + +USER $NB_USER + +RUN pip install --no-cache-dir jupyter-remote-desktop-proxy + +# Install Allen SDK +RUN mamba create --yes -n allen -c conda-forge python=3.8 pip ipykernel 'h5py>=3.4=mpi*' \ + && /opt/conda/envs/allen/bin/pip install --no-cache-dir allensdk \ + && conda clean --all -f -y && rm -rf /tmp/* + +RUN mamba install --yes 'datalad>=0.18' rclone 'h5py>3.3=mpi*' ipykernel zarr blosc eccodes websockify \ + && wget --quiet https://raw.githubusercontent.com/DanielDent/git-annex-remote-rclone/v0.8/git-annex-remote-rclone \ + && chmod +x git-annex-remote-rclone && mv git-annex-remote-rclone /opt/conda/bin \ + && conda clean --all -f -y && rm -rf /tmp/* + +RUN /opt/conda/envs/allen/bin/python -m ipykernel install --user --name allen \ + --display-name="Allen SDK" + +RUN pip install --no-cache-dir -U plotly jupyter_bokeh jupytext nbgitpuller datalad_container \ + datalad-osf dandi nibabel nilearn pybids spikeinterface neo itkwidgets \ + 'pydra>=0.17' 'pynwb>=2.3.1' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ + aicsimageio kerchunk 'neuroglancer>=2.28' cloud-volume ipywidgets ome-zarr \ + webio_jupyter_extension https://github.com/balbasty/dandi-io/archive/refs/heads/main.zip \ + tensorstore anndata + +RUN pip install --no-cache-dir multiscale_spatial_image jupyterlab-git \ + datalad-fuse + +RUN pip install --no-cache-dir brainrender napari From 66735365634fd79bdb934a5cb154a73d386dee1a Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 13 Dec 2024 14:31:30 -0600 Subject: [PATCH 20/30] Update GA build matrix with AllenSDK Dockerfiles --- .github/workflows/docker-push.yaml | 4 ++++ .github/workflows/docker-test.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/docker-push.yaml b/.github/workflows/docker-push.yaml index 941fea7..66f4a32 100644 --- a/.github/workflows/docker-push.yaml +++ b/.github/workflows/docker-push.yaml @@ -17,8 +17,12 @@ jobs: include: - tag: latest dockerfile: Dockerfile + - tag: latest + dockerfile: Dockerfile.allensdk - tag: latest-gpu dockerfile: Dockerfile.gpu + - tag: latest-gpu-allensdk + dockerfile: Dockerfile.gpu.allensdk steps: - name: Checkout repository diff --git a/.github/workflows/docker-test.yaml b/.github/workflows/docker-test.yaml index 14e23d9..8161b4a 100644 --- a/.github/workflows/docker-test.yaml +++ b/.github/workflows/docker-test.yaml @@ -15,8 +15,12 @@ jobs: include: - tag: latest dockerfile: Dockerfile + - tag: latest + dockerfile: Dockerfile.allensdk - tag: latest-gpu dockerfile: Dockerfile.gpu + - tag: latest-gpu-allensdk + dockerfile: Dockerfile.gpu.allensdk steps: - name: Checkout repository From 84696b4f61f8f925d2c41c10a90d511a3fd39402 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 13 Dec 2024 14:35:03 -0600 Subject: [PATCH 21/30] Remove dependencies from AllenSDK images --- images/Dockerfile.allensdk | 6 ++---- images/Dockerfile.gpu.allensdk | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/images/Dockerfile.allensdk b/images/Dockerfile.allensdk index aa20736..638e96e 100644 --- a/images/Dockerfile.allensdk +++ b/images/Dockerfile.allensdk @@ -58,13 +58,11 @@ RUN /opt/conda/envs/allen/bin/python -m ipykernel install --user --name allen \ --display-name="Allen SDK" RUN pip install --no-cache-dir -U plotly jupyter_bokeh jupytext nbgitpuller datalad_container \ - datalad-osf dandi nibabel nilearn pybids spikeinterface neo itkwidgets \ + datalad-osf dandi pybids spikeinterface neo itkwidgets \ 'pydra>=0.17' 'pynwb>=2.3.1' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ - aicsimageio kerchunk 'neuroglancer>=2.28' cloud-volume ipywidgets ome-zarr \ + aicsimageio kerchunk cloud-volume ipywidgets ome-zarr \ webio_jupyter_extension https://github.com/balbasty/dandi-io/archive/refs/heads/main.zip \ tensorstore anndata RUN pip install --no-cache-dir multiscale_spatial_image jupyterlab-git \ datalad-fuse - -RUN pip install --no-cache-dir brainrender napari diff --git a/images/Dockerfile.gpu.allensdk b/images/Dockerfile.gpu.allensdk index 45808eb..625f999 100644 --- a/images/Dockerfile.gpu.allensdk +++ b/images/Dockerfile.gpu.allensdk @@ -58,9 +58,9 @@ RUN /opt/conda/envs/allen/bin/python -m ipykernel install --user --name allen \ --display-name="Allen SDK" RUN pip install --no-cache-dir -U plotly jupyter_bokeh jupytext nbgitpuller datalad_container \ - datalad-osf dandi nibabel nilearn pybids spikeinterface neo itkwidgets \ + datalad-osf dandi pybids spikeinterface neo itkwidgets \ 'pydra>=0.17' 'pynwb>=2.3.1' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ - aicsimageio kerchunk 'neuroglancer>=2.28' cloud-volume ipywidgets ome-zarr \ + aicsimageio kerchunk cloud-volume ipywidgets ome-zarr \ webio_jupyter_extension https://github.com/balbasty/dandi-io/archive/refs/heads/main.zip \ tensorstore anndata && rm -rf /tmp/* @@ -69,5 +69,3 @@ RUN mamba install --yes -c "nvidia/label/cuda-11.8.0" cuda-toolkit cudnn \ RUN pip install --no-cache-dir multiscale_spatial_image jupyterlab-git \ datalad-fuse jupyterlab_nvdashboard - -RUN pip install --no-cache-dir brainrender napari From 6556e96a4187c6a87e74fb2a3f33fef9ecf17f58 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 13 Dec 2024 14:55:50 -0600 Subject: [PATCH 22/30] Add AllenSDK images to profileList --- envs/shared/jupyterhub.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/envs/shared/jupyterhub.yaml b/envs/shared/jupyterhub.yaml index 5a3e3ed..a08abbe 100755 --- a/envs/shared/jupyterhub.yaml +++ b/envs/shared/jupyterhub.yaml @@ -51,6 +51,11 @@ singleuser: default: true kubespawner_override: image: "${singleuser_image_repo}:${singleuser_image_tag}" + allensdk: + display_name: "AllenSDK" + default: true + kubespawner_override: + image: "${singleuser_image_repo}:${singleuser_image_tag}-allensdk" kubespawner_override: image_pull_policy: Always cpu_limit: 2 @@ -70,6 +75,11 @@ singleuser: # default: true # kubespawner_override: # image: "${singleuser_image_repo}:${singleuser_image_tag}" + # allensdk: + # display_name: "AllenSDK" + # default: true + # kubespawner_override: + # image: "${singleuser_image_repo}:${singleuser_image_tag}-allensdk" # matlab: # display_name: "MATLAB (must provide your own license)" # kubespawner_override: @@ -94,6 +104,11 @@ singleuser: default: true kubespawner_override: image: "${singleuser_image_repo}:${singleuser_image_tag}" + allensdk: + display_name: "AllenSDK" + default: true + kubespawner_override: + image: "${singleuser_image_repo}:${singleuser_image_tag}-allensdk" matlab: display_name: "MATLAB (must provide your own license)" kubespawner_override: @@ -119,6 +134,11 @@ singleuser: default: true kubespawner_override: image: "${singleuser_image_repo}:${singleuser_image_tag}" + allensdk: + display_name: "AllenSDK" + default: true + kubespawner_override: + image: "${singleuser_image_repo}:${singleuser_image_tag}-allensdk" matlab: display_name: "MATLAB (must provide your own license)" kubespawner_override: @@ -142,6 +162,11 @@ singleuser: default: true kubespawner_override: image: "${singleuser_image_repo}:${singleuser_image_tag}" + allensdk: + display_name: "AllenSDK" + default: true + kubespawner_override: + image: "${singleuser_image_repo}:${singleuser_image_tag}-allensdk" matlab: display_name: "MATLAB (must provide your own license)" kubespawner_override: @@ -165,6 +190,11 @@ singleuser: default: true kubespawner_override: image: "${singleuser_image_repo}:${singleuser_image_tag}-gpu" + allensdk: + display_name: "AllenSDK GPU" + default: true + kubespawner_override: + image: "${singleuser_image_repo}:${singleuser_image_tag}-gpu-allensdk" matlab: display_name: "MATLAB GPU (must provide your own license)" kubespawner_override: From 3923c528b3b21750de73d2011f457cf0ddef1058 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 13 Dec 2024 14:56:47 -0600 Subject: [PATCH 23/30] Fix tag --- .github/workflows/docker-push.yaml | 2 +- .github/workflows/docker-test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-push.yaml b/.github/workflows/docker-push.yaml index 66f4a32..5453265 100644 --- a/.github/workflows/docker-push.yaml +++ b/.github/workflows/docker-push.yaml @@ -17,7 +17,7 @@ jobs: include: - tag: latest dockerfile: Dockerfile - - tag: latest + - tag: latest-allensdk dockerfile: Dockerfile.allensdk - tag: latest-gpu dockerfile: Dockerfile.gpu diff --git a/.github/workflows/docker-test.yaml b/.github/workflows/docker-test.yaml index 8161b4a..b13db86 100644 --- a/.github/workflows/docker-test.yaml +++ b/.github/workflows/docker-test.yaml @@ -15,7 +15,7 @@ jobs: include: - tag: latest dockerfile: Dockerfile - - tag: latest + - tag: latest-allensdk dockerfile: Dockerfile.allensdk - tag: latest-gpu dockerfile: Dockerfile.gpu From 440b075082b93158364743426f6d441b83c642d0 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Wed, 18 Dec 2024 17:31:53 -0600 Subject: [PATCH 24/30] Update base image --- images/Dockerfile | 2 +- images/Dockerfile.allensdk | 2 +- images/Dockerfile.gpu | 2 +- images/Dockerfile.gpu.allensdk | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/images/Dockerfile b/images/Dockerfile index f94a11c..c7bc6c9 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/jupyter/datascience-notebook:2024-02-06 +FROM quay.io/jupyter/datascience-notebook:python-3.11.10 USER root diff --git a/images/Dockerfile.allensdk b/images/Dockerfile.allensdk index 638e96e..2cd7a50 100644 --- a/images/Dockerfile.allensdk +++ b/images/Dockerfile.allensdk @@ -1,4 +1,4 @@ -FROM quay.io/jupyter/datascience-notebook:2024-02-06 +FROM quay.io/jupyter/datascience-notebook:python-3.11.10 USER root diff --git a/images/Dockerfile.gpu b/images/Dockerfile.gpu index 6d80774..49d0fba 100644 --- a/images/Dockerfile.gpu +++ b/images/Dockerfile.gpu @@ -1,4 +1,4 @@ -FROM quay.io/jupyter/datascience-notebook:2024-02-06 +FROM quay.io/jupyter/datascience-notebook:python-3.11.10 USER root diff --git a/images/Dockerfile.gpu.allensdk b/images/Dockerfile.gpu.allensdk index 625f999..5f456f5 100644 --- a/images/Dockerfile.gpu.allensdk +++ b/images/Dockerfile.gpu.allensdk @@ -1,4 +1,4 @@ -FROM quay.io/jupyter/datascience-notebook:2024-02-06 +FROM quay.io/jupyter/datascience-notebook:python-3.11.10 USER root From ca646313f1026aeb5163733bc08290989b89eacf Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 20 Dec 2024 12:06:19 -0600 Subject: [PATCH 25/30] Fix cuda installation --- images/Dockerfile.gpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/Dockerfile.gpu b/images/Dockerfile.gpu index 49d0fba..0afbcea 100644 --- a/images/Dockerfile.gpu +++ b/images/Dockerfile.gpu @@ -56,7 +56,7 @@ RUN pip install --no-cache-dir -U plotly jupyter_bokeh jupytext nbgitpuller data webio_jupyter_extension https://github.com/balbasty/dandi-io/archive/refs/heads/main.zip \ tensorstore anndata && rm -rf /tmp/* -RUN mamba install --yes -c "nvidia/label/cuda-11.8.0" cuda-toolkit cudnn \ +RUN mamba install --yes -c "nvidia/label/cuda-12.6.3" cuda-toolkit cudnn \ && conda clean --all -f -y && rm -rf /tmp/* RUN pip install --no-cache-dir multiscale_spatial_image jupyterlab-git \ From 8d7f44c773bcc5c792d31a1d8d72ff6276bcc29d Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 20 Dec 2024 13:51:19 -0600 Subject: [PATCH 26/30] Convert to markdown syntax --- images/README.adoc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/images/README.adoc b/images/README.adoc index 195d13f..b03e952 100644 --- a/images/README.adoc +++ b/images/README.adoc @@ -1,13 +1,13 @@ :matlab_byoi: https://github.com/mathworks-ref-arch/matlab-integration-for-jupyter/tree/main/matlab -= Dandi Docker Images +# Dandi Docker Images This folder contains Dockerfiles to build various Docker images for Dandi: * `Dockerfile` and `Dockerfile.gpu` provide Jupyter(R) notebooks for datascience. The `Dockerfile.gpu` enables the use of GPU. * `Dockerfile.matlab` provides a jupyter environment with MATLAB(R) installed. This image requires you to bring your own licence. -== MATLAB Docker Image +## MATLAB Docker Image The MATLAB Docker image relies on the {matlab_byoi}[MATLAB Integration for Jupyter in a Docker Container] BYOI (Build Your Own Image). It is shipped with https://github.com/mathworks/matlab-proxy[MATLAB-proxy] which enables communication with MATLAB from a web-browser, and with https://github.com/mathworks/jupyter-matlab-proxy[MATLAB-proxy-jupyter] which adds MATLAB integration for Jupyter. @@ -17,7 +17,7 @@ This Dockerfile includes the following add-ons: * matnwb v2.6.0.0 * Brain-Observatory-Toolbox v0.9.2 -=== How to Build +### How to Build Building the MATLAB Docker image is straight forward. The following lines consider that you already cloned the repository and that you are positioned in the `docker` folder in the cloned repository on your file system. @@ -29,7 +29,7 @@ docker build -t dandi-matlab - < Dockerfile.matlab This will build the image tagging it as `dandi-matlab`. -=== How to Run +### How to Run Running a container for the built image requires that a port is passed to the command line to tell the container which internal port needs to be exposed and on which port to map it in the host system. @@ -57,7 +57,7 @@ To open locally the Jupyter, copy/paste the `127.0.0.1:8888/xxxxx` address in yo CAUTION: If you changed the port on which will be mapped the internal container port, do not forget to change it also in the address you copy/paste from the logs. -=== Closing your Session +### Closing your Session Be careful while closing your session. If you don't close the session properly prior to stop your container, _i.e_: closing the MATLAB session and disconnecting yourself, there is chances that the MATLAB licencing system sees yourself as still connected and you'll have to wait the timeout of the session to be able to log/connect again after restarting the container. @@ -65,7 +65,7 @@ If you don't close the session properly prior to stop your container, _i.e_: clo To properly close your session, click on the `MATLAB Jupyter Setting` button which appears above the MATLAB top bar. From there, if you really want to close your session, clic on "Stop MATLAB Session", and if you really want to stop your Jupyter session, clic on "Sign Out". -=== Add new Add-Ons +### Add new Add-Ons By default, the `Dockerfile.matlab` image is shipped with two addons already installed and accessible from MATLAB. You can easily add/remove addons by changing some lines in the Dockerfile: the addons links to download/install are defined by the `ARG ADDONS` variable. @@ -95,7 +95,7 @@ generateCore(); % Generate the most recent nwb-schema \n\ clear" >> /opt/conda/lib/python3.10/site-packages/matlab_proxy/matlab/startup.m ---- -=== Customize your Container +### Customize your Container You can customize some parameter of your container changing some variables in the `Dockerfile.matlab`. From 33652db29ec04e1ae4b57998c9450db469e446d3 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 20 Dec 2024 13:51:47 -0600 Subject: [PATCH 27/30] Convert to markdown --- images/{README.adoc => README.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename images/{README.adoc => README.md} (100%) diff --git a/images/README.adoc b/images/README.md similarity index 100% rename from images/README.adoc rename to images/README.md From 82d16ab12d906bd16253aac0057ad48bb54e941d Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 20 Dec 2024 14:04:34 -0600 Subject: [PATCH 28/30] Convert syntax to markdown --- images/README.md | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/images/README.md b/images/README.md index b03e952..3302983 100644 --- a/images/README.md +++ b/images/README.md @@ -1,5 +1,3 @@ -:matlab_byoi: https://github.com/mathworks-ref-arch/matlab-integration-for-jupyter/tree/main/matlab - # Dandi Docker Images This folder contains Dockerfiles to build various Docker images for Dandi: @@ -9,8 +7,8 @@ This folder contains Dockerfiles to build various Docker images for Dandi: ## MATLAB Docker Image -The MATLAB Docker image relies on the {matlab_byoi}[MATLAB Integration for Jupyter in a Docker Container] BYOI (Build Your Own Image). -It is shipped with https://github.com/mathworks/matlab-proxy[MATLAB-proxy] which enables communication with MATLAB from a web-browser, and with https://github.com/mathworks/jupyter-matlab-proxy[MATLAB-proxy-jupyter] which adds MATLAB integration for Jupyter. +The MATLAB Docker image relies on the [MATLAB Integration for Jupyter in a Docker Container](https://github.com/mathworks-ref-arch/matlab-integration-for-jupyter). +It is shipped with [MATLAB-proxy](https://github.com/mathworks/matlab-proxy) which enables communication with MATLAB from a web-browser, and with [MATLAB-proxy-jupyter](https://github.com/mathworks/jupyter-matlab-proxy) which adds MATLAB integration for Jupyter. This Dockerfile includes the following add-ons: @@ -22,10 +20,9 @@ This Dockerfile includes the following add-ons: Building the MATLAB Docker image is straight forward. The following lines consider that you already cloned the repository and that you are positioned in the `docker` folder in the cloned repository on your file system. -[source, bash] ----- +```bash docker build -t dandi-matlab - < Dockerfile.matlab ----- +``` This will build the image tagging it as `dandi-matlab`. @@ -33,10 +30,9 @@ This will build the image tagging it as `dandi-matlab`. Running a container for the built image requires that a port is passed to the command line to tell the container which internal port needs to be exposed and on which port to map it in the host system. -[source, bash] ----- +```bash docker run -p 8888:8888 dandi-matlab:latest ----- +``` This command considers the exposition of port `8888` and maps it to the port `8888` in the host. The syntax of the option is `-p [host port]:[container port]`. @@ -44,14 +40,13 @@ The port to expose in the container is always `8888`, but the host port can be c After the container started, you can check the logs and you will see lines giving you the address you can open in your web browser to start the Jupyter instance. -[source] ----- +``` To access the server, open this file in a browser: file:///home/jovyan/.local/share/jupyter/runtime/jpserver-6-open.html Or copy and paste one of these URLs: http://78bd0f342a19:8888/lab?token=6bf3ad4d468ab3532fab610f5ff28dcf27b1b60300ec8e0c or http://127.0.0.1:8888/lab?token=6bf3ad4d468ab3532fab610f5ff28dcf27b1b60300ec8e0c ----- +``` To open locally the Jupyter, copy/paste the `127.0.0.1:8888/xxxxx` address in your browser. @@ -72,7 +67,7 @@ You can easily add/remove addons by changing some lines in the Dockerfile: the a CAUTION: The download links have to be release links towards `.zip` files. -==== How the Add-On Registration is Working +#### How the Add-On Registration is Working The add-ons registration is actually performed in two steps happening at two differents times: at "docker image construction" time, and at MATLAB startup time. @@ -82,8 +77,7 @@ At startup-time, this folder is automatically scanned by MATLAB and all download The code responsible for the auto-scan of the add-ons folder is directly injected in the `startup.m` file during the docker image construction. If some add-ons require extra actions after being installed/added to the path, you can modify these lines to add extra action before the `clear`: -[source,dockerfile] ----- +```dockerfile RUN echo -e "\n\ addons = dir('${ADDONS_DIR}'); \n\ addons = setdiff({addons([addons.isdir]).name}, {'.', '..'}); \n\ @@ -93,7 +87,7 @@ end \n\ generateCore(); % Generate the most recent nwb-schema \n\ % ADD HERE EXTRA ACTIONS FOR YOUR ADD-ON IF REQUIRED! \n\ clear" >> /opt/conda/lib/python3.10/site-packages/matlab_proxy/matlab/startup.m ----- +``` ### Customize your Container @@ -101,12 +95,12 @@ You can customize some parameter of your container changing some variables in th You can impact those parameters: -`ADDONS_DIR`:: +`ADDONS_DIR`: This variable defines where the add-ons must be downloaded/extracted and what will be the folder scanned by MATLAB at startup time. If you change this folder, the Jupyter user needs to have read/write access to it. This comes from a specificity of `matnwb` which requires the execution of some extra actions for its activation. -`ADDONS_RELEASE`:: +`ADDONS_RELEASE`: This variable defines the list of add-ons to download and install. You can add as much add-ons as you want as long as they are compatible with MATLAB-R22. -`ADDONS_LATEST`:: +`ADDONS_LATEST`: This variable defines the list of add-ons to download and install directly from the lastest version identified in the github repository. \ No newline at end of file From 48d021bb251a4902b37f35222b644fb930766711 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 20 Dec 2024 14:07:22 -0600 Subject: [PATCH 29/30] Upgrade pynwb version --- images/Dockerfile | 2 +- images/Dockerfile.allensdk | 2 +- images/Dockerfile.gpu | 2 +- images/Dockerfile.gpu.allensdk | 2 +- images/Dockerfile.gpu.matlab | 2 +- images/Dockerfile.matlab | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/images/Dockerfile b/images/Dockerfile index c7bc6c9..442e3fc 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -51,7 +51,7 @@ RUN mamba install --yes 'datalad>=0.18' rclone 'h5py>3.3=mpi*' ipykernel zarr bl RUN pip install --no-cache-dir -U plotly jupyter_bokeh jupytext nbgitpuller datalad_container \ datalad-osf dandi nibabel nilearn pybids spikeinterface neo itkwidgets \ - 'pydra>=0.17' 'pynwb>=2.3.1' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ + 'pydra>=0.17' 'pynwb>=2.8.3' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ aicsimageio kerchunk 'neuroglancer>=2.28' cloud-volume ipywidgets ome-zarr \ webio_jupyter_extension https://github.com/balbasty/dandi-io/archive/refs/heads/main.zip \ tensorstore anndata diff --git a/images/Dockerfile.allensdk b/images/Dockerfile.allensdk index 2cd7a50..75694c7 100644 --- a/images/Dockerfile.allensdk +++ b/images/Dockerfile.allensdk @@ -59,7 +59,7 @@ RUN /opt/conda/envs/allen/bin/python -m ipykernel install --user --name allen \ RUN pip install --no-cache-dir -U plotly jupyter_bokeh jupytext nbgitpuller datalad_container \ datalad-osf dandi pybids spikeinterface neo itkwidgets \ - 'pydra>=0.17' 'pynwb>=2.3.1' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ + 'pydra>=0.17' 'pynwb>=2.8.3' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ aicsimageio kerchunk cloud-volume ipywidgets ome-zarr \ webio_jupyter_extension https://github.com/balbasty/dandi-io/archive/refs/heads/main.zip \ tensorstore anndata diff --git a/images/Dockerfile.gpu b/images/Dockerfile.gpu index 0afbcea..378c328 100644 --- a/images/Dockerfile.gpu +++ b/images/Dockerfile.gpu @@ -51,7 +51,7 @@ RUN mamba install --yes datalad rclone 'h5py>3.3=mpi*' ipykernel zarr blosc ecco RUN pip install --no-cache-dir -U plotly jupyter_bokeh jupytext nbgitpuller datalad_container \ datalad-osf dandi nibabel nilearn pybids spikeinterface neo itkwidgets \ - 'pydra>=0.17' 'pynwb>=2.3.1' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ + 'pydra>=0.17' 'pynwb>=2.8.3' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ aicsimageio kerchunk 'neuroglancer>=2.28' cloud-volume ipywidgets ome-zarr \ webio_jupyter_extension https://github.com/balbasty/dandi-io/archive/refs/heads/main.zip \ tensorstore anndata && rm -rf /tmp/* diff --git a/images/Dockerfile.gpu.allensdk b/images/Dockerfile.gpu.allensdk index 5f456f5..d756822 100644 --- a/images/Dockerfile.gpu.allensdk +++ b/images/Dockerfile.gpu.allensdk @@ -59,7 +59,7 @@ RUN /opt/conda/envs/allen/bin/python -m ipykernel install --user --name allen \ RUN pip install --no-cache-dir -U plotly jupyter_bokeh jupytext nbgitpuller datalad_container \ datalad-osf dandi pybids spikeinterface neo itkwidgets \ - 'pydra>=0.17' 'pynwb>=2.3.1' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ + 'pydra>=0.17' 'pynwb>=2.8.3' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ aicsimageio kerchunk cloud-volume ipywidgets ome-zarr \ webio_jupyter_extension https://github.com/balbasty/dandi-io/archive/refs/heads/main.zip \ tensorstore anndata && rm -rf /tmp/* diff --git a/images/Dockerfile.gpu.matlab b/images/Dockerfile.gpu.matlab index cd0b914..1bf523d 100644 --- a/images/Dockerfile.gpu.matlab +++ b/images/Dockerfile.gpu.matlab @@ -72,7 +72,7 @@ RUN mamba install --yes datalad rclone 'h5py>3.3=mpi*' ipykernel zarr blosc gcc RUN pip install --no-cache-dir plotly jupyter_bokeh jupytext nbgitpuller datalad_container \ datalad-osf dandi nibabel nilearn pybids spikeinterface neo \ - 'pydra>=0.17' 'pynwb>=2.3.1' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ + 'pydra>=0.17' 'pynwb>=2.8.3' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ aicsimageio kerchunk 'neuroglancer>=2.28' cloud-volume ipywidgets ome-zarr \ webio_jupyter_extension https://github.com/balbasty/dandi-io/archive/refs/heads/main.zip \ tensorstore anndata "tensorflow[and-cuda]==2.14" diff --git a/images/Dockerfile.matlab b/images/Dockerfile.matlab index bc3d791..9c3a216 100644 --- a/images/Dockerfile.matlab +++ b/images/Dockerfile.matlab @@ -64,7 +64,7 @@ RUN mamba install --yes datalad rclone 'h5py>3.3=mpi*' ipykernel zarr blosc gcc RUN pip install --no-cache-dir plotly jupyter_bokeh jupytext nbgitpuller datalad_container \ datalad-osf dandi nibabel nilearn pybids spikeinterface neo \ - 'pydra>=0.17' 'pynwb>=2.3.1' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ + 'pydra>=0.17' 'pynwb>=2.8.3' 'nwbwidgets>=0.10.2' hdf5plugin s3fs h5netcdf "xarray[io]" \ aicsimageio kerchunk 'neuroglancer>=2.28' cloud-volume ipywidgets ome-zarr \ webio_jupyter_extension https://github.com/balbasty/dandi-io/archive/refs/heads/main.zip \ tensorstore anndata tensorflow From 18ae25361f353b085f820d53c31ffea29c450ee1 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 20 Dec 2024 14:23:51 -0600 Subject: [PATCH 30/30] Add environment variable to prevent timeout --- images/Dockerfile | 2 ++ images/Dockerfile.allensdk | 2 ++ images/Dockerfile.gpu | 2 ++ images/Dockerfile.gpu.allensdk | 2 ++ images/Dockerfile.gpu.matlab | 2 ++ images/Dockerfile.matlab | 2 +- 6 files changed, 11 insertions(+), 1 deletion(-) diff --git a/images/Dockerfile b/images/Dockerfile index 442e3fc..8b0ce42 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -44,6 +44,8 @@ USER $NB_USER RUN pip install --no-cache-dir jupyter-remote-desktop-proxy +ENV MAMBA_NO_LOW_SPEED_LIMIT=1 + RUN mamba install --yes 'datalad>=0.18' rclone 'h5py>3.3=mpi*' ipykernel zarr blosc eccodes websockify \ && wget --quiet https://raw.githubusercontent.com/DanielDent/git-annex-remote-rclone/v0.8/git-annex-remote-rclone \ && chmod +x git-annex-remote-rclone && mv git-annex-remote-rclone /opt/conda/bin \ diff --git a/images/Dockerfile.allensdk b/images/Dockerfile.allensdk index 75694c7..37c64ad 100644 --- a/images/Dockerfile.allensdk +++ b/images/Dockerfile.allensdk @@ -44,6 +44,8 @@ USER $NB_USER RUN pip install --no-cache-dir jupyter-remote-desktop-proxy +ENV MAMBA_NO_LOW_SPEED_LIMIT=1 + # Install Allen SDK RUN mamba create --yes -n allen -c conda-forge python=3.8 pip ipykernel 'h5py>=3.4=mpi*' \ && /opt/conda/envs/allen/bin/pip install --no-cache-dir allensdk \ diff --git a/images/Dockerfile.gpu b/images/Dockerfile.gpu index 378c328..607199d 100644 --- a/images/Dockerfile.gpu +++ b/images/Dockerfile.gpu @@ -44,6 +44,8 @@ USER $NB_USER RUN pip install --no-cache-dir jupyter-remote-desktop-proxy +ENV MAMBA_NO_LOW_SPEED_LIMIT=1 + RUN mamba install --yes datalad rclone 'h5py>3.3=mpi*' ipykernel zarr blosc eccodes websockify \ && wget --quiet https://raw.githubusercontent.com/DanielDent/git-annex-remote-rclone/v0.8/git-annex-remote-rclone \ && chmod +x git-annex-remote-rclone && mv git-annex-remote-rclone /opt/conda/bin \ diff --git a/images/Dockerfile.gpu.allensdk b/images/Dockerfile.gpu.allensdk index d756822..ef35404 100644 --- a/images/Dockerfile.gpu.allensdk +++ b/images/Dockerfile.gpu.allensdk @@ -44,6 +44,8 @@ USER $NB_USER RUN pip install --no-cache-dir jupyter-remote-desktop-proxy +ENV MAMBA_NO_LOW_SPEED_LIMIT=1 + # Install Allen SDK RUN mamba create --yes -n allen -c conda-forge python=3.11 pip ipykernel 'h5py>=3.4=mpi*' \ && /opt/conda/envs/allen/bin/pip install --no-cache-dir allensdk \ diff --git a/images/Dockerfile.gpu.matlab b/images/Dockerfile.gpu.matlab index 1bf523d..8e40bcf 100644 --- a/images/Dockerfile.gpu.matlab +++ b/images/Dockerfile.gpu.matlab @@ -55,6 +55,8 @@ USER $NB_USER RUN pip install --no-cache-dir jupyter-remote-desktop-proxy jupyterlab_nvdashboard +ENV MAMBA_NO_LOW_SPEED_LIMIT=1 + # Install CUDA toolkit and extension for GPU usage display RUN CONDA_OVERRIDE_CUDA="12.3" mamba install --yes -c "nvidia/label/cuda-12.3.0" cuda-toolkit cudnn \ && conda clean --all -f -y && rm -rf /tmp/* diff --git a/images/Dockerfile.matlab b/images/Dockerfile.matlab index 9c3a216..60eac31 100644 --- a/images/Dockerfile.matlab +++ b/images/Dockerfile.matlab @@ -55,7 +55,7 @@ USER $NB_USER RUN pip install --no-cache-dir jupyter-remote-desktop-proxy jupyterlab_nvdashboard - +ENV MAMBA_NO_LOW_SPEED_LIMIT=1 RUN mamba install --yes datalad rclone 'h5py>3.3=mpi*' ipykernel zarr blosc gcc eccodes websockify \ && wget --quiet https://raw.githubusercontent.com/DanielDent/git-annex-remote-rclone/v0.7/git-annex-remote-rclone \