Merge RUN commands to reduce storage footprint of build

rapidsai · Sep 27, 2024 · f7c702a · f7c702a
1 parent 27c529f
commit f7c702a
Show file tree

Hide file tree

Showing 3 changed files with 48 additions and 79 deletions.
diff --git a/dask/Dockerfile b/dask/Dockerfile
@@ -11,38 +11,27 @@ ARG UCX_PY_VER=unset
 
 COPY environment.yml /rapids.yml
 ADD https://raw.githubusercontent.com/dask/dask/main/continuous_integration/environment-$PYTHON_VER.yaml /dask.yml
-
 ADD https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz /tools.tar.gz
-RUN tar -xzvf /tools.tar.gz -C /usr/local/bin --strip-components=1 \
-    && rm /tools.tar.gz
-
-RUN conda config --set ssl_verify false
-
-RUN rapids-mamba-retry install conda-merge git
-
-RUN cat /rapids.yml \
-    | sed -r "s/CUDA_VER/$(echo $CUDA_VER | cut -d. -f1,2)/g" \
-    | sed -r "s/RAPIDS_VER/${RAPIDS_VER}/g" \
-    | sed -r "s/UCX_PY_VER/${UCX_PY_VER}/g" \
-    > /rapids_pinned.yml
 
-# unpin problematic CI dependencies
-RUN cat /dask.yml \
-    | sed -r "s/pyarrow=/pyarrow>=/g" \
-    | sed -r "s/pandas=/pandas>=/g" \
-    | sed -r "s/numpy=/numpy>=/g" \
-    > /dask_unpinned.yml
-
-RUN conda-merge /rapids_pinned.yml /dask_unpinned.yml > /dask.yml
-
-RUN rapids-mamba-retry env create -n dask --file /dask.yml
-
-# Clean up pkgs to reduce image size and chmod for all users
-RUN chmod -R ugo+rw /opt/conda \
+RUN tar -xzvf /tools.tar.gz -C /usr/local/bin --strip-components=1 \
+    && rm /tools.tar.gz \
+    && conda config --set ssl_verify false \
+    && rapids-mamba-retry install conda-merge git \
+    && cat /rapids.yml \
+        | sed -r "s/CUDA_VER/$(echo $CUDA_VER | cut -d. -f1,2)/g" \
+        | sed -r "s/RAPIDS_VER/${RAPIDS_VER}/g" \
+        | sed -r "s/UCX_PY_VER/${UCX_PY_VER}/g" \
+        > /rapids_pinned.yml \
+    && cat /dask.yml \
+        | sed -r "s/pyarrow=/pyarrow>=/g" \
+        | sed -r "s/pandas=/pandas>=/g" \
+        | sed -r "s/numpy=/numpy>=/g" \
+        > /dask_unpinned.yml \
+    && conda-merge /rapids_pinned.yml /dask_unpinned.yml > /dask.yml \
+    && rapids-mamba-retry env create -n dask --file /dask.yml \
+    && chmod -R ugo+rw /opt/conda \
     && conda clean -tipy \
-    && chmod -R ugo+rw /opt/conda
-
-# need a user with access to conda
-RUN useradd -r -g conda -u 10000 dask
+    && chmod -R ugo+rw /opt/conda \
+    && useradd -r -g conda -u 10000 dask
 
 CMD [ "/bin/bash" ]
diff --git a/dask_image/Dockerfile b/dask_image/Dockerfile
@@ -9,29 +9,20 @@ ARG PYTHON_VER=unset
 
 COPY environment.yml /rapids.yml
 ADD https://raw.githubusercontent.com/dask/dask-image/main/continuous_integration/environment-$PYTHON_VER.yml /dask.yml
-
 ADD https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz /tools.tar.gz
-RUN tar -xzvf /tools.tar.gz -C /usr/local/bin --strip-components=1 \
-    && rm /tools.tar.gz
-
-RUN conda config --set ssl_verify false
-
-RUN rapids-mamba-retry install conda-merge git
-
-RUN cat /rapids.yml \
-    | sed -r "s/CUDA_VER/$(echo $CUDA_VER | cut -d. -f1,2)/g" \
-    > /rapids_pinned.yml
 
-RUN conda-merge /rapids_pinned.yml /dask.yml > /dask_image.yml
-
-RUN rapids-mamba-retry env create -n dask_image --file /dask_image.yml
-
-# Clean up pkgs to reduce image size and chmod for all users
-RUN chmod -R ugo+rw /opt/conda \
+RUN tar -xzvf /tools.tar.gz -C /usr/local/bin --strip-components=1 \
+    && rm /tools.tar.gz \
+    && conda config --set ssl_verify false \
+    && rapids-mamba-retry install conda-merge git \
+    && cat /rapids.yml \
+        | sed -r "s/CUDA_VER/$(echo $CUDA_VER | cut -d. -f1,2)/g" \
+        > /rapids_pinned.yml \
+    && conda-merge /rapids_pinned.yml /dask.yml > /dask_image.yml \
+    && rapids-mamba-retry env create -n dask_image --file /dask_image.yml \
+    && chmod -R ugo+rw /opt/conda \
     && conda clean -tipy \
-    && chmod -R ugo+rw /opt/conda
-
-# need a user with access to conda
-RUN useradd -r -g conda -u 10000 dask
+    && chmod -R ugo+rw /opt/conda \
+    && useradd -r -g conda -u 10000 dask
 
 CMD [ "/bin/bash" ]
diff --git a/distributed/Dockerfile b/distributed/Dockerfile
@@ -11,36 +11,25 @@ ARG UCX_PY_VER=unset
 
 COPY environment.yml /rapids.yml
 ADD https://raw.githubusercontent.com/dask/distributed/main/continuous_integration/environment-$PYTHON_VER.yaml /dask.yml
-
 ADD https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz /tools.tar.gz
-RUN tar -xzvf /tools.tar.gz -C /usr/local/bin --strip-components=1 \
-    && rm /tools.tar.gz
-
-RUN conda config --set ssl_verify false
-
-RUN rapids-mamba-retry install conda-merge git
-
-RUN cat /rapids.yml \
-    | sed -r "s/CUDA_VER/$(echo $CUDA_VER | cut -d. -f1,2)/g" \
-    | sed -r "s/RAPIDS_VER/${RAPIDS_VER}/g" \
-    | sed -r "s/UCX_PY_VER/${UCX_PY_VER}/g" \
-    > /rapids_pinned.yml
 
-# unpin problematic CI dependencies
-RUN cat /dask.yml \
-    | sed -r "s/pyarrow=/pyarrow>=/g" \
-    > /dask_unpinned.yml
-
-RUN conda-merge /rapids_pinned.yml /dask_unpinned.yml > /distributed.yml
-
-RUN rapids-mamba-retry env create -n dask --file /distributed.yml
-
-# Clean up pkgs to reduce image size and chmod for all users
-RUN chmod -R ugo+rw /opt/conda \
+RUN tar -xzvf /tools.tar.gz -C /usr/local/bin --strip-components=1 \
+    && rm /tools.tar.gz \
+    && conda config --set ssl_verify false \
+    && rapids-mamba-retry install conda-merge git \
+    && cat /rapids.yml \
+        | sed -r "s/CUDA_VER/$(echo $CUDA_VER | cut -d. -f1,2)/g" \
+        | sed -r "s/RAPIDS_VER/${RAPIDS_VER}/g" \
+        | sed -r "s/UCX_PY_VER/${UCX_PY_VER}/g" \
+    > /rapids_pinned.yml \
+    && cat /dask.yml \
+        | sed -r "s/pyarrow=/pyarrow>=/g" \
+        > /dask_unpinned.yml \
+    && conda-merge /rapids_pinned.yml /dask_unpinned.yml > /distributed.yml \
+    && rapids-mamba-retry env create -n dask --file /distributed.yml \
+    && chmod -R ugo+rw /opt/conda \
     && conda clean -tipy \
-    && chmod -R ugo+rw /opt/conda
-
-# need a user with access to conda
-RUN useradd -r -g conda -u 10000 dask
+    && chmod -R ugo+rw /opt/conda \
+    && useradd -r -g conda -u 10000 dask
 
 CMD [ "/bin/bash" ]