diff --git a/Dockerfile b/Dockerfile
index b5bfdb9..529b2ce 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -75,20 +75,6 @@ RUN conda install -y \
     conda build purge-all; sync && \
     conda clean -tipsy; sync
 
-# Precaching fonts, set "Agg" as default backend for matplotlib
-RUN python -c "from matplotlib import font_manager" && \
-    sed -i 's/\(backend *: \).*$/\1Agg/g' $( python -c "import matplotlib; print(matplotlib.matplotlib_fname())" )
-
-# Precaching atlases: UPDATE TEMPLATEFLOW VERSION TO MATCH XCP_D
-RUN python -c "from templateflow import api as tfapi; \
-               tfapi.get('MNI152NLin2009cAsym', resolution=2, suffix='T1w', desc=None); \
-               tfapi.get(template='MNI152NLin6Asym', resolution=2, suffix='T1w'); \
-               tfapi.get('MNI152NLin2009cAsym', resolution=1, desc='carpet',suffix='dseg'); \
-               tfapi.get(template='MNI152NLin2009cAsym', mode='image', suffix='xfm', extension='.h5'); \
-               tfapi.get('fsLR', density='32k');" && \
-    find $HOME/.cache/templateflow -type d -exec chmod go=u {} + && \
-    find $HOME/.cache/templateflow -type f -exec chmod go=u {} +
-
 # Set up NeuroDebian
 RUN curl -sSL "http://neuro.debian.net/lists/$( lsb_release -c | cut -f2 ).us-ca.full" >> /etc/apt/sources.list.d/neurodebian.sources.list && \
     apt-key add /usr/local/etc/neurodebian.gpg && \
@@ -213,6 +199,21 @@ RUN useradd -m -s /bin/bash -G users xcp_d
 WORKDIR /home/xcp_d
 ENV HOME="/home/xcp_d"
 
+# Must do this after setting up home directory
+# Precaching fonts, set "Agg" as default backend for matplotlib
+RUN python -c "from matplotlib import font_manager" && \
+    sed -i 's/\(backend *: \).*$/\1Agg/g' $( python -c "import matplotlib; print(matplotlib.matplotlib_fname())" )
+
+# Precaching atlases: UPDATE TEMPLATEFLOW VERSION TO MATCH XCP_D
+RUN python -c "from templateflow import api as tfapi; \
+               tfapi.get('MNI152NLin2009cAsym', resolution=2, suffix='T1w', desc=None); \
+               tfapi.get(template='MNI152NLin6Asym', resolution=2, suffix='T1w'); \
+               tfapi.get('MNI152NLin2009cAsym', resolution=1, desc='carpet',suffix='dseg'); \
+               tfapi.get(template='MNI152NLin2009cAsym', mode='image', suffix='xfm', extension='.h5'); \
+               tfapi.get('fsLR', density='32k');" && \
+    find $HOME/.cache/templateflow -type d -exec chmod go=u {} + && \
+    find $HOME/.cache/templateflow -type f -exec chmod go=u {} +
+
 # Install pandoc (for HTML/LaTeX reports)
 RUN curl -o pandoc-2.2.2.1-1-amd64.deb -sSL "https://github.com/jgm/pandoc/releases/download/2.2.2.1/pandoc-2.2.2.1-1-amd64.deb" && \
     dpkg -i pandoc-2.2.2.1-1-amd64.deb && \