From 997931ee3507a037081b8dc050987f74be1ea1ef Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 26 Feb 2021 14:06:25 -0500 Subject: [PATCH 01/89] [WIP] working in freesurfer from source --- neurodocker/templates/freesurfer.yaml | 41 +++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index a762a964..686e8ee1 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -86,3 +86,44 @@ binaries: {%- endif -%} {%- endfor %} {%- endif %} + +source: + arguments: + required: + - version + optional: + repo: https://github.com/freesurfer/freesurfer + source_path: /tmp/freesurfer/freesurfer-{{ self.version }} + pkg_uri: hello + pkg_path: /tmp/freesurfer/pkg + install_path: /opt/freesurfer-{{ self.version }} + cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS=OFF -DMINIMAL=ON -DFS_PACKAGES_DIR={{ self.pkg_path }} + make_opts: -j1 + dependencies: + apt: + - ca-certificates + - cmake + - g++ + - gcc + - git + - make + yum: + - ca-certificates + - cmake + - gcc-c++ + - git + - make + env: + PATH: "{{ self.install_path }}/bin:$PATH" + LD_LIBRARY_PATH: "{{ self.install_path }}/lib:$LD_LIBRARY_PATH" + instructions: | + mkdir -p {{ self.install_path }} + git clone {{ self.repo }} {{ self.source_path }} + {% if self.version != "master" and self.version != "latest" -%} + cd {{ self.source_path }} + git fetch --tags + git checkout {{ self.version }} + {% endif -%} + cd {{ self.install_path }} + cmake {{ self.cmake_opts }} {{ self.source_path }} + make {{ self.make_opts }} From 2bc36ec747e62bcb0a5661980bd5999aca4579f6 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 26 Feb 2021 14:07:05 -0500 Subject: [PATCH 02/89] [WIP] trying to port mcr template --- neurodocker/templates/matlabmcr.yaml | 66 ++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 neurodocker/templates/matlabmcr.yaml diff --git a/neurodocker/templates/matlabmcr.yaml b/neurodocker/templates/matlabmcr.yaml new file mode 100644 index 00000000..873ab3bd --- /dev/null +++ b/neurodocker/templates/matlabmcr.yaml @@ -0,0 +1,66 @@ +--- +# Instructions to install MATLAB Compiler Runtime. +# +# Website: https://www.mathworks.com/products/compiler/matlab-runtime.html + +name: matlabmcr +binaries: + arguments: + required: + - version + optional: + curl_opts: -v + install_path: /opt/MCR-{{ self.version }} + urls: + "2019b": http://ssd.mathworks.com/supportfiles/downloads/R2019b/Release/1/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2019b_Update_1_glnxa64.zip + "2019a": https://ssd.mathworks.com/supportfiles/downloads/R2019a/Release/6/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2019a_Update_6_glnxa64.zip + "2018b": https://ssd.mathworks.com/supportfiles/downloads/R2018b/deployment_files/R2018b/installers/glnxa64/MCR_R2018b_glnxa64_installer.zip + "2018a": https://ssd.mathworks.com/supportfiles/downloads/R2018a/deployment_files/R2018a/installers/glnxa64/MCR_R2018a_glnxa64_installer.zip + "2017b": https://ssd.mathworks.com/supportfiles/downloads/R2017b/deployment_files/R2017b/installers/glnxa64/MCR_R2017b_glnxa64_installer.zip + "2017a": https://ssd.mathworks.com/supportfiles/downloads/R2017a/deployment_files/R2017a/installers/glnxa64/MCR_R2017a_glnxa64_installer.zip + "2016b": https://ssd.mathworks.com/supportfiles/downloads/R2016b/deployment_files/R2016b/installers/glnxa64/MCR_R2016b_glnxa64_installer.zip + "2016a": https://ssd.mathworks.com/supportfiles/downloads/R2016a/deployment_files/R2016a/installers/glnxa64/MCR_R2016a_glnxa64_installer.zip + "2015b": https://ssd.mathworks.com/supportfiles/downloads/R2015b/deployment_files/R2015b/installers/glnxa64/MCR_R2015b_glnxa64_installer.zip + "2015aSP1": https://ssd.mathworks.com/supportfiles/downloads/R2015a/deployment_files/R2015aSP1/installers/glnxa64/MCR_R2015aSP1_glnxa64_installer.zip + "2015a": https://ssd.mathworks.com/supportfiles/downloads/R2015a/deployment_files/R2015a/installers/glnxa64/MCR_R2015a_glnxa64_installer.zip + "2014a": https://ssd.mathworks.com/supportfiles/downloads/R2014a/deployment_files/R2014a/installers/glnxa64/MCR_R2014a_glnxa64_installer.zip + "2013b": https://ssd.mathworks.com/supportfiles/downloads/R2013b/deployment_files/R2013b/installers/glnxa64/MCR_R2013b_glnxa64_installer.zip + "2013a": https://ssd.mathworks.com/supportfiles/MCR_Runtime/R2013a/MCR_R2013a_glnxa64_installer.zip + "2012b": https://ssd.mathworks.com/supportfiles/MCR_Runtime/R2012b/MCR_R2012b_glnxa64_installer.zip + "2012a": https://ssd.mathworks.com/supportfiles/MCR_Runtime/R2012a/MCR_R2012a_glnxa64_installer.zip + "2010a": https://dl.dropbox.com/s/zz6me0c3v4yq5fd/MCR_R2010a_glnxa64_installer.bin + dependencies: + apt: + - bc + - libncurses5 + - libxext6 + - libxmu6 + - libxpm-dev + - libxt6 + yum: + - bc + - libXext.x86_64 + - libXmu + - libXpm + - libXt.x86_64 + debs: + - http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb + env: + LD_LIBRARY_PATH: "$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu:{{ self.install_path }}/{{ self.version }}/runtime/glnxa64:{{ self.install_path }}/{{ self.version }}/bin/glnxa64:{{ self.install_path }}/{{ self.version }}/sys/os/glnxa64:{{ self.install_path }}/{{ self.version }}/extern/bin/glnxa64" + MATLABCMD: "{{ self.install_path }}/{{ self.version }}/toolbox/matlab" + instructions: | + export TMPDIR="$(mktemp -d)" + {{ self.install_dependencies() }} + echo "Downloading MATLAB Compiler Runtime ..." + {% if self.version == "2010a" -%} + {{ self.install_debs() }} + curl {{ self.curl_opts }} -o "$TMPDIR/MCRInstaller.bin" {{ self.urls[self.version] }} + chmod +x "$TMPDIR/MCRInstaller.bin" + "$TMPDIR/MCRInstaller.bin" -silent -P installLocation="{{ self.install_path }}" + {% else -%} + curl {{ self.curl_opts }} -o "$TMPDIR/mcr.zip" {{ self.urls[self.version] }} + unzip -q "$TMPDIR/mcr.zip" -d "$TMPDIR/mcrtmp" + "$TMPDIR/mcrtmp/install" -destinationFolder {{ self.install_path }} -mode silent -agreeToLicense yes + {% endif -%} + rm -rf "$TMPDIR" + unset TMPDIR From 21222313c86b38c678e3893d6b79025273d5224f Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 26 Feb 2021 14:10:25 -0500 Subject: [PATCH 03/89] [WIP] env vars for fs from source --- neurodocker/templates/freesurfer.yaml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 686e8ee1..b9d817ed 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -114,8 +114,28 @@ source: - git - make env: - PATH: "{{ self.install_path }}/bin:$PATH" - LD_LIBRARY_PATH: "{{ self.install_path }}/lib:$LD_LIBRARY_PATH" + # From https://github.com/freesurfer/freesurfer/blob/54018f7d6f620d6288b28f50e14a0a4ba421757c/Dockerfile#L20-L42 + # freesurfer env + OS: "Linux" + PATH: "{{ self.install_path }}/bin:{{ self.install_path }}/fsfast/bin:{{ self.install_path }}/tktools:{{ self.install_path }}/mni/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + FREESURFER_HOME: "{{ self.install_path }}" + FREESURFER: "{{ self.install_path }}" + SUBJECTS_DIR: "{{ self.install_path }}/subjects" + LOCAL_DIR: "{{ self.install_path }}/local" + FSFAST_HOME: "{{ self.install_path }}/fsfast" + FMRI_ANALYSIS_DIR: "{{ self.install_path }}/fsfast" + FUNCTIONALS_DIR: "{{ self.install_path }}/sessions" + # default freesurfer options + FS_OVERRIDE: "0" + FIX_VERTEX_AREA: "" + FSF_OUTPUT_FORMAT: "nii.gz# mni env requirements" + # mni environment requirements + MINC_BIN_DIR: "{{ self.install_path }}/mni/bin" + MINC_LIB_DIR: "{{ self.install_path }}/mni/lib" + MNI_DIR: "{{ self.install_path }}/mni" + MNI_DATAPATH: "{{ self.install_path }}/mni/data" + MNI_PERL5LIB: "{{ self.install_path }}/mni/share/perl5" + PERL5LIB: "{{ self.install_path }}/mni/share/perl5" instructions: | mkdir -p {{ self.install_path }} git clone {{ self.repo }} {{ self.source_path }} From e04b2cadcec5d58a5258c3c7ca865a464eb17cba Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 26 Feb 2021 14:26:11 -0500 Subject: [PATCH 04/89] [WIP] tweaks --- neurodocker/templates/freesurfer.yaml | 11 +++-------- neurodocker/templates/matlabmcr.yaml | 3 ++- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index b9d817ed..38206dcf 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -64,7 +64,7 @@ binaries: # default freesurfer options FS_OVERRIDE: "0" FIX_VERTEX_AREA: "" - FSF_OUTPUT_FORMAT: "nii.gz# mni env requirements" + FSF_OUTPUT_FORMAT: "nii.gz" # mni environment requirements MINC_BIN_DIR: "{{ self.install_path }}/mni/bin" MINC_LIB_DIR: "{{ self.install_path }}/mni/lib" @@ -128,7 +128,7 @@ source: # default freesurfer options FS_OVERRIDE: "0" FIX_VERTEX_AREA: "" - FSF_OUTPUT_FORMAT: "nii.gz# mni env requirements" + FSF_OUTPUT_FORMAT: "nii.gz" # mni environment requirements MINC_BIN_DIR: "{{ self.install_path }}/mni/bin" MINC_LIB_DIR: "{{ self.install_path }}/mni/lib" @@ -138,12 +138,7 @@ source: PERL5LIB: "{{ self.install_path }}/mni/share/perl5" instructions: | mkdir -p {{ self.install_path }} - git clone {{ self.repo }} {{ self.source_path }} - {% if self.version != "master" and self.version != "latest" -%} - cd {{ self.source_path }} - git fetch --tags - git checkout {{ self.version }} - {% endif -%} + git clone --single-branch --branch {{ self.version }} {{ self.repo }} {{ self.source_path }} cd {{ self.install_path }} cmake {{ self.cmake_opts }} {{ self.source_path }} make {{ self.make_opts }} diff --git a/neurodocker/templates/matlabmcr.yaml b/neurodocker/templates/matlabmcr.yaml index 873ab3bd..ce2024a0 100644 --- a/neurodocker/templates/matlabmcr.yaml +++ b/neurodocker/templates/matlabmcr.yaml @@ -9,7 +9,7 @@ binaries: required: - version optional: - curl_opts: -v + curl_opts: "" install_path: /opt/MCR-{{ self.version }} urls: "2019b": http://ssd.mathworks.com/supportfiles/downloads/R2019b/Release/1/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2019b_Update_1_glnxa64.zip @@ -24,6 +24,7 @@ binaries: "2015aSP1": https://ssd.mathworks.com/supportfiles/downloads/R2015a/deployment_files/R2015aSP1/installers/glnxa64/MCR_R2015aSP1_glnxa64_installer.zip "2015a": https://ssd.mathworks.com/supportfiles/downloads/R2015a/deployment_files/R2015a/installers/glnxa64/MCR_R2015a_glnxa64_installer.zip "2014a": https://ssd.mathworks.com/supportfiles/downloads/R2014a/deployment_files/R2014a/installers/glnxa64/MCR_R2014a_glnxa64_installer.zip + "2014b": https://ssd.mathworks.com/supportfiles/downloads/R2014b/deployment_files/R2014b/installers/glnxa64/MCR_R2014b_glnxa64_installer.zip "2013b": https://ssd.mathworks.com/supportfiles/downloads/R2013b/deployment_files/R2013b/installers/glnxa64/MCR_R2013b_glnxa64_installer.zip "2013a": https://ssd.mathworks.com/supportfiles/MCR_Runtime/R2013a/MCR_R2013a_glnxa64_installer.zip "2012b": https://ssd.mathworks.com/supportfiles/MCR_Runtime/R2012b/MCR_R2012b_glnxa64_installer.zip From e389f8c5bce911a3675c1dfeec34b02a25131918 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 2 Mar 2021 23:39:04 +0000 Subject: [PATCH 05/89] [WIP] up to `make install` --- neurodocker/templates/freesurfer.yaml | 82 +++++++++++++++++++++------ 1 file changed, 66 insertions(+), 16 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 38206dcf..183f3666 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -94,25 +94,11 @@ source: optional: repo: https://github.com/freesurfer/freesurfer source_path: /tmp/freesurfer/freesurfer-{{ self.version }} - pkg_uri: hello + pkg_url: http://surfer.nmr.mgh.harvard.edu/pub/data/fspackages/prebuilt/centos7-packages.tar.gz pkg_path: /tmp/freesurfer/pkg install_path: /opt/freesurfer-{{ self.version }} cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS=OFF -DMINIMAL=ON -DFS_PACKAGES_DIR={{ self.pkg_path }} make_opts: -j1 - dependencies: - apt: - - ca-certificates - - cmake - - g++ - - gcc - - git - - make - yum: - - ca-certificates - - cmake - - gcc-c++ - - git - - make env: # From https://github.com/freesurfer/freesurfer/blob/54018f7d6f620d6288b28f50e14a0a4ba421757c/Dockerfile#L20-L42 # freesurfer env @@ -136,9 +122,73 @@ source: MNI_DATAPATH: "{{ self.install_path }}/mni/data" MNI_PERL5LIB: "{{ self.install_path }}/mni/share/perl5" PERL5LIB: "{{ self.install_path }}/mni/share/perl5" + dependencies: + apt: + - ca-certificates + - cmake + - git + - make + - bc + - binutils + - bzip2 + - curl + - g++-4.8 + - gcc-4.8-base + - gcc-4.8 + - gfortran-4.8 + - libbz2-dev + - libfreetype6-dev + - libgfortran-4.8-dev + - libglu1-mesa-dev + - libgomp1 + - libjpeg62-dev + - libopenblas-dev + - libssl-dev + - libtool + - libtool-bin + - libx11-dev + - libxaw7-dev + - libxi-dev + - libxml2-utils + - libxmu-dev + - libxmu-headers + - libxmu6 + - libxt-dev + - libxt6 + - perl + - sudo + - tar + - tcsh + - uuid-dev + - vim-common + - wget + yum: + - ca-certificates + - cmake + - gcc-c++ + - git + - make instructions: | + {{ self.install_dependencies() }} + # Compile with gcc 4.8 + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100 + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100 + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-4.8 100 + # FreeSurfer needs CMake 3.9 or higher + mkdir -p /tmp/cmake + cd /tmp/cmake + curl -sSL --retry 5 https://cmake.org/files/v3.12/cmake-3.12.3.tar.gz | tar -xz + cd cmake-3.12.3 + ./configure + make + make install + ### + mkdir -p {{ self.pkg_path }} mkdir -p {{ self.install_path }} + wget -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} + mv {{ self.pkg_path }}/packages/* {{ self.pkg_path }} + rm -rf {{ self.pkg_path }}/packages git clone --single-branch --branch {{ self.version }} {{ self.repo }} {{ self.source_path }} cd {{ self.install_path }} cmake {{ self.cmake_opts }} {{ self.source_path }} - make {{ self.make_opts }} + make {{ self.make_opts }} \ No newline at end of file From 2dad8b56acb35b8aed39e628b446943f024b5d77 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 4 Mar 2021 00:54:26 +0000 Subject: [PATCH 06/89] [WIP] having issues with git annex --- neurodocker/templates/freesurfer.yaml | 61 ++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 183f3666..0a480610 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -98,7 +98,7 @@ source: pkg_path: /tmp/freesurfer/pkg install_path: /opt/freesurfer-{{ self.version }} cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS=OFF -DMINIMAL=ON -DFS_PACKAGES_DIR={{ self.pkg_path }} - make_opts: -j1 + make_opts: -j4 env: # From https://github.com/freesurfer/freesurfer/blob/54018f7d6f620d6288b28f50e14a0a4ba421757c/Dockerfile#L20-L42 # freesurfer env @@ -127,6 +127,7 @@ source: - ca-certificates - cmake - git + - git-annex - make - bc - binutils @@ -171,24 +172,62 @@ source: instructions: | {{ self.install_dependencies() }} # Compile with gcc 4.8 + # -------------------- update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100 update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100 update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-4.8 100 - # FreeSurfer needs CMake 3.9 or higher - mkdir -p /tmp/cmake - cd /tmp/cmake - curl -sSL --retry 5 https://cmake.org/files/v3.12/cmake-3.12.3.tar.gz | tar -xz - cd cmake-3.12.3 + # FreeSurfer needs python > 3.6 (? todo: verify) + # From: https://github.com/pwighton/fs-docker/blob/09cc6b0b6c0538cbb182b04435daf85c123a63d5/build/Dockerfile#L66 + # Alternative is https://github.com/ReproNim/neurodocker/blob/master/neurodocker/templates/afni.yaml + # But it installs python 3.5 + # ------------------------------------------------------------------------- + curl -sSL --retry 5 https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz | tar -xz -C /tmp + cd /tmp/Python-3.6.5 ./configure make - make install - ### + make install + cd / + rm -rf /tmp/Python-3.6.5 + pip3 install -q --no-cache-dir -U pip + pip3 install -q --no-cache-dir wheel + sync + # FreeSurfer needs CMake 3.9 or higher + # Taken from: https://github.com/Rikorose/gcc-cmake/blob/master/gcc-4/Dockerfile + # ------------------------------------------------------------------------------ + wget https://github.com/Kitware/CMake/releases/download/v3.19.0/cmake-3.19.0-Linux-x86_64.sh -q -O /tmp/cmake-install.sh + chmod u+x /tmp/cmake-install.sh + mkdir -p /usr/share/cmake-3.19.0 + /tmp/cmake-install.sh --skip-license --prefix=/usr/share/cmake-3.19.0 + CMAKE_BINPATH=`which cmake` + rm -rf $CMAKE_BINPATH + ln -s /usr/share/cmake-3.19.0/bin/cmake $CMAKE_BINPATH + rm /tmp/cmake-install.sh + # FreeSurfer needs a modern (>6.0? todo verify) version of git annex + # ---------------------------------------------------------------- + curl -sSL --retry 5 https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz | tar -xz -C /usr/share + GITANNEX_BINPATH=`which git-annex` + rm -rf $GITANNEX_BINPATH + ln -s /usr/share/git-annex.linux/git-annex $GITANNEX_BINPATH + # Compile FreeSurfer + # https://github.com/pwighton/fs-docker (todo: maybe a newer way?) + # ---------------------------------------------------------------- mkdir -p {{ self.pkg_path }} mkdir -p {{ self.install_path }} wget -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} mv {{ self.pkg_path }}/packages/* {{ self.pkg_path }} rm -rf {{ self.pkg_path }}/packages git clone --single-branch --branch {{ self.version }} {{ self.repo }} {{ self.source_path }} - cd {{ self.install_path }} - cmake {{ self.cmake_opts }} {{ self.source_path }} - make {{ self.make_opts }} \ No newline at end of file + # Files needed to be downloaded from the annex before we can run `make install` + # todo: tag files in annex so we can run `git annex get --metadata fstags=makeinstall` (faster) + # ------------------------------------------------- + cd {{ self.source_path }} + # git remote add datasrc https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/repo/annex.git + # git fetch datasrc + # git annex enableremote datasrc + # git annex get . + # Compile and install + # ------------------------------------------------- + # cd {{ self.install_path }} + # cmake {{ self.cmake_opts }} {{ self.source_path }} + # make {{ self.make_opts }} + # make install From c35aa7f800a06aa5b3fa38e6b511de22a94a611e Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 4 Mar 2021 17:22:17 +0000 Subject: [PATCH 07/89] [WIP] working verison? --- neurodocker/templates/freesurfer.yaml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 0a480610..864860ef 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -202,12 +202,6 @@ source: rm -rf $CMAKE_BINPATH ln -s /usr/share/cmake-3.19.0/bin/cmake $CMAKE_BINPATH rm /tmp/cmake-install.sh - # FreeSurfer needs a modern (>6.0? todo verify) version of git annex - # ---------------------------------------------------------------- - curl -sSL --retry 5 https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz | tar -xz -C /usr/share - GITANNEX_BINPATH=`which git-annex` - rm -rf $GITANNEX_BINPATH - ln -s /usr/share/git-annex.linux/git-annex $GITANNEX_BINPATH # Compile FreeSurfer # https://github.com/pwighton/fs-docker (todo: maybe a newer way?) # ---------------------------------------------------------------- @@ -221,13 +215,15 @@ source: # todo: tag files in annex so we can run `git annex get --metadata fstags=makeinstall` (faster) # ------------------------------------------------- cd {{ self.source_path }} - # git remote add datasrc https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/repo/annex.git - # git fetch datasrc - # git annex enableremote datasrc - # git annex get . + git remote add datasrc https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/repo/annex.git + git fetch datasrc + # PW 2021/03/04: Werid.. this returns an error code, but things still work? + # So hardwire the exit code to zero so `docker build` doesn't error out + git annex enableremote datasrc || true + git annex get -q . # Compile and install # ------------------------------------------------- - # cd {{ self.install_path }} - # cmake {{ self.cmake_opts }} {{ self.source_path }} - # make {{ self.make_opts }} - # make install + cd {{ self.install_path }} + cmake {{ self.cmake_opts }} {{ self.source_path }} + make {{ self.make_opts }} + make install From 0bebb0424808a8efd579d8c164b8bea8862497bd Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 4 Mar 2021 17:42:04 +0000 Subject: [PATCH 08/89] [FIX] tweaks --- neurodocker/templates/freesurfer.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 864860ef..093c3c08 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -171,6 +171,9 @@ source: - make instructions: | {{ self.install_dependencies() }} + # git config + git config --global user.email "CI@example.com" + git config --global user.name "CI" # Compile with gcc 4.8 # -------------------- update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100 @@ -207,7 +210,7 @@ source: # ---------------------------------------------------------------- mkdir -p {{ self.pkg_path }} mkdir -p {{ self.install_path }} - wget -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} + wget -q -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} mv {{ self.pkg_path }}/packages/* {{ self.pkg_path }} rm -rf {{ self.pkg_path }}/packages git clone --single-branch --branch {{ self.version }} {{ self.repo }} {{ self.source_path }} From 63d1bb892a9f78e4387c15abe3b2786523f44e18 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 4 Mar 2021 17:54:49 +0000 Subject: [PATCH 09/89] [WIP] this builds fs --- neurodocker/templates/freesurfer.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 093c3c08..ef7290de 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -222,6 +222,7 @@ source: git fetch datasrc # PW 2021/03/04: Werid.. this returns an error code, but things still work? # So hardwire the exit code to zero so `docker build` doesn't error out + # THis is because surfer.nmr now runs git-annex 8. todo: upgrade git annex enableremote datasrc || true git annex get -q . # Compile and install From ca1617098d2c9f5176b3f1a763ed369d2b5e92ca Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 4 Mar 2021 22:57:10 +0000 Subject: [PATCH 10/89] [WIP] a lot more params --- neurodocker/templates/freesurfer.yaml | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index ef7290de..6cfbb3cb 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -92,14 +92,24 @@ source: required: - version optional: + license_base64: "" repo: https://github.com/freesurfer/freesurfer - source_path: /tmp/freesurfer/freesurfer-{{ self.version }} pkg_url: http://surfer.nmr.mgh.harvard.edu/pub/data/fspackages/prebuilt/centos7-packages.tar.gz + annex: https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/repo/annex.git + minimal: "ON" + build_guis: "OFF" + infant_module: "OFF" + install_python_deps: "OFF" + distribute_fspython: "OFF" + fortran_libs: /usr/lib/gcc/x86_64-linux-gnu/5/libgfortran.so + source_path: /tmp/freesurfer/freesurfer-{{ self.version }} pkg_path: /tmp/freesurfer/pkg install_path: /opt/freesurfer-{{ self.version }} - cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS=OFF -DMINIMAL=ON -DFS_PACKAGES_DIR={{ self.pkg_path }} + license_path: /opt/freesurfer/license.txt + cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} make_opts: -j4 env: + FS_LICENSE: "{{ self.license_path }}" # From https://github.com/freesurfer/freesurfer/blob/54018f7d6f620d6288b28f50e14a0a4ba421757c/Dockerfile#L20-L42 # freesurfer env OS: "Linux" @@ -218,11 +228,11 @@ source: # todo: tag files in annex so we can run `git annex get --metadata fstags=makeinstall` (faster) # ------------------------------------------------- cd {{ self.source_path }} - git remote add datasrc https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/repo/annex.git + git remote add datasrc {{ self.annex }} git fetch datasrc # PW 2021/03/04: Werid.. this returns an error code, but things still work? # So hardwire the exit code to zero so `docker build` doesn't error out - # THis is because surfer.nmr now runs git-annex 8. todo: upgrade + # This is because surfer.nmr now runs git-annex 8(?) todo: upgrade git annex enableremote datasrc || true git annex get -q . # Compile and install @@ -231,3 +241,9 @@ source: cmake {{ self.cmake_opts }} {{ self.source_path }} make {{ self.make_opts }} make install + # If self.license_base64 populated, decode from base64 and write to self.license_path + # https://github.com/pwighton/fs-docker/blob/master/run/entrypoint.freesurfer-run.bash + # ------------------------------------------------- + if [ -n "{{ self.license_base64 }}" ]; then \ + echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}; \ + fi From c181b6569c07abc2cf8d2e81fd02bdd9757d2cc3 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 4 Mar 2021 23:27:13 +0000 Subject: [PATCH 11/89] [FIX] tweaks --- neurodocker/templates/freesurfer.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 6cfbb3cb..37e5f2e9 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -105,7 +105,8 @@ source: source_path: /tmp/freesurfer/freesurfer-{{ self.version }} pkg_path: /tmp/freesurfer/pkg install_path: /opt/freesurfer-{{ self.version }} - license_path: /opt/freesurfer/license.txt + # This one has to be quoted with \"s for some reason + license_path: "{{ self.install_path }}/license.txt" cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} make_opts: -j4 env: From db108aa813bac02165d6e145bc8db912cbb6bbcf Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 5 Mar 2021 22:17:25 +0000 Subject: [PATCH 12/89] [WIP] working on infant stream from source --- neurodocker/templates/freesurfer.yaml | 65 +++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 37e5f2e9..bfdce5fb 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -93,15 +93,23 @@ source: - version optional: license_base64: "" + # The subject dir is not in /opt, like everything else FreeSurfer-related + # Because if we want to run `neurodocker minify` we want to operate over the `/opt` + # dir and have the subjects dir excluded + subjects_dir: "/ext/fs-subjects" + infant_model_dir: "/opt/fs-infant-model" repo: https://github.com/freesurfer/freesurfer pkg_url: http://surfer.nmr.mgh.harvard.edu/pub/data/fspackages/prebuilt/centos7-packages.tar.gz annex: https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/repo/annex.git + # set to niftyreg.yaml default + niftyreg_path: /opt/niftyreg-master minimal: "ON" build_guis: "OFF" infant_module: "OFF" install_python_deps: "OFF" distribute_fspython: "OFF" fortran_libs: /usr/lib/gcc/x86_64-linux-gnu/5/libgfortran.so + # --- Probably don't have to change args below here (?) --- source_path: /tmp/freesurfer/freesurfer-{{ self.version }} pkg_path: /tmp/freesurfer/pkg install_path: /opt/freesurfer-{{ self.version }} @@ -111,21 +119,26 @@ source: make_opts: -j4 env: FS_LICENSE: "{{ self.license_path }}" + SUBJECTS_DIR: "{{ self.subjects_dir }}" + FS_INFANT_MODEL: "{{ self.infant_model_dir }}" # From https://github.com/freesurfer/freesurfer/blob/54018f7d6f620d6288b28f50e14a0a4ba421757c/Dockerfile#L20-L42 - # freesurfer env OS: "Linux" - PATH: "{{ self.install_path }}/bin:{{ self.install_path }}/fsfast/bin:{{ self.install_path }}/tktools:{{ self.install_path }}/mni/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + PATH: "{{ self.niftyreg_path }}/bin:{{ self.pkg_path }}/mni/current/bin:{{ self.install_path }}/bin:{{ self.install_path }}/fsfast/bin:{{ self.install_path }}/tktools:{{ self.install_path }}/mni/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" FREESURFER_HOME: "{{ self.install_path }}" FREESURFER: "{{ self.install_path }}" - SUBJECTS_DIR: "{{ self.install_path }}/subjects" + PERL5LIB: "{{ self.install_path }}/mni/share/perl5" LOCAL_DIR: "{{ self.install_path }}/local" FSFAST_HOME: "{{ self.install_path }}/fsfast" FMRI_ANALYSIS_DIR: "{{ self.install_path }}/fsfast" + FSF_OUTPUT_FORMAT: "nii.gz" FUNCTIONALS_DIR: "{{ self.install_path }}/sessions" # default freesurfer options FS_OVERRIDE: "0" FIX_VERTEX_AREA: "" - FSF_OUTPUT_FORMAT: "nii.gz" + # To supress a `Could not set locale` error when `mri_convert` is run in recon-all + FS_DISABLE_LANG: "1" + # Just in case: https://github.com/freesurfer/freesurfer/issues/776 + FS_TIME_ALLOW: "0" # mni environment requirements MINC_BIN_DIR: "{{ self.install_path }}/mni/bin" MINC_LIB_DIR: "{{ self.install_path }}/mni/lib" @@ -133,6 +146,23 @@ source: MNI_DATAPATH: "{{ self.install_path }}/mni/data" MNI_PERL5LIB: "{{ self.install_path }}/mni/share/perl5" PERL5LIB: "{{ self.install_path }}/mni/share/perl5" + # infant model setup, see: + # - https://github.com/pwighton/fs-docker/blob/f691232312e97f2ea87b8ae45bfd5c365e2bc91c/baby/dockerfile#L85 + # - https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/infant/freesurfer-linux-centos7_x86_64-infant.tar.gz + # This temporary and should migrate somewhere else + CNYBCH_TEMPLATE_SUBJECTS_DIR: "{{ self.infant_model_dir }}/CNYBCH" + CNYBCH_SUBJECTS: "Template1 Template2 Template3 Template4 Template5 Template6 Template7 Template8 Template9 Template10 Template11 Template12 Template13 Template14 Template15 Template16 Template17 Template18 Template19 Template20 Template21 Template22 Template23 Template24 Template25 Template26" + # age at MRI scan (mo) + CNYBCH_AGES: "9 7 6 5 18 12 0 0 3 8 10 10 18 4 2 14 3 16 0 12 0 15 5 17 16 0" + CNYBCH_GMWM_SUBJECTS: "Template5 Template6 Template8 Template10 Template13 Template18 Template20 Template22" + # mo + CNYBCH_GMWM_AGES: "18 12 0 8 18 16 12 15" + CNYBCH_NEONATES: "Template7 Template8 Template19 Template21 Template26" + # days (first two computed from weeks) + CNYBCH_NEONATEAGES: "3 4 1 2 4" + CNYBCH_AROUNDONE: "Template11 Template12 Template6 Template20 Template16" + #mo + CNYBCH_AROUNDONEAGES: "10 10 12 12 14" dependencies: apt: - ca-certificates @@ -143,6 +173,7 @@ source: - bc - binutils - bzip2 + - coreutils - curl - g++-4.8 - gcc-4.8-base @@ -185,15 +216,16 @@ source: # git config git config --global user.email "CI@example.com" git config --global user.name "CI" + # Following along with: + # - https://travis-ci.com/github/freesurfer/freesurfer/builds/219074161 # Compile with gcc 4.8 # -------------------- update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100 update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100 update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-4.8 100 - # FreeSurfer needs python > 3.6 (? todo: verify) + # FreeSurfer needs python > 3.5 # From: https://github.com/pwighton/fs-docker/blob/09cc6b0b6c0538cbb182b04435daf85c123a63d5/build/Dockerfile#L66 # Alternative is https://github.com/ReproNim/neurodocker/blob/master/neurodocker/templates/afni.yaml - # But it installs python 3.5 # ------------------------------------------------------------------------- curl -sSL --retry 5 https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz | tar -xz -C /tmp cd /tmp/Python-3.6.5 @@ -206,7 +238,7 @@ source: pip3 install -q --no-cache-dir wheel sync # FreeSurfer needs CMake 3.9 or higher - # Taken from: https://github.com/Rikorose/gcc-cmake/blob/master/gcc-4/Dockerfile + # From: https://github.com/Rikorose/gcc-cmake/blob/master/gcc-4/Dockerfile # ------------------------------------------------------------------------------ wget https://github.com/Kitware/CMake/releases/download/v3.19.0/cmake-3.19.0-Linux-x86_64.sh -q -O /tmp/cmake-install.sh chmod u+x /tmp/cmake-install.sh @@ -217,7 +249,7 @@ source: ln -s /usr/share/cmake-3.19.0/bin/cmake $CMAKE_BINPATH rm /tmp/cmake-install.sh # Compile FreeSurfer - # https://github.com/pwighton/fs-docker (todo: maybe a newer way?) + # https://github.com/pwighton/fs-docker # ---------------------------------------------------------------- mkdir -p {{ self.pkg_path }} mkdir -p {{ self.install_path }} @@ -233,7 +265,7 @@ source: git fetch datasrc # PW 2021/03/04: Werid.. this returns an error code, but things still work? # So hardwire the exit code to zero so `docker build` doesn't error out - # This is because surfer.nmr now runs git-annex 8(?) todo: upgrade + # This is because surfer.nmr now runs git-annex 8 todo: upgrade git annex enableremote datasrc || true git annex get -q . # Compile and install @@ -248,3 +280,18 @@ source: if [ -n "{{ self.license_base64 }}" ]; then \ echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}; \ fi + # Setup mount points + # ------------------------------------------------- + mkdir -p {{ self.subjects_dir }} + mkdir -p {{ self.infant_model_dir }} + # Install python reqs for infant module + # - https://github.com/pwighton/fs-docker/blob/f691232312e97f2ea87b8ae45bfd5c365e2bc91c/baby/entrypoint.freesurfer-run.bash#L9 + # - infant entrypoint is make executable here, but needs to be activates with `--entrypoint` + # ------------------------------------------------- + if [ "{{ self.infant_module }}" = "ON" ]; then \ + pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt; \ + mkdir -p {{ self.install_path }}/average/sscnn_skullstripping; \ + chmod +x {{ self.source_path }}/infant/docker/entrypoint.bash; \ + fi + # Todo: + # - delete the /tmp dir, but first ensure that it's not needed From 57bfc55310b207abae506907614c0d991094b12e Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 27 Apr 2021 11:18:08 -0400 Subject: [PATCH 13/89] [FIX] installs python reqs --- neurodocker/templates/freesurfer.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 37e5f2e9..87b7104f 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -242,6 +242,7 @@ source: cmake {{ self.cmake_opts }} {{ self.source_path }} make {{ self.make_opts }} make install + pip3 install -r {{ self.source_path }}/python/requirements.txt; # If self.license_base64 populated, decode from base64 and write to self.license_path # https://github.com/pwighton/fs-docker/blob/master/run/entrypoint.freesurfer-run.bash # ------------------------------------------------- From 716ffeb3122bd01bf2d0aad5b133ab2155fe1c3f Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 27 Apr 2021 11:25:36 -0400 Subject: [PATCH 14/89] [FIX] typo --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 87b7104f..9c1becc6 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -242,7 +242,7 @@ source: cmake {{ self.cmake_opts }} {{ self.source_path }} make {{ self.make_opts }} make install - pip3 install -r {{ self.source_path }}/python/requirements.txt; + pip3 install -r {{ self.source_path }}/python/requirements.txt # If self.license_base64 populated, decode from base64 and write to self.license_path # https://github.com/pwighton/fs-docker/blob/master/run/entrypoint.freesurfer-run.bash # ------------------------------------------------- From e764a3a131fd8a0ab53160356e516d267008fbfd Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Mon, 3 May 2021 18:35:59 -0400 Subject: [PATCH 15/89] [WIP] aws buildspec --- buildspec.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 buildspec.yaml diff --git a/buildspec.yaml b/buildspec.yaml new file mode 100644 index 00000000..dfe6a173 --- /dev/null +++ b/buildspec.yaml @@ -0,0 +1,25 @@ +# see https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html +version: 0.2 + +env: + + shell: bash + parameter-store: + DOCKERHUB_PASS_PARAM: /dockerhub-access-token-for-pwighton + +phases: + install: + on-failure: ABORT + commands: + - export DOCKERHUB_PASS=$DOCKERHUB_PASS_PARAM + - docker login --username pwighton --password $DOCKERHUB_PASS + + build: + on-failure: ABORT + commands: + - docker build -t pwighton/neurodocker . + +post_build: + on-failure: ABORT + commands: + - docker push pwighton/neurodocker:latest From 080f7edb1925ab961cf5351d2c958b28c2bd2443 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Mon, 3 May 2021 18:39:25 -0400 Subject: [PATCH 16/89] [WIP] fix --- buildspec.yaml => buildspec.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename buildspec.yaml => buildspec.yml (100%) diff --git a/buildspec.yaml b/buildspec.yml similarity index 100% rename from buildspec.yaml rename to buildspec.yml From 8044d6ece510e38db39fcf8425c3d3cf3902f2f7 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Mon, 3 May 2021 18:53:30 -0400 Subject: [PATCH 17/89] [WIP] fix --- buildspec.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index dfe6a173..afa8982f 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -6,20 +6,18 @@ env: shell: bash parameter-store: DOCKERHUB_PASS_PARAM: /dockerhub-access-token-for-pwighton - + phases: install: on-failure: ABORT commands: - export DOCKERHUB_PASS=$DOCKERHUB_PASS_PARAM - docker login --username pwighton --password $DOCKERHUB_PASS - build: on-failure: ABORT commands: - docker build -t pwighton/neurodocker . - -post_build: + post_build: on-failure: ABORT commands: - docker push pwighton/neurodocker:latest From 10a81b56d1665b7bd380e19f3f8b1c0ec40a1056 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 7 May 2021 14:47:14 -0400 Subject: [PATCH 18/89] [FIX] wrap around martinos specific niftireg install --- neurodocker/templates/freesurfer.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 9c1becc6..de7977c8 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -101,13 +101,14 @@ source: infant_module: "OFF" install_python_deps: "OFF" distribute_fspython: "OFF" + install_niftireg_martinos: "OFF" fortran_libs: /usr/lib/gcc/x86_64-linux-gnu/5/libgfortran.so source_path: /tmp/freesurfer/freesurfer-{{ self.version }} pkg_path: /tmp/freesurfer/pkg install_path: /opt/freesurfer-{{ self.version }} # This one has to be quoted with \"s for some reason license_path: "{{ self.install_path }}/license.txt" - cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} + cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} -DINSTALL_NIFTIREG_MARTINOS={{ self.install_niftireg_martinos }} make_opts: -j4 env: FS_LICENSE: "{{ self.license_path }}" @@ -225,6 +226,11 @@ source: mv {{ self.pkg_path }}/packages/* {{ self.pkg_path }} rm -rf {{ self.pkg_path }}/packages git clone --single-branch --branch {{ self.version }} {{ self.repo }} {{ self.source_path }} + # Compile + # ------------------------------------------------- + cd {{ self.install_path }} + cmake {{ self.cmake_opts }} {{ self.source_path }} + make {{ self.make_opts }} # Files needed to be downloaded from the annex before we can run `make install` # todo: tag files in annex so we can run `git annex get --metadata fstags=makeinstall` (faster) # ------------------------------------------------- @@ -236,11 +242,9 @@ source: # This is because surfer.nmr now runs git-annex 8(?) todo: upgrade git annex enableremote datasrc || true git annex get -q . - # Compile and install - # ------------------------------------------------- + # Install + # ------------------------------------------------- cd {{ self.install_path }} - cmake {{ self.cmake_opts }} {{ self.source_path }} - make {{ self.make_opts }} make install pip3 install -r {{ self.source_path }}/python/requirements.txt # If self.license_base64 populated, decode from base64 and write to self.license_path From 40fad5ef7dafd64da8f48e661e7aadd436efa837 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 7 May 2021 16:37:24 -0400 Subject: [PATCH 19/89] [FIX] var rename --- neurodocker/templates/freesurfer.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index de7977c8..3cf8eb7f 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -101,14 +101,14 @@ source: infant_module: "OFF" install_python_deps: "OFF" distribute_fspython: "OFF" - install_niftireg_martinos: "OFF" + martinos_build: "OFF" fortran_libs: /usr/lib/gcc/x86_64-linux-gnu/5/libgfortran.so source_path: /tmp/freesurfer/freesurfer-{{ self.version }} pkg_path: /tmp/freesurfer/pkg install_path: /opt/freesurfer-{{ self.version }} # This one has to be quoted with \"s for some reason license_path: "{{ self.install_path }}/license.txt" - cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} -DINSTALL_NIFTIREG_MARTINOS={{ self.install_niftireg_martinos }} + cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} -DMARTINOS_BUILD={{ self.martinos_build }} make_opts: -j4 env: FS_LICENSE: "{{ self.license_path }}" From 071f4800b974c7da142bf98a1475b388d5970526 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 7 May 2021 17:50:18 -0400 Subject: [PATCH 20/89] [WIP] adding unzip --- neurodocker/templates/freesurfer.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 3cf8eb7f..f21668d4 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -172,6 +172,7 @@ source: - sudo - tar - tcsh + - unzip - uuid-dev - vim-common - wget From a5625b44a0ccafacfc88c4926f23f714741cfb89 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 11 May 2021 17:26:27 -0400 Subject: [PATCH 21/89] [FIX] upgrating fs python env to 3.6.8 --- neurodocker/templates/freesurfer.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index f21668d4..07866c9b 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -192,18 +192,18 @@ source: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100 update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100 update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-4.8 100 - # FreeSurfer needs python > 3.6 (? todo: verify) + # FreeSurfer needs python v3.6.8 # From: https://github.com/pwighton/fs-docker/blob/09cc6b0b6c0538cbb182b04435daf85c123a63d5/build/Dockerfile#L66 # Alternative is https://github.com/ReproNim/neurodocker/blob/master/neurodocker/templates/afni.yaml # But it installs python 3.5 # ------------------------------------------------------------------------- - curl -sSL --retry 5 https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz | tar -xz -C /tmp - cd /tmp/Python-3.6.5 + curl -sSL --retry 5 https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz | tar -xz -C /tmp + cd /tmp/Python-3.6.8 ./configure make make install cd / - rm -rf /tmp/Python-3.6.5 + rm -rf /tmp/Python-3.6.8 pip3 install -q --no-cache-dir -U pip pip3 install -q --no-cache-dir wheel sync From 943679f508962dbf57ed5b1e5cd755123d50dbc8 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 13 May 2021 09:52:29 -0400 Subject: [PATCH 22/89] [WIP] install python env for non infant stream --- neurodocker/templates/freesurfer.yaml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 3560f7f6..d7a48598 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -279,7 +279,18 @@ source: # ------------------------------------------------- cd {{ self.install_path }} make install - pip3 install -r {{ self.source_path }}/python/requirements.txt + # Install python reqs + # For infant: + # - https://github.com/pwighton/fs-docker/blob/f691232312e97f2ea87b8ae45bfd5c365e2bc91c/baby/entrypoint.freesurfer-run.bash#L9 + # - infant entrypoint is made executable here, but needs to be activates with `--entrypoint` + if [ "{{ self.infant_module }}" = ON ] | [ "{{ self.infant_module }}" = on ]; then \ + pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt; \ + mkdir -p {{ self.install_path }}/average/sscnn_skullstripping; \ + chmod +x {{ self.source_path }}/infant/docker/entrypoint.bash; \ + else \ + pip3 install -r {{ self.source_path }}/python/requirements.txt; \ + pip3 install -r {{ self.source_path }}/python/requirements-extra.txt; \ + fi # If self.license_base64 populated, decode from base64 and write to self.license_path # https://github.com/pwighton/fs-docker/blob/master/run/entrypoint.freesurfer-run.bash # ------------------------------------------------- @@ -290,14 +301,5 @@ source: # ------------------------------------------------- mkdir -p {{ self.subjects_dir }} mkdir -p {{ self.infant_model_dir }} - # Install python reqs for infant module - # - https://github.com/pwighton/fs-docker/blob/f691232312e97f2ea87b8ae45bfd5c365e2bc91c/baby/entrypoint.freesurfer-run.bash#L9 - # - infant entrypoint is make executable here, but needs to be activates with `--entrypoint` - # ------------------------------------------------- - if [ "{{ self.infant_module }}" = "ON" ]; then \ - pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt; \ - mkdir -p {{ self.install_path }}/average/sscnn_skullstripping; \ - chmod +x {{ self.source_path }}/infant/docker/entrypoint.bash; \ - fi # Todo: # - delete the /tmp dir, but first ensure that it's not needed From 5a3f2e5f82ab3d9b724a26bdd1cc7ab33d1998d7 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 13 May 2021 09:54:55 -0400 Subject: [PATCH 23/89] [WIP] re-arrange cause git annex is slow --- neurodocker/templates/freesurfer.yaml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index d7a48598..774e9c8c 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -264,6 +264,19 @@ source: cd {{ self.install_path }} cmake {{ self.cmake_opts }} {{ self.source_path }} make {{ self.make_opts }} + # Install python reqs + # ------------------------------------------------- + # For infant: + # - https://github.com/pwighton/fs-docker/blob/f691232312e97f2ea87b8ae45bfd5c365e2bc91c/baby/entrypoint.freesurfer-run.bash#L9 + # - infant entrypoint is made executable here, but needs to be activates with `--entrypoint` + if [ "{{ self.infant_module }}" = ON ] | [ "{{ self.infant_module }}" = on ]; then \ + pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt; \ + mkdir -p {{ self.install_path }}/average/sscnn_skullstripping; \ + chmod +x {{ self.source_path }}/infant/docker/entrypoint.bash; \ + else \ + pip3 install -r {{ self.source_path }}/python/requirements.txt; \ + pip3 install -r {{ self.source_path }}/python/requirements-extra.txt; \ + fi # Files needed to be downloaded from the annex before we can run `make install` # todo: tag files in annex so we can run `git annex get --metadata fstags=makeinstall` (faster) # ------------------------------------------------- @@ -279,18 +292,6 @@ source: # ------------------------------------------------- cd {{ self.install_path }} make install - # Install python reqs - # For infant: - # - https://github.com/pwighton/fs-docker/blob/f691232312e97f2ea87b8ae45bfd5c365e2bc91c/baby/entrypoint.freesurfer-run.bash#L9 - # - infant entrypoint is made executable here, but needs to be activates with `--entrypoint` - if [ "{{ self.infant_module }}" = ON ] | [ "{{ self.infant_module }}" = on ]; then \ - pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt; \ - mkdir -p {{ self.install_path }}/average/sscnn_skullstripping; \ - chmod +x {{ self.source_path }}/infant/docker/entrypoint.bash; \ - else \ - pip3 install -r {{ self.source_path }}/python/requirements.txt; \ - pip3 install -r {{ self.source_path }}/python/requirements-extra.txt; \ - fi # If self.license_base64 populated, decode from base64 and write to self.license_path # https://github.com/pwighton/fs-docker/blob/master/run/entrypoint.freesurfer-run.bash # ------------------------------------------------- From ecc824fb9f5560ef8eb22c4aabdb0e3cb7d52e73 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 13 May 2021 12:29:53 -0400 Subject: [PATCH 24/89] [WIP] debug echos --- neurodocker/templates/freesurfer.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 774e9c8c..6b90e64e 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -258,6 +258,10 @@ source: wget -q -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} mv {{ self.pkg_path }}/packages/* {{ self.pkg_path }} rm -rf {{ self.pkg_path }}/packages + echo "Cloning..." + echo " remote: "{{ self.repo }} + echo " branch: "{{ self.repo }} + echo " destination: "{{ self.source_path }} git clone --single-branch --branch {{ self.version }} {{ self.repo }} {{ self.source_path }} # Compile # ------------------------------------------------- From 294f31e980a86c69c7734eeda0b3a55b4024f0dc Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 19 May 2021 22:24:27 -0400 Subject: [PATCH 25/89] [FIX] bash bugfix --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 6b90e64e..b0406c18 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -273,7 +273,7 @@ source: # For infant: # - https://github.com/pwighton/fs-docker/blob/f691232312e97f2ea87b8ae45bfd5c365e2bc91c/baby/entrypoint.freesurfer-run.bash#L9 # - infant entrypoint is made executable here, but needs to be activates with `--entrypoint` - if [ "{{ self.infant_module }}" = ON ] | [ "{{ self.infant_module }}" = on ]; then \ + if [ "{{ self.infant_module }}" = ON ] || [ "{{ self.infant_module }}" = on ]; then \ pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt; \ mkdir -p {{ self.install_path }}/average/sscnn_skullstripping; \ chmod +x {{ self.source_path }}/infant/docker/entrypoint.bash; \ From efcea8c2182c50eb0c884df50a66640510013a8c Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 20 May 2021 09:03:31 -0400 Subject: [PATCH 26/89] [WIP] --- neurodocker/templates/freesurfer.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index b0406c18..c87012b4 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -275,8 +275,7 @@ source: # - infant entrypoint is made executable here, but needs to be activates with `--entrypoint` if [ "{{ self.infant_module }}" = ON ] || [ "{{ self.infant_module }}" = on ]; then \ pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt; \ - mkdir -p {{ self.install_path }}/average/sscnn_skullstripping; \ - chmod +x {{ self.source_path }}/infant/docker/entrypoint.bash; \ + ln -s {{ self.install_path }}/bin/infant-container-entrypoint.bash /bin/infant-container-entrypoint.bash else \ pip3 install -r {{ self.source_path }}/python/requirements.txt; \ pip3 install -r {{ self.source_path }}/python/requirements-extra.txt; \ From c6f5fd3914554e3ca22cd184da29d7e42933747a Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 20 May 2021 09:21:13 -0400 Subject: [PATCH 27/89] [WIP] syntax --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index c87012b4..a77c5043 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -275,7 +275,7 @@ source: # - infant entrypoint is made executable here, but needs to be activates with `--entrypoint` if [ "{{ self.infant_module }}" = ON ] || [ "{{ self.infant_module }}" = on ]; then \ pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt; \ - ln -s {{ self.install_path }}/bin/infant-container-entrypoint.bash /bin/infant-container-entrypoint.bash + ln -s {{ self.install_path }}/bin/infant-container-entrypoint.bash /bin/infant-container-entrypoint.bash; \ else \ pip3 install -r {{ self.source_path }}/python/requirements.txt; \ pip3 install -r {{ self.source_path }}/python/requirements-extra.txt; \ From 16a2378497644dd3b7e79d479d132f2a30da20cf Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 26 May 2021 15:54:47 -0400 Subject: [PATCH 28/89] [FIX] to support samseg vis --- neurodocker/templates/freesurfer.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index a77c5043..bf9723a4 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -183,6 +183,7 @@ source: - libbz2-dev - libfreetype6-dev - libgfortran-4.8-dev + - libglib2.0-0 - libglu1-mesa-dev - libgomp1 - libjpeg62-dev @@ -260,7 +261,7 @@ source: rm -rf {{ self.pkg_path }}/packages echo "Cloning..." echo " remote: "{{ self.repo }} - echo " branch: "{{ self.repo }} + echo " branch: "{{ self.version }} echo " destination: "{{ self.source_path }} git clone --single-branch --branch {{ self.version }} {{ self.repo }} {{ self.source_path }} # Compile @@ -272,7 +273,6 @@ source: # ------------------------------------------------- # For infant: # - https://github.com/pwighton/fs-docker/blob/f691232312e97f2ea87b8ae45bfd5c365e2bc91c/baby/entrypoint.freesurfer-run.bash#L9 - # - infant entrypoint is made executable here, but needs to be activates with `--entrypoint` if [ "{{ self.infant_module }}" = ON ] || [ "{{ self.infant_module }}" = on ]; then \ pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt; \ ln -s {{ self.install_path }}/bin/infant-container-entrypoint.bash /bin/infant-container-entrypoint.bash; \ From 05701e1dcc539ec5b5ca3f7b97e2a4014498a94b Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 8 Jun 2021 17:27:51 -0400 Subject: [PATCH 29/89] [WIP] to support samseg atlas building utils --- neurodocker/templates/freesurfer.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index bf9723a4..907229f8 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -109,6 +109,7 @@ source: install_python_deps: "OFF" distribute_fspython: "OFF" martinos_build: "OFF" + samseg_atlas_build: "OFF" fortran_libs: /usr/lib/gcc/x86_64-linux-gnu/5/libgfortran.so # --- Probably don't have to change args below here (?) --- source_path: /tmp/freesurfer/freesurfer-{{ self.version }} @@ -301,6 +302,15 @@ source: if [ -n "{{ self.license_base64 }}" ]; then \ echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}; \ fi + # Compile some utils to support samseg atlas creation, if enabled (see `fs_build_GEMS2`) + # ---------------------------------------------------- + if [ "{{ self.samseg_atlas_build }}" = ON ] || [ "{{ self.samseg_atlas_build }}" = on ]; then \ + ln -s /usr/local/bin/python3 /usr/local/bin/python; \ + {{ self.source_path }}/packages/build_packages.py --only-itk {{ self.install_path }}/packages; \ + cd {{ self.source_path }}/gems; \ + cmake . -G "Unix Makefiles" -DBUILD_CUDA=OFF -DBUILD_EXECUTABLES=OFF -DBUILD_GUI=OFF -DBUILD_MATLAB=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF -DITK_DIR=/opt/freesurfer-20210513-fs-infant-dev-merge/packages/itk/4.13.0/src/build -DGFORTRAN_LIBRARIES=/usr/lib/gcc/x86_64-linux-gnu/5/libgfortran.so -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_CXX_COMPILER_AR=/usr/bin/ar -DCMAKE_CXX_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_CXX_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_C_COMPILER_AR=/usr/bin/ar -DCMAKE_C_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_C_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_INSTALL_PREFIX=.; \ + make {{ self.make_opts }} + fi # Setup mount points # ------------------------------------------------- mkdir -p {{ self.subjects_dir }} From b572b5b30c8340dd44d74fd7cfe08c0dac74f636 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 8 Jun 2021 17:29:33 -0400 Subject: [PATCH 30/89] [WIP] re-arrange --- neurodocker/templates/freesurfer.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 907229f8..ceeec66e 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -280,7 +280,16 @@ source: else \ pip3 install -r {{ self.source_path }}/python/requirements.txt; \ pip3 install -r {{ self.source_path }}/python/requirements-extra.txt; \ - fi + fi + # Compile some utils to support samseg atlas creation, if enabled (see `fs_build_GEMS2`) + # ---------------------------------------------------- + if [ "{{ self.samseg_atlas_build }}" = ON ] || [ "{{ self.samseg_atlas_build }}" = on ]; then \ + ln -s /usr/local/bin/python3 /usr/local/bin/python; \ + {{ self.source_path }}/packages/build_packages.py --only-itk {{ self.install_path }}/packages; \ + cd {{ self.source_path }}/gems; \ + cmake . -G "Unix Makefiles" -DBUILD_CUDA=OFF -DBUILD_EXECUTABLES=OFF -DBUILD_GUI=OFF -DBUILD_MATLAB=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF -DITK_DIR=/opt/freesurfer-20210513-fs-infant-dev-merge/packages/itk/4.13.0/src/build -DGFORTRAN_LIBRARIES=/usr/lib/gcc/x86_64-linux-gnu/5/libgfortran.so -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_CXX_COMPILER_AR=/usr/bin/ar -DCMAKE_CXX_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_CXX_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_C_COMPILER_AR=/usr/bin/ar -DCMAKE_C_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_C_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_INSTALL_PREFIX=.; \ + make {{ self.make_opts }}; \ + fi # Files needed to be downloaded from the annex before we can run `make install` # todo: tag files in annex so we can run `git annex get --metadata fstags=makeinstall` (faster) # ------------------------------------------------- @@ -302,15 +311,6 @@ source: if [ -n "{{ self.license_base64 }}" ]; then \ echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}; \ fi - # Compile some utils to support samseg atlas creation, if enabled (see `fs_build_GEMS2`) - # ---------------------------------------------------- - if [ "{{ self.samseg_atlas_build }}" = ON ] || [ "{{ self.samseg_atlas_build }}" = on ]; then \ - ln -s /usr/local/bin/python3 /usr/local/bin/python; \ - {{ self.source_path }}/packages/build_packages.py --only-itk {{ self.install_path }}/packages; \ - cd {{ self.source_path }}/gems; \ - cmake . -G "Unix Makefiles" -DBUILD_CUDA=OFF -DBUILD_EXECUTABLES=OFF -DBUILD_GUI=OFF -DBUILD_MATLAB=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF -DITK_DIR=/opt/freesurfer-20210513-fs-infant-dev-merge/packages/itk/4.13.0/src/build -DGFORTRAN_LIBRARIES=/usr/lib/gcc/x86_64-linux-gnu/5/libgfortran.so -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_CXX_COMPILER_AR=/usr/bin/ar -DCMAKE_CXX_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_CXX_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_C_COMPILER_AR=/usr/bin/ar -DCMAKE_C_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_C_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_INSTALL_PREFIX=.; \ - make {{ self.make_opts }} - fi # Setup mount points # ------------------------------------------------- mkdir -p {{ self.subjects_dir }} From 955b927dd2086441a31c6fc74d2a190e2cee0cd4 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 8 Jun 2021 18:44:09 -0400 Subject: [PATCH 31/89] [WIP] samseg atlas utils --- neurodocker/templates/freesurfer.yaml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index ceeec66e..50bd0423 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -281,15 +281,6 @@ source: pip3 install -r {{ self.source_path }}/python/requirements.txt; \ pip3 install -r {{ self.source_path }}/python/requirements-extra.txt; \ fi - # Compile some utils to support samseg atlas creation, if enabled (see `fs_build_GEMS2`) - # ---------------------------------------------------- - if [ "{{ self.samseg_atlas_build }}" = ON ] || [ "{{ self.samseg_atlas_build }}" = on ]; then \ - ln -s /usr/local/bin/python3 /usr/local/bin/python; \ - {{ self.source_path }}/packages/build_packages.py --only-itk {{ self.install_path }}/packages; \ - cd {{ self.source_path }}/gems; \ - cmake . -G "Unix Makefiles" -DBUILD_CUDA=OFF -DBUILD_EXECUTABLES=OFF -DBUILD_GUI=OFF -DBUILD_MATLAB=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF -DITK_DIR=/opt/freesurfer-20210513-fs-infant-dev-merge/packages/itk/4.13.0/src/build -DGFORTRAN_LIBRARIES=/usr/lib/gcc/x86_64-linux-gnu/5/libgfortran.so -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_CXX_COMPILER_AR=/usr/bin/ar -DCMAKE_CXX_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_CXX_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_C_COMPILER_AR=/usr/bin/ar -DCMAKE_C_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_C_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_INSTALL_PREFIX=.; \ - make {{ self.make_opts }}; \ - fi # Files needed to be downloaded from the annex before we can run `make install` # todo: tag files in annex so we can run `git annex get --metadata fstags=makeinstall` (faster) # ------------------------------------------------- @@ -305,6 +296,16 @@ source: # ------------------------------------------------- cd {{ self.install_path }} make install + # Compile some utils to support samseg atlas creation, if enabled (see `fs_build_GEMS2`) + # (needs to be run after git annex get because it depends on `packages/source/itk-4.13.0.tar.gz` + # ---------------------------------------------------- + if [ "{{ self.samseg_atlas_build }}" = ON ] || [ "{{ self.samseg_atlas_build }}" = on ]; then \ + ln -s /usr/local/bin/python3 /usr/local/bin/python; \ + {{ self.source_path }}/packages/build_packages.py --only-itk {{ self.install_path }}/packages; \ + cd {{ self.source_path }}/gems; \ + cmake . -G "Unix Makefiles" -DBUILD_CUDA=OFF -DBUILD_EXECUTABLES=OFF -DBUILD_GUI=OFF -DBUILD_MATLAB=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF -DITK_DIR={{ self.install_path }}/packages/itk/4.13.0/src/build -DGFORTRAN_LIBRARIES=/usr/lib/gcc/x86_64-linux-gnu/5/libgfortran.so -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_CXX_COMPILER_AR=/usr/bin/ar -DCMAKE_CXX_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_CXX_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_C_COMPILER_AR=/usr/bin/ar -DCMAKE_C_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_C_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_INSTALL_PREFIX=.; \ + make {{ self.make_opts }}; \ + fi # If self.license_base64 populated, decode from base64 and write to self.license_path # https://github.com/pwighton/fs-docker/blob/master/run/entrypoint.freesurfer-run.bash # ------------------------------------------------- From 3922ff74e1eac1d9f3430a4fd00f785ba1fad3b4 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 15 Jun 2021 17:27:57 -0400 Subject: [PATCH 32/89] [FIX] unlocking git annex before `make install` --- neurodocker/templates/freesurfer.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 50bd0423..ef2c7ebd 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -292,6 +292,7 @@ source: # This is because surfer.nmr now runs git-annex 8 todo: upgrade git annex enableremote datasrc || true git annex get -q . + git annex unlock # Install # ------------------------------------------------- cd {{ self.install_path }} From 216af347c3d1f857e3ad3858d9b01eb0fe5e6449 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 23 Jun 2021 12:08:02 -0400 Subject: [PATCH 33/89] [WIP] using -DGEMS_BUILD_EXECUTABLES to build samseg atlas tools --- neurodocker/templates/freesurfer.yaml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index ef2c7ebd..cf011d41 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -117,7 +117,7 @@ source: install_path: /opt/freesurfer-{{ self.version }} # This one has to be quoted with \"s for some reason license_path: "{{ self.install_path }}/license.txt" - cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} -DMARTINOS_BUILD={{ self.martinos_build }} + cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} -DMARTINOS_BUILD={{ self.martinos_build }} -DGEMS_BUILD_EXECUTABLES={{ self.samseg_atlas_build }} make_opts: -j4 env: FS_LICENSE: "{{ self.license_path }}" @@ -297,16 +297,6 @@ source: # ------------------------------------------------- cd {{ self.install_path }} make install - # Compile some utils to support samseg atlas creation, if enabled (see `fs_build_GEMS2`) - # (needs to be run after git annex get because it depends on `packages/source/itk-4.13.0.tar.gz` - # ---------------------------------------------------- - if [ "{{ self.samseg_atlas_build }}" = ON ] || [ "{{ self.samseg_atlas_build }}" = on ]; then \ - ln -s /usr/local/bin/python3 /usr/local/bin/python; \ - {{ self.source_path }}/packages/build_packages.py --only-itk {{ self.install_path }}/packages; \ - cd {{ self.source_path }}/gems; \ - cmake . -G "Unix Makefiles" -DBUILD_CUDA=OFF -DBUILD_EXECUTABLES=OFF -DBUILD_GUI=OFF -DBUILD_MATLAB=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF -DITK_DIR={{ self.install_path }}/packages/itk/4.13.0/src/build -DGFORTRAN_LIBRARIES=/usr/lib/gcc/x86_64-linux-gnu/5/libgfortran.so -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_CXX_COMPILER_AR=/usr/bin/ar -DCMAKE_CXX_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_CXX_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_C_COMPILER_AR=/usr/bin/ar -DCMAKE_C_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_C_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_INSTALL_PREFIX=.; \ - make {{ self.make_opts }}; \ - fi # If self.license_base64 populated, decode from base64 and write to self.license_path # https://github.com/pwighton/fs-docker/blob/master/run/entrypoint.freesurfer-run.bash # ------------------------------------------------- From 8d6abc0f199c1c9034356723fd0c20c19436d49f Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 23 Jun 2021 12:29:45 -0400 Subject: [PATCH 34/89] [WIP] test.. --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index cf011d41..05000391 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -117,7 +117,7 @@ source: install_path: /opt/freesurfer-{{ self.version }} # This one has to be quoted with \"s for some reason license_path: "{{ self.install_path }}/license.txt" - cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} -DMARTINOS_BUILD={{ self.martinos_build }} -DGEMS_BUILD_EXECUTABLES={{ self.samseg_atlas_build }} + cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} -DMARTINOS_BUILD={{ self.martinos_build }} -DGEMS_BUILD_EXECUTABLES={{ self.samseg_atlas_build }} -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_CXX_COMPILER_AR=/usr/bin/ar -DCMAKE_CXX_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_CXX_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_C_COMPILER_AR=/usr/bin/ar -DCMAKE_C_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_C_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" make_opts: -j4 env: FS_LICENSE: "{{ self.license_path }}" From 19d46788407f571d636e358def0790f0335a902c Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 2 Jul 2021 15:46:08 -0400 Subject: [PATCH 35/89] [FIX] moving default fs pkg dir, so /tmp can safely be removed --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 05000391..3278c3d2 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -113,7 +113,7 @@ source: fortran_libs: /usr/lib/gcc/x86_64-linux-gnu/5/libgfortran.so # --- Probably don't have to change args below here (?) --- source_path: /tmp/freesurfer/freesurfer-{{ self.version }} - pkg_path: /tmp/freesurfer/pkg + pkg_path: /opt/freesurfer-pkg install_path: /opt/freesurfer-{{ self.version }} # This one has to be quoted with \"s for some reason license_path: "{{ self.install_path }}/license.txt" From 6d672ba170c50ba629b18d073321e878b562f68d Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 7 Jul 2021 16:17:06 -0400 Subject: [PATCH 36/89] [FIX] adds pythonpath env var for fs packages --- neurodocker/templates/freesurfer.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 3278c3d2..b3ed3d06 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -134,6 +134,7 @@ source: FMRI_ANALYSIS_DIR: "{{ self.install_path }}/fsfast" FSF_OUTPUT_FORMAT: "nii.gz" FUNCTIONALS_DIR: "{{ self.install_path }}/sessions" + PYTHONPATH: "{{ self.install_path }}/python/packages" # default freesurfer options FS_OVERRIDE: "0" FIX_VERTEX_AREA: "" From 72a9c8c4d3bbc064c30ac596d43aec2c84155420 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 13 Jul 2021 10:12:47 -0400 Subject: [PATCH 37/89] [WIP] the beginnings of dev options --- neurodocker/templates/freesurfer.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index b3ed3d06..67aa9206 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -191,6 +191,7 @@ source: - libjpeg62-dev - libopenblas-dev - libssl-dev + - libsqlite3-dev - libtool - libtool-bin - libx11-dev @@ -304,6 +305,9 @@ source: if [ -n "{{ self.license_base64 }}" ]; then \ echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}; \ fi + ## *** Dev options; this should become it's own variable + pip3 install jupyter + ## *** End dev options # Setup mount points # ------------------------------------------------- mkdir -p {{ self.subjects_dir }} From 358f9d84fc1a39c7bee1997113e24e1cc4f09ce7 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 13 Jul 2021 17:59:08 -0400 Subject: [PATCH 38/89] [FIX] fs path stomp fix --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 67aa9206..fc9be3a9 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -53,7 +53,7 @@ binaries: # From https://github.com/freesurfer/freesurfer/blob/54018f7d6f620d6288b28f50e14a0a4ba421757c/Dockerfile#L20-L42 # freesurfer env OS: "Linux" - PATH: "{{ self.install_path }}/bin:{{ self.install_path }}/fsfast/bin:{{ self.install_path }}/tktools:{{ self.install_path }}/mni/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + PATH: "{{ self.install_path }}/bin:{{ self.install_path }}/fsfast/bin:{{ self.install_path }}/tktools:{{ self.install_path }}/mni/bin:$PATH" FREESURFER_HOME: "{{ self.install_path }}" FREESURFER: "{{ self.install_path }}" SUBJECTS_DIR: "{{ self.install_path }}/subjects" From 04d1a8a1fc0fd71adc01ebef7abfb06e6779712a Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 13 Jul 2021 18:24:22 -0400 Subject: [PATCH 39/89] [FIX] path stomp bug take 2: electric boogaloo --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index fc9be3a9..d0fed3fe 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -125,7 +125,7 @@ source: FS_INFANT_MODEL: "{{ self.infant_model_dir }}" # From https://github.com/freesurfer/freesurfer/blob/54018f7d6f620d6288b28f50e14a0a4ba421757c/Dockerfile#L20-L42 OS: "Linux" - PATH: "{{ self.niftyreg_path }}/bin:{{ self.pkg_path }}/mni/current/bin:{{ self.install_path }}/bin:{{ self.install_path }}/fsfast/bin:{{ self.install_path }}/tktools:{{ self.install_path }}/mni/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + PATH: "{{ self.niftyreg_path }}/bin:{{ self.pkg_path }}/mni/current/bin:{{ self.install_path }}/bin:{{ self.install_path }}/fsfast/bin:{{ self.install_path }}/tktools:{{ self.install_path }}/mni/bin:$PATH" FREESURFER_HOME: "{{ self.install_path }}" FREESURFER: "{{ self.install_path }}" PERL5LIB: "{{ self.install_path }}/mni/share/perl5" From 3a1d539d5741eaf1f12ea5501e24aa04ac904468 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 11 Aug 2021 11:57:52 -0400 Subject: [PATCH 40/89] [WIP] tweaks for jupyter --- neurodocker/templates/freesurfer.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index d0fed3fe..d1039a86 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -191,7 +191,7 @@ source: - libjpeg62-dev - libopenblas-dev - libssl-dev - - libsqlite3-dev + - libsqlite3 - libtool - libtool-bin - libx11-dev @@ -306,6 +306,7 @@ source: echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}; \ fi ## *** Dev options; this should become it's own variable + ## Note jupyter relies on `libsqlite3` being present before python is compiled pip3 install jupyter ## *** End dev options # Setup mount points From e728e7a8c89a12392e74177d7d4c7c5d188b0003 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 13 Aug 2021 12:11:11 -0400 Subject: [PATCH 41/89] [WIP] prereqs for jupyter --- neurodocker/templates/freesurfer.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index d1039a86..99749204 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -191,7 +191,7 @@ source: - libjpeg62-dev - libopenblas-dev - libssl-dev - - libsqlite3 + - libsqlite3-dev - libtool - libtool-bin - libx11-dev @@ -306,7 +306,7 @@ source: echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}; \ fi ## *** Dev options; this should become it's own variable - ## Note jupyter relies on `libsqlite3` being present before python is compiled + ## Note jupyter relies on `libsqlite3-dev` being present before python is compiled pip3 install jupyter ## *** End dev options # Setup mount points From ba3eb9971fe2154d3527516968ce0f4d79e8c03a Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 13 Aug 2021 17:43:02 -0400 Subject: [PATCH 42/89] [WIP] New Params - git_annex_get: "ON" - dev_tools: "OFF" - cxx_flags: "-msse2 -mfpmath=sse -fPIC -fpermissive" - c_flags: "-msse2 -mfpmath=sse -fPIC -fpermissive" - `make install` not run if `git_annex_get` is `OFF`. - `make install` will run in the gems dir if `samseg_atlas_build` is `ON` - even if `git_annex_get` is `OFF` --- neurodocker/templates/freesurfer.yaml | 50 +++++++++++++++++---------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 99749204..47a94352 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -110,6 +110,8 @@ source: distribute_fspython: "OFF" martinos_build: "OFF" samseg_atlas_build: "OFF" + git_annex_get: "ON" + dev_tools: "OFF" fortran_libs: /usr/lib/gcc/x86_64-linux-gnu/5/libgfortran.so # --- Probably don't have to change args below here (?) --- source_path: /tmp/freesurfer/freesurfer-{{ self.version }} @@ -117,7 +119,9 @@ source: install_path: /opt/freesurfer-{{ self.version }} # This one has to be quoted with \"s for some reason license_path: "{{ self.install_path }}/license.txt" - cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} -DMARTINOS_BUILD={{ self.martinos_build }} -DGEMS_BUILD_EXECUTABLES={{ self.samseg_atlas_build }} -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_CXX_COMPILER_AR=/usr/bin/ar -DCMAKE_CXX_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_CXX_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_C_COMPILER_AR=/usr/bin/ar -DCMAKE_C_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_C_FLAGS="-msse2 -mfpmath=sse -fPIC -fpermissive" + cxx_flags: "-msse2 -mfpmath=sse -fPIC -fpermissive" + c_flags: "-msse2 -mfpmath=sse -fPIC -fpermissive" + cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} -DMARTINOS_BUILD={{ self.martinos_build }} -DGEMS_BUILD_EXECUTABLES={{ self.samseg_atlas_build }} -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_CXX_COMPILER_AR=/usr/bin/ar -DCMAKE_CXX_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_CXX_FLAGS="{{ self.cxx_flags }}" -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_C_COMPILER_AR=/usr/bin/ar -DCMAKE_C_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_C_FLAGS="{{ self.c_flags }}" make_opts: -j4 env: FS_LICENSE: "{{ self.license_path }}" @@ -281,34 +285,44 @@ source: ln -s {{ self.install_path }}/bin/infant-container-entrypoint.bash /bin/infant-container-entrypoint.bash; \ else \ pip3 install -r {{ self.source_path }}/python/requirements.txt; \ - pip3 install -r {{ self.source_path }}/python/requirements-extra.txt; \ fi + # PW 2021/08/13: moving comments outside of `git_annex_get` if block + # PW 2021/03/04: Weird.. this (`git annex enableremote datasrc`) returns an + # error code, but things still work? + # So hardwire the exit code to zero so `docker build` doesn't error out + # This is because surfer.nmr now runs git-annex 8? todo: upgrade # Files needed to be downloaded from the annex before we can run `make install` # todo: tag files in annex so we can run `git annex get --metadata fstags=makeinstall` (faster) # ------------------------------------------------- - cd {{ self.source_path }} - git remote add datasrc {{ self.annex }} - git fetch datasrc - # PW 2021/03/04: Werid.. this returns an error code, but things still work? - # So hardwire the exit code to zero so `docker build` doesn't error out - # This is because surfer.nmr now runs git-annex 8 todo: upgrade - git annex enableremote datasrc || true - git annex get -q . - git annex unlock - # Install - # ------------------------------------------------- - cd {{ self.install_path }} - make install + if [ "{{ self.git_annex_get }}" = ON ] || [ "{{ self.git_annex_get }}" = on ]; then \ + cd {{ self.source_path }}; \ + git remote add datasrc {{ self.annex }}; \ + git fetch datasrc; \ + git annex enableremote datasrc || true; \ + git annex get -q .; \ + git annex unlock; \ + cd {{ self.install_path }}; \ + make install; \ + fi + # Since the gems subdir doesn't rely on the annex remote to execute `make install`, + # If were building for samseg atlases, run make install in the gems subdir + # This lets us build the container with git_annex_get=off, and still have gems binaries installed + if [ "{{ self.samseg_atlas_build }}" = ON ] || [ "{{ self.samseg_atlas_build }}" = on ]; then \ + cd {{ self.source_path }}/gems; \ + make install; \ + fi # If self.license_base64 populated, decode from base64 and write to self.license_path # https://github.com/pwighton/fs-docker/blob/master/run/entrypoint.freesurfer-run.bash # ------------------------------------------------- if [ -n "{{ self.license_base64 }}" ]; then \ echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}; \ fi - ## *** Dev options; this should become it's own variable + ## *** Dev tools ** ## Note jupyter relies on `libsqlite3-dev` being present before python is compiled - pip3 install jupyter - ## *** End dev options + if [ "{{ self.dev_tools }}" = ON ] || [ "{{ self.dev_tools }}" = on ]; then \ + pip3 install jupyter; \ + pip3 install -r {{ self.source_path }}/python/requirements-extra.txt; \ + fi # Setup mount points # ------------------------------------------------- mkdir -p {{ self.subjects_dir }} From 396248ca76dbf1076f194b4503b8bfef60322583 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 25 Aug 2021 18:26:13 -0400 Subject: [PATCH 43/89] [WIP] refactor to use jinja conditionals --- neurodocker/templates/freesurfer.yaml | 114 ++++++++++++-------------- 1 file changed, 52 insertions(+), 62 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 47a94352..5b3aded1 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -120,7 +120,7 @@ source: # This one has to be quoted with \"s for some reason license_path: "{{ self.install_path }}/license.txt" cxx_flags: "-msse2 -mfpmath=sse -fPIC -fpermissive" - c_flags: "-msse2 -mfpmath=sse -fPIC -fpermissive" + c_flags: "-msse2 -mfpmath=sse -fPIC" cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} -DMARTINOS_BUILD={{ self.martinos_build }} -DGEMS_BUILD_EXECUTABLES={{ self.samseg_atlas_build }} -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_CXX_COMPILER_AR=/usr/bin/ar -DCMAKE_CXX_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_CXX_FLAGS="{{ self.cxx_flags }}" -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_C_COMPILER_AR=/usr/bin/ar -DCMAKE_C_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_C_FLAGS="{{ self.c_flags }}" make_opts: -j4 env: @@ -224,18 +224,15 @@ source: instructions: | {{ self.install_dependencies() }} # git config + # ------------------------------------------------------------------------- git config --global user.email "CI@example.com" git config --global user.name "CI" - # Following along with: - # - https://travis-ci.com/github/freesurfer/freesurfer/builds/219074161 - # Compile with gcc 4.8 - # -------------------- + # Switch to gcc 4.8 + # ------------------------------------------------------------------------- update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100 update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100 update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-4.8 100 - # FreeSurfer needs python v3.6.8 - # From: https://github.com/pwighton/fs-docker/blob/09cc6b0b6c0538cbb182b04435daf85c123a63d5/build/Dockerfile#L66 - # Alternative is https://github.com/ReproNim/neurodocker/blob/master/neurodocker/templates/afni.yaml + # Python v3.6.8 # ------------------------------------------------------------------------- curl -sSL --retry 5 https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz | tar -xz -C /tmp cd /tmp/Python-3.6.8 @@ -247,9 +244,8 @@ source: pip3 install -q --no-cache-dir -U pip pip3 install -q --no-cache-dir wheel sync - # FreeSurfer needs CMake 3.9 or higher - # From: https://github.com/Rikorose/gcc-cmake/blob/master/gcc-4/Dockerfile - # ------------------------------------------------------------------------------ + # Cmake v3.19.0 (must be > v3.6.8) + # ------------------------------------------------------------------------- wget https://github.com/Kitware/CMake/releases/download/v3.19.0/cmake-3.19.0-Linux-x86_64.sh -q -O /tmp/cmake-install.sh chmod u+x /tmp/cmake-install.sh mkdir -p /usr/share/cmake-3.19.0 @@ -257,9 +253,8 @@ source: CMAKE_BINPATH=`which cmake` rm -rf $CMAKE_BINPATH ln -s /usr/share/cmake-3.19.0/bin/cmake $CMAKE_BINPATH - rm /tmp/cmake-install.sh + rm /tmp/cmake-install.sh # Compile FreeSurfer - # https://github.com/pwighton/fs-docker # ---------------------------------------------------------------- mkdir -p {{ self.pkg_path }} mkdir -p {{ self.install_path }} @@ -271,61 +266,56 @@ source: echo " branch: "{{ self.version }} echo " destination: "{{ self.source_path }} git clone --single-branch --branch {{ self.version }} {{ self.repo }} {{ self.source_path }} - # Compile - # ------------------------------------------------- cd {{ self.install_path }} cmake {{ self.cmake_opts }} {{ self.source_path }} make {{ self.make_opts }} # Install python reqs - # ------------------------------------------------- - # For infant: - # - https://github.com/pwighton/fs-docker/blob/f691232312e97f2ea87b8ae45bfd5c365e2bc91c/baby/entrypoint.freesurfer-run.bash#L9 - if [ "{{ self.infant_module }}" = ON ] || [ "{{ self.infant_module }}" = on ]; then \ - pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt; \ - ln -s {{ self.install_path }}/bin/infant-container-entrypoint.bash /bin/infant-container-entrypoint.bash; \ - else \ - pip3 install -r {{ self.source_path }}/python/requirements.txt; \ - fi - # PW 2021/08/13: moving comments outside of `git_annex_get` if block - # PW 2021/03/04: Weird.. this (`git annex enableremote datasrc`) returns an - # error code, but things still work? - # So hardwire the exit code to zero so `docker build` doesn't error out - # This is because surfer.nmr now runs git-annex 8? todo: upgrade - # Files needed to be downloaded from the annex before we can run `make install` - # todo: tag files in annex so we can run `git annex get --metadata fstags=makeinstall` (faster) - # ------------------------------------------------- - if [ "{{ self.git_annex_get }}" = ON ] || [ "{{ self.git_annex_get }}" = on ]; then \ - cd {{ self.source_path }}; \ - git remote add datasrc {{ self.annex }}; \ - git fetch datasrc; \ - git annex enableremote datasrc || true; \ - git annex get -q .; \ - git annex unlock; \ - cd {{ self.install_path }}; \ - make install; \ - fi - # Since the gems subdir doesn't rely on the annex remote to execute `make install`, + # ---------------------------------------------------------------- + {%- if self.infant_module.lower() in ["on", "true", "1", "y"] %} + pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt + ln -s {{ self.install_path }}/bin/infant-container-entrypoint.bash /bin/infant-container-entrypoint.bash + {% else %} + pip3 install -r {{ self.source_path }}/python/requirements.txt + {% endif -%} + # Setup annex remote + # ---------------------------------------------------------------- + cd {{ self.source_path }} + git remote add datasrc {{ self.annex }} + git fetch datasrc + git annex enableremote datasrc || true + # Pull from annex and install + # (`make install` will fail if `git annex get` isn't run) + # ---------------------------------------------------------------- + {%- if self.git_annex_get.lower() in ["on", "true", "1", "y"] %} + git annex get -q . + git annex unlock + cd {{ self.install_path }} + make install + {% endif -%} # If were building for samseg atlases, run make install in the gems subdir + # Since the gems subdir doesn't rely on the annex remote to execute `make install`, # This lets us build the container with git_annex_get=off, and still have gems binaries installed - if [ "{{ self.samseg_atlas_build }}" = ON ] || [ "{{ self.samseg_atlas_build }}" = on ]; then \ - cd {{ self.source_path }}/gems; \ - make install; \ - fi - # If self.license_base64 populated, decode from base64 and write to self.license_path - # https://github.com/pwighton/fs-docker/blob/master/run/entrypoint.freesurfer-run.bash - # ------------------------------------------------- - if [ -n "{{ self.license_base64 }}" ]; then \ - echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}; \ - fi - ## *** Dev tools ** - ## Note jupyter relies on `libsqlite3-dev` being present before python is compiled - if [ "{{ self.dev_tools }}" = ON ] || [ "{{ self.dev_tools }}" = on ]; then \ - pip3 install jupyter; \ - pip3 install -r {{ self.source_path }}/python/requirements-extra.txt; \ - fi + # ---------------------------------------------------------------- + {%- if self.samseg_atlas_build.lower() in ["on", "true", "1", "y"] %} + cd {{ self.source_path }}/gems + make install + {% endif -%} # Setup mount points - # ------------------------------------------------- + # ---------------------------------------------------------------- mkdir -p {{ self.subjects_dir }} mkdir -p {{ self.infant_model_dir }} - # Todo: - # - delete the /tmp dir, but first ensure that it's not needed + # If self.license_base64 is populated, decode from base64 and write to self.license_path + # ------------------------------------------------- + {%- if self.license_base64 is defined and self.license_base64|length %} + echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }} + {% endif -%} + # Dev tools + # - jupyter relies on `libsqlite3-dev` being present before python is compiled + # - the /tmp dir isn't deleted (to facilitate recompile/install/test) + # ------------------------------------------------- + {%- if self.dev_tools.lower() in ["on", "true", "1", "y"] %} + pip3 install jupyter; \ + pip3 install -r {{ self.source_path }}/python/requirements-extra.txt; \ + {% else %} + rm -rf /tmp/* + {% endif -%} From aeac0d0577daa980d41051d56938d75401e195a8 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 26 Aug 2021 11:22:40 -0400 Subject: [PATCH 44/89] [FIX] syntax --- neurodocker/templates/freesurfer.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 5b3aded1..77dd2d29 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -305,17 +305,17 @@ source: mkdir -p {{ self.subjects_dir }} mkdir -p {{ self.infant_model_dir }} # If self.license_base64 is populated, decode from base64 and write to self.license_path - # ------------------------------------------------- + # ---------------------------------------------------------------- {%- if self.license_base64 is defined and self.license_base64|length %} echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }} {% endif -%} # Dev tools # - jupyter relies on `libsqlite3-dev` being present before python is compiled # - the /tmp dir isn't deleted (to facilitate recompile/install/test) - # ------------------------------------------------- + # ---------------------------------------------------------------- {%- if self.dev_tools.lower() in ["on", "true", "1", "y"] %} - pip3 install jupyter; \ - pip3 install -r {{ self.source_path }}/python/requirements-extra.txt; \ + pip3 install jupyter + pip3 install -r {{ self.source_path }}/python/requirements-extra.txt {% else %} rm -rf /tmp/* {% endif -%} From 244e9135c7af110285b023a184b9b610f9838009 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 26 Aug 2021 11:58:49 -0400 Subject: [PATCH 45/89] [WIP] test --- neurodocker/templates/freesurfer.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 77dd2d29..90f2fc7d 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -292,14 +292,6 @@ source: cd {{ self.install_path }} make install {% endif -%} - # If were building for samseg atlases, run make install in the gems subdir - # Since the gems subdir doesn't rely on the annex remote to execute `make install`, - # This lets us build the container with git_annex_get=off, and still have gems binaries installed - # ---------------------------------------------------------------- - {%- if self.samseg_atlas_build.lower() in ["on", "true", "1", "y"] %} - cd {{ self.source_path }}/gems - make install - {% endif -%} # Setup mount points # ---------------------------------------------------------------- mkdir -p {{ self.subjects_dir }} From 38fdf90fac510bbcc6c9a4a007bcd059ab7ad040 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 26 Aug 2021 12:51:45 -0400 Subject: [PATCH 46/89] [WIP] debug --- neurodocker/templates/freesurfer.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 90f2fc7d..8e1d79ce 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -296,11 +296,6 @@ source: # ---------------------------------------------------------------- mkdir -p {{ self.subjects_dir }} mkdir -p {{ self.infant_model_dir }} - # If self.license_base64 is populated, decode from base64 and write to self.license_path - # ---------------------------------------------------------------- - {%- if self.license_base64 is defined and self.license_base64|length %} - echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }} - {% endif -%} # Dev tools # - jupyter relies on `libsqlite3-dev` being present before python is compiled # - the /tmp dir isn't deleted (to facilitate recompile/install/test) From e7a211346e9d9d569e4a76e34a79ab9dffd4c7b7 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 27 Aug 2021 11:24:23 -0400 Subject: [PATCH 47/89] [WIP] adding license stuff back --- neurodocker/templates/freesurfer.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 8e1d79ce..4e9c6767 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -223,6 +223,12 @@ source: - make instructions: | {{ self.install_dependencies() }} + # If self.license_base64 is populated, decode from base64 and write to self.license_path + # ---------------------------------------------------------------- + {%- if self.license_base64 is defined and self.license_base64|length %} + echo " fs license (base64): "{{ self.license_base64 }} + echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }} + {% endif -%} # git config # ------------------------------------------------------------------------- git config --global user.email "CI@example.com" @@ -287,6 +293,7 @@ source: # (`make install` will fail if `git annex get` isn't run) # ---------------------------------------------------------------- {%- if self.git_annex_get.lower() in ["on", "true", "1", "y"] %} + cd {{ self.source_path }} git annex get -q . git annex unlock cd {{ self.install_path }} From d0005e9d8cfd3eea25b8a6bd422566cca9d9881d Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 31 Aug 2021 11:59:37 -0400 Subject: [PATCH 48/89] [FIX] --- neurodocker/templates/freesurfer.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 4e9c6767..f04ae397 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -223,12 +223,6 @@ source: - make instructions: | {{ self.install_dependencies() }} - # If self.license_base64 is populated, decode from base64 and write to self.license_path - # ---------------------------------------------------------------- - {%- if self.license_base64 is defined and self.license_base64|length %} - echo " fs license (base64): "{{ self.license_base64 }} - echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }} - {% endif -%} # git config # ------------------------------------------------------------------------- git config --global user.email "CI@example.com" @@ -264,6 +258,11 @@ source: # ---------------------------------------------------------------- mkdir -p {{ self.pkg_path }} mkdir -p {{ self.install_path }} + {%- if self.license_base64 is defined and self.license_base64|length %} + echo " fs license (base64): "{{ self.license_base64 }} + mkdir -p `dirname {{ self.license_path }}` + echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }} + {% endif -%} wget -q -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} mv {{ self.pkg_path }}/packages/* {{ self.pkg_path }} rm -rf {{ self.pkg_path }}/packages From a352ac292ab37c3e6faa1fba3627f00602888b28 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 8 Sep 2021 12:16:32 -0400 Subject: [PATCH 49/89] [FIX] formatting --- neurodocker/templates/freesurfer.yaml | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index f04ae397..bf85f605 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -259,9 +259,9 @@ source: mkdir -p {{ self.pkg_path }} mkdir -p {{ self.install_path }} {%- if self.license_base64 is defined and self.license_base64|length %} - echo " fs license (base64): "{{ self.license_base64 }} - mkdir -p `dirname {{ self.license_path }}` - echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }} + echo " fs license (base64): "{{ self.license_base64 }} + mkdir -p `dirname {{ self.license_path }}` + echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }} {% endif -%} wget -q -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} mv {{ self.pkg_path }}/packages/* {{ self.pkg_path }} @@ -277,10 +277,10 @@ source: # Install python reqs # ---------------------------------------------------------------- {%- if self.infant_module.lower() in ["on", "true", "1", "y"] %} - pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt - ln -s {{ self.install_path }}/bin/infant-container-entrypoint.bash /bin/infant-container-entrypoint.bash + pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt + ln -s {{ self.install_path }}/bin/infant-container-entrypoint.bash /bin/infant-container-entrypoint.bash {% else %} - pip3 install -r {{ self.source_path }}/python/requirements.txt + pip3 install -r {{ self.source_path }}/python/requirements.txt {% endif -%} # Setup annex remote # ---------------------------------------------------------------- @@ -292,11 +292,11 @@ source: # (`make install` will fail if `git annex get` isn't run) # ---------------------------------------------------------------- {%- if self.git_annex_get.lower() in ["on", "true", "1", "y"] %} - cd {{ self.source_path }} - git annex get -q . - git annex unlock - cd {{ self.install_path }} - make install + cd {{ self.source_path }} + git annex get -q . + git annex unlock + cd {{ self.install_path }} + make install {% endif -%} # Setup mount points # ---------------------------------------------------------------- @@ -307,8 +307,8 @@ source: # - the /tmp dir isn't deleted (to facilitate recompile/install/test) # ---------------------------------------------------------------- {%- if self.dev_tools.lower() in ["on", "true", "1", "y"] %} - pip3 install jupyter - pip3 install -r {{ self.source_path }}/python/requirements-extra.txt + pip3 install jupyter + pip3 install -r {{ self.source_path }}/python/requirements-extra.txt {% else %} - rm -rf /tmp/* + rm -rf /tmp/* {% endif -%} From 510a9e7c924e94dded2d034eff3b79a77b0088c5 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 8 Sep 2021 12:24:20 -0400 Subject: [PATCH 50/89] [FIX] formatting --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index bf85f605..fb62a28f 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -263,7 +263,7 @@ source: mkdir -p `dirname {{ self.license_path }}` echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }} {% endif -%} - wget -q -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} + wget -q -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} mv {{ self.pkg_path }}/packages/* {{ self.pkg_path }} rm -rf {{ self.pkg_path }}/packages echo "Cloning..." From 5236bfda69e1b51af49b3f55fb133d4566ed010f Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 8 Sep 2021 12:32:55 -0400 Subject: [PATCH 51/89] [WIP] syntax text --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index fb62a28f..95f52311 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -260,7 +260,7 @@ source: mkdir -p {{ self.install_path }} {%- if self.license_base64 is defined and self.license_base64|length %} echo " fs license (base64): "{{ self.license_base64 }} - mkdir -p `dirname {{ self.license_path }}` + mkdir -p $(dirname {{ self.license_path }}) echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }} {% endif -%} wget -q -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} From 4b57ea8baf366bcf666d3368d35d638afd5775e1 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 8 Sep 2021 12:40:54 -0400 Subject: [PATCH 52/89] [WIP] test --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 95f52311..009a025c 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -118,7 +118,7 @@ source: pkg_path: /opt/freesurfer-pkg install_path: /opt/freesurfer-{{ self.version }} # This one has to be quoted with \"s for some reason - license_path: "{{ self.install_path }}/license.txt" + license_path: {{ self.install_path }}/license.txt cxx_flags: "-msse2 -mfpmath=sse -fPIC -fpermissive" c_flags: "-msse2 -mfpmath=sse -fPIC" cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} -DMARTINOS_BUILD={{ self.martinos_build }} -DGEMS_BUILD_EXECUTABLES={{ self.samseg_atlas_build }} -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_CXX_COMPILER_AR=/usr/bin/ar -DCMAKE_CXX_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_CXX_FLAGS="{{ self.cxx_flags }}" -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_C_COMPILER_AR=/usr/bin/ar -DCMAKE_C_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_C_FLAGS="{{ self.c_flags }}" From 0ea929f1f2065baa123cc626ddfe84fd1f9ca59e Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 8 Sep 2021 12:45:43 -0400 Subject: [PATCH 53/89] [WIP] test --- neurodocker/templates/freesurfer.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 009a025c..1f7b985a 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -118,7 +118,8 @@ source: pkg_path: /opt/freesurfer-pkg install_path: /opt/freesurfer-{{ self.version }} # This one has to be quoted with \"s for some reason - license_path: {{ self.install_path }}/license.txt + license_path: "{{ self.install_path }}" + license_file: license.txt cxx_flags: "-msse2 -mfpmath=sse -fPIC -fpermissive" c_flags: "-msse2 -mfpmath=sse -fPIC" cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} -DMARTINOS_BUILD={{ self.martinos_build }} -DGEMS_BUILD_EXECUTABLES={{ self.samseg_atlas_build }} -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_CXX_COMPILER_AR=/usr/bin/ar -DCMAKE_CXX_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_CXX_FLAGS="{{ self.cxx_flags }}" -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_C_COMPILER_AR=/usr/bin/ar -DCMAKE_C_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_C_FLAGS="{{ self.c_flags }}" @@ -260,8 +261,8 @@ source: mkdir -p {{ self.install_path }} {%- if self.license_base64 is defined and self.license_base64|length %} echo " fs license (base64): "{{ self.license_base64 }} - mkdir -p $(dirname {{ self.license_path }}) - echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }} + mkdir -p {{ self.license_path }}) + echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}/{{ self.license_file }} {% endif -%} wget -q -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} mv {{ self.pkg_path }}/packages/* {{ self.pkg_path }} From 21fa66a57c9d279177e380f9304b1544874cf692 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 8 Sep 2021 12:58:31 -0400 Subject: [PATCH 54/89] [FIX] wip --- neurodocker/templates/freesurfer.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 1f7b985a..e9f6915c 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -118,14 +118,14 @@ source: pkg_path: /opt/freesurfer-pkg install_path: /opt/freesurfer-{{ self.version }} # This one has to be quoted with \"s for some reason - license_path: "{{ self.install_path }}" + license_path: "/opt" license_file: license.txt cxx_flags: "-msse2 -mfpmath=sse -fPIC -fpermissive" c_flags: "-msse2 -mfpmath=sse -fPIC" cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} -DMARTINOS_BUILD={{ self.martinos_build }} -DGEMS_BUILD_EXECUTABLES={{ self.samseg_atlas_build }} -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_CXX_COMPILER_AR=/usr/bin/ar -DCMAKE_CXX_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_CXX_FLAGS="{{ self.cxx_flags }}" -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_C_COMPILER_AR=/usr/bin/ar -DCMAKE_C_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_C_FLAGS="{{ self.c_flags }}" make_opts: -j4 env: - FS_LICENSE: "{{ self.license_path }}" + FS_LICENSE: "{{ self.license_path }}/{{ license_file }}" SUBJECTS_DIR: "{{ self.subjects_dir }}" FS_INFANT_MODEL: "{{ self.infant_model_dir }}" # From https://github.com/freesurfer/freesurfer/blob/54018f7d6f620d6288b28f50e14a0a4ba421757c/Dockerfile#L20-L42 @@ -261,7 +261,7 @@ source: mkdir -p {{ self.install_path }} {%- if self.license_base64 is defined and self.license_base64|length %} echo " fs license (base64): "{{ self.license_base64 }} - mkdir -p {{ self.license_path }}) + mkdir -p {{ self.license_path }} echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}/{{ self.license_file }} {% endif -%} wget -q -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} From a44107eca6262cda6521205742bfec4ff7eb0443 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 8 Sep 2021 13:02:28 -0400 Subject: [PATCH 55/89] [FIX] wip --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index e9f6915c..7ac7fae4 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -125,7 +125,7 @@ source: cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} -DMARTINOS_BUILD={{ self.martinos_build }} -DGEMS_BUILD_EXECUTABLES={{ self.samseg_atlas_build }} -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_CXX_COMPILER_AR=/usr/bin/ar -DCMAKE_CXX_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_CXX_FLAGS="{{ self.cxx_flags }}" -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_C_COMPILER_AR=/usr/bin/ar -DCMAKE_C_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_C_FLAGS="{{ self.c_flags }}" make_opts: -j4 env: - FS_LICENSE: "{{ self.license_path }}/{{ license_file }}" + FS_LICENSE: "{{ self.license_path }}/{{ self.license_file }}" SUBJECTS_DIR: "{{ self.subjects_dir }}" FS_INFANT_MODEL: "{{ self.infant_model_dir }}" # From https://github.com/freesurfer/freesurfer/blob/54018f7d6f620d6288b28f50e14a0a4ba421757c/Dockerfile#L20-L42 From 79b02f6c4b1410257b89c189f3116d78325b7d86 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 9 Sep 2021 12:19:36 -0400 Subject: [PATCH 56/89] [WIP] adding awscli as a dev tool (prolly not the best palce) --- neurodocker/templates/freesurfer.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 7ac7fae4..32521890 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -306,10 +306,15 @@ source: # Dev tools # - jupyter relies on `libsqlite3-dev` being present before python is compiled # - the /tmp dir isn't deleted (to facilitate recompile/install/test) + # - awscli is installed (todo, best place for this?) # ---------------------------------------------------------------- {%- if self.dev_tools.lower() in ["on", "true", "1", "y"] %} pip3 install jupyter pip3 install -r {{ self.source_path }}/python/requirements-extra.txt + cd /tmp + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + ./aws/install {% else %} rm -rf /tmp/* {% endif -%} From 319d14b2178a18741fd0ea3cee02c1d92d197f22 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 10 Sep 2021 13:07:27 -0400 Subject: [PATCH 57/89] [WIP] updating mni envs --- neurodocker/templates/freesurfer.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 32521890..f08c3d56 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -148,12 +148,12 @@ source: # Just in case: https://github.com/freesurfer/freesurfer/issues/776 FS_TIME_ALLOW: "0" # mni environment requirements - MINC_BIN_DIR: "{{ self.install_path }}/mni/bin" - MINC_LIB_DIR: "{{ self.install_path }}/mni/lib" - MNI_DIR: "{{ self.install_path }}/mni" - MNI_DATAPATH: "{{ self.install_path }}/mni/data" - MNI_PERL5LIB: "{{ self.install_path }}/mni/share/perl5" - PERL5LIB: "{{ self.install_path }}/mni/share/perl5" + MINC_BIN_DIR: "{{ self.pkg_path }}/mni/current/bin" + MINC_LIB_DIR: "{{ self.pkg_path }}/mni/current/lib" + MNI_DIR: "{{ self.pkg_path }}/mni" + MNI_DATAPATH: "{{ self.pkg_path }}/mni/current/data" + MNI_PERL5LIB: "{{ self.pkg_path }}/mni/current/share/perl5" + PERL5LIB: "{{ self.pkg_path }}/mni/current/share/perl5" # infant model setup, see: # - https://github.com/pwighton/fs-docker/blob/f691232312e97f2ea87b8ae45bfd5c365e2bc91c/baby/dockerfile#L85 # - https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/infant/freesurfer-linux-centos7_x86_64-infant.tar.gz From bb0d6156b31847bebf7b155f472c237028e011ce Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 22 Sep 2021 13:45:50 -0400 Subject: [PATCH 58/89] [WIP] adding the ability to select indv commit --- neurodocker/templates/freesurfer.yaml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index f08c3d56..8efce749 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -90,8 +90,9 @@ binaries: source: arguments: required: - - version + - branch optional: + commit: "" license_base64: "" # The subject dir is not in /opt, like everything else FreeSurfer-related # Because if we want to run `neurodocker minify` we want to operate over the `/opt` @@ -114,9 +115,9 @@ source: dev_tools: "OFF" fortran_libs: /usr/lib/gcc/x86_64-linux-gnu/5/libgfortran.so # --- Probably don't have to change args below here (?) --- - source_path: /tmp/freesurfer/freesurfer-{{ self.version }} + source_path: /tmp/freesurfer/freesurfer-{{ self.branch }} pkg_path: /opt/freesurfer-pkg - install_path: /opt/freesurfer-{{ self.version }} + install_path: /opt/freesurfer-{{ self.branch }} # This one has to be quoted with \"s for some reason license_path: "/opt" license_file: license.txt @@ -269,9 +270,13 @@ source: rm -rf {{ self.pkg_path }}/packages echo "Cloning..." echo " remote: "{{ self.repo }} - echo " branch: "{{ self.version }} + echo " branch: "{{ self.branch }} echo " destination: "{{ self.source_path }} - git clone --single-branch --branch {{ self.version }} {{ self.repo }} {{ self.source_path }} + git clone --single-branch --branch {{ self.branch }} {{ self.repo }} {{ self.source_path }} + {%- if self.commit is defined and self.commit|length %} + cd {{ self.source_path }} + git checkout {{ self.commit }} + {% endif -%} cd {{ self.install_path }} cmake {{ self.cmake_opts }} {{ self.source_path }} make {{ self.make_opts }} From ed37f87b557730708209c4b7da1070ce860ceb96 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 22 Sep 2021 14:19:39 -0400 Subject: [PATCH 59/89] [FIX] dropping --single-branch from git clone --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 8efce749..4cb47168 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -272,7 +272,7 @@ source: echo " remote: "{{ self.repo }} echo " branch: "{{ self.branch }} echo " destination: "{{ self.source_path }} - git clone --single-branch --branch {{ self.branch }} {{ self.repo }} {{ self.source_path }} + git clone --branch {{ self.branch }} {{ self.repo }} {{ self.source_path }} {%- if self.commit is defined and self.commit|length %} cd {{ self.source_path }} git checkout {{ self.commit }} From 7f17fa1f212622fc6d7cf8d756086aac4c07f433 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 23 Sep 2021 14:59:54 -0400 Subject: [PATCH 60/89] [WIP] test.. --- neurodocker/templates/freesurfer.yaml | 77 +-------------------------- 1 file changed, 1 insertion(+), 76 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 4cb47168..c27c0bdd 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -229,33 +229,6 @@ source: # ------------------------------------------------------------------------- git config --global user.email "CI@example.com" git config --global user.name "CI" - # Switch to gcc 4.8 - # ------------------------------------------------------------------------- - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100 - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100 - update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-4.8 100 - # Python v3.6.8 - # ------------------------------------------------------------------------- - curl -sSL --retry 5 https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz | tar -xz -C /tmp - cd /tmp/Python-3.6.8 - ./configure - make - make install - cd / - rm -rf /tmp/Python-3.6.8 - pip3 install -q --no-cache-dir -U pip - pip3 install -q --no-cache-dir wheel - sync - # Cmake v3.19.0 (must be > v3.6.8) - # ------------------------------------------------------------------------- - wget https://github.com/Kitware/CMake/releases/download/v3.19.0/cmake-3.19.0-Linux-x86_64.sh -q -O /tmp/cmake-install.sh - chmod u+x /tmp/cmake-install.sh - mkdir -p /usr/share/cmake-3.19.0 - /tmp/cmake-install.sh --skip-license --prefix=/usr/share/cmake-3.19.0 - CMAKE_BINPATH=`which cmake` - rm -rf $CMAKE_BINPATH - ln -s /usr/share/cmake-3.19.0/bin/cmake $CMAKE_BINPATH - rm /tmp/cmake-install.sh # Compile FreeSurfer # ---------------------------------------------------------------- mkdir -p {{ self.pkg_path }} @@ -265,61 +238,13 @@ source: mkdir -p {{ self.license_path }} echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}/{{ self.license_file }} {% endif -%} - wget -q -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} - mv {{ self.pkg_path }}/packages/* {{ self.pkg_path }} - rm -rf {{ self.pkg_path }}/packages echo "Cloning..." echo " remote: "{{ self.repo }} echo " branch: "{{ self.branch }} + echo " commit: "{{ self.commit }} echo " destination: "{{ self.source_path }} git clone --branch {{ self.branch }} {{ self.repo }} {{ self.source_path }} {%- if self.commit is defined and self.commit|length %} cd {{ self.source_path }} git checkout {{ self.commit }} {% endif -%} - cd {{ self.install_path }} - cmake {{ self.cmake_opts }} {{ self.source_path }} - make {{ self.make_opts }} - # Install python reqs - # ---------------------------------------------------------------- - {%- if self.infant_module.lower() in ["on", "true", "1", "y"] %} - pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt - ln -s {{ self.install_path }}/bin/infant-container-entrypoint.bash /bin/infant-container-entrypoint.bash - {% else %} - pip3 install -r {{ self.source_path }}/python/requirements.txt - {% endif -%} - # Setup annex remote - # ---------------------------------------------------------------- - cd {{ self.source_path }} - git remote add datasrc {{ self.annex }} - git fetch datasrc - git annex enableremote datasrc || true - # Pull from annex and install - # (`make install` will fail if `git annex get` isn't run) - # ---------------------------------------------------------------- - {%- if self.git_annex_get.lower() in ["on", "true", "1", "y"] %} - cd {{ self.source_path }} - git annex get -q . - git annex unlock - cd {{ self.install_path }} - make install - {% endif -%} - # Setup mount points - # ---------------------------------------------------------------- - mkdir -p {{ self.subjects_dir }} - mkdir -p {{ self.infant_model_dir }} - # Dev tools - # - jupyter relies on `libsqlite3-dev` being present before python is compiled - # - the /tmp dir isn't deleted (to facilitate recompile/install/test) - # - awscli is installed (todo, best place for this?) - # ---------------------------------------------------------------- - {%- if self.dev_tools.lower() in ["on", "true", "1", "y"] %} - pip3 install jupyter - pip3 install -r {{ self.source_path }}/python/requirements-extra.txt - cd /tmp - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip awscliv2.zip - ./aws/install - {% else %} - rm -rf /tmp/* - {% endif -%} From e5fc06e9cd443ee875afb059ba7d5727c66782d2 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 23 Sep 2021 15:46:46 -0400 Subject: [PATCH 61/89] [WIP] full test --- neurodocker/templates/freesurfer.yaml | 76 +++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index c27c0bdd..390df6d7 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -229,6 +229,33 @@ source: # ------------------------------------------------------------------------- git config --global user.email "CI@example.com" git config --global user.name "CI" + # Switch to gcc 4.8 + # ------------------------------------------------------------------------- + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100 + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100 + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-4.8 100 + # Python v3.6.8 + # ------------------------------------------------------------------------- + curl -sSL --retry 5 https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz | tar -xz -C /tmp + cd /tmp/Python-3.6.8 + ./configure + make + make install + cd / + rm -rf /tmp/Python-3.6.8 + pip3 install -q --no-cache-dir -U pip + pip3 install -q --no-cache-dir wheel + sync + # Cmake v3.19.0 (must be > v3.6.8) + # ------------------------------------------------------------------------- + wget https://github.com/Kitware/CMake/releases/download/v3.19.0/cmake-3.19.0-Linux-x86_64.sh -q -O /tmp/cmake-install.sh + chmod u+x /tmp/cmake-install.sh + mkdir -p /usr/share/cmake-3.19.0 + /tmp/cmake-install.sh --skip-license --prefix=/usr/share/cmake-3.19.0 + CMAKE_BINPATH=`which cmake` + rm -rf $CMAKE_BINPATH + ln -s /usr/share/cmake-3.19.0/bin/cmake $CMAKE_BINPATH + rm /tmp/cmake-install.sh # Compile FreeSurfer # ---------------------------------------------------------------- mkdir -p {{ self.pkg_path }} @@ -238,6 +265,9 @@ source: mkdir -p {{ self.license_path }} echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}/{{ self.license_file }} {% endif -%} + wget -q -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} + mv {{ self.pkg_path }}/packages/* {{ self.pkg_path }} + rm -rf {{ self.pkg_path }}/packages echo "Cloning..." echo " remote: "{{ self.repo }} echo " branch: "{{ self.branch }} @@ -248,3 +278,49 @@ source: cd {{ self.source_path }} git checkout {{ self.commit }} {% endif -%} + cd {{ self.install_path }} + cmake {{ self.cmake_opts }} {{ self.source_path }} + make {{ self.make_opts }} + # Install python reqs + # ---------------------------------------------------------------- + {%- if self.infant_module.lower() in ["on", "true", "1", "y"] %} + pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt + ln -s {{ self.install_path }}/bin/infant-container-entrypoint.bash /bin/infant-container-entrypoint.bash + {% else %} + pip3 install -r {{ self.source_path }}/python/requirements.txt + {% endif -%} + # Setup annex remote + # ---------------------------------------------------------------- + cd {{ self.source_path }} + git remote add datasrc {{ self.annex }} + git fetch datasrc + git annex enableremote datasrc || true + # Pull from annex and install + # (`make install` will fail if `git annex get` isn't run) + # ---------------------------------------------------------------- + {%- if self.git_annex_get.lower() in ["on", "true", "1", "y"] %} + cd {{ self.source_path }} + git annex get -q . + git annex unlock + cd {{ self.install_path }} + make install + {% endif -%} + # Setup mount points + # ---------------------------------------------------------------- + mkdir -p {{ self.subjects_dir }} + mkdir -p {{ self.infant_model_dir }} + # Dev tools + # - jupyter relies on `libsqlite3-dev` being present before python is compiled + # - the /tmp dir isn't deleted (to facilitate recompile/install/test) + # - awscli is installed (todo, best place for this?) + # ---------------------------------------------------------------- + {%- if self.dev_tools.lower() in ["on", "true", "1", "y"] %} + pip3 install jupyter + pip3 install -r {{ self.source_path }}/python/requirements-extra.txt + cd /tmp + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + ./aws/install + {% else %} + rm -rf /tmp/* + {% endif -%} From e412c5c802ea59aff4a288ce2f6f7041fa06cf10 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 23 Sep 2021 16:23:21 -0400 Subject: [PATCH 62/89] [WIP] syntax test --- neurodocker/templates/freesurfer.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 390df6d7..5d6c2566 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -273,11 +273,11 @@ source: echo " branch: "{{ self.branch }} echo " commit: "{{ self.commit }} echo " destination: "{{ self.source_path }} - git clone --branch {{ self.branch }} {{ self.repo }} {{ self.source_path }} + git clone --branch {{ self.branch }} {{ self.repo }} {{ self.source_path }} {%- if self.commit is defined and self.commit|length %} cd {{ self.source_path }} git checkout {{ self.commit }} - {% endif -%} + {% endif %} cd {{ self.install_path }} cmake {{ self.cmake_opts }} {{ self.source_path }} make {{ self.make_opts }} From c8b87cfae3da62e6ea96ea17c6f732bcd939f1f0 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 24 Sep 2021 15:17:03 -0400 Subject: [PATCH 63/89] [FIX] adding back matlabmcr --- neurodocker/templates/matlabmcr.yaml | 67 ++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 neurodocker/templates/matlabmcr.yaml diff --git a/neurodocker/templates/matlabmcr.yaml b/neurodocker/templates/matlabmcr.yaml new file mode 100644 index 00000000..ce2024a0 --- /dev/null +++ b/neurodocker/templates/matlabmcr.yaml @@ -0,0 +1,67 @@ +--- +# Instructions to install MATLAB Compiler Runtime. +# +# Website: https://www.mathworks.com/products/compiler/matlab-runtime.html + +name: matlabmcr +binaries: + arguments: + required: + - version + optional: + curl_opts: "" + install_path: /opt/MCR-{{ self.version }} + urls: + "2019b": http://ssd.mathworks.com/supportfiles/downloads/R2019b/Release/1/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2019b_Update_1_glnxa64.zip + "2019a": https://ssd.mathworks.com/supportfiles/downloads/R2019a/Release/6/deployment_files/installer/complete/glnxa64/MATLAB_Runtime_R2019a_Update_6_glnxa64.zip + "2018b": https://ssd.mathworks.com/supportfiles/downloads/R2018b/deployment_files/R2018b/installers/glnxa64/MCR_R2018b_glnxa64_installer.zip + "2018a": https://ssd.mathworks.com/supportfiles/downloads/R2018a/deployment_files/R2018a/installers/glnxa64/MCR_R2018a_glnxa64_installer.zip + "2017b": https://ssd.mathworks.com/supportfiles/downloads/R2017b/deployment_files/R2017b/installers/glnxa64/MCR_R2017b_glnxa64_installer.zip + "2017a": https://ssd.mathworks.com/supportfiles/downloads/R2017a/deployment_files/R2017a/installers/glnxa64/MCR_R2017a_glnxa64_installer.zip + "2016b": https://ssd.mathworks.com/supportfiles/downloads/R2016b/deployment_files/R2016b/installers/glnxa64/MCR_R2016b_glnxa64_installer.zip + "2016a": https://ssd.mathworks.com/supportfiles/downloads/R2016a/deployment_files/R2016a/installers/glnxa64/MCR_R2016a_glnxa64_installer.zip + "2015b": https://ssd.mathworks.com/supportfiles/downloads/R2015b/deployment_files/R2015b/installers/glnxa64/MCR_R2015b_glnxa64_installer.zip + "2015aSP1": https://ssd.mathworks.com/supportfiles/downloads/R2015a/deployment_files/R2015aSP1/installers/glnxa64/MCR_R2015aSP1_glnxa64_installer.zip + "2015a": https://ssd.mathworks.com/supportfiles/downloads/R2015a/deployment_files/R2015a/installers/glnxa64/MCR_R2015a_glnxa64_installer.zip + "2014a": https://ssd.mathworks.com/supportfiles/downloads/R2014a/deployment_files/R2014a/installers/glnxa64/MCR_R2014a_glnxa64_installer.zip + "2014b": https://ssd.mathworks.com/supportfiles/downloads/R2014b/deployment_files/R2014b/installers/glnxa64/MCR_R2014b_glnxa64_installer.zip + "2013b": https://ssd.mathworks.com/supportfiles/downloads/R2013b/deployment_files/R2013b/installers/glnxa64/MCR_R2013b_glnxa64_installer.zip + "2013a": https://ssd.mathworks.com/supportfiles/MCR_Runtime/R2013a/MCR_R2013a_glnxa64_installer.zip + "2012b": https://ssd.mathworks.com/supportfiles/MCR_Runtime/R2012b/MCR_R2012b_glnxa64_installer.zip + "2012a": https://ssd.mathworks.com/supportfiles/MCR_Runtime/R2012a/MCR_R2012a_glnxa64_installer.zip + "2010a": https://dl.dropbox.com/s/zz6me0c3v4yq5fd/MCR_R2010a_glnxa64_installer.bin + dependencies: + apt: + - bc + - libncurses5 + - libxext6 + - libxmu6 + - libxpm-dev + - libxt6 + yum: + - bc + - libXext.x86_64 + - libXmu + - libXpm + - libXt.x86_64 + debs: + - http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb + env: + LD_LIBRARY_PATH: "$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu:{{ self.install_path }}/{{ self.version }}/runtime/glnxa64:{{ self.install_path }}/{{ self.version }}/bin/glnxa64:{{ self.install_path }}/{{ self.version }}/sys/os/glnxa64:{{ self.install_path }}/{{ self.version }}/extern/bin/glnxa64" + MATLABCMD: "{{ self.install_path }}/{{ self.version }}/toolbox/matlab" + instructions: | + export TMPDIR="$(mktemp -d)" + {{ self.install_dependencies() }} + echo "Downloading MATLAB Compiler Runtime ..." + {% if self.version == "2010a" -%} + {{ self.install_debs() }} + curl {{ self.curl_opts }} -o "$TMPDIR/MCRInstaller.bin" {{ self.urls[self.version] }} + chmod +x "$TMPDIR/MCRInstaller.bin" + "$TMPDIR/MCRInstaller.bin" -silent -P installLocation="{{ self.install_path }}" + {% else -%} + curl {{ self.curl_opts }} -o "$TMPDIR/mcr.zip" {{ self.urls[self.version] }} + unzip -q "$TMPDIR/mcr.zip" -d "$TMPDIR/mcrtmp" + "$TMPDIR/mcrtmp/install" -destinationFolder {{ self.install_path }} -mode silent -agreeToLicense yes + {% endif -%} + rm -rf "$TMPDIR" + unset TMPDIR From 93f689e4a017a99ba2eaf8c68c417e8c66b4e824 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 15 Oct 2021 16:24:21 -0400 Subject: [PATCH 64/89] [FIX] applying @kaczmarj's diff --- neurodocker/templates/matlabmcr.yaml | 31 +++++++++++++++++----------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/neurodocker/templates/matlabmcr.yaml b/neurodocker/templates/matlabmcr.yaml index ce2024a0..b95cf918 100644 --- a/neurodocker/templates/matlabmcr.yaml +++ b/neurodocker/templates/matlabmcr.yaml @@ -31,23 +31,30 @@ binaries: "2012a": https://ssd.mathworks.com/supportfiles/MCR_Runtime/R2012a/MCR_R2012a_glnxa64_installer.zip "2010a": https://dl.dropbox.com/s/zz6me0c3v4yq5fd/MCR_R2010a_glnxa64_installer.bin dependencies: - apt: - - bc - - libncurses5 - - libxext6 - - libxmu6 - - libxpm-dev + apt: + - bc + - curl + - libncurses5 + - libxext6 + - libxmu6 + - libxpm-dev - libxt6 - yum: - - bc - - libXext.x86_64 - - libXmu - - libXpm + - multiarch-support + - unzip + yum: + - bc + - curl + - libXext.x86_64 + - libXmu + - libXpm - libXt.x86_64 + - unzip debs: - http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb env: - LD_LIBRARY_PATH: "$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu:{{ self.install_path }}/{{ self.version }}/runtime/glnxa64:{{ self.install_path }}/{{ self.version }}/bin/glnxa64:{{ self.install_path }}/{{ self.version }}/sys/os/glnxa64:{{ self.install_path }}/{{ self.version }}/extern/bin/glnxa64" + LD_LIBRARY_PATH: | + {% set versionTovXX = {"2019b": "v97", "2019a": "v98", "2018b": "v95", "2018a": "v94", "2017b": "v93", "2017a": "v92", "2016b": "v91", "2016a": "v901", "2015b": "v90", "2015aSP1": "v851", "2015a": "v85", "2014b": "v84", "2014a": "v83", "2013b": "v82", "2013a": "v81", "2012b": "v80", "2012a": "v717"} -%} + $LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu:{{ self.install_path }}/{{ versionTovXX[self.version] }}/runtime/glnxa64:{{ self.install_path }}/{{ versionTovXX[self.version] }}/bin/glnxa64:{{ self.install_path }}/{{ versionTovXX[self.version] }}/sys/os/glnxa64:{{ self.install_path }}/{{ versionTovXX[self.version] }}/extern/bin/glnxa64 MATLABCMD: "{{ self.install_path }}/{{ self.version }}/toolbox/matlab" instructions: | export TMPDIR="$(mktemp -d)" From e174eb5625972590290b18b51b7f8bdeb8d95894 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Sat, 16 Oct 2021 01:49:47 -0400 Subject: [PATCH 65/89] [WIP] infant tensorflow fix --- neurodocker/templates/freesurfer.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 5d6c2566..070291c0 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -316,7 +316,10 @@ source: # ---------------------------------------------------------------- {%- if self.dev_tools.lower() in ["on", "true", "1", "y"] %} pip3 install jupyter + # Don't insall extra python packages with infant model due to tensorflow version mismatch + {%- if not self.infant_module.lower() in ["on", "true", "1", "y"] %} pip3 install -r {{ self.source_path }}/python/requirements-extra.txt + {% endif -%} cd /tmp curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip From 99393785a44dbfe61df0b75a43095f9fa6afca4a Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Mon, 18 Oct 2021 12:24:46 -0400 Subject: [PATCH 66/89] [WIP] not installing extra reqs with dev tools --- neurodocker/templates/freesurfer.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 070291c0..dd6930d1 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -316,10 +316,6 @@ source: # ---------------------------------------------------------------- {%- if self.dev_tools.lower() in ["on", "true", "1", "y"] %} pip3 install jupyter - # Don't insall extra python packages with infant model due to tensorflow version mismatch - {%- if not self.infant_module.lower() in ["on", "true", "1", "y"] %} - pip3 install -r {{ self.source_path }}/python/requirements-extra.txt - {% endif -%} cd /tmp curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip From 6f1aa33dde236a2f77edbd4ab0dbf3227a6ee463 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 5 Nov 2021 14:40:19 -0400 Subject: [PATCH 67/89] [WIP] tweaks for singularity --- neurodocker/templates/freesurfer.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index dd6930d1..4e0d5a7f 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -320,6 +320,12 @@ source: curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip ./aws/install + rm -rf /tmp/aws + rm -rf /tmp/awscliv2.zip {% else %} - rm -rf /tmp/* + # Note: don't run `rm -rf /tmp/*` since singularity bind-mounts /tmp by default + # See: https://github.com/ReproNim/neurodocker/issues/246 + rm -rf /tmp/freesurfer + rm -rf /tmp/aws + rm -rf /tmp/awscliv2.zip {% endif -%} From da4c72ac4106a87d3d1b2e4bf9e0402b47ccec34 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 16 Nov 2021 18:24:41 -0500 Subject: [PATCH 68/89] [FIX] moving from /tmp to /stage for singularity --- neurodocker/templates/freesurfer.yaml | 33 ++++++++++++++------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 4e0d5a7f..12d1d16f 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -234,28 +234,32 @@ source: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100 update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100 update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-4.8 100 + # Make a staging directory, we don't want to use /tmp since it doesn't play nice with singularity + # See: https://github.com/ReproNim/neurodocker/issues/246 + # ------------------------------------------------------------------------- + mkdir -p /stage # Python v3.6.8 # ------------------------------------------------------------------------- - curl -sSL --retry 5 https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz | tar -xz -C /tmp - cd /tmp/Python-3.6.8 + curl -sSL --retry 5 https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz | tar -xz -C /stage + cd /stage/Python-3.6.8 ./configure make make install cd / - rm -rf /tmp/Python-3.6.8 + rm -rf /stage/Python-3.6.8 pip3 install -q --no-cache-dir -U pip pip3 install -q --no-cache-dir wheel sync # Cmake v3.19.0 (must be > v3.6.8) # ------------------------------------------------------------------------- - wget https://github.com/Kitware/CMake/releases/download/v3.19.0/cmake-3.19.0-Linux-x86_64.sh -q -O /tmp/cmake-install.sh - chmod u+x /tmp/cmake-install.sh + wget https://github.com/Kitware/CMake/releases/download/v3.19.0/cmake-3.19.0-Linux-x86_64.sh -q -O /stage/cmake-install.sh + chmod u+x /stage/cmake-install.sh mkdir -p /usr/share/cmake-3.19.0 - /tmp/cmake-install.sh --skip-license --prefix=/usr/share/cmake-3.19.0 + /stage/cmake-install.sh --skip-license --prefix=/usr/share/cmake-3.19.0 CMAKE_BINPATH=`which cmake` rm -rf $CMAKE_BINPATH ln -s /usr/share/cmake-3.19.0/bin/cmake $CMAKE_BINPATH - rm /tmp/cmake-install.sh + rm /stage/cmake-install.sh # Compile FreeSurfer # ---------------------------------------------------------------- mkdir -p {{ self.pkg_path }} @@ -286,6 +290,7 @@ source: {%- if self.infant_module.lower() in ["on", "true", "1", "y"] %} pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt ln -s {{ self.install_path }}/bin/infant-container-entrypoint.bash /bin/infant-container-entrypoint.bash + # todo, edit freesurfer so infant-container-entrypoint-aws.bash gets installed to /opt {% else %} pip3 install -r {{ self.source_path }}/python/requirements.txt {% endif -%} @@ -311,21 +316,17 @@ source: mkdir -p {{ self.infant_model_dir }} # Dev tools # - jupyter relies on `libsqlite3-dev` being present before python is compiled - # - the /tmp dir isn't deleted (to facilitate recompile/install/test) + # - the /stage dir isn't deleted (to facilitate recompile/install/test) # - awscli is installed (todo, best place for this?) # ---------------------------------------------------------------- {%- if self.dev_tools.lower() in ["on", "true", "1", "y"] %} pip3 install jupyter - cd /tmp + cd /stage curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip ./aws/install - rm -rf /tmp/aws - rm -rf /tmp/awscliv2.zip + rm -rf /stage/aws + rm -rf /stage/awscliv2.zip {% else %} - # Note: don't run `rm -rf /tmp/*` since singularity bind-mounts /tmp by default - # See: https://github.com/ReproNim/neurodocker/issues/246 - rm -rf /tmp/freesurfer - rm -rf /tmp/aws - rm -rf /tmp/awscliv2.zip + rm -rf /stage {% endif -%} From 94e704f617057b007429b59d2b4f0366979c1755 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 17 Nov 2021 16:54:47 -0500 Subject: [PATCH 69/89] [WIP] changing fs source dir to /stage and symlinking infant aws entrypoint to /bin --- neurodocker/templates/freesurfer.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 12d1d16f..92033df3 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -114,8 +114,11 @@ source: git_annex_get: "ON" dev_tools: "OFF" fortran_libs: /usr/lib/gcc/x86_64-linux-gnu/5/libgfortran.so - # --- Probably don't have to change args below here (?) --- - source_path: /tmp/freesurfer/freesurfer-{{ self.branch }} + # --- Probably don't have to change args below here --- + # Were using a source path of `/stage` instead of `/temp` in case we want to + # keep the source dir around for debugging (dev_tools=ON) with singularity containers + # (which bind mount the `/tmp` dir by default) + source_path: /stage/freesurfer/freesurfer-{{ self.branch }} pkg_path: /opt/freesurfer-pkg install_path: /opt/freesurfer-{{ self.branch }} # This one has to be quoted with \"s for some reason @@ -290,7 +293,7 @@ source: {%- if self.infant_module.lower() in ["on", "true", "1", "y"] %} pip3 install -r {{ self.source_path }}/infant/docker/requirements.txt ln -s {{ self.install_path }}/bin/infant-container-entrypoint.bash /bin/infant-container-entrypoint.bash - # todo, edit freesurfer so infant-container-entrypoint-aws.bash gets installed to /opt + ln -s {{ self.install_path }}/bin/infant-container-entrypoint-aws.bash /bin/infant-container-entrypoint-aws.bash {% else %} pip3 install -r {{ self.source_path }}/python/requirements.txt {% endif -%} From 9da3823845e2c4588c0b44dc682fec87a2f1c37c Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 15 Feb 2022 15:44:48 -0500 Subject: [PATCH 70/89] [WIP] trying to support deb and rpm installations --- neurodocker/templates/freesurfer.yaml | 49 +++++++++++++++++++++------ 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 3773df94..d08ed665 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -22,6 +22,10 @@ binaries: subjects/fsaverage_sym trctrain urls: + "7.2.0-centos6": https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.2.0/freesurfer-CentOS6-7.2.0-1.x86_64.rpm + "7.2.0-centos7": https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.2.0/freesurfer-CentOS7-7.2.0-1.x86_64.rpm + "7.2.0-centos8": https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.2.0/freesurfer-CentOS8-7.2.0-1.x86_64.rpm + "7.2.0-deb": https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.2.0/freesurfer_7.2.0_amd64.deb "7.2.0": https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.2.0/freesurfer-linux-centos6_x86_64-7.2.0.tar.gz "7.1.1": https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.1.1/freesurfer-linux-centos6_x86_64-7.1.1.tar.gz "7.1.1-min": https://dl.dropbox.com/s/c3earkfhhvdyuo4/freesurfer-7.1.1-min.tgz @@ -75,18 +79,43 @@ binaries: PERL5LIB: "{{ self.install_path }}/mni/share/perl5" instructions: | {{ self.install_dependencies() }} - echo "Downloading FreeSurfer ..." - mkdir -p {{ self.install_path }} - curl -fL {{ self.urls[self.version] }} \ - | tar -xz -C {{ self.install_path }} --owner root --group root --no-same-owner --strip-components 1 {% if self.exclude_paths -%}\ - {%- for exclude_path in self.exclude_paths.split() %} - {% if not loop.last -%} - --exclude='{{ exclude_path }}' \ + {%- if self.version in ["6.0.0-min", "6.0.0", "6.0.1", "7.1.0", "7.1.1-min", "7.1.1", "7.2.0"] %} + # Install the old way (.tar.gz) + echo "Downloading FreeSurfer ..." + mkdir -p {{ self.install_path }} + curl -fL {{ self.urls[self.version] }} \ + | tar -xz -C {{ self.install_path }} --owner root --group root --no-same-owner --strip-components 1 {% if self.exclude_paths -%}\ + {%- for exclude_path in self.exclude_paths.split() %} + {% if not loop.last -%} + --exclude='{{ exclude_path }}' \ + {%- else -%} + --exclude='{{ exclude_path }}' + {%- endif -%} + {%- endfor %} + {%- endif %} + {%- else -%} + # Install the new way (.deb or .rpm package) + {%- if self.version in ["apt"] %} + apt-get update -qq + apt-get install --fix-missing -y \ + apt-file apt-utils apt-transport-https dpkg-dev debhelper devscripts librpm-dev rpm + curl -o /stage/freesurfer_amd64.deb {{ self.urls[self.version] }} + dpkg -i /stage/freesurfer_amd64.deb + # To supress interactive keyboard-configuration + DEBIAN_FRONTEND=noninteractive apt-get -f -y install + # todo: find install dir and set env vars accordingly + apt-get clean + rm -rf /var/lib/apt/lists/* + sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' {%- else -%} - --exclude='{{ exclude_path }}' + # todo: wip + yum -y update --exclude=filesystem* + yum -y install dnf-plugins-core epel-release + yum -y config-manager --set-enabled powertools + yum -y update --exclude=filesystem* {%- endif -%} - {%- endfor %} - {%- endif %} + echo "{{ self.pkg_manager }}" + {%- endif -%} source: arguments: From 84636a3b2c162f46763e4e3948d66ab8b58fc405 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 15 Feb 2022 16:16:41 -0500 Subject: [PATCH 71/89] [DOC] --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index d08ed665..ef4d69e5 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -103,7 +103,7 @@ binaries: dpkg -i /stage/freesurfer_amd64.deb # To supress interactive keyboard-configuration DEBIAN_FRONTEND=noninteractive apt-get -f -y install - # todo: find install dir and set env vars accordingly + # todo: find install dir (/usr/local/freesurfer/7.2.0/) and set env vars accordingly apt-get clean rm -rf /var/lib/apt/lists/* sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' From 300796f9c8c7a11ee94e405173f5478634575e41 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Mon, 28 Mar 2022 18:35:19 -0400 Subject: [PATCH 72/89] [WIP] infant model inside container --- neurodocker/templates/freesurfer.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index ef4d69e5..ecc93966 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -129,6 +129,8 @@ source: # dir and have the subjects dir excluded subjects_dir: "/ext/fs-subjects" infant_model_dir: "/opt/fs-infant-model" + infant_model_s3: "s3://freesurfer-annex/infant/model/dev/" + infant_model_s3_region: "us-east-2" repo: https://github.com/freesurfer/freesurfer pkg_url: http://surfer.nmr.mgh.harvard.edu/pub/data/fspackages/prebuilt/centos7-packages.tar.gz annex: https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/repo/annex.git @@ -277,7 +279,7 @@ source: cd /stage/Python-3.6.8 ./configure make - make install + make install cd / rm -rf /stage/Python-3.6.8 pip3 install -q --no-cache-dir -U pip @@ -360,6 +362,9 @@ source: ./aws/install rm -rf /stage/aws rm -rf /stage/awscliv2.zip + {%- if self.infant_model_s3 is defined and self.infant_model_s3|length and self.infant_model_s3_region is defined and self.infant_model_s3_region|length %} + echo "You should copy the atlas from s3 here" + {% endif -%} {% else %} rm -rf /stage {% endif -%} From cda0118d1650066c0cad80eb8ef3263b72060f88 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 29 Mar 2022 11:11:08 -0400 Subject: [PATCH 73/89] [WIP] copying the infant model into the container, but not by default --- neurodocker/templates/freesurfer.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index ecc93966..420f0067 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -129,8 +129,8 @@ source: # dir and have the subjects dir excluded subjects_dir: "/ext/fs-subjects" infant_model_dir: "/opt/fs-infant-model" - infant_model_s3: "s3://freesurfer-annex/infant/model/dev/" - infant_model_s3_region: "us-east-2" + infant_model_s3: "" + infant_model_s3_region: "" repo: https://github.com/freesurfer/freesurfer pkg_url: http://surfer.nmr.mgh.harvard.edu/pub/data/fspackages/prebuilt/centos7-packages.tar.gz annex: https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/repo/annex.git @@ -363,7 +363,7 @@ source: rm -rf /stage/aws rm -rf /stage/awscliv2.zip {%- if self.infant_model_s3 is defined and self.infant_model_s3|length and self.infant_model_s3_region is defined and self.infant_model_s3_region|length %} - echo "You should copy the atlas from s3 here" + aws s3 cp --no-sign-request --region={{ self.infant_model_s3_region }} --recursive --include '*' {{ self.infant_model_s3 }} {{ self.infant_model_dir }}/ {% endif -%} {% else %} rm -rf /stage From f2721cc0a1389bc6d2cba81ef8ac5cd6c24a80f8 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 8 Apr 2022 12:59:02 -0400 Subject: [PATCH 74/89] [FIX] syntax tweak --- neurodocker/templates/freesurfer.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 420f0067..59815e70 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -303,7 +303,7 @@ source: echo " fs license (base64): "{{ self.license_base64 }} mkdir -p {{ self.license_path }} echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}/{{ self.license_file }} - {% endif -%} + {% endif %} wget -q -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} mv {{ self.pkg_path }}/packages/* {{ self.pkg_path }} rm -rf {{ self.pkg_path }}/packages @@ -328,7 +328,7 @@ source: ln -s {{ self.install_path }}/bin/infant-container-entrypoint-aws.bash /bin/infant-container-entrypoint-aws.bash {% else %} pip3 install -r {{ self.source_path }}/python/requirements.txt - {% endif -%} + {% endif %} # Setup annex remote # ---------------------------------------------------------------- cd {{ self.source_path }} @@ -344,7 +344,7 @@ source: git annex unlock cd {{ self.install_path }} make install - {% endif -%} + {% endif %} # Setup mount points # ---------------------------------------------------------------- mkdir -p {{ self.subjects_dir }} @@ -364,7 +364,7 @@ source: rm -rf /stage/awscliv2.zip {%- if self.infant_model_s3 is defined and self.infant_model_s3|length and self.infant_model_s3_region is defined and self.infant_model_s3_region|length %} aws s3 cp --no-sign-request --region={{ self.infant_model_s3_region }} --recursive --include '*' {{ self.infant_model_s3 }} {{ self.infant_model_dir }}/ - {% endif -%} + {% endif %} {% else %} rm -rf /stage - {% endif -%} + {% endif %} From 95055e07e4ff64a454382ee03aa32b5a559e0548 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 8 Apr 2022 13:19:49 -0400 Subject: [PATCH 75/89] [WIP] syntax tweak --- neurodocker/templates/freesurfer.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 59815e70..ace235b6 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -328,7 +328,7 @@ source: ln -s {{ self.install_path }}/bin/infant-container-entrypoint-aws.bash /bin/infant-container-entrypoint-aws.bash {% else %} pip3 install -r {{ self.source_path }}/python/requirements.txt - {% endif %} + {% endif -%} # Setup annex remote # ---------------------------------------------------------------- cd {{ self.source_path }} @@ -367,4 +367,4 @@ source: {% endif %} {% else %} rm -rf /stage - {% endif %} + {% endif -%} From a9071f9bfb9ada2538816ea8c52492b644f75b8f Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 8 Apr 2022 13:32:26 -0400 Subject: [PATCH 76/89] [WIP] syntax tweak --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index ace235b6..f5730144 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -344,7 +344,7 @@ source: git annex unlock cd {{ self.install_path }} make install - {% endif %} + {% endif -%} # Setup mount points # ---------------------------------------------------------------- mkdir -p {{ self.subjects_dir }} From 7eabc6d59a624a3a4bba2f22da21d23b877f521e Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Mon, 11 Apr 2022 16:12:43 -0400 Subject: [PATCH 77/89] [FIX] syntax --- neurodocker/templates/freesurfer.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index f5730144..34ba2e0c 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -80,7 +80,6 @@ binaries: instructions: | {{ self.install_dependencies() }} {%- if self.version in ["6.0.0-min", "6.0.0", "6.0.1", "7.1.0", "7.1.1-min", "7.1.1", "7.2.0"] %} - # Install the old way (.tar.gz) echo "Downloading FreeSurfer ..." mkdir -p {{ self.install_path }} curl -fL {{ self.urls[self.version] }} \ @@ -94,16 +93,13 @@ binaries: {%- endfor %} {%- endif %} {%- else -%} - # Install the new way (.deb or .rpm package) - {%- if self.version in ["apt"] %} + {%- if self.version in ["deb"] %} apt-get update -qq apt-get install --fix-missing -y \ apt-file apt-utils apt-transport-https dpkg-dev debhelper devscripts librpm-dev rpm curl -o /stage/freesurfer_amd64.deb {{ self.urls[self.version] }} dpkg -i /stage/freesurfer_amd64.deb - # To supress interactive keyboard-configuration DEBIAN_FRONTEND=noninteractive apt-get -f -y install - # todo: find install dir (/usr/local/freesurfer/7.2.0/) and set env vars accordingly apt-get clean rm -rf /var/lib/apt/lists/* sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' From 632b11e3d2c287e87fee21dc61913e66aa57648f Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 12 Apr 2022 12:45:24 -0400 Subject: [PATCH 78/89] [WIP] --- neurodocker/templates/freesurfer.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 34ba2e0c..71295ba8 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -256,6 +256,11 @@ source: - make instructions: | {{ self.install_dependencies() }} + {% if self.license_base64 is defined and self.license_base64|length %} + echo " fs license (base64): "{{ self.license_base64 }} + mkdir -p {{ self.license_path }} + echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}/{{ self.license_file }} + {% endif %} # git config # ------------------------------------------------------------------------- git config --global user.email "CI@example.com" @@ -295,11 +300,6 @@ source: # ---------------------------------------------------------------- mkdir -p {{ self.pkg_path }} mkdir -p {{ self.install_path }} - {%- if self.license_base64 is defined and self.license_base64|length %} - echo " fs license (base64): "{{ self.license_base64 }} - mkdir -p {{ self.license_path }} - echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}/{{ self.license_file }} - {% endif %} wget -q -c {{ self.pkg_url }} -O - | tar -xz -C {{ self.pkg_path }} mv {{ self.pkg_path }}/packages/* {{ self.pkg_path }} rm -rf {{ self.pkg_path }}/packages From 669fbaf87d443057aa1714710aacfea8eebf307a Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Tue, 12 Apr 2022 12:59:54 -0400 Subject: [PATCH 79/89] [WIP] --- neurodocker/templates/freesurfer.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 71295ba8..13fb9491 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -256,11 +256,11 @@ source: - make instructions: | {{ self.install_dependencies() }} - {% if self.license_base64 is defined and self.license_base64|length %} + {%- if self.license_base64 is defined and self.license_base64|length %} echo " fs license (base64): "{{ self.license_base64 }} mkdir -p {{ self.license_path }} echo {{ self.license_base64 }} | base64 -d > {{ self.license_path }}/{{ self.license_file }} - {% endif %} + {% endif -%} # git config # ------------------------------------------------------------------------- git config --global user.email "CI@example.com" From 68dc87a3340d365d5a56e5e978fc5c03e4a729cf Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 13 Apr 2022 18:12:17 -0400 Subject: [PATCH 80/89] [FIX] defining SSCNN_MODEL_DIR --- neurodocker/templates/freesurfer.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 13fb9491..ba7bfb74 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -7,6 +7,9 @@ binaries: - version optional: install_path: /opt/freesurfer-{{ self.version }} + license_base64: "" + license_path: "/opt" + license_file: license.txt exclude_paths: | average/mult-comp-cor lib/cuda @@ -120,6 +123,8 @@ source: optional: commit: "" license_base64: "" + license_path: "/opt" + license_file: license.txt # The subject dir is not in /opt, like everything else FreeSurfer-related # Because if we want to run `neurodocker minify` we want to operate over the `/opt` # dir and have the subjects dir excluded @@ -149,9 +154,6 @@ source: source_path: /stage/freesurfer/freesurfer-{{ self.branch }} pkg_path: /opt/freesurfer-pkg install_path: /opt/freesurfer-{{ self.branch }} - # This one has to be quoted with \"s for some reason - license_path: "/opt" - license_file: license.txt cxx_flags: "-msse2 -mfpmath=sse -fPIC -fpermissive" c_flags: "-msse2 -mfpmath=sse -fPIC" cmake_opts: -DCMAKE_INSTALL_PREFIX={{ self.install_path }} -DBUILD_GUIS={{ self.build_guis }} -DMINIMAL={{ self.minimal }} -DINFANT_MODULE={{ self.infant_module }} -DFS_PACKAGES_DIR={{ self.pkg_path }} -DGFORTRAN_LIBRARIES={{ self.fortran_libs }} -DINSTALL_PYTHON_DEPENDENCIES={{ self.install_python_deps }} -DDISTRIBUTE_FSPYTHON={{ self.distribute_fspython }} -DMARTINOS_BUILD={{ self.martinos_build }} -DGEMS_BUILD_EXECUTABLES={{ self.samseg_atlas_build }} -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_CXX_COMPILER_AR=/usr/bin/ar -DCMAKE_CXX_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_CXX_FLAGS="{{ self.cxx_flags }}" -DCMAKE_C_COMPILER=/usr/bin/cc -DCMAKE_C_COMPILER_AR=/usr/bin/ar -DCMAKE_C_COMPILER_RANLIB=/usr/bin/ranlib -DCMAKE_C_FLAGS="{{ self.c_flags }}" @@ -160,6 +162,7 @@ source: FS_LICENSE: "{{ self.license_path }}/{{ self.license_file }}" SUBJECTS_DIR: "{{ self.subjects_dir }}" FS_INFANT_MODEL: "{{ self.infant_model_dir }}" + SSCNN_MODEL_DIR: "{{ self.infant_model_dir }}"/sscnn_skullstrip # From https://github.com/freesurfer/freesurfer/blob/54018f7d6f620d6288b28f50e14a0a4ba421757c/Dockerfile#L20-L42 OS: "Linux" PATH: "{{ self.niftyreg_path }}/bin:{{ self.pkg_path }}/mni/current/bin:{{ self.install_path }}/bin:{{ self.install_path }}/fsfast/bin:{{ self.install_path }}/tktools:{{ self.install_path }}/mni/bin:$PATH" From ca592a3415f5461d343ea6c84984cad86645f14b Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 13 Apr 2022 18:16:44 -0400 Subject: [PATCH 81/89] [FIX] defining SSCNN_MODEL_DIR --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index ba7bfb74..bc478e18 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -162,7 +162,7 @@ source: FS_LICENSE: "{{ self.license_path }}/{{ self.license_file }}" SUBJECTS_DIR: "{{ self.subjects_dir }}" FS_INFANT_MODEL: "{{ self.infant_model_dir }}" - SSCNN_MODEL_DIR: "{{ self.infant_model_dir }}"/sscnn_skullstrip + SSCNN_MODEL_DIR: "{{ self.infant_model_dir }}/sscnn_skullstrip" # From https://github.com/freesurfer/freesurfer/blob/54018f7d6f620d6288b28f50e14a0a4ba421757c/Dockerfile#L20-L42 OS: "Linux" PATH: "{{ self.niftyreg_path }}/bin:{{ self.pkg_path }}/mni/current/bin:{{ self.install_path }}/bin:{{ self.install_path }}/fsfast/bin:{{ self.install_path }}/tktools:{{ self.install_path }}/mni/bin:$PATH" From a05eef7a767411853b32ea86c27d09bcdc4628b3 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 14 Apr 2022 14:51:15 -0400 Subject: [PATCH 82/89] [ENH] making `branch` optional --- neurodocker/templates/freesurfer.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index bc478e18..651852ce 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -118,9 +118,9 @@ binaries: source: arguments: - required: - - branch optional: + repo: https://github.com/freesurfer/freesurfer + branch: "dev" commit: "" license_base64: "" license_path: "/opt" @@ -132,7 +132,6 @@ source: infant_model_dir: "/opt/fs-infant-model" infant_model_s3: "" infant_model_s3_region: "" - repo: https://github.com/freesurfer/freesurfer pkg_url: http://surfer.nmr.mgh.harvard.edu/pub/data/fspackages/prebuilt/centos7-packages.tar.gz annex: https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/repo/annex.git # set to niftyreg.yaml default From 12d0f21e119ad595ccf2487b4edc788a6e032dc3 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 14 Apr 2022 15:32:58 -0400 Subject: [PATCH 83/89] [GIT] removing buildspec before PR --- buildspec.yml | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 buildspec.yml diff --git a/buildspec.yml b/buildspec.yml deleted file mode 100644 index afa8982f..00000000 --- a/buildspec.yml +++ /dev/null @@ -1,23 +0,0 @@ -# see https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html -version: 0.2 - -env: - - shell: bash - parameter-store: - DOCKERHUB_PASS_PARAM: /dockerhub-access-token-for-pwighton - -phases: - install: - on-failure: ABORT - commands: - - export DOCKERHUB_PASS=$DOCKERHUB_PASS_PARAM - - docker login --username pwighton --password $DOCKERHUB_PASS - build: - on-failure: ABORT - commands: - - docker build -t pwighton/neurodocker . - post_build: - on-failure: ABORT - commands: - - docker push pwighton/neurodocker:latest From 1fd938c6276c0f87bbb2c963e5eea572040e8a1a Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 9 Jun 2022 14:24:26 -0400 Subject: [PATCH 84/89] [FIX] remove whitespace --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 651852ce..3df8d385 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -314,7 +314,7 @@ source: {%- if self.commit is defined and self.commit|length %} cd {{ self.source_path }} git checkout {{ self.commit }} - {% endif %} + {% endif %} cd {{ self.install_path }} cmake {{ self.cmake_opts }} {{ self.source_path }} make {{ self.make_opts }} From 368b2f7e95161b24c55e835caf45005cce0345bb Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 9 Jun 2022 14:33:18 -0400 Subject: [PATCH 85/89] [WIP] adding back buildspec --- buildspec.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 buildspec.yml diff --git a/buildspec.yml b/buildspec.yml new file mode 100644 index 00000000..afa8982f --- /dev/null +++ b/buildspec.yml @@ -0,0 +1,23 @@ +# see https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html +version: 0.2 + +env: + + shell: bash + parameter-store: + DOCKERHUB_PASS_PARAM: /dockerhub-access-token-for-pwighton + +phases: + install: + on-failure: ABORT + commands: + - export DOCKERHUB_PASS=$DOCKERHUB_PASS_PARAM + - docker login --username pwighton --password $DOCKERHUB_PASS + build: + on-failure: ABORT + commands: + - docker build -t pwighton/neurodocker . + post_build: + on-failure: ABORT + commands: + - docker push pwighton/neurodocker:latest From e6a2af8d2061ee29fa489994ba39bbce516f7c9b Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 9 Jun 2022 14:41:58 -0400 Subject: [PATCH 86/89] [WIP] syntax tweak to support commit param --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index 3df8d385..b1bc6d48 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -314,7 +314,7 @@ source: {%- if self.commit is defined and self.commit|length %} cd {{ self.source_path }} git checkout {{ self.commit }} - {% endif %} + {% endif -%} cd {{ self.install_path }} cmake {{ self.cmake_opts }} {{ self.source_path }} make {{ self.make_opts }} From 2900180048a28629c37c3f7d98eebb95276995a0 Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Thu, 9 Jun 2022 14:49:16 -0400 Subject: [PATCH 87/89] [GIT] removing buildspec before pushing upstream --- buildspec.yml | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 buildspec.yml diff --git a/buildspec.yml b/buildspec.yml deleted file mode 100644 index afa8982f..00000000 --- a/buildspec.yml +++ /dev/null @@ -1,23 +0,0 @@ -# see https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html -version: 0.2 - -env: - - shell: bash - parameter-store: - DOCKERHUB_PASS_PARAM: /dockerhub-access-token-for-pwighton - -phases: - install: - on-failure: ABORT - commands: - - export DOCKERHUB_PASS=$DOCKERHUB_PASS_PARAM - - docker login --username pwighton --password $DOCKERHUB_PASS - build: - on-failure: ABORT - commands: - - docker build -t pwighton/neurodocker . - post_build: - on-failure: ABORT - commands: - - docker push pwighton/neurodocker:latest From 730aff39d16b7ab7fde19fd3429b0e301edc468c Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Fri, 10 Jun 2022 10:49:42 -0400 Subject: [PATCH 88/89] [WIP] updating fs python to 3.6.8 --- neurodocker/templates/freesurfer.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index b1bc6d48..77dceb6a 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -276,15 +276,15 @@ source: # See: https://github.com/ReproNim/neurodocker/issues/246 # ------------------------------------------------------------------------- mkdir -p /stage - # Python v3.6.8 + # Python v3.8.9 # ------------------------------------------------------------------------- - curl -sSL --retry 5 https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz | tar -xz -C /stage - cd /stage/Python-3.6.8 + curl -sSL --retry 5 https://www.python.org/ftp/python/3.8.9/Python-3.8.9.tgz | tar -xz -C /stage + cd /stage/Python-3.8.9 ./configure make make install cd / - rm -rf /stage/Python-3.6.8 + rm -rf /stage/Python-3.8.9 pip3 install -q --no-cache-dir -U pip pip3 install -q --no-cache-dir wheel sync From 26251841ea865d6922e7f211b7fc13a08661676e Mon Sep 17 00:00:00 2001 From: Paul Wighton Date: Wed, 3 Aug 2022 15:03:19 -0400 Subject: [PATCH 89/89] [FIX] fix for issue #449 --- neurodocker/templates/freesurfer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/templates/freesurfer.yaml b/neurodocker/templates/freesurfer.yaml index e7e15a9b..bdd32442 100644 --- a/neurodocker/templates/freesurfer.yaml +++ b/neurodocker/templates/freesurfer.yaml @@ -56,7 +56,7 @@ binaries: # From https://github.com/freesurfer/freesurfer/blob/54018f7d6f620d6288b28f50e14a0a4ba421757c/Dockerfile#L20-L42 # freesurfer env OS: "Linux" - PATH: "{{ self.install_path }}/bin:{{ self.install_path }}/fsfast/bin:{{ self.install_path }}/tktools:{{ self.install_path }}/mni/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + PATH: "{{ self.install_path }}/bin:{{ self.install_path }}/fsfast/bin:{{ self.install_path }}/tktools:{{ self.install_path }}/mni/bin:$PATH" FREESURFER_HOME: "{{ self.install_path }}" FREESURFER: "{{ self.install_path }}" SUBJECTS_DIR: "{{ self.install_path }}/subjects"