Allow dicom files or directories and fix field map json completion (#7)

Summary --------- ENH: Better completion of fieldmap jsons ENH: Input data can be tarball/tar.gz or directory, bidsifier will detect the type and apply accordingly * Major Refactoring * More changes * Fix * Fix * Update bidsify.py * Update bidsify.py * Update bidsify.py * Update bidsify.py * Update bidsify.py * Update bidsify.py * Update bidsify.py * Update bidsify.py * Update bidsify.py * Update bidsify.py * Update bidsify.py * Update bidsify.py * moved files * Update bidsconvert.sh * Update bidsify.py * Update bidsconvert.sh * Fixed some behavior * Update clean_metadata.py * Preparing for docker build * Fixed Line Endings * Update Dockerfile * Update bidsconvert.sh * first pass * Refactoring * Debug * Update complete_jsons.py * Update complete_jsons.py * Update complete_jsons.py * Update complete_jsons.py * Debug * Debug * Update complete_jsons.py * Added helper function for cleaning up directory ENH: Added Helper function maintain_bids to clean up working directories post run MAINT: Removed troubleshooting prints * Update bidsconvert.sh * Update Dockerfile * Update Dockerfile * Update Dockerfile * Update Dockerfile * Cleaning * Update bidsify.py * Update bidsify.py * Update bidsify.py * Update bidsify.py * Update complete_jsons.py * Update complete_jsons.py * Update complete_jsons.py * debugging * Update complete_jsons.py * FINALLY * Update complete_jsons.py * Update complete_jsons.py * Update complete_jsons.py * Requested Changes Modified clean_metadata to not remove fields, only unwrap global keys * Update bidsify.py * Update bidsify.py Co-Authored-By: Taylor Salo <[email protected]> * Update clean_metadata.py Co-Authored-By: Taylor Salo <[email protected]> * Update clean_metadata.py * Update clean_metadata.py * Update clean_metadata.py Co-Authored-By: Taylor Salo <[email protected]> * Update clean_metadata.py * Update Dockerfile
fiuneuro · Oct 10, 2019 · 426f825 · 426f825
1 parent 5fc7597
commit 426f825
Show file tree

Hide file tree

Showing 6 changed files with 248 additions and 296 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -45,7 +45,7 @@ RUN apt-get update -qq \
 
 # nvm environment variables
 ENV NVM_DIR /usr/local/nvm
-ENV NODE_VERSION 8.0.0
+ENV NODE_VERSION 10.16.3
 
 # install nvm
 # https://github.com/creationix/nvm#install-script
@@ -62,15 +62,15 @@ ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules
 ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
 
 #------------------------
-# Install dcm2niix v1.0.20171215
+# Install dcm2niix v1.0.20190410
 #------------------------
 WORKDIR /tmp
 RUN deps='cmake g++ gcc git make pigz zlib1g-dev' \
     && apt-get update -qq && apt-get install -yq --no-install-recommends $deps \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
     && mkdir dcm2niix \
-    && curl -sSL https://github.com/rordenlab/dcm2niix/tarball/v1.0.20171215 | tar xz -C dcm2niix --strip-components 1 \
+    && curl -sSL https://github.com/rordenlab/dcm2niix/tarball/v1.0.20190410 | tar xz -C dcm2niix --strip-components 1 \
     && mkdir dcm2niix/build && cd dcm2niix/build \
     && cmake .. && make \
     && make install \
@@ -99,14 +99,15 @@ RUN conda create -y -q --name neuro python=3 \
     && sync && conda clean -tipsy && sync \
     && /bin/bash -c "source activate neuro \
       && pip install git+git://github.com/FIU-Neuro/dcmstack \
-      && pip install numpy pandas pybids nibabel heudiconv pydicom python-dateutil" \
+      && pip install numpy pandas pybids nibabel pydicom python-dateutil \
+      && pip install git+git://github.com/nipy/heudiconv@202f9434819318055e5293486f6bdac489989c52" \
     && sync \
     && sed -i '$isource activate neuro' $ND_ENTRYPOINT
 
 #---------------
 # BIDS-validator
 #---------------
-RUN npm install -g bids-validator@0.27.5
+RUN npm install -g bids-validator@1.3.0
 
 #--------------------------------------------------
 # Add NeuroDebian repository
@@ -166,4 +167,7 @@ ENV SINGULARITY_TMPDIR /scratch
 # Set entrypoint script
 #----------------------
 COPY ./ /scripts/
+USER root
+RUN chmod 755 -R /scripts/
+USER neuro
 ENTRYPOINT ["/neurodocker/startup.sh", "/scripts/bidsify.py"]
diff --git a/bidsconvert.sh b/bidsconvert.sh
@@ -9,12 +9,12 @@
 ##########################
 # Get command line options
 ##########################
-scratchdir=$1
-heuristics=$2
-project=$3
-sub=$4
-sess=${5:-None}
-
+dir_type=$1
+dicom_dir=$2
+heuristics=$3
+out_dir=$4
+sub=$5
+sess=${6:-None}
 ######################################
 ######################################
 
@@ -23,40 +23,40 @@ sess=${5:-None}
 #############################################
 if [ "$sess" = "None" ]; then
   # Put data in BIDS format
-  heudiconv -d $scratchdir/sub-{subject}.tar -s $sub -f \
-    $heuristics -c dcm2niix -o $scratchdir/bids/ --bids --overwrite
+  heudiconv $dir_type $dicom_dir -s $sub -f \
+    $heuristics -c dcm2niix -o $out_dir --bids --overwrite --minmeta
   minipath=sub-$sub
 else
   # Put data in BIDS format
-  heudiconv -d $scratchdir/sub-{subject}-ses-{session}.tar -s $sub -ss $sess -f \
-    $heuristics -c dcm2niix -o $scratchdir/bids/ --bids --overwrite
+  heudiconv $dir_type $dicom_dir -s $sub -ss $sess -f \
+    $heuristics -c dcm2niix -o $out_dir --bids --overwrite --minmeta
   minipath=sub-$sub/ses-$sess
 fi
 
 ##############################################
 # Check results, anonymize, and clean metadata
 ##############################################
-if [ -d $scratchdir/bids/$minipath ]; then
-  chmod -R 774 $scratchdir/bids/$minipath
+if [ -d $out_dir/$minipath ]; then
+  chmod -R 774 $out_dir/$minipath
 
   # Deface structural scans
-  imglist=$(ls $scratchdir/bids/$minipath/anat/*.nii.gz)
-  for tmpimg in $imglist; do
-    mri_deface $tmpimg /src/deface/talairach_mixed_with_skull.gca \
+  if [ -d $out_dir/$minipath/anat/ ]; then
+    imglist=$(ls $out_dir/$minipath/anat/*.nii.gz)
+    for tmpimg in $imglist; do
+      mri_deface $tmpimg /src/deface/talairach_mixed_with_skull.gca \
       /src/deface/face.gca $tmpimg
-  done
-  rm ./*.log
+    done
+  fi
+  #rm ./*.log
 
   # Add IntendedFor and TotalReadoutTime fields to jsons
-  python /scripts/complete_jsons.py -d $scratchdir/bids/ -s $sub -ss $sess --overwrite
-
+  python /scripts/complete_jsons.py -d $out_dir -s $sub -ss $sess --overwrite
   # Remove extraneous fields from jsons
-  python /scripts/clean_metadata.py $scratchdir/bids/
-
+  python /scripts/clean_metadata.py $out_dir $sub $sess
   # Validate dataset and, if it passes, copy files to outdir
-  bids-validator $scratchdir/bids/ --ignoreWarnings > $scratchdir/validator.txt
+  bids-validator $out_dir --ignoreWarnings > $out_dir/validator.txt
 else
-  echo "FAILED" > $scratchdir/validator.txt
+  echo "FAILED" > $out_dir/validator.txt
   echo "Heudiconv failed to convert this dataset to BIDS format."
 fi
 ######################################

diff --git a/bidsify.py b/bidsify.py
@@ -1,20 +1,79 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 """
 From https://github.com/BIDS-Apps/example/blob/aa0d4808974d79c9fbe54d56d3b47bb2cf4e0a0d/run.py
 """
 import os
-import os.path as op
 import tarfile
+import pathlib
 import argparse
 import subprocess
-
+import shutil
 import pydicom
 import numpy as np
 import pandas as pd
 from dateutil.parser import parse
 
 
+def manage_dicom_dir(dicom_dir):
+    '''
+    Helper function to grab data from dicom header depending on the type of dicom
+    directory given
+
+    Parameters
+    ----------
+    dicom_dir: Directory containing dicoms for processing
+    '''
+    if dicom_dir.suffix in ('.gz', '.tar'):
+        open_type = 'r'
+        if dicom_dir.suffix == '.gz':
+            open_type = 'r:gz'
+        with tarfile.open(dicom_dir, open_type) as tar:
+            dicoms = [mem for mem in tar.getmembers() if
+                      mem.name.endswith('.dcm')]
+            f_obj = tar.extractfile(dicoms[0])
+            data = pydicom.read_file(f_obj)
+    elif dicom_dir.is_dir():
+        f_obj = [x for x in pathlib.Path(dicom_dir).glob('**/*.dcm')][0].as_posix()
+        data = pydicom.read_file(f_obj)
+    return data
+
+
+def maintain_bids(output_dir, sub, ses):
+    '''
+    Function that cleans up working directories when called,
+    if all work is complete, will return directory to bids standard
+    (removing .heudiconv and tmp directories)
+
+    Parameters
+    ----------
+    output_dir: Path object of bids directory
+    sub: Subject ID
+    ses: Session ID, if required
+    '''
+    for root in ['.heudiconv', 'tmp']:
+        if ses:
+            if root == '.heudiconv':
+                shutil.rmtree(output_dir / root / sub / f'ses-{ses}')
+            else:
+                shutil.rmtree(output_dir / root / sub / ses)
+        if (output_dir / root / sub).is_dir():
+            if not [x for x in (output_dir / root / sub).iterdir()]:
+                shutil.rmtree((output_dir / root / sub))
+        if (output_dir / root).is_dir():
+            if not [x for x in (output_dir / root).iterdir()]:
+                shutil.rmtree((output_dir / root))
+
+
 def run(command, env={}):
+    '''
+    Helper function that runs a given command and allows for specification of
+    environment information
+
+    Parameters
+    ----------
+    command: command to be sent to system
+    env: parameters to be added to environment
+    '''
     merged_env = os.environ
     merged_env.update(env)
     process = subprocess.Popen(command, stdout=subprocess.PIPE,
@@ -34,59 +93,83 @@ def run(command, env={}):
 
 
 def get_parser():
+    '''
+    Sets up argument parser for scripts
+
+    Parameters
+    ----------
+    '''
     parser = argparse.ArgumentParser(description='BIDS conversion and '
                                                  'anonymization for the FIU '
                                                  'scanner.')
     parser.add_argument('-d', '--dicomdir', required=True, dest='dicom_dir',
                         help='Directory containing raw data.')
-    parser.add_argument('--heuristics', required=True, dest='heuristics',
+    parser.add_argument('-f', '--heuristics', required=True, dest='heuristics',
                         help='Path to the heuristics file.')
-    parser.add_argument('--project', required=True, dest='project',
-                        help='Name of the project.')
-    parser.add_argument('--sub', required=True, dest='sub',
+    parser.add_argument('-s', '--sub', required=True, dest='sub',
                         help='The label of the subject to analyze.')
-    parser.add_argument('--ses', required=False, dest='ses',
+    parser.add_argument('-ss', '--ses', required=False, dest='ses',
                         help='Session number', default=None)
+    parser.add_argument('-o', '--output_dir', dest='output_dir', required=True)
     return parser
 
 
 def main(argv=None):
+    '''
+    Function that executes when bidsify.py is called
+
+    Parameters inherited from argparser
+    ----------
+    dicom_dir: Directory cointaining dicom data to be processed
+    heuristics: Path to heuristics file
+    sub: Subject ID
+    ses: Session ID, if required
+    output_dir: Directory to output bidsified data
+    '''
     args = get_parser().parse_args(argv)
 
-    # Check inputs
-    if args.ses is None:
-        tar_file = op.join(args.dicom_dir, 'sub-{0}.tar'.format(args.sub))
+    args.dicom_dir = pathlib.Path(args.dicom_dir)
+    args.heuristics = pathlib.Path(args.heuristics)
+    args.output_dir = pathlib.Path(args.output_dir)
+    if args.dicom_dir.is_file():
+        dir_type = '-d'
+        heudiconv_input = args.dicom_dir.as_posix().replace(args.sub, '{subject}')
+        if args.ses:
+            heudiconv_input = heudiconv_input.replace(args.ses, '{session}')
     else:
-        tar_file = op.join(args.dicom_dir,
-                           'sub-{0}-ses-{1}.tar'.format(args.sub, args.ses))
-
-    if not args.dicom_dir.startswith('/scratch'):
-        raise ValueError('Dicom files must be in scratch.')
-
-    if not op.isfile(tar_file):
-        raise ValueError('Argument "dicom_dir" must contain a file '
-                         'named {0}.'.format(op.basename(tar_file)))
-
-    if not op.isfile(args.heuristics):
+        dir_type = '--files'
+        heudiconv_input = args.dicom_dir.as_posix()
+    #if not args.dicom_dir.startswith('/scratch'):
+    #    raise ValueError('Dicom files must be in scratch.')
+    if not args.heuristics.is_file():
         raise ValueError('Argument "heuristics" must be an existing file.')
 
     # Compile and run command
-    cmd = ('/scripts/bidsconvert.sh {0} {1} {2} {3} {4}'.format(args.dicom_dir,
-                                                                args.heuristics,
-                                                                args.project,
-                                                                args.sub, args.ses))
-    run(cmd, env={'TMPDIR': args.dicom_dir})
+    cmd = ('/scripts/bidsconvert.sh {0} {1} {2} {3} {4} {5}'.format(dir_type,
+                                                                    heudiconv_input,
+                                                                    args.heuristics,
+                                                                    args.output_dir,
+                                                                    args.sub,
+                                                                    args.ses))
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    tmp_path = args.output_dir / 'tmp' / args.sub
+    if not (args.output_dir / '.bidsignore').is_file():
+        with (args.output_dir / '.bidsignore').open('a') as wk_file:
+            wk_file.write('.heudiconv/\n')
+            wk_file.write('tmp/\n')
+            wk_file.write('validator.txt\n')
+    if args.ses:
+        tmp_path = tmp_path / args.ses
+    tmp_path.mkdir(parents=True, exist_ok=True)
+    run(cmd, env={'TMPDIR': tmp_path.name})
+    #Cleans up output directory, returning it to bids standard
+    maintain_bids(args.output_dir, args.sub, args.ses)
 
     # Grab some info to add to the participants file
-    participants_file = op.join(args.dicom_dir, 'bids/participants.tsv')
-    if op.isfile(participants_file):
+    participants_file = args.output_dir / 'participants.tsv'
+    if participants_file.is_file():
         df = pd.read_csv(participants_file, sep='\t')
-        with tarfile.open(tar_file, 'r') as tar:
-            dicoms = [mem for mem in tar.getmembers() if
-                      mem.name.endswith('.dcm')]
-            f_obj = tar.extractfile(dicoms[0])
-            data = pydicom.read_file(f_obj)
-
+        data = manage_dicom_dir(args.dicom_dir)
         if data.get('PatientAge'):
             age = data.PatientAge.replace('Y', '')
             try:
@@ -100,7 +183,7 @@ def main(argv=None):
             age = np.nan
         df2 = pd.DataFrame(columns=['age', 'sex', 'weight'],
                            data=[[age, data.PatientSex, data.PatientWeight]])
-        df = pd.concat((df, df2), axis=1)
+        df = pd.concat([df, df2], axis=1)
         df.to_csv(participants_file, sep='\t', index=False)
 
 

diff --git a/build_image.sh b/build_image.sh