🔀 Merge main into C-PAC-1.8.5

.gitignore

@@ -0,0 +1 @@
+.DS_Store

LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2021,  Lifespan Informatics and Neuroimaging Center
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -0,0 +1,2 @@
+# TheWay
+Boilerplate scripts for running workflows on Penn clusters

Untitled Diagram.drawio

@@ -0,0 +1,97 @@
+<mxfile host="app.diagrams.net" modified="2021-10-11T17:38:49.174Z" agent="5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Safari/605.1.15" etag="qySIbvWX67eLjWm0QLuf" version="15.5.1" type="github">
+  <diagram id="C5RBs43oDa-KdzZeNtuy" name="Page-1">
+    <mxGraphModel dx="788" dy="531" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+      <root>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-0" />
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-1" parent="WIyWlLk6GJQsqaUBKTNV-0" />
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-27" value="" style="rounded=1;whiteSpace=wrap;html=1;fontSize=10;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="210" y="60" width="330" height="350" as="geometry" />
+        </mxCell>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-3" value="hcpya" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#dae8fc;strokeColor=#6c8ebf;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
+          <mxGeometry x="80" y="80" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-6" value="minimal preprocessing" style="rhombus;whiteSpace=wrap;html=1;shadow=0;fontFamily=Helvetica;fontSize=10;align=center;strokeWidth=1;spacing=6;spacingTop=-4;fillColor=#d5e8d4;strokeColor=#82b366;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
+          <mxGeometry x="90" y="180" width="100" height="80" as="geometry" />
+        </mxCell>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-10" value="dcan&amp;nbsp;&lt;br style=&quot;font-size: 10px;&quot;&gt;preprocessing&lt;br style=&quot;font-size: 10px;&quot;&gt;" style="rhombus;whiteSpace=wrap;html=1;shadow=0;fontFamily=Helvetica;fontSize=10;align=center;strokeWidth=1;spacing=6;spacingTop=-4;fillColor=#d5e8d4;strokeColor=#82b366;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
+          <mxGeometry x="330" y="170" width="100" height="80" as="geometry" />
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-0" value="hcp-d" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="240" y="80" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-1" value="abcd" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="400" y="80" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-2" value="hbn" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="560" y="80" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-8" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="qOXWICZVVOMb8-LOwz_p-1" target="WIyWlLk6GJQsqaUBKTNV-10">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="360" y="200" as="sourcePoint" />
+            <mxPoint x="410" y="150" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-9" value="" style="endArrow=classic;html=1;rounded=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-3" target="WIyWlLk6GJQsqaUBKTNV-6">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="360" y="200" as="sourcePoint" />
+            <mxPoint x="410" y="150" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-11" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="qOXWICZVVOMb8-LOwz_p-0" target="WIyWlLk6GJQsqaUBKTNV-10">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="360" y="200" as="sourcePoint" />
+            <mxPoint x="410" y="150" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-15" value="" style="endArrow=classic;html=1;rounded=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-10" target="qOXWICZVVOMb8-LOwz_p-30">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="360" y="330" as="sourcePoint" />
+            <mxPoint x="410" y="280" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-16" value="xcp outputs" style="rhombus;whiteSpace=wrap;html=1;shadow=0;fontFamily=Helvetica;fontSize=12;align=center;strokeWidth=1;spacing=6;spacingTop=-4;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="90" y="320" width="100" height="80" as="geometry" />
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-18" value="" style="endArrow=classic;html=1;rounded=0;fontSize=10;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-6" target="qOXWICZVVOMb8-LOwz_p-16">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="360" y="250" as="sourcePoint" />
+            <mxPoint x="410" y="200" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-20" value="fmriprep (v21.11?)" style="rhombus;whiteSpace=wrap;html=1;shadow=0;fontFamily=Helvetica;fontSize=10;align=center;strokeWidth=1;spacing=6;spacingTop=-4;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="570" y="170" width="100" height="80" as="geometry" />
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-28" value="data already exists" style="rounded=1;whiteSpace=wrap;html=1;fontSize=10;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="280" y="60" width="210" height="20" as="geometry" />
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-30" value="dcan&lt;br&gt;outputs" style="rhombus;whiteSpace=wrap;html=1;shadow=0;fontFamily=Helvetica;fontSize=10;align=center;strokeWidth=1;spacing=6;spacingTop=-4;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="330" y="320" width="100" height="80" as="geometry" />
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-33" value="" style="endArrow=none;dashed=1;html=1;rounded=0;fontSize=10;exitX=0;exitY=0;exitDx=0;exitDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="qOXWICZVVOMb8-LOwz_p-20" target="qOXWICZVVOMb8-LOwz_p-1">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="480" y="190" as="sourcePoint" />
+            <mxPoint x="530" y="140" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-34" value="" style="endArrow=none;dashed=1;html=1;rounded=0;fontSize=10;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="qOXWICZVVOMb8-LOwz_p-0" target="qOXWICZVVOMb8-LOwz_p-20">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="480" y="190" as="sourcePoint" />
+            <mxPoint x="530" y="140" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-36" value="" style="endArrow=none;dashed=1;html=1;rounded=0;fontSize=10;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="qOXWICZVVOMb8-LOwz_p-20" target="qOXWICZVVOMb8-LOwz_p-2">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="480" y="190" as="sourcePoint" />
+            <mxPoint x="530" y="140" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="qOXWICZVVOMb8-LOwz_p-40" value="" style="endArrow=none;dashed=1;html=1;rounded=0;fontSize=10;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="qOXWICZVVOMb8-LOwz_p-16" target="qOXWICZVVOMb8-LOwz_p-20">
+          <mxGeometry width="50" height="50" relative="1" as="geometry">
+            <mxPoint x="430" y="190" as="sourcePoint" />
+            <mxPoint x="480" y="140" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>

scripts/cubic/add_archive_content.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+#$ -l h_vmem=25G
+#$ -R y 
+
+# Adds output files to archive so thier filestubs are accessible by users 
+
+# Set up the correct conda environment
+source ${CONDA_PREFIX}/bin/activate base
+echo I\'m in $PWD using `which python`
+
+cd ${PWD}
+
+BOOTSTRAP_DIR=$1
+
+# clone output ria of bootstrap dir 
+datalad clone ria+file:///${BOOTSTRAP_DIR}/output_ria#~data archive_clone
+
+# cd into clone 
+cd archive_clone
+
+for zip in *.zip; do
+    echo Adding archive content for ${zip}
+    datalad get ${zip}
+    datalad add-archive-content -e 'logs/.*' -e '.bidsignore' -e 'dataset_description.json' -e 'dwiqc.json' --drop-after ${zip}
+done
+
+datalad push

scripts/cubic/add_urls_hcpd.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+#$ -l h_vmem=25G
+#$ -R y 
+# Set up the correct conda environment
+source ${CONDA_PREFIX}/bin/activate base
+echo I\'m in $PWD using `which python`
+
+SUBJECTIDCSV=/cbica/projects/RBC/testing/hcpd/hcpd_subject_ids.csv
+S3CSV=/cbica/projects/RBC/testing/hcpd/hcpd_s3.csv
+
+# create HCPD datalad dataset
+datalad create -c text2git HCPD
+
+# cd into HCPD datalad dataset 
+cd HCPD
+
+# initialize git annex remote
+git annex initremote datalad type=external externaltype=datalad encryption=none autoenable=true
+
+# get the dataset config file from github
+wget https://raw.githubusercontent.com/TobiasKadelka/build_hcp/master/code/cfg_hcp_dataset.sh
+
+# get list of subject IDs and run addurls for each subject
+SUBJECTS=$(cut -d, -f1 ${SUBJECTIDCSV})
+
+# heredoc for subject csv creator 
+cat > get_subject_csv.py << "EOT"
+#!/usr/bin/env python
+"""
+USAGE:
+python participant_csv.py subid
+Run this inside of participant_job.sh
+Creates csv for one single participant 
+"""
+import pandas as pd
+import sys
+hcpdcsv = sys.argv[1]
+subid = sys.argv[2]
+df = pd.read_csv(hcpdcsv)
+df2=df[df.filename.str.startswith(subid)]
+df3 = df2.drop_duplicates(subset ="filename", keep = 'first', ignore_index=True)
+df3.to_csv("/cbica/projects/RBC/testing/hcpd/subject_csvs/" + subid + ".csv", index=False)
+EOT
+
+
+
+chmod +x get_subject_csv.py
+
+datalad save -m "Added python file to create subject CSVs and HCPD dataset config"
+
+for subject in ${SUBJECTS}; do
+    echo Creating subject csv for ${subject}
+    python get_subject_csv.py ${S3CSV} ${subject}
+
+    echo Adding URLS for ${subject}
+    datalad addurls -c hcp_dataset -d ${subject} ~/testing/hcpd/subject_csvs/${subject}.csv '{associated_file}' '{filename}'
+    datalad save -m "Added URLs for ${subject}"
+done