diff --git a/.dockstore.yml b/.dockstore.yml index 8bc0a06..4ce4927 100644 --- a/.dockstore.yml +++ b/.dockstore.yml @@ -119,8 +119,13 @@ workflows: primaryDescriptorPath: /CleanupFailedSubmissions/Cleanup_Failed_Submissions.wdl testParameterFiles: - /CleanupFailedSubmissions/Cleanup_Failed_Submissions.inputs.json + - name: TAG_Mop + subclass: WDL + primaryDescriptorPath: /TAG_Mop/TAG_Mop.wdl + testParameterFiles: + - /TAG_Mop/TAG_Mop.inputs.json - name: QUICviz subclass: WDL primaryDescriptorPath: /PECGS-QUICviz/QUICviz.wdl testParameterFiles: - - /PECGS-QUICviz/QUICviz.inputs.json \ No newline at end of file + - /PECGS-QUICviz/QUICviz.inputs.json diff --git a/CleanupFailedSubmissions/Cleanup_Failed_Submissions.wdl b/CleanupFailedSubmissions/Cleanup_Failed_Submissions.wdl index 43b7f36..25bc6c1 100644 --- a/CleanupFailedSubmissions/Cleanup_Failed_Submissions.wdl +++ b/CleanupFailedSubmissions/Cleanup_Failed_Submissions.wdl @@ -35,6 +35,10 @@ workflow Cleanup_Failed_Submissions { submission_id = sid } } + +output { + Int num_failed_submissions = length(GetWorkspaceInfo.failed_submissions) + } } task GetWorkspaceInfo { @@ -87,7 +91,14 @@ task CleanupAFolder { } command <<< - timeout 23h gsutil -q rm -rf gs://~{bucket_name}/submissions/~{submission_id} || echo "Timed out. Please try again." + # Older version of Terra does not have the submission folder + if gsutil -q ls gs://~{bucket_name}/submissions/~{submission_id} >/dev/null 2>&1; then + timeout 23h gsutil -q rm -rf gs://~{bucket_name}/submissions/~{submission_id} || echo "Timed out. Please try again." + elif gsutil -q ls gs://~{bucket_name}/~{submission_id} >/dev/null 2>&1; then + timeout 23h gsutil -q rm -rf gs://~{bucket_name}/~{submission_id} || echo "Timed out. Please try again." + else + echo "Failed submission folder not found. This workspace has been cleaned up already." + fi >>> runtime { diff --git a/TAG_Mop/TAG_Mop.inputs.json b/TAG_Mop/TAG_Mop.inputs.json new file mode 100644 index 0000000..0c75a57 --- /dev/null +++ b/TAG_Mop/TAG_Mop.inputs.json @@ -0,0 +1,7 @@ +{ + "TAG_Mop.mopDocker": "String (optional, default = \"us.gcr.io/tag-team-160914/neovax-parsley:2.2.1.0\")", + "TAG_Mop.workspaceName": "String", + "TAG_Mop.namespace": "String (optional, default = \"broadtagteam\")", + "TAG_Mop.runMop": "Boolean" +} + diff --git a/TAG_Mop/TAG_Mop.wdl b/TAG_Mop/TAG_Mop.wdl new file mode 100644 index 0000000..d78c51c --- /dev/null +++ b/TAG_Mop/TAG_Mop.wdl @@ -0,0 +1,136 @@ +version 1.0 + +workflow TAG_Mop{ + input{ + String namespace = "broadtagteam" + String workspaceName + String mopDocker = "us.gcr.io/tag-team-160914/neovax-parsley:2.2.1.0" + Boolean runMop + } + + call rmSysfiles { + input: + namespace = namespace, + workspaceName = workspaceName, + mopDocker = mopDocker + } + + if (runMop){ + call mop { + input: + namespace = namespace, + workspaceName = workspaceName, + mopDocker = mopDocker, + sysfiles = rmSysfiles.deleted_sys_files + + } + } + + + output{ + Int num_deleted_sys_files = rmSysfiles.deleted_sys_files + File deleted_sys_files = rmSysfiles.sys_files_to_delete + Int? num_mopped_files = mop.num_of_files_to_mop + File? mopped_files = mop.mopped_files + } + + meta { + author: "Yueyao Gao" + email: "gaoyueya@broadinstitute.org" + description: "TAG Mop contains three sub-workflows: rmSysfiles and mop. rmSysfiles removes system files that were generated from submissions from a Terra workspace. mop runs the FISS Mop function. Suggest to run after cleanupFailedSubmission.wdl" + } + + } + + task rmSysfiles { + input{ + String namespace + String workspaceName + String mopDocker + Int memory = 32 + Int cpu = 8 + } + command <<< + source activate NeoVax-Input-Parser + python <>> + output{ + Int deleted_sys_files = read_int("num_of_sys_files_to_delete.txt") + File sys_files_to_delete = "sys_files_to_delete.txt" + } + runtime { + docker: mopDocker + memory: memory + " GiB" + cpu: cpu + } + } + + task mop { + input{ + String namespace + String workspaceName + String mopDocker + Int sysfiles + Int memory = 32 + Int cpu = 8 + } + command <<< + source activate NeoVax-Input-Parser + # The number of system files that were deleted + echo "System Files Deleted: ~{sysfiles}" + # Dry run Mop + fissfc mop -w ~{workspaceName} -p ~{namespace} --dry-run > mop_dry_run.txt + echo Files to mop:" $(cat mop_dry_run.txt | wc -l)" + cat mop_dry_run.txt | wc -l > num_of_files_to_mop.txt + + # Mop + if [ $(cat mop_dry_run.txt | wc -l) -eq 0 ]; then + echo "No files to mop" + else + fissfc mop -w ~{workspaceName} -p ~{namespace} + fi + >>> + output{ + Int num_of_files_to_mop = read_int("num_of_files_to_mop.txt") + File mopped_files = "mop_dry_run.txt" + } + runtime { + docker: mopDocker + memory: memory + " GiB" + cpu: cpu + } + }