Merge branch 'main' into bs_docker_usage_sum

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.0.3
+current_version = 4.0.8
 commit = True
 tag = True
 tag_name = {new_version}

.dockstore.yml

@@ -94,5 +94,5 @@ workflows:
   subclass: wdl
   primaryDescriptorPath: /wdl/pipelines/ONT/Preprocessing/ONTFlowcellFromMultipleBasecalls.wdl
 - name: CleanupIntermediate
-  subclass:wdl
+  subclass: wdl
   primaryDescriptorPath: /wdl/pipelines/TechAgnostic/Utility/CleanupIntermediate.wdl

.github/workflows/ci_nightly.yml

@@ -68,7 +68,7 @@ jobs:
       env:
         SLACK_MESSAGE: 'Nightly test of the main branch failed.'
         SLACK_COLOR: '#DF5A49'
-        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_NIGHTLY }}
+        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         SLACK_ICON: https://github.com/broadinstitute/long-read-pipelines/blob/main/.github/workflows/dnabad.png?raw=true
         SLACK_USERNAME: long-read-pipelines
 
@@ -79,6 +79,6 @@ jobs:
       env:
         SLACK_MESSAGE: 'Nightly test of the main branch successful!'
         SLACK_COLOR: '#50D9C9'
-        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_NIGHTLY }}
+        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         SLACK_ICON: https://github.com/broadinstitute/long-read-pipelines/blob/main/.github/workflows/dnagood.png?raw=true
         SLACK_USERNAME: long-read-pipelines

.github/workflows/validate_dockstore_yaml.yml

@@ -7,11 +7,11 @@ jobs:
             - name: Checkout branch
               uses: actions/checkout@v3
 
-            - name: Setup Java 11
+            - name: Setup Java 17
               uses: actions/setup-java@v2
               with:
                 distribution: 'temurin'
-                java-version: '11'
+                java-version: '17'
 
             - name: Cache node_modules
               uses: actions/cache@v2
@@ -22,7 +22,7 @@ jobs:
             - name: Install dockstore cli
               run: |
                 mkdir -p ~/bin
-                curl -L -o ~/bin/dockstore https://github.com/dockstore/dockstore-cli/releases/download/1.13.1/dockstore
+                curl -L -o ~/bin/dockstore https://github.com/dockstore/dockstore-cli/releases/download/1.14.0/dockstore
                 chmod +x ~/bin/dockstore
                 export PATH=~/bin:$PATH
                 
@@ -34,5 +34,11 @@ jobs:
             - name: Validate dockstore.yml
               run: |
                 export PATH=~/bin:$PATH
-                dockstore yaml validate --path . 2>&1 | tee dockstore_result.txt
-                if grep -q "error" dockstore_result.txt ; then exit 1 ; fi
+
+                validation_output=$(dockstore yaml validate --path .)
+                echo "$validation_output"
+                                
+                if [[ $validation_output == *"Your .dockstore.yml has the following errors"* || $validation_output == *"Your .dockstore.yml is invalid"* ]]; then
+                  echo "Dockstore validation failed."
+                  exit 1
+                fi

.gitignore

@@ -10,3 +10,4 @@ venv/
 .tox
 test/test_output/
 __pycache__
+.ipynb_checkpoints/

README.md

@@ -1,4 +1,4 @@
-[![Generic badge](https://img.shields.io/badge/version-4.0.3-blue.svg)](https://shields.io/)
+[![Generic badge](https://img.shields.io/badge/version-4.0.8-blue.svg)](https://shields.io/)
 ![CI/CD](https://github.com/broadinstitute/long-read-pipelines/workflows/CI/CD/badge.svg)
 ![Nightly](https://github.com/broadinstitute/long-read-pipelines/workflows/Nightly/badge.svg)
 

VERSION

@@ -1,2 +1,2 @@
 
-version=4.0.3
+version=4.0.8

wdl/deprecated/ONT10x.wdl

@@ -69,7 +69,7 @@ workflow ONT10x {
                 Int splint_num = a.left
                 File fq = a.right
 
-#                call Utils.FastaToSam as FastaToSam { input: fasta = C3POa.consensus }
+#                call FastaToSam { input: fasta = C3POa.consensus }
 #                call AnnotateAdapters { input: bam = FastaToSam.output_bam }
 
                 String rg_consensus = "@RG\\tID:~{SID}.consensus~{splint_num}\\tSM:~{SM}\\tPL:~{PL}\\tPU:~{PU}\\tDT:~{DT}"
@@ -82,7 +82,7 @@ workflow ONT10x {
                         map_preset = "splice"
                 }
 
-                #call Utils.CountFastaRecords as CountConsensusReadsInPartition { input: fasta = fq }
+                #call CountFastaRecords as CountConsensusReadsInPartition { input: fasta = fq }
             }
 
 #            File align_subreads_bam = AlignSubreads.aligned_bam
@@ -97,17 +97,17 @@ workflow ONT10x {
             call CountNumPasses as CountNumPasses3 { input: fastq = C3POa.subreads3 }
             call CountNumPasses as CountNumPasses4 { input: fastq = C3POa.subreads4 }
 
-            call Utils.CountFastqRecords as CountSubreadsInPartition1 { input: fastq = C3POa.subreads1 }
-            call Utils.CountFastqRecords as CountSubreadsInPartition2 { input: fastq = C3POa.subreads2 }
-            call Utils.CountFastqRecords as CountSubreadsInPartition3 { input: fastq = C3POa.subreads3 }
-            call Utils.CountFastqRecords as CountSubreadsInPartition4 { input: fastq = C3POa.subreads4 }
+            call CountFastqRecords as CountSubreadsInPartition1 { input: fastq = C3POa.subreads1 }
+            call CountFastqRecords as CountSubreadsInPartition2 { input: fastq = C3POa.subreads2 }
+            call CountFastqRecords as CountSubreadsInPartition3 { input: fastq = C3POa.subreads3 }
+            call CountFastqRecords as CountSubreadsInPartition4 { input: fastq = C3POa.subreads4 }
 
-#            call Utils.CountFastqRecords as CountAnnotatedReadsInPartition { input: fastq = AnnotateAdapters.annotated_fq }
+#            call CountFastqRecords as CountAnnotatedReadsInPartition { input: fastq = AnnotateAdapters.annotated_fq }
 
-            call Utils.CountFastaRecords as CountConsensusReadsInPartition1 { input: fasta = C3POa.consensus1 }
-            call Utils.CountFastaRecords as CountConsensusReadsInPartition2 { input: fasta = C3POa.consensus2 }
-            call Utils.CountFastaRecords as CountConsensusReadsInPartition3 { input: fasta = C3POa.consensus3 }
-            call Utils.CountFastaRecords as CountConsensusReadsInPartition4 { input: fasta = C3POa.consensus4 }
+            call CountFastaRecords as CountConsensusReadsInPartition1 { input: fasta = C3POa.consensus1 }
+            call CountFastaRecords as CountConsensusReadsInPartition2 { input: fasta = C3POa.consensus2 }
+            call CountFastaRecords as CountConsensusReadsInPartition3 { input: fasta = C3POa.consensus3 }
+            call CountFastaRecords as CountConsensusReadsInPartition4 { input: fasta = C3POa.consensus4 }
         }
 
         call Utils.Sum as CountNoSplintReadsInRun  { input: ints = C3POa.no_splint_reads }
@@ -365,3 +365,160 @@ task CountNumPasses {
         docker:                 select_first([runtime_attr.docker,            default_attr.docker])
     }
 }
+
+task FastaToSam {
+
+    meta {
+        description: "Convert a fasta file to a sam file"
+    }
+
+    parameter_meta {
+        fasta: "The fasta file"
+        runtime_attr_override: "Override the default runtime attributes"
+    }
+
+    input {
+        File fasta
+
+        RuntimeAttr? runtime_attr_override
+    }
+
+    Float fasta_sam_disk_multiplier = 3.25
+    Int disk_size = ceil(fasta_sam_disk_multiplier * size(fasta, "GiB")) + 20
+
+    command <<<
+        python /usr/local/bin/prepare_run.py ~{fasta}
+    >>>
+
+    output {
+        File output_bam = "unmapped.bam"
+    }
+
+    #########################
+    RuntimeAttr default_attr = object {
+        cpu_cores:          1,
+        mem_gb:             1,
+        disk_gb:            disk_size,
+        boot_disk_gb:       10,
+        preemptible_tries:  2,
+        max_retries:        1,
+        docker:             "us.gcr.io/broad-dsp-lrma/lr-align:0.1.28"
+    }
+    RuntimeAttr runtime_attr = select_first([runtime_attr_override, default_attr])
+    runtime {
+        cpu:                    select_first([runtime_attr.cpu_cores,         default_attr.cpu_cores])
+        memory:                 select_first([runtime_attr.mem_gb,            default_attr.mem_gb]) + " GiB"
+        disks: "local-disk " +  select_first([runtime_attr.disk_gb,           default_attr.disk_gb]) + " HDD"
+        bootDiskSizeGb:         select_first([runtime_attr.boot_disk_gb,      default_attr.boot_disk_gb])
+        preemptible:            select_first([runtime_attr.preemptible_tries, default_attr.preemptible_tries])
+        maxRetries:             select_first([runtime_attr.max_retries,       default_attr.max_retries])
+        docker:                 select_first([runtime_attr.docker,            default_attr.docker])
+    }
+}
+
+task CountFastqRecords {
+
+    meta {
+        description: "Count the number of records in a fastq file"
+    }
+
+    parameter_meta {
+        fastq: "The fastq file"
+        runtime_attr_override: "Override the default runtime attributes"
+    }
+
+    input {
+        File fastq
+
+        RuntimeAttr? runtime_attr_override
+    }
+
+    Int disk_size = 1 + ceil(2 * size(fastq, "GiB"))
+
+    command <<<
+        set -euxo pipefail
+
+        FILE="~{fastq}"
+        if [[ "$FILE" =~ \.fastq$ ]] || [[ "$FILE" =~ \.fq$ ]]; then
+            cat ~{fastq} | awk '{s++}END{print s/4}'
+        elif [[ "$FILE" =~ \.fastq.gz$ ]] || [[ "$FILE" =~ \.fq.gz$ ]]; then
+            zcat ~{fastq} | awk '{s++}END{print s/4}'
+        fi
+    >>>
+
+    output {
+        Int num_records = read_int(stdout())
+    }
+
+    #########################
+    RuntimeAttr default_attr = object {
+        cpu_cores:          1,
+        mem_gb:             1,
+        disk_gb:            disk_size,
+        boot_disk_gb:       10,
+        preemptible_tries:  2,
+        max_retries:        1,
+        docker:             "us.gcr.io/broad-dsp-lrma/lr-align:0.1.28"
+    }
+    RuntimeAttr runtime_attr = select_first([runtime_attr_override, default_attr])
+    runtime {
+        cpu:                    select_first([runtime_attr.cpu_cores,         default_attr.cpu_cores])
+        memory:                 select_first([runtime_attr.mem_gb,            default_attr.mem_gb]) + " GiB"
+        disks: "local-disk " +  select_first([runtime_attr.disk_gb,           default_attr.disk_gb]) + " HDD"
+        bootDiskSizeGb:         select_first([runtime_attr.boot_disk_gb,      default_attr.boot_disk_gb])
+        preemptible:            select_first([runtime_attr.preemptible_tries, default_attr.preemptible_tries])
+        maxRetries:             select_first([runtime_attr.max_retries,       default_attr.max_retries])
+        docker:                 select_first([runtime_attr.docker,            default_attr.docker])
+    }
+}
+
+task CountFastaRecords {
+
+    meta {
+        description: "Count the number of records in a fasta file"
+    }
+
+    parameter_meta {
+        fasta: "The fasta file"
+        runtime_attr_override: "Override the default runtime attributes"
+    }
+
+    input {
+        File fasta
+
+        RuntimeAttr? runtime_attr_override
+    }
+
+    Int disk_size = 1 + 2*ceil(size(fasta, "GiB"))
+
+    command <<<
+        grep -c '>' ~{fasta}
+
+        exit 0
+    >>>
+
+    output {
+        Int num_records = read_int(stdout())
+    }
+
+    #########################
+    RuntimeAttr default_attr = object {
+        cpu_cores:          1,
+        mem_gb:             1,
+        disk_gb:            disk_size,
+        boot_disk_gb:       10,
+        preemptible_tries:  2,
+        max_retries:        1,
+        docker:             "us.gcr.io/broad-dsp-lrma/lr-align:0.1.28"
+    }
+    RuntimeAttr runtime_attr = select_first([runtime_attr_override, default_attr])
+    runtime {
+        cpu:                    select_first([runtime_attr.cpu_cores,         default_attr.cpu_cores])
+        memory:                 select_first([runtime_attr.mem_gb,            default_attr.mem_gb]) + " GiB"
+        disks: "local-disk " +  select_first([runtime_attr.disk_gb,           default_attr.disk_gb]) + " HDD"
+        bootDiskSizeGb:         select_first([runtime_attr.boot_disk_gb,      default_attr.boot_disk_gb])
+        preemptible:            select_first([runtime_attr.preemptible_tries, default_attr.preemptible_tries])
+        maxRetries:             select_first([runtime_attr.max_retries,       default_attr.max_retries])
+        docker:                 select_first([runtime_attr.docker,            default_attr.docker])
+    }
+}