From fd193f0e7117a9ea0a9cefc73b08874664ac4643 Mon Sep 17 00:00:00 2001 From: Sebastian Uhrig Date: Wed, 29 Mar 2023 17:11:52 +0200 Subject: [PATCH 01/17] number groups after removing irrelevant components from sample name --- bin/deseq2_qc.r | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/deseq2_qc.r b/bin/deseq2_qc.r index fa2e2c78a..d99bc7c2b 100755 --- a/bin/deseq2_qc.r +++ b/bin/deseq2_qc.r @@ -77,10 +77,10 @@ decompose <- n_components!=1 && all(sapply(name_components, length)==n_com coldata <- data.frame(samples.vec, sample=samples.vec, row.names=1) if (decompose) { groupings <- as.data.frame(lapply(1:n_components, function(i) sapply(name_components, "[[", i))) - names(groupings) <- paste0("Group", 1:n_components) n_distinct <- sapply(groupings, function(grp) length(unique(grp))) groupings <- groupings[n_distinct!=1 & n_distinct!=length(samples.vec)] if (ncol(groupings)!=0) { + names(groupings) <- paste0("Group", 1:ncol(groupings)) coldata <- cbind(coldata, groupings) } else { decompose <- FALSE From cd48584185044a011449114eb6e4ee9306463097 Mon Sep 17 00:00:00 2001 From: Sebastian Uhrig Date: Wed, 29 Mar 2023 18:21:04 +0200 Subject: [PATCH 02/17] update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42f7496ea..9528e55d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements - [[#961](https://github.com/nf-core/rnaseq/issues/961)] - Add warnings to STDOUT for all skipped and failed strandedness check samples - [[#975](https://github.com/nf-core/rnaseq/issues/975)] - `SALMON_INDEX` runs when using `--aligner star_rsem` even if samples have explicit strandedness - Remove HISAT2 from automated AWS full-sized tests +- [[#988](https://github.com/nf-core/rnaseq/issues/988)] - `DESEQ2_QC_STAR_SALMON` fails when sample names have many components ### Parameters From 930ab4d5b9d09a94f639d75f2701290f30759d6b Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 30 Mar 2023 15:00:34 +0100 Subject: [PATCH 03/17] Remove wait option from Tower Actions --- .github/workflows/cloud_tests_full.yml | 3 --- .github/workflows/cloud_tests_small.yml | 3 --- 2 files changed, 6 deletions(-) diff --git a/.github/workflows/cloud_tests_full.yml b/.github/workflows/cloud_tests_full.yml index 92d784131..d0c3d6e3e 100644 --- a/.github/workflows/cloud_tests_full.yml +++ b/.github/workflows/cloud_tests_full.yml @@ -36,7 +36,6 @@ jobs: "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}", "outdir": "${{ secrets.TOWER_BUCKET_AWS }}/rnaseq/results-${{ github.sha }}/aligner_${{ matrix.aligner }}" } - wait: false - uses: actions/upload-artifact@v3 with: name: Tower debug log file @@ -61,7 +60,6 @@ jobs: "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}", "outdir": "${{ secrets.TOWER_BUCKET_GCP }}/rnaseq/results-${{ github.sha }}/aligner_${{ matrix.aligner }}" } - wait: false - uses: actions/upload-artifact@v3 with: name: Tower debug log file @@ -87,7 +85,6 @@ jobs: "outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/rnaseq/results-${{ github.sha }}/aligner_${{ matrix.aligner }}", "igenomes_base": "${{ secrets.TOWER_IGENOMES_BASE_AZURE }}" } - wait: false - uses: actions/upload-artifact@v3 with: name: Tower debug log file diff --git a/.github/workflows/cloud_tests_small.yml b/.github/workflows/cloud_tests_small.yml index 318601912..6f2e6d8a9 100644 --- a/.github/workflows/cloud_tests_small.yml +++ b/.github/workflows/cloud_tests_small.yml @@ -30,7 +30,6 @@ jobs: { "outdir": "${{ secrets.TOWER_BUCKET_AWS }}/rnaseq/results-test-${{ github.sha }}" } - wait: false - uses: actions/upload-artifact@v3 with: name: Tower debug log file @@ -51,7 +50,6 @@ jobs: { "outdir": "${{ secrets.TOWER_BUCKET_GCP }}/rnaseq/results-test-${{ github.sha }}" } - wait: false - uses: actions/upload-artifact@v3 with: name: Tower debug log file @@ -72,7 +70,6 @@ jobs: { "outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/rnaseq/results-test-${{ github.sha }}" } - wait: false - uses: actions/upload-artifact@v3 with: name: Tower debug log file From 851e2c57e47f028c0f0a3f1c096739f9c955d67c Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 30 Mar 2023 15:04:30 +0100 Subject: [PATCH 04/17] Update CHANGELOG --- CHANGELOG.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ecef3a5c..c4034f5eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [[3.11.1](https://github.com/nf-core/rnaseq/releases/tag/3.11.1)] - 2023-03-30 + +### Credits + +Special thanks to the following for their code contributions to the release: + +- [Rob Syme](https://github.com/robsyme) +- [suhrig](https://github.com/suhrig) + +### Enhancements & fixes + +- [[#988](https://github.com/nf-core/rnaseq/issues/988)] - `DESEQ2_QC_STAR_SALMON` fails when sample names have many components +- Remove `wait: false` option from Tower Actions which is the default +- Fix release trigger for full-sized multi-cloud tests + ## [[3.11.0](https://github.com/nf-core/rnaseq/releases/tag/3.11.0)] - 2023-03-30 ### Credits @@ -37,7 +52,6 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements - [[#961](https://github.com/nf-core/rnaseq/issues/961)] - Add warnings to STDOUT for all skipped and failed strandedness check samples - [[#975](https://github.com/nf-core/rnaseq/issues/975)] - `SALMON_INDEX` runs when using `--aligner star_rsem` even if samples have explicit strandedness - Remove HISAT2 from automated AWS full-sized tests -- [[#988](https://github.com/nf-core/rnaseq/issues/988)] - `DESEQ2_QC_STAR_SALMON` fails when sample names have many components ### Parameters From f801b1eda588b5247a027806562dc880f872a981 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 30 Mar 2023 15:05:08 +0100 Subject: [PATCH 05/17] Bump pipeline version to 3.11.1 --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index c5ca21943..a5b94fd04 100644 --- a/nextflow.config +++ b/nextflow.config @@ -261,7 +261,7 @@ manifest { description = """RNA sequencing analysis pipeline for gene/isoform quantification and extensive quality control.""" mainScript = 'main.nf' nextflowVersion = '!>=22.10.1' - version = '3.11.0' + version = '3.11.1' doi = 'https://doi.org/10.5281/zenodo.1400710' } From 302254cdfe97f5132a5c8cc2a06b1836f317ee2a Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 30 Mar 2023 15:18:56 +0100 Subject: [PATCH 06/17] Update umitools modules to use different labels --- modules.json | 6 +++--- modules/nf-core/umitools/dedup/main.nf | 2 +- modules/nf-core/umitools/extract/main.nf | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules.json b/modules.json index 018c1ecae..a209ca3d2 100644 --- a/modules.json +++ b/modules.json @@ -211,13 +211,13 @@ }, "umitools/dedup": { "branch": "master", - "git_sha": "6d9c7e43404e20a97d2f6f88548456afe78282e6", + "git_sha": "ffe2f5865f608848e440a52b73c304ea79aaf818", "installed_by": ["bam_dedup_stats_samtools_umitools"] }, "umitools/extract": { "branch": "master", - "git_sha": "6d9c7e43404e20a97d2f6f88548456afe78282e6", - "installed_by": ["fastq_fastqc_umitools_trimgalore", "fastq_fastqc_umitools_fastp"] + "git_sha": "ffe2f5865f608848e440a52b73c304ea79aaf818", + "installed_by": ["fastq_fastqc_umitools_fastp", "fastq_fastqc_umitools_trimgalore"] }, "untar": { "branch": "master", diff --git a/modules/nf-core/umitools/dedup/main.nf b/modules/nf-core/umitools/dedup/main.nf index 68fc9b9e7..aa6fdd24e 100644 --- a/modules/nf-core/umitools/dedup/main.nf +++ b/modules/nf-core/umitools/dedup/main.nf @@ -1,6 +1,6 @@ process UMITOOLS_DEDUP { tag "$meta.id" - label "process_single" + label "process_medium" conda "bioconda::umi_tools=1.1.4" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? diff --git a/modules/nf-core/umitools/extract/main.nf b/modules/nf-core/umitools/extract/main.nf index ba2826e1f..64fdd367f 100644 --- a/modules/nf-core/umitools/extract/main.nf +++ b/modules/nf-core/umitools/extract/main.nf @@ -1,6 +1,7 @@ process UMITOOLS_EXTRACT { tag "$meta.id" label "process_single" + label "process_long" conda "bioconda::umi_tools=1.1.4" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? From 5c2456aabbf44be3c29b4abbf1a8e68480d96a00 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 30 Mar 2023 16:40:09 +0100 Subject: [PATCH 07/17] Test [ci fast] --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc63a1509..19fd3004a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,7 @@ jobs: star_salmon: name: Test STAR Salmon with workflow parameters - if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }} + if: ${{ (github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq')) && !contains(github.event.head_commit.message, '[ci fast]') }} runs-on: ubuntu-latest strategy: matrix: @@ -122,7 +122,7 @@ jobs: star_rsem: name: Test STAR RSEM with workflow parameters - if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }} + if: ${{ (github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq')) && !contains(github.event.head_commit.message, '[ci fast]') }} runs-on: ubuntu-latest strategy: matrix: @@ -168,7 +168,7 @@ jobs: hisat2: name: Test HISAT2 with workflow parameters - if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }} + if: ${{ (github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq')) && !contains(github.event.head_commit.message, '[ci fast]') }} runs-on: ubuntu-latest strategy: matrix: @@ -214,7 +214,7 @@ jobs: salmon: name: Test Salmon with workflow parameters - if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq') }} + if: ${{ (github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnaseq')) && !contains(github.event.head_commit.message, '[ci fast]') }} runs-on: ubuntu-latest strategy: matrix: From 490f319d52c9d07eb2f8c4aeef9f708d1cf9ca92 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 30 Mar 2023 16:55:19 +0100 Subject: [PATCH 08/17] Update CHANGELOG [ci fast] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4034f5eb..2ae20d8b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Special thanks to the following for their code contributions to the release: - [[#988](https://github.com/nf-core/rnaseq/issues/988)] - `DESEQ2_QC_STAR_SALMON` fails when sample names have many components - Remove `wait: false` option from Tower Actions which is the default - Fix release trigger for full-sized multi-cloud tests +- Adding `[ci fast]` to commit message now skips all tests except for standard `-profile test` pipeline run ## [[3.11.0](https://github.com/nf-core/rnaseq/releases/tag/3.11.0)] - 2023-03-30 From 00f578086c4f900c48695a6fe0e18f9d282ef2fc Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Wed, 29 Mar 2023 15:39:03 +0100 Subject: [PATCH 09/17] Create hash of github.workspace ID and use that as cache key --- .github/workflows/ci.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc63a1509..dd08f3c22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,8 +52,8 @@ jobs: - name: Replace remote paths in samplesheets run: | - for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do - sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f + for f in ./test-datasets/samplesheet/v3.10/*.csv; do + sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=./test-datasets/=g" $f echo "========== $f ============" cat $f echo "========================================" @@ -61,7 +61,7 @@ jobs: - name: Run pipeline with test data run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/ + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results --test_data_base ./test-datasets/ star_salmon: name: Test STAR Salmon with workflow parameters @@ -104,8 +104,8 @@ jobs: - name: Replace remote paths in samplesheets run: | - for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do - sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f + for f in ./test-datasets/samplesheet/v3.10/*.csv; do + sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=./test-datasets/=g" $f echo "========== $f ============" cat $f echo "========================================" @@ -118,7 +118,7 @@ jobs: - name: Run pipeline with STAR and various parameters run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_salmon ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/ + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_salmon ${{ matrix.parameters }} --outdir ./results --test_data_base ./test-datasets/ star_rsem: name: Test STAR RSEM with workflow parameters @@ -150,8 +150,8 @@ jobs: - name: Replace remote paths in samplesheets run: | - for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do - sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f + for f in ./test-datasets/samplesheet/v3.10/*.csv; do + sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=./test-datasets/=g" $f echo "========== $f ============" cat $f echo "========================================" @@ -164,7 +164,7 @@ jobs: - name: Run pipeline with RSEM STAR and various parameters run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_rsem ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/ + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_rsem ${{ matrix.parameters }} --outdir ./results --test_data_base ./test-datasets/ hisat2: name: Test HISAT2 with workflow parameters @@ -196,8 +196,8 @@ jobs: - name: Replace remote paths in samplesheets run: | - for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do - sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f + for f in ./test-datasets/samplesheet/v3.10/*.csv; do + sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=./test-datasets/=g" $f echo "========== $f ============" cat $f echo "========================================" @@ -210,7 +210,7 @@ jobs: - name: Run pipeline with HISAT2 and various parameters run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner hisat2 ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/ + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner hisat2 ${{ matrix.parameters }} --outdir ./results --test_data_base ./test-datasets/ salmon: name: Test Salmon with workflow parameters @@ -242,8 +242,8 @@ jobs: - name: Replace remote paths in samplesheets run: | - for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do - sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f + for f in ./test-datasets/samplesheet/v3.10/*.csv; do + sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=./test-datasets/=g" $f echo "========== $f ============" cat $f echo "========================================" @@ -256,4 +256,4 @@ jobs: - name: Run pipeline with Salmon and various parameters run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --pseudo_aligner salmon ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/ + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --pseudo_aligner salmon ${{ matrix.parameters }} --outdir ./results --test_data_base ./test-datasets/ From 1ac037e9e8fa6d4a1d7a701e00810e58dd90decf Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 30 Mar 2023 17:03:20 +0100 Subject: [PATCH 10/17] fixup --- .github/workflows/ci.yml | 66 ++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd08f3c22..1a932c551 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,12 +35,17 @@ jobs: with: version: "${{ matrix.NXF_VER }}" + - name: Hash Github Workspace + id: hash_workspace + run: | + echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT + - name: Cache test data id: cache-testdata uses: actions/cache@v3 with: path: test-datasets/ - key: rnaseq3_10-test-data + key: ${{ steps.hash_workspace.outputs.digest }} - name: Check out test data if: steps.cache-testdata.outputs.cache-hit != 'true' @@ -52,8 +57,8 @@ jobs: - name: Replace remote paths in samplesheets run: | - for f in ./test-datasets/samplesheet/v3.10/*.csv; do - sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=./test-datasets/=g" $f + for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do + sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f echo "========== $f ============" cat $f echo "========================================" @@ -61,7 +66,7 @@ jobs: - name: Run pipeline with test data run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results --test_data_base ./test-datasets/ + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/ star_salmon: name: Test STAR Salmon with workflow parameters @@ -87,12 +92,17 @@ jobs: - name: Check out pipeline code uses: actions/checkout@v2 + - name: Hash Github Workspace + id: hash_workspace + run: | + echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT + - name: Cache test data id: cache-testdata uses: actions/cache@v3 with: path: test-datasets/ - key: rnaseq3_10-test-data + key: ${{ steps.hash_workspace.outputs.digest }} - name: Check out test data if: steps.cache-testdata.outputs.cache-hit != 'true' @@ -104,8 +114,8 @@ jobs: - name: Replace remote paths in samplesheets run: | - for f in ./test-datasets/samplesheet/v3.10/*.csv; do - sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=./test-datasets/=g" $f + for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do + sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f echo "========== $f ============" cat $f echo "========================================" @@ -118,7 +128,7 @@ jobs: - name: Run pipeline with STAR and various parameters run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_salmon ${{ matrix.parameters }} --outdir ./results --test_data_base ./test-datasets/ + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_salmon ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/ star_rsem: name: Test STAR RSEM with workflow parameters @@ -133,12 +143,17 @@ jobs: - name: Check out pipeline code uses: actions/checkout@v2 + - name: Hash Github Workspace + id: hash_workspace + run: | + echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT + - name: Cache test data id: cache-testdata uses: actions/cache@v3 with: path: test-datasets/ - key: rnaseq3_10-test-data + key: rnaseq-${{ steps.hash_workspace.outputs.digest }} - name: Check out test data if: steps.cache-testdata.outputs.cache-hit != 'true' @@ -150,8 +165,8 @@ jobs: - name: Replace remote paths in samplesheets run: | - for f in ./test-datasets/samplesheet/v3.10/*.csv; do - sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=./test-datasets/=g" $f + for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do + sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f echo "========== $f ============" cat $f echo "========================================" @@ -164,7 +179,7 @@ jobs: - name: Run pipeline with RSEM STAR and various parameters run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_rsem ${{ matrix.parameters }} --outdir ./results --test_data_base ./test-datasets/ + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_rsem ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/ hisat2: name: Test HISAT2 with workflow parameters @@ -179,12 +194,17 @@ jobs: - name: Check out pipeline code uses: actions/checkout@v2 + - name: Hash Github Workspace + id: hash_workspace + run: | + echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT + - name: Cache test data id: cache-testdata uses: actions/cache@v3 with: path: test-datasets/ - key: rnaseq3_10-test-data + key: rnaseq-${{ steps.hash_workspace.outputs.digest }} - name: Check out test data if: steps.cache-testdata.outputs.cache-hit != 'true' @@ -196,8 +216,8 @@ jobs: - name: Replace remote paths in samplesheets run: | - for f in ./test-datasets/samplesheet/v3.10/*.csv; do - sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=./test-datasets/=g" $f + for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do + sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f echo "========== $f ============" cat $f echo "========================================" @@ -210,7 +230,7 @@ jobs: - name: Run pipeline with HISAT2 and various parameters run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner hisat2 ${{ matrix.parameters }} --outdir ./results --test_data_base ./test-datasets/ + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner hisat2 ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/ salmon: name: Test Salmon with workflow parameters @@ -225,12 +245,18 @@ jobs: - name: Check out pipeline code uses: actions/checkout@v2 + - name: Hash Github Workspace + id: hash_workspace + run: | + echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT + + - name: Cache test data id: cache-testdata uses: actions/cache@v3 with: path: test-datasets/ - key: rnaseq3_10-test-data + key: ${{ steps.hash_workspace.outputs.digest }} - name: Check out test data if: steps.cache-testdata.outputs.cache-hit != 'true' @@ -242,8 +268,8 @@ jobs: - name: Replace remote paths in samplesheets run: | - for f in ./test-datasets/samplesheet/v3.10/*.csv; do - sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=./test-datasets/=g" $f + for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do + sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f echo "========== $f ============" cat $f echo "========================================" @@ -256,4 +282,4 @@ jobs: - name: Run pipeline with Salmon and various parameters run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --pseudo_aligner salmon ${{ matrix.parameters }} --outdir ./results --test_data_base ./test-datasets/ + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --pseudo_aligner salmon ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/ From 1f228a2f209fbf81c01884af94e28dfd8dd83f3c Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 30 Mar 2023 17:04:10 +0100 Subject: [PATCH 11/17] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42f7496ea..a98da1e30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements - [[#961](https://github.com/nf-core/rnaseq/issues/961)] - Add warnings to STDOUT for all skipped and failed strandedness check samples - [[#975](https://github.com/nf-core/rnaseq/issues/975)] - `SALMON_INDEX` runs when using `--aligner star_rsem` even if samples have explicit strandedness - Remove HISAT2 from automated AWS full-sized tests +- Fix issue with incorrect cacheing of test datasets during CI/CD ### Parameters From 1964135eb783a25eaf913e96f8a226d5f5c8c085 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 30 Mar 2023 17:06:13 +0100 Subject: [PATCH 12/17] Fix Actions to trigger full-sized tests on release --- .github/workflows/cloud_tests_full.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cloud_tests_full.yml b/.github/workflows/cloud_tests_full.yml index d0c3d6e3e..3c4b23106 100644 --- a/.github/workflows/cloud_tests_full.yml +++ b/.github/workflows/cloud_tests_full.yml @@ -17,7 +17,7 @@ on: - gcp jobs: run-full-tests-on-aws: - if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'aws' }} + if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'aws' || !github.event.inputs }} runs-on: ubuntu-latest strategy: matrix: @@ -34,6 +34,7 @@ jobs: parameters: | { "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}", + "aligner": "${{ matrix.aligner }}", "outdir": "${{ secrets.TOWER_BUCKET_AWS }}/rnaseq/results-${{ github.sha }}/aligner_${{ matrix.aligner }}" } - uses: actions/upload-artifact@v3 @@ -41,7 +42,7 @@ jobs: name: Tower debug log file path: tower_action_*.log run-full-tests-on-gcp: - if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'gcp' }} + if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'gcp' || !github.event.inputs }} runs-on: ubuntu-latest strategy: matrix: @@ -58,6 +59,7 @@ jobs: parameters: | { "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}", + "aligner": "${{ matrix.aligner }}", "outdir": "${{ secrets.TOWER_BUCKET_GCP }}/rnaseq/results-${{ github.sha }}/aligner_${{ matrix.aligner }}" } - uses: actions/upload-artifact@v3 @@ -65,7 +67,7 @@ jobs: name: Tower debug log file path: tower_action_*.log run-full-tests-on-azure: - if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'azure' }} + if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'azure' || !github.event.inputs }} runs-on: ubuntu-latest strategy: matrix: @@ -82,6 +84,7 @@ jobs: parameters: | { "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}", + "aligner": "${{ matrix.aligner }}", "outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/rnaseq/results-${{ github.sha }}/aligner_${{ matrix.aligner }}", "igenomes_base": "${{ secrets.TOWER_IGENOMES_BASE_AZURE }}" } From c3cc602244cea3ee92c75961b7d1aa3d276f1f8f Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 30 Mar 2023 17:06:38 +0100 Subject: [PATCH 13/17] Add Adam to CHANGELOG [ci skip] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ae20d8b9..06be5ef99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Special thanks to the following for their code contributions to the release: +- [Adam Talbot](https://github.com/adamrtalbot) - [Rob Syme](https://github.com/robsyme) - [suhrig](https://github.com/suhrig) From 18e910b05f5d6f5f99db9f232b05370d8455e119 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 30 Mar 2023 17:14:58 +0100 Subject: [PATCH 14/17] [ci skip] Changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ace6f5cf..920fa29af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,7 +38,7 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements - [[#975](https://github.com/nf-core/rnaseq/issues/975)] - `SALMON_INDEX` runs when using `--aligner star_rsem` even if samples have explicit strandedness - Remove HISAT2 from automated AWS full-sized tests - [[#988](https://github.com/nf-core/rnaseq/issues/988)] - `DESEQ2_QC_STAR_SALMON` fails when sample names have many components -- Fix issue with incorrect cacheing of test datasets during CI/CD +- [[#987](https://github.com/nf-core/rnaseq/pull/987)] - Fix issue with incorrect cacheing of test datasets during CI/CD ### Parameters From d010be381541d9fc67ab227e9464af510d5306f2 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 30 Mar 2023 16:18:09 +0000 Subject: [PATCH 15/17] [automated] Fix linting with Prettier --- .github/workflows/ci.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a932c551..21771e649 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: id: hash_workspace run: | echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT - + - name: Cache test data id: cache-testdata uses: actions/cache@v3 @@ -96,7 +96,7 @@ jobs: id: hash_workspace run: | echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT - + - name: Cache test data id: cache-testdata uses: actions/cache@v3 @@ -147,7 +147,7 @@ jobs: id: hash_workspace run: | echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT - + - name: Cache test data id: cache-testdata uses: actions/cache@v3 @@ -198,7 +198,7 @@ jobs: id: hash_workspace run: | echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT - + - name: Cache test data id: cache-testdata uses: actions/cache@v3 @@ -249,7 +249,6 @@ jobs: id: hash_workspace run: | echo "digest=$(echo RNA_3.10.1_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT - - name: Cache test data id: cache-testdata From 1333a7adf75d9c863b0643ec9d97ef4aa9ccd428 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 30 Mar 2023 17:21:45 +0100 Subject: [PATCH 16/17] fixup --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21771e649..393c9bd24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,7 +153,7 @@ jobs: uses: actions/cache@v3 with: path: test-datasets/ - key: rnaseq-${{ steps.hash_workspace.outputs.digest }} + key: ${{ steps.hash_workspace.outputs.digest }} - name: Check out test data if: steps.cache-testdata.outputs.cache-hit != 'true' @@ -204,7 +204,7 @@ jobs: uses: actions/cache@v3 with: path: test-datasets/ - key: rnaseq-${{ steps.hash_workspace.outputs.digest }} + key: ${{ steps.hash_workspace.outputs.digest }} - name: Check out test data if: steps.cache-testdata.outputs.cache-hit != 'true' From fcb218926c78766c47bffee384262075c513d4b8 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 31 Mar 2023 14:38:45 +0100 Subject: [PATCH 17/17] Update date in CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7b5011f6..2d5191059 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [[3.11.1](https://github.com/nf-core/rnaseq/releases/tag/3.11.1)] - 2023-03-30 +## [[3.11.1](https://github.com/nf-core/rnaseq/releases/tag/3.11.1)] - 2023-03-31 ### Credits