Merge pull request #126 from nf-core/dev

Dev > Master for 1.1.0 release

.github/markdownlint.yml

@@ -3,3 +3,7 @@ default: true,
 line-length: false
 no-duplicate-header:
     siblings_only: true
+MD033:
+    allowed_elements: [details, summary, p, img]
+MD007:
+    indent: 4

.github/workflows/awsfulltest.yml

@@ -0,0 +1,36 @@
+name: nf-core AWS full test
+# This workflow is triggered on releases.
+# It runs the -profile 'test_full' on AWS batch
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  run-awstest:
+    if: github.repository == 'nf-core/viralrecon'
+    name: Run AWS test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Miniconda
+        uses: goanpeca/[email protected]
+        with:
+          auto-update-conda: true
+          python-version: 3.7
+      - name: Install awscli
+        run: conda install -c conda-forge awscli
+      - name: Start AWS batch job
+        env:
+          AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
+          TOWER_ACCESS_TOKEN: ${{secrets.AWS_TOWER_TOKEN}}
+          #AWS_JOB_DEFINITION: ${{secrets.AWS_JOB_DEFINITION}}
+          AWS_JOB_QUEUE: ${{secrets.AWS_JOB_QUEUE}}
+          AWS_S3_BUCKET: ${{secrets.AWS_S3_BUCKET}}
+        run: | # Submits job to AWS batch using a 'nextflow-4GiB' job definition. Setting JVM options to "-XX:+UseG1GC" for more efficient garbage collection when staging remote files.
+          aws batch submit-job \
+          --region eu-west-1 \
+          --job-name nf-core-viralrecon \
+          --job-queue $AWS_JOB_QUEUE \
+          --job-definition nextflow-4GiB \
+          --container-overrides '{"command": ["nf-core/viralrecon", "-r '"${GITHUB_SHA}"' -profile test_full --outdir s3://'"${AWS_S3_BUCKET}"'/viralrecon/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/viralrecon/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}, {"name": "NXF_OPTS", "value": "-XX:+UseG1GC"}]}'

.github/workflows/awstest.yml

@@ -1,16 +1,15 @@
 name: nf-core AWS test
-# This workflow is triggered on PRs to the master branch.
-# It runs the -profile 'test_full' on AWS batch
+# This workflow is triggered on push to the master branch.
+# It runs the -profile 'test' on AWS batch
 
 on:
   push:
     branches:
       - master
-  release:
-    types: [published]
-
+      - dev # only for testing purposes, to be removed
 jobs:
   run-awstest:
+    if: github.repository == 'nf-core/viralrecon'
     name: Run AWS test
     runs-on: ubuntu-latest
     steps:
@@ -23,8 +22,16 @@ jobs:
         run: conda install -c conda-forge awscli
       - name: Start AWS batch job
         env:
-          AWS_ACCESS_KEY_ID: ${{secrets.AWS_KEY_ID}}
-          AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_KEY_SECRET}}
-          TOWER_ACCESS_TOKEN: ${{secrets.TOWER_ACCESS_TOKEN}}
-        run: | # Submits job to AWS batch using a 'nextflow-big' instance. Setting JVM options to "-XX:+UseG1GC" for more efficient garbage collection when staging remote files.
-          aws batch submit-job --region eu-west-1 --job-name nf-core-viralrecon --job-queue 'default-8b3836e0-5eda-11ea-96e5-0a2c3f6a2a32' --job-definition nextflow-4GiB --container-overrides '{"command": ["nf-core/viralrecon", "-r '"${GITHUB_SHA}"' -profile test_full --outdir s3://nf-core-awsmegatests/viralrecon/results-'"${GITHUB_SHA}"' -w s3://nf-core-awsmegatests/viralrecon/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}, {"name": "NXF_OPTS", "value": "-XX:+UseG1GC"}]}'
+          AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
+          TOWER_ACCESS_TOKEN: ${{secrets.AWS_TOWER_TOKEN}}
+          #AWS_JOB_DEFINITION: ${{secrets.AWS_JOB_DEFINITION}}
+          AWS_JOB_QUEUE: ${{secrets.AWS_JOB_QUEUE}}
+          AWS_S3_BUCKET: ${{secrets.AWS_S3_BUCKET}}
+        run: | # Submits job to AWS batch using a 'nextflow-4GiB' job definition. Setting JVM options to "-XX:+UseG1GC" for more efficient garbage collection when staging remote files.
+          aws batch submit-job \
+          --region eu-west-1 \
+          --job-name nf-core-viralrecon \
+          --job-queue $AWS_JOB_QUEUE \
+          --job-definition nextflow-4GiB \
+          --container-overrides '{"command": ["nf-core/viralrecon", "-r '"${GITHUB_SHA}"' -profile test --outdir s3://'"${AWS_S3_BUCKET}"'/viralrecon/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/viralrecon/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}, {"name": "NXF_OPTS", "value": "-XX:+UseG1GC"}]}'

.github/workflows/branch.yml

@@ -1,16 +1,16 @@
-name: nf-core branch protection
-# This workflow is triggered on PRs to master branch on the repository
-# It fails when someone tries to make a PR against the nf-core `master` branch instead of `dev`
-on:
-  pull_request:
-    branches:
-    - master
-
-jobs:
-  test:
-    runs-on: ubuntu-18.04
-    steps:
-      # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch
-      - name: Check PRs
-        run: |
-          { [[ $(git remote get-url origin) == *nf-core/viralrecon ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]]
+name: nf-core branch protection
+# This workflow is triggered on PRs to master branch on the repository
+# It fails when someone tries to make a PR against the nf-core `master` branch instead of `dev`
+on:
+  pull_request:
+    branches:
+    - master
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+    steps:
+      # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch
+      - name: Check PRs
+        run: |
+          { [[ $(git remote get-url origin) == *nf-core/viralrecon ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]]

.github/workflows/ci.yml

@@ -1,94 +1,205 @@
 name: nf-core CI
-# This workflow is triggered on pushes and PRs to the repository.
+# This workflow is triggered on releases and pull-requests.
 # It runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - dev
+  pull_request:
+  release:
+    types: [published]
 
 jobs:
   test:
+    name: Test default workflow
+    # Only run on push if this is the nf-core dev branch (merged PRs)
+    if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/viralrecon') }}
+    runs-on: ubuntu-latest
     env:
       NXF_VER: ${{ matrix.nxf_ver }}
       NXF_ANSI_LOG: false
-    runs-on: ubuntu-latest
     strategy:
       matrix:
         # Nextflow versions: check pipeline minimum and current latest
         nxf_ver: ['19.10.0', '']
     steps:
-      - uses: actions/checkout@v2
+      - name: Check out pipeline code
+        uses: actions/checkout@v2
+
+      - name: Check if Dockerfile or Conda environment changed
+        uses: technote-space/get-diff-action@v1
+        with:
+          PREFIX_FILTER: |
+            Dockerfile
+            environment.yml
+
+      - name: Build new docker image
+        if: env.GIT_DIFF
+        run: docker build --no-cache . -t nfcore/viralrecon:1.1.0
+
+      - name: Pull docker image
+        if: ${{ !env.GIT_DIFF }}
+        run: |
+          docker pull nfcore/viralrecon:dev
+          docker tag nfcore/viralrecon:dev nfcore/viralrecon:1.1.0
+
       - name: Install Nextflow
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
-      - name: Pull docker image
-        run: |
-          docker pull nfcore/viralrecon:dev
-          docker tag nfcore/viralrecon:dev nfcore/viralrecon:1.0.0
+
       - name: Run pipeline with test data
         run: |
           nextflow run ${GITHUB_WORKSPACE} -profile test,docker
 
   parameters:
+    name: Test workflow parameters
+    if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/viralrecon') }}
+    runs-on: ubuntu-latest
     env:
       NXF_VER: '19.10.0'
       NXF_ANSI_LOG: false
-    runs-on: ubuntu-latest
     strategy:
       matrix:
-        parameters: [--skip_adapter_trimming, --skip_markduplicates, --skip_variants, --skip_amplicon_trimming, --skip_kraken2, --skip_assembly]
+        parameters: [--skip_adapter_trimming, --skip_markduplicates, --skip_variants, --skip_amplicon_trimming, --skip_kraken2, --skip_assembly, '--callers ivar --assemblers spades']
     steps:
-      - uses: actions/checkout@v2
+      - name: Check out pipeline code
+        uses: actions/checkout@v2
+
+      - name: Check if Dockerfile or Conda environment changed
+        uses: technote-space/get-diff-action@v1
+        with:
+          PREFIX_FILTER: |
+            Dockerfile
+            environment.yml
+
+      - name: Build new docker image
+        if: env.GIT_DIFF
+        run: docker build --no-cache . -t nfcore/viralrecon:1.1.0
+
+      - name: Pull docker image
+        if: ${{ !env.GIT_DIFF }}
+        run: |
+          docker pull nfcore/viralrecon:dev
+          docker tag nfcore/viralrecon:dev nfcore/viralrecon:1.1.0
+
       - name: Install Nextflow
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
-      - name: Pull docker image
-        run: |
-          docker pull nfcore/viralrecon:dev
-          docker tag nfcore/viralrecon:dev nfcore/viralrecon:1.0.0
+
       - name: Run pipeline with test amplicon data with various options
         run: |
           nextflow run ${GITHUB_WORKSPACE} -profile test,docker ${{ matrix.parameters }}
 
   test_sra:
-     env:
-       NXF_VER: '19.10.0'
-       NXF_ANSI_LOG: false
-     runs-on: ubuntu-latest
-     strategy:
-       matrix:
-         parameters: [--skip_sra, '']
-     steps:
-       - uses: actions/checkout@v2
-       - name: Install Nextflow
-         run: |
-           wget -qO- get.nextflow.io | bash
-           sudo mv nextflow /usr/local/bin/
-       - name: Pull docker image
-         run: |
-           docker pull nfcore/viralrecon:dev
-           docker tag nfcore/viralrecon:dev nfcore/viralrecon:1.0.0
-       - name: Run pipeline with minimal data via SRA ids and various options
-         run: |
-           nextflow run ${GITHUB_WORKSPACE} -profile test_sra,docker ${{ matrix.parameters }}
+    name: Test SRA workflow
+    if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/viralrecon') }}
+    runs-on: ubuntu-latest
+    env:
+      NXF_VER: '19.10.0'
+      NXF_ANSI_LOG: false
+    strategy:
+      matrix:
+        parameters: [--skip_sra, '']
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@v2
+
+      - name: Check if Dockerfile or Conda environment changed
+        uses: technote-space/get-diff-action@v1
+        with:
+          PREFIX_FILTER: |
+            Dockerfile
+            environment.yml
+
+      - name: Build new docker image
+        if: env.GIT_DIFF
+        run: docker build --no-cache . -t nfcore/viralrecon:1.1.0
+
+      - name: Pull docker image
+        if: ${{ !env.GIT_DIFF }}
+        run: |
+          docker pull nfcore/viralrecon:dev
+          docker tag nfcore/viralrecon:dev nfcore/viralrecon:1.1.0
+
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+
+      - name: Run pipeline with minimal data via SRA ids and various options
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} -profile test_sra,docker ${{ matrix.parameters }}
 
   test_sispa:
-     env:
-       NXF_VER: '19.10.0'
-       NXF_ANSI_LOG: false
-     runs-on: ubuntu-latest
-     strategy:
-       matrix:
-         parameters: [--gff false, '']
-     steps:
-       - uses: actions/checkout@v2
-       - name: Install Nextflow
-         run: |
-           wget -qO- get.nextflow.io | bash
-           sudo mv nextflow /usr/local/bin/
-       - name: Pull docker image
-         run: |
-           docker pull nfcore/viralrecon:dev
-           docker tag nfcore/viralrecon:dev nfcore/viralrecon:1.0.0
-       - name: Run pipeline with minimal SISPA data and various options
-         run: |
-           nextflow run ${GITHUB_WORKSPACE} -profile test_sispa,docker ${{ matrix.parameters }}
+    name: Test SISPA workflow
+    if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/viralrecon') }}
+    runs-on: ubuntu-latest
+    env:
+      NXF_VER: '19.10.0'
+      NXF_ANSI_LOG: false
+    strategy:
+      matrix:
+        parameters: [--gff false, '']
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@v2
+
+      - name: Check if Dockerfile or Conda environment changed
+        uses: technote-space/get-diff-action@v1
+        with:
+          PREFIX_FILTER: |
+            Dockerfile
+            environment.yml
+
+      - name: Build new docker image
+        if: env.GIT_DIFF
+        run: docker build --no-cache . -t nfcore/viralrecon:1.1.0
+
+      - name: Pull docker image
+        if: ${{ !env.GIT_DIFF }}
+        run: |
+          docker pull nfcore/viralrecon:dev
+          docker tag nfcore/viralrecon:dev nfcore/viralrecon:1.1.0
+
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+
+      - name: Run pipeline with minimal SISPA data and various options
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} -profile test_sispa,docker ${{ matrix.parameters }}
+
+  push_dockerhub:
+    name: Push new Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    # Only run if the tests passed
+    needs: test
+    # Only run for the nf-core repo, for releases and merged PRs
+    if: ${{ github.repository == 'nf-core/viralrecon' && (github.event_name == 'release' || github.event_name == 'push') }}
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }}
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@v2
+
+      - name: Build new docker image
+        run: docker build --no-cache . -t nfcore/viralrecon:latest
+
+      - name: Push Docker image to DockerHub (dev)
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
+          docker tag nfcore/viralrecon:latest nfcore/viralrecon:dev
+          docker push nfcore/viralrecon:dev
+
+      - name: Push Docker image to DockerHub (release)
+        if: ${{ github.event_name == 'release' }}
+        run: |
+          echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
+          docker push nfcore/viralrecon:latest
+          docker tag nfcore/viralrecon:latest nfcore/viralrecon:${{ github.ref }}
+          docker push nfcore/viralrecon:${{ github.ref }}