Merge branch 'release/5.7.0'

.circleci/config.yml

@@ -0,0 +1,62 @@
+# 1. Set your GitHub default branch to dev/develop
+# 2. In circleci project settings->advanced set:
+#    - GitHub Status Updates (this should be on already)
+#    - Only build pull requests
+#    - Auto-cancel redundant builds
+#    - Free and Open Source (this should be on already)
+# 3. If existing project with quay.io, remove the quay build triggers.
+#    - as circleci is pushing the image
+#    - reevaluate when/if quay.io adds ability to authenticate pulls
+
+version: 2.1
+jobs:
+  build:
+    environment:
+      IMAGE_NAME: quay.io/wtsicgp/pcap-core
+    docker:
+      - image: circleci/buildpack-deps:stretch
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: Build Docker image
+          command: |
+            set -e
+            BRANCH_OR_TAG="${CIRCLE_TAG:-$CIRCLE_BRANCH}"
+            CLEAN_BRANCH=$(echo $BRANCH_OR_TAG | tr / -)
+            echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USER" --password-stdin
+            echo "$QUAYIO_PASSWORD" | docker login -u "$QUAYIO_USER" --password-stdin quay.io
+            docker build -t $IMAGE_NAME:$CLEAN_BRANCH .
+            echo 'Verify program(s) from base image'
+            docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bam2bedgraph --version
+            docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bwjoin --version
+            docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bam2bw --version
+            docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bwcat --version
+            docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bam2bwbases --version
+            docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bg2bw --version
+            docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH detectExtremeDepth --version
+            echo 'Verify program(s) from this repo'
+            docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH diff_bams --version
+            docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH reheadSQ --version
+            docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bam_stats --version
+            docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH mismatchQc --version
+            docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bwa_mem.pl --version
+            docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH merge_or_mark.pl --version
+            if [ "$CIRCLE_TAG" = "$BRANCH_OR_TAG" ]; then
+              echo 'Pushing image'
+              docker push $IMAGE_NAME:$CLEAN_BRANCH
+            else
+              echo "I am not a tag so not pushing image"
+            fi
+
+workflows:
+  version: 2.1
+  build_test:
+    jobs:
+      - build:
+          context:
+            - dockerhub-casmservice
+            - quayio-casmservice
+          filters:
+            tags:
+              only: /.+/

.dockerignore

@@ -10,3 +10,4 @@
 /setup.sh
 /prerelease.sh
 /blib
+*.code-workspace

.gitignore

@@ -20,3 +20,4 @@ examples/cgp_gnos_pull.ini
 /c/c_tests/01_bam_stats_output_tests
 /c/c_tests/02_bam_access_tests
 /c/c_tests/03_bam_stats_calcs_tests
+*.code-workspace

CHANGES.md

@@ -1,5 +1,10 @@
 # CHANGES
 
+## 5.7.0
+
+* FASTQ tag preservation during mapping (bam/cram handled unless "-legacy")
+* bwakit option added
+
 ## 5.6.1
 
 * htslib/samtools 1.12 - came out hours after we released 5.6.0

Dockerfile

@@ -12,9 +12,11 @@ ARG BWAMEM2_GIT="https://github.com/bwa-mem2/bwa-mem2.git"
 ARG BWAMEM2_TAG="v2.2.1"
 ARG STADEN="https://iweb.dl.sourceforge.net/project/staden/staden/2.0.0b11/staden-2.0.0b11-2016-linux-x86_64.tar.gz"
 ARG VER_BIODBHTS="3.01"
+# needed for bwakit as well as bwa
 ARG VER_BWA="v0.7.17"
 ARG VER_HTSLIB="1.12"
 ARG VER_SAMTOOLS="1.12"
+ARG VER_K8="0.2.5"
 
 RUN apt-get -yq update
 RUN apt-get install -yq --no-install-recommends apt-transport-https

Makefile.PL

@@ -3,6 +3,7 @@
 ##########LICENCE##########
 # PCAP - NGS reference implementations and helper code for the ICGC/TCGA Pan-Cancer Analysis Project
 # Copyright (C) 2014-2018 ICGC PanCancer Project
+# Copyright (C) 2018-2021 Cancer, Ageing and Somatic Mutation, Genome Research Limited
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License

README.md

@@ -6,9 +6,7 @@ This has been forked from the [ICGC-TCGA-PanCancer/PCAP-core][PCAP-core]
 repository as this codebase was created by [cancerit](cancerit_github) and continues to be developed.
 This version strips out PCAWG related elements and incorporates more efficient code.
 
-| Master                                        | Develop                                         |
-| --------------------------------------------- | ----------------------------------------------- |
-| [![Master Badge][travis-master]][travis-base] | [![Develop Badge][travis-develop]][travis-base] |
+[![CircleCI](https://circleci.com/gh/cancerit/PCAP-core/tree/develop.svg?style=svg)](https://circleci.com/gh/cancerit/PCAP-core/tree/develop)
 
 This repository contains code to run genomic alignments of paired end data
 and subsequent calling algorithms.
@@ -79,8 +77,3 @@ Please see the respective licence for each before use.
 [old_repo]: https://github.com/ICGC-TCGA-PanCancer/PCAP-core
 [dockstore-cgpmap]: https://github.com/cancerit/dockstore-cgpmap
 [quay-io-pcap-core]: https://quay.io/repository/wtsicgp/pcap-core
-
-<!-- Travis -->
-[travis-base]: https://travis-ci.org/cancerit/PCAP-core
-[travis-master]: https://travis-ci.org/cancerit/PCAP-core.svg?branch=master
-[travis-develop]: https://travis-ci.org/cancerit/PCAP-core.svg?branch=develop

bin/bamToBw.pl

@@ -3,6 +3,7 @@
 ##########LICENCE##########
 # PCAP - NGS reference implementations and helper code for the ICGC/TCGA Pan-Cancer Analysis Project
 # Copyright (C) 2014-2018 ICGC PanCancer Project
+# Copyright (C) 2018-2021 Cancer, Ageing and Somatic Mutation, Genome Research Limited
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License

bin/bwa_aln.pl

@@ -2,7 +2,8 @@
 
 ##########LICENCE##########
 # PCAP - NGS reference implementations and helper code for the ICGC/TCGA Pan-Cancer Analysis Project
-# Copyright (C) 2014 ICGC PanCancer Project
+# Copyright (C) 2014-2018 ICGC PanCancer Project
+# Copyright (C) 2018-2021 Cancer, Ageing and Somatic Mutation, Genome Research Limited
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License

bin/bwa_mem.pl

@@ -2,7 +2,8 @@
 
 ##########LICENCE##########
 # PCAP - NGS reference implementations and helper code for the ICGC/TCGA Pan-Cancer Analysis Project
-# Copyright (C) 2014-2020 ICGC PanCancer Project
+# Copyright (C) 2014-2018 ICGC PanCancer Project
+# Copyright (C) 2018-2021 Cancer, Ageing and Somatic Mutation, Genome Research Limited
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -111,6 +112,7 @@ sub setup {
               'dupmode' => 't',
               'seqslice' => 10000,
               'csi' => undef,
+              'tags' => undef,
              );
 
   GetOptions( 'h|help' => \$opts{'h'},
@@ -137,7 +139,9 @@ sub setup {
               'bm2|bwamem2' => \$opts{'bwamem2'},
               'd|dupmode:s' => \$opts{'dupmode'},
               'legacy' => \$opts{'legacy'},
-              'ss|seqslice:i' => $opts{'seqslice'},
+              'ss|seqslice:i' => \$opts{'seqslice'},
+              'k|bwakit' => \$opts{'bwakit'},
+              'tags:s' => \$opts{'tags'},
   ) or pod2usage(2);
 
   pod2usage(-verbose => 1, -exitval => 0) if(defined $opts{'h'});
@@ -180,6 +184,8 @@ sub setup {
   delete $opts{'csi'} unless(defined $opts{'csi'});
   delete $opts{'bwamem2'} unless(defined $opts{'bwamem2'});
   delete $opts{'legacy'} unless(defined $opts{'legacy'});
+  delete $opts{'bwakit'} unless(defined $opts{'bwakit'});
+  delete $opts{'tags'} unless(defined $opts{'tags'});
 
   if(defined $opts{'bwamem2'} && defined $opts{'legacy'}) {
     warn "WARN: Use of options bwamem2 and legacy is suboptimal, proceeding but memory will be excessive.\n";
@@ -250,7 +256,8 @@ =head1 SYNOPSIS
     -threads     -t    Number of threads to use. [1]
 
   Optional parameters:
-    -bwamem2     -bm2  Use bwa-mem2 instead of bwa (experimental).
+    -bwamem2     -bm2  Use bwa-mem2 instead of bwa.
+    -bwakit      -k    Include 'bwa-postalt.js' processing from bwakit.
     -fragment    -f    Split input into fragments of X million repairs [10]
                         - only applies to fastq[.gz] input
     -nomarkdup   -n    Don't mark duplicates [flag]
@@ -259,8 +266,10 @@ =head1 SYNOPSIS
     -seqslice    -ss   seqs_per_slice for CRAM compression [samtools default: 10000]
     -scramble    -sc   DEPRECATED
     -bwa         -b    Single quoted string of additional parameters to pass to BWA
-                        - '-t,-p,-R' are used internally and should not be provided.
+                        - '-t,-p,-R,-C' are used internally and should not be provided.
                         - '-v' is set to 1 unless '-bwa' is set.
+    -tags              Propogate these tags from input data into final alignment, recommend:
+                          BC,QT
     -map_threads -mt   Number of cores applied to each parallel BWA job when '-t' exceeds this value
                        and '-i' is not in use [6]
     -groupinfo   -g    Readgroup information metadata file, values are not validated (yaml) [file]

bin/xam_coverage_bins.pl

@@ -2,7 +2,8 @@
 
 ##########LICENCE##########
 # PCAP - NGS reference implementations and helper code for the ICGC/TCGA Pan-Cancer Analysis Project
-# Copyright (C) 2014-2016 ICGC PanCancer Project
+# Copyright (C) 2014-2018 ICGC PanCancer Project
+# Copyright (C) 2018-2021 Cancer, Ageing and Somatic Mutation, Genome Research Limited
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License

build/opt-build.sh

@@ -49,10 +49,30 @@ export MANPATH=`echo $INST_PATH/man:$BB_INST/man:$INST_PATH/share/man:$MANPATH |
 export PERL5LIB=`echo $INST_PATH/lib/perl5:$PERL5LIB | perl -pe 's/:\$//;'`
 set -u
 
-## biobambam2 first
+## k8 javascript client for bwakit
+if [ ! -e $SETUP_DIR/k8.sucess ]; then
+  curl -sSL --retry 10 -o distro.tar.bz2 https://github.com/attractivechaos/k8/releases/download/${VER_K8}/k8-${VER_K8}.tar.bz2
+  rm -rf distro/*
+  tar --strip-components 1 -C distro -xjf distro.tar.bz2
+  cp distro/k8-Linux $INST_PATH/bin/k8
+  chmod ugo+x $INST_PATH/bin/k8
+  rm -rf distro.* distro/*
+  touch $SETUP_DIR/k8.success
+fi
+
+## add relevant script from bwakit
+if [ ! -e $SETUP_DIR/bwakit.sucess ]; then
+  # slight hack to make this executable
+  echo "#!$INST_PATH/bin/k8" > $INST_PATH/bin/bwa-postalt
+  curl -sSL https://raw.githubusercontent.com/lh3/bwa/${VER_BWA}/bwakit/bwa-postalt.js >> $INST_PATH/bin/bwa-postalt
+  chmod ugo+x $INST_PATH/bin/bwa-postalt
+  touch $SETUP_DIR/bwakit.success
+fi
+
+## biobambam2
 BB_INST=$INST_PATH/biobambam2
 if [ ! -e $SETUP_DIR/bbb2.sucess ]; then
-  curl -sSL --retry 10 $BBB2_URL > distro.tar.xz
+  curl -sSL --retry 10 -o distro.tar.xz $BBB2_URL
   mkdir -p $BB_INST
   tar --strip-components 3 -C $BB_INST -Jxf distro.tar.xz
   rm -f $BB_INST/bin/curl # don't let this file in SSL doesn't work
@@ -76,7 +96,7 @@ rm -f $SETUP_DIR/cpanm
 
 ## scramble (from staden)
 if [ ! -e $SETUP_DIR/staden.success ]; then
-  curl -sSL --retry 10 $STADEN > distro.tar.gz
+  curl -sSL --retry 10 -o distro.tar.gz $STADEN
   rm -rf distro/* $OPT/scramble
   mkdir -p $OPT/scramble
   tar --strip-components 1 -C distro -xzf distro.tar.gz
@@ -88,7 +108,7 @@ fi
 
 ## SAMTOOLS (tar.bz2)
 if [ ! -e $SETUP_DIR/samtools.success ]; then
-  curl -sSL --retry 10 https://github.com/samtools/samtools/releases/download/${VER_SAMTOOLS}/samtools-${VER_SAMTOOLS}.tar.bz2 > distro.tar.bz2
+  curl -sSL --retry 10 -o distro.tar.bz2 https://github.com/samtools/samtools/releases/download/${VER_SAMTOOLS}/samtools-${VER_SAMTOOLS}.tar.bz2
   rm -rf distro/*
   tar --strip-components 1 -C distro -xjf distro.tar.bz2
   cd distro
@@ -105,7 +125,7 @@ fi
 
 ## build BWA (tar.gz)
 if [ ! -e $SETUP_DIR/bwa.success ]; then
-  curl -sSL --retry 10 https://github.com/lh3/bwa/archive/${VER_BWA}.tar.gz > distro.tar.gz
+  curl -sSL --retry 10 -o distro.tar.gz https://github.com/lh3/bwa/archive/${VER_BWA}.tar.gz
   rm -rf distro/*
   tar --strip-components 1 -C distro -zxf distro.tar.gz
   make -C distro -j$CPU
@@ -134,7 +154,7 @@ if [ ! -e $SETUP_DIR/Bio-DB-HTS.success ]; then
   cpanm --no-wget --no-interactive --notest --mirror http://cpan.metacpan.org -l $INST_PATH XML::Parser
   cpanm --no-wget --no-interactive --notest --mirror http://cpan.metacpan.org -l $INST_PATH Bio::Root::Version
 
-  curl -sSL --retry 10 https://github.com/Ensembl/Bio-DB-HTS/archive/${VER_BIODBHTS}.tar.gz > distro.tar.gz
+  curl -sSL --retry 10 -o distro.tar.gz https://github.com/Ensembl/Bio-DB-HTS/archive/${VER_BIODBHTS}.tar.gz
   rm -rf distro/*
   tar --strip-components 1 -C distro -zxf distro.tar.gz
   cd distro

c/Makefile

@@ -1,6 +1,7 @@
 ##########LICENCE##########
 # PCAP - NGS reference implementations and helper code for the ICGC/TCGA Pan-Cancer Analysis Project
 # Copyright (C) 2014-2018 ICGC PanCancer Project
+# Copyright (C) 2018-2021 Cancer, Ageing and Somatic Mutation, Genome Research Limited
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License

c/bam_access.c

@@ -1,6 +1,7 @@
 /*       LICENCE
 * PCAP - NGS reference implementations and helper code for the ICGC/TCGA Pan-Cancer Analysis Project
 * Copyright (C) 2014-2018 ICGC PanCancer Project
+* Copyright (C) 2018-2021 Cancer, Ageing and Somatic Mutation, Genome Research Limited
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License

c/bam_access.h

@@ -1,6 +1,7 @@
 /*       LICENCE
 * PCAP - NGS reference implementations and helper code for the ICGC/TCGA Pan-Cancer Analysis Project
 * Copyright (C) 2014-2018 ICGC PanCancer Project
+* Copyright (C) 2018-2021 Cancer, Ageing and Somatic Mutation, Genome Research Limited
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License

c/bam_stats_output.c

@@ -1,6 +1,7 @@
 /*       LICENCE
 * PCAP - NGS reference implementations and helper code for the ICGC/TCGA Pan-Cancer Analysis Project
 * Copyright (C) 2014-2018 ICGC PanCancer Project
+* Copyright (C) 2018-2021 Cancer, Ageing and Somatic Mutation, Genome Research Limited
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License

c/c_tests/01_bam_stats_output_tests.c

@@ -1,6 +1,7 @@
 /*########LICENCE#########
 * PCAP - NGS reference implementations and helper code for the ICGC/TCGA Pan-Cancer Analysis Project
 * Copyright (C) 2014-2018 ICGC PanCancer Project
+* Copyright (C) 2018-2021 Cancer, Ageing and Somatic Mutation, Genome Research Limited
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License

c/c_tests/02_bam_access_tests.c

@@ -1,6 +1,7 @@
 /*########LICENCE#########
 * PCAP - NGS reference implementations and helper code for the ICGC/TCGA Pan-Cancer Analysis Project
 * Copyright (C) 2014-2018 ICGC PanCancer Project
+* Copyright (C) 2018-2021 Cancer, Ageing and Somatic Mutation, Genome Research Limited
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License

c/c_tests/runtests.sh

@@ -1,6 +1,7 @@
 ##########LICENCE##########
 # PCAP - NGS reference implementations and helper code for the ICGC/TCGA Pan-Cancer Analysis Project
 # Copyright (C) 2014-2018 ICGC PanCancer Project
+# Copyright (C) 2018-2021 Cancer, Ageing and Somatic Mutation, Genome Research Limited
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License

c/c_tests/test_05_mmFlagModifier.sh

@@ -2,7 +2,8 @@
 
 ##########LICENCE##########
 # PCAP - NGS reference implementations and helper code for the ICGC/TCGA Pan-Cancer Analysis Project
-# Copyright (C) 2014-2019 Genome Research Limited
+# Copyright (C) 2014-2018 ICGC PanCancer Project
+# Copyright (C) 2018-2021 Cancer, Ageing and Somatic Mutation, Genome Research Limited
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License

c/diff_bams.c

@@ -1,6 +1,7 @@
 /*       LICENCE
 * PCAP - NGS reference implementations and helper code for mapping (originally part of ICGC/TCGA PanCancer)
-# Copyright (C) 2014-2018 ICGC PanCancer Project
+* Copyright (C) 2014-2018 ICGC PanCancer Project
+* Copyright (C) 2018-2021 Cancer, Ageing and Somatic Mutation, Genome Research Limited
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License

c/mmFlagModifier.c

@@ -1,6 +1,7 @@
 /*       LICENCE
 * PCAP - NGS reference implementations and helper code for mapping (originally part of ICGC/TCGA PanCancer)
-# Copyright (C) 2014-2019 Genome Research Ltd.
+# Copyright (C) 2014-2018 ICGC PanCancer Project
+# Copyright (C) 2018-2021 Cancer, Ageing and Somatic Mutation, Genome Research Limited
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License