Merge branch 'main' of github.com:populationgenomics/images into push…

…-to-azure

.github/workflows/deploy_config.yaml

@@ -19,10 +19,10 @@ jobs:
       matrix: ${{ steps.set_matrix.outputs.matrix }}
     steps:
       - name: "Checkout repo"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: "Checkout ref before change"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           ref: ${{ github.event.before }}
           path: 'before'
@@ -62,12 +62,14 @@ jobs:
     outputs:
       version: ${{ steps.get_version.outputs.version }}
     steps:
-    - uses: actions/checkout@main
+    - uses: actions/checkout@v3
       with:
         repository: "populationgenomics/production-pipelines"
         ref: "main"
         path: "production-pipelines"
         submodules: recursive
+        # Comment this line out to revert to using the default $GITHUB_TOKEN.
+        token: ${{ secrets.PRODUCTION_PIPELINES_RAW_REPO_TOKEN }}
 
     - id: get_version
       run: |
@@ -109,15 +111,15 @@ jobs:
   # special-case cpg_workflows.
   deploy_config:
     runs-on: ubuntu-latest
-    needs: 
+    needs:
       - deploy_images
       - deploy_cpg_workflows
     if: ${{ ! failure() }}
     env:
       CLOUDSDK_CORE_DISABLE_PROMPTS: 1
     steps:
       - name: "checkout repo"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: "gcloud setup"
         uses: google-github-actions/setup-gcloud@v0

.github/workflows/deploy_container.yaml

@@ -44,13 +44,16 @@ jobs:
       DOCKER_BUILDKIT: 1
       BUILDKIT_PROGRESS: plain
       CLOUDSDK_CORE_DISABLE_PROMPTS: 1
+      DOCKER_DEV: australia-southeast1-docker.pkg.dev/cpg-common/images-dev/
+      DOCKER_MAIN: australia-southeast1-docker.pkg.dev/cpg-common/images/
       DOCKER_IMAGE: ${{ github.event.inputs.image_name }}:${{ github.event.inputs.image_tag }}
       IMAGE_NAME: ${{ github.event.inputs.image_name }}
       IMAGE_TAG: ${{ github.event.inputs.image_tag }}
 
     steps:
+
       - name: "checkout repo"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: "gcloud setup"
         uses: google-github-actions/setup-gcloud@v0
@@ -82,12 +85,16 @@ jobs:
             --tag $DOCKER_IMAGE \
             images/$IMAGE_NAME
 
-      - name: "tag and push gcp image"
+      - name: "Push non-main branch to dev artifactory"
+        if: ${{ github.ref_name != 'main' }}
         run: |
-          docker image tag $DOCKER_IMAGE $GCP_CONTAINER_REGISTRY/$DOCKER_IMAGE &&
-          docker push $GCP_CONTAINER_REGISTRY/$DOCKER_IMAGE
+          docker tag $DOCKER_IMAGE $DOCKER_DEV$DOCKER_IMAGE
+          docker push $DOCKER_DEV$DOCKER_IMAGE
 
-      - name: "tag and push azure image"
+      - name: "Push main branch to core artifactory"
+        if: ${{ github.ref_name == 'main' }}
         run: |
-          docker image tag $DOCKER_IMAGE $AZURE_CONTAINER_REGISTRY/$DOCKER_IMAGE &&
-          $AZURE_CONTAINER_REGISTRY/$DOCKER_IMAGE
+          docker tag $DOCKER_IMAGE $DOCKER_MAIN$DOCKER_IMAGE
+          docker tag $DOCKER_IMAGE $AZURE_CONTAINER_REGISTRY/$DOCKER_IMAGE
+          docker push $DOCKER_MAIN$DOCKER_IMAGE
+          docker push $AZURE_CONTAINER_REGISTRY/$DOCKER_IMAGE

.github/workflows/lint.yaml

@@ -9,9 +9,9 @@ jobs:
         shell: bash -l {0}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.11"
 

.pre-commit-config.yaml

@@ -11,6 +11,8 @@ repos:
   - id: detect-private-key
   - id: debug-statements
   - id: check-added-large-files
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
 
 - repo: https://github.com/igorshubovych/markdownlint-cli
   rev: v0.26.0

README.md

@@ -34,7 +34,7 @@ To update the tool version, modify the corresponding entry in TOML and create a
 Whenever possible, avoid moving images with `scopeo`, and build them with a `Dockefile` instead following the section above. Moving is not recommended for two reasons:
 
 - It hides how the image was build originally (and sometimes the original `Dockefile` is not even shared),
-- The source image can be removed by the author in the future, making your analysis no longer reproducible. 
+- The source image can be removed by the author in the future, making your analysis no longer reproducible.
 
 If you still need to move, create a script named `move-{IMAGE_NAME}:{IMAGE_VERSION}.sh` following the template below:
 

images.toml

@@ -3,25 +3,31 @@ bcftools = '1.16'
 bedtools = '2.30.0'
 bwa = '0.7.17'
 cellregmap = '0.0.3'
-dragmap = '1.3.0'
+dragmap = '1.3.0-tokenizer-next-fix'
+ensembl-vep = 'release_110.1'
 expansionhunter = '5.0.0'
+expansionhunter_bw2 = 'b85cba004fba4e0223e632fae576bc7fad0da804'
+fastp = '0.23.4'
 fastqc = '0.11.9'
 gatk = '4.2.6.1'
 hap-py = '0.3.15'
 haplocheckcli = '64293b3481d52025a01a293f6cc891546c30660e'
+mitoreport = '1.1.0'
 multipy = '0.16'
 multiqc = '1.14'
 peer = '1.3'
 picard = '2.27.4'
 reviewer = '0.2.7'
+reviewer_bw2 = '0.2.7-bw2-fork-0.2'
 saige-qtl = '1bdfe8408a17f7da8b0453a697a867f412f48bcc'
 samblaster = '0.1.26'
 samtools = '1.16.1'
+scanpy = '1.9.3'
 somalier = '0.2.15'
 trtools = '4.0.2'
 vep = '105.0'
 verifybamid = '2.0.1'
-stripy = '2.2'
+stripy = '4e393f921d15bfba8e3c2234b20c483c205ac295'
 gangstr = '2.5.0'
 hipstr = 'v0.6.2'
 bgt = 'v1.0'

images/basespace/Dockerfile

@@ -0,0 +1,6 @@
+FROM australia-southeast1-docker.pkg.dev/analysis-runner/images/driver-base:1.4
+
+RUN apt-get update && \
+    mkdir $HOME/bin && \
+    wget "https://launch.basespace.illumina.com/CLI/latest/amd64-linux/bs" -O $HOME/bin/bs && \
+    chmod u+x $HOME/bin/bs

images/dragmap/Dockerfile

@@ -13,6 +13,6 @@ RUN apt-get update && apt-get install -y \
     micromamba install -y --prefix ${MAMBA_ROOT_PREFIX} -c bioconda -c conda-forge \
         bazam biobambam google-cloud-sdk samtools \
     && rm -r /root/micromamba/pkgs && \
-    git clone --depth 1 --branch ${VERSION} https://github.com/Illumina/DRAGMAP && \
+    git clone --depth 1 --branch ${VERSION} https://github.com/populationgenomics/DRAGMAP && \
     HAS_GTEST=0 make -C DRAGMAP -j 8 install && \
     rm -r DRAGMAP

images/ensembl-vep/Dockerfile

@@ -0,0 +1,62 @@
+ARG VERSION=${VERSION:-release_110.1}
+
+FROM ensemblorg/ensembl-vep:${VERSION}
+
+USER root
+
+# This Dockerfile is a departure from the style used in `images/vep`
+# Instead of building up a fresh VEP install using Miniconda, we are
+# using the official VEP image and adding plugins and data files to it.
+# This is being built in parallel to the existing `vep` image so that we
+# can maintain the existing image for backwards compatibility, and only
+# switch once the newer VEP version has been evaluated.
+
+# The AlphaMissense plugin at time of writing (26.09.2023) is available
+# in the vep-plugin repository, but not installed within the latest image
+
+# The PLI, LoF, & UTR annotator data files are not available by default,
+# but only add minimally to the image size.
+
+# LOFTEE is installed by default, but the standard version doesn't work
+# for GRCh38, so we overwrite the standard files
+
+# GCSfuse and the Google Cloud SDK are not installed, as the workflow
+# using this image is intended to mount data files from GCS using Hail
+# Batch, so that functionality is not required inside the image.
+
+# AlphaMissense
+# print AlphaMissense scores and predictions
+# only report results for the transcripts in the AlphaMissense prediction
+# ./vep -i variations.vcf --plugin AlphaMissense,file=/full/path/to/file.tsv.gz,transcript_match=1
+# file path = gs://dm_alphamissense/AlphaMissense_aa_substitutions.tsv.gz
+ENV AlphaMissensePlugin=${VEP_DIR_PLUGINS}/AlphaMissense.pm
+
+# pLI https://raw.githubusercontent.com/Ensembl/VEP_plugins/release/110/pLI_values.txt
+# ./vep -i variants.vcf --plugin pLI,values_file.txt
+ENV PLI_SCORES=/data/pli_scores.txt
+
+# Usage: vep -i test.vcf --tab -plugin UTRannotator,/path/to/uORF_starts_ends_GRCh38_PUBLIC.txt -o test.output
+# Does not (currently) support writing  JSON output
+ENV UTR38=/data/utr_annotator_38.txt
+
+# LoFTool
+# https://github.com/Ensembl/VEP_plugins/blob/release/110/LoFtool.pm
+# https://raw.githubusercontent.com/Ensembl/VEP_plugins/release/110/LoFtool_scores.txt
+# ./vep -i variants.vcf --plugin LoFtool,scores_file.txt
+ENV LOFTOOL_SCORES=/data/loftool_110_scores.txt
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        git \
+    && curl https://raw.githubusercontent.com/Ensembl/UTRannotator/master/uORF_5UTR_GRCh38_PUBLIC.txt > ${UTR38} \
+    && curl https://raw.githubusercontent.com/Ensembl/VEP_plugins/release/110/pLI_values.txt > ${PLI_SCORES} \
+    && curl https://raw.githubusercontent.com/Ensembl/VEP_plugins/release/110/AlphaMissense.pm > ${VEP_DIR_PLUGINS}/AlphaMissense.pm \
+    && curl https://raw.githubusercontent.com/Ensembl/VEP_plugins/release/110/LoFtool_scores.txt > ${LOFTOOL_SCORES} \
+    && git clone -b grch38 https://github.com/konradjk/loftee.git \
+    && mv loftee/* ${VEP_DIR_PLUGINS} \
+    && rm -rf loftee \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/* \
+    && PERL_MM_USE_DEFAULT=1 cpan Ensembl::XS Bio::DB::HTS

images/expansionhunter_bw2/Dockerfile

@@ -0,0 +1,32 @@
+FROM ubuntu:20.04
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG VERSION=${VERSION:-b85cba004fba4e0223e632fae576bc7fad0da804}
+
+# Install Git to clone the repository
+RUN apt-get update && \
+    apt-get install -y \
+    cmake \
+    g++ \
+    git \
+    libbz2-dev \
+    libcurl4-openssl-dev \
+    liblzma-dev \
+    libssl-dev \
+    zlib1g-dev \
+    && \
+    apt-get clean -y && \
+    rm -rf /var/lib/apt/lists/*
+
+# Clone the repository, jump into it
+# build the ExpansionHunter executable, +x, move to /usr/bin
+RUN git clone https://github.com/bw2/ExpansionHunter && \
+    cd ExpansionHunter && \
+    git checkout $VERSION && \
+    mkdir build && \
+    cd build && \
+    cmake .. && \
+    make && \
+    chmod +x /ExpansionHunter/build/install/bin/ExpansionHunter && \
+    mv /ExpansionHunter/build/install/bin/ExpansionHunter /usr/bin && \
+    rm -rf /ExpansionHunter

images/fastp/Dockerfile

@@ -0,0 +1,10 @@
+FROM debian:bookworm-slim
+
+ARG VERSION=${VERSION:-0.23.4}
+
+RUN apt-get update && apt-get install -y git wget bash bzip2 zip && \
+    rm -r /var/lib/apt/lists/* && \
+    rm -r /var/cache/apt/* && \
+    wget http://opengene.org/fastp/fastp.${VERSION} && \
+    chmod a+x fastp.${VERSION} && \
+    ln -s /fastp.${VERSION} /usr/local/bin/fastp

images/hail-gcp-notebook/Dockerfile

@@ -22,7 +22,7 @@ RUN apt-get update && \
         gnomad \
         hail==${VERSION} \
         pyarrow \
-        sample-metadata \
+        metamist \
         selenium==3.8.0 && \
     curl -sSL https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 | tar xjf - && \
     mv phantomjs-2.1.1-linux-x86_64/bin/phantomjs /usr/local/bin && \

images/hipstr/Dockerfile

@@ -4,7 +4,7 @@ ARG VERSION=${VERSION:-v0.6.2}
 
 # hadolint ignore=DL3008
 RUN apt-get update -y && apt-get install -y --no-install-recommends -qqy \
-    apt-transport-https \ 
+    apt-transport-https \
     ca-certificates \
     g++ \
     git \
@@ -27,4 +27,4 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends -qqy \
     && cp HipSTR /usr/local/bin/ \
     && chmod a+x HipSTR \
     && cd .. \
-    && rm -rf HipSTR
+    && rm -rf HipSTR

images/mitoreport/Dockerfile

@@ -0,0 +1,58 @@
+FROM ubuntu:22.04
+
+ARG VERSION=${VERSION:-1.1.0}
+
+LABEL \
+  version="v${VERSION}" \
+  description="mitoreport"
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/London
+
+# Run update and install necessary libraries, mitoreport requires java11
+RUN apt-get update -y && apt-get install -y --no-install-recommends \
+    openjdk-11-jdk \
+    apt-transport-https \
+    build-essential \
+    bzip2 \
+    ca-certificates \
+    curl \
+    gcc \
+    gnupg \
+    gpg-agent \
+    libbz2-dev \
+    libcurl4-openssl-dev \
+    libffi-dev \
+    liblzma-dev \
+    libncurses5-dev \
+    libssl-dev \
+    make \
+    software-properties-common \
+    wget \
+    zlib1g-dev
+
+## Install Google cloud-sdk
+RUN curl https://sdk.cloud.google.com > install.sh && \
+    bash install.sh --disable-prompts --install-dir=/opt && \
+    rm install.sh
+
+ENV PATH=$PATH:/opt/google-cloud-sdk/bin
+
+WORKDIR /usr/local/bin/
+
+# Install mitoreport and resourses
+RUN curl -L -O https://github.com/bioinfomethods/mitoreport/releases/download/${VERSION}/mitoreport-${VERSION}-all.jar \
+    && mv mitoreport-${VERSION}-all.jar mitoreport.jar \
+    && curl -L -O https://github.com/bioinfomethods/mitoreport/releases/download/${VERSION}/resources.tgz \
+    && tar -zxvf resources.tgz \
+    && java -jar mitoreport.jar mito-map-download \
+        --output resources/mito_map_annotations.json
+
+# Requires samtools for local cram > bam
+ENV MAMBA_ROOT_PREFIX /root/micromamba
+ENV PATH $MAMBA_ROOT_PREFIX/bin:$PATH
+RUN wget -qO- https://api.anaconda.org/download/conda-forge/micromamba/0.8.2/linux-64/micromamba-0.8.2-he9b6cbd_0.tar.bz2 | tar -xvj -C /usr/local bin/micromamba && \
+    mkdir ${MAMBA_ROOT_PREFIX} && \
+    micromamba install -y --prefix ${MAMBA_ROOT_PREFIX} -c bioconda -c conda-forge \
+    samtools && \
+    rm -r /root/micromamba/pkgs

images/multipy/Dockerfile

@@ -6,4 +6,3 @@ FROM australia-southeast1-docker.pkg.dev/analysis-runner/images/driver:$DRIVER_T
 # Install a Python3-compatible version that hasn't been released on PyPI yet.
 # See https://github.com/puolival/multipy/issues/6 and https://github.com/puolival/multipy/issues/7.
 RUN pip install git+https://github.com/puolival/multipy.git@ef9d2e2ffd6a7e079e29268dfbd59e630dd346b4
-

images/reviewer_bw2/Dockerfile

@@ -0,0 +1,27 @@
+FROM ubuntu:20.04
+ARG DEBIAN_FRONTEND=noninteractive
+ARG VERSION=${VERSION:-0.2.7-bw2-fork-0.2}
+# Install Git to clone the repository
+RUN apt-get update && \
+    apt-get install -y \
+    cmake \
+    g++ \
+    git \
+    libbz2-dev \
+    libcurl4-openssl-dev \
+    liblzma-dev \
+    zlib1g-dev
+# Clone the repository, jump into it
+# build the REViewer executable, +x, move to /usr/bin
+RUN git clone https://github.com/bw2/REViewer && \
+    cd REViewer && \
+    mkdir build && \
+    cd build && \
+    cmake .. && \
+    make && \
+    chmod +x /REViewer/build/install/bin/REViewer && \
+    mv /REViewer/build/install/bin/* /usr/bin && \
+    cd . && \
+    rm -rf REViewer
+# Set the entry point for the Docker container
+ENTRYPOINT ["REViewer"]

images/saige-qtl/Dockerfile

@@ -27,4 +27,4 @@ RUN apt-get update && \
               /usr/local/bin/step3_LDmat.R \
               /usr/local/bin/createSparseGRM.R && \
     cd .. && \
-    rm -r qtl
+    rm -r qtl