Skip to content
Test Pull Requests

Test Pull Requests #1013

Sign in to view logs

Test Pull Requests

Test Pull Requests #1013

Workflow file for this run

.github/workflows/workflow_modules.yml at cf5c770
name: Test Pull Requests
on: [workflow_dispatch]
concurrency:
group: ${{ github.ref_name }}
cancel-in-progress: true
env:
WORK_DIR: "/vol/spool/${{ github.ref_name }}"
PROFILE: "slurm"
PR_NUMBER: ${{ github.event.pull_request.number }}
EMGB_KEGG_DB: "/vol/spool/emgb/annotatedgenes2json_db_kegg-mirror-2022-12"
EMGB_DB_TITLES: "/vol/spool/emgb/uniref90.titles.tsv.gz"
jobs:
full_pipeline:
timeout-minutes: 2500
runs-on: [ self-hosted, slurm]
needs: [codePolicy]
steps:
- uses: actions/checkout@v4
- name: Set Secrets
run: |
make set_secrets SECRET_NAME=S3_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
- name: Test if SRA S3 Input module works
run: |
bash ./scripts/test_fullPipeline.sh " -c test_data/assets/aws.config " \
./example_params/fullPipelineSRAQC.yml "${WORK_DIR}" ${PROFILE} || exit 1
- name: Test if SRA S3 Input module works with ONT data
run: |
bash ./scripts/test_fullPipeline.sh " -c test_data/assets/aws.config " \
./example_params/fullPipelineSRAQCONT.yml "${WORK_DIR}" ${PROFILE} || exit 1
- name: Test Full Pipeline Run
run: |
VERSION=$(sort VERSIONS.txt | tail -n 1)
bash ./scripts/test_fullPipeline.sh \
" --smetana_image=pbelmann/metabolomics:0.1.0 --databases=/vol/scratch/databases/ --output=output " \
"" "${WORK_DIR}" ${PROFILE} ${VERSION} || exit 1
- name: Test EMGB import tools
run: |
./bin/emgb.sh --output=output/test1 --runid=1 --binsdir=$(find output/test1/ -name "metabat") \
--blastdb=uniref90 \
--titles=${EMGB_DB_TITLES} \
--db=${EMGB_KEGG_DB} \
--workdir="${WORK_DIR}_wFullPipeline" --name=test1
- name: Test Full Pipeline run that ends with magAttributes
run: |
bash ./scripts/test_fullPipeline.sh \
" --databases=/vol/scratch/databases/ " \
example_params/fullPipeline_fraction/fullPipeline_fraction_magAttributes.yml "${WORK_DIR}" ${PROFILE} || exit 1
- name: Test Full Pipeline run that ends with plasmids
run: |
bash ./scripts/test_fullPipeline.sh " --databases=/vol/scratch/databases/ " \
example_params/fullPipeline_fraction/fullPipeline_fraction_plasmid.yml "${WORK_DIR}" ${PROFILE} || exit 1
- name: Test Full Pipeline run that ends with binning
run: |
bash ./scripts/test_fullPipeline.sh " --databases=/vol/scratch/databases/ " \
example_params/fullPipeline_fraction/fullPipeline_fraction_binning_metabinner.yml \
"${WORK_DIR}" ${PROFILE} || exit 1
full_pipeline_ont:
timeout-minutes: 2500
runs-on: [ self-hosted, slurm]
needs: [codePolicy]
steps:
- uses: actions/checkout@v4
- name: Set Secrets
run: |
make set_secrets SECRET_NAME=S3_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
- name: Test Full Pipeline ONT Run
run: |
VERSION=$(sort VERSIONS.txt | tail -n 1)
bash ./scripts/test_fullPipeline.sh \
" --databases=/vol/scratch/databases/ " \
./example_params/fullPipelineONT.yml "${WORK_DIR}" \
${PROFILE} ${VERSION} || exit 1
- name: Cleanup previous checkm results for correct aggregation run
run: |
rm -rf output
- name: Test Full Pipeline ONT and Illumina Run
run: |
VERSION=$(sort -r VERSIONS.txt | tail -n 1)
bash ./scripts/test_fullPipeline.sh \
" --databases=/vol/scratch/databases/ --output=output \
--smetana_image=pbelmann/metabolomics:0.1.0 --steps.cooccurrence.beforeProcessScript=/vol/spool/dockerPull.sh " \
./example_params/fullPipelineIlluminaOrONT.yml \
"${WORK_DIR}" ${PROFILE} ${VERSION} || exit 1
- name: Test aggregate workflow based on previous test
run: |
bash ./scripts/test_fullPipelineAggregate.sh \
" --output=output --smetana_image=pbelmann/metabolomics:0.1.0 \
--steps.cooccurrence.beforeProcessScript=/vol/spool/dockerPull.sh " ./example_params/fullPipelineAggregate.yml \
"${WORK_DIR}" ${PROFILE} || exit 1
- name: Test if SRA S3 Input module works with ONT data
run: |
bash ./scripts/test_fullPipeline.sh " -c test_data/assets/aws.config " \
./example_params/fullPipelineSRAQCONT.yml "${WORK_DIR}" ${PROFILE} || exit 1
qc:
timeout-minutes: 2500
needs: [codePolicy]
runs-on: [ self-hosted, slurm]
steps:
- name: QC Filter DB Test
run: |
make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/human_filter.db.20231218v2" \
MODULE_DB_TEST_MD5SUM="cc92c0f926656565b1156d66a0db5a3c" MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/human_filter.db.20231218v2.gz" \
MODULE_DB_TEST_PATH="/vol/spool/toolkit/human_filter.db.20231218v2.gz" MODULE_DB_TEST_S3PATH="s3://databases/human_filter.db.20231218v2.gz" \
MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/human_filter.db.20231218v2" \
MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --no-sign-request --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \
MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials MODULE_DB_TEST_GENERATED_YML=${WORK_DIR}/generated_yamls/qc MODULE_DB_TEST_YML=example_params/qc.yml \
MODULE_DB_TEST_YML_PATH="'.steps.qc.filterHuman.database=env(database)'" \
MODULE_DB_TEST_YML_SCRIPT="./scripts/test_qc.sh" \
MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/qc_yaml_database_tests" \
MODULE_DB_TEST_REMOVE_DB="yes"
- name: QC ONT Filter DB Test
run: |
make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/human_filter.db.20231218v2" \
MODULE_DB_TEST_MD5SUM="cc92c0f926656565b1156d66a0db5a3c" MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/human_filter.db.20231218v2.gz" \
MODULE_DB_TEST_PATH="/vol/spool/toolkit/human_filter.db.20231218v2.gz" MODULE_DB_TEST_S3PATH="s3://databases/human_filter.db.20231218v2.gz" \
MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/human_filter.db.20231218v2" \
MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --no-sign-request --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \
MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials MODULE_DB_TEST_GENERATED_YML=${WORK_DIR}/generated_yamls/qcONT MODULE_DB_TEST_YML=example_params/qcONT.yml \
MODULE_DB_TEST_YML_PATH="'.steps.qcONT.filterHumanONT.database=env(database)'" \
MODULE_DB_TEST_YML_SCRIPT="./scripts/test_qcONT.sh" \
MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/qcONT_yaml_database_tests" \
MODULE_DB_TEST_REMOVE_DB="yes"
external_SRA:
timeout-minutes: 800
runs-on: [ self-hosted, slurm]
needs: [codePolicy]
steps:
- uses: actions/checkout@v4
- name: Set Secrets
run: |
make set_secrets SECRET_NAME=S3_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
- name: Test if SRA S3 Entrypoint works
run: |
NCBI_API_KEY=${{ secrets.NCBI_API_KEY }} bash ./scripts/test_SRA.sh " -c test_data/assets/aws.config " "" "${WORK_DIR}" ${PROFILE} || exit 1
- name: Test if just the input SRA datasets are reported
run: |
bash ./scripts/test_SRA_large.sh "${WORK_DIR}" ${PROFILE} || exit 1
mag_attributes:
timeout-minutes: 2500
needs: [codePolicy]
runs-on: [ self-hosted, slurm]
steps:
- uses: actions/checkout@v4
- name: Set Secrets
run: |
make set_secrets SECRET_NAME=S3_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
make set_secrets SECRET_NAME=S3_checkm_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_checkm_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
make set_secrets SECRET_NAME=S3_checkm2_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_checkm2_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
make set_secrets SECRET_NAME=S3_gtdb_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_gtdb_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
- name: Test MAG Attributes
run: |
bash ./scripts/test_magAttributes.sh \
" --databases=/vol/scratch/databases/ " \
"" "${WORK_DIR}" ${PROFILE} || exit 1
- name: Test MAG Attributes Fraction
run: |
bash ./scripts/test_magAttributes.sh \
" --databases=/vol/scratch/databases/ " \
example_params/magAttributes_fraction/magAttributes.yml "${WORK_DIR}" ${PROFILE} || exit 1
- name: Test Checkm1 runs against different database parameters
run: |
make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/checkm/" \
MODULE_DB_TEST_MD5SUM="0963b301dfe9345ea4be1246e32f6728" \
MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/checkm_data_2015_01_16.tar.gz" \
MODULE_DB_TEST_PATH="/vol/spool/toolkit/checkm_data_2015_01_16.tar.gz" MODULE_DB_TEST_S3PATH="s3://databases/checkm_data_2015_01_16.tar.gz" \
MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/checkm/*" \
MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \
MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials MODULE_DB_TEST_GENERATED_YML=${WORK_DIR}/generated_yamls/checkm \
MODULE_DB_TEST_YML=example_params/magAttributes.yml \
MODULE_DB_TEST_YML_PATH="'.steps.magAttributes.checkm.database=env(database),del(.steps.magAttributes.gtdb,.steps.magAttributes.checkm2,.steps.magAttributes.prokka)'" \
MODULE_DB_TEST_YML_SCRIPT="./scripts/test_magAttributes.sh" MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/checkm_yaml_database_tests" \
MODULE_DB_TEST_REMOVE_DB="yes"
- name: Test Checkm2 runs against different database parameters
run: |
make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/checkm2/" \
MODULE_DB_TEST_MD5SUM="a634cb3d31a1f56f2912b74005f25f09" \
MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/checkm2_v2.tar.gz" \
MODULE_DB_TEST_PATH="/vol/spool/toolkit/checkm2_v2.tar.gz" MODULE_DB_TEST_S3PATH="s3://databases/checkm2_v2.tar.gz" \
MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/checkm2/*" \
MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \
MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials MODULE_DB_TEST_GENERATED_YML=${WORK_DIR}/generated_yamls/checkm2 \
MODULE_DB_TEST_YML=example_params/magAttributes.yml \
MODULE_DB_TEST_YML_PATH="'.steps.magAttributes.checkm2.database=env(database),del(.steps.magAttributes.gtdb,.steps.magAttributes.checkm,.steps.magAttributes.prokka)'" \
MODULE_DB_TEST_YML_SCRIPT="./scripts/test_magAttributes.sh" MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/checkm2_yaml_database_tests" \
MODULE_DB_TEST_REMOVE_DB="yes"
- name: Test GTDB runs against different database parameters
run: |
make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/gtdbtk_r214_data/release214" \
MODULE_DB_TEST_MD5SUM="390e16b3f7b0c4463eb7a3b2149261d9" \
MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/gtdbtk_r214_data.tar.gz" \
MODULE_DB_TEST_PATH="/vol/spool/toolkit/gtdbtk_r214_data.tar.gz" MODULE_DB_TEST_S3PATH="s3://databases/gtdbtk_r214_data.tar.gz" \
MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/gtdbtk_r214_data/*" \
MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \
MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials \
MODULE_DB_TEST_GENERATED_YML=${WORK_DIR}/generated_yamls/gtdb \
MODULE_DB_TEST_YML=example_params/magAttributes.yml \
MODULE_DB_TEST_YML_PATH="'.steps.magAttributes.gtdb.database=env(database),del(.steps.magAttributes.checkm,.steps.magAttributes.checkm2,.steps.magAttributes.prokka)'" \
MODULE_DB_TEST_YML_SCRIPT="./scripts/test_magAttributes.sh" \
MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/gtdb_yaml_database_tests" \
MODULE_DB_TEST_SKIP_TESTS="s3File,s3Directory,https" \
MODULE_DB_TEST_REMOVE_DB="yes"
dereplication:
timeout-minutes: 800
runs-on: [ self-hosted, slurm]
needs: [codePolicy]
steps:
- uses: actions/checkout@v4
- name: Test Dereplication
run: |
bash ./scripts/test_dereplication.sh " " "" "${WORK_DIR}" ${PROFILE} || exit 1
- name: Test Species Dereplication
run: |
bash ./scripts/test_dereplication.sh " " example_params/dereplication_fraction/dereplication.yml \
"${WORK_DIR}" ${PROFILE} || exit 1
quickstart:
timeout-minutes: 800
runs-on: [ self-hosted, slurm]
needs: [codePolicy]
steps:
- uses: actions/checkout@v4
- name: Test Quickstart
run: |
bash ./scripts/test_quickstart.sh || exit 1
readmapping:
timeout-minutes: 800
needs: [codePolicy]
runs-on: [ self-hosted, slurm]
steps:
- uses: actions/checkout@v4
- name: Test Read Mapping
run: |
bash ./scripts/test_readMapping.sh " " "" "${WORK_DIR}" ${PROFILE} || exit 1
assembly:
timeout-minutes: 800
needs: [codePolicy]
runs-on: [ self-hosted, slurm]
steps:
- uses: actions/checkout@v4
- name: Test Megahit Assembly
run: |
bash ./scripts/test_assembly.sh " " "" "${WORK_DIR}" ${PROFILE} || exit 1
- name: Test Metaspades Assembly
run: |
bash ./scripts/test_assembly.sh " " example_params/assemblyMetaspades.yml "${WORK_DIR}" ${PROFILE} || exit 1
- name: Test Metaflye Assembly
run: |
bash ./scripts/test_assemblyONT.sh " " example_params/assemblyONT.yml "${WORK_DIR}" ${PROFILE} || exit 1
cooccurrence:
timeout-minutes: 800
needs: [codePolicy]
runs-on: [ self-hosted, slurm]
steps:
- uses: actions/checkout@v4
- name: Test cooccurrence graph building
run: |
bash ./scripts/test_cooccurrence.sh "" "" "${WORK_DIR}/${{ github.ref_name }}" ${PROFILE} || exit 1
- name: Test cooccurrence graph building with edge analysis
run: |
bash ./scripts/test_cooccurrence.sh " --smetana_image=pbelmann/metabolomics:0.1.0 --steps.cooccurrence.beforeProcessScript=/vol/spool/dockerPull.sh " \
./example_params/cooccurrenceMetabolom.yml "${WORK_DIR}" ${PROFILE} || exit 1
cplex_docker:
timeout-minutes: 800
runs-on: [ self-hosted, slurm]
steps:
- uses: actions/checkout@v4
- name: Test if cplex docker container can be build
run: |
cp /vol/spool/cos_installer-test.bin cplex/docker/ \
&& cd cplex/docker \
&& docker build --no-cache -t cplextest:${PR_NUMBER} .
- name: Cleanup image
if: always()
run: |
docker images --format "{{.Repository}}:{{.Tag}}" | grep cplextest:${PR_NUMBER} | xargs -I {} docker rmi {}
fragment_recruitment:
timeout-minutes: 800
needs: [codePolicy]
runs-on: [ self-hosted, slurm]
steps:
- uses: actions/checkout@v4
- name: Test Fragment Recruitment via Mash Screen with BWA
run: |
bash ./scripts/test_fragmentRecruitment.sh " " "example_params/fragmentRecruitmentMashScreen.yml" "${WORK_DIR}" ${PROFILE} || exit 1
- name: Test Fragment Recruitment via Mash Screen with Bowtie
run: |
bash ./scripts/test_fragmentRecruitment.sh " " \
"example_params/fragmentRecruitment_fraction/fragmentRecruitmentMashScreenBowtie.yml" "${WORK_DIR}" ${PROFILE} || exit 1
annotation:
timeout-minutes: 2200
needs: [codePolicy]
runs-on: [ self-hosted, slurm]
steps:
- uses: actions/checkout@v4
- name: Set Secrets
run: |
make set_secrets SECRET_NAME=S3_kegg_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_kegg_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
- name: Test Annotation
run: |
bash ./scripts/test_annotation.sh \
" --steps.annotation.mmseqs.kegg.database.extractedDBPath=/vol/spool/toolkit/kegg-mirror-2021-01_mmseqs/sequenceDB \
--steps.annotation.keggFromMMseqs2.database.extractedDBPath=/vol/spool/toolkit/kegg_2021-01/ --databases=/vol/scratch/databases/ " "" \
"${WORK_DIR}" ${PROFILE} || exit 1
annotation_database_tests_part_1:
timeout-minutes: 2200
needs: [codePolicy]
runs-on: [ self-hosted, slurm]
steps:
- uses: actions/checkout@v4
- name: Set Secrets
run: |
make set_secrets SECRET_NAME=S3_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
make set_secrets SECRET_NAME=S3_rgi_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_rgi_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
make set_secrets SECRET_NAME=S3_kegg_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_kegg_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
- name: Test rgi against different database configurations
run: |
make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/card_20221209/card.json" \
MODULE_DB_TEST_MD5SUM="d7e627221a1d4584d1c1795cda774cdb" \
MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/card_20221209.tar.bz2" \
MODULE_DB_TEST_PATH="/vol/spool/toolkit/card_20221209.tar.bz2" MODULE_DB_TEST_S3PATH="s3://databases/card_20221209.tar.bz2" \
MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/card_20221209/*" \
MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \
MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials \
MODULE_DB_TEST_GENERATED_YML=${WORK_DIR}/generated_yamls/rgi \
MODULE_DB_TEST_YML=example_params/annotation.yml \
MODULE_DB_TEST_YML_PATH="'.steps.annotation.rgi.database=env(database),del(.steps.annotation.mmseqs2,.steps.annotation.keggFromMMseqs2)'" \
MODULE_DB_TEST_YML_SCRIPT="./scripts/test_annotation.sh" MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/rgi_yaml_database_tests" \
MODULE_DB_TEST_SKIP_TESTS="https" \
MODULE_DB_TEST_REMOVE_DB="yes"
- name: Test keggFromMMseqs2 against different database configurations
run: |
make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/kegg_2021-01" \
MODULE_DB_TEST_MD5SUM="69538151ecf4128f4cb1e0ac21ff55b3" \
MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases_internal/kegg-links-mirror-2021-01.tar.gz" \
MODULE_DB_TEST_PATH="/vol/spool/toolkit/kegg.tar.gz" MODULE_DB_TEST_S3PATH="s3://databases_internal/kegg-links-mirror-2021-01.tar.gz" \
MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases_internal/kegg-links-mirror-2021-01/*" \
MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \
MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials \
MODULE_DB_TEST_GENERATED_YML=${WORK_DIR}/generated_yamls/kegg \
MODULE_DB_TEST_YML=example_params/annotation.yml \
MODULE_DB_TEST_YML_PATH="'.steps.annotation.keggFromMMseqs2.database=env(database),del(.steps.annotation.rgi)'" \
MODULE_DB_TEST_YML_SCRIPT="./scripts/test_annotation.sh" MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/kegg_yaml_database_tests" \
MODULE_DB_TEST_SKIP_TESTS="https" \
MODULE_DB_TEST_REMOVE_DB="yes"
annotation_database_tests_part_2:
timeout-minutes: 2200
needs: [codePolicy]
runs-on: [ self-hosted, slurm]
steps:
- uses: actions/checkout@v4
- name: Set Secrets
run: |
make set_secrets SECRET_NAME=S3_kegg_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_kegg_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
make set_secrets SECRET_NAME=S3_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
- name: Test mmseqs2 against different database configurations
run: |
make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/kegg-mirror-2021-01_mmseqs/sequenceDB" \
MODULE_DB_TEST_MD5SUM="0d20db97b3e7ee6571ca1fd5ad3a87f1" \
MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases_internal/kegg-mirror-2021-01_mmseqs.tar.zst" \
MODULE_DB_TEST_PATH="/vol/spool/toolkit/kegg-mirror-2021-01_mmseqs.tar.zst" MODULE_DB_TEST_S3PATH="s3://databases_internal/kegg-mirror-2021-01_mmseqs.tar.zst" \
MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases_internal/kegg-mirror-2021-01_mmseqs/*" \
MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \
MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials \
MODULE_DB_TEST_GENERATED_YML=${WORK_DIR}/generated_yamls/mmseqs2 \
MODULE_DB_TEST_YML=example_params/annotation.yml \
MODULE_DB_TEST_YML_PATH="'.steps.annotation.mmseqs2.kegg.database=env(database),del(.steps.annotation.rgi,.steps.annotation.keggFromMMseqs2)'" \
MODULE_DB_TEST_YML_SCRIPT="./scripts/test_annotation.sh" \
MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/mmseqs2_yaml_database_tests" \
MODULE_DB_TEST_SKIP_TESTS="s3File,s3Directory,https" \
MODULE_DB_TEST_REMOVE_DB="yes"
external_S3_annotation_database:
timeout-minutes: 2200
needs: [codePolicy]
runs-on: [ self-hosted, slurm]
steps:
- uses: actions/checkout@v4
- name: Set Secrets
run: |
make set_secrets SECRET_NAME=S3_kegg_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_kegg_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
make set_secrets SECRET_NAME=S3_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
- name: Test mmseqs2 against different database configurations
run: |
make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/kegg-mirror-2021-01_mmseqs/sequenceDB" \
MODULE_DB_TEST_MD5SUM="0d20db97b3e7ee6571ca1fd5ad3a87f1" \
MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases_internal/kegg-mirror-2021-01_mmseqs.tar.zst" \
MODULE_DB_TEST_PATH="/vol/spool/toolkit/kegg-mirror-2021-01_mmseqs.tar.zst" MODULE_DB_TEST_S3PATH="s3://databases_internal/kegg-mirror-2021-01_mmseqs.tar.zst" \
MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases_internal/kegg-mirror-2021-01_mmseqs/*" \
MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \
MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials \
MODULE_DB_TEST_GENERATED_YML=${WORK_DIR}/generated_yamls/mmseqs2 \
MODULE_DB_TEST_YML=example_params/annotation.yml \
MODULE_DB_TEST_YML_PATH="'.steps.annotation.mmseqs2.kegg.database=env(database),del(.steps.annotation.rgi,.steps.annotation.keggFromMMseqs2)'" \
MODULE_DB_TEST_YML_SCRIPT="./scripts/test_annotation.sh" \
MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/mmseqs2_yaml_database_tests" \
MODULE_DB_TEST_SKIP_TESTS="extracted,https,compressed" \
MODULE_DB_TEST_REMOVE_DB="yes"
external_gtdb_S3_mag_attributes:
timeout-minutes: 2500
needs: [codePolicy]
runs-on: [ self-hosted, slurm]
steps:
- uses: actions/checkout@v4
- name: Set Secrets
run: |
make set_secrets SECRET_NAME=S3_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
make set_secrets SECRET_NAME=S3_gtdb_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_gtdb_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
- name: Test GTDB runs against different database parameters
run: |
make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/gtdbtk_r214_data/release214" \
MODULE_DB_TEST_MD5SUM="390e16b3f7b0c4463eb7a3b2149261d9" \
MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/gtdbtk_r214_data.tar.gz" \
MODULE_DB_TEST_PATH="/vol/spool/toolkit/gtdbtk_r214_data.tar.gz" MODULE_DB_TEST_S3PATH="s3://databases/gtdbtk_r214_data.tar.gz" \
MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/gtdbtk_r214_data/*" \
MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \
MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials \
MODULE_DB_TEST_GENERATED_YML=${WORK_DIR}/generated_yamls/gtdb \
MODULE_DB_TEST_YML=example_params/magAttributes.yml \
MODULE_DB_TEST_YML_PATH="'.steps.magAttributes.gtdb.database=env(database),del(.steps.magAttributes.checkm,.steps.magAttributes.checkm2,.steps.magAttributes.prokka)'" \
MODULE_DB_TEST_YML_SCRIPT="./scripts/test_magAttributes.sh" \
MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/gtdb_yaml_database_tests" \
MODULE_DB_TEST_SKIP_TESTS="extracted,https,compressed" \
MODULE_DB_TEST_REMOVE_DB="yes"
metabolomics:
timeout-minutes: 700
runs-on: [ self-hosted, slurm]
needs: [codePolicy]
steps:
- uses: actions/checkout@v4
- name: Metabolomics
run: |
bash ./scripts/test_metabolomics.sh " -c test_data/assets/aws.config \
--steps.metabolomics.smetana.beforeProcessScript=/vol/spool/dockerPull.sh \
--steps.metabolomics.memote.beforeProcessScript=/vol/spool/dockerPull.sh \
--smetana_image=pbelmann/metabolomics:0.1.0 " \
"" "${WORK_DIR}" "slurm"
plasmid:
timeout-minutes: 800
needs: [codePolicy]
runs-on: [ self-hosted, slurm]
steps:
- uses: actions/checkout@v4
- name: Set Secrets
run: |
make set_secrets SECRET_NAME=S3_PLSDB_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_PLSDB_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
make set_secrets SECRET_NAME=S3_ViralVerifyPlasmid_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_ViralVerifyPlasmid_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
make set_secrets SECRET_NAME=S3_Platon_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_Platon_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
make set_secrets SECRET_NAME=S3_MobTyper_ACCESS SECRET_VALUE=${{ secrets.S3_ACCESS }}
make set_secrets SECRET_NAME=S3_MobTyper_SECRET SECRET_VALUE=${{ secrets.S3_VALUE }}
- name: Plasmid Test
run: |
bash ./scripts/test_plasmids.sh " --databases=/vol/scratch/databases/ " "" "${WORK_DIR}/${{ github.ref_name }}" ${PROFILE} || exit 1
- name: Plasmids Database Test
run: |
make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/plasmids_plsdb_20220929" \
MODULE_DB_TEST_MD5SUM="13c1078e6cd6a46e3f508c24ca07cc18" \
MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/plasmids_plsdb_20220929.tar.bz2" \
MODULE_DB_TEST_PATH="/vol/spool/toolkit/plasmids_plsdb_20220929.tar.bz2" \
MODULE_DB_TEST_S3PATH="s3://databases/plasmids_plsdb_20220929.tar.bz2" \
MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/plasmids_plsdb_20220929/*" \
MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \
MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials \
MODULE_DB_TEST_GENERATED_YML=${WORK_DIR}/generated_yamls/plasmids \
MODULE_DB_TEST_YML=example_params/plasmid.yml MODULE_DB_TEST_YML_PATH="'.steps.plasmid.PLSDB.database=env(database),del(.steps.plasmid.Platon,.steps.plasmid.ViralVerifyPlasmid,.steps.plasmid.MobTyper)'" \
MODULE_DB_TEST_YML_SCRIPT="./scripts/test_plasmids.sh" MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/plasmid_yaml_database_tests" \
MODULE_DB_TEST_REMOVE_DB="yes"
- name: ViralVerify Plasmid Test
run: |
make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/pfam-A_35.0.hmm" \
MODULE_DB_TEST_MD5SUM="c80b75bd48ec41760bbca19c70616e36" MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/pfam-A_35.0.hmm.gz" \
MODULE_DB_TEST_PATH="/vol/spool/toolkit/pfam-A_35.0.hmm.gz" MODULE_DB_TEST_S3PATH="s3://databases/pfam-A_35.0.hmm.gz" \
MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/pfam-A_35.*.hmm" \
MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --no-sign-request --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \
MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials MODULE_DB_TEST_GENERATED_YML=${WORK_DIR}/generated_yamls/plasmids MODULE_DB_TEST_YML=example_params/plasmid.yml \
MODULE_DB_TEST_YML_PATH="'.steps.plasmid.ViralVerifyPlasmid.database=env(database),del(.steps.plasmid.Platon,.steps.plasmid.PlasClass,.steps.plasmid.MobTyper,.steps.plasmid.PLSDB)'" \
MODULE_DB_TEST_YML_SCRIPT="./scripts/test_plasmids.sh" \
MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/plasmid_yaml_database_tests" \
MODULE_DB_TEST_REMOVE_DB="yes"
- name: Platon Test
run: |
make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/platon_20220929" \
MODULE_DB_TEST_MD5SUM="f6d1701704396182c6c9daca053eb9d6" \
MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/platon_20220929.tar.gz" \
MODULE_DB_TEST_PATH="/vol/spool/toolkit/platon_20220929.tar.gz" MODULE_DB_TEST_S3PATH="s3://databases/platon_20220929.tar.gz" \
MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/platon/*" \
MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --no-sign-request --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \
MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials MODULE_DB_TEST_GENERATED_YML=${WORK_DIR}/generated_yamls/plasmids MODULE_DB_TEST_YML=example_params/plasmid.yml \
MODULE_DB_TEST_YML_PATH="'.steps.plasmid.Platon.database=env(database),del(.steps.plasmid.ViralVerifyPlasmid,.steps.plasmid.PlasClass,.steps.plasmid.MobTyper,.steps.plasmid.PLSDB)'" \
MODULE_DB_TEST_YML_SCRIPT="./scripts/test_plasmids.sh" MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/plasmid_yaml_database_tests" \
MODULE_DB_TEST_REMOVE_DB="yes"
- name: MobTyper Test
run: |
make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/mob_20220929" \
MODULE_DB_TEST_MD5SUM="21fcaf9c3754a985d1d6875939d71e28" \
MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/mob_20220929.gz" \
MODULE_DB_TEST_PATH="/vol/spool/toolkit/mob_20220929.gz" MODULE_DB_TEST_S3PATH="s3://databases/mob_20220929.gz" \
MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/mob_20220929/*" \
MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --no-sign-request --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \
MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials MODULE_DB_TEST_GENERATED_YML=${WORK_DIR}/generated_yamls/plasmids MODULE_DB_TEST_YML=example_params/plasmid.yml \
MODULE_DB_TEST_YML_PATH="'.steps.plasmid.MobTyper.database=env(database),del(.steps.plasmid.ViralVerifyPlasmid,.steps.plasmid.PlasClass,.steps.plasmid.Platon,.steps.plasmid.PLSDB)'" \
MODULE_DB_TEST_YML_SCRIPT="./scripts/test_plasmids.sh" MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/plasmid_yaml_database_tests" \
MODULE_DB_TEST_REMOVE_DB="yes"
settings:
timeout-minutes: 2500
runs-on: [ self-hosted, slurm]
steps:
- uses: actions/checkout@v4
- name: Test whether settings in easy mode can be updated
run: |
VERSION=$(sort VERSIONS.txt | tail -n 1)
OUTPUT=outputEasy
bash ./scripts/test_settings.sh \
" --preset --template default/fullPipeline_illumina_nanpore.yml --scratch /vol/scratch --input.paired.path test_data/fullPipeline/reads_split.tsv --highmemLarge=28,2000 --s3SignIn false --databases=/vol/scratch/databases/ --output=${OUTPUT} " \
" " "${WORK_DIR}" ${PROFILE} ${VERSION} "preset" || exit 1
bash ./scripts/check_parameter.sh ${OUTPUT} || exit 1
- name: Test whether settings in default mode can be updated
run: |
VERSION=$(sort VERSIONS.txt | tail -n 1)
OUTPUT=outputDefault
bash ./scripts/test_settings.sh \
" --scratch /vol/scratch --template default/fullPipeline_illumina_nanpore.yml --resources.highmemLarge.memory=2000 --s3SignIn false --databases=/vol/scratch/databases/ --output=${OUTPUT} " \
"" "${WORK_DIR}" ${PROFILE} ${VERSION} "" || exit 1
bash ./scripts/check_parameter.sh ${OUTPUT} || exit 1
codePolicy:
runs-on: [ self-hosted, slurm]
needs: [settings]
steps:
- uses: actions/checkout@v4
- name: Checks if every process defines a publishDirMode
run: |
make checkPublisDirMode || exit 1