diff --git a/.github/workflows/build_feature_branch.yml b/.github/workflows/build_feature_branch.yml deleted file mode 100644 index b62fc1c89..000000000 --- a/.github/workflows/build_feature_branch.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Build Feature Branches - -on: [pull_request] - -jobs: - main: - runs-on: ubuntu-20.04 - steps: - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ secrets.GHCR_USERNAME }} - password: ${{ secrets.GHCR_TOKEN }} - - - name: Build and push this feature branch - id: docker_build - uses: docker/build-push-action@v2 - with: - push: true - tags: ghcr.io/${{ github.repository }}:${{ github.head_ref }} - - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/build_on_push.yaml b/.github/workflows/build_on_push.yaml deleted file mode 100644 index d610acfae..000000000 --- a/.github/workflows/build_on_push.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: Build Main/Develop Branches on push - -on: - push: - branches: - - main - - master - - develop - -jobs: - main: - runs-on: ubuntu-20.04 - steps: - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ secrets.GHCR_USERNAME }} - password: ${{ secrets.GHCR_TOKEN }} - - - name: Build and push the main branch - id: docker_build - uses: docker/build-push-action@v2 - with: - push: true - tags: ghcr.io/${{ github.repository }}:${ GITHUB_REF##*/ } - - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml index c4920ce01..16c264e83 100644 --- a/.github/workflows/ee2-tests.yml +++ b/.github/workflows/ee2-tests.yml @@ -11,7 +11,7 @@ name: Execution Engine 2 Test Suite on: - [push, pull_request] + [pull_request] jobs: build: @@ -43,4 +43,3 @@ jobs: cp test/env/test.travis.env test.env make test-coverage codecov - diff --git a/.github/workflows/manual-build.yml b/.github/workflows/manual-build.yml new file mode 100644 index 000000000..944f9035a --- /dev/null +++ b/.github/workflows/manual-build.yml @@ -0,0 +1,11 @@ +--- +name: Manual Build & Push +on: + workflow_dispatch: +jobs: + build-push: + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}-develop' + tags: br-${{ github.ref_name }} + secrets: inherit diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml new file mode 100644 index 000000000..bf5d7e076 --- /dev/null +++ b/.github/workflows/pr_build.yml @@ -0,0 +1,43 @@ +--- +name: Pull Request Build, Tag, & Push +on: + pull_request: + branches: + - develop + - main + - master + types: + - opened + - reopened + - synchronize + - merged +jobs: + build-develop-open: + if: github.base_ref == 'develop' && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_build.yml@main + secrets: inherit + build-develop-merge: + if: github.base_ref == 'develop' && github.event.pull_request.merged == true + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}-develop' + tags: pr-${{ github.event.number }},latest + secrets: inherit + build-main-open: + if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: pr-${{ github.event.number }} + secrets: inherit + build-main-merge: + if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == true + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: pr-${{ github.event.number }},latest-rc + secrets: inherit + trivy-scans: + if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main + secrets: inherit diff --git a/.github/workflows/prod_release.yml b/.github/workflows/prod_release.yml new file mode 100644 index 000000000..a25467818 --- /dev/null +++ b/.github/workflows/prod_release.yml @@ -0,0 +1,25 @@ +--- +name: Release - Build & Push Image +on: + release: + branches: + - main + - master + types: [ published ] +jobs: + check-source-branch: + uses: kbase/.github/.github/workflows/reusable_validate-branch.yml@main + with: + build_branch: '${{ github.event.release.target_commitish }}' + validate-release-tag: + needs: check-source-branch + uses: kbase/.github/.github/workflows/reusable_validate-release-tag.yml@main + with: + release_tag: '${{ github.event.release.tag_name }}' + build-push: + needs: validate-release-tag + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: '${{ github.event.release.tag_name }},latest' + secrets: inherit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d3cb732c7..4556abb64 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,10 @@ repos: - repo: https://github.com/ambv/black - rev: 21.5b0 + rev: 22.1.0 hooks: - id: black exclude: '.+Impl.py' - repo: https://gitlab.com/pycqa/flake8 - rev: '' + rev: '3.9.2' hooks: - id: flake8 diff --git a/Dockerfile b/Dockerfile index 4939fde50..02fcbf9d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -FROM quay.io/kbase/sdkbase2:python +FROM kbase/sdkpython:3.8.0 MAINTAINER KBase Developer -RUN apt-get clean all && apt-get update --fix-missing -y +RUN apt-get clean all && apt-get update --fix-missing -y && apt-get upgrade -y # ----------------------------------------- # In this section, you can install any system dependencies required @@ -11,21 +11,19 @@ RUN apt-get clean all && apt-get update --fix-missing -y RUN apt-get install -y gcc wget vim htop tmpreaper RUN mkdir -p /etc/apt/sources.list.d +# Install condor +RUN curl -fsSL https://get.htcondor.org | /bin/bash -s -- --no-dry-run -RUN DEBIAN_FRONTEND=noninteractive wget -qO - https://research.cs.wisc.edu/htcondor/debian/HTCondor-Release.gpg.key | apt-key add - \ - && echo "deb http://research.cs.wisc.edu/htcondor/debian/8.8/stretch stretch contrib" >> /etc/apt/sources.list \ - && echo "deb-src http://research.cs.wisc.edu/htcondor/debian/8.8/stretch stretch contrib" >> /etc/apt/sources.list \ - && apt-get update -y \ - && apt-get install -y condor +# Install jars for testing purposes +# Uncomment this if you want to run tests inside the ee2 container on MacOSX +# RUN cd /opt && git clone https://github.com/kbase/jars && cd - -# install jars -# perhaps we should have test and prod dockerfiles to avoid jars and mongo installs in prod -RUN cd /opt \ - && git clone https://github.com/kbase/jars \ - && cd - - -# Remove due to cve-2021-4104 issue in spin (log4j) -RUN rm /opt/jars/lib/jars/dockerjava/docker-java-shaded-3.0.14.jar + +# Install DOCKERIZE +RUN curl -o /tmp/dockerize.tgz https://raw.githubusercontent.com/kbase/dockerize/dist/dockerize-linux-amd64-v0.5.0.tar.gz && \ + cd /usr/bin && \ + tar xvzf /tmp/dockerize.tgz && \ + rm /tmp/dockerize.tgz # install mongodb @@ -65,6 +63,10 @@ RUN mkdir -p /kb/module/work && chmod -R a+rw /kb/module && mkdir -p /etc/condor WORKDIR /kb/module RUN make all +# Remove Jars and old Conda for Trivy Scans and after compilation is done +RUN rm -rf /sdk && rm -rf /opt +RUN rm -rf /miniconda-latest/pkgs/conda-4.12.0-py39h06a4308_0/info/test/tests/data/env_metadata + WORKDIR /kb/module/scripts RUN chmod +x download_runner.sh && ./download_runner.sh diff --git a/Pipfile b/Pipfile index ec7e9deb0..5de7bfa43 100644 --- a/Pipfile +++ b/Pipfile @@ -64,7 +64,7 @@ slackclient = "==2.7.1" toml = "==0.10.1" tqdm = "==4.42.1" typing-extensions = "==3.7.4.3" -ujson = "==1.35" +ujson = "==5.2.0" urllib3 = "==1.25.8" uvloop = "==0.12.2" websockets = "==6.0" diff --git a/Pipfile.lock b/Pipfile.lock index 5df9ba0ff..e238e5484 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "96e75d4a0d655bab93d08d5d163c1bdb458d7ff8bc22b3e48af30e07797d0340" + "sha256": "131a9560c753b4c2a11c262f60221ca37fbf88b824191cfd09e123bd9e63c170" }, "pipfile-spec": 6, "requires": { @@ -199,32 +199,42 @@ }, "coverage": { "hashes": [ + "sha256:0c5fe441b9cfdab64719f24e9684502a59432df7570521563d7b1aff27ac755f", + "sha256:2b412abc4c7d6e019ce7c27cbc229783035eef6d5401695dccba80f481be4eb3", "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", + "sha256:42692db854d13c6c5e9541b6ffe0fe921fe16c9c446358d642ccae1462582d3b", "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", + "sha256:4ec30ade438d1711562f3786bea33a9da6107414aed60a5daa974d50a8c2c351", "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", + "sha256:6899797ac384b239ce1926f3cb86ffc19996f6fa3a1efbb23cb49e0c12d8c18c", "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", + "sha256:8e679d1bde5e2de4a909efb071f14b472a678b788904440779d2c449c0355b27", "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", + "sha256:93f965415cc51604f571e491f280cff0f5be35895b4eb5e55b47ae90c02a497b", "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", + "sha256:a9abc8c480e103dc05d9b332c6cc9fb1586330356fc14f1aa9c0ca5745097d19", "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", + "sha256:c22ab9f96cbaff05c6a84e20ec856383d27eae09e511d3e6ac4479489195861d", "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", + "sha256:ca58eba39c68010d7e87a823f22a081b5290e3e3c64714aac3c91481d8b34d22", "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", @@ -563,10 +573,11 @@ }, "packaging": { "hashes": [ - "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5", - "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a" + "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", + "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" ], - "version": "==20.9" + "markers": "python_version >= '3.6'", + "version": "==21.3" }, "pluggy": { "hashes": [ @@ -619,6 +630,7 @@ "hashes": [ "sha256:32421df60d06f479d71b6b539642e410ece3006e8910688e68df962c8eb40a21", "sha256:324b22a8443e11faca44c96b20e7ec8a9e59a1e664457edeeb4f796080b31cde", + "sha256:3b6336b1d2a1ac2fcc8f629070016f3c76ad7dc969f269232471953d6dd17c0d", "sha256:4505ff8b7923dd7a8bed1bf25c9c4d0df5ab0b8b2821f2296533f2149a55f401", "sha256:460b224681ea711e48e3638d15be2249024031b7dcb9622ba19c2e85bd5a26cc", "sha256:47473b70c5f3cd5ddd2c49ab3b9ceafdafbbed5bc963f147df22a9343d7978f5", @@ -630,6 +642,7 @@ "sha256:61cad83637ae12c1c825130d7f9325cd6c162e3a64e8747a8144866020be3ff4", "sha256:61e8e1c58b4fdf47ab79b7c7db8bb022c1e40b3b5fcbbaeea5fc94dc5c75638d", "sha256:6e04e496af7d156b66cce70460011c621ecbadf5dcdce325c7acbb3cd6ea245d", + "sha256:74838f04da0b3995b830fe1f00f9b200831582cbc42a22b77e04dfb717cb0d56", "sha256:7ef89ec435e89da902451dde6845066fe2770befaf0301fe2a1ac426b51fced3", "sha256:854e8425e5eb775ccfffad04ecd094c99923d60a2c2d49babb5c435e836a91fa", "sha256:9569796d48498e4db4e1d56284b626a8ed15f641ce3a8b2085f06bb03f4c2c88", @@ -639,6 +652,7 @@ "sha256:aef7d88384ada699976350a285c7a333f96ebc959e98e7d2c98589f47bbf3b7f", "sha256:b4d7ff9957ee770cf03bd7156a68a2f2e838e60712d9608eadc8741c15d01e72", "sha256:c1db85c39e6a60588f855dbc7bd68fb0dab796096148ab5aa4abecaff19e1c6e", + "sha256:c3e813b1bd0b883639e30170dc9daccb9b6ef7e81836188b88d3fc7364892b35", "sha256:cee2fc0b94e66e7230da12fc4b3d34793c49957e16ee04f6468a94e264a1e41d", "sha256:cf1dea28379a16b23e47db312883f07b3ba8d9d6abc1c59e51d4c8ae1820ab43", "sha256:d1cd175df7c8b5fc976bade78bf4d9fb5aa7ab465c0f59931e380bbe188ef8fc", @@ -692,6 +706,8 @@ }, "pytest-profiling": { "hashes": [ + "sha256:3b255f9db36cb2dd7536a8e7e294c612c0be7f7850a7d30754878e4315d56600", + "sha256:6bce4e2edc04409d2f3158c16750fab8074f62d404cc38eeb075dff7fcbb996c", "sha256:93938f147662225d2b8bd5af89587b979652426a8a6ffd7e73ec4a23e24b7f29", "sha256:999cc9ac94f2e528e3f5d43465da277429984a1c237ae9818f8cfd0b06acb019" ], @@ -789,6 +805,14 @@ "index": "pypi", "version": "==0.14.3" }, + "setuptools": { + "hashes": [ + "sha256:68e45d17c9281ba25dc0104eadd2647172b3472d9e01f911efa57965e8d51a36", + "sha256:a43bdedf853c670e5fed28e5623403bad2f73cf02f9a2774e91def6bda8265a7" + ], + "markers": "python_version >= '3.7'", + "version": "==62.3.2" + }, "six": { "hashes": [ "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", @@ -832,10 +856,59 @@ }, "ujson": { "hashes": [ - "sha256:f66073e5506e91d204ab0c614a148d5aa938bdbf104751be66f8ad7a222f5f86" + "sha256:04a8c388b2d16316df3365c81f368955662581f6a4ff033e9aba2dd1ffc9e05e", + "sha256:080da13f81740c076e5f16c254a10d0e32f45d225a5e6b0687a86493cfcfbafb", + "sha256:0b47a138203bb06bdac03b2a89ac9b2993fd32cb7daded06c966dd84300a5786", + "sha256:102b8eb5e15e6c5537426414d180c28dbf0489e51f7c22b706511ac84aae4458", + "sha256:11f735870f189bff1841c720115226894415ab6a7796dee8ab46bc767ea2e743", + "sha256:163191b88842d874e081707d35de2e205e0e396e70fd068d1038879bca8b17ad", + "sha256:25522c674b35c33f375586ac98d92ce731e79059424507ecbccbfcbce832d597", + "sha256:27a254a150e46980608b16ef3b609e703173492cfa738f4644c81d7e7d77494c", + "sha256:2c04456de1fc92cc7062904c176c74e6ea220469b949508be42e819646a28457", + "sha256:2c7712da662b92f80442a8efc0df09cea3a5efb42b0dd6a642e36b1b40a260d4", + "sha256:350a3010db0045e1306bbdf889d1bdaee9bb095856c317716f0a74108cf4afe9", + "sha256:468d7d8dcbafc3fd40cc73e4a533a7a1d4f935f605c15ae6cac32c6d53c4c6aa", + "sha256:489d495431c80dc0048c4551a0d6cdbf1209e2d274f47c3f72415c91842eeb68", + "sha256:49ce8521b0cdf210481bd89887fd1bd0a975f66088b1256dafc77c67c8ccb89d", + "sha256:4d1ed3897e45477b2a4a1371186df299b13938d4d44d850953a4bb0ea4cb38f3", + "sha256:54ee7c46615b42f7ae9dca90f54d204a4d2041a4c926b08fffa953aa3a246e54", + "sha256:584c558c23ddc21f5b07d2c54ee527731bd9716101c27829023ab7f3ffbaa8fc", + "sha256:6227597d0201ceadc902d1a8edaffaeb244050b197368ed25e6f6be0df170a6f", + "sha256:6677bee8690c71f5e6cf519a6d8400f04fbd3ff9f6c50f35f1b664bc94546f84", + "sha256:6b455a62bd20e890b2124a65df45313b4292dbea851ef38574e5e2de94691ad5", + "sha256:6c5bbe6de6c9a5fe8dca56e36fb5c4a42e1a01d4aae1ac20cd8d7d82ccff9430", + "sha256:729af63e4de30c54b527b54b4100266f79833c1e8ba35e784f01b44c2aca88d8", + "sha256:754e9da96a24535ae5ab2a52e1d1dfc65a6a717c14063855b83f327fdf2173ea", + "sha256:75a886bd89d8e5a004a39a6c5dc8a43bb7fcf05129d2dccd16a59602a612823a", + "sha256:8c3f7578a62d9255650ef32e78d3345e98262e064c9ba3f205311b4c9eb507a6", + "sha256:90de04391916c5adc7bbcc69bd778e263ed45cc83c070099cb07ed25068d6a12", + "sha256:940f35e9a0969440621445dbb6adffaa2cea77d0262abc74fce78704120c4534", + "sha256:9acc874128baddeff908736db251597e4cbd007a384730377a59a61b08886599", + "sha256:a1a55b3310632661a03ce68ccfb92264031aea21626d6fa5c8f6c32e769be7b6", + "sha256:a3c6798035b574ceba747de83f3223a622622b7ab77a24f8b4fbea2cb92f14b0", + "sha256:a5e374e793b0a3c7df20ee4c8234e89859ddb2b2821cc3300ae94ab5b08fa6d0", + "sha256:a6f3ad3b11578bc4e25d5bd256c938fe2c7c015d8f504bc7835f127ed26a0818", + "sha256:b3671e1dfc49a4b4453d89fd7438aa9d7cca28afe329c70eba84e2a5778dbf3f", + "sha256:b5fcbaabf3d115cb816eb165f3fa5de5c5bc795473a554ae55620d134ddf2d36", + "sha256:bc1a619bad9894dad144184b735c98179c7d92d7b40fbda28eb8b0857bdfdf52", + "sha256:be909514a47b6272e34cd1213feee324ca35a354e07f1ae3aba12d3694a5279f", + "sha256:c519743a53bbe8aac6b743bcf50eb83057d1e0341e1ca8f8491f729a885af640", + "sha256:c549d5a7652c3a0dd00ef6ff910fb01878bc116c66c94ac455a55cffa32cc229", + "sha256:d1e5c635b7c3465ab8d2e3dc97c341ef1801c53a378f1d1d4cb934f6c90ec66c", + "sha256:d2357ce7d93eadd29b6efbe72228809948cc59ec6682c20fa6de08aeef1703f8", + "sha256:d38c2a58c892c680080b22b59eebd77b7c6f4ae24361111fba115f9ed3651dcf", + "sha256:d57a87bbc77d66b8a2b74bab66357c3bb6194f5d248f1053fb8044787abde73f", + "sha256:d9b1c3d2b22c040a81ff4e5927ce307919f7ac8bf888afded714d925edc8d0a4", + "sha256:dc5fd1d5b48edd3cc64e89ea94abe231509fdc938bdeafafe9aef3a05810159f", + "sha256:dc71ead5706e81fdf1054c8c11e4aaab43527da450a2701213c20717852d1a51", + "sha256:e53388fb092197cb8f956673792aca994872917d897ca42a0abf7a35e293575a", + "sha256:e991b7b3a08ac9e9d3a51589ef1c359c8d44ece730351cfac055684bf3787372", + "sha256:ed78a5b169ece75a1e1368935ce6ab051dcbcd5c158b9796b2f1fa6cc467a651", + "sha256:ef868bf01851869a26c0ca5f88036903836c3a6b463c74d96b37f294f6bdeea4", + "sha256:fb4555df1fe018806ba14cc38786269c8e213930103c6d0ac81e506d09d1de7e" ], "index": "pypi", - "version": "==1.35" + "version": "==5.2.0" }, "urllib3": { "hashes": [ diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 62948402e..855352c76 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,19 @@ # execution_engine2 (ee2) release notes ========================================= +## 0.0.9 +* Update GHA with latest actions, remove old actions +* Change job defaults to result in +* NJS and KB_UPLOAD 5 Jobs +* Bigmem nodes with 250GB of ram take 1 job +* Bigmem nodes with 1TB of ram take 4 jobs +* Remove Jars from built image / cleanup for trivy + +## 0.0.81 +* Updated HTCondor Clients, New Base Image +* Use default GH actions +* Updated precommit hooks + ## 0.0.8 * Fixed a bug that could, seemingly rarely, cause job and log updates to be applied to the wrong Mongo collection. diff --git a/bin/PurgeBadJobs.py b/bin/PurgeBadJobs.py index d2a182c84..b1b1758f3 100644 --- a/bin/PurgeBadJobs.py +++ b/bin/PurgeBadJobs.py @@ -44,6 +44,7 @@ CREATED_MINUTES_AGO = 5 QUEUE_THRESHOLD_DAYS = 14 +RUNNING_THRESHOLD_DAYS = 8 def cancel(record): @@ -65,32 +66,47 @@ def cancel(record): sleep(1) -def cancel_jobs_stuck_in_queue(): - """ - For jobs over 14 days old, cancel them - Update a completed Job as necessary to test this out: - ee2.update_job_status({'job_id': '601af2afeeb773acaf9de80d', 'as_admin': True, 'status': 'queued'}) - :return: - """ - queue_threshold_days = QUEUE_THRESHOLD_DAYS - before_days = ( - datetime.today() - timedelta(days=queue_threshold_days + 1) - ).timestamp() - print({"status": "queued", "queued": {"$lt": before_days}}) +def cancel_jobs_stuck_in_state(threshold_days, state): + before_days = (datetime.today() - timedelta(days=threshold_days + 1)).timestamp() + print({"status": state, state: {"$lt": before_days}}) stuck_jobs = ee2_jobs_collection.find( - {"status": Status.queued.value, "queued": {"$lt": before_days}} + {"status": state, state: {"$lt": before_days}} ) print( - f"Found {stuck_jobs.count()} jobs that were stuck in the {Status.queued.value} state over {queue_threshold_days} days" + f"Found {stuck_jobs.count()} jobs that were stuck in the {state} state over {threshold_days} days" ) for record in stuck_jobs: - queued_time = record["queued"] + queued_time = record[state] now = datetime.now(timezone.utc).timestamp() elapsed = now - queued_time - print("queued days=", elapsed / 86000) + print(f"{state} days=", elapsed / 86000) cancel(record) +def cancel_jobs_stuck_in_running(): + """ + For running jobs over 8 days old, cancel them + Update a completed Job as necessary to test this out: + ee2.update_job_status({'job_id': '601af2afeeb773acaf9de80d', 'as_admin': True, 'status': 'queued'}) + :return: + """ + cancel_jobs_stuck_in_state( + threshold_days=RUNNING_THRESHOLD_DAYS, state=Status.running.value + ) + + +def cancel_jobs_stuck_in_queue(): + """ + For queued jobs over 14 days old, cancel them + Update a completed Job as necessary to test this out: + ee2.update_job_status({'job_id': '601af2afeeb773acaf9de80d', 'as_admin': True, 'status': 'running'}) + :return: + """ + cancel_jobs_stuck_in_state( + threshold_days=QUEUE_THRESHOLD_DAYS, state=Status.queued.value + ) + + def cancel_created(): """ For jobs that are not batch jobs, and have been in the created state for more than 5 minutes, uh oh, spaghettio, time to go @@ -116,6 +132,8 @@ def clean_retried_jobs(): def purge(): cancel_jobs_stuck_in_queue() + # Use this after an outage + # cancel_jobs_stuck_in_running() cancel_created() diff --git a/bin/cron_vars b/bin/cron_vars index d7b9cec77..a90ea97b6 100644 --- a/bin/cron_vars +++ b/bin/cron_vars @@ -1,2 +1,3 @@ EE2_ADMIN_SERVICE_TOKEN=$EE2_ADMIN_SERVICE_TOKEN -KB_DEPLOYMENT_CONFIG=$KB_DEPLOYMENT_CONFIG \ No newline at end of file +KB_DEPLOYMENT_CONFIG=$KB_DEPLOYMENT_CONFIG +PYTHONPATH=/kb/module:/kb/module/lib diff --git a/bin/ee2_cronjobs b/bin/ee2_cronjobs index 036231a15..55524e4e8 100644 --- a/bin/ee2_cronjobs +++ b/bin/ee2_cronjobs @@ -3,4 +3,6 @@ BASH_ENV=/etc/environment # Check the cron-purge.log for issues why the script isn't running, such as missing `EE2_ADMIN_SERVICE_TOKEN` # m h dom mon dow user command - * * * * * root . /etc/environment; /miniconda-latest/bin/python3 /kb/module/bin/PurgeBadJobs.py >> /root/cron-purge.log 2>&1 + * * * * * root . /etc/environment; /miniconda-latest/bin/python3 /kb/module/bin/PurgeBadJobs.py >> /root/cron-purge.log 2>&1 + + diff --git a/deploy.cfg b/deploy.cfg index 9618cb902..72062fd35 100644 --- a/deploy.cfg +++ b/deploy.cfg @@ -51,13 +51,13 @@ request_memory = 23000M request_disk = 100GB [njs] -request_cpus = {{ default .Env.njs_default_cores "24" }} -request_memory = 23000M +request_cpus = {{ default .Env.njs_default_cores "12" }} +request_memory = 10000M request_disk = 100GB [bigmem] -request_cpus = {{ default .Env.bigmem_default_cores "32" }} -request_memory = 204800M +request_cpus = {{ default .Env.bigmem_default_cores "16" }} +request_memory = 250000M request_disk = 100GB [bigmemlong] @@ -81,8 +81,8 @@ request_memory = 204800M request_disk = 100GB [kb_upload] -request_cpus = {{ default .Env.kb_upload_default_cores "24" }} -request_memory = 4500M +request_cpus = {{ default .Env.kb_upload_default_cores "12" }} +request_memory = 1000M request_disk = 50GB [kb_upload_bulk] diff --git a/docker-compose.yml b/docker-compose.yml index 5eeba3d70..46896e28c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ services: - "-stdout" - "/kb/deployment/jettybase/logs/request.log" - "./scripts/entrypoint.sh" - entrypoint: [ "/kb/deployment/bin/dockerize" ] + entrypoint: [ "dockerize" ] depends_on: ["mongodb","condor"] environment: - POOL_PASSWORD=weakpassword @@ -42,7 +42,7 @@ services: - "-stdout" - "/kb/deployment/jettybase/logs/request.log" - "./scripts/entrypoint.sh" - entrypoint: [ "/kb/deployment/bin/dockerize" ] + entrypoint: [ "dockerize" ] depends_on: ["mongodb","condor"] environment: - POOL_PASSWORD=weakpassword diff --git a/requirements-dev.txt b/requirements-dev.txt index 06e45c8e1..a6676536b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,10 @@ +# +# These requirements were autogenerated by pipenv +# To regenerate from the project's Pipfile, run: +# +# pipenv lock --requirements +# + -i https://pypi.org/simple/ aiofiles==0.4.0 aiohttp==3.7.4 @@ -37,7 +44,7 @@ mock==3.0.5 mongoengine==0.23.0 multidict==4.5.2 nose==1.3.7 -packaging==20.9 +packaging==21.3; python_version >= '3.6' pluggy==0.13.1 psutil==5.6.6 py==1.10.0 @@ -59,12 +66,13 @@ rfc3986==1.3.2 ruamel.yaml==0.15.87 sanic==19.6.0 sentry-sdk==0.14.3 +setuptools==62.3.2; python_version >= '3.7' six==1.14.0 slackclient==2.7.1 toml==0.10.1 tqdm==4.42.1 typing-extensions==3.7.4.3 -ujson==1.35 +ujson==5.2.0 urllib3==1.25.8 uvloop==0.12.2 websocket-client==0.57.0 diff --git a/test/dockerfiles/condor/requirements.txt b/test/dockerfiles/condor/requirements.txt index a7f499bb6..b9d0de8fa 100644 --- a/test/dockerfiles/condor/requirements.txt +++ b/test/dockerfiles/condor/requirements.txt @@ -16,8 +16,8 @@ pymongo==3.8.0 requests==2.22.0 requests-async==0.5.0 rfc3986==1.3.2 -sanic==19.6.0 -ujson==1.35 +sanic==20.12.6 +ujson==5.2.0 urllib3==1.26.5 uvloop==0.12.2 websockets==6.0 diff --git a/test/tests_for_sdkmr/job_submission_parameters_test.py b/test/tests_for_sdkmr/job_submission_parameters_test.py index 33ab3e16b..1bd123424 100644 --- a/test/tests_for_sdkmr/job_submission_parameters_test.py +++ b/test/tests_for_sdkmr/job_submission_parameters_test.py @@ -216,37 +216,31 @@ def test_job_req_check_parameters_no_input(): def test_job_req_check_parameters_full_input(): - assert ( - JobRequirements.check_parameters( - 1, - 1, - 1, - " b ", - "x", - " user ", - 890, - {"proc": "x286", "maxmem": "640k"}, - [], - ) - == (1, 1, 1, "b", True, "user", True, {"proc": "x286", "maxmem": "640k"}, False) - ) + assert JobRequirements.check_parameters( + 1, + 1, + 1, + " b ", + "x", + " user ", + 890, + {"proc": "x286", "maxmem": "640k"}, + [], + ) == (1, 1, 1, "b", True, "user", True, {"proc": "x286", "maxmem": "640k"}, False) def test_job_req_check_parameters_whitespace_as_user(): - assert ( - JobRequirements.check_parameters( - 1, - 1, - 1, - " b ", - 0, - " \t ", - 890, - {"proc": "x286", "maxmem": "640k"}, - 1, - ) - == (1, 1, 1, "b", False, None, True, {"proc": "x286", "maxmem": "640k"}, True) - ) + assert JobRequirements.check_parameters( + 1, + 1, + 1, + " b ", + 0, + " \t ", + 890, + {"proc": "x286", "maxmem": "640k"}, + 1, + ) == (1, 1, 1, "b", False, None, True, {"proc": "x286", "maxmem": "640k"}, True) def test_job_req_check_parameters_fail():