From 8c4f7d794c247a99f25c718c2c3f7e7209f8f75a Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 30 Nov 2021 13:25:59 -0600 Subject: [PATCH 01/43] Update PurgeBadJobs.py --- bin/PurgeBadJobs.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/bin/PurgeBadJobs.py b/bin/PurgeBadJobs.py index d2a182c84..0b2144ef7 100644 --- a/bin/PurgeBadJobs.py +++ b/bin/PurgeBadJobs.py @@ -44,6 +44,7 @@ CREATED_MINUTES_AGO = 5 QUEUE_THRESHOLD_DAYS = 14 +RUNNING_THRESHOLD_DAYS = 8 def cancel(record): @@ -65,6 +66,28 @@ def cancel(record): sleep(1) +def cancel_jobs_stuck_in_running(): + """ + Same as cancel_jobs_stuck_in queue except they are in the running state + """ + threshold_days = RUNNING_THRESHOLD_DAYS + before_days = ( + datetime.today() - timedelta(days=queue_threshold_days + 1) + ).timestamp() + print({"status": "queued", "queued": {"$lt": before_days}}) + stuck_jobs = ee2_jobs_collection.find( + {"status": Status.queued.value, "queued": {"$lt": before_days}} + ) + print( + f"Found {stuck_jobs.count()} jobs that were stuck in the {Status.queued.value} state over {queue_threshold_days} days" + ) + for record in stuck_jobs: + queued_time = record["queued"] + now = datetime.now(timezone.utc).timestamp() + elapsed = now - queued_time + print("queued days=", elapsed / 86000) + cancel(record) + def cancel_jobs_stuck_in_queue(): """ For jobs over 14 days old, cancel them From 6c0bd87846308aae790c703ff6b6e27e1f6bbaf8 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 30 Nov 2021 13:50:40 -0600 Subject: [PATCH 02/43] Fix running jobs --- bin/PurgeBadJobs.py | 57 +++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/bin/PurgeBadJobs.py b/bin/PurgeBadJobs.py index 0b2144ef7..b1b1758f3 100644 --- a/bin/PurgeBadJobs.py +++ b/bin/PurgeBadJobs.py @@ -66,52 +66,45 @@ def cancel(record): sleep(1) -def cancel_jobs_stuck_in_running(): - """ - Same as cancel_jobs_stuck_in queue except they are in the running state - """ - threshold_days = RUNNING_THRESHOLD_DAYS - before_days = ( - datetime.today() - timedelta(days=queue_threshold_days + 1) - ).timestamp() - print({"status": "queued", "queued": {"$lt": before_days}}) +def cancel_jobs_stuck_in_state(threshold_days, state): + before_days = (datetime.today() - timedelta(days=threshold_days + 1)).timestamp() + print({"status": state, state: {"$lt": before_days}}) stuck_jobs = ee2_jobs_collection.find( - {"status": Status.queued.value, "queued": {"$lt": before_days}} + {"status": state, state: {"$lt": before_days}} ) print( - f"Found {stuck_jobs.count()} jobs that were stuck in the {Status.queued.value} state over {queue_threshold_days} days" + f"Found {stuck_jobs.count()} jobs that were stuck in the {state} state over {threshold_days} days" ) for record in stuck_jobs: - queued_time = record["queued"] + queued_time = record[state] now = datetime.now(timezone.utc).timestamp() elapsed = now - queued_time - print("queued days=", elapsed / 86000) + print(f"{state} days=", elapsed / 86000) cancel(record) - -def cancel_jobs_stuck_in_queue(): + + +def cancel_jobs_stuck_in_running(): """ - For jobs over 14 days old, cancel them + For running jobs over 8 days old, cancel them Update a completed Job as necessary to test this out: ee2.update_job_status({'job_id': '601af2afeeb773acaf9de80d', 'as_admin': True, 'status': 'queued'}) :return: """ - queue_threshold_days = QUEUE_THRESHOLD_DAYS - before_days = ( - datetime.today() - timedelta(days=queue_threshold_days + 1) - ).timestamp() - print({"status": "queued", "queued": {"$lt": before_days}}) - stuck_jobs = ee2_jobs_collection.find( - {"status": Status.queued.value, "queued": {"$lt": before_days}} + cancel_jobs_stuck_in_state( + threshold_days=RUNNING_THRESHOLD_DAYS, state=Status.running.value ) - print( - f"Found {stuck_jobs.count()} jobs that were stuck in the {Status.queued.value} state over {queue_threshold_days} days" + + +def cancel_jobs_stuck_in_queue(): + """ + For queued jobs over 14 days old, cancel them + Update a completed Job as necessary to test this out: + ee2.update_job_status({'job_id': '601af2afeeb773acaf9de80d', 'as_admin': True, 'status': 'running'}) + :return: + """ + cancel_jobs_stuck_in_state( + threshold_days=QUEUE_THRESHOLD_DAYS, state=Status.queued.value ) - for record in stuck_jobs: - queued_time = record["queued"] - now = datetime.now(timezone.utc).timestamp() - elapsed = now - queued_time - print("queued days=", elapsed / 86000) - cancel(record) def cancel_created(): @@ -139,6 +132,8 @@ def clean_retried_jobs(): def purge(): cancel_jobs_stuck_in_queue() + # Use this after an outage + # cancel_jobs_stuck_in_running() cancel_created() From 76cf1ffc900f68cff10ddf9cb8d3b7ac4445b50b Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 10 Dec 2021 10:58:15 -0600 Subject: [PATCH 03/43] Update ee2_cronjobs --- bin/ee2_cronjobs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/ee2_cronjobs b/bin/ee2_cronjobs index 036231a15..1368ea6c6 100644 --- a/bin/ee2_cronjobs +++ b/bin/ee2_cronjobs @@ -3,4 +3,6 @@ BASH_ENV=/etc/environment # Check the cron-purge.log for issues why the script isn't running, such as missing `EE2_ADMIN_SERVICE_TOKEN` # m h dom mon dow user command - * * * * * root . /etc/environment; /miniconda-latest/bin/python3 /kb/module/bin/PurgeBadJobs.py >> /root/cron-purge.log 2>&1 + * * * * * root PYTHONPATH=/kb/module:/kb/module/lib . /etc/environment; /miniconda-latest/bin/python3 /kb/module/bin/PurgeBadJobs.py >> /root/cron-purge.log 2>&1 + + From cac11cfcb5c4fa55a9a11f4b8685f98d5f799535 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 10 Dec 2021 11:02:57 -0600 Subject: [PATCH 04/43] Update ee2_cronjobs --- bin/ee2_cronjobs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ee2_cronjobs b/bin/ee2_cronjobs index 1368ea6c6..55524e4e8 100644 --- a/bin/ee2_cronjobs +++ b/bin/ee2_cronjobs @@ -3,6 +3,6 @@ BASH_ENV=/etc/environment # Check the cron-purge.log for issues why the script isn't running, such as missing `EE2_ADMIN_SERVICE_TOKEN` # m h dom mon dow user command - * * * * * root PYTHONPATH=/kb/module:/kb/module/lib . /etc/environment; /miniconda-latest/bin/python3 /kb/module/bin/PurgeBadJobs.py >> /root/cron-purge.log 2>&1 + * * * * * root . /etc/environment; /miniconda-latest/bin/python3 /kb/module/bin/PurgeBadJobs.py >> /root/cron-purge.log 2>&1 From 84af6694cc8b78054df926f65bf69194f2f247cc Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 10 Dec 2021 11:03:10 -0600 Subject: [PATCH 05/43] Update cron_vars --- bin/cron_vars | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/cron_vars b/bin/cron_vars index d7b9cec77..a90ea97b6 100644 --- a/bin/cron_vars +++ b/bin/cron_vars @@ -1,2 +1,3 @@ EE2_ADMIN_SERVICE_TOKEN=$EE2_ADMIN_SERVICE_TOKEN -KB_DEPLOYMENT_CONFIG=$KB_DEPLOYMENT_CONFIG \ No newline at end of file +KB_DEPLOYMENT_CONFIG=$KB_DEPLOYMENT_CONFIG +PYTHONPATH=/kb/module:/kb/module/lib From 4c42129c0f61e94d7cda1acbb9048cb8c729b1f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Feb 2022 14:49:18 +0000 Subject: [PATCH 06/43] Bump sanic from 19.6.0 to 20.12.6 in /test/dockerfiles/condor Bumps [sanic](https://github.com/sanic-org/sanic) from 19.6.0 to 20.12.6. - [Release notes](https://github.com/sanic-org/sanic/releases) - [Changelog](https://github.com/sanic-org/sanic/blob/main/CHANGELOG.rst) - [Commits](https://github.com/sanic-org/sanic/compare/v19.6.0...v20.12.6) --- updated-dependencies: - dependency-name: sanic dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- test/dockerfiles/condor/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/dockerfiles/condor/requirements.txt b/test/dockerfiles/condor/requirements.txt index a7f499bb6..9cc91a5b8 100644 --- a/test/dockerfiles/condor/requirements.txt +++ b/test/dockerfiles/condor/requirements.txt @@ -16,7 +16,7 @@ pymongo==3.8.0 requests==2.22.0 requests-async==0.5.0 rfc3986==1.3.2 -sanic==19.6.0 +sanic==20.12.6 ujson==1.35 urllib3==1.26.5 uvloop==0.12.2 From 12222cd1708c26ef8982d35824de47a97871a38a Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 22 Feb 2022 16:17:58 -0600 Subject: [PATCH 07/43] Create build_on_tag.yaml --- .github/workflows/build_on_tag.yaml | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/build_on_tag.yaml diff --git a/.github/workflows/build_on_tag.yaml b/.github/workflows/build_on_tag.yaml new file mode 100644 index 000000000..04bef1581 --- /dev/null +++ b/.github/workflows/build_on_tag.yaml @@ -0,0 +1,31 @@ +name: Build Tag +on: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + branches: + - main + - master + +jobs: + main: + runs-on: ubuntu-20.04 + steps: + - + name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + - + name: Build and push this feature branch + id: docker_build + uses: docker/build-push-action@v2 + with: + push: true + tags: ghcr.io/${{ github.repository }}:${{ github.head_ref }} + + - + name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} From 3ec91af0f5f9232c9804d8c1cabf4b17d93fefb4 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 22 Feb 2022 16:34:05 -0600 Subject: [PATCH 08/43] Update build_on_tag.yaml --- .github/workflows/build_on_tag.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_on_tag.yaml b/.github/workflows/build_on_tag.yaml index 04bef1581..95a8cb245 100644 --- a/.github/workflows/build_on_tag.yaml +++ b/.github/workflows/build_on_tag.yaml @@ -24,7 +24,7 @@ jobs: uses: docker/build-push-action@v2 with: push: true - tags: ghcr.io/${{ github.repository }}:${{ github.head_ref }} + tags: ghcr.io/${{ github.repository }}:${{ GITHUB_REF_NAME }} - name: Image digest From 25d16151a3e0751abe83363a225e444bfbb2bd6f Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 22 Feb 2022 16:44:47 -0600 Subject: [PATCH 09/43] Ran with latest black --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d3cb732c7..6cbe3988b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/ambv/black - rev: 21.5b0 + rev: 22.1.0 hooks: - id: black exclude: '.+Impl.py' From e02b79887916ea60ed47cb359b10f56ac2232273 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 22 Feb 2022 16:45:25 -0600 Subject: [PATCH 10/43] Ran with latest black --- .pre-commit-config.yaml | 2 +- .../job_submission_parameters_test.py | 50 ++++++++----------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6cbe3988b..4556abb64 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,6 +5,6 @@ repos: - id: black exclude: '.+Impl.py' - repo: https://gitlab.com/pycqa/flake8 - rev: '' + rev: '3.9.2' hooks: - id: flake8 diff --git a/test/tests_for_sdkmr/job_submission_parameters_test.py b/test/tests_for_sdkmr/job_submission_parameters_test.py index 33ab3e16b..1bd123424 100644 --- a/test/tests_for_sdkmr/job_submission_parameters_test.py +++ b/test/tests_for_sdkmr/job_submission_parameters_test.py @@ -216,37 +216,31 @@ def test_job_req_check_parameters_no_input(): def test_job_req_check_parameters_full_input(): - assert ( - JobRequirements.check_parameters( - 1, - 1, - 1, - " b ", - "x", - " user ", - 890, - {"proc": "x286", "maxmem": "640k"}, - [], - ) - == (1, 1, 1, "b", True, "user", True, {"proc": "x286", "maxmem": "640k"}, False) - ) + assert JobRequirements.check_parameters( + 1, + 1, + 1, + " b ", + "x", + " user ", + 890, + {"proc": "x286", "maxmem": "640k"}, + [], + ) == (1, 1, 1, "b", True, "user", True, {"proc": "x286", "maxmem": "640k"}, False) def test_job_req_check_parameters_whitespace_as_user(): - assert ( - JobRequirements.check_parameters( - 1, - 1, - 1, - " b ", - 0, - " \t ", - 890, - {"proc": "x286", "maxmem": "640k"}, - 1, - ) - == (1, 1, 1, "b", False, None, True, {"proc": "x286", "maxmem": "640k"}, True) - ) + assert JobRequirements.check_parameters( + 1, + 1, + 1, + " b ", + 0, + " \t ", + 890, + {"proc": "x286", "maxmem": "640k"}, + 1, + ) == (1, 1, 1, "b", False, None, True, {"proc": "x286", "maxmem": "640k"}, True) def test_job_req_check_parameters_fail(): From f205d4062284eb2c61959e2ed4cb42936f2ded97 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 22 Feb 2022 16:47:07 -0600 Subject: [PATCH 11/43] Update build_on_tag.yaml --- .github/workflows/build_on_tag.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_on_tag.yaml b/.github/workflows/build_on_tag.yaml index 95a8cb245..530226036 100644 --- a/.github/workflows/build_on_tag.yaml +++ b/.github/workflows/build_on_tag.yaml @@ -19,7 +19,7 @@ jobs: username: ${{ secrets.GHCR_USERNAME }} password: ${{ secrets.GHCR_TOKEN }} - - name: Build and push this feature branch + name: Build and push this tag from the main branch id: docker_build uses: docker/build-push-action@v2 with: From 1e7e7993b231eadb551d91182f221ca9f7d0c980 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 1 Mar 2022 11:01:03 -0600 Subject: [PATCH 12/43] Update build_on_tag.yaml --- .github/workflows/build_on_tag.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_on_tag.yaml b/.github/workflows/build_on_tag.yaml index 530226036..7f1f64abb 100644 --- a/.github/workflows/build_on_tag.yaml +++ b/.github/workflows/build_on_tag.yaml @@ -6,6 +6,7 @@ on: branches: - main - master + - develop jobs: main: From e041645e52f8cd0efb71d482f780d1971f8d4ad2 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 18 Mar 2022 12:06:53 -0500 Subject: [PATCH 13/43] Update Dockerfile --- Dockerfile | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4939fde50..49e8995a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM quay.io/kbase/sdkbase2:python MAINTAINER KBase Developer -RUN apt-get clean all && apt-get update --fix-missing -y +RUN apt-get clean all && apt-get update --fix-missing -y && apt-get upgrade # ----------------------------------------- # In this section, you can install any system dependencies required @@ -11,12 +11,8 @@ RUN apt-get clean all && apt-get update --fix-missing -y RUN apt-get install -y gcc wget vim htop tmpreaper RUN mkdir -p /etc/apt/sources.list.d - -RUN DEBIAN_FRONTEND=noninteractive wget -qO - https://research.cs.wisc.edu/htcondor/debian/HTCondor-Release.gpg.key | apt-key add - \ - && echo "deb http://research.cs.wisc.edu/htcondor/debian/8.8/stretch stretch contrib" >> /etc/apt/sources.list \ - && echo "deb-src http://research.cs.wisc.edu/htcondor/debian/8.8/stretch stretch contrib" >> /etc/apt/sources.list \ - && apt-get update -y \ - && apt-get install -y condor +# Install condor +RUN curl -fsSL https://get.htcondor.org | sudo /bin/bash -s -- --no-dry-run # install jars # perhaps we should have test and prod dockerfiles to avoid jars and mongo installs in prod From 7e228bae6609137e1f6a8ea04f92ba489153bbf9 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 18 Mar 2022 12:10:54 -0500 Subject: [PATCH 14/43] dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 49e8995a8..16980f505 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM quay.io/kbase/sdkbase2:python MAINTAINER KBase Developer -RUN apt-get clean all && apt-get update --fix-missing -y && apt-get upgrade +RUN apt-get clean all && apt-get update --fix-missing -y && apt-get upgrade -y # ----------------------------------------- # In this section, you can install any system dependencies required From 0b5b836fc3a366d12908902b310230fc317b1e3a Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 18 Mar 2022 12:53:34 -0500 Subject: [PATCH 15/43] Update base image and htcondor to latest --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 16980f505..4a4cd5040 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/kbase/sdkbase2:python +FROM kbase/sdkpython:3.8.0 MAINTAINER KBase Developer RUN apt-get clean all && apt-get update --fix-missing -y && apt-get upgrade -y @@ -12,7 +12,7 @@ RUN apt-get install -y gcc wget vim htop tmpreaper RUN mkdir -p /etc/apt/sources.list.d # Install condor -RUN curl -fsSL https://get.htcondor.org | sudo /bin/bash -s -- --no-dry-run +RUN curl -fsSL https://get.htcondor.org | /bin/bash -s -- --no-dry-run # install jars # perhaps we should have test and prod dockerfiles to avoid jars and mongo installs in prod From badba4b0eacafae3e4f94e7544bff7a55532557b Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 18 Mar 2022 13:12:30 -0500 Subject: [PATCH 16/43] Update base image and htcondor to latest --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 4a4cd5040..fd4b051ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,12 @@ RUN cd /opt \ # Remove due to cve-2021-4104 issue in spin (log4j) RUN rm /opt/jars/lib/jars/dockerjava/docker-java-shaded-3.0.14.jar +# Install DOCKERIZE +RUN curl -o /tmp/dockerize.tgz https://raw.githubusercontent.com/kbase/dockerize/dist/dockerize-linux-amd64-v0.5.0.tar.gz && \ + cd /usr/bin && \ + tar xvzf /tmp/dockerize.tgz && \ + rm /tmp/dockerize.tgz + # install mongodb RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5 \ From b34486880371f18649bf0d5bb4867e7bd6613cc6 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 18 Mar 2022 13:39:56 -0500 Subject: [PATCH 17/43] Update base image and htcondor to latest --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5eeba3d70..46896e28c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ services: - "-stdout" - "/kb/deployment/jettybase/logs/request.log" - "./scripts/entrypoint.sh" - entrypoint: [ "/kb/deployment/bin/dockerize" ] + entrypoint: [ "dockerize" ] depends_on: ["mongodb","condor"] environment: - POOL_PASSWORD=weakpassword @@ -42,7 +42,7 @@ services: - "-stdout" - "/kb/deployment/jettybase/logs/request.log" - "./scripts/entrypoint.sh" - entrypoint: [ "/kb/deployment/bin/dockerize" ] + entrypoint: [ "dockerize" ] depends_on: ["mongodb","condor"] environment: - POOL_PASSWORD=weakpassword From d51d1385f16c0b2b99b1de8aac632d11b50d31d1 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 18 Mar 2022 14:51:33 -0500 Subject: [PATCH 18/43] Update RELEASE_NOTES.md --- RELEASE_NOTES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 62948402e..4a96200f5 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,9 @@ # execution_engine2 (ee2) release notes ========================================= +## 0.0.81 +* Updated HTCondor Clients, New Base Image + ## 0.0.8 * Fixed a bug that could, seemingly rarely, cause job and log updates to be applied to the wrong Mongo collection. From f30a402d99808288dd3a9e6f0cc025313ff39072 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 18 Mar 2022 14:57:30 -0500 Subject: [PATCH 19/43] Update RELEASE_NOTES.md --- RELEASE_NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 4a96200f5..102b1617a 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -3,6 +3,7 @@ ## 0.0.81 * Updated HTCondor Clients, New Base Image +* Use default GH actions ## 0.0.8 * Fixed a bug that could, seemingly rarely, cause job and log updates to be applied to the From 8b904fa896184936b91401b2e5f6ae6055f2ea98 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 18 Mar 2022 14:59:45 -0500 Subject: [PATCH 20/43] updated gha --- .github/workflows/build_feature_branch.yml | 26 ----- .github/workflows/build_on_push.yaml | 31 ------ .github/workflows/build_on_tag.yaml | 32 ------ .github/workflows/build_prodrc_pr.yaml | 31 ++++++ .github/workflows/build_test_pr.yaml | 27 +++++ .github/workflows/non_sdk_test.yml | 103 ++++++++++++++++++ .github/workflows/prod_release.yaml | 38 +++++++ .github/workflows/scripts/build_prodrc_pr.sh | 17 +++ .github/workflows/scripts/build_test_pr.sh | 17 +++ .github/workflows/scripts/deploy_tag.sh | 34 ++++++ .github/workflows/scripts/prod_release.sh | 24 ++++ .github/workflows/scripts/tag_environments.sh | 22 ++++ .github/workflows/scripts/tag_prod_latest.sh | 12 ++ .github/workflows/scripts/tag_test_latest.sh | 12 ++ .github/workflows/tag_environments.yaml | 19 ++++ .github/workflows/tag_prod_latest.yaml | 27 +++++ .github/workflows/tag_test_latest.yaml | 26 +++++ 17 files changed, 409 insertions(+), 89 deletions(-) delete mode 100644 .github/workflows/build_feature_branch.yml delete mode 100644 .github/workflows/build_on_push.yaml delete mode 100644 .github/workflows/build_on_tag.yaml create mode 100644 .github/workflows/build_prodrc_pr.yaml create mode 100644 .github/workflows/build_test_pr.yaml create mode 100644 .github/workflows/non_sdk_test.yml create mode 100644 .github/workflows/prod_release.yaml create mode 100755 .github/workflows/scripts/build_prodrc_pr.sh create mode 100755 .github/workflows/scripts/build_test_pr.sh create mode 100755 .github/workflows/scripts/deploy_tag.sh create mode 100755 .github/workflows/scripts/prod_release.sh create mode 100755 .github/workflows/scripts/tag_environments.sh create mode 100755 .github/workflows/scripts/tag_prod_latest.sh create mode 100755 .github/workflows/scripts/tag_test_latest.sh create mode 100644 .github/workflows/tag_environments.yaml create mode 100644 .github/workflows/tag_prod_latest.yaml create mode 100644 .github/workflows/tag_test_latest.yaml diff --git a/.github/workflows/build_feature_branch.yml b/.github/workflows/build_feature_branch.yml deleted file mode 100644 index b62fc1c89..000000000 --- a/.github/workflows/build_feature_branch.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Build Feature Branches - -on: [pull_request] - -jobs: - main: - runs-on: ubuntu-20.04 - steps: - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ secrets.GHCR_USERNAME }} - password: ${{ secrets.GHCR_TOKEN }} - - - name: Build and push this feature branch - id: docker_build - uses: docker/build-push-action@v2 - with: - push: true - tags: ghcr.io/${{ github.repository }}:${{ github.head_ref }} - - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/build_on_push.yaml b/.github/workflows/build_on_push.yaml deleted file mode 100644 index d610acfae..000000000 --- a/.github/workflows/build_on_push.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: Build Main/Develop Branches on push - -on: - push: - branches: - - main - - master - - develop - -jobs: - main: - runs-on: ubuntu-20.04 - steps: - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ secrets.GHCR_USERNAME }} - password: ${{ secrets.GHCR_TOKEN }} - - - name: Build and push the main branch - id: docker_build - uses: docker/build-push-action@v2 - with: - push: true - tags: ghcr.io/${{ github.repository }}:${ GITHUB_REF##*/ } - - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/build_on_tag.yaml b/.github/workflows/build_on_tag.yaml deleted file mode 100644 index 7f1f64abb..000000000 --- a/.github/workflows/build_on_tag.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: Build Tag -on: - push: - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - branches: - - main - - master - - develop - -jobs: - main: - runs-on: ubuntu-20.04 - steps: - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ secrets.GHCR_USERNAME }} - password: ${{ secrets.GHCR_TOKEN }} - - - name: Build and push this tag from the main branch - id: docker_build - uses: docker/build-push-action@v2 - with: - push: true - tags: ghcr.io/${{ github.repository }}:${{ GITHUB_REF_NAME }} - - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/build_prodrc_pr.yaml b/.github/workflows/build_prodrc_pr.yaml new file mode 100644 index 000000000..2e5034e7c --- /dev/null +++ b/.github/workflows/build_prodrc_pr.yaml @@ -0,0 +1,31 @@ +--- +name: Build Prod RC Image +'on': + pull_request: + branches: + - master + - main + types: + - opened + - synchronize + - ready_for_review +jobs: + docker_build: + runs-on: ubuntu-latest + steps: + - name: Verify merge is develop -> main + if: github.head_ref != 'develop' + run: echo "Must merge from develop -> main/master"; exit 1 + - name: Check out GitHub Repo + if: github.event.pull_request.draft == false && github.head_ref == 'develop' + with: + ref: "${{ github.event.pull_request.head.sha }}" + uses: actions/checkout@v2 + - name: Build and Push to Packages + if: github.event.pull_request.draft == false && github.head_ref == 'develop' + env: + PR: "${{ github.event.pull_request.number }}" + SHA: "${{ github.event.pull_request.head.sha }}" + DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" + DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" + run: "./.github/workflows/scripts/build_prodrc_pr.sh\n" diff --git a/.github/workflows/build_test_pr.yaml b/.github/workflows/build_test_pr.yaml new file mode 100644 index 000000000..b6b53286f --- /dev/null +++ b/.github/workflows/build_test_pr.yaml @@ -0,0 +1,27 @@ +--- +name: Build Test Image +'on': + pull_request: + branches: + - develop + types: + - opened + - synchronize + - ready_for_review +jobs: + docker_build: + runs-on: ubuntu-latest + steps: + - name: Check out GitHub Repo + if: github.event.pull_request.draft == false + with: + ref: "${{ github.event.pull_request.head.sha }}" + uses: actions/checkout@v2 + - name: Build and Push to Packages + if: github.event.pull_request.draft == false + env: + PR: "${{ github.event.pull_request.number }}" + SHA: "${{ github.event.pull_request.head.sha }}" + DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" + DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" + run: "./.github/workflows/scripts/build_test_pr.sh\n" diff --git a/.github/workflows/non_sdk_test.yml b/.github/workflows/non_sdk_test.yml new file mode 100644 index 000000000..e4fb6a906 --- /dev/null +++ b/.github/workflows/non_sdk_test.yml @@ -0,0 +1,103 @@ +name: non-sdk tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + workflow_dispatch: + +jobs: + non_sdk_tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7] + steps: + + - name: Check out GitHub repo + if: "!contains(github.event.head_commit.message, 'skip ci')" + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Set up test environment + if: "!contains(github.event.head_commit.message, 'skip ci')" + shell: bash + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + MONGODB_VER: mongodb-linux-x86_64-3.6.16 + ARANGODB_VER: 3.5.1 + ARANGODB_V: 35 + KAFKA_VER: 2.8.1 + SCALA_VER: 2.12 + run: | + # upgrade and update + apt update && apt upgrade + # Download necessary components for testing + # mongo is only a test dependency, no point in matrixing it + cd .. + wget http://fastdl.mongodb.org/linux/$MONGODB_VER.tgz + # file may be gzipped over transfer + gzip -d $MONGODB_VER.tgz + tar xf $MONGODB_VER.tar + export MONGOD=`pwd`/$MONGODB_VER/bin/mongod + cd - + echo "Done with Mongo" + + # arango + cd .. + curl -O https://download.arangodb.com/arangodb$ARANGODB_V/Community/Linux/arangodb3-linux-$ARANGODB_VER.tar.gz + tar -xf arangodb3-linux-$ARANGODB_VER.tar.gz + export ARANGO_EXE=$(pwd)/arangodb3-$ARANGODB_VER/usr/sbin/arangod + export ARANGO_JS=$(pwd)/arangodb3-$ARANGODB_VER/usr/share/arangodb3/js/ + cd - + echo "Done with Arango" + + # kafka + cd .. + curl -O http://mirror.metrocast.net/apache/kafka/$KAFKA_VER/kafka_$SCALA_VER-$KAFKA_VER.tgz + tar -xzf kafka_$SCALA_VER-$KAFKA_VER.tgz + export KAFKA_BIN_DIR=$(pwd)/kafka_$SCALA_VER-$KAFKA_VER/bin + cd - + echo "Done with Kafka" + + # jars + cd .. + git clone https://github.com/kbase/jars + export JARS=$(pwd)/jars/lib/jars + cd - + echo "Done with Jars" + + # copy to test.cfg file + cd test + cp test.cfg.example test.cfg + sed -i "s#^test.jars.dir=.*#test.jars.dir=$JARS#" test.cfg + sed -i "s#^test.temp.dir=.*#test.temp.dir=temp_test_dir#" test.cfg + sed -i "s#^test.arango.exe.*#test.arango.exe=$ARANGO_EXE#" test.cfg + sed -i "s#^test.arango.js.*#test.arango.js=$ARANGO_JS#" test.cfg + sed -i "s#^test.mongo.exe.*#test.mongo.exe=$MONGOD#" test.cfg + sed -i "s#^test.kafka.bin.dir.*#test.kafka.bin.dir=$KAFKA_BIN_DIR#" test.cfg + sed -i "s#^test.mongo.wired_tiger.*#test.mongo.wired_tiger=true#" test.cfg + cat test.cfg + cd - + + - name: Install Python dependencies + shell: bash + run: | + pip install --upgrade pip + pip install -r requirements.txt + + - name: Run tests + shell: bash + run: | + echo "Create fake deployment config" + echo "[SampleService]" > service.cfg + echo "srv_wiz_url = https://ci.kbase.us/services/service_wizard" >> service.cfg + export KB_DEPLOYMENT_CONFIG=`pwd`/service.cfg + pipenv install --dev + pipenv run make test-sdkless + diff --git a/.github/workflows/prod_release.yaml b/.github/workflows/prod_release.yaml new file mode 100644 index 000000000..ffa145332 --- /dev/null +++ b/.github/workflows/prod_release.yaml @@ -0,0 +1,38 @@ +--- +name: Publish Release Image +'on': + release: + branches: + - main + - master + types: + - published +jobs: + docker_build: + runs-on: ubuntu-latest + steps: + - name: Check Tag + id: check-tag + run: |- + if [[ ${{ github.ref_name }} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo ::set-output name=match::true + fi + - name: Report SemVer Check + if: steps.check-tag.outputs.match != 'true' + run: echo "Release version must follow semantic naming (e.g. 1.0.2)"; exit 1 + - name: Check Source Branch + if: github.event.release.target_commitish != 'master' && github.event.release.target_commitish != 'main' + run: echo "Releases must be built from master/main branch"; exit 1 + - name: Check out GitHub Repo + with: + ref: "${{ github.event.pull_request.head.sha }}" + uses: actions/checkout@v2 + - name: Build and Push to Packages + env: + ISH: "${{ github.event.release.target_commitish }}" + PR: "${{ github.event.pull_request.number }}" + SHA: "${{ github.event.pull_request.head.sha }}" + VER: "${{ github.event.release.tag_name }}" + DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" + DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" + run: "./.github/workflows/scripts/prod_release.sh\n" diff --git a/.github/workflows/scripts/build_prodrc_pr.sh b/.github/workflows/scripts/build_prodrc_pr.sh new file mode 100755 index 000000000..4c7bdf277 --- /dev/null +++ b/.github/workflows/scripts/build_prodrc_pr.sh @@ -0,0 +1,17 @@ +#! /usr/bin/env bash + +export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') +export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}') +export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export COMMIT=$(echo "$SHA" | cut -c -7) + +echo "Branch is:" ${GITHUB_HEAD_REF} +docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io +docker build --build-arg BUILD_DATE="$DATE" \ + --build-arg COMMIT="$COMMIT" \ + --build-arg BRANCH="$GITHUB_HEAD_REF" \ + --build-arg PULL_REQUEST="$PR" \ + --label us.kbase.vcs-pull-req="$PR" \ + -t ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" . +docker push ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" diff --git a/.github/workflows/scripts/build_test_pr.sh b/.github/workflows/scripts/build_test_pr.sh new file mode 100755 index 000000000..546b1b422 --- /dev/null +++ b/.github/workflows/scripts/build_test_pr.sh @@ -0,0 +1,17 @@ +#! /usr/bin/env bash + +export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') +export MY_APP=$(echo $(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')"-develop") +export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export COMMIT=$(echo "$SHA" | cut -c -7) + +echo $DOCKER_TOKEN | docker login ghcr.io -u $DOCKER_ACTOR --password-stdin +docker build --build-arg BUILD_DATE="$DATE" \ + --build-arg COMMIT="$COMMIT" \ + --build-arg BRANCH="$GITHUB_HEAD_REF" \ + --build-arg PULL_REQUEST="$PR" \ + --label us.kbase.vcs-pull-req="$PR" \ + -t ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" . +docker push ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" + \ No newline at end of file diff --git a/.github/workflows/scripts/deploy_tag.sh b/.github/workflows/scripts/deploy_tag.sh new file mode 100755 index 000000000..5fb928ab4 --- /dev/null +++ b/.github/workflows/scripts/deploy_tag.sh @@ -0,0 +1,34 @@ +#! /usr/bin/env bash + +# Usage: ./deploy_tag.sh -e TARGET -o ORG -r REPO -s DEV_PROD -t IMAGE_TAG +# +# Example 1: ./deploy_tag.sh -o "kbase" -r "narrative-traefiker" -s "dev" -t "pr-9001" -e "ci" +# Example 2: ./deploy_tag.sh -o "kbase" -r "narrative" -s "prod" -t "latest" -e "next" +# +# Where: +# -o ORG is the organization (`kbase`, `kbaseapps`, etc.) +# -r REPO is the repository (e.g. `narrative`) +# -s DEV_PROD determines whether to pull the development {APPNAME}-develop or production {APPNAME} image. +# -t IMAGE_TAG is the *current* Docker image tag, typically `pr-#` or `latest` +# -e TARGET is one of: `appdsshev`, `ci`, or `next` +# +# Be sure to set $TOKEN first! +# See: https://docs.github.com/en/packages/getting-started-with-github-container-registry/migrating-to-github-container-registry-for-docker-images#authenticating-with-the-container-registry + + +while getopts e:o:r:s:t: option + do + case "${option}" + in + e) TARGET=${OPTARG};; + o) ORG=${OPTARG};; + r) REPO=${OPTARG};; + s) DEV_PROD=${OPTARG};; + t) IMAGE_TAG=${OPTARG};; + esac +done + +curl -H "Authorization: token $TOKEN" \ + -H 'Accept: application/vnd.github.everest-preview+json' \ + "https://api.github.com/repos/$ORG/$REPO/dispatches" \ + -d '{"event_type":"Tag '"$DEV_PROD"' '"$IMAGE_TAG"' for '"$TARGET"'", "client_payload": {"image_tag": "'"$IMAGE_TAG"'","target": "'"$TARGET"'","dev_prod": "'"$DEV_PROD"'"}}' diff --git a/.github/workflows/scripts/prod_release.sh b/.github/workflows/scripts/prod_release.sh new file mode 100755 index 000000000..46d008c69 --- /dev/null +++ b/.github/workflows/scripts/prod_release.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') +export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}') +export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export COMMIT=$(echo "$SHA" | cut -c -7) + +echo "ISH is:" $ISH +echo "GITHUB_REF is:" $GITHUB_REF +echo "HEAD_REF is:" $GITHUB_HEAD_REF +echo "BASE_REF is:" $GITHUB_BASE_REF +echo "Release is:" $GITHUB_REF_NAME +echo $DOCKER_TOKEN | docker login ghcr.io -u $DOCKER_ACTOR --password-stdin +docker build --build-arg BUILD_DATE="$DATE" \ + --build-arg COMMIT="$COMMIT" \ + --build-arg BRANCH="$GITHUB_HEAD_REF" \ + --build-arg PULL_REQUEST="$PR" \ + --build-arg VERSION="$VER" \ + --label us.kbase.vcs-pull-req="$PR" \ + -t ghcr.io/"$MY_ORG"/"$MY_APP":"$VER" \ + -t ghcr.io/"$MY_ORG"/"$MY_APP":"latest" . +docker push ghcr.io/"$MY_ORG"/"$MY_APP":"$VER" +docker push ghcr.io/"$MY_ORG"/"$MY_APP":"latest" diff --git a/.github/workflows/scripts/tag_environments.sh b/.github/workflows/scripts/tag_environments.sh new file mode 100755 index 000000000..b39732a09 --- /dev/null +++ b/.github/workflows/scripts/tag_environments.sh @@ -0,0 +1,22 @@ + +#! /usr/bin/env bash +# Add vars for PR & environments to yaml, as called from external script + +export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') +export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}') +export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export COMMIT=$(echo "$SHA" | cut -c -7) + +if [ $DEV_PROD = "dev" ] || [ $DEV_PROD = "develop" ] +then + IMAGE=$MY_APP"-develop" +else + IMAGE=$MY_APP +fi + +echo "Dev or Prod:" $DEV_PROD +docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io +docker pull ghcr.io/"$MY_ORG"/"$IMAGE":"$IMAGE_TAG" +docker tag ghcr.io/"$MY_ORG"/"$IMAGE":"$IMAGE_TAG" ghcr.io/"$MY_ORG"/"$IMAGE":"$TARGET" +docker push ghcr.io/"$MY_ORG"/"$IMAGE":"$TARGET" diff --git a/.github/workflows/scripts/tag_prod_latest.sh b/.github/workflows/scripts/tag_prod_latest.sh new file mode 100755 index 000000000..c3c422526 --- /dev/null +++ b/.github/workflows/scripts/tag_prod_latest.sh @@ -0,0 +1,12 @@ +#! /usr/bin/env bash + +export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') +export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}') +export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export COMMIT=$(echo "$SHA" | cut -c -7) + +docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io +docker pull ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" +docker tag ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" ghcr.io/"$MY_ORG"/"$MY_APP":"latest-rc" +docker push ghcr.io/"$MY_ORG"/"$MY_APP":"latest-rc" diff --git a/.github/workflows/scripts/tag_test_latest.sh b/.github/workflows/scripts/tag_test_latest.sh new file mode 100755 index 000000000..c0dc504a0 --- /dev/null +++ b/.github/workflows/scripts/tag_test_latest.sh @@ -0,0 +1,12 @@ +#! /usr/bin/env bash + +export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') +export MY_APP=$(echo $(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')"-develop") +export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +export COMMIT=$(echo "$SHA" | cut -c -7) + +docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io +docker pull ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" +docker tag ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" ghcr.io/"$MY_ORG"/"$MY_APP":"latest" +docker push ghcr.io/"$MY_ORG"/"$MY_APP":"latest" diff --git a/.github/workflows/tag_environments.yaml b/.github/workflows/tag_environments.yaml new file mode 100644 index 000000000..6dba7431f --- /dev/null +++ b/.github/workflows/tag_environments.yaml @@ -0,0 +1,19 @@ +--- +name: Tag Image For Deploy +'on': + repository_dispatch +jobs: + tag_environments: + runs-on: ubuntu-latest + steps: + - name: Check out GitHub Repo + uses: actions/checkout@v2 + - name: Tag Deploy Environments + env: + DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" + DOCKER_TOKEN: ${{ secrets.GHCR_TOKEN }} + IMAGE_TAG: ${{ github.event.client_payload.image_tag }} + SHA: ${{ github.event.pull_request.head.sha }} + TARGET: ${{ github.event.client_payload.target }} + DEV_PROD: ${{ github.event.client_payload.dev_prod }} + run: './.github/workflows/scripts/tag_environments.sh' diff --git a/.github/workflows/tag_prod_latest.yaml b/.github/workflows/tag_prod_latest.yaml new file mode 100644 index 000000000..12b23df0c --- /dev/null +++ b/.github/workflows/tag_prod_latest.yaml @@ -0,0 +1,27 @@ +--- +name: Tag Prod Latest +'on': + pull_request: + branches: + - master + - main + types: + - closed +jobs: + docker_tag: + runs-on: ubuntu-latest + steps: + - name: Check out GitHub Repo + if: github.event_name == 'pull_request' && github.event.action == 'closed' && + github.event.pull_request.merged == true + with: + ref: "${{ github.event.pull_request.head.sha }}" + uses: actions/checkout@v2 + - name: Build and Push to Packages + if: github.event.pull_request.draft == false + env: + PR: "${{ github.event.pull_request.number }}" + SHA: "${{ github.event.pull_request.head.sha }}" + DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" + DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" + run: "./.github/workflows/scripts/tag_prod_latest.sh\n" diff --git a/.github/workflows/tag_test_latest.yaml b/.github/workflows/tag_test_latest.yaml new file mode 100644 index 000000000..d8cac4654 --- /dev/null +++ b/.github/workflows/tag_test_latest.yaml @@ -0,0 +1,26 @@ +--- +name: Tag Latest Test Image +'on': + pull_request: + branches: + - develop + types: + - closed +jobs: + docker_tag: + runs-on: ubuntu-latest + steps: + - name: Check out GitHub Repo + if: github.event_name == 'pull_request' && github.event.action == 'closed' && + github.event.pull_request.merged == true + with: + ref: "${{ github.event.pull_request.head.sha }}" + uses: actions/checkout@v2 + - name: Build and Push to Packages + if: github.event.pull_request.draft == false + env: + PR: "${{ github.event.pull_request.number }}" + SHA: "${{ github.event.pull_request.head.sha }}" + DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" + DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" + run: "./.github/workflows/scripts/tag_test_latest.sh\n" From 5196ea08a63fa1de478f00592432eeb971fcf57c Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 18 Mar 2022 15:22:14 -0500 Subject: [PATCH 21/43] Update RELEASE_NOTES.md --- RELEASE_NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 102b1617a..2b073d7bf 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -4,6 +4,7 @@ ## 0.0.81 * Updated HTCondor Clients, New Base Image * Use default GH actions +* Updated precommit hooks ## 0.0.8 * Fixed a bug that could, seemingly rarely, cause job and log updates to be applied to the From 444d520c4b1ff992e592a16b92f3b1f6f6f62c53 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 18 Mar 2022 15:31:38 -0500 Subject: [PATCH 22/43] Delete non_sdk_test.yml --- .github/workflows/non_sdk_test.yml | 103 ----------------------------- 1 file changed, 103 deletions(-) delete mode 100644 .github/workflows/non_sdk_test.yml diff --git a/.github/workflows/non_sdk_test.yml b/.github/workflows/non_sdk_test.yml deleted file mode 100644 index e4fb6a906..000000000 --- a/.github/workflows/non_sdk_test.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: non-sdk tests - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - workflow_dispatch: - -jobs: - non_sdk_tests: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.7] - steps: - - - name: Check out GitHub repo - if: "!contains(github.event.head_commit.message, 'skip ci')" - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Set up test environment - if: "!contains(github.event.head_commit.message, 'skip ci')" - shell: bash - env: - KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} - MONGODB_VER: mongodb-linux-x86_64-3.6.16 - ARANGODB_VER: 3.5.1 - ARANGODB_V: 35 - KAFKA_VER: 2.8.1 - SCALA_VER: 2.12 - run: | - # upgrade and update - apt update && apt upgrade - # Download necessary components for testing - # mongo is only a test dependency, no point in matrixing it - cd .. - wget http://fastdl.mongodb.org/linux/$MONGODB_VER.tgz - # file may be gzipped over transfer - gzip -d $MONGODB_VER.tgz - tar xf $MONGODB_VER.tar - export MONGOD=`pwd`/$MONGODB_VER/bin/mongod - cd - - echo "Done with Mongo" - - # arango - cd .. - curl -O https://download.arangodb.com/arangodb$ARANGODB_V/Community/Linux/arangodb3-linux-$ARANGODB_VER.tar.gz - tar -xf arangodb3-linux-$ARANGODB_VER.tar.gz - export ARANGO_EXE=$(pwd)/arangodb3-$ARANGODB_VER/usr/sbin/arangod - export ARANGO_JS=$(pwd)/arangodb3-$ARANGODB_VER/usr/share/arangodb3/js/ - cd - - echo "Done with Arango" - - # kafka - cd .. - curl -O http://mirror.metrocast.net/apache/kafka/$KAFKA_VER/kafka_$SCALA_VER-$KAFKA_VER.tgz - tar -xzf kafka_$SCALA_VER-$KAFKA_VER.tgz - export KAFKA_BIN_DIR=$(pwd)/kafka_$SCALA_VER-$KAFKA_VER/bin - cd - - echo "Done with Kafka" - - # jars - cd .. - git clone https://github.com/kbase/jars - export JARS=$(pwd)/jars/lib/jars - cd - - echo "Done with Jars" - - # copy to test.cfg file - cd test - cp test.cfg.example test.cfg - sed -i "s#^test.jars.dir=.*#test.jars.dir=$JARS#" test.cfg - sed -i "s#^test.temp.dir=.*#test.temp.dir=temp_test_dir#" test.cfg - sed -i "s#^test.arango.exe.*#test.arango.exe=$ARANGO_EXE#" test.cfg - sed -i "s#^test.arango.js.*#test.arango.js=$ARANGO_JS#" test.cfg - sed -i "s#^test.mongo.exe.*#test.mongo.exe=$MONGOD#" test.cfg - sed -i "s#^test.kafka.bin.dir.*#test.kafka.bin.dir=$KAFKA_BIN_DIR#" test.cfg - sed -i "s#^test.mongo.wired_tiger.*#test.mongo.wired_tiger=true#" test.cfg - cat test.cfg - cd - - - - name: Install Python dependencies - shell: bash - run: | - pip install --upgrade pip - pip install -r requirements.txt - - - name: Run tests - shell: bash - run: | - echo "Create fake deployment config" - echo "[SampleService]" > service.cfg - echo "srv_wiz_url = https://ci.kbase.us/services/service_wizard" >> service.cfg - export KB_DEPLOYMENT_CONFIG=`pwd`/service.cfg - pipenv install --dev - pipenv run make test-sdkless - From b81da2920c42d95f3078a14407ef2521fed53a8b Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 19 Apr 2022 14:13:52 -0500 Subject: [PATCH 23/43] Create pr_build.yml --- .github/workflows/pr_build.yml | 47 ++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/pr_build.yml diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml new file mode 100644 index 000000000..fb8fb6783 --- /dev/null +++ b/.github/workflows/pr_build.yml @@ -0,0 +1,47 @@ +--- +name: Build & Tag Image on PR +'on': + pull_request: + branches: + - develop + - main + - master + types: + - closed + - edited + - opened + - ready_for_review + - reopened + - synchronize + - workflow_dispatch +jobs: + docker_build: + runs-on: ubuntu-latest + steps: + - name: Check Out GitHub Repo + if: github.event.pull_request.draft == false + with: + ref: "${{ github.event.pull_request.head.sha }}" + uses: actions/checkout@v2 + - name: Check Source Branch + if: github.head_ref != 'develop' && (github.base_ref == 'master' || github.base_ref == 'main') + run: echo "PRs must be made to develop branch before merging to main/master"; exit 1 + - name: Build And Push To Packages + if: github.event.pull_request.draft == false && github.event.action != 'closed' && github.event.pull_request.merged != true + env: + PR: "${{ github.event.pull_request.number }}" + SHA: "${{ github.event.pull_request.head.sha }}" + DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" + DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" + run: "curl https://raw.githubusercontent.com/kbase/.github/main/workflow-templates/scripts/pr_build.sh | bash" + tag_on_merge: + runs-on: ubuntu-latest + steps: + - name: Tag Latest Image On Merge + if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + env: + PR: "${{ github.event.pull_request.number }}" + SHA: "${{ github.event.pull_request.head.sha }}" + DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" + DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" + run: "curl https://raw.githubusercontent.com/kbase/.github/main/workflow-templates/scripts/tag_on_pr_merge.sh | bash" From 9e992833af3b95e526d75b2750900a8a0938908a Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 19 Apr 2022 14:14:31 -0500 Subject: [PATCH 24/43] Delete .github/workflows/scripts directory --- .github/workflows/scripts/build_prodrc_pr.sh | 17 ---------- .github/workflows/scripts/build_test_pr.sh | 17 ---------- .github/workflows/scripts/deploy_tag.sh | 34 ------------------- .github/workflows/scripts/prod_release.sh | 24 ------------- .github/workflows/scripts/tag_environments.sh | 22 ------------ .github/workflows/scripts/tag_prod_latest.sh | 12 ------- .github/workflows/scripts/tag_test_latest.sh | 12 ------- 7 files changed, 138 deletions(-) delete mode 100755 .github/workflows/scripts/build_prodrc_pr.sh delete mode 100755 .github/workflows/scripts/build_test_pr.sh delete mode 100755 .github/workflows/scripts/deploy_tag.sh delete mode 100755 .github/workflows/scripts/prod_release.sh delete mode 100755 .github/workflows/scripts/tag_environments.sh delete mode 100755 .github/workflows/scripts/tag_prod_latest.sh delete mode 100755 .github/workflows/scripts/tag_test_latest.sh diff --git a/.github/workflows/scripts/build_prodrc_pr.sh b/.github/workflows/scripts/build_prodrc_pr.sh deleted file mode 100755 index 4c7bdf277..000000000 --- a/.github/workflows/scripts/build_prodrc_pr.sh +++ /dev/null @@ -1,17 +0,0 @@ -#! /usr/bin/env bash - -export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') -export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}') -export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -export COMMIT=$(echo "$SHA" | cut -c -7) - -echo "Branch is:" ${GITHUB_HEAD_REF} -docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io -docker build --build-arg BUILD_DATE="$DATE" \ - --build-arg COMMIT="$COMMIT" \ - --build-arg BRANCH="$GITHUB_HEAD_REF" \ - --build-arg PULL_REQUEST="$PR" \ - --label us.kbase.vcs-pull-req="$PR" \ - -t ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" . -docker push ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" diff --git a/.github/workflows/scripts/build_test_pr.sh b/.github/workflows/scripts/build_test_pr.sh deleted file mode 100755 index 546b1b422..000000000 --- a/.github/workflows/scripts/build_test_pr.sh +++ /dev/null @@ -1,17 +0,0 @@ -#! /usr/bin/env bash - -export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') -export MY_APP=$(echo $(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')"-develop") -export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -export COMMIT=$(echo "$SHA" | cut -c -7) - -echo $DOCKER_TOKEN | docker login ghcr.io -u $DOCKER_ACTOR --password-stdin -docker build --build-arg BUILD_DATE="$DATE" \ - --build-arg COMMIT="$COMMIT" \ - --build-arg BRANCH="$GITHUB_HEAD_REF" \ - --build-arg PULL_REQUEST="$PR" \ - --label us.kbase.vcs-pull-req="$PR" \ - -t ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" . -docker push ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" - \ No newline at end of file diff --git a/.github/workflows/scripts/deploy_tag.sh b/.github/workflows/scripts/deploy_tag.sh deleted file mode 100755 index 5fb928ab4..000000000 --- a/.github/workflows/scripts/deploy_tag.sh +++ /dev/null @@ -1,34 +0,0 @@ -#! /usr/bin/env bash - -# Usage: ./deploy_tag.sh -e TARGET -o ORG -r REPO -s DEV_PROD -t IMAGE_TAG -# -# Example 1: ./deploy_tag.sh -o "kbase" -r "narrative-traefiker" -s "dev" -t "pr-9001" -e "ci" -# Example 2: ./deploy_tag.sh -o "kbase" -r "narrative" -s "prod" -t "latest" -e "next" -# -# Where: -# -o ORG is the organization (`kbase`, `kbaseapps`, etc.) -# -r REPO is the repository (e.g. `narrative`) -# -s DEV_PROD determines whether to pull the development {APPNAME}-develop or production {APPNAME} image. -# -t IMAGE_TAG is the *current* Docker image tag, typically `pr-#` or `latest` -# -e TARGET is one of: `appdsshev`, `ci`, or `next` -# -# Be sure to set $TOKEN first! -# See: https://docs.github.com/en/packages/getting-started-with-github-container-registry/migrating-to-github-container-registry-for-docker-images#authenticating-with-the-container-registry - - -while getopts e:o:r:s:t: option - do - case "${option}" - in - e) TARGET=${OPTARG};; - o) ORG=${OPTARG};; - r) REPO=${OPTARG};; - s) DEV_PROD=${OPTARG};; - t) IMAGE_TAG=${OPTARG};; - esac -done - -curl -H "Authorization: token $TOKEN" \ - -H 'Accept: application/vnd.github.everest-preview+json' \ - "https://api.github.com/repos/$ORG/$REPO/dispatches" \ - -d '{"event_type":"Tag '"$DEV_PROD"' '"$IMAGE_TAG"' for '"$TARGET"'", "client_payload": {"image_tag": "'"$IMAGE_TAG"'","target": "'"$TARGET"'","dev_prod": "'"$DEV_PROD"'"}}' diff --git a/.github/workflows/scripts/prod_release.sh b/.github/workflows/scripts/prod_release.sh deleted file mode 100755 index 46d008c69..000000000 --- a/.github/workflows/scripts/prod_release.sh +++ /dev/null @@ -1,24 +0,0 @@ -#! /usr/bin/env bash - -export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') -export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}') -export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -export COMMIT=$(echo "$SHA" | cut -c -7) - -echo "ISH is:" $ISH -echo "GITHUB_REF is:" $GITHUB_REF -echo "HEAD_REF is:" $GITHUB_HEAD_REF -echo "BASE_REF is:" $GITHUB_BASE_REF -echo "Release is:" $GITHUB_REF_NAME -echo $DOCKER_TOKEN | docker login ghcr.io -u $DOCKER_ACTOR --password-stdin -docker build --build-arg BUILD_DATE="$DATE" \ - --build-arg COMMIT="$COMMIT" \ - --build-arg BRANCH="$GITHUB_HEAD_REF" \ - --build-arg PULL_REQUEST="$PR" \ - --build-arg VERSION="$VER" \ - --label us.kbase.vcs-pull-req="$PR" \ - -t ghcr.io/"$MY_ORG"/"$MY_APP":"$VER" \ - -t ghcr.io/"$MY_ORG"/"$MY_APP":"latest" . -docker push ghcr.io/"$MY_ORG"/"$MY_APP":"$VER" -docker push ghcr.io/"$MY_ORG"/"$MY_APP":"latest" diff --git a/.github/workflows/scripts/tag_environments.sh b/.github/workflows/scripts/tag_environments.sh deleted file mode 100755 index b39732a09..000000000 --- a/.github/workflows/scripts/tag_environments.sh +++ /dev/null @@ -1,22 +0,0 @@ - -#! /usr/bin/env bash -# Add vars for PR & environments to yaml, as called from external script - -export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') -export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}') -export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -export COMMIT=$(echo "$SHA" | cut -c -7) - -if [ $DEV_PROD = "dev" ] || [ $DEV_PROD = "develop" ] -then - IMAGE=$MY_APP"-develop" -else - IMAGE=$MY_APP -fi - -echo "Dev or Prod:" $DEV_PROD -docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io -docker pull ghcr.io/"$MY_ORG"/"$IMAGE":"$IMAGE_TAG" -docker tag ghcr.io/"$MY_ORG"/"$IMAGE":"$IMAGE_TAG" ghcr.io/"$MY_ORG"/"$IMAGE":"$TARGET" -docker push ghcr.io/"$MY_ORG"/"$IMAGE":"$TARGET" diff --git a/.github/workflows/scripts/tag_prod_latest.sh b/.github/workflows/scripts/tag_prod_latest.sh deleted file mode 100755 index c3c422526..000000000 --- a/.github/workflows/scripts/tag_prod_latest.sh +++ /dev/null @@ -1,12 +0,0 @@ -#! /usr/bin/env bash - -export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') -export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}') -export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -export COMMIT=$(echo "$SHA" | cut -c -7) - -docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io -docker pull ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" -docker tag ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" ghcr.io/"$MY_ORG"/"$MY_APP":"latest-rc" -docker push ghcr.io/"$MY_ORG"/"$MY_APP":"latest-rc" diff --git a/.github/workflows/scripts/tag_test_latest.sh b/.github/workflows/scripts/tag_test_latest.sh deleted file mode 100755 index c0dc504a0..000000000 --- a/.github/workflows/scripts/tag_test_latest.sh +++ /dev/null @@ -1,12 +0,0 @@ -#! /usr/bin/env bash - -export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}') -export MY_APP=$(echo $(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')"-develop") -export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -export COMMIT=$(echo "$SHA" | cut -c -7) - -docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io -docker pull ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" -docker tag ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" ghcr.io/"$MY_ORG"/"$MY_APP":"latest" -docker push ghcr.io/"$MY_ORG"/"$MY_APP":"latest" From 377b7d57a9608612911dc7cf002181770840c0c9 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 19 Apr 2022 14:15:01 -0500 Subject: [PATCH 25/43] Delete .github/workflows directory --- .github/workflows/build_prodrc_pr.yaml | 31 ---------------- .github/workflows/build_test_pr.yaml | 27 -------------- .github/workflows/ee2-tests.yml | 46 ------------------------ .github/workflows/pr_build.yml | 47 ------------------------- .github/workflows/prod_release.yaml | 38 -------------------- .github/workflows/tag_environments.yaml | 19 ---------- .github/workflows/tag_prod_latest.yaml | 27 -------------- .github/workflows/tag_test_latest.yaml | 26 -------------- 8 files changed, 261 deletions(-) delete mode 100644 .github/workflows/build_prodrc_pr.yaml delete mode 100644 .github/workflows/build_test_pr.yaml delete mode 100644 .github/workflows/ee2-tests.yml delete mode 100644 .github/workflows/pr_build.yml delete mode 100644 .github/workflows/prod_release.yaml delete mode 100644 .github/workflows/tag_environments.yaml delete mode 100644 .github/workflows/tag_prod_latest.yaml delete mode 100644 .github/workflows/tag_test_latest.yaml diff --git a/.github/workflows/build_prodrc_pr.yaml b/.github/workflows/build_prodrc_pr.yaml deleted file mode 100644 index 2e5034e7c..000000000 --- a/.github/workflows/build_prodrc_pr.yaml +++ /dev/null @@ -1,31 +0,0 @@ ---- -name: Build Prod RC Image -'on': - pull_request: - branches: - - master - - main - types: - - opened - - synchronize - - ready_for_review -jobs: - docker_build: - runs-on: ubuntu-latest - steps: - - name: Verify merge is develop -> main - if: github.head_ref != 'develop' - run: echo "Must merge from develop -> main/master"; exit 1 - - name: Check out GitHub Repo - if: github.event.pull_request.draft == false && github.head_ref == 'develop' - with: - ref: "${{ github.event.pull_request.head.sha }}" - uses: actions/checkout@v2 - - name: Build and Push to Packages - if: github.event.pull_request.draft == false && github.head_ref == 'develop' - env: - PR: "${{ github.event.pull_request.number }}" - SHA: "${{ github.event.pull_request.head.sha }}" - DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" - DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" - run: "./.github/workflows/scripts/build_prodrc_pr.sh\n" diff --git a/.github/workflows/build_test_pr.yaml b/.github/workflows/build_test_pr.yaml deleted file mode 100644 index b6b53286f..000000000 --- a/.github/workflows/build_test_pr.yaml +++ /dev/null @@ -1,27 +0,0 @@ ---- -name: Build Test Image -'on': - pull_request: - branches: - - develop - types: - - opened - - synchronize - - ready_for_review -jobs: - docker_build: - runs-on: ubuntu-latest - steps: - - name: Check out GitHub Repo - if: github.event.pull_request.draft == false - with: - ref: "${{ github.event.pull_request.head.sha }}" - uses: actions/checkout@v2 - - name: Build and Push to Packages - if: github.event.pull_request.draft == false - env: - PR: "${{ github.event.pull_request.number }}" - SHA: "${{ github.event.pull_request.head.sha }}" - DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" - DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" - run: "./.github/workflows/scripts/build_test_pr.sh\n" diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml deleted file mode 100644 index c4920ce01..000000000 --- a/.github/workflows/ee2-tests.yml +++ /dev/null @@ -1,46 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -# To ssh into this build add the following: -#- name: Start SSH session -# uses: luchihoratiu/debug-via-ssh@main -# with: -# NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }} -# SSH_PASS: ${{ secrets.SSH_PASS }} - -name: Execution Engine 2 Test Suite - -on: - [push, pull_request] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Lint with flake8 and black - run: | - python -m pip install --upgrade pip - pip install flake8 black pytest - flake8 ./lib ./test - black --check ./lib ./test - - name: Install dependencies - run: | - if [ -f requirements.txt ]; then pip install -r requirements-dev.txt; fi - cd /opt - git clone https://github.com/kbase/jars - cd - - - name: Build Docker Image - run: | - docker build . -t execution_engine2:test - - name: Run Tests - run: | - docker-compose up -d - cp test/env/test.travis.env test.env - make test-coverage - codecov - diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml deleted file mode 100644 index fb8fb6783..000000000 --- a/.github/workflows/pr_build.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -name: Build & Tag Image on PR -'on': - pull_request: - branches: - - develop - - main - - master - types: - - closed - - edited - - opened - - ready_for_review - - reopened - - synchronize - - workflow_dispatch -jobs: - docker_build: - runs-on: ubuntu-latest - steps: - - name: Check Out GitHub Repo - if: github.event.pull_request.draft == false - with: - ref: "${{ github.event.pull_request.head.sha }}" - uses: actions/checkout@v2 - - name: Check Source Branch - if: github.head_ref != 'develop' && (github.base_ref == 'master' || github.base_ref == 'main') - run: echo "PRs must be made to develop branch before merging to main/master"; exit 1 - - name: Build And Push To Packages - if: github.event.pull_request.draft == false && github.event.action != 'closed' && github.event.pull_request.merged != true - env: - PR: "${{ github.event.pull_request.number }}" - SHA: "${{ github.event.pull_request.head.sha }}" - DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" - DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" - run: "curl https://raw.githubusercontent.com/kbase/.github/main/workflow-templates/scripts/pr_build.sh | bash" - tag_on_merge: - runs-on: ubuntu-latest - steps: - - name: Tag Latest Image On Merge - if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true - env: - PR: "${{ github.event.pull_request.number }}" - SHA: "${{ github.event.pull_request.head.sha }}" - DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" - DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" - run: "curl https://raw.githubusercontent.com/kbase/.github/main/workflow-templates/scripts/tag_on_pr_merge.sh | bash" diff --git a/.github/workflows/prod_release.yaml b/.github/workflows/prod_release.yaml deleted file mode 100644 index ffa145332..000000000 --- a/.github/workflows/prod_release.yaml +++ /dev/null @@ -1,38 +0,0 @@ ---- -name: Publish Release Image -'on': - release: - branches: - - main - - master - types: - - published -jobs: - docker_build: - runs-on: ubuntu-latest - steps: - - name: Check Tag - id: check-tag - run: |- - if [[ ${{ github.ref_name }} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo ::set-output name=match::true - fi - - name: Report SemVer Check - if: steps.check-tag.outputs.match != 'true' - run: echo "Release version must follow semantic naming (e.g. 1.0.2)"; exit 1 - - name: Check Source Branch - if: github.event.release.target_commitish != 'master' && github.event.release.target_commitish != 'main' - run: echo "Releases must be built from master/main branch"; exit 1 - - name: Check out GitHub Repo - with: - ref: "${{ github.event.pull_request.head.sha }}" - uses: actions/checkout@v2 - - name: Build and Push to Packages - env: - ISH: "${{ github.event.release.target_commitish }}" - PR: "${{ github.event.pull_request.number }}" - SHA: "${{ github.event.pull_request.head.sha }}" - VER: "${{ github.event.release.tag_name }}" - DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" - DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" - run: "./.github/workflows/scripts/prod_release.sh\n" diff --git a/.github/workflows/tag_environments.yaml b/.github/workflows/tag_environments.yaml deleted file mode 100644 index 6dba7431f..000000000 --- a/.github/workflows/tag_environments.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Tag Image For Deploy -'on': - repository_dispatch -jobs: - tag_environments: - runs-on: ubuntu-latest - steps: - - name: Check out GitHub Repo - uses: actions/checkout@v2 - - name: Tag Deploy Environments - env: - DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" - DOCKER_TOKEN: ${{ secrets.GHCR_TOKEN }} - IMAGE_TAG: ${{ github.event.client_payload.image_tag }} - SHA: ${{ github.event.pull_request.head.sha }} - TARGET: ${{ github.event.client_payload.target }} - DEV_PROD: ${{ github.event.client_payload.dev_prod }} - run: './.github/workflows/scripts/tag_environments.sh' diff --git a/.github/workflows/tag_prod_latest.yaml b/.github/workflows/tag_prod_latest.yaml deleted file mode 100644 index 12b23df0c..000000000 --- a/.github/workflows/tag_prod_latest.yaml +++ /dev/null @@ -1,27 +0,0 @@ ---- -name: Tag Prod Latest -'on': - pull_request: - branches: - - master - - main - types: - - closed -jobs: - docker_tag: - runs-on: ubuntu-latest - steps: - - name: Check out GitHub Repo - if: github.event_name == 'pull_request' && github.event.action == 'closed' && - github.event.pull_request.merged == true - with: - ref: "${{ github.event.pull_request.head.sha }}" - uses: actions/checkout@v2 - - name: Build and Push to Packages - if: github.event.pull_request.draft == false - env: - PR: "${{ github.event.pull_request.number }}" - SHA: "${{ github.event.pull_request.head.sha }}" - DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" - DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" - run: "./.github/workflows/scripts/tag_prod_latest.sh\n" diff --git a/.github/workflows/tag_test_latest.yaml b/.github/workflows/tag_test_latest.yaml deleted file mode 100644 index d8cac4654..000000000 --- a/.github/workflows/tag_test_latest.yaml +++ /dev/null @@ -1,26 +0,0 @@ ---- -name: Tag Latest Test Image -'on': - pull_request: - branches: - - develop - types: - - closed -jobs: - docker_tag: - runs-on: ubuntu-latest - steps: - - name: Check out GitHub Repo - if: github.event_name == 'pull_request' && github.event.action == 'closed' && - github.event.pull_request.merged == true - with: - ref: "${{ github.event.pull_request.head.sha }}" - uses: actions/checkout@v2 - - name: Build and Push to Packages - if: github.event.pull_request.draft == false - env: - PR: "${{ github.event.pull_request.number }}" - SHA: "${{ github.event.pull_request.head.sha }}" - DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" - DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" - run: "./.github/workflows/scripts/tag_test_latest.sh\n" From fb3cbd1e88bb7b0bba6552f170e0f1ebfd8a8d69 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 19 Apr 2022 14:15:48 -0500 Subject: [PATCH 26/43] Create pr_build.yml --- .github/workflows/pr_build.yml | 47 ++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/pr_build.yml diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml new file mode 100644 index 000000000..fb8fb6783 --- /dev/null +++ b/.github/workflows/pr_build.yml @@ -0,0 +1,47 @@ +--- +name: Build & Tag Image on PR +'on': + pull_request: + branches: + - develop + - main + - master + types: + - closed + - edited + - opened + - ready_for_review + - reopened + - synchronize + - workflow_dispatch +jobs: + docker_build: + runs-on: ubuntu-latest + steps: + - name: Check Out GitHub Repo + if: github.event.pull_request.draft == false + with: + ref: "${{ github.event.pull_request.head.sha }}" + uses: actions/checkout@v2 + - name: Check Source Branch + if: github.head_ref != 'develop' && (github.base_ref == 'master' || github.base_ref == 'main') + run: echo "PRs must be made to develop branch before merging to main/master"; exit 1 + - name: Build And Push To Packages + if: github.event.pull_request.draft == false && github.event.action != 'closed' && github.event.pull_request.merged != true + env: + PR: "${{ github.event.pull_request.number }}" + SHA: "${{ github.event.pull_request.head.sha }}" + DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" + DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" + run: "curl https://raw.githubusercontent.com/kbase/.github/main/workflow-templates/scripts/pr_build.sh | bash" + tag_on_merge: + runs-on: ubuntu-latest + steps: + - name: Tag Latest Image On Merge + if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + env: + PR: "${{ github.event.pull_request.number }}" + SHA: "${{ github.event.pull_request.head.sha }}" + DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" + DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" + run: "curl https://raw.githubusercontent.com/kbase/.github/main/workflow-templates/scripts/tag_on_pr_merge.sh | bash" From ec1f81b21977452f6b684e6337741228a0a41702 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 19 Apr 2022 14:19:57 -0500 Subject: [PATCH 27/43] Create ee2-tests.yml --- .github/workflows/ee2-tests.yml | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/ee2-tests.yml diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml new file mode 100644 index 000000000..422e97e2b --- /dev/null +++ b/.github/workflows/ee2-tests.yml @@ -0,0 +1,45 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +# To ssh into this build add the following: +#- name: Start SSH session +# uses: luchihoratiu/debug-via-ssh@main +# with: +# NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }} +# SSH_PASS: ${{ secrets.SSH_PASS }} + +name: Execution Engine 2 Test Suite + +on: + [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Lint with flake8 and black + run: | + python -m pip install --upgrade pip + pip install flake8 black pytest + flake8 ./lib ./test + black --check ./lib ./test + - name: Install dependencies + run: | + if [ -f requirements.txt ]; then pip install -r requirements-dev.txt; fi + cd /opt + git clone https://github.com/kbase/jars + cd - + - name: Build Docker Image + run: | + docker build . -t execution_engine2:test + - name: Run Tests + run: | + docker-compose up -d + cp test/env/test.travis.env test.env + make test-coverage + codecov From ec0913cd08d9c3dd54575243c68b62ccac4af088 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 19 Apr 2022 14:21:11 -0500 Subject: [PATCH 28/43] Update ee2-tests.yml --- .github/workflows/ee2-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml index 422e97e2b..16c264e83 100644 --- a/.github/workflows/ee2-tests.yml +++ b/.github/workflows/ee2-tests.yml @@ -11,7 +11,7 @@ name: Execution Engine 2 Test Suite on: - [push, pull_request] + [pull_request] jobs: build: From 9f07d956007f14b10adb21efef5755ad028beb1b Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 19 Apr 2022 14:34:48 -0500 Subject: [PATCH 29/43] Update pr_build.yml --- .github/workflows/pr_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index fb8fb6783..874a843cf 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -13,7 +13,7 @@ name: Build & Tag Image on PR - ready_for_review - reopened - synchronize - - workflow_dispatch + workflow_dispatch: jobs: docker_build: runs-on: ubuntu-latest From 31c32c75f6347511b537e3cc5c88b8ec80d55efb Mon Sep 17 00:00:00 2001 From: bio-boris Date: Tue, 19 Apr 2022 14:39:49 -0500 Subject: [PATCH 30/43] Create prod_release.yml --- .github/workflows/prod_release.yml | 37 ++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/prod_release.yml diff --git a/.github/workflows/prod_release.yml b/.github/workflows/prod_release.yml new file mode 100644 index 000000000..8c5462f57 --- /dev/null +++ b/.github/workflows/prod_release.yml @@ -0,0 +1,37 @@ +--- +name: Publish Release Image +'on': + release: + branches: + - main + - master + types: + - published +jobs: + docker_build: + runs-on: ubuntu-latest + steps: + - name: Check Tag + id: check-tag + run: |- + if [[ ${{ github.ref_name }} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo ::set-output name=match::true + fi + - name: Report SemVer Check + if: steps.check-tag.outputs.match != 'true' + run: echo "Release version must follow semantic naming (e.g. 1.0.2)"; exit 1 + - name: Check Source Branch + if: github.event.release.target_commitish != 'master' && github.event.release.target_commitish != 'main' + run: echo "Releases must be built from master/main branch"; exit 1 + - name: Check out GitHub Repo + with: + ref: "${{ github.event.pull_request.head.sha }}" + uses: actions/checkout@v2 + - name: Build and Push to Packages + env: + PR: "${{ github.event.pull_request.number }}" + SHA: "${{ github.event.pull_request.head.sha }}" + VER: "${{ github.event.release.tag_name }}" + DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" + DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" + run: "curl https://raw.githubusercontent.com/kbase/.github/main/workflow-templates/scripts/prod_release.sh | bash" From 43138e3743731090c2fa85e219cbc51271e0c211 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 May 2022 01:07:56 +0000 Subject: [PATCH 31/43] Bump ujson from 1.35 to 5.2.0 in /test/dockerfiles/condor Bumps [ujson](https://github.com/ultrajson/ultrajson) from 1.35 to 5.2.0. - [Release notes](https://github.com/ultrajson/ultrajson/releases) - [Commits](https://github.com/ultrajson/ultrajson/compare/v1.35...5.2.0) --- updated-dependencies: - dependency-name: ujson dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- test/dockerfiles/condor/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/dockerfiles/condor/requirements.txt b/test/dockerfiles/condor/requirements.txt index 9cc91a5b8..b9d0de8fa 100644 --- a/test/dockerfiles/condor/requirements.txt +++ b/test/dockerfiles/condor/requirements.txt @@ -17,7 +17,7 @@ requests==2.22.0 requests-async==0.5.0 rfc3986==1.3.2 sanic==20.12.6 -ujson==1.35 +ujson==5.2.0 urllib3==1.26.5 uvloop==0.12.2 websockets==6.0 From 96d5745bbad50cbb0845eb054acd623e3f4e3e68 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 May 2022 01:23:16 +0000 Subject: [PATCH 32/43] Bump ujson from 1.35 to 5.2.0 Bumps [ujson](https://github.com/ultrajson/ultrajson) from 1.35 to 5.2.0. - [Release notes](https://github.com/ultrajson/ultrajson/releases) - [Commits](https://github.com/ultrajson/ultrajson/compare/v1.35...5.2.0) --- updated-dependencies: - dependency-name: ujson dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Pipfile | 2 +- Pipfile.lock | 85 ++++++++++++++++++++++++++++++++++++++++---- requirements-dev.txt | 12 +++++-- 3 files changed, 90 insertions(+), 9 deletions(-) diff --git a/Pipfile b/Pipfile index ec7e9deb0..5de7bfa43 100644 --- a/Pipfile +++ b/Pipfile @@ -64,7 +64,7 @@ slackclient = "==2.7.1" toml = "==0.10.1" tqdm = "==4.42.1" typing-extensions = "==3.7.4.3" -ujson = "==1.35" +ujson = "==5.2.0" urllib3 = "==1.25.8" uvloop = "==0.12.2" websockets = "==6.0" diff --git a/Pipfile.lock b/Pipfile.lock index 5df9ba0ff..e238e5484 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "96e75d4a0d655bab93d08d5d163c1bdb458d7ff8bc22b3e48af30e07797d0340" + "sha256": "131a9560c753b4c2a11c262f60221ca37fbf88b824191cfd09e123bd9e63c170" }, "pipfile-spec": 6, "requires": { @@ -199,32 +199,42 @@ }, "coverage": { "hashes": [ + "sha256:0c5fe441b9cfdab64719f24e9684502a59432df7570521563d7b1aff27ac755f", + "sha256:2b412abc4c7d6e019ce7c27cbc229783035eef6d5401695dccba80f481be4eb3", "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", + "sha256:42692db854d13c6c5e9541b6ffe0fe921fe16c9c446358d642ccae1462582d3b", "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", + "sha256:4ec30ade438d1711562f3786bea33a9da6107414aed60a5daa974d50a8c2c351", "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", + "sha256:6899797ac384b239ce1926f3cb86ffc19996f6fa3a1efbb23cb49e0c12d8c18c", "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", + "sha256:8e679d1bde5e2de4a909efb071f14b472a678b788904440779d2c449c0355b27", "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", + "sha256:93f965415cc51604f571e491f280cff0f5be35895b4eb5e55b47ae90c02a497b", "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", + "sha256:a9abc8c480e103dc05d9b332c6cc9fb1586330356fc14f1aa9c0ca5745097d19", "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", + "sha256:c22ab9f96cbaff05c6a84e20ec856383d27eae09e511d3e6ac4479489195861d", "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", + "sha256:ca58eba39c68010d7e87a823f22a081b5290e3e3c64714aac3c91481d8b34d22", "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", @@ -563,10 +573,11 @@ }, "packaging": { "hashes": [ - "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5", - "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a" + "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", + "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" ], - "version": "==20.9" + "markers": "python_version >= '3.6'", + "version": "==21.3" }, "pluggy": { "hashes": [ @@ -619,6 +630,7 @@ "hashes": [ "sha256:32421df60d06f479d71b6b539642e410ece3006e8910688e68df962c8eb40a21", "sha256:324b22a8443e11faca44c96b20e7ec8a9e59a1e664457edeeb4f796080b31cde", + "sha256:3b6336b1d2a1ac2fcc8f629070016f3c76ad7dc969f269232471953d6dd17c0d", "sha256:4505ff8b7923dd7a8bed1bf25c9c4d0df5ab0b8b2821f2296533f2149a55f401", "sha256:460b224681ea711e48e3638d15be2249024031b7dcb9622ba19c2e85bd5a26cc", "sha256:47473b70c5f3cd5ddd2c49ab3b9ceafdafbbed5bc963f147df22a9343d7978f5", @@ -630,6 +642,7 @@ "sha256:61cad83637ae12c1c825130d7f9325cd6c162e3a64e8747a8144866020be3ff4", "sha256:61e8e1c58b4fdf47ab79b7c7db8bb022c1e40b3b5fcbbaeea5fc94dc5c75638d", "sha256:6e04e496af7d156b66cce70460011c621ecbadf5dcdce325c7acbb3cd6ea245d", + "sha256:74838f04da0b3995b830fe1f00f9b200831582cbc42a22b77e04dfb717cb0d56", "sha256:7ef89ec435e89da902451dde6845066fe2770befaf0301fe2a1ac426b51fced3", "sha256:854e8425e5eb775ccfffad04ecd094c99923d60a2c2d49babb5c435e836a91fa", "sha256:9569796d48498e4db4e1d56284b626a8ed15f641ce3a8b2085f06bb03f4c2c88", @@ -639,6 +652,7 @@ "sha256:aef7d88384ada699976350a285c7a333f96ebc959e98e7d2c98589f47bbf3b7f", "sha256:b4d7ff9957ee770cf03bd7156a68a2f2e838e60712d9608eadc8741c15d01e72", "sha256:c1db85c39e6a60588f855dbc7bd68fb0dab796096148ab5aa4abecaff19e1c6e", + "sha256:c3e813b1bd0b883639e30170dc9daccb9b6ef7e81836188b88d3fc7364892b35", "sha256:cee2fc0b94e66e7230da12fc4b3d34793c49957e16ee04f6468a94e264a1e41d", "sha256:cf1dea28379a16b23e47db312883f07b3ba8d9d6abc1c59e51d4c8ae1820ab43", "sha256:d1cd175df7c8b5fc976bade78bf4d9fb5aa7ab465c0f59931e380bbe188ef8fc", @@ -692,6 +706,8 @@ }, "pytest-profiling": { "hashes": [ + "sha256:3b255f9db36cb2dd7536a8e7e294c612c0be7f7850a7d30754878e4315d56600", + "sha256:6bce4e2edc04409d2f3158c16750fab8074f62d404cc38eeb075dff7fcbb996c", "sha256:93938f147662225d2b8bd5af89587b979652426a8a6ffd7e73ec4a23e24b7f29", "sha256:999cc9ac94f2e528e3f5d43465da277429984a1c237ae9818f8cfd0b06acb019" ], @@ -789,6 +805,14 @@ "index": "pypi", "version": "==0.14.3" }, + "setuptools": { + "hashes": [ + "sha256:68e45d17c9281ba25dc0104eadd2647172b3472d9e01f911efa57965e8d51a36", + "sha256:a43bdedf853c670e5fed28e5623403bad2f73cf02f9a2774e91def6bda8265a7" + ], + "markers": "python_version >= '3.7'", + "version": "==62.3.2" + }, "six": { "hashes": [ "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", @@ -832,10 +856,59 @@ }, "ujson": { "hashes": [ - "sha256:f66073e5506e91d204ab0c614a148d5aa938bdbf104751be66f8ad7a222f5f86" + "sha256:04a8c388b2d16316df3365c81f368955662581f6a4ff033e9aba2dd1ffc9e05e", + "sha256:080da13f81740c076e5f16c254a10d0e32f45d225a5e6b0687a86493cfcfbafb", + "sha256:0b47a138203bb06bdac03b2a89ac9b2993fd32cb7daded06c966dd84300a5786", + "sha256:102b8eb5e15e6c5537426414d180c28dbf0489e51f7c22b706511ac84aae4458", + "sha256:11f735870f189bff1841c720115226894415ab6a7796dee8ab46bc767ea2e743", + "sha256:163191b88842d874e081707d35de2e205e0e396e70fd068d1038879bca8b17ad", + "sha256:25522c674b35c33f375586ac98d92ce731e79059424507ecbccbfcbce832d597", + "sha256:27a254a150e46980608b16ef3b609e703173492cfa738f4644c81d7e7d77494c", + "sha256:2c04456de1fc92cc7062904c176c74e6ea220469b949508be42e819646a28457", + "sha256:2c7712da662b92f80442a8efc0df09cea3a5efb42b0dd6a642e36b1b40a260d4", + "sha256:350a3010db0045e1306bbdf889d1bdaee9bb095856c317716f0a74108cf4afe9", + "sha256:468d7d8dcbafc3fd40cc73e4a533a7a1d4f935f605c15ae6cac32c6d53c4c6aa", + "sha256:489d495431c80dc0048c4551a0d6cdbf1209e2d274f47c3f72415c91842eeb68", + "sha256:49ce8521b0cdf210481bd89887fd1bd0a975f66088b1256dafc77c67c8ccb89d", + "sha256:4d1ed3897e45477b2a4a1371186df299b13938d4d44d850953a4bb0ea4cb38f3", + "sha256:54ee7c46615b42f7ae9dca90f54d204a4d2041a4c926b08fffa953aa3a246e54", + "sha256:584c558c23ddc21f5b07d2c54ee527731bd9716101c27829023ab7f3ffbaa8fc", + "sha256:6227597d0201ceadc902d1a8edaffaeb244050b197368ed25e6f6be0df170a6f", + "sha256:6677bee8690c71f5e6cf519a6d8400f04fbd3ff9f6c50f35f1b664bc94546f84", + "sha256:6b455a62bd20e890b2124a65df45313b4292dbea851ef38574e5e2de94691ad5", + "sha256:6c5bbe6de6c9a5fe8dca56e36fb5c4a42e1a01d4aae1ac20cd8d7d82ccff9430", + "sha256:729af63e4de30c54b527b54b4100266f79833c1e8ba35e784f01b44c2aca88d8", + "sha256:754e9da96a24535ae5ab2a52e1d1dfc65a6a717c14063855b83f327fdf2173ea", + "sha256:75a886bd89d8e5a004a39a6c5dc8a43bb7fcf05129d2dccd16a59602a612823a", + "sha256:8c3f7578a62d9255650ef32e78d3345e98262e064c9ba3f205311b4c9eb507a6", + "sha256:90de04391916c5adc7bbcc69bd778e263ed45cc83c070099cb07ed25068d6a12", + "sha256:940f35e9a0969440621445dbb6adffaa2cea77d0262abc74fce78704120c4534", + "sha256:9acc874128baddeff908736db251597e4cbd007a384730377a59a61b08886599", + "sha256:a1a55b3310632661a03ce68ccfb92264031aea21626d6fa5c8f6c32e769be7b6", + "sha256:a3c6798035b574ceba747de83f3223a622622b7ab77a24f8b4fbea2cb92f14b0", + "sha256:a5e374e793b0a3c7df20ee4c8234e89859ddb2b2821cc3300ae94ab5b08fa6d0", + "sha256:a6f3ad3b11578bc4e25d5bd256c938fe2c7c015d8f504bc7835f127ed26a0818", + "sha256:b3671e1dfc49a4b4453d89fd7438aa9d7cca28afe329c70eba84e2a5778dbf3f", + "sha256:b5fcbaabf3d115cb816eb165f3fa5de5c5bc795473a554ae55620d134ddf2d36", + "sha256:bc1a619bad9894dad144184b735c98179c7d92d7b40fbda28eb8b0857bdfdf52", + "sha256:be909514a47b6272e34cd1213feee324ca35a354e07f1ae3aba12d3694a5279f", + "sha256:c519743a53bbe8aac6b743bcf50eb83057d1e0341e1ca8f8491f729a885af640", + "sha256:c549d5a7652c3a0dd00ef6ff910fb01878bc116c66c94ac455a55cffa32cc229", + "sha256:d1e5c635b7c3465ab8d2e3dc97c341ef1801c53a378f1d1d4cb934f6c90ec66c", + "sha256:d2357ce7d93eadd29b6efbe72228809948cc59ec6682c20fa6de08aeef1703f8", + "sha256:d38c2a58c892c680080b22b59eebd77b7c6f4ae24361111fba115f9ed3651dcf", + "sha256:d57a87bbc77d66b8a2b74bab66357c3bb6194f5d248f1053fb8044787abde73f", + "sha256:d9b1c3d2b22c040a81ff4e5927ce307919f7ac8bf888afded714d925edc8d0a4", + "sha256:dc5fd1d5b48edd3cc64e89ea94abe231509fdc938bdeafafe9aef3a05810159f", + "sha256:dc71ead5706e81fdf1054c8c11e4aaab43527da450a2701213c20717852d1a51", + "sha256:e53388fb092197cb8f956673792aca994872917d897ca42a0abf7a35e293575a", + "sha256:e991b7b3a08ac9e9d3a51589ef1c359c8d44ece730351cfac055684bf3787372", + "sha256:ed78a5b169ece75a1e1368935ce6ab051dcbcd5c158b9796b2f1fa6cc467a651", + "sha256:ef868bf01851869a26c0ca5f88036903836c3a6b463c74d96b37f294f6bdeea4", + "sha256:fb4555df1fe018806ba14cc38786269c8e213930103c6d0ac81e506d09d1de7e" ], "index": "pypi", - "version": "==1.35" + "version": "==5.2.0" }, "urllib3": { "hashes": [ diff --git a/requirements-dev.txt b/requirements-dev.txt index 06e45c8e1..a6676536b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,10 @@ +# +# These requirements were autogenerated by pipenv +# To regenerate from the project's Pipfile, run: +# +# pipenv lock --requirements +# + -i https://pypi.org/simple/ aiofiles==0.4.0 aiohttp==3.7.4 @@ -37,7 +44,7 @@ mock==3.0.5 mongoengine==0.23.0 multidict==4.5.2 nose==1.3.7 -packaging==20.9 +packaging==21.3; python_version >= '3.6' pluggy==0.13.1 psutil==5.6.6 py==1.10.0 @@ -59,12 +66,13 @@ rfc3986==1.3.2 ruamel.yaml==0.15.87 sanic==19.6.0 sentry-sdk==0.14.3 +setuptools==62.3.2; python_version >= '3.7' six==1.14.0 slackclient==2.7.1 toml==0.10.1 tqdm==4.42.1 typing-extensions==3.7.4.3 -ujson==1.35 +ujson==5.2.0 urllib3==1.25.8 uvloop==0.12.2 websocket-client==0.57.0 From 2c365d8f398115d66d80fe38a7fae45094d8f33a Mon Sep 17 00:00:00 2001 From: bio-boris Date: Mon, 20 Jun 2022 22:53:11 -0500 Subject: [PATCH 33/43] Update pr_build.yml --- .github/workflows/pr_build.yml | 80 ++++++++++++++++------------------ 1 file changed, 38 insertions(+), 42 deletions(-) diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 874a843cf..bf5d7e076 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -1,47 +1,43 @@ --- -name: Build & Tag Image on PR -'on': +name: Pull Request Build, Tag, & Push +on: pull_request: branches: - - develop - - main - - master + - develop + - main + - master types: - - closed - - edited - - opened - - ready_for_review - - reopened - - synchronize - workflow_dispatch: + - opened + - reopened + - synchronize + - merged jobs: - docker_build: - runs-on: ubuntu-latest - steps: - - name: Check Out GitHub Repo - if: github.event.pull_request.draft == false - with: - ref: "${{ github.event.pull_request.head.sha }}" - uses: actions/checkout@v2 - - name: Check Source Branch - if: github.head_ref != 'develop' && (github.base_ref == 'master' || github.base_ref == 'main') - run: echo "PRs must be made to develop branch before merging to main/master"; exit 1 - - name: Build And Push To Packages - if: github.event.pull_request.draft == false && github.event.action != 'closed' && github.event.pull_request.merged != true - env: - PR: "${{ github.event.pull_request.number }}" - SHA: "${{ github.event.pull_request.head.sha }}" - DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" - DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" - run: "curl https://raw.githubusercontent.com/kbase/.github/main/workflow-templates/scripts/pr_build.sh | bash" - tag_on_merge: - runs-on: ubuntu-latest - steps: - - name: Tag Latest Image On Merge - if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true - env: - PR: "${{ github.event.pull_request.number }}" - SHA: "${{ github.event.pull_request.head.sha }}" - DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" - DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" - run: "curl https://raw.githubusercontent.com/kbase/.github/main/workflow-templates/scripts/tag_on_pr_merge.sh | bash" + build-develop-open: + if: github.base_ref == 'develop' && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_build.yml@main + secrets: inherit + build-develop-merge: + if: github.base_ref == 'develop' && github.event.pull_request.merged == true + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}-develop' + tags: pr-${{ github.event.number }},latest + secrets: inherit + build-main-open: + if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: pr-${{ github.event.number }} + secrets: inherit + build-main-merge: + if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == true + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: pr-${{ github.event.number }},latest-rc + secrets: inherit + trivy-scans: + if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main + secrets: inherit From 2b867e8d2c4f1f85111fe89249f49083bf71582a Mon Sep 17 00:00:00 2001 From: bio-boris Date: Mon, 20 Jun 2022 22:53:49 -0500 Subject: [PATCH 34/43] Update prod_release.yml --- .github/workflows/prod_release.yml | 54 ++++++++++++------------------ 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/.github/workflows/prod_release.yml b/.github/workflows/prod_release.yml index 8c5462f57..a25467818 100644 --- a/.github/workflows/prod_release.yml +++ b/.github/workflows/prod_release.yml @@ -1,37 +1,25 @@ --- -name: Publish Release Image -'on': +name: Release - Build & Push Image +on: release: branches: - - main - - master - types: - - published + - main + - master + types: [ published ] jobs: - docker_build: - runs-on: ubuntu-latest - steps: - - name: Check Tag - id: check-tag - run: |- - if [[ ${{ github.ref_name }} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo ::set-output name=match::true - fi - - name: Report SemVer Check - if: steps.check-tag.outputs.match != 'true' - run: echo "Release version must follow semantic naming (e.g. 1.0.2)"; exit 1 - - name: Check Source Branch - if: github.event.release.target_commitish != 'master' && github.event.release.target_commitish != 'main' - run: echo "Releases must be built from master/main branch"; exit 1 - - name: Check out GitHub Repo - with: - ref: "${{ github.event.pull_request.head.sha }}" - uses: actions/checkout@v2 - - name: Build and Push to Packages - env: - PR: "${{ github.event.pull_request.number }}" - SHA: "${{ github.event.pull_request.head.sha }}" - VER: "${{ github.event.release.tag_name }}" - DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}" - DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}" - run: "curl https://raw.githubusercontent.com/kbase/.github/main/workflow-templates/scripts/prod_release.sh | bash" + check-source-branch: + uses: kbase/.github/.github/workflows/reusable_validate-branch.yml@main + with: + build_branch: '${{ github.event.release.target_commitish }}' + validate-release-tag: + needs: check-source-branch + uses: kbase/.github/.github/workflows/reusable_validate-release-tag.yml@main + with: + release_tag: '${{ github.event.release.tag_name }}' + build-push: + needs: validate-release-tag + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: '${{ github.event.release.tag_name }},latest' + secrets: inherit From 726edcfe4c13ead79300e9d65c65f5dfdcd53640 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Mon, 20 Jun 2022 22:54:14 -0500 Subject: [PATCH 35/43] Create manual-build.yml --- .github/workflows/manual-build.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/workflows/manual-build.yml diff --git a/.github/workflows/manual-build.yml b/.github/workflows/manual-build.yml new file mode 100644 index 000000000..944f9035a --- /dev/null +++ b/.github/workflows/manual-build.yml @@ -0,0 +1,11 @@ +--- +name: Manual Build & Push +on: + workflow_dispatch: +jobs: + build-push: + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}-develop' + tags: br-${{ github.ref_name }} + secrets: inherit From 79cc8532daaf1df0851b1f5204e30ef7e65efb4f Mon Sep 17 00:00:00 2001 From: bio-boris Date: Mon, 20 Jun 2022 23:07:36 -0500 Subject: [PATCH 36/43] Update deploy.cfg --- deploy.cfg | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy.cfg b/deploy.cfg index 9618cb902..02fccd0bc 100644 --- a/deploy.cfg +++ b/deploy.cfg @@ -51,13 +51,13 @@ request_memory = 23000M request_disk = 100GB [njs] -request_cpus = {{ default .Env.njs_default_cores "24" }} -request_memory = 23000M +request_cpus = {{ default .Env.njs_default_cores "12" }} +request_memory = 10000M request_disk = 100GB [bigmem] request_cpus = {{ default .Env.bigmem_default_cores "32" }} -request_memory = 204800M +request_memory = 250000M request_disk = 100GB [bigmemlong] @@ -81,8 +81,8 @@ request_memory = 204800M request_disk = 100GB [kb_upload] -request_cpus = {{ default .Env.kb_upload_default_cores "24" }} -request_memory = 4500M +request_cpus = {{ default .Env.kb_upload_default_cores "12" }} +request_memory = 1000M request_disk = 50GB [kb_upload_bulk] From ce70b9c818aa84f77a9dec310ef5c5bf256b04c9 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Mon, 20 Jun 2022 23:08:44 -0500 Subject: [PATCH 37/43] Update deploy.cfg --- deploy.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy.cfg b/deploy.cfg index 02fccd0bc..72062fd35 100644 --- a/deploy.cfg +++ b/deploy.cfg @@ -56,7 +56,7 @@ request_memory = 10000M request_disk = 100GB [bigmem] -request_cpus = {{ default .Env.bigmem_default_cores "32" }} +request_cpus = {{ default .Env.bigmem_default_cores "16" }} request_memory = 250000M request_disk = 100GB From 2bf1266e81a50b2756a0ffb9a4915bd8efb66e6d Mon Sep 17 00:00:00 2001 From: Boris Sadkhin Date: Mon, 20 Jun 2022 23:38:58 -0500 Subject: [PATCH 38/43] Removed jars from build --- Dockerfile | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index fd4b051ed..1a244a3a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,14 +14,10 @@ RUN mkdir -p /etc/apt/sources.list.d # Install condor RUN curl -fsSL https://get.htcondor.org | /bin/bash -s -- --no-dry-run -# install jars -# perhaps we should have test and prod dockerfiles to avoid jars and mongo installs in prod -RUN cd /opt \ - && git clone https://github.com/kbase/jars \ - && cd - +# Install jars for testing purposes +# Uncomment this if you want to run tests inside the ee2 container on MacOSX +# RUN cd /opt && git clone https://github.com/kbase/jars && cd - -# Remove due to cve-2021-4104 issue in spin (log4j) -RUN rm /opt/jars/lib/jars/dockerjava/docker-java-shaded-3.0.14.jar # Install DOCKERIZE RUN curl -o /tmp/dockerize.tgz https://raw.githubusercontent.com/kbase/dockerize/dist/dockerize-linux-amd64-v0.5.0.tar.gz && \ From b35919efacac94bccc1d64f6bd0976aadd923ab6 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Mon, 20 Jun 2022 23:42:17 -0500 Subject: [PATCH 39/43] Update RELEASE_NOTES.md --- RELEASE_NOTES.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 2b073d7bf..855352c76 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,14 @@ # execution_engine2 (ee2) release notes ========================================= +## 0.0.9 +* Update GHA with latest actions, remove old actions +* Change job defaults to result in +* NJS and KB_UPLOAD 5 Jobs +* Bigmem nodes with 250GB of ram take 1 job +* Bigmem nodes with 1TB of ram take 4 jobs +* Remove Jars from built image / cleanup for trivy + ## 0.0.81 * Updated HTCondor Clients, New Base Image * Use default GH actions From aee4d50dbb1b2cebc8869b4341e229e53ee58d5c Mon Sep 17 00:00:00 2001 From: Boris Sadkhin Date: Tue, 21 Jun 2022 00:15:20 -0500 Subject: [PATCH 40/43] Remove more jars --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1a244a3a3..13d8e9671 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,9 @@ RUN curl -fsSL https://get.htcondor.org | /bin/bash -s -- --no-dry-run # Install jars for testing purposes # Uncomment this if you want to run tests inside the ee2 container on MacOSX # RUN cd /opt && git clone https://github.com/kbase/jars && cd - - + +# Remove Jars for Trivy Scans +RUN rm -rf /sdk # Install DOCKERIZE RUN curl -o /tmp/dockerize.tgz https://raw.githubusercontent.com/kbase/dockerize/dist/dockerize-linux-amd64-v0.5.0.tar.gz && \ From bc229d4730d9548d56af83059b85f3445dce02cb Mon Sep 17 00:00:00 2001 From: Boris Date: Tue, 21 Jun 2022 00:27:43 -0500 Subject: [PATCH 41/43] Remove more jars --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 13d8e9671..185bea020 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,8 +18,6 @@ RUN curl -fsSL https://get.htcondor.org | /bin/bash -s -- --no-dry-run # Uncomment this if you want to run tests inside the ee2 container on MacOSX # RUN cd /opt && git clone https://github.com/kbase/jars && cd - -# Remove Jars for Trivy Scans -RUN rm -rf /sdk # Install DOCKERIZE RUN curl -o /tmp/dockerize.tgz https://raw.githubusercontent.com/kbase/dockerize/dist/dockerize-linux-amd64-v0.5.0.tar.gz && \ @@ -65,6 +63,10 @@ RUN mkdir -p /kb/module/work && chmod -R a+rw /kb/module && mkdir -p /etc/condor WORKDIR /kb/module RUN make all +# Remove Jars for Trivy Scans and after compilation is done +RUN rm -rf /sdk + + WORKDIR /kb/module/scripts RUN chmod +x download_runner.sh && ./download_runner.sh From 6a7c585efa31212832d1831e0c417b487fe943ba Mon Sep 17 00:00:00 2001 From: Boris Date: Tue, 21 Jun 2022 00:46:39 -0500 Subject: [PATCH 42/43] Remove python failures --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 185bea020..4520edb9d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,8 +63,8 @@ RUN mkdir -p /kb/module/work && chmod -R a+rw /kb/module && mkdir -p /etc/condor WORKDIR /kb/module RUN make all -# Remove Jars for Trivy Scans and after compilation is done -RUN rm -rf /sdk +# Remove Jars and old Conda for Trivy Scans and after compilation is done +RUN rm -rf /sdk && rm -rf /opt WORKDIR /kb/module/scripts From eb7c49a79920669d6bf5d6b7cfe483469b9705b1 Mon Sep 17 00:00:00 2001 From: Boris Date: Tue, 21 Jun 2022 01:09:54 -0500 Subject: [PATCH 43/43] Remove more python crup --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4520edb9d..02fcbf9d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,7 +65,7 @@ RUN make all # Remove Jars and old Conda for Trivy Scans and after compilation is done RUN rm -rf /sdk && rm -rf /opt - +RUN rm -rf /miniconda-latest/pkgs/conda-4.12.0-py39h06a4308_0/info/test/tests/data/env_metadata WORKDIR /kb/module/scripts RUN chmod +x download_runner.sh && ./download_runner.sh