D->M (#463)

0.0.12

    Forcing black to 22.1.0 to make sure that GHA doesn't suddenly fail
    Prevent jobs that never ran from submitting job execution stats

0.0.11

    Add ability for kbase user to contact condor via token

0.0.10

    Fixes bug with ee2 not recording all jobs with the catalog during the process
    of finishing a job
    Updates GHA with black and flake8
    Fix flake8 and black formatting issues by formatting MANY files
    Updated docs for installing htcondor
    Update many python libs in requirements.txt

.github/workflows/ee2-tests.yml

@@ -1,4 +1,4 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# This workflow will install Python dependencies, run tests and lint
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
 # To ssh into this build add the following:
@@ -11,35 +11,61 @@
 name: Execution Engine 2 Test Suite
 
 on:
-  [pull_request]
+  [ pull_request ]
 
 jobs:
-  build:
+  Lint_with_Black:
     runs-on: ubuntu-latest
+    name: Lint With Black
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.8
-    - name: Lint with flake8 and black
-      run: |
-        python -m pip install --upgrade pip
-        pip install flake8 black pytest
-        flake8 ./lib ./test
-        black --check ./lib ./test
-    - name: Install dependencies
-      run: |
-        if [ -f requirements.txt ]; then pip install -r requirements-dev.txt; fi
-        cd /opt
-        git clone https://github.com/kbase/jars
-        cd -
-    - name: Build Docker Image
-      run: |
-        docker build . -t execution_engine2:test
-    - name: Run Tests
-      run: |
-        docker-compose up -d
-        cp test/env/test.travis.env test.env
-        make test-coverage
-        codecov
+      - uses: actions/checkout@v3
+      - uses: psf/black@stable
+        with:
+          options: "--check --verbose"
+          src: "./lib"
+          version: "22.10.0"
+      - uses: psf/black@stable
+        with:
+          options: "--check --verbose"
+          src: "./test"
+          version: "22.10.0"
+
+  Lint_with_Flake8:
+    runs-on: ubuntu-latest
+    name: Lint With Flake8
+    steps:
+      - name: Check out source repository
+        uses: actions/checkout@v3
+      - name: Set up Python environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+      - name: flake8 Lint Lib
+        uses: py-actions/flake8@v2
+        with:
+          path: "./lib"
+      - name: flake8 Lint Test
+        uses: py-actions/flake8@v2
+        with:
+          path: "./test"
+
+
+  Build_and_Run_Tests_and_CodeCov:
+    name: Build and Run Tests and CodeCov
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out source repository
+        uses: actions/checkout@v3
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          git clone https://github.com/kbase/jars /opt/jars
+      - name: Build Docker Image
+        run: |
+          docker build . -t execution_engine2:test
+      - name: Run Tests
+        run: |
+          docker-compose up -d
+          cp test/env/test.travis.env test.env
+          make test-coverage
+          codecov

.github/workflows/pr_build.yml

@@ -10,7 +10,7 @@ on:
       - opened
       - reopened
       - synchronize
-      - merged
+      - closed
 jobs:
   build-develop-open:
     if: github.base_ref == 'develop' && github.event.pull_request.merged == false

.pre-commit-config.yaml

@@ -1,9 +1,9 @@
 repos:
 -   repo: https://github.com/ambv/black
-    rev: 22.1.0
+    rev: 22.10.0
     hooks:
     - id: black
-      exclude: '.+Impl.py'
+      exclude: ".*execution_engine2Impl.py"
 -   repo: https://gitlab.com/pycqa/flake8
     rev: '3.9.2'
     hooks:

Dockerfile

@@ -61,7 +61,9 @@ COPY ./ /kb/module
 RUN mkdir -p /kb/module/work && chmod -R a+rw /kb/module && mkdir -p /etc/condor/
 
 WORKDIR /kb/module
-RUN make all
+# Due to older kb-sdk in this base image, getting some compilation results we don't want
+# Will have to manually use the correct version of kbase-sdk to compile impl/Server files
+RUN make build
 
 # Remove Jars and old Conda for Trivy Scans and after compilation is done
 RUN rm -rf /sdk && rm -rf /opt

Makefile

@@ -21,6 +21,7 @@ compile:
 		--pysrvname $(SERVICE_CAPS).$(SERVICE_CAPS)Server \
 		--pyimplname $(SERVICE_CAPS).$(SERVICE_CAPS)Impl;
 
+
 	kb-sdk compile $(SPEC_FILE) \
 		--out . \
 		--html \
@@ -41,4 +42,4 @@ test-coverage:
 
 build-condor-test-image:
 	cd test/dockerfiles/condor && echo `pwd` && docker build -f Dockerfile . -t $(CONDOR_DOCKER_IMAGE_TAG_NAME)
-	docker push $(CONDOR_DOCKER_IMAGE_TAG_NAME)
+	docker push $(CONDOR_DOCKER_IMAGE_TAG_NAME)

Pipfile

@@ -15,7 +15,7 @@ cachetools = "==3.1.1"
 certifi = "==2019.6.16"
 cffi = "==1.14.0"
 chardet = "==3.0.4"
-codecov = "==2.0.15"
+codecov = "==2.0.16"
 configparser = "==3.7.4"
 confluent-kafka = "==1.5.0"
 coverage = "==4.5.3"

README.md

@@ -67,11 +67,7 @@ pre-commit uninstall
 ```
 
 ## Installing HTCondor Bindings from the mac
-* You may not be able to load without disabling the mac Security Gatekeeper with `sudo spctl --master-disable`
-* The HTCondor bindings only work on the Python.org install of python or your system install of python2.7. They will not work with anaconda. So download python from python.org
-* Download the mac bindings at https://research.cs.wisc.edu/htcondor/tarball/current/8.9.10/release/
-* Current version is [8.9.10](https://research.cs.wisc.edu/htcondor/tarball/current/8.9.10/release/condor-8.9.10-x86_64_MacOSX-unstripped.tar.gz)
-* Add <condor>/lib/python3 to PYTHONPATH.
+* `conda install -c conda-forge python-htcondor`
 * `import htcondor`
 
 ## Test Running Options  

RELEASE_NOTES.md

@@ -1,6 +1,23 @@
 # execution_engine2 (ee2) release notes
 =========================================
 
+## 0.0.12
+* Forcing black to 22.1.0 to make sure that GHA doesn't suddenly fail
+* Prevent jobs that never ran from submitting job execution stats
+
+
+## 0.0.11
+* Add ability for `kbase` user to contact condor via token
+
+## 0.0.10
+* Fixes bug with ee2 not recording all jobs with the catalog during the process 
+of finishing a job
+* Updates GHA with black and flake8
+* Fix flake8 and black formatting issues by formatting MANY files
+* Updated docs for installing htcondor
+* Update many python libs in requirements.txt
+
+
 ## 0.0.9
 * Update GHA with latest actions, remove old actions
 * Change job defaults to result in 

lib/execution_engine2/authorization/authstrategy.py

@@ -2,7 +2,7 @@
 A module with commands for checking user privileges for jobs.
 This doesn't include checking admin rights.
 """
-from typing import Dict, List
+from typing import List
 from lib.execution_engine2.authorization.workspaceauth import WorkspaceAuth
 from lib.execution_engine2.db.models.models import Job
 from collections import defaultdict

lib/execution_engine2/db/models/models.py

@@ -414,9 +414,9 @@ def save(self, *args, **kwargs):
         return super(HeldJob, self).save(*args, **kwargs)
 
 
-###
-### Unused fields that we might want
-###
+#
+# Unused fields that we might want
+#
 
 result_example = {
     "shocknodes": [],
@@ -442,9 +442,9 @@ def save(self, *args, **kwargs):
 }
 
 
-####
-#### Unused Stuff to look at
-####
+#
+# Unused Stuff to look at
+#
 
 
 class Results(EmbeddedDocument):

lib/execution_engine2/db/models/transfer/check_jobs_for_finish_times.py

@@ -1,15 +1,15 @@
 #!/usr/bin/env python
-# type: ignore
+from bson import ObjectId
+
 try:
     from .transfer_ujs_njs import MigrateDatabases
     from lib.execution_engine2.db.models.models import Status, valid_status
 except Exception:
     from transfer_ujs_njs import MigrateDatabases
-    from models import *
+    from models import Status, valid_status
 
 
 ee2_jobs = MigrateDatabases().ee2_jobs
-from bson import ObjectId
 
 count = 0
 for job in ee2_jobs.find():

lib/execution_engine2/db/models/transfer/check_njs_jobs_for_ujs_id.py

@@ -1,20 +1,17 @@
 #!/usr/bin/env python
 # type: ignore
+from collections import defaultdict
+
+from bson import ObjectId
+
 try:
     from .transfer_ujs_njs import MigrateDatabases
-    from lib.execution_engine2.db.models.models import Status, valid_status
 except Exception:
     from transfer_ujs_njs import MigrateDatabases
-    from models import *
-from bson import ObjectId
-
-from pprint import pprint
 
 njs_jobs_db = MigrateDatabases().njs_jobs
 ujs_jobs_db = MigrateDatabases().ujs_jobs
 
-from collections import defaultdict
-
 count = 0
 missing_ujs = []
 c = defaultdict(int)
@@ -27,7 +24,6 @@
         print(f"Couldn't find {job_id}, ")
         missing_ujs.append(job_id)
 
-
 print("Max occurences", max(c.values()))
 
 print("Number of njs jobs", count)

lib/execution_engine2/db/models/transfer/fix_transfer_ujs_njs.py

@@ -1,19 +1,17 @@
 #!/usr/bin/env python
-# type: ignore
 import os
 from collections import Counter
 from configparser import ConfigParser
+from datetime import datetime
 
-jobs_database_name = "ee2_jobs"
 from mongoengine import connect
 
-from datetime import datetime
-
 try:
-    from lib.execution_engine2.db.models.models import Job, Status, JobInput
-
+    from lib.execution_engine2.db.models.models import Job, Status
 except Exception:
-    from models import Status, Job, JobInput
+    from models import Status, Job
+
+jobs_database_name = "ee2_jobs"
 
 
 class FixEE2JobsDatabase: