D->M (#463)

0.0.12

    Forcing black to 22.1.0 to make sure that GHA doesn't suddenly fail
    Prevent jobs that never ran from submitting job execution stats

0.0.11

    Add ability for kbase user to contact condor via token

0.0.10

    Fixes bug with ee2 not recording all jobs with the catalog during the process
    of finishing a job
    Updates GHA with black and flake8
    Fix flake8 and black formatting issues by formatting MANY files
    Updated docs for installing htcondor
    Update many python libs in requirements.txt

Author: bio-boris

.flake8

@@ -1,4 +1,4 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# This workflow will install Python dependencies, run tests and lint
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
 # To ssh into this build add the following:
@@ -11,35 +11,61 @@
 name: Execution Engine 2 Test Suite
 
 on:
-  [pull_request]
+  [ pull_request ]
 
 jobs:
-  build:
+  Lint_with_Black:
     runs-on: ubuntu-latest
+    name: Lint With Black
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.8
-    - name: Lint with flake8 and black
-      run: |
-        python -m pip install --upgrade pip
-        pip install flake8 black pytest
-        flake8 ./lib ./test
-        black --check ./lib ./test
-    - name: Install dependencies
-      run: |
-        if [ -f requirements.txt ]; then pip install -r requirements-dev.txt; fi
-        cd /opt
-        git clone https://github.com/kbase/jars
-        cd -
-    - name: Build Docker Image
-      run: |
-        docker build . -t execution_engine2:test
-    - name: Run Tests
-      run: |
-        docker-compose up -d
-        cp test/env/test.travis.env test.env
-        make test-coverage
-        codecov
+      - uses: actions/checkout@v3
+      - uses: psf/black@stable
+        with:
+          options: "--check --verbose"
+          src: "./lib"
+          version: "22.10.0"
+      - uses: psf/black@stable
+        with:
+          options: "--check --verbose"
+          src: "./test"
+          version: "22.10.0"
+
+  Lint_with_Flake8:
+    runs-on: ubuntu-latest
+    name: Lint With Flake8
+    steps:
+      - name: Check out source repository
+        uses: actions/checkout@v3
+      - name: Set up Python environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+      - name: flake8 Lint Lib
+        uses: py-actions/flake8@v2
+        with:
+          path: "./lib"
+      - name: flake8 Lint Test
+        uses: py-actions/flake8@v2
+        with:
+          path: "./test"
+
+
+  Build_and_Run_Tests_and_CodeCov:
+    name: Build and Run Tests and CodeCov
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out source repository
+        uses: actions/checkout@v3
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          git clone https://github.com/kbase/jars /opt/jars
+      - name: Build Docker Image
+        run: |
+          docker build . -t execution_engine2:test
+      - name: Run Tests
+        run: |
+          docker-compose up -d
+          cp test/env/test.travis.env test.env
+          make test-coverage
+          codecov

.github/workflows/ee2-tests.yml

@@ -10,7 +10,7 @@ on:
       - opened
       - reopened
       - synchronize
-      - merged
+      - closed
 jobs:
   build-develop-open:
     if: github.base_ref == 'develop' && github.event.pull_request.merged == false

.github/workflows/pr_build.yml

@@ -1,9 +1,9 @@
 repos:
 -   repo: https://github.com/ambv/black
-    rev: 22.1.0
+    rev: 22.10.0
     hooks:
     - id: black
-      exclude: '.+Impl.py'
+      exclude: ".*execution_engine2Impl.py"
 -   repo: https://gitlab.com/pycqa/flake8
     rev: '3.9.2'
     hooks:

.pre-commit-config.yaml

@@ -61,7 +61,9 @@ COPY ./ /kb/module
 RUN mkdir -p /kb/module/work && chmod -R a+rw /kb/module && mkdir -p /etc/condor/
 
 WORKDIR /kb/module
-RUN make all
+# Due to older kb-sdk in this base image, getting some compilation results we don't want
+# Will have to manually use the correct version of kbase-sdk to compile impl/Server files
+RUN make build
 
 # Remove Jars and old Conda for Trivy Scans and after compilation is done
 RUN rm -rf /sdk && rm -rf /opt

Dockerfile

@@ -21,6 +21,7 @@ compile:
 		--pysrvname $(SERVICE_CAPS).$(SERVICE_CAPS)Server \
 		--pyimplname $(SERVICE_CAPS).$(SERVICE_CAPS)Impl;
 
+
 	kb-sdk compile $(SPEC_FILE) \
 		--out . \
 		--html \
@@ -41,4 +42,4 @@ test-coverage:
 
 build-condor-test-image:
 	cd test/dockerfiles/condor && echo `pwd` && docker build -f Dockerfile . -t $(CONDOR_DOCKER_IMAGE_TAG_NAME)
-	docker push $(CONDOR_DOCKER_IMAGE_TAG_NAME)
+	docker push $(CONDOR_DOCKER_IMAGE_TAG_NAME)

Makefile

@@ -15,7 +15,7 @@ cachetools = "==3.1.1"
 certifi = "==2019.6.16"
 cffi = "==1.14.0"
 chardet = "==3.0.4"
-codecov = "==2.0.15"
+codecov = "==2.0.16"
 configparser = "==3.7.4"
 confluent-kafka = "==1.5.0"
 coverage = "==4.5.3"

Pipfile

@@ -67,11 +67,7 @@ pre-commit uninstall
 ```
 
 ## Installing HTCondor Bindings from the mac
-* You may not be able to load without disabling the mac Security Gatekeeper with `sudo spctl --master-disable`
-* The HTCondor bindings only work on the Python.org install of python or your system install of python2.7. They will not work with anaconda. So download python from python.org
-* Download the mac bindings at https://research.cs.wisc.edu/htcondor/tarball/current/8.9.10/release/
-* Current version is [8.9.10](https://research.cs.wisc.edu/htcondor/tarball/current/8.9.10/release/condor-8.9.10-x86_64_MacOSX-unstripped.tar.gz)
-* Add <condor>/lib/python3 to PYTHONPATH.
+* `conda install -c conda-forge python-htcondor`
 * `import htcondor`
 
 ## Test Running Options  

Pipfile.lock

@@ -1,6 +1,23 @@
 # execution_engine2 (ee2) release notes
 =========================================
 
+## 0.0.12
+* Forcing black to 22.1.0 to make sure that GHA doesn't suddenly fail
+* Prevent jobs that never ran from submitting job execution stats
+
+
+## 0.0.11
+* Add ability for `kbase` user to contact condor via token
+
+## 0.0.10
+* Fixes bug with ee2 not recording all jobs with the catalog during the process 
+of finishing a job
+* Updates GHA with black and flake8
+* Fix flake8 and black formatting issues by formatting MANY files
+* Updated docs for installing htcondor
+* Update many python libs in requirements.txt
+
+
 ## 0.0.9
 * Update GHA with latest actions, remove old actions
 * Change job defaults to result in