Merge pull request #58 from kbase/scanon/gha

Add Github Actions Round 1

.github/dependabot.yml

@@ -0,0 +1,23 @@
+version: 2
+updates:
+
+  # Docker
+  - package-ecosystem: docker
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    open-pull-requests-limit: 25
+
+  # Python
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "monthly"
+    open-pull-requests-limit: 25
+
+  # GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: ".github/workflows"
+    schedule:
+      interval: "monthly"
+    open-pull-requests-limit: 25

.github/workflows/codeql.yml

@@ -0,0 +1,38 @@
+name: "CodeQL"
+
+on:
+  pull_request:
+    types:
+    - opened
+    - reopened
+    - synchronize
+    - ready_for_review
+  push:
+    # run workflow when merging to main or develop
+    branches:
+      - main
+      - master
+      - develop
+
+jobs:
+  CodeQL-Build:
+    # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest
+    runs-on: ubuntu-latest
+
+    permissions:
+      # required for all workflows
+      security-events: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        # Override language selection by uncommenting this and choosing your languages
+        with:
+          languages: python
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2

.github/workflows/manual-build.yml

@@ -0,0 +1,11 @@
+---
+name: Manual Build & Push
+on:
+ workflow_dispatch:
+jobs:
+  build-push:
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}-develop'
+      tags: br-${{ github.ref_name }}
+    secrets: inherit

.github/workflows/pr_build.yml

@@ -0,0 +1,43 @@
+---
+name: Pull Request Build, Tag, & Push
+on:
+  pull_request:
+    branches:
+      - develop
+      - main
+      - master
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - closed
+jobs:
+  build-develop-open:
+    if: github.base_ref == 'develop' && github.event.pull_request.merged == false
+    uses: kbase/.github/.github/workflows/reusable_build.yml@main
+    secrets: inherit
+  build-develop-merge:
+    if: github.base_ref == 'develop' && github.event.pull_request.merged == true
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: 'jobrunner-develop'
+      tags: pr-${{ github.event.number }},latest
+    secrets: inherit
+  build-main-open:
+    if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == false
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: jobrunner
+      tags: pr-${{ github.event.number }}
+    secrets: inherit
+  build-main-merge:
+    if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == true
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: jobrunner
+      tags: pr-${{ github.event.number }},latest-rc
+    secrets: inherit
+  trivy-scans:
+    if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false
+    uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main
+    secrets: inherit

.github/workflows/release-main.yml

@@ -0,0 +1,25 @@
+---
+name: Release - Build & Push Image
+on:
+  release:
+    branches:
+      - main
+      - master
+    types: [ published ]
+jobs:
+  check-source-branch:
+    uses: kbase/.github/.github/workflows/reusable_validate-branch.yml@main
+    with:
+      build_branch: '${{ github.event.release.target_commitish }}'
+  validate-release-tag:
+    needs: check-source-branch
+    uses: kbase/.github/.github/workflows/reusable_validate-release-tag.yml@main
+    with:
+      release_tag: '${{ github.event.release.tag_name }}'
+  build-push:
+    needs: validate-release-tag
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}'
+      tags: '${{ github.event.release.tag_name }},latest'
+    secrets: inherit

.github/workflows/test.yml

@@ -0,0 +1,65 @@
+name: KBase JobRunner tests
+
+on:
+  pull_request:
+    types:
+    - opened
+    - reopened
+    - synchronize
+    - ready_for_review
+  push:
+    # run workflow when merging to main or develop
+    branches:
+      - main
+      - master
+      - develop
+      - scanon/gha
+
+jobs:
+
+  jobsrunner_tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9"]
+
+    steps:
+    - name: Install Docker
+      uses: docker-practice/actions-setup-docker@master
+      timeout-minutes: 12
+
+    - name: Repo checkout
+      uses: actions/checkout@v3
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependecies
+      # tried VaultVulp/action-pipenv but pytest wasn't on the path post action
+      shell: bash
+      run: |
+        pip install -r requirements.txt
+        pip install -r requirements-dev.txt
+        pip freeze
+        make mock
+        docker pull kbase/runtester
+        docker tag kbase/runtester test/runtester
+        curl -q -L -o $HOME/cromwell-44.jar https://github.com/broadinstitute/cromwell/releases/download/44/cromwell-44.jar
+        touch ~/cromwell.conf
+        ulimit -a
+     
+
+    - name: Run tests
+      shell: bash
+      env:
+          KB_AUTH_TOKEN: ${{ secrets.KBASE_CI_TOKEN  }}
+          KB_ADMIN_AUTH_TOKEN: ${{ secrets.KBASE_CI_TOKEN  }}
+          KB_BASE_URL: https://ci.kbase.us/services
+      run: make test
+
+#    - name: Upload coverage to Codecov
+#      uses: codecov/codecov-action@v3
+#      with:
+#        fail_ci_if_error: true

Dockerfile

@@ -7,4 +7,6 @@ WORKDIR "/src"
 RUN pip install -r requirements.txt
 ADD . /src
 
+EXPOSE 9999
+
 CMD [ "python3", "-m", "JobRunner.Callback"]

requirements-dev.txt

@@ -1,2 +1,5 @@
 nose==1.3.7
-aiohttp
+aiohttp==3.8.4
+nose-cov
+sanic-testing==0.8.3
+

requirements.txt

@@ -1,2 +1,7 @@
 sanic==21.9.3
 docker==3.6.0
+websockets==10.0
+requests==2.27.1
+uvloop==0.17.0
+websocket-client==1.6.1
+

test/test_jobrunner.py

@@ -3,6 +3,7 @@
 import unittest
 from copy import deepcopy
 from time import time as _time
+from time import sleep
 from unittest.mock import patch, MagicMock
 
 from nose.plugins.attrib import attr
@@ -72,8 +73,6 @@ def setUpClass(cls):
         if "KB_ADMIN_AUTH_TOKEN" not in os.environ:
             os.environ["KB_ADMIN_AUTH_TOKEN"] = "bogus"
 
-
-
     def _cleanup(self, job):
         d = os.path.join(self.workdir, "workdir")
         if os.path.exists(d):
@@ -82,16 +81,15 @@ def _cleanup(self, job):
                     os.unlink(os.path.join(d, fn))
             try:
                 os.rmdir(d)
-            except:
+            except OSError:
                 pass
 
-    @attr("offline")
+    @attr("online")
     @patch("JobRunner.JobRunner.KBaseAuth", autospec=True)
     @patch("JobRunner.JobRunner.EE2", autospec=True)
     def test_run_sub(self, mock_ee2, mock_auth):
         """
-        This test is expected to run for 50-60 seconds?
-
+        Test running a subjob
         """
         self._cleanup(self.jobid)
         params = deepcopy(EE2_JOB_PARAMS)
@@ -118,6 +116,7 @@ def test_run_sub(self, mock_ee2, mock_auth):
         out = jr.run()
         self.assertIn("result", out)
         self.assertNotIn("error", out)
+        sleep(1)
 
     @attr("offline")
     @patch("JobRunner.JobRunner.KBaseAuth", autospec=True)
@@ -169,7 +168,7 @@ def test_run_volume(self, mock_ee2, mock_auth):
         jr.cc.catalog.get_secure_config_params = MagicMock(return_value=None)
         jr.logger.ee2.add_job_logs = MagicMock(return_value=rv)
         jr.ee2.get_job_params.return_value = params
-        jr.ee2.list_config.return_value = EE2_LIST_CONFIG
+        jr.ee2.list_config.return_value = deepcopy(EE2_LIST_CONFIG)
         jr.ee2.check_job_canceled.return_value = {"finished": False}
         jr.auth.get_user.return_value = "bogus"
         jr._get_token_lifetime = MagicMock(return_value=self.future)
@@ -182,31 +181,30 @@ def test_run_volume(self, mock_ee2, mock_auth):
         self.assertIn("result", out)
         self.assertNotIn("error", out)
 
-    @attr("offline")
+    @attr("online")
     @patch("JobRunner.JobRunner.KBaseAuth", autospec=True)
     @patch("JobRunner.JobRunner.EE2", autospec=True)
     def test_cancel(self, mock_ee2, mock_auth):
         """
-        This test is expected to run for 30 seconds?
-
+        Test canceling a job
         """
         self._cleanup(self.jobid)
         params = deepcopy(EE2_JOB_PARAMS)
         params["method"] = "RunTester.run_RunTester"
         params["params"] = [{"depth": 3, "size": 1000, "parallel": 4}]
         jr = JobRunner(
-            self.config, self.ee2_url, self.jobid, self.token, self.admin_token
+            self.config, self.ee2_url, self.jobid, self.token, self.admin_token,
         )
         rv = deepcopy(CATALOG_GET_MODULE_VERSION)
         rv["docker_img_name"] = "test/runtester:latest"
-        jr._get_cgroup = MagicMock(return_value='cgroup')
+        jr._get_cgroup = MagicMock(return_value=None)
         jr.cc.catalog.get_module_version = MagicMock(return_value=rv)
         jr.cc.catalog.list_volume_mounts = MagicMock(return_value=[])
         jr.cc.catalog.get_secure_config_params = MagicMock(return_value=None)
         jr.logger.ee2.add_job_logs = MagicMock(return_value=rv)
         jr._get_token_lifetime = MagicMock(return_value=self.future)
         jr.ee2.get_job_params.return_value = params
-        jr.ee2.list_config.return_value = EE2_LIST_CONFIG
+        jr.ee2.list_config.return_value = deepcopy(EE2_LIST_CONFIG)
         nf = {"finished": False}
         jr.ee2.check_job_canceled.side_effect = [nf, nf, nf, nf, nf, {"finished": True}]
         jr.auth.get_user.return_value = "bogus"
@@ -241,7 +239,7 @@ def test_max_jobs(self, mock_ee2, mock_auth):
         jr.logger.ee2.add_job_logs = MagicMock(return_value=rv)
         jr._get_token_lifetime = MagicMock(return_value=self.future)
         jr.ee2.get_job_params.return_value = params
-        jr.ee2.list_config.return_value = EE2_LIST_CONFIG
+        jr.ee2.list_config.return_value = deepcopy(EE2_LIST_CONFIG)
         jr.ee2.check_job_canceled.return_value = {"finished": False}
         jr.auth.get_user.return_value = "bogus"
         out = jr.run()
@@ -361,7 +359,7 @@ def test_expire_loop(self, mock_auth, mock_ee2):
         jr._get_cgroup = MagicMock(return_value=None)
         jr.logger.ee2.add_job_logs = MagicMock(return_value=rv)
         jr.ee2.get_job_params.return_value = params
-        jr.ee2.list_config.return_value = EE2_LIST_CONFIG
+        jr.ee2.list_config.return_value = deepcopy(EE2_LIST_CONFIG)
         jr.ee2.check_job_canceled.return_value = {"finished": False}
         jr.auth.get_user.return_value = "bogus"
         jr._get_token_lifetime = MagicMock(return_value=_time())
@@ -386,7 +384,7 @@ def test_special(self, mock_ee2, mock_auth):
         params["params"] = [{"submit_script": "submit.sl"}]
         jr._submit_special(self.config, "1234", params)
 
-    @attr("offline")
+    @attr("online")
     @patch("JobRunner.JobRunner.KBaseAuth", autospec=True)
     @patch("JobRunner.JobRunner.EE2", autospec=True)
     def test_run_slurm(self, mock_ee2, mock_auth):
@@ -415,7 +413,7 @@ def test_run_slurm(self, mock_ee2, mock_auth):
         out = jr.run()
         self.assertNotIn("error", out)
 
-    @attr("offline")
+    @attr("online")
     @patch("JobRunner.JobRunner.KBaseAuth", autospec=True)
     @patch("JobRunner.JobRunner.EE2", autospec=True)
     def test_run_wdl(self, mock_ee2, mock_auth):