v0.8.0

CHANGELOG.md

@@ -1,5 +1,14 @@
 # CHANGELOG
 
+## v0.8.0 (2020-12-20)
+
+* Refactored the `image` module and added unit tests
+* Added a fallback variable for `MODE` set to `production`
+* Created `conftest` file for test suite, started shifting fixtures around
+* Bumped Docker API version from `1.40` to `1.41`, there should be no change in behavior
+* Fixed a bug where if a container didn't exist yet, it would still try to wait, stop, and remove it on the deploy stage. The output would also blow up as it was impossible to do because it didn't exist. Now we check if a container exists prior to running those commands on the deploy stage and skip if no container exists
+* Various bug fixes and optimizations
+
 ## v0.7.0 (2020-10-26)
 
 * Refactored the `webhook` module and added unit tests

Makefile

@@ -32,10 +32,14 @@ lint:
 	venv/bin/flake8 harvey/*.py
 	venv/bin/flake8 test/*.py
 
-## test - Test the project
+## test - Test the project (unit tests)
 test:
 	venv/bin/pytest
 
+## integration test - Test the project (integration tests)
+integration_test:
+	venv/bin/python test/integration/test_pipeline.py
+
 ## coverage - Test the project and generate an HTML coverage report
 coverage:
 	venv/bin/pytest --cov=harvey --cov-branch --cov-report=html

README.md

@@ -102,7 +102,7 @@ Environment Variables:
     SLACK_CHANNEL   The Slack channel to send messages to
     SLACK_BOT_TOKEN The Slackbot token to use to authenticate each request to Slack
     WEBHOOK_SECRET  The Webhook secret required by GitHub (if enabled) to secure your webhooks
-    MODE            Set to "test" to bypass the header and auth data from GitHub to test
+    MODE            Set to "test" to bypass the header and auth data from GitHub to test. Default: production
     HOST            The host Harvey will run on. Default: 127.0.0.1
     PORT            The port Harvey will run on. Default: 5000
     DEBUG           Whether the Flask API will run in debug mode or not

docs/README.md

@@ -14,7 +14,9 @@
 - Each `harvey.json` config file will house information about your tests and build/deploy
 - This file must follow proper JSON standards (start and end with `{ }`, contain commas after each item, no trailing commas, and be surrounded by quotes)
 
-The following example will run a full pipeline (tests, build and deploy), tag it with a unique name based on the GitHub project. Provide the language and version for the test stage:
+The following example will run a full pipeline (tests, build and deploy), tag it with a unique name based on the GitHub project. Provide the language and version for the test stage.
+
+**Note:** All keys must be lowercase!
 
 ```javascript
 {

examples/examples.py

@@ -4,19 +4,19 @@
 
 
 # """Retrieve a single image"""
-# image = harvey.Image.retrieve('harvey/python-test')
+# image = harvey.Image.retrieve_image('harvey/python-test')
 # print(image)
 
 # """Retrieve a list of images"""
-# images = harvey.Image.all()
+# images = harvey.Image.retrieve_all_images()
 # print(json.dumps(images, indent=4))
 
 # """Remove an image"""
-# remove_image = harvey.Image.remove('c1d7538e38f74ea6ba43920eaabd27b8')
+# remove_image = harvey.Image.remove_image('c1d7538e38f74ea6ba43920eaabd27b8')
 # print(remove_image)
 
 # """Build an image"""
-# image = harvey.Image.build(
+# image = harvey.build_image(
 #     config={
 #         'language': 'python',
 #         'version': '3.7',

harvey/app.py

@@ -45,8 +45,8 @@ def start_pipeline_compose():
 def retrieve_pipeline(pipeline_id):
     """Retrieve a pipeline's logs by ID
     """
-    # This is a hacky temporary solution until we can store
-    # this data in a database and is not meant to remain
+    # TODO: This is a hacky temporary solution until we can
+    # store this data in a database and is not meant to remain
     # as a long-term solution
     file = f'{pipeline_id}.log'
     for root, dirs, files in os.walk(Global.PROJECTS_LOG_PATH):
@@ -61,8 +61,8 @@ def retrieve_pipeline(pipeline_id):
 def retrieve_pipelines():
     """Retrieve a list of pipelines
     """
-    # This is a hacky temporary solution until we can store
-    # this data in a database and is not meant to remain
+    # TODO: This is a hacky temporary solution until we can
+    # store this data in a database and is not meant to remain
     # as a long-term solution
     pipelines = {'pipelines': []}
     for root, dirs, files in os.walk(Global.PROJECTS_LOG_PATH, topdown=True):
@@ -144,28 +144,28 @@ def retrieve_pipelines():
 #     data = json.loads(request.data)
 #     tag = json.loads(request.tag)
 #     context = json.loads(request.context)
-#     build = harvey.Image.build(data, tag, context)
+#     build = harvey.build_image(data, tag, context)
 #     return build
 
 
 # @API.route('/images/<image_id>', methods=['GET'])
 # def retrieve_image(image_id):
 #     """Retrieve a Docker image"""
-#     response = json.dumps(harvey.Image.retrieve(image_id))
+#     response = json.dumps(harvey.Image.retrieve_image(image_id))
 #     return response
 
 
 # @API.route('/images', methods=['GET'])
 # def all_images():
 #     """Retrieve all Docker images"""
-#     response = json.dumps(harvey.Image.all())
+#     response = json.dumps(harvey.Image.retrieve_all_images())
 #     return response
 
 
 # @API.route('/images/<image_id>/remove', methods=['DELETE'])
 # def remove_image(image_id):
 #     """Remove (delete) a Docker image"""
-#     remove = harvey.Image.remove(image_id)
+#     remove = harvey.Image.remove_image(image_id)
 #     response = str(remove)
 #     return response
 

harvey/globals.py

@@ -1,9 +1,12 @@
+import os
+
+
 class Global():
     """Contains global configuration for Harvey
     """
-    DOCKER_VERSION = 'v1.40'  # Docker API version
+    DOCKER_VERSION = 'v1.41'  # Docker API version
     # TODO: Figure out how to sync this version number with the one in `setup.py`
-    HARVEY_VERSION = '0.7.0'  # Harvey release
+    HARVEY_VERSION = '0.8.0'  # Harvey release
     PROJECTS_PATH = 'projects'
     PROJECTS_LOG_PATH = 'logs/projects'
     HARVEY_LOG_PATH = 'logs/harvey'
@@ -12,6 +15,7 @@ class Global():
     BASE_URL = f'http+unix://%2Fvar%2Frun%2Fdocker.sock/{DOCKER_VERSION}/'
     JSON_HEADERS = {'Content-Type': 'application/json'}
     TAR_HEADERS = {'Content-Type': 'application/tar'}
+    APP_MODE = os.getenv('MODE', 'production').lower()
 
     @classmethod
     def repo_name(cls, webhook):

harvey/image.py

@@ -1,4 +1,3 @@
-import json
 import os
 import subprocess
 import requests
@@ -7,8 +6,8 @@
 
 class Image():
     @classmethod
-    def build(cls, config, webhook, context=''):
-        """Build a Docker image
+    def build_image(cls, config, webhook, context=''):
+        """Build a Docker image by shelling out and running Docker commands.
         """
         # TODO: Use the Docker API for building instead of a shell \
         # command (haven't because I can't get it working)
@@ -17,61 +16,53 @@ def build(cls, config, webhook, context=''):
         # data = requests.post(Global.BASE_URL + 'build', \
         # params=json, data=tar, headers=Global.TAR_HEADERS)
 
-        # Global variables
-        if "dockerfile" in config:
-            dockerfile = f'-f {config["dockerfile"]}'
-        else:
-            dockerfile = ''
-
         # Set variables based on the context (test vs deploy vs full)
         if context == 'test':
+            language = f'--build-arg LANGUAGE={config.get("language", "")}'
+            version = f'--build-arg VERSION={config.get("version", "")}'
             project = f'--build-arg PROJECT={Global.repo_full_name(webhook)}'
-            path = Global.PROJECTS_PATH
-        else:
-            project = ''
-            path = os.path.join(Global.PROJECTS_PATH,
-                                Global.repo_full_name(webhook))
-
-        tag_arg = f'-t {Global.docker_project_name(webhook)}'
-
-        # For testing only:
-        if "language" in config and context == 'test':
-            language = f'--build-arg LANGUAGE={config["language"]}'
+            path = f'{Global.PROJECTS_PATH}'
         else:
             language = ''
-        if "version" in config and context == 'test':
-            version = f'--build-arg VERSION={config["version"]}'
-        else:
             version = ''
+            project = ''
+            path = os.path.join(Global.PROJECTS_PATH, Global.repo_full_name(webhook))
+        dockerfile = f'-f {config["dockerfile"]}' if config.get("dockerfile") else ''
+        tag_arg = f'-t {Global.docker_project_name(webhook)}'
 
-        # Build the image (exceptions handled at stage level)
+        # Build the image (exceptions bubble up to the stage module)
+        # We cd into the directory here so we have access to the files to copy into the container
         image = subprocess.check_output(
             f'cd {path} && docker build {dockerfile} {tag_arg} {language} {version} {project} .',
-            stdin=None, stderr=None, shell=True, timeout=Global.BUILD_TIMEOUT)
+            stdin=None,
+            stderr=None,
+            shell=True,
+            timeout=Global.BUILD_TIMEOUT
+        )
 
         return image.decode('UTF-8')
 
     @classmethod
-    def retrieve(cls, image_id):
+    def retrieve_image(cls, image_id):
         """Retrieve a Docker image
         """
-        data = requests.get(Global.BASE_URL + f'images/{image_id}/json')
-        return data.json()
+        response = requests.get(Global.BASE_URL + f'images/{image_id}/json')
+        return response
 
     @classmethod
-    def all(cls):
+    def retrieve_all_images(cls):
         """Retrieve all Docker images
         """
-        data = requests.get(Global.BASE_URL + 'images/json')
-        return data.json()
+        response = requests.get(Global.BASE_URL + 'images/json')
+        return response
 
     @classmethod
-    def remove(cls, image_id):
+    def remove_image(cls, image_id):
         """Remove (delete) a Docker image
         """
-        data = requests.delete(
+        response = requests.delete(
             Global.BASE_URL + f'images/{image_id}',
-            data=json.dumps({'force': True}),
+            json={'force': True},
             headers=Global.JSON_HEADERS
         )
-        return data
+        return response

harvey/pipeline.py

@@ -9,7 +9,7 @@
 class Pipeline():
     @classmethod
     def test(cls, config, webhook, output):
-        """Pull changes and run tests (no deploy)
+        """Pull changes and run tests (will not deploy code)
         """
         start_time = datetime.now()
         if os.getenv('SLACK'):

harvey/stage.py

@@ -19,7 +19,7 @@ def test(cls, config, webhook, output):
 
         # Build the image
         try:
-            image = Image.build(config, webhook, context)
+            image = Image.build_image(config, webhook, context)
             image_output = f'Test image created.\n{image}'
             print(image_output)
         except subprocess.TimeoutExpired:
@@ -40,7 +40,7 @@ def test(cls, config, webhook, output):
         else:
             final_output = output + image_output + \
                 '\nError: Harvey could not create the Test container.'
-            Image.remove(test_project_name)
+            Image.remove_image(test_project_name)
             Utils.kill(final_output, webhook)
 
         # Start the container
@@ -51,7 +51,7 @@ def test(cls, config, webhook, output):
         else:
             final_output = output + image_output + container_output + \
                 '\nError: Harvey could not start the container.'
-            Image.remove(image[0])
+            Image.remove_image(image[0])
             Container.remove_container(test_project_name)
             Utils.kill(final_output, webhook)
 
@@ -63,7 +63,7 @@ def test(cls, config, webhook, output):
         else:
             final_output = output + image_output + container_output + start_output + \
                 '\nError: Harvey could not wait for the container.'
-            Image.remove(image[0])
+            Image.remove_image(image[0])
             Container.remove_container(test_project_name)
             Utils.kill(final_output, webhook)
 
@@ -77,21 +77,21 @@ def test(cls, config, webhook, output):
         else:
             final_output = output + image_output + container_output + start_output + wait_output + \
                 '\nError: Harvey could not create the container logs.'
-            Image.remove(image[0])
+            Image.remove_image(image[0])
             Container.remove_container(test_project_name)
             Utils.kill(final_output, webhook)
 
         # Remove container and image after it's done
         remove = Container.remove_container(test_project_name)
         if remove is not False:
-            Image.remove(image[0])
+            Image.remove_image(image[0])
             remove_output = 'Test container and image removed.'
             print(remove_output)
         else:
             final_output = output + image_output + container_output + start_output + \
                 wait_output + logs_output + \
                 '\nError: Harvey could not remove the container and/or image.'
-            Image.remove(image[0])
+            Image.remove_image(image[0])
             Container.remove_container(test_project_name)
             Utils.kill(final_output, webhook)
 
@@ -110,8 +110,8 @@ def build(cls, config, webhook, output):
 
         # Build the image
         try:
-            Image.remove(Global.docker_project_name(webhook))
-            image = Image.build(config, webhook)
+            Image.remove_image(Global.docker_project_name(webhook))
+            image = Image.build_image(config, webhook)
             image_output = f'Project image created\n{image}'
             print(image_output)
         except subprocess.TimeoutExpired:
@@ -146,9 +146,9 @@ def deploy(cls, webhook, output):
             stop_output = f'Stopping old {Global.docker_project_name(webhook)} container.'
             print(stop_output)
         elif stop_container.status_code == 304:
-            # TODO: Add missing logic here
-            stop_output = f'Error: {Global.docker_project_name(webhook)} is already stopped.'
-            print(stop_output)
+            stop_output = ''
+        elif stop_container.status_code == 404:
+            stop_output = ''
         else:
             # TODO: Add missing logic here
             stop_output = 'Error: Harvey could not stop the container.'
@@ -157,6 +157,8 @@ def deploy(cls, webhook, output):
         if wait_container.status_code == 200:
             wait_output = f'Waiting for old {Global.docker_project_name(webhook)} container to exit...'
             print(wait_output)
+        elif wait_container.status_code == 404:
+            wait_output = ''
         else:
             # TODO: Add missing logic here
             print(
@@ -166,6 +168,8 @@ def deploy(cls, webhook, output):
         if remove_container.status_code == 204:
             remove_output = f'Removing old {Global.docker_project_name(webhook)} container.'
             print(remove_output)
+        elif remove_container.status_code == 404:
+            remove_output = ''
         else:
             # TODO: Add missing logic here
             print(