v0.9.0 - Overhaul Pipelines (#42)

* v0.9.0 - Overhaul Pipelines

* Fix Pipeline tests

* 100% test coverage on pipelines

* Update CHANGELOG

CHANGELOG.md

@@ -1,5 +1,12 @@
 # CHANGELOG
 
+## v0.9.0 (2021-03-09)
+
+* Overhauled the Pipeline/Webhook modules and removed lots of duplicate code
+* Fixed a bug where the pipeline timer wouldn't account for startup time (closes #35)
+* Added unit tests for the `pipelines` modules
+* Various other bug fixes
+
 ## v0.8.2 (2021-03-06)
 
 * Disabled container healthchecks temporarily for docker-compose workflows

harvey/__init__.py

@@ -1,10 +1,10 @@
 # flake8: noqa
-from harvey.globals import Global
-from harvey.container import Container
+from harvey.containers import Container
 from harvey.git import Git
-from harvey.image import Image
-from harvey.message import Message
-from harvey.pipeline import Pipeline
-from harvey.stage import Stage
-from harvey.utils import Utils, Logs
-from harvey.webhook import Webhook
+from harvey.globals import Global
+from harvey.images import Image
+from harvey.messages import Message
+from harvey.pipelines import Pipeline
+from harvey.stages import Stage
+from harvey.utils import Logs, Utils
+from harvey.webhooks import Webhook

harvey/app.py

@@ -7,7 +7,7 @@
 from flask import Flask, abort, request
 
 from harvey.globals import Global
-from harvey.webhook import Webhook
+from harvey.webhooks import Webhook
 
 load_dotenv()  # must remain at the top of this file
 API = Flask(__name__)

harvey/globals.py

@@ -6,7 +6,7 @@ class Global():
     """
     DOCKER_VERSION = 'v1.41'  # Docker API version
     # TODO: Figure out how to sync this version number with the one in `setup.py`
-    HARVEY_VERSION = '0.8.2'  # Harvey release
+    HARVEY_VERSION = '0.9.0'  # Harvey release
     PROJECTS_PATH = 'projects'
     PROJECTS_LOG_PATH = 'logs/projects'
     HARVEY_LOG_PATH = 'logs/harvey'
@@ -16,6 +16,12 @@ class Global():
     JSON_HEADERS = {'Content-Type': 'application/json'}
     TAR_HEADERS = {'Content-Type': 'application/tar'}
     APP_MODE = os.getenv('MODE', 'production').lower()
+    SUPPORTED_PIPELINES = [
+        'pull',
+        'test',
+        'deploy',
+        'full'
+    ]
 
     @classmethod
     def repo_name(cls, webhook):

harvey/pipelines.py

@@ -0,0 +1,144 @@
+import json
+import os
+from datetime import datetime
+
+from harvey.git import Git
+from harvey.globals import Global
+from harvey.messages import Message
+from harvey.stages import Stage
+from harvey.utils import Utils
+
+SLACK = os.getenv('SLACK')
+
+
+class Pipeline():
+    @classmethod
+    def initialize_pipeline(cls, webhook):
+        """Initialize the setup for a pipeline by cloning/pulling the project
+        and setting up standard logging info
+        """
+        start_time = datetime.now()
+        config = cls.open_project_config(webhook)
+
+        if SLACK:
+            Message.send_slack_message(
+                f'Harvey has started a `{config["pipeline"]}` pipeline for `{Global.repo_full_name(webhook)}`.'
+            )
+
+        preamble = f'Running Harvey v{Global.HARVEY_VERSION}\n{config["pipeline"].title()} Pipeline Started: {start_time}'  # noqa
+        pipeline_id = f'Pipeline ID: {Global.repo_commit_id(webhook)}\n'
+        print(preamble)
+        git_message = (f'New commit by: {Global.repo_commit_author(webhook)}.'
+                       f'\nCommit made on repo: {Global.repo_full_name(webhook)}.')
+
+        git = Git.update_git_repo(webhook)
+
+        execution_time = f'Startup execution time: {datetime.now() - start_time}\n'
+        output = (f'{preamble}\n{pipeline_id}Configuration:\n{json.dumps(config, indent=4)}'
+                  f'\n\n{git_message}\n{git}\n{execution_time}')
+        print(execution_time)
+
+        return config, output, start_time
+
+    @classmethod
+    def start_pipeline(cls, webhook, use_compose=False):
+        """After receiving a webhook, spin up a pipeline based on the config
+        If a Pipeline fails, it fails early in the individual functions being called
+        """
+        webhook_config, webhook_output, start_time = cls.initialize_pipeline(webhook)
+        pipeline = webhook_config.get('pipeline').lower()
+
+        if pipeline in Global.SUPPORTED_PIPELINES:
+            if pipeline == 'pull':
+                final_output = f'{webhook_output}\nHarvey pulled the project successfully.'
+            if pipeline in ['test', 'full']:
+                test = cls.test(webhook_config, webhook, webhook_output, start_time)
+
+                end_time = datetime.now()
+                execution_time = f'Pipeline execution time: {end_time - start_time}'
+                pipeline_status = 'Pipeline succeeded!'
+
+                final_output = f'{webhook_output}\n{test}\n{execution_time}\n{pipeline_status}'
+            if pipeline in ['deploy', 'full']:
+                build, deploy, healthcheck = cls.deploy(webhook_config, webhook, webhook_output, start_time, use_compose)  # noqa
+
+                stage_output = build + '\n' + deploy
+                healthcheck_message = f'Project passed healthcheck: {healthcheck}'
+                end_time = datetime.now()
+                execution_time = f'Pipeline execution time: {end_time - start_time}'
+                pipeline_status = 'Pipeline succeeded!'
+
+                final_output = f'{webhook_output}\n{stage_output}\n{execution_time}\n{healthcheck_message}\n{pipeline_status}'  # noqa
+
+            Utils.success(final_output, webhook)
+        else:
+            final_output = webhook_output + '\nError: Harvey could not run, there was no acceptable pipeline specified.'
+            pipeline = Utils.kill(final_output, webhook)
+
+        return final_output
+
+    @classmethod
+    def open_project_config(cls, webhook):
+        """Open the project's config file to assign pipeline variables.
+
+        Project configs look like the following:
+        {
+            "pipeline": "full",
+            "language": "php",
+            "version": "7.4"
+        }
+        """
+        # TODO: Add the ability to configure projects on the Harvey side
+        # (eg: save values to a database via a UI) instead of only from
+        # within a JSON file in the repo
+        try:
+            filename = os.path.join(
+                Global.PROJECTS_PATH, Global.repo_full_name(webhook), 'harvey.json'
+            )
+            with open(filename, 'r') as file:
+                config = json.loads(file.read())
+                print(json.dumps(config, indent=4))
+            return config
+        except FileNotFoundError:
+            final_output = f'Error: Harvey could not find a "harvey.json" file in {Global.repo_full_name(webhook)}.'
+            print(final_output)
+            Utils.kill(final_output, webhook)
+
+    @classmethod
+    def test(cls, config, webhook, output, start_time):
+        """Run the test stage in a pipeline
+        """
+        test = Stage.test(config, webhook, output)
+        if 'Error: the above command exited with code' in test:
+            # TODO: Ensure this works, it may be broken
+            end_time = datetime.now()
+            pipeline_status = 'Pipeline failed!'
+            execution_time = f'Pipeline execution time: {end_time - start_time}'
+            final_output = f'{output}\n{test}\n{execution_time}\n{pipeline_status}'
+            Utils.kill(final_output, webhook)
+
+        return test
+
+    @classmethod
+    def deploy(cls, config, webhook, output, start_time, use_compose):
+        """Run the build and deploy stages in a pipeline
+        """
+        if use_compose:
+            build = ''  # When using compose, there is no build step
+            deploy = Stage.build_deploy_compose(config, webhook, output)
+            # healthcheck = Stage.run_container_healthcheck(webhook)  # TODO: Correct healthchecks for compose
+            healthcheck = True
+        else:
+            build = Stage.build(config, webhook, output)
+            deploy = Stage.deploy(webhook, output)
+            healthcheck = Stage.run_container_healthcheck(webhook)
+
+        if healthcheck is False:
+            end_time = datetime.now()
+            pipeline_status = 'Pipeline failed due to a bad healthcheck.'
+            execution_time = f'Pipeline execution time: {end_time - start_time}'
+            healthcheck_message = f'Project passed healthcheck: {healthcheck}'
+            final_output = f'{output}\n{build}\n{deploy}\n{execution_time}\n{healthcheck_message}\n{pipeline_status}'
+            Utils.kill(final_output, webhook)
+
+        return build, deploy, healthcheck