diff --git a/.github/workflows/create-tag-workflow.yml b/.github/workflows/test-and-tag.yml similarity index 62% rename from .github/workflows/create-tag-workflow.yml rename to .github/workflows/test-and-tag.yml index 503146b..308a5d2 100644 --- a/.github/workflows/create-tag-workflow.yml +++ b/.github/workflows/test-and-tag.yml @@ -1,20 +1,43 @@ -name: Create Tag +name: Test and Tag -# This workflow reads the current version from setup.py and then creates a new tag and release named +# Runs on every push to run the unit tests. + +# Additionally, if on main, reads the current version from setup.py and then creates a new tag and release named # for that version. # If a tag already exists with that name, the Create Release step is skipped. -# Currently version must be manually updated in setup.py to enable this workflow to run +# Currently version must be manually updated in setup.py to enable this tagging job to run + on: workflow_dispatch: push: - # Should only trigger on main branch - branches: [ "main" ] + paths-ignore: + - "**/README.md" jobs: + run-unit-tests: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v3 + + - name: Setup python + id: setup_python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Run tests + run: | + python -m venv .venv + source .venv/bin/activate && python -m pip install --upgrade pip && pip install -r requirements-dev.txt + pytest create-release: runs-on: ubuntu-latest + needs: run-unit-tests + if: github.ref == 'refs/heads/main' permissions: contents: write steps: diff --git a/.gitignore b/.gitignore index 2532ffd..022f823 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,9 @@ Temporary Items # *.iml # *.ipr +# VS Code +.vscode/ + # CMake cmake-build-*/ diff --git a/README.md b/README.md index 51153c1..4f36771 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,21 @@ Shared library for funding service design apps. This library can be installed into other python repos and the packages used by those repos. +# Dev setup +In order to run the unit tests, setup a virtual env and install requirements +1. Checkout the code +1. Setup a venv and activate: `python3 -m venv .venv && source .venv/bin/activate` +1. Install dev requirements: `pip install -r requirements-dev.txt` +1. Install pre-commit hook: `pre-commit install` +1. Run tests with `pytest` +1. If you add any packages needed by services that consume `fsd_utils`, add them into `setup.py`. + # Releasing To create a new release of funding-service-design-utils: 1. Make and test your changes as normal in this repo 2. Update the `version` tag in `setup.py` 3. Push your changes to `main`. -4. The Action at `.github/workflows/create-tag-workflow.yml` will create a new tag and release, named for the version in `setup.py`. This is triggered automatically on a push to main. +4. The Action at `.github/workflows/test-and-tag.yml` will create a new tag and release, named for the version in `setup.py`. This is triggered automatically on a push to main. # Usage Either of the following options will install the funding-service-design-utils into your python project. The package `fsd_utils` can then be imported. @@ -27,6 +36,18 @@ To reference the latest commit from a particular branch from pip, add the follow ## The configclass Currently the configclass allows for pretty print debugging of config keys and the class from which they are created. This allows devs to quickly diagnoise problems arrising from incorrectly set config. To activate this functionality, one must decorate each config class with the `@configclass` decorator. +## Common Config +This defines config values that are common across different services, eg. url patterns. Usage example: + +``` +from fsd_utils import CommonConfig + +@configclass +class DefaultConfig: + SECRET_KEY = CommonConfig.SECRET_KEY + +``` + ## Gunicorn The gunicorn utility allows consistent configuration of gunicorn across microservices. @@ -101,3 +122,33 @@ Then - to use the `@login_required` decorator just add it to routes you need to def example_route(account_id): #...account_id will be available here if the user is authenticated #...if not logged in the user will be redirected to re-authenticate + +## Healthchecks +Adds the route `/healthcheck` to an application. On hitting this endpoint, a customisable set of checks are run to confirm the application is functioning as expected and a JSON response is returned. + +Response codes: +- 200: All checks were successful, application is healthy +- 500: One or more checks failed, application is unhealthy + +Example usage: +``` +from fsd_utils.healthchecks.healthcheck import Healthcheck +from fsd_utils.healthchecks.checkers import DbChecker, FlaskRunningChecker + +health = Healthcheck(flask_app) +health.add_check(FlaskRunningChecker()) +health.add_check(DbChecker(db)) +``` +The above will initialise the `/healthcheck` url and adds 2 checks. + +### Checks +The following 2 checks are provided in `fsd_utils` in `checkers.py`: +- `FlaskRunningChecker`: Checks whether a `current_app` is available from flask, returns `True` if it is, `False` if not. +- `DbChecker`: Runs a simple query against the DB to confirm database availability. + +### Implementing custom checks +Custom checks can be created as subclasses of `CheckerInterface` and should contain a method with the following signature: +`def check(self) -> Tuple[bool, str]:` +Where +- `bool` is a True or False whether the check was successful +- `str` is a message to display in the result JSON, typically `OK` or `Fail` diff --git a/fsd_utils/__init__.py b/fsd_utils/__init__.py index 6941d2c..5c070ad 100644 --- a/fsd_utils/__init__.py +++ b/fsd_utils/__init__.py @@ -1,7 +1,15 @@ from fsd_utils import authentication # noqa from fsd_utils import gunicorn # noqa +from fsd_utils import healthchecks from fsd_utils import logging # noqa -from fsd_utils.config.configclass import configclass # noqa from fsd_utils.config.commonconfig import CommonConfig # noqa +from fsd_utils.config.configclass import configclass # noqa -__all__ = [configclass, logging, gunicorn, authentication, CommonConfig] +__all__ = [ + configclass, + logging, + gunicorn, + authentication, + CommonConfig, + healthchecks, +] diff --git a/fsd_utils/config/commonconfig.py b/fsd_utils/config/commonconfig.py index c33751f..7443d84 100644 --- a/fsd_utils/config/commonconfig.py +++ b/fsd_utils/config/commonconfig.py @@ -1,7 +1,6 @@ -from distutils.log import WARN import logging import os -from pathlib import Path + class CommonConfig: @@ -10,7 +9,7 @@ class CommonConfig: "unit_test": logging.DEBUG, "dev": logging.INFO, "test": logging.WARN, - "production": logging.ERROR + "production": logging.ERROR, } # --------------- @@ -23,7 +22,7 @@ class CommonConfig: if not FLASK_ENV: raise KeyError("FLASK_ENV is not present in environment") try: - FSD_LOG_LEVEL = FSD_LOG_LEVELS['FLASK_ENV'] + FSD_LOG_LEVEL = FSD_LOG_LEVELS["FLASK_ENV"] except KeyError: FSD_LOG_LEVEL = FSD_LOG_LEVELS["production"] @@ -49,7 +48,9 @@ class CommonConfig: # Application hosts, endpoints # --------------- - APPLICATION_STORE_API_HOST = os.getenv("APPLICATION_STORE_API_HOST", TEST_APPLICATION_STORE_API_HOST) + APPLICATION_STORE_API_HOST = os.getenv( + "APPLICATION_STORE_API_HOST", TEST_APPLICATION_STORE_API_HOST + ) APPLICATIONS_ENDPOINT = "/applications" APPLICATION_ENDPOINT = "/applications/{application_id}" APPLICATION_STATUS_ENDPOINT = "/applications/{application_id}/status" @@ -59,15 +60,19 @@ class CommonConfig: # Assessment hosts, endpoints # --------------- - ASSESSMENT_STORE_API_HOST = os.getenv("ASSESSMENT_STORE_API_HOST", TEST_ASSESSMENT_STORE_API_HOST) + ASSESSMENT_STORE_API_HOST = os.getenv( + "ASSESSMENT_STORE_API_HOST", TEST_ASSESSMENT_STORE_API_HOST + ) # --------------- # Fund hosts, endpoints # --------------- - FUND_STORE_API_HOST = os.getenv("FUND_STORE_API_HOST", TEST_FUND_STORE_API_HOST) + FUND_STORE_API_HOST = os.getenv( + "FUND_STORE_API_HOST", TEST_FUND_STORE_API_HOST + ) FUNDS_ENDPOINT = "/funds" - FUND_ENDPOINT = "/funds/{fund_id}" #account id in assessment store + FUND_ENDPOINT = "/funds/{fund_id}" # account id in assessment store ROUNDS_ENDPOINT = "/funds/{fund_id}/rounds" ROUND_ENDPOINT = "/funds/{fund_id}/rounds/{round_id}" @@ -143,4 +148,4 @@ class CommonConfig: "session_cookie_samesite": FSD_SESSION_COOKIE_SAMESITE, "x_content_type_options": True, "x_xss_protection": True, - } \ No newline at end of file + } diff --git a/fsd_utils/healthchecks/__init__.py b/fsd_utils/healthchecks/__init__.py new file mode 100644 index 0000000..abee005 --- /dev/null +++ b/fsd_utils/healthchecks/__init__.py @@ -0,0 +1,2 @@ +from . import checkers # noqa +from . import healthcheck # noqa diff --git a/fsd_utils/healthchecks/checkers.py b/fsd_utils/healthchecks/checkers.py new file mode 100644 index 0000000..b008dc6 --- /dev/null +++ b/fsd_utils/healthchecks/checkers.py @@ -0,0 +1,35 @@ +from typing import Tuple + +from flask import current_app + + +class CheckerInterface: + def check(self) -> Tuple[bool, str]: + pass + + +class FlaskRunningChecker(CheckerInterface): + def __init__(self): + self.name = "check_running" + + def check(self): + if current_app: + return True, "OK" + else: + return False, "Fail" + + +class DbChecker(CheckerInterface): + def __init__(self, db): + self.db = db + self.name = "check_db" + + def check(self): + from sqlalchemy.exc import SQLAlchemyError + + try: + self.db.session.execute("SELECT 1") + return True, "OK" + except SQLAlchemyError: + current_app.logger.exception("DB Check failed") + return False, "Fail" diff --git a/fsd_utils/healthchecks/healthcheck.py b/fsd_utils/healthchecks/healthcheck.py new file mode 100644 index 0000000..8fedafe --- /dev/null +++ b/fsd_utils/healthchecks/healthcheck.py @@ -0,0 +1,35 @@ +from flask import current_app + + +class Healthcheck(object): + def __init__(self, app): + self.flask_app = app + self.flask_app.add_url_rule( + "/healthcheck", view_func=self.healthcheck_view + ) + self.checkers = [] + + def healthcheck_view(self): + responseCode = 200 + response = {"checks": []} + for checker in self.checkers: + try: + result = checker.check() + current_app.logger.debug( + f"Check {checker.name} returned {result}" + ) + response["checks"].append({checker.name: result[1]}) + if result[0] is False: + responseCode = 500 + except Exception: + response["checks"].append( + {checker.name: "Failed - check logs"} + ) + current_app.logger.exception( + f"Check {checker.name} failed with an exception" + ) + responseCode = 500 + return response, responseCode + + def add_check(self, checker): + self.checkers.append(checker) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..f8f5caa --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +env = + FLASK_ENV=unit_test + FLASK_DEBUG=1 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..4c12eda --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +-e . + +pre-commit +pytest-env>=0.6.2 +sqlalchemy==1.4.39 diff --git a/setup.py b/setup.py index 5612f23..600f534 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup_kwargs = { "name": "funding-service-design-utils", - "version": "0.0.11", + "version": "0.0.12", "description": "Utils for the fsd-tech team", "long_description": None, "author": "DLUHC", diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..8f9e621 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,23 @@ +""" +Contains test configuration. +""" +import pytest +from flask import Flask + + +def create_app(): + app = Flask("test") + return app + + +@pytest.fixture(scope="function") +def flask_test_client(): + """ + Creates the test client we will be using to test the responses + from our app, this is a test fixture. + :return: A flask test client. + """ + + with create_app().app_context() as app_context: + with app_context.app.test_client() as test_client: + yield test_client diff --git a/tests/test_healthcheck.py b/tests/test_healthcheck.py new file mode 100644 index 0000000..5f852e2 --- /dev/null +++ b/tests/test_healthcheck.py @@ -0,0 +1,124 @@ +from unittest.mock import ANY +from unittest.mock import Mock + +from fsd_utils.healthchecks.checkers import DbChecker +from fsd_utils.healthchecks.checkers import FlaskRunningChecker +from fsd_utils.healthchecks.healthcheck import Healthcheck +from sqlalchemy.exc import ArgumentError + + +class TestHealthcheck: + def testHealthChecksSetup(self): + test_app = Mock() + health = Healthcheck(test_app) + test_app.add_url_rule.assert_called_with("/healthcheck", view_func=ANY) + assert health.checkers == [], "Checks not initialised" + + def testWithNoChecks(self): + mock_app = Mock() + health = Healthcheck(mock_app) + mock_app.add_url_rule.assert_called_with("/healthcheck", view_func=ANY) + + expected_dict = {"checks": []} + + result = health.healthcheck_view() + assert result[0] == expected_dict, "Unexpected response body" + assert result[1] == 200, "Unexpected status code" + + def testWithChecksPassing_mocks(self, flask_test_client): + test_app = Mock() + health = Healthcheck(test_app) + test_app.add_url_rule.assert_called_with("/healthcheck", view_func=ANY) + + expected_dict = {"checks": [{"check_a": "ok"}, {"check_b": "ok"}]} + + check_a = Mock() + check_a.check.return_value = True, "ok" + check_a.name = "check_a" + health.add_check(check_a) + check_b = Mock() + check_b.check.return_value = True, "ok" + check_b.name = "check_b" + health.add_check(check_b) + + result = health.healthcheck_view() + assert result[0] == expected_dict, "Unexpected response body" + assert result[1] == 200, "Unexpected status code" + + def testWithChecksFailing_mocks(self, flask_test_client): + + test_app = Mock() + health = Healthcheck(test_app) + test_app.add_url_rule.assert_called_with("/healthcheck", view_func=ANY) + + expected_dict = {"checks": [{"check_a": "fail"}, {"check_b": "ok"}]} + + check_a = Mock() + check_a.check.return_value = False, "fail" + check_a.name = "check_a" + health.add_check(check_a) + check_b = Mock() + check_b.check.return_value = True, "ok" + check_b.name = "check_b" + health.add_check(check_b) + + result = health.healthcheck_view() + assert result[0] == expected_dict, "Unexpected response body" + assert result[1] == 500, "Unexpected status code" + + def testWithChecksException_mocks(self, flask_test_client): + + test_app = Mock() + health = Healthcheck(test_app) + test_app.add_url_rule.assert_called_with("/healthcheck", view_func=ANY) + + expected_dict = { + "checks": [{"check_a": "fail"}, {"check_b": "Failed - check logs"}] + } + + check_a = Mock() + check_a.check.return_value = False, "fail" + check_a.name = "check_a" + health.add_check(check_a) + check_b = Mock() + check_b.check.side_effect = TypeError + check_b.name = "check_b" + health.add_check(check_b) + + result = health.healthcheck_view() + assert result[0] == expected_dict, "Unexpected response body" + assert result[1] == 500, "Unexpected status code" + + def testRunningCheck_pass(self, flask_test_client): + result = FlaskRunningChecker().check() + assert result[0] is True, "Unexpected check result" + assert result[1] == "OK", "Unexpected check message" + + # Don't pass in flask_test_client, therefore no flask.current_app, + # therefore this check returns false + def testRunningCheck_fail(self): + result = FlaskRunningChecker().check() + assert result[0] is False, "Unexpected check result" + assert result[1] == "Fail", "Unexpected check message" + + def testDbCheck_pass(self): + mock_db = Mock() + mock_db.session = Mock() + mock_db.session.execute.return_value = True + db_checker = DbChecker(mock_db) + + result = db_checker.check() + assert result[0] is True, "Unexpected check result" + assert result[1] == "OK", "Unexpected check message" + mock_db.session.execute.assert_called_with("SELECT 1") + + def testDbCheck_fail(self, flask_test_client): + mock_db = Mock() + mock_db.session = Mock() + mock_db.session.execute.side_effect = ArgumentError + db_checker = DbChecker(mock_db) + + result = db_checker.check() + assert result[0] is False, "Unexpected check result" + assert result[1] == "Fail", "Unexpected check message" + mock_db.session.execute.assert_called_with("SELECT 1")