From a135c283420d0c8a01432b96b130c6d032465836 Mon Sep 17 00:00:00 2001 From: Mohamed Nur <85885638+mohamed-cds@users.noreply.github.com> Date: Thu, 2 Sep 2021 11:39:51 -0400 Subject: [PATCH] feat: bootstrapped basic web rendering using jinja2 (#102) * feat: bootstrapped basic web rendering using jinja2 * Formatting and remove unused import * chore: removed reference to Form component * chore: ignore unreferenced objects since import is required * chore: api is now versioned and front end has been extracted to its own module * feat: i18n for en & fr added * chore: renamed pydantic organisation model to OrganisationFilter * chore: formatting * feat: add test * chore: formatting isnt my friend * chore: remove unused import after refactor * chore: another test * chore: remove unused imports from test * chore: test removal of db initialize * chore: remove import of disabled feature * chore: remove broken test and database reset debug code * chore: remove unused import * chore: cleaned up and new db session code * chore: less ugly page --- .devcontainer/docker-compose.yml | 2 + api/api_gateway/api.py | 47 ----------- api/api_gateway/v1/__init__.py | 0 api/api_gateway/v1/api.py | 76 ++++++++++++++++++ api/front_end/templates/base.html | 49 ++++++++++++ api/front_end/templates/dashboard.html | 40 ++++++++++ api/front_end/templates/index.html | 7 ++ api/front_end/view.py | 104 +++++++++++++++++++++++++ api/i18n/en.json | 11 +++ api/i18n/fr.json | 11 +++ api/main.py | 25 +++++- api/models/Organisation.py | 1 - api/models/__init__.py | 6 -- api/requirements.txt | 3 + api/schemas/Organization.py | 12 +++ api/schemas/__init__.py | 0 api/tests/api_gateway/test_api.py | 16 ++-- api/tests/conftest.py | 10 +++ api/tests/front_end/test_view.py | 11 +++ bin/api_create_organisation.sh | 17 ++++ bin/api_healthcheck.sh | 2 +- bin/api_version.sh | 2 +- bin/app_dashboard.sh | 17 ++++ bin/app_index.sh | 17 ++++ 24 files changed, 420 insertions(+), 66 deletions(-) delete mode 100644 api/api_gateway/api.py create mode 100644 api/api_gateway/v1/__init__.py create mode 100644 api/api_gateway/v1/api.py create mode 100644 api/front_end/templates/base.html create mode 100644 api/front_end/templates/dashboard.html create mode 100644 api/front_end/templates/index.html create mode 100644 api/front_end/view.py create mode 100644 api/i18n/en.json create mode 100644 api/i18n/fr.json create mode 100644 api/schemas/Organization.py create mode 100644 api/schemas/__init__.py create mode 100644 api/tests/front_end/test_view.py create mode 100755 bin/api_create_organisation.sh create mode 100755 bin/app_dashboard.sh create mode 100755 bin/app_index.sh diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 32318d1b..8f6d8070 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -36,6 +36,8 @@ services: AWS_LOCALSTACK: "True" AXE_CORE_URLS_TOPIC: "arn:aws:sns:us-east-1:000000000000:axe-core-urls-topic" + ports: + - "8080:8080" db: image: postgres:11.2 volumes: diff --git a/api/api_gateway/api.py b/api/api_gateway/api.py deleted file mode 100644 index fb77e7c0..00000000 --- a/api/api_gateway/api.py +++ /dev/null @@ -1,47 +0,0 @@ -from os import environ -from fastapi import FastAPI -from sqlalchemy.exc import SQLAlchemyError -from database.db import db_session -from logger import log - -# from crawler.crawler import crawl -# import uuid -from pydantic import BaseModel - - -app = FastAPI() - - -@app.get("/version") -async def version(): - return {"version": environ.get("GIT_SHA", "unknown")} - - -def get_db_version(sessionmaker): - session = sessionmaker() - - query = "SELECT version_num FROM alembic_version" - full_name = session.execute(query).fetchone()[0] - return full_name - - -@app.get("/healthcheck") -async def healthcheck(): - try: - full_name = get_db_version(db_session) - db_status = {"able_to_connect": True, "db_version": full_name} - except SQLAlchemyError as err: - log.error(err) - db_status = {"able_to_connect": False} - - return {"database": db_status} - - -class CrawlUrl(BaseModel): - url: str - - -# @app.post("/crawl") -# async def crawl_endpoint(crawl_url: CrawlUrl): -# log.info(f"Crawling {crawl_url}") -# crawl(uuid.uuid4(), crawl_url.url) diff --git a/api/api_gateway/v1/__init__.py b/api/api_gateway/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/api/api_gateway/v1/api.py b/api/api_gateway/v1/api.py new file mode 100644 index 00000000..0b490306 --- /dev/null +++ b/api/api_gateway/v1/api.py @@ -0,0 +1,76 @@ +from os import environ +from fastapi import Depends, FastAPI, HTTPException +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session +from fastapi.responses import RedirectResponse +from database.db import db_session +from logger import log + +from models.Organisation import Organisation +from schemas.Organization import OrganizationCreate + +# from crawler.crawler import crawl +# import uuid +from pydantic import BaseModel + +app = FastAPI() + + +# Dependency +def get_db(): + db = db_session() + try: + yield db + finally: + db.close() + + +@app.get("/api/v1/version") +def version(): + return {"version": environ.get("GIT_SHA", "unknown")} + + +def get_db_version(session): + + query = "SELECT version_num FROM alembic_version" + full_name = session.execute(query).fetchone()[0] + return full_name + + +@app.get("/api/v1/healthcheck") +def healthcheck(session: Session = Depends(get_db)): + try: + full_name = get_db_version(session) + db_status = {"able_to_connect": True, "db_version": full_name} + except SQLAlchemyError as err: + log.error(err) + db_status = {"able_to_connect": False} + + return {"database": db_status} + + +# TODO Require auth and redirect to home +# TODO Push errors to cloudwatch metric and response when debug enabled +@app.post("/api/v1/organisation", response_class=RedirectResponse) +def create_organisation( + organisation: OrganizationCreate, session: Session = Depends(get_db) +): + + try: + new_organisation = Organisation(name=organisation.name) + session.add(new_organisation) + session.commit() + return RedirectResponse("/dashboard") + except Exception as e: + log.error(e) + raise HTTPException(status_code=500, detail=str(e)) + + +class CrawlUrl(BaseModel): + url: str + + +# @app.post("/crawl") +# def crawl_endpoint(crawl_url: CrawlUrl): +# log.info(f"Crawling {crawl_url}") +# crawl(uuid.uuid4(), crawl_url.url) diff --git a/api/front_end/templates/base.html b/api/front_end/templates/base.html new file mode 100644 index 00000000..a5dc7949 --- /dev/null +++ b/api/front_end/templates/base.html @@ -0,0 +1,49 @@ + + + + + + + + + {{ webpage_title }} + + +
+
+ +
+
+
+
+ {% block body %}{% endblock %} +
+
+
+
+ {% block footer %} +
+
+ {% endblock %} +
+
+
+
+ + + \ No newline at end of file diff --git a/api/front_end/templates/dashboard.html b/api/front_end/templates/dashboard.html new file mode 100644 index 00000000..0e62f79d --- /dev/null +++ b/api/front_end/templates/dashboard.html @@ -0,0 +1,40 @@ +{% extends "base.html" %} +{% block body %} +

{{ organisations_locale }}

+
+
+
+
+ + + + + + + + + {% for organisation in organisations %} + + + + + {% endfor %} + +
+ Name + + Configure +
+
+ {{ organisation.name }} +
+ 		+ Edit +
+
+
+
+
+ + +{% endblock %} \ No newline at end of file diff --git a/api/front_end/templates/index.html b/api/front_end/templates/index.html new file mode 100644 index 00000000..ca54e95b --- /dev/null +++ b/api/front_end/templates/index.html @@ -0,0 +1,7 @@ +{% extends "base.html" %} +{% block content %} +
{{ welcome }}
+ {% block footer %} + {{super()}} + {% endblock %} +{% endblock %} \ No newline at end of file diff --git a/api/front_end/view.py b/api/front_end/view.py new file mode 100644 index 00000000..ce306df1 --- /dev/null +++ b/api/front_end/view.py @@ -0,0 +1,104 @@ +from fastapi import Depends, FastAPI, Request, HTTPException +from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi.templating import Jinja2Templates +from babel.plural import PluralRule +from database.db import db_session +from logger import log +from sqlalchemy.orm import Session + +from models.Organisation import Organisation + +import glob +import json +import os + +app = FastAPI() + + +# Dependency +def get_db(): + db = db_session() + try: + yield db + finally: + db.close() + + +templates = Jinja2Templates(directory="front_end/templates") +default_fallback = "en" +languages = {} + + +def generate_languages(locale_files): + language_list = glob.glob(locale_files) + for lang in language_list: + filename = lang.split(os.path.sep) + lang_code = filename[1].split(".")[0] + + with open(lang, "r", encoding="utf8") as file: + languages[lang_code] = json.load(file) + + +generate_languages("i18n/*.json") + + +# custom filters for Jinja2 +def plural_formatting(key_value, input, locale): + plural_rule = PluralRule({"one": "n in 0..1"}) + key = "" + for i in languages[locale]: + if key_value == languages[locale][i]: + key = i + break + + if not key: + return key_value + + plural_key = f"{key}_plural" + + if plural_rule(input) != "one" and plural_key in languages[locale]: + key = plural_key + + return languages[locale][key] + + +# assign filter to Jinja2 +templates.env.filters["plural_formatting"] = plural_formatting + + +@app.get("/", response_class=HTMLResponse) +async def force_lang(): + return RedirectResponse("/en") + + +@app.get("/{locale}", response_class=HTMLResponse) +async def home(request: Request, locale: str): + try: + if locale not in languages: + locale = default_fallback + + result = {"request": request} + result.update(languages[locale]) + return templates.TemplateResponse("index.html", result) + except Exception as e: + log.error(e) + raise HTTPException(status_code=500, detail=str(e)) + + +# TODO Require auth & limit to users current organisation +# TODO Push errors to cloudwatch metric and response when debug enabled +# TODO Enable detailed error messages via debug flag +@app.get("/{locale}/dashboard", response_class=HTMLResponse) +async def dashboard(request: Request, locale: str, session: Session = Depends(get_db)): + try: + if locale not in languages: + locale = default_fallback + + organisation_list = session.query(Organisation).all() + result = {"request": request} + result.update(languages[locale]) + result.update({"organisations": organisation_list}) + except Exception as e: + log.error(e) + raise HTTPException(status_code=500, detail=str(e)) + return templates.TemplateResponse("dashboard.html", result) diff --git a/api/i18n/en.json b/api/i18n/en.json new file mode 100644 index 00000000..0a6aed14 --- /dev/null +++ b/api/i18n/en.json @@ -0,0 +1,11 @@ +{ + "alpha": "Alpha", + "goc": "Government of Canada", + "goc_banner": "https://ssl-templates.services.gc.ca/app/cls/wet/gcintranet/v4_0_20/assets/sig-blk-en.svg", + "lang": "en", + "organisations_locale" : "Organisations", + "other_lang": "fr", + "other_language": "Français", + "webpage_title": "Scan websites", + "welcome": "Welcome to Scan websites" +} \ No newline at end of file diff --git a/api/i18n/fr.json b/api/i18n/fr.json new file mode 100644 index 00000000..4484c513 --- /dev/null +++ b/api/i18n/fr.json @@ -0,0 +1,11 @@ +{ + "alpha": "Alpha", + "goc": "Gouvernement du Canada", + "goc_banner": "https://ssl-templates.services.gc.ca/app/cls/wet/gcintranet/v4_0_20/assets/sig-blk-fr.svg", + "lang": "fr", + "organisations_locale" : "Organisations", + "other_lang": "en", + "other_language": "English", + "webpage_title": "Analyser les sites web", + "welcome": "Bienvenue sur les sites Web de Scan" +} \ No newline at end of file diff --git a/api/main.py b/api/main.py index 27005f07..2392a3e3 100644 --- a/api/main.py +++ b/api/main.py @@ -1,11 +1,26 @@ from mangum import Mangum -from api_gateway import api +from api_gateway.v1 import api +from front_end import view from logger import log from database.migrate import migrate_head from storage import storage import os -app = api.app + +# Import so that the application is aware of these Models +# Required so that models are initialized before they're referenced +from models.A11yReport import A11yReport # noqa: F401 +from models.A11yViolation import A11yViolation # noqa: F401 +from models.Organisation import Organisation # noqa: F401 +from models.Scan import Scan # noqa: F401 +from models.Template import Template # noqa: F401 +from models.TemplateScan import TemplateScan # noqa: F401 +from models.TemplateScanTrigger import TemplateScanTrigger # noqa: F401 +from models.User import User # noqa: F401 + + +api_v1 = api.app +app = view.app def print_env_variables(): @@ -20,7 +35,13 @@ def handler(event, context): if "httpMethod" in event: # Assume it is an API Gateway event asgi_handler = Mangum(app) + + if "path" in event: + if event["path"].lower().startswith("/api/v1"): + asgi_handler = Mangum(api_v1) + response = asgi_handler(event, context) + return response elif "Records" in event: diff --git a/api/models/Organisation.py b/api/models/Organisation.py index cbb1d94e..c66c18b9 100644 --- a/api/models/Organisation.py +++ b/api/models/Organisation.py @@ -5,7 +5,6 @@ from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import relationship, validates - from models import Base diff --git a/api/models/__init__.py b/api/models/__init__.py index 07830b05..31d07d5e 100644 --- a/api/models/__init__.py +++ b/api/models/__init__.py @@ -3,9 +3,3 @@ DBSession = scoped_session(sessionmaker()) Base = declarative_base() - - -def initialize_sql(engine): - DBSession.configure(bind=engine) - Base.metadata.bind = engine - Base.metadata.create_all(engine) diff --git a/api/requirements.txt b/api/requirements.txt index 21ba215a..5a711f53 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,7 +1,10 @@ alembic==1.6.5 +aiofiles==0.7.0 +babel==2.9.1 boto3==1.17.84 bcrypt==3.2.0 fastapi==0.67.0 +jinja2==3.0.1 logzero==1.7.0 mangum==0.12.1 psycopg2-binary==2.9.1 diff --git a/api/schemas/Organization.py b/api/schemas/Organization.py new file mode 100644 index 00000000..78ab81f9 --- /dev/null +++ b/api/schemas/Organization.py @@ -0,0 +1,12 @@ +from pydantic import BaseModel + + +class OrganisationFilter(BaseModel): + name: str + + +class OrganizationCreate(OrganisationFilter): + class Config: + extra = "forbid" + + pass diff --git a/api/schemas/__init__.py b/api/schemas/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/api/tests/api_gateway/test_api.py b/api/tests/api_gateway/test_api.py index f07b2941..871d473b 100644 --- a/api/tests/api_gateway/test_api.py +++ b/api/tests/api_gateway/test_api.py @@ -3,39 +3,39 @@ from fastapi.testclient import TestClient from unittest.mock import patch -from api_gateway import api +from api_gateway.v1 import api from sqlalchemy.exc import SQLAlchemyError client = TestClient(api.app) def test_version_with_no_GIT_SHA(): - response = client.get("/version") + response = client.get("/api/v1/version") assert response.status_code == 200 assert response.json() == {"version": "unknown"} @patch.dict(os.environ, {"GIT_SHA": "foo"}, clear=True) def test_version_with_GIT_SHA(): - response = client.get("/version") + response = client.get("/api/v1/version") assert response.status_code == 200 assert response.json() == {"version": "foo"} -@patch("api_gateway.api.get_db_version") +@patch("api_gateway.v1.api.get_db_version") def test_healthcheck_success(mock_get_db_version): mock_get_db_version.return_value = "foo" - response = client.get("/healthcheck") + response = client.get("/api/v1/healthcheck") assert response.status_code == 200 expected_val = {"database": {"able_to_connect": True, "db_version": "foo"}} assert response.json() == expected_val -@patch("api_gateway.api.get_db_version") -@patch("api_gateway.api.log") +@patch("api_gateway.v1.api.get_db_version") +@patch("api_gateway.v1.api.log") def test_healthcheck_failure(mock_log, mock_get_db_version): mock_get_db_version.side_effect = SQLAlchemyError() - response = client.get("/healthcheck") + response = client.get("/api/v1/healthcheck") assert response.status_code == 200 expected_val = {"database": {"able_to_connect": False}} assert response.json() == expected_val diff --git a/api/tests/conftest.py b/api/tests/conftest.py index ee4e73f7..09b317bd 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -10,6 +10,7 @@ from models.ScanType import ScanType from models.Template import Template from models.TemplateScan import TemplateScan +from models.User import User from sqlalchemy import create_engine @@ -44,6 +45,7 @@ def f(model): def organisation_fixture(session): organisation = Organisation(name="fixture_name") session.add(organisation) + return organisation @@ -59,6 +61,7 @@ def setup_db(): os.environ["SQLALCHEMY_DATABASE_URI"] = os.environ.get( "SQLALCHEMY_DATABASE_TEST_URI" ) + alembic_cfg = Config("./db_migrations/alembic.ini") alembic_cfg.set_main_option("script_location", "./db_migrations") command.downgrade(alembic_cfg, "base") @@ -101,3 +104,10 @@ def template_scan_fixture(scan_type_fixture, template_fixture, session): ) session.add(template_scan) return template_scan + + +@pytest.fixture(scope="session") +def user_fixture(session): + user = User(name="fixture_name") + session.add(user) + return user diff --git a/api/tests/front_end/test_view.py b/api/tests/front_end/test_view.py new file mode 100644 index 00000000..c3b5381c --- /dev/null +++ b/api/tests/front_end/test_view.py @@ -0,0 +1,11 @@ +from fastapi.testclient import TestClient + +from front_end import view + +client = TestClient(view.app) + + +def test_langing_page_redirect_to_en(): + response = client.get("/", allow_redirects=False) + assert response.status_code == 307 + assert response.headers["location"] == "/en" diff --git a/bin/api_create_organisation.sh b/bin/api_create_organisation.sh new file mode 100755 index 00000000..37b8b916 --- /dev/null +++ b/bin/api_create_organisation.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +echo hitting create organisation endpoint +curl "http://api:8080/2015-03-31/functions/function/invocations" -d '{ + "resource": "/", + "path": "/api/v1/organisation", + "requestContext": {}, + "httpMethod": "POST", + "headers": {}, + "multiValueHeaders": { }, + "queryStringParameters": null, + "multiValueQueryStringParameters": null, + "pathParameters": null, + "stageVariables": null, + "body": "{\"name\": \"ABCDE\"}", + "isBase64Encoded": false +}' |jq diff --git a/bin/api_healthcheck.sh b/bin/api_healthcheck.sh index 2f359ae2..59f5e84b 100755 --- a/bin/api_healthcheck.sh +++ b/bin/api_healthcheck.sh @@ -3,7 +3,7 @@ echo hitting api healthcheck endpoint curl "http://api:8080/2015-03-31/functions/function/invocations" -d '{ "resource": "/", - "path": "/healthcheck", + "path": "/api/v1/healthcheck", "requestContext": {}, "httpMethod": "GET", "headers": {}, diff --git a/bin/api_version.sh b/bin/api_version.sh index 3892a648..92b25aa8 100755 --- a/bin/api_version.sh +++ b/bin/api_version.sh @@ -3,7 +3,7 @@ echo hitting api version endpoint curl "http://api:8080/2015-03-31/functions/function/invocations" -d '{ "resource": "/", - "path": "/version", + "path": "/api/v1/version", "requestContext": {}, "httpMethod": "GET", "headers": {}, diff --git a/bin/app_dashboard.sh b/bin/app_dashboard.sh new file mode 100755 index 00000000..b35e0992 --- /dev/null +++ b/bin/app_dashboard.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +echo hitting app dashboard endpoint +curl "http://api:8080/2015-03-31/functions/function/invocations" -d '{ + "resource": "/", + "path": "/en/dashboard", + "requestContext": {}, + "httpMethod": "GET", + "headers": {}, + "multiValueHeaders": { }, + "queryStringParameters": null, + "multiValueQueryStringParameters": null, + "pathParameters": null, + "stageVariables": null, + "body": null, + "isBase64Encoded": false +}' |jq diff --git a/bin/app_index.sh b/bin/app_index.sh new file mode 100755 index 00000000..64466c7f --- /dev/null +++ b/bin/app_index.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +echo hitting app root endpoint +curl "http://api:8080/2015-03-31/functions/function/invocations" -d '{ + "resource": "/", + "path": "/en", + "requestContext": {}, + "httpMethod": "GET", + "headers": {}, + "multiValueHeaders": { }, + "queryStringParameters": null, + "multiValueQueryStringParameters": null, + "pathParameters": null, + "stageVariables": null, + "body": null, + "isBase64Encoded": false +}' |jq