diff --git a/README.md b/README.md index e89d4af..0b7b31f 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,12 @@ Contained in [export_tasks.py](./tasks/export_tasks.py) The configuration output is generated by the [config_generator](./app/config_generator/README.md) module. This module contains functions to generate fund and round configuration, form JSONs, and HTML representations for a given funding round. ## Database +### Schema +The database schema is defined in [app/db/models.py](./app/db/models.py) and is managed by Alembic. The migrations are stored in [app/db/migrations/versions](./app/db/migrations/versions/) + +### Entity Relationship Diagram +See [Here](./app/db/database_ERD_9-8-24.png) + ### Recreate Local DBs For both `DATABASE_URL` and `DATABASE_URL_UNIT_TEST`, drops the database if it exists and then recreates it. diff --git a/app/config_generator/scripts/generate_fund_round_config.py b/app/config_generator/scripts/generate_fund_round_config.py index 080c5e3..0600cc9 100644 --- a/app/config_generator/scripts/generate_fund_round_config.py +++ b/app/config_generator/scripts/generate_fund_round_config.py @@ -4,7 +4,8 @@ from typing import Dict from typing import Optional -from app.app import app +from flask import current_app + from app.config_generator.scripts.helpers import write_config from app.db import db from app.db.models import Form @@ -12,6 +13,8 @@ from app.db.queries.fund import get_fund_by_id from app.db.queries.round import get_round_by_id +# TODO : The Round path might be better as a placeholder to avoid conflict in the actual fund store. +# Decide on this further down the line. ROUND_BASE_PATHS = { # Should increment for each new round, anything that shares the same base path will also share # the child tree path config. @@ -66,7 +69,7 @@ def generate_application_display_config(round_id): round_base_path = ROUND_BASE_PATHS[round.short_name] "sort by Section.index" sections = db.session.query(Section).filter(Section.round_id == round_id).order_by(Section.index).all() - app.logger.info(f"Generating application display config for round {round_id}") + current_app.logger.info(f"Generating application display config for round {round_id}") for original_section in sections: section = copy.deepcopy(original_section) @@ -192,7 +195,7 @@ def generate_fund_config(round_id): round = get_round_by_id(round_id) fund_id = round.fund_id fund = get_fund_by_id(fund_id) - app.logger.info(f"Generating fund config for fund {fund_id}") + current_app.logger.info(f"Generating fund config for fund {fund_id}") fund_export = FundExport( id=str(fund.fund_id), @@ -210,7 +213,7 @@ def generate_fund_config(round_id): def generate_round_config(round_id): round = get_round_by_id(round_id) - app.logger.info(f"Generating round config for round {round_id}") + current_app.logger.info(f"Generating round config for round {round_id}") round_export = RoundExport( id=str(round.round_id), diff --git a/app/config_generator/scripts/generate_fund_round_form_jsons.py b/app/config_generator/scripts/generate_fund_round_form_jsons.py index 01cd83c..8093b2a 100644 --- a/app/config_generator/scripts/generate_fund_round_form_jsons.py +++ b/app/config_generator/scripts/generate_fund_round_form_jsons.py @@ -1,10 +1,90 @@ import json -from app.app import app +from flask import current_app + from app.config_generator.generate_form import build_form_json +from app.config_generator.scripts.helpers import validate_json from app.config_generator.scripts.helpers import write_config from app.db.queries.round import get_round_by_id +form_schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "name": {"type": "string"}, + "metadata": {"type": "object"}, + "startPage": {"type": "string"}, + "backLinkText": {"type": "string"}, + "sections": {"type": "array"}, + "pages": { + "type": "array", + "items": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "title": {"type": "string"}, + "components": { + "type": "array", + "items": { + "type": "object", + "properties": { + "options": { + "type": "object", + "properties": {"hideTitle": {"type": "boolean"}, "classes": {"type": "string"}}, + }, + "type": {"type": "string"}, + "title": {"type": "string"}, + "hint": {"type": "string"}, + "schema": {"type": "object"}, + "name": {"type": "string"}, + "metadata": { + "type": "object", + }, + }, + }, + }, + }, + "required": ["path", "title", "components"], + }, + }, + "lists": {"type": "array"}, + "conditions": {"type": "array"}, + "fees": {"type": "array"}, + "outputs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "title": {"type": "string"}, + "type": {"type": "string"}, + "outputConfiguration": { + "type": "object", + "properties": {"savePerPageUrl": {"type": "boolean"}}, + "required": ["savePerPageUrl"], + }, + }, + "required": ["name", "title", "type", "outputConfiguration"], + }, + }, + "skipSummary": {"type": "boolean"}, + # Add other top-level keys as needed + }, + "required": [ + "metadata", + "startPage", + "backLinkText", + "pages", + "lists", + "conditions", + "fees", + "outputs", + "skipSummary", + "name", + "sections", + ], +} + def generate_form_jsons_for_round(round_id): """ @@ -22,13 +102,14 @@ def generate_form_jsons_for_round(round_id): """ if not round_id: raise ValueError("Round ID is required to generate form JSONs.") - forms_for_round = [] round = get_round_by_id(round_id) - app.logger.info(f"Generating form JSONs for round {round_id}.") + current_app.logger.info(f"Generating form JSONs for round {round_id}.") for section in round.sections: for form in section.forms: result = build_form_json(form) form_json = json.dumps(result, indent=4) - forms_for_round.append({"form_name": form.runner_publish_name, "json_str": form_json}) - for form in forms_for_round: - write_config(form["json_str"], form["form_name"], round.short_name, "form_json") + valid_json = validate_json(result, form_schema) + if valid_json: + write_config(form_json, form.runner_publish_name, round.short_name, "form_json") + else: + current_app.logger.error(f"Form JSON for {form.runner_publish_name} is invalid.") diff --git a/app/config_generator/scripts/generate_fund_round_html.py b/app/config_generator/scripts/generate_fund_round_html.py index 4b42a83..b6c8937 100644 --- a/app/config_generator/scripts/generate_fund_round_html.py +++ b/app/config_generator/scripts/generate_fund_round_html.py @@ -1,5 +1,6 @@ +from flask import current_app + from app.all_questions.metadata_utils import generate_print_data_for_sections -from app.app import app from app.config_generator.generate_all_questions import print_html from app.config_generator.generate_form import build_form_json from app.config_generator.scripts.helpers import write_config @@ -31,7 +32,7 @@ def generate_all_round_html(round_id): """ if not round_id: raise ValueError("Round ID is required to generate HTML.") - app.logger.info(f"Generating HTML for round {round_id}.") + current_app.logger.info(f"Generating HTML for round {round_id}.") round = get_round_by_id(round_id) sections_in_round = round.sections section_data = [] diff --git a/app/config_generator/scripts/helpers.py b/app/config_generator/scripts/helpers.py index 7f6871b..73b0ee5 100644 --- a/app/config_generator/scripts/helpers.py +++ b/app/config_generator/scripts/helpers.py @@ -3,6 +3,10 @@ from dataclasses import is_dataclass from datetime import date +import jsonschema +from flask import current_app +from jsonschema import validate + from app.blueprints.self_serve.routes import human_to_kebab_case from app.blueprints.self_serve.routes import human_to_snake_case @@ -45,3 +49,15 @@ def write_config(config, filename, round_short_name, config_type): print(content_to_write, file=f) # Print the dictionary for non-JSON types elif config_type == "html": f.write(content_to_write) + + +# Function to validate JSON data against the schema +def validate_json(data, schema): + try: + validate(instance=data, schema=schema) + current_app.logger.info("Given JSON data is valid") + return True + except jsonschema.exceptions.ValidationError as err: + current_app.logger.error("Given JSON data is invalid") + current_app.logger.error(err) + return False diff --git a/app/db/database_ERD_9-8-24.png b/app/db/database_ERD_9-8-24.png new file mode 100644 index 0000000..5fc5855 Binary files /dev/null and b/app/db/database_ERD_9-8-24.png differ diff --git a/requirements-dev.txt b/requirements-dev.txt index 9a81990..d7f8cf0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile requirements-dev.in @@ -10,6 +10,10 @@ alembic==1.13.2 # via flask-migrate async-timeout==4.0.3 # via redis +attrs==24.2.0 + # via + # jsonschema + # referencing babel==2.15.0 # via flask-babel beautifulsoup4==4.12.3 @@ -67,8 +71,6 @@ editorconfig==0.12.4 # via # cssbeautifier # jsbeautifier -exceptiongroup==1.2.2 - # via pytest filelock==3.15.4 # via virtualenv flake8==7.0.0 @@ -156,6 +158,10 @@ jsmin==3.0.1 # via -r requirements.in json5==0.9.25 # via djlint +jsonschema==4.23.0 + # via -r requirements.in +jsonschema-specifications==2023.12.1 + # via jsonschema mako==1.3.5 # via alembic markupsafe==2.1.5 @@ -210,9 +216,7 @@ pyflakes==3.2.0 pygments==2.18.0 # via rich pyjwt[crypto]==2.8.0 - # via - # funding-service-design-utils - # pyjwt + # via funding-service-design-utils pyscss==1.4.0 # via -r requirements.in pytest==8.2.2 @@ -253,6 +257,10 @@ redis==4.6.0 # via # flask-redis # flipper-client +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications regex==2023.12.25 # via djlint requests==2.32.3 @@ -262,6 +270,10 @@ requests==2.32.3 # python-consul rich==12.6.0 # via funding-service-design-utils +rpds-py==0.20.0 + # via + # jsonschema + # referencing s3transfer==0.10.2 # via boto3 sentry-sdk[flask]==2.10.0 @@ -282,7 +294,6 @@ sqlalchemy[mypy]==2.0.31 # alembic # flask-sqlalchemy # marshmallow-sqlalchemy - # sqlalchemy # sqlalchemy-json # sqlalchemy-utils sqlalchemy-json==0.7.0 @@ -293,20 +304,11 @@ sqlalchemy-utils==0.41.2 # funding-service-design-utils thrift==0.20.0 # via flipper-client -tomli==2.0.1 - # via - # black - # djlint - # flake8-pyproject - # mypy - # pytest - # pytest-env tqdm==4.66.4 # via djlint typing-extensions==4.12.2 # via # alembic - # black # mypy # sqlalchemy urllib3==2.2.2 diff --git a/requirements.in b/requirements.in index 5546570..f63a0ad 100644 --- a/requirements.in +++ b/requirements.in @@ -58,3 +58,4 @@ Flask-WTF==1.2.1 # Utils #----------------------------------- funding-service-design-utils>=4.0.1 +jsonschema diff --git a/requirements.txt b/requirements.txt index 5eaf1b7..99cf5cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile requirements.in @@ -10,6 +10,10 @@ alembic==1.13.2 # via flask-migrate async-timeout==4.0.3 # via redis +attrs==24.2.0 + # via + # jsonschema + # referencing babel==2.15.0 # via flask-babel beautifulsoup4==4.12.3 @@ -106,6 +110,10 @@ jmespath==1.0.1 # botocore jsmin==3.0.1 # via -r requirements.in +jsonschema==4.23.0 + # via -r requirements.in +jsonschema-specifications==2023.12.1 + # via jsonschema mako==1.3.5 # via alembic markupsafe==2.1.5 @@ -136,9 +144,7 @@ pyee==6.0.0 pygments==2.18.0 # via rich pyjwt[crypto]==2.8.0 - # via - # funding-service-design-utils - # pyjwt + # via funding-service-design-utils pyscss==1.4.0 # via -r requirements.in python-consul==1.1.0 @@ -159,6 +165,10 @@ redis==4.6.0 # via # flask-redis # flipper-client +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications requests==2.32.3 # via # -r requirements.in @@ -166,6 +176,10 @@ requests==2.32.3 # python-consul rich==12.6.0 # via funding-service-design-utils +rpds-py==0.20.0 + # via + # jsonschema + # referencing s3transfer==0.10.2 # via boto3 sentry-sdk[flask]==2.10.0 @@ -184,7 +198,6 @@ sqlalchemy[mypy]==2.0.31 # alembic # flask-sqlalchemy # marshmallow-sqlalchemy - # sqlalchemy # sqlalchemy-json # sqlalchemy-utils sqlalchemy-json==0.7.0 @@ -195,8 +208,6 @@ sqlalchemy-utils==0.41.2 # funding-service-design-utils thrift==0.20.0 # via flipper-client -tomli==2.0.1 - # via mypy typing-extensions==4.12.2 # via # alembic diff --git a/tasks/export_tasks.py b/tasks/export_tasks.py index 925a69d..4dce9de 100644 --- a/tasks/export_tasks.py +++ b/tasks/export_tasks.py @@ -67,7 +67,7 @@ def publish_form_json_to_runner(c, filename): ] = f"http://{Config.FAB_HOST}{Config.FAB_SAVE_PER_PAGE}" try: publish_response = requests.post( - url="http://localhost:3009/publish", + url=f"{Config.FORM_RUNNER_URL}/publish", json={"id": human_to_kebab_case(form_dict["name"]), "configuration": form_dict}, ) if not str(publish_response.status_code).startswith("2"): diff --git a/tests/test_config_generators.py b/tests/test_config_generators.py index be8ac97..50b6e77 100644 --- a/tests/test_config_generators.py +++ b/tests/test_config_generators.py @@ -15,6 +15,7 @@ from app.config_generator.scripts.generate_fund_round_html import ( generate_all_round_html, ) +from app.config_generator.scripts.helpers import validate_json output_base_path = Path("app") / "config_generator" / "output" @@ -221,3 +222,33 @@ def test_generate_fund_round_html_invalid_input(seed_dynamic_data): # Execute and Assert: Ensure the function raises an exception for invalid inputs with pytest.raises(ValueError): generate_all_round_html(round_id) + + +test_json_schema = { + "type": "object", + "properties": {"name": {"type": "string"}, "age": {"type": "number"}}, + "required": ["name", "age"], +} + + +def test_valid_data_validate_json(): + # Data that matches the schema + data = {"name": "John Doe", "age": 30} + result = validate_json(data, test_json_schema) + assert result, "The data should be valid according to the schema" + + +@pytest.mark.parametrize( + "data", + [ + ({"age": 30}), # Missing 'name' + ({"name": 123}), # 'name' should be a string + ({"name": ""}), # 'name' is empty + ({}), # Empty object + ({"name": "John Doe", "extra_field": "not allowed"}), # Extra field not defined in schema + # Add more invalid cases as needed + ], +) +def test_invalid_data_validate_json(data): + result = validate_json(data, test_json_schema) + assert not result, "The data should be invalid according to the schema"