From 5636633ef735cfd7954d127e8f6aac187e5548b4 Mon Sep 17 00:00:00 2001 From: srh-sloan Date: Wed, 16 Oct 2024 13:42:12 +0000 Subject: [PATCH] Fix path errors --- app/blueprints/fund_builder/routes.py | 16 +-- config/envs/default.py | 3 +- tests/conftest.py | 9 ++ tests/test_config_export.py | 146 ++++++++++++-------------- tests/test_integration.py | 107 +++++++++---------- 5 files changed, 136 insertions(+), 145 deletions(-) diff --git a/app/blueprints/fund_builder/routes.py b/app/blueprints/fund_builder/routes.py index e7286ca..4ad8753 100644 --- a/app/blueprints/fund_builder/routes.py +++ b/app/blueprints/fund_builder/routes.py @@ -522,6 +522,14 @@ def view_form_questions(round_id, form_id): "view_questions.html", round=round, fund=fund, question_html=html, title=form.name_in_apply_json["en"] ) +def create_export_zip(directory_to_zip, zip_file_name) -> str: + # Output zip file path (temporary) + output_zip_path = Config.TEMP_FILE_PATH / zip_file_name + + # Create a zip archive of the directory + shutil.make_archive(base_name=output_zip_path, format="zip", root_dir=directory_to_zip) + return f"{output_zip_path}.zip" + @build_fund_bp.route("/create_export_files/", methods=["GET"]) def create_export_files(round_id): @@ -530,13 +538,7 @@ def create_export_files(round_id): generate_config_for_round(round_id) round_short_name = get_round_by_id(round_id).short_name - # Directory to zip - directory_to_zip = Config.TEMP_FILE_PATH / "round_short_name" - # Output zip file path (temporary) - output_zip_path = Config.TEMP_FILE_PATH / f"{round_short_name}.zip" - - # Create a zip archive of the directory - shutil.make_archive(output_zip_path.replace(".zip", ""), "zip", directory_to_zip) + output_zip_path = create_export_zip(directory_to_zip=Config.TEMP_FILE_PATH / round_short_name, zip_file_name=round_short_name) # Ensure the file is removed after sending it @after_this_request diff --git a/config/envs/default.py b/config/envs/default.py index 1d93e22..2b2b033 100644 --- a/config/envs/default.py +++ b/config/envs/default.py @@ -1,6 +1,7 @@ import logging from os import environ from os import getenv +from pathlib import Path from fsd_utils import CommonConfig from fsd_utils import configclass @@ -19,4 +20,4 @@ class DefaultConfig(object): FORM_RUNNER_URL_REDIRECT = getenv("FORM_RUNNER_EXTERNAL_HOST", "http://localhost:3009") SQLALCHEMY_DATABASE_URI = environ.get("DATABASE_URL") - TEMP_FILE_PATH="/tmp" \ No newline at end of file + TEMP_FILE_PATH=Path("/tmp") \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 15ff5ed..3104a0f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,10 +8,19 @@ from tests.helpers import create_test_fund from tests.helpers import create_test_organisation from tests.helpers import create_test_round +from config import Config +import shutil pytest_plugins = ["fsd_test_utils.fixtures.db_fixtures"] +@pytest.fixture(scope="session") +def temp_output_dir(): + temp_dir = Config.TEMP_FILE_PATH + yield temp_dir + if temp_dir.exists(): + shutil.rmtree(temp_dir) + @pytest.fixture def test_fund(flask_test_client, _db, clear_test_data): """ diff --git a/tests/test_config_export.py b/tests/test_config_export.py index b831834..8756ab0 100644 --- a/tests/test_config_export.py +++ b/tests/test_config_export.py @@ -5,6 +5,7 @@ import pytest +from app.blueprints.fund_builder.routes import create_export_zip from app.export_config.generate_fund_round_config import generate_config_for_round from app.export_config.generate_fund_round_form_jsons import ( generate_form_jsons_for_round, @@ -13,9 +14,6 @@ from app.export_config.generate_fund_round_html import frontend_html_suffix from app.export_config.generate_fund_round_html import generate_all_round_html from app.export_config.helpers import validate_json -from config import Config - -output_base_path = Config.TEMP_FILE_PATH def read_data_from_output_file(file): @@ -27,7 +25,7 @@ def read_data_from_output_file(file): return data -def test_generate_config_for_round_valid_input(seed_dynamic_data, monkeypatch): +def test_generate_config_for_round_valid_input(seed_dynamic_data, monkeypatch, temp_output_dir): # Setup: Prepare valid input parameters fund_short_name = seed_dynamic_data["funds"][0].short_name round_id = seed_dynamic_data["rounds"][0].round_id @@ -45,7 +43,7 @@ def test_generate_config_for_round_valid_input(seed_dynamic_data, monkeypatch): # Assert: Check if the directory structure and files are created as expected expected_files = [ { - "path": output_base_path / round_short_name / "fund_store" / "round_config.py", + "path": temp_output_dir / round_short_name / "fund_store" / "round_config.py", "expected_output": { "sections_config": [ { @@ -97,49 +95,43 @@ def test_generate_config_for_round_valid_input(seed_dynamic_data, monkeypatch): }, }, ] - try: - for expected_file in expected_files: - path = expected_file["path"] - assert path.exists(), f"Expected file {path} does not exist." - - with open(expected_file["path"], "r") as file: - data = read_data_from_output_file(file=file) - - if expected_file["expected_output"].get("fund_config", None): - # remove keys that can't be accurately compared - keys_to_remove = ["base_path"] - keys_to_remove_fund_config = ["id"] - keys_to_remove_round_config = [ - "id", - "fund_id", - "reminder_date", - "assessment_start", - "assessment_deadline", - "deadline", - "opens", - ] - keys_to_remove_section_config = ["tree_path"] - data = {k: v for k, v in data.items() if k not in keys_to_remove} - data["fund_config"] = { - k: v for k, v in data["fund_config"].items() if k not in keys_to_remove_fund_config - } - data["round_config"] = { - k: v for k, v in data["round_config"].items() if k not in keys_to_remove_round_config - } - data["sections_config"] = [ - {k: v for k, v in section.items() if k not in keys_to_remove_section_config} - for section in data["sections_config"] - ] - assert expected_file["expected_output"]["fund_config"] == data["fund_config"] - assert expected_file["expected_output"]["round_config"] == data["round_config"] - assert expected_file["expected_output"]["sections_config"] == data["sections_config"] - else: - assert data == expected_file["expected_output"] - finally: - # Cleanup step to remove the directory - directory_path = output_base_path / round_short_name - if directory_path.exists(): - shutil.rmtree(directory_path) + for expected_file in expected_files: + path = expected_file["path"] + assert path.exists(), f"Expected file {path} does not exist." + + with open(expected_file["path"], "r") as file: + data = read_data_from_output_file(file=file) + + if expected_file["expected_output"].get("fund_config", None): + # remove keys that can't be accurately compared + keys_to_remove = ["base_path"] + keys_to_remove_fund_config = ["id"] + keys_to_remove_round_config = [ + "id", + "fund_id", + "reminder_date", + "assessment_start", + "assessment_deadline", + "deadline", + "opens", + ] + keys_to_remove_section_config = ["tree_path"] + data = {k: v for k, v in data.items() if k not in keys_to_remove} + data["fund_config"] = { + k: v for k, v in data["fund_config"].items() if k not in keys_to_remove_fund_config + } + data["round_config"] = { + k: v for k, v in data["round_config"].items() if k not in keys_to_remove_round_config + } + data["sections_config"] = [ + {k: v for k, v in section.items() if k not in keys_to_remove_section_config} + for section in data["sections_config"] + ] + assert expected_file["expected_output"]["fund_config"] == data["fund_config"] + assert expected_file["expected_output"]["round_config"] == data["round_config"] + assert expected_file["expected_output"]["sections_config"] == data["sections_config"] + else: + assert data == expected_file["expected_output"] def test_generate_config_for_round_invalid_input(seed_dynamic_data): @@ -150,7 +142,7 @@ def test_generate_config_for_round_invalid_input(seed_dynamic_data): generate_config_for_round(round_id) -def test_generate_form_jsons_for_round_valid_input(seed_dynamic_data): +def test_generate_form_jsons_for_round_valid_input(seed_dynamic_data,temp_output_dir): # Setup: Prepare valid input parameters round_id = seed_dynamic_data["rounds"][0].round_id round_short_name = seed_dynamic_data["rounds"][0].short_name @@ -161,7 +153,7 @@ def test_generate_form_jsons_for_round_valid_input(seed_dynamic_data): # Assert: Check if the directory structure and files are created as expected expected_files = [ { - "path": output_base_path / round_short_name / "form_runner" / f"{form_publish_name}.json", + "path": temp_output_dir / round_short_name / "form_runner" / f"{form_publish_name}.json", "expected_output": { "startPage": "/intro-about-your-organisation", "pages": [ @@ -234,22 +226,16 @@ def test_generate_form_jsons_for_round_valid_input(seed_dynamic_data): }, } ] - try: - for expected_file in expected_files: - path = expected_file["path"] - assert path.exists(), f"Expected file {path} does not exist." + for expected_file in expected_files: + path = expected_file["path"] + assert path.exists(), f"Expected file {path} does not exist." - with open(expected_file["path"], "r") as file: - data = json.load(file) - for page in data["pages"]: - for component in page["components"]: - component.pop("metadata", None) - assert data == expected_file["expected_output"] - finally: - # Cleanup step to remove the directory - directory_path = output_base_path / round_short_name - if directory_path.exists(): - shutil.rmtree(directory_path) + with open(expected_file["path"], "r") as file: + data = json.load(file) + for page in data["pages"]: + for component in page["components"]: + component.pop("metadata", None) + assert data == expected_file["expected_output"] def test_generate_form_jsons_for_round_invalid_input(seed_dynamic_data): @@ -260,7 +246,7 @@ def test_generate_form_jsons_for_round_invalid_input(seed_dynamic_data): generate_form_jsons_for_round(round_id) -def test_generate_fund_round_html(seed_dynamic_data): +def test_generate_fund_round_html(seed_dynamic_data, temp_output_dir): # Setup: Prepare valid input parameters round_id = seed_dynamic_data["rounds"][0].round_id @@ -271,7 +257,7 @@ def test_generate_fund_round_html(seed_dynamic_data): # Assert: Check if the directory structure and files are created as expected expected_files = [ { - "path": output_base_path + "path": temp_output_dir / round_short_name / "html" / f"{fund_short_name.casefold()}_{round_short_name.casefold()}_all_questions_en.html", @@ -280,19 +266,13 @@ def test_generate_fund_round_html(seed_dynamic_data): + frontend_html_suffix, } ] - try: - for expected_file in expected_files: - path = expected_file["path"] - assert path.exists(), f"Expected file {path} does not exist." + for expected_file in expected_files: + path = expected_file["path"] + assert path.exists(), f"Expected file {path} does not exist." - with open(expected_file["path"], "r") as file: - data = file.read() - assert data == expected_file["expected_output"] - finally: - # Cleanup step to remove the directory - directory_path = output_base_path / round_short_name - if directory_path.exists(): - shutil.rmtree(directory_path) + with open(expected_file["path"], "r") as file: + data = file.read() + assert data == expected_file["expected_output"] def test_generate_fund_round_html_invalid_input(seed_dynamic_data): @@ -331,3 +311,11 @@ def test_valid_data_validate_json(): def test_invalid_data_validate_json(data): result = validate_json(data, test_json_schema) assert not result, "The data should be invalid according to the schema" + + +def test_create_export_zip(temp_output_dir): + test_data_path = Path("tests") / "test_data" + output = create_export_zip(directory_to_zip=test_data_path, zip_file_name="test_zip") + assert output + output_path = Path(output) + assert output_path.exists() diff --git a/tests/test_integration.py b/tests/test_integration.py index 57c5890..d2576cb 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -285,9 +285,6 @@ def test_list_relationship(seed_dynamic_data): assert result.lizt.name == "classifications_list" -output_base_path = Config.TEMP_FILE_PATH - - # add files in /test_data t orun the below test against each file @pytest.mark.parametrize( "input_filename, output_filename,,expected_page_count_for_form,expected_component_count_for_form", @@ -307,7 +304,7 @@ def test_generate_config_for_round_valid_input( input_filename, output_filename, expected_page_count_for_form, - expected_component_count_for_form, + expected_component_count_for_form,temp_output_dir ): form_configs = [] script_dir = os.path.dirname(__file__) @@ -349,60 +346,54 @@ def test_generate_config_for_round_valid_input( # Simply writes the files to the output directory so no result is given directly assert result is None - try: - # Check if the directory is created - generated_json_form = output_base_path / round_short_name / "form_runner" / output_filename - assert generated_json_form - - # compare the import file with the generated file - with open(generated_json_form, "r") as file: - output_form = json.load(file) - - # Compare the contents of the files - - # ensure the keys of the output form are in the input form keys - assert set(output_form.keys()) - {"name"} <= set( - input_form.keys() - ), "Output form keys are not a subset of input form keys, ignoring 'name'" - - # check conditions length is equal - input_condition_count = len(input_form.get("conditions", [])) - output_condition_count = len(output_form.get("conditions", [])) - assert output_condition_count <= input_condition_count # sometime we remove specified but unused conditions - - # check that content of each page (including page[components] and page[next] within form[pages] is the same - for input_page in input_form["pages"]: - - # find page in output pages - output_page = next((p for p in output_form["pages"] if p["path"] == input_page["path"]), None) - assert input_page["path"] == output_page["path"] - assert input_page["title"] == output_page["title"] - for next_dict in input_page["next"]: - # find next in output page - output_next = next((n for n in output_page["next"] if n["path"] == next_dict["path"]), None) - assert next_dict["path"] == output_next["path"] - assert next_dict.get("condition", None) == output_next.get("condition", None) - - # compare components - for input_component in input_page["components"]: - # find component in output page - output_component = None - for c in output_page.get("components", []): - # Get name or content for both components safely - output_name_or_content = c.get("name") or c.get("content") - input_name_or_content = input_component.get("name") or input_component.get("content") - print(f"Checking output: {output_name_or_content} vs input: {input_name_or_content}") - if output_name_or_content == input_name_or_content: - output_component = c - break - - for key in input_component: - assert input_component[key] == output_component[key] - finally: - # Cleanup step to remove the directory - directory_path = output_base_path / round_short_name - if directory_path.exists(): - shutil.rmtree(directory_path) + # Check if the directory is created + generated_json_form = temp_output_dir / round_short_name / "form_runner" / output_filename + assert generated_json_form + + # compare the import file with the generated file + with open(generated_json_form, "r") as file: + output_form = json.load(file) + + # Compare the contents of the files + + # ensure the keys of the output form are in the input form keys + assert set(output_form.keys()) - {"name"} <= set( + input_form.keys() + ), "Output form keys are not a subset of input form keys, ignoring 'name'" + + # check conditions length is equal + input_condition_count = len(input_form.get("conditions", [])) + output_condition_count = len(output_form.get("conditions", [])) + assert output_condition_count <= input_condition_count # sometime we remove specified but unused conditions + + # check that content of each page (including page[components] and page[next] within form[pages] is the same + for input_page in input_form["pages"]: + + # find page in output pages + output_page = next((p for p in output_form["pages"] if p["path"] == input_page["path"]), None) + assert input_page["path"] == output_page["path"] + assert input_page["title"] == output_page["title"] + for next_dict in input_page["next"]: + # find next in output page + output_next = next((n for n in output_page["next"] if n["path"] == next_dict["path"]), None) + assert next_dict["path"] == output_next["path"] + assert next_dict.get("condition", None) == output_next.get("condition", None) + + # compare components + for input_component in input_page["components"]: + # find component in output page + output_component = None + for c in output_page.get("components", []): + # Get name or content for both components safely + output_name_or_content = c.get("name") or c.get("content") + input_name_or_content = input_component.get("name") or input_component.get("content") + print(f"Checking output: {output_name_or_content} vs input: {input_name_or_content}") + if output_name_or_content == input_name_or_content: + output_component = c + break + + for key in input_component: + assert input_component[key] == output_component[key] # colour_json={