FAB Import Script (#336)

* temp changes

* temp changes

* Updating loading script to work for fab

* Updating data scripts to work with new flask and wsgi config

* Rename script

---------

Co-authored-by: srh-sloan <[email protected]>

.vscode/launch.json

@@ -96,7 +96,21 @@
             "cwd": "${workspaceFolder}",
             "env": { "PYTHONPATH":"${workspaceFolder}"},
             "envFile": "${workspaceFolder}/.env.development",
+            "justMyCode": false
+        },
+        {
+            "name": "Load FAB Round",
+            "type": "python",
+            "request": "launch",
+            "program": "${workspaceFolder}/scripts/fund_round_loaders/load_fund_round_from_fab.py",
+            "console": "integratedTerminal",
+            "host": "localhost",
+            "port": 9091,
+            "cwd": "${workspaceFolder}",
+            "env": { "PYTHONPATH":"${workspaceFolder}"},
+            "envFile": "${workspaceFolder}/.env.development",
             "justMyCode": false,
+            "args": ["--fund_short_code", "COF25-EOI"]
         },
         {
             "name": "Amend Round dates",

config/fund_loader_config/FAB/__init__.py

@@ -0,0 +1,39 @@
+import ast
+import os
+from pathlib import Path
+
+"""
+Goes through all the files in config/fund_loader_config/FAB and adds each round to FAB_FUND_ROUND_CONFIGS
+
+Each file in that directory needs to be python format as per the FAB exports,
+containing one property called LOADER_CONFIG
+See test_fab_round_config.py for example
+"""
+
+FAB_FUND_ROUND_CONFIGS = {}
+
+this_dir = Path("config") / "fund_loader_config" / "FAB"
+
+for file in os.listdir(this_dir):
+    if "__" in file.title():
+        continue
+    with open(this_dir / file, "r") as json_file:
+
+        content = json_file.read().split("LOADER_CONFIG=")[1]
+        loader_config = ast.literal_eval(content)
+        if not loader_config.get("fund_config", None):
+            print("No fund config found in the loader config.")
+            raise ValueError(f"No fund_config found in {file}")
+        if not loader_config.get("round_config", None):
+            print("No round config found in the loader config.")
+            raise ValueError(f"No round_config found in {file}")
+        fund_short_name = loader_config["fund_config"]["short_name"]
+        round_short_name = loader_config["round_config"]["short_name"]
+        FAB_FUND_ROUND_CONFIGS[fund_short_name] = loader_config["fund_config"]
+        if not FAB_FUND_ROUND_CONFIGS[fund_short_name].get("rounds", None):
+            FAB_FUND_ROUND_CONFIGS[fund_short_name]["rounds"] = {}
+        FAB_FUND_ROUND_CONFIGS[fund_short_name]["rounds"][round_short_name] = loader_config["round_config"]
+        FAB_FUND_ROUND_CONFIGS[fund_short_name]["rounds"][round_short_name]["sections_config"] = loader_config[
+            "sections_config"
+        ]
+        FAB_FUND_ROUND_CONFIGS[fund_short_name]["rounds"][round_short_name]["base_path"] = loader_config["base_path"]

config/fund_loader_config/common_fund_config/fund_base_tree_paths.py

@@ -1,5 +1,6 @@
 # Should increment for each new round, anything that shares the same base path will also share
 # the child tree path config.
+#
 COF_R2_W2_BASE_PATH = 1
 COF_R2_W3_BASE_PATH = 1
 COF_R3_W1_BASE_PATH = 2
@@ -12,3 +13,5 @@
 COF_R4_W1_BASE_PATH = 9
 HSRA_BASE_PATH = 10
 COF_R4_W2_BASE_PATH = 11
+
+FAB_BASE_PATH = 0

db/queries.py

@@ -235,7 +235,7 @@ def insert_sections(sections):
     db.session.commit()
 
 
-def insert_fund_data(fund_config):
+def insert_fund_data(fund_config, commit: bool = True):
     stmt = (
         (
             postgres_insert(Fund).values(
@@ -283,101 +283,116 @@ def insert_fund_data(fund_config):
 
     result = db.session.execute(stmt, update_params)
     inserted_fund_ids = [row.id for row in result]
-    db.session.commit()
-    print(f"Inserted funds: '{inserted_fund_ids}'.")
+
+    print(f"Prepared fund for insert: '{inserted_fund_ids}'.")
+    if commit:
+        db.session.commit()
+        print("DB changes committed")
     return inserted_fund_ids
 
 
-def insert_round_data(round_config):
+def upsert_round_data(round_configs, commit: bool = True):
     # Create dictionary to store updated records
     updated_rounds = {}
 
-    for item in round_config:
+    for round_config in round_configs:
         # Check if record exist
-        round_record = Round.query.filter_by(id=item["id"]).first()
+        round_record = Round.query.filter_by(id=round_config["id"]).first()
 
         if round_record is not None:
             # Update existing round record
-            round_record.title_json = item["title_json"]
-            round_record.short_name = item["short_name"]
-            round_record.opens = item["opens"]
-            round_record.assessment_start = item["assessment_start"]
-            round_record.deadline = item["deadline"]
-            round_record.application_reminder_sent = item["application_reminder_sent"]
-            round_record.reminder_date = item["reminder_date"]
-            round_record.fund_id = item["fund_id"]
-            round_record.assessment_deadline = item["assessment_deadline"]
-            round_record.prospectus = item["prospectus"]
-            round_record.privacy_notice = item["privacy_notice"]
-            round_record.reference_contact_page_over_email = item["reference_contact_page_over_email"]
-            round_record.contact_us_banner_json = item["contact_us_banner_json"]
-            round_record.contact_email = item["contact_email"]
-            round_record.contact_phone = item["contact_phone"]
-            round_record.contact_textphone = item["contact_textphone"]
-            round_record.support_times = item["support_times"]
-            round_record.support_days = item["support_days"]
-            round_record.instructions_json = item["instructions_json"]
-            round_record.project_name_field_id = item["project_name_field_id"]
-            round_record.feedback_link = item["feedback_link"]
-            round_record.application_guidance_json = item["application_guidance_json"]
-            round_record.guidance_url = item["guidance_url"]
-            round_record.all_uploaded_documents_section_available = item["all_uploaded_documents_section_available"]
-            round_record.application_fields_download_available = item["application_fields_download_available"]
-            round_record.display_logo_on_pdf_exports = item["display_logo_on_pdf_exports"]
-            round_record.feedback_survey_config = item["feedback_survey_config"]
-            round_record.mark_as_complete_enabled = item["mark_as_complete_enabled"]
-            round_record.is_expression_of_interest = item["is_expression_of_interest"]
-            round_record.eligibility_config = item["eligibility_config"]
-            round_record.eoi_decision_schema = item["eoi_decision_schema"]
-
-            updated_rounds[item["id"]] = round_record
+            round_record.title_json = round_config["title_json"]
+            round_record.short_name = round_config["short_name"]
+            round_record.opens = round_config["opens"]
+            round_record.assessment_start = round_config["assessment_start"]
+            round_record.deadline = round_config["deadline"]
+            round_record.application_reminder_sent = round_config["application_reminder_sent"]
+            round_record.reminder_date = round_config["reminder_date"]
+            round_record.fund_id = round_config["fund_id"]
+            round_record.assessment_deadline = round_config["assessment_deadline"]
+            round_record.prospectus = round_config["prospectus"]
+            round_record.privacy_notice = round_config["privacy_notice"]
+            round_record.reference_contact_page_over_email = round_config["reference_contact_page_over_email"]
+            round_record.contact_us_banner_json = round_config["contact_us_banner_json"]
+            round_record.contact_email = round_config["contact_email"]
+            round_record.contact_phone = round_config["contact_phone"]
+            round_record.contact_textphone = round_config["contact_textphone"]
+            round_record.support_times = round_config["support_times"]
+            round_record.support_days = round_config["support_days"]
+            round_record.instructions_json = round_config["instructions_json"]
+            round_record.project_name_field_id = round_config["project_name_field_id"]
+            round_record.feedback_link = round_config["feedback_link"]
+            round_record.application_guidance_json = round_config["application_guidance_json"]
+            round_record.guidance_url = round_config["guidance_url"]
+            round_record.all_uploaded_documents_section_available = round_config[
+                "all_uploaded_documents_section_available"
+            ]
+            round_record.application_fields_download_available = round_config["application_fields_download_available"]
+            round_record.display_logo_on_pdf_exports = round_config["display_logo_on_pdf_exports"]
+            round_record.feedback_survey_config = round_config["feedback_survey_config"]
+            round_record.mark_as_complete_enabled = round_config["mark_as_complete_enabled"]
+            round_record.is_expression_of_interest = round_config["is_expression_of_interest"]
+            round_record.eligibility_config = round_config["eligibility_config"]
+            round_record.eoi_decision_schema = round_config["eoi_decision_schema"]
+
+            updated_rounds[round_config["id"]] = round_record
 
         else:
             # Insert new round record
             new_round = Round(
-                id=item["id"],
-                title_json=item["title_json"],
-                short_name=item["short_name"],
-                opens=item["opens"],
-                assessment_start=item["assessment_start"],
-                deadline=item["deadline"],
-                application_reminder_sent=item["application_reminder_sent"],
-                reminder_date=item["reminder_date"],
-                fund_id=item["fund_id"],
-                assessment_deadline=item["assessment_deadline"],
-                prospectus=item["prospectus"],
-                privacy_notice=item["privacy_notice"],
-                reference_contact_page_over_email=item["reference_contact_page_over_email"],
-                contact_us_banner_json=item["contact_us_banner_json"],
-                contact_email=item["contact_email"],
-                contact_phone=item["contact_phone"],
-                contact_textphone=item["contact_textphone"],
-                support_times=item["support_times"],
-                support_days=item["support_days"],
-                instructions_json=item["instructions_json"],
-                project_name_field_id=item["project_name_field_id"],
-                feedback_link=item["feedback_link"],
-                application_guidance_json=item["application_guidance_json"],
-                guidance_url=item["guidance_url"],
-                all_uploaded_documents_section_available=item["all_uploaded_documents_section_available"],
-                application_fields_download_available=item["application_fields_download_available"],
-                display_logo_on_pdf_exports=item["display_logo_on_pdf_exports"],
-                feedback_survey_config=item["feedback_survey_config"],
-                mark_as_complete_enabled=item["mark_as_complete_enabled"],
-                is_expression_of_interest=item["is_expression_of_interest"],
-                eligibility_config=item["eligibility_config"],
-                eoi_decision_schema=item["eoi_decision_schema"],
+                id=round_config["id"],
+                title_json=round_config["title_json"],
+                short_name=round_config["short_name"],
+                opens=round_config["opens"],
+                assessment_start=round_config["assessment_start"],
+                deadline=round_config["deadline"],
+                application_reminder_sent=round_config["application_reminder_sent"],
+                reminder_date=round_config["reminder_date"],
+                fund_id=round_config["fund_id"],
+                assessment_deadline=round_config["assessment_deadline"],
+                prospectus=round_config["prospectus"],
+                privacy_notice=round_config["privacy_notice"],
+                reference_contact_page_over_email=round_config["reference_contact_page_over_email"],
+                contact_us_banner_json=round_config["contact_us_banner_json"],
+                contact_email=round_config["contact_email"],
+                contact_phone=round_config["contact_phone"],
+                contact_textphone=round_config["contact_textphone"],
+                support_times=round_config["support_times"],
+                support_days=round_config["support_days"],
+                instructions_json=round_config["instructions_json"],
+                project_name_field_id=round_config["project_name_field_id"],
+                feedback_link=round_config["feedback_link"],
+                application_guidance_json=round_config["application_guidance_json"],
+                guidance_url=round_config["guidance_url"],
+                all_uploaded_documents_section_available=round_config["all_uploaded_documents_section_available"],
+                application_fields_download_available=round_config["application_fields_download_available"],
+                display_logo_on_pdf_exports=round_config["display_logo_on_pdf_exports"],
+                feedback_survey_config=round_config["feedback_survey_config"],
+                mark_as_complete_enabled=round_config["mark_as_complete_enabled"],
+                is_expression_of_interest=round_config["is_expression_of_interest"],
+                eligibility_config=round_config["eligibility_config"],
+                eoi_decision_schema=round_config["eoi_decision_schema"],
             )
             db.session.add(new_round)
 
-            updated_rounds[item["id"]] = new_round
+            updated_rounds[round_config["id"]] = new_round
 
-    db.session.commit()
-    print(f"Inserted rounds: '{updated_rounds}'.")
+    print(f"Prepared rounds for insert: '{updated_rounds}'.")
+    if commit:
+        db.session.commit()
+        print("DB changes committed")
     return updated_rounds
 
 
 def insert_base_sections(APPLICATION_BASE_PATH, ASSESSMENT_BASE_PATH, round_id):
+    """
+    Insert base sections for a fund round.
+
+    :param APPLICATION_BASE_PATH: The base path for the application sections.
+    :param ASSESSMENT_BASE_PATH: The base path for the assessment sections.
+    :param round_id: The id of the round to insert the sections for.
+    :return: A dictionary of the inserted sections.
+    """
     tree_base_sections = [
         {
             "section_name": {"en": "Application", "cy": "Application"},
@@ -419,13 +434,12 @@ def insert_base_sections(APPLICATION_BASE_PATH, ASSESSMENT_BASE_PATH, round_id):
 
             updated_sections[section["tree_path"]] = new_section
 
-    db.session.commit()
-    print(f"Inserted sections: '{updated_sections}'.")
+    print(f"Prepared sections for insert: '{updated_sections}'.")
     return updated_sections
 
 
 def insert_or_update_application_sections(round_id, sorted_application_sections: dict):
-    print(f"Inserting forms config: '{sorted_application_sections}'.")
+    print(f"Preparing insert for sections config: '{sorted_application_sections}'.")
     updated_sections = {}
     for section in sorted_application_sections:
         section_record = Section.query.filter(
@@ -469,7 +483,7 @@ def insert_or_update_application_sections(round_id, sorted_application_sections:
                 db.session.add(new_form_record)
                 print(f"Inserted form name information: '{new_form_record}'.")
     db.session.commit()
-    print("Section UPDATES and INSERTS committed.")
+    print("Section UPDATES and INSERTS Prepared for insert.")
 
     return updated_sections
 

scripts/data_updates/FS-2433-application_guidance.py

@@ -32,5 +32,5 @@ def main() -> None:
 if __name__ == "__main__":
     from app import app
 
-    with app.app_context():
+    with app.app.app_context():
         main()

scripts/data_updates/FS-2900-ns-section-update.py

@@ -51,5 +51,5 @@ def main() -> None:
 if __name__ == "__main__":
     from app import app
 
-    with app.app_context():
+    with app.app.app_context():
         main()

scripts/data_updates/FS-2965_ns_guidance.py

@@ -34,5 +34,5 @@ def main() -> None:
 if __name__ == "__main__":
     from app import app
 
-    with app.app_context():
+    with app.app.app_context():
         main()

scripts/data_updates/FS-2982-date-logic.py

@@ -34,5 +34,5 @@ def main() -> None:
 if __name__ == "__main__":
     from app import app
 
-    with app.app_context():
+    with app.app.app_context():
         main()

scripts/data_updates/FS-3471-application_guidance.py

@@ -30,5 +30,5 @@ def main() -> None:
 if __name__ == "__main__":
     from app import app
 
-    with app.app_context():
+    with app.app.app_context():
         main()

scripts/data_updates/FS-3749_dpif_guidance_url.py

@@ -34,5 +34,5 @@ def main() -> None:
 if __name__ == "__main__":
     from app import app
 
-    with app.app_context():
+    with app.app.app_context():
         main()

scripts/data_updates/FS-3808_dpif_application_fields_download_available.py

@@ -34,5 +34,5 @@ def main() -> None:
 if __name__ == "__main__":
     from app import app
 
-    with app.app_context():
+    with app.app.app_context():
         main()

scripts/data_updates/FS-3859_patch_cof_r3w3_section_names.py

@@ -12,5 +12,5 @@ def main() -> None:
 if __name__ == "__main__":
     from app import app
 
-    with app.app_context():
+    with app.app.app_context():
         main()

scripts/data_updates/FS-3866_fix_incorrect_instructions_url.py

@@ -34,5 +34,5 @@ def main() -> None:
 if __name__ == "__main__":
     from app import app
 
-    with app.app_context():
+    with app.app.app_context():
         main()

scripts/data_updates/FS-4264_update_eoi_welsh_instructions.py

@@ -31,5 +31,5 @@ def main() -> None:
 if __name__ == "__main__":
     from app import app
 
-    with app.app_context():
+    with app.app.app_context():
         main()

scripts/data_updates/FS2910_ns_links.py

@@ -35,5 +35,5 @@ def main() -> None:
 if __name__ == "__main__":
     from app import app
 
-    with app.app_context():
+    with app.app.app_context():
         main()

scripts/data_updates/FS2956_ns_weightings.py

@@ -32,5 +32,5 @@ def main() -> None:
 if __name__ == "__main__":
     from app import app
 
-    with app.app_context():
+    with app.app.app_context():
         main()

scripts/data_updates/patch_cof_r3w1_section_names.py

@@ -12,5 +12,5 @@ def main() -> None:
 if __name__ == "__main__":
     from app import app
 
-    with app.app_context():
+    with app.app.app_context():
         main()

scripts/data_updates/patch_cyp_name.py

@@ -24,5 +24,5 @@ def main() -> None:
 if __name__ == "__main__":
     from app import app
 
-    with app.app_context():
+    with app.app.app_context():
         main()