Merge pull request #76 from communitiesuk/include-psql-client-dockerfile

added psql client in dockerfile & updated tasks file

Dockerfile

@@ -4,6 +4,7 @@ WORKDIR /app
 COPY requirements-dev.txt requirements-dev.txt
 RUN pip --no-cache-dir install --ignore-installed distlib -r requirements-dev.txt
 RUN pip install gunicorn
+RUN apt-get update && apt-get install -y postgresql-client
 COPY . .
 
 EXPOSE 8080

README.md

@@ -107,11 +107,11 @@ You can also force all defined rounds to be open by setting the environment vari
 To seed fund & round data to db
 
 ```
-docker exec -ti <fund_store_container_id> scripts/load_cof_r2.py
+docker exec -ti <fund_store_container_id> python -m scripts.load_cof_r2
 
 ```
 ```
-docker exec -ti <fund_store_container_id> scripts/load_cof_r3w1.py
+docker exec -ti <fund_store_container_id> python -m scripts.load_cof_r3w1
 
 ```
 To amend the round dates

db/README

@@ -2,7 +2,18 @@
 
 Tasks to help with dropping and recreating, then repopulating the db during development are in `tasks.py`
 
-1. Set local DATABASE_URL in environment (default in config is `postgresql://postgres:[email protected]:5432/fsd_fund_store_1`)
+Tasks can be run locally and in container
+
+- To run locally, make sure `psql` client is installed on your machine(https://www.postgresql.org/download/) & set the environment variable `DATABASE_URL`,
+```bash
+export DATABASE_URL=postgresql://postgres:[email protected]:5432/fund_store
+```
+
+- To run the tasks in the container, bash into the container
+```bash
+docker exec -it <container_id> bash
+```
+
 1. To recreate the db instance, run
 
         inv recreate-local-db

scripts/load_cof_r2.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 # flake8: noqa
-from app import app  # noqa: E402
 from config import Config  # noqa: E402
 from db.queries import insert_application_sections
 from db.queries import insert_assessment_sections
@@ -51,87 +50,93 @@ def create_sections(path_prefix, round_id, forms_config):
     # assessment_result = insert_assessment_sections(cof_form_config.COF_ROUND_2_ID, assessment_config)
 
 
-if __name__ == "__main__":
-    with app.app_context():
+def main() -> None:
 
-        # -- load fund and rounds --
-        fund_config = {
-            "id": "47aef2f5-3fcb-4d45-acb5-f0152b5f03c4",
-            "name": "Community Ownership Fund",
-            "title": "funding to save an asset in your community",
-            "short_name": "COF",
-            "description": (
-                "The Community Ownership Fund is a £150 million fund over 4 years"
-                " to support community groups across England, Wales, Scotland and"
-                " Northern Ireland to take ownership of assets which are at risk"
-                " of being lost to the community."
+    # -- load fund and rounds --
+    fund_config = {
+        "id": "47aef2f5-3fcb-4d45-acb5-f0152b5f03c4",
+        "name": "Community Ownership Fund",
+        "title": "funding to save an asset in your community",
+        "short_name": "COF",
+        "description": (
+            "The Community Ownership Fund is a £150 million fund over 4 years"
+            " to support community groups across England, Wales, Scotland and"
+            " Northern Ireland to take ownership of assets which are at risk"
+            " of being lost to the community."
+        ),
+    }
+
+    rounds_config = [
+        {
+            "id": "c603d114-5364-4474-a0c4-c41cbf4d3bbd",
+            "title": "Round 2 Window 2",
+            "short_name": "R2W2",
+            "opens": "2022-10-04 12:00:00",
+            "deadline": "2022-12-14 11:59:00",
+            "fund_id": "47aef2f5-3fcb-4d45-acb5-f0152b5f03c4",
+            "assessment_deadline": "2023-03-30 12:00:00",
+            "prospectus": "https://www.gov.uk/government/publications/community-ownership-fund-prospectus",
+            "privacy_notice": "https://www.gov.uk/government/publications/community-ownership-fund-privacy-notice/community-ownership-fund-privacy-notice",
+            "contact_email": "[email protected]",
+            "contact_phone": None,
+            "contact_textphone": None,
+            "support_times": "9am to 5pm",
+            "support_days": "Monday to Friday",
+            "instructions": (
+                "You must have received an invitation to apply. If we did not invite"
+                " you, first <a"
+                ' href="https://www.gov.uk/government/publications/community-ownership-fund-prospectus">'
+                " express your interest in the fund</a>."
+            ),
+        },
+        {
+            "id": "5cf439bf-ef6f-431e-92c5-a1d90a4dd32f",
+            "title": "Round 2 Window 3",
+            "short_name": "R2W3",
+            "opens": "2022-10-04 12:00:00",
+            "deadline": "2022-12-14 11:59:00",
+            "fund_id": "47aef2f5-3fcb-4d45-acb5-f0152b5f03c4",
+            "assessment_deadline": "2023-03-30 12:00:00",
+            "prospectus": "https://www.gov.uk/government/publications/community-ownership-fund-prospectus",
+            "privacy_notice": "https://www.gov.uk/government/publications/community-ownership-fund-privacy-notice/community-ownership-fund-privacy-notice",
+            "contact_email": "[email protected]",
+            "contact_phone": None,
+            "contact_textphone": None,
+            "support_times": "9am to 5pm",
+            "support_days": "Monday to Friday",
+            "instructions": (
+                "You must have received an invitation to apply. If we did not invite"
+                " you, first <a"
+                ' href="https://www.gov.uk/government/publications/community-ownership-fund-prospectus">'
+                " express your interest in the fund</a>."
             ),
-        }
+        },
+    ]
+
+    inserted_fund = insert_fund_data(fund_config)
+    print("Fund inserted:")
+    print(inserted_fund)
+    inserted_rounds = insert_round_data(rounds_config)
+    print("Rounds inserted:")
+    print(inserted_rounds)
 
-        rounds_config = [
-            {
-                "id": "c603d114-5364-4474-a0c4-c41cbf4d3bbd",
-                "title": "Round 2 Window 2",
-                "short_name": "R2W2",
-                "opens": "2022-10-04 12:00:00",
-                "deadline": "2022-12-14 11:59:00",
-                "fund_id": "47aef2f5-3fcb-4d45-acb5-f0152b5f03c4",
-                "assessment_deadline": "2023-03-30 12:00:00",
-                "prospectus": "https://www.gov.uk/government/publications/community-ownership-fund-prospectus",
-                "privacy_notice": "https://www.gov.uk/government/publications/community-ownership-fund-privacy-notice/community-ownership-fund-privacy-notice",
-                "contact_email": "[email protected]",
-                "contact_phone": None,
-                "contact_textphone": None,
-                "support_times": "9am to 5pm",
-                "support_days": "Monday to Friday",
-                "instructions": (
-                    "You must have received an invitation to apply. If we did not"
-                    " invite you, first <a"
-                    ' href="https://www.gov.uk/government/publications/community-ownership-fund-prospectus">'
-                    " express your interest in the fund</a>."
-                ),
-            },
-            {
-                "id": "5cf439bf-ef6f-431e-92c5-a1d90a4dd32f",
-                "title": "Round 2 Window 3",
-                "short_name": "R2W3",
-                "opens": "2022-10-04 12:00:00",
-                "deadline": "2022-12-14 11:59:00",
-                "fund_id": "47aef2f5-3fcb-4d45-acb5-f0152b5f03c4",
-                "assessment_deadline": "2023-03-30 12:00:00",
-                "prospectus": "https://www.gov.uk/government/publications/community-ownership-fund-prospectus",
-                "privacy_notice": "https://www.gov.uk/government/publications/community-ownership-fund-privacy-notice/community-ownership-fund-privacy-notice",
-                "contact_email": "[email protected]",
-                "contact_phone": None,
-                "contact_textphone": None,
-                "support_times": "9am to 5pm",
-                "support_days": "Monday to Friday",
-                "instructions": (
-                    "You must have received an invitation to apply. If we did not"
-                    " invite you, first <a"
-                    ' href="https://www.gov.uk/government/publications/community-ownership-fund-prospectus">'
-                    " express your interest in the fund</a>."
-                ),
-            },
-        ]
+    # Do we want to reuse the application sections config?
 
-        inserted_fund = insert_fund_data(fund_config)
-        print("Fund inserted:")
-        print(inserted_fund)
-        inserted_rounds = insert_round_data(rounds_config)
-        print("Rounds inserted:")
-        print(inserted_rounds)
+    # Separate config for r2w2 and r2w3
+    # create_sections("1", cof_form_config.COF_ROUND_2_ID, cof_form_config.COF_R2_ORDERED_FORMS_CONFIG, True)
+    # create_sections("2", cof_form_config.COF_ROUND_2_W3_ID, cof_form_config.COF_R2_ORDERED_FORMS_CONFIG, True)
 
-        # Do we want to reuse the application sections config?
+    # reuse config between r2w2 and r2w3
+    create_sections(
+        "1",
+        cof_form_config.COF_ROUND_2_ID,
+        cof_form_config.COF_R2_ORDERED_FORMS_CONFIG,
+    )
+    create_sections("1", cof_form_config.COF_ROUND_2_W3_ID, None)
 
-        # Separate config for r2w2 and r2w3
-        # create_sections("1", cof_form_config.COF_ROUND_2_ID, cof_form_config.COF_R2_ORDERED_FORMS_CONFIG, True)
-        # create_sections("2", cof_form_config.COF_ROUND_2_W3_ID, cof_form_config.COF_R2_ORDERED_FORMS_CONFIG, True)
 
-        # reuse config between r2w2 and r2w3
-        create_sections(
-            "1",
-            cof_form_config.COF_ROUND_2_ID,
-            cof_form_config.COF_R2_ORDERED_FORMS_CONFIG,
-        )
-        create_sections("1", cof_form_config.COF_ROUND_2_W3_ID, None)
+if __name__ == "__main__":
+    from app import app
+
+    with app.app_context():
+        main()

scripts/load_cof_r3w1.py

@@ -1,4 +1,4 @@
-from app import app  # noqa: E402
+#!/usr/bin/env python3
 from db.queries import insert_application_sections
 from db.queries import insert_fund_data
 from db.queries import insert_round_data
@@ -183,15 +183,22 @@
     }
 ]
 
+
+def main() -> None:
+    print("Inserting fund and round data.")
+    insert_fund_data(fund_config)
+    insert_round_data(round_config)
+
+    print("Inserting base sections config.")
+    insert_application_sections(COF_ROUND_3_WINDOW_1_ID, tree_base_sections)
+    print("Inserting sections.")
+    insert_application_sections(
+        COF_ROUND_3_WINDOW_1_ID, sorted_application_r3w1_sections
+    )
+
+
 if __name__ == "__main__":
-    with app.app_context():
-        print("Inserting fund and round data.")
-        insert_fund_data(fund_config)
-        insert_round_data(round_config)
+    from app import app
 
-        print("Inserting base sections config.")
-        insert_application_sections(COF_ROUND_3_WINDOW_1_ID, tree_base_sections)
-        print("Inserting sections.")
-        insert_application_sections(
-            COF_ROUND_3_WINDOW_1_ID, sorted_application_r3w1_sections
-        )
+    with app.app_context():
+        main()

tasks.py

@@ -1,3 +1,6 @@
+import os
+from urllib.parse import urlparse
+
 from colored import attr
 from colored import fg
 from colored import stylize
@@ -7,20 +10,29 @@
 
 
 @task
-def recreate_local_db(c, database_host="localhost", db_name="fsd_fund_store_1"):
+def recreate_local_db(c):
     """Create a clean database for testing"""
-    c.run(f"dropdb -h {database_host} --if-exists {db_name} --force")
+
+    # As we assume "db_name" is not yet created. First we need to connect to psql with an existing database
+    # Replace database in database_url with "postgres" db
+    database_url = os.environ.get("DATABASE_URL")
+    if not database_url:
+        raise Exception("Please set the environmental variable 'DATABASE_URL'!")
+    parsed_db_url = urlparse(database_url)
+    database_host = parsed_db_url.hostname
+    db_name = parsed_db_url.path.lstrip("/")
+    parsed_db_url = parsed_db_url._replace(path="/postgres")
+    database_url = parsed_db_url.geturl()
+
+    c.run(f'psql {database_url} -c "DROP DATABASE IF EXISTS {db_name} WITH (FORCE);"')
     print(
         stylize(
             f"{db_name} db dropped from {database_host}...",
             ECHO_STYLE,
         )
     )
-    c.run(f"createdb -h {database_host} {db_name}")
-    c.run(
-        f'psql -h {database_host} -d {db_name} -c "create extension if not'
-        ' exists ltree;"'
-    )
+    c.run(f'psql {database_url} -c "CREATE DATABASE {db_name};"')
+    c.run(f'psql {database_url} -c "create extension if not exists ltree;"')
     print(stylize(f"{db_name} db created on {database_host}...", ECHO_STYLE))
 
 
@@ -45,40 +57,44 @@ def init_migr(c, database_host="localhost", db_name="fsd_fund_store_1"):
 
 
 @task
-def seed_db(c, database_host="localhost", db_name="fsd_fund_store_1"):
+def seed_db(c):
+    """Seed the Fund data into the database."""
+
+    database_url = os.environ.get("DATABASE_URL")
+    if not database_url:
+        raise Exception("Please set the environmental variable 'DATABASE_URL'!")
+
     c.run("flask db upgrade")
     print(
         stylize(
             "Run upgrade",
             ECHO_STYLE,
         )
     )
-    c.run(f"psql -h {database_host} -d {db_name} -a -f db/cof_sql/fund.sql")
-    c.run(f"psql -h {database_host} -d {db_name} -a -f db/cof_sql/rounds.sql")
-    c.run(f"psql -h {database_host} -d {db_name} -a -f db/cof_sql/sections.sql")
-    c.run(
-        f"psql -h {database_host} -d {db_name} -a -f db/cof_sql/assessment_fields.sql"
-    )
-    c.run(f"psql -h {database_host} -d {db_name} -a -f db/cof_sql/section_fields.sql")
+    c.run(f"psql {database_url} -a -f db/cof_sql/fund.sql")
+    c.run(f"psql {database_url} -a -f db/cof_sql/rounds.sql")
+    c.run(f"psql {database_url} -a -f db/cof_sql/sections.sql")
+    c.run(f"psql {database_url} -a -f db/cof_sql/assessment_fields.sql")
+    c.run(f"psql {database_url} -a -f db/cof_sql/section_fields.sql")
     # c.run(
-    #     f"psql -h {database_host} -d {db_name} -a -f"
+    #     f"psql {database_url} -a -f"
     #     " db/cof_sql/translations.sql"
     # )
-    c.run(f"psql -h {database_host} -d {db_name} -a -f db/cof_sql/form_name.sql")
+    c.run(f"psql {database_url} -a -f db/cof_sql/form_name.sql")
     c.run(
-        f'psql -h {database_host} -d {db_name} -c "select f.short_name as'
+        f'psql {database_url} -c "select f.short_name as'
         " Fund, r.short_name as Round from round r join fund f on r.fund_id ="
         ' f.id";'
     )
     c.run(
-        f'psql -h {database_host} -d {db_name} -c "select f.short_name,'
+        f'psql {database_url} -c "select f.short_name,'
         " r.short_name, s.title, s.weighting, s.path from section s join"
         " round r on round_id=r.id join fund f on r.fund_id = f.id order by"
         ' path;"'
     )
 
     c.run(
-        f'psql -h {database_host} -d {db_name} -c "select s.title, f.title'
+        f'psql {database_url} -c "select s.title, f.title'
         " from section s left outer join section_field sf on s.id ="
         " sf.section_id left outer join assessment_field f on sf.field_id ="
         ' f.id order by s.path, sf.display_order;"'