From 7e13ea41bae775447e2108547382fc766ad5a3c5 Mon Sep 17 00:00:00 2001
From: David Kane <david@dkane.net>
Date: Fri, 17 Jan 2025 17:27:26 +0000
Subject: [PATCH] fix for notes change cleaning status to use funder financial
 year

---
 docs/data_model.md                            |   1 -
 .../grantmakers/partials/funderyear.html.j2   |   8 +-
 jinja2/grantmakers/partials/notes.html.j2     |  13 +-
 jinja2/grantmakers/task/detail.html.j2        |  37 +-
 readme.md                                     |  16 +
 ukgrantmaking/admin/funder.py                 |   3 +-
 ukgrantmaking/admin/funder_year.py            |  13 -
 ...1_alter_cleaningstatus_sort_by_and_more.py | 319 ++++++++++++++++++
 ...move_funderfinancialyear_notes_and_more.py |  32 ++
 ...move_funderfinancialyear_notes_and_more.py |  83 +++++
 ...move_funderfinancialyear_notes_and_more.py |  42 +++
 ukgrantmaking/models/cleaningstatus.py        |  52 +--
 ukgrantmaking/models/funder.py                |  11 +-
 ukgrantmaking/models/funder_financial_year.py |   2 -
 ukgrantmaking/models/funder_year.py           |  36 +-
 ukgrantmaking/models/grant.py                 |   4 +-
 ukgrantmaking/tests/conftest.py               |   2 +-
 ukgrantmaking/tests/test_funderyear_htmx.py   |   5 +-
 ukgrantmaking/views/grantmakers.py            |  27 +-
 19 files changed, 584 insertions(+), 122 deletions(-)
 create mode 100644 ukgrantmaking/migrations/0101_alter_cleaningstatus_sort_by_and_more.py
 create mode 100644 ukgrantmaking/migrations/0102_remove_funderfinancialyear_notes_and_more.py
 create mode 100644 ukgrantmaking/migrations/0103_remove_funderfinancialyear_notes_and_more.py
 create mode 100644 ukgrantmaking/migrations/0104_remove_funderfinancialyear_notes_and_more.py

diff --git a/docs/data_model.md b/docs/data_model.md
index 3ad0a1a..7fbcd7b 100644
--- a/docs/data_model.md
+++ b/docs/data_model.md
@@ -80,7 +80,6 @@ the `Funder` model in these attributes will be copied to the equivalent `FunderF
 | `checked`                     | Whether this record has been checked by a user                                          |
 | `checked_on`                  | The time at which this record was marked as checked                                     |
 | `checked_by`                  | The user who marked this record as checked                                              |
-| `notes`                       | Any notes made about this financial year when checking                                  |
 
 #### FunderFinancialYear financial data
 
diff --git a/jinja2/grantmakers/partials/funderyear.html.j2 b/jinja2/grantmakers/partials/funderyear.html.j2
index 4152561..6e23468 100644
--- a/jinja2/grantmakers/partials/funderyear.html.j2
+++ b/jinja2/grantmakers/partials/funderyear.html.j2
@@ -37,10 +37,12 @@
     {% if funder_year.account_url %}
     <br><a href="{{ funder_year.account_url }}" target="_blank">Accounts PDF</a>
     {% endif %}
-    {% if funder_year.notes %}
+    {% if funder_year.notes.count() %}
     <details>
         <summary>Note</summary>
-        {{ funder_year.notes|markdownify|safe }}<br>
+        {% for note in funder_year.notes.all() %}
+        {{ note.note|markdownify|safe }}<br>
+        {% endfor %}
     </details>
     {% endif %}
 </td>
@@ -245,7 +247,7 @@
             <tr>
                 <td class="tr">Note</td>
                 <td colspan="{% if funder_year_py %}3{% else %}2{% endif%}">
-                    <textarea name="note" class="funder-year-edit" rows="7">{{ funder_year.notes or '' }}</textarea>
+                    <textarea name="note" class="funder-year-edit" rows="7"></textarea>
                 </td>
             </tr>
         </tbody>
diff --git a/jinja2/grantmakers/partials/notes.html.j2 b/jinja2/grantmakers/partials/notes.html.j2
index 6b13d42..6b2cd55 100644
--- a/jinja2/grantmakers/partials/notes.html.j2
+++ b/jinja2/grantmakers/partials/notes.html.j2
@@ -8,17 +8,20 @@
         {{ note.note|markdownify|safe }}
     </li>
     {% endfor %}
-    {% for funder_financial_year in object.funder_financial_years.filter(notes__isnull=false).exclude(notes__exact='') %}
+    {% for funder_financial_year in object.funder_financial_years.all() %}
+    {% for funder_year in funder_financial_year.funder_years.all() %}
+    {% for note in funder_year.notes.all() %}
     <li class="note">
         <header>
             <strong>FY{{ funder_financial_year.financial_year_id }}</strong>
-            <time
-                value="{{ funder_financial_year.checked_on }}">{{ funder_financial_year.checked_on|naturalday }}</time>
-            by <strong>{{ funder_financial_year.checked_by|user_name }}</strong>
+            <time value="{{ note.date_added }}">{{ note.date_added|naturalday }}</time>
+            by <strong>{{ note.added_by }}</strong>
         </header>
-        {{ funder_financial_year.notes|markdownify|safe }}
+        {{ note.note|markdownify|safe }}
     </li>
     {% endfor %}
+    {% endfor %}
+    {% endfor %}
     <li class="note">
         <header>Add note</header>
         <form action="{{ url_for('grantmakers:add_note', org_id=object.org_id) }}" method="post"
diff --git a/jinja2/grantmakers/task/detail.html.j2 b/jinja2/grantmakers/task/detail.html.j2
index 4420a08..e5f2843 100644
--- a/jinja2/grantmakers/task/detail.html.j2
+++ b/jinja2/grantmakers/task/detail.html.j2
@@ -30,6 +30,12 @@
             <li>Showing all records</li>
             {% endif %}
         </ul>
+        {% if object._sql_query %}
+        <details>
+            <summary>SQL Query</summary>
+            <pre style="white-space: pre-wrap;">{{ object._sql_query }}</pre>
+        </details>
+        {% endif %}
     </details>
     {% if object.description %}
     <p>{{ object.description }}</p>
@@ -61,26 +67,25 @@
 {% if page_obj.object_list %}
 {% call wrap_table(class_="sticky") %}
 {% for object in page_obj.object_list %}
-<tr class="{% if object.funder_financial_year.checked == 'Checked' %}positive{% endif %}">
+<tr class="{% if object.checked == 'Checked' %}positive{% endif %}">
     <td><a class="b"
-            href="{{ url_for('grantmakers:detail', org_id=object.funder_financial_year.funder.org_id) }}">{{ object.funder_financial_year.funder.name }}</a>
-    <td><a
-            href="{{ url_for('grantmakers:detail', org_id=object.funder_financial_year.funder.org_id) }}">{{ object.funder_financial_year.funder.org_id }}</a>
+            href="{{ url_for('grantmakers:detail', org_id=object.funder.org_id) }}">{{ object.funder.name }}</a>
+    <td><a href="{{ url_for('grantmakers:detail', org_id=object.funder.org_id) }}">{{ object.funder.org_id }}</a>
     </td>
     </td>
-    <td>{{ display_text(object.funder_financial_year.segment) }}</td>
-    <td class="tc">{{ display_boolean(object.funder_financial_year.included) }}</td>
-    <td class="tc">{{ display_boolean(object.funder_financial_year.makes_grants_to_individuals) }}</td>
+    <td>{{ display_text(object.segment) }}</td>
+    <td class="tc">{{ display_boolean(object.included) }}</td>
+    <td class="tc">{{ display_boolean(object.makes_grants_to_individuals) }}</td>
     <td>
-        <ul class="tags">{% for tag in object.funder_financial_year.tags.values_list("tag", flat=true) %}<li>{{ tag }}
+        <ul class="tags">{% for tag in object.tags.values_list("tag", flat=true) %}<li>{{ tag }}
             </li>
             {% endfor %}</ul>
     </td>
     <td class="tc">
         <div class="flex-row">
-            {{ display_boolean(object.funder_financial_year.checked == "Checked") }}
-            {% if object.funder_financial_year.checked_by %}
-            {{ object.funder_financial_year.checked_by|user_name }}
+            {{ display_boolean(object.checked == "Checked") }}
+            {% if object.checked_by %}
+            {{ object.checked_by|user_name }}
             {% endif %}
         </div>
     </td>
@@ -113,13 +118,13 @@
     <td class="tr">{{ display_number(object.funds_unrestricted, format_str="£{:,.0f}") }}</td>
     <td class="tr">{{ display_number(object.employees, format_str="{:,.0f}") }}</td>
     <td class="tr">
-        This: {{ display_number(object.funder_financial_year.scaling, format_str="{:,.0f}") }}<br>
-        {% if object.funder_financial_year.funder.current_year %}
+        This: {{ display_number(object.scaling, format_str="{:,.0f}") }}<br>
+        {% if object.funder.current_year %}
         Current:
-        {{ display_number(object.funder_financial_year.funder.current_year.scaling, format_str="{:,.0f}") }}<br>
+        {{ display_number(object.funder.current_year.scaling, format_str="{:,.0f}") }}<br>
         {% endif %}
-        {% if object.funder_financial_year.funder.latest_year %}
-        Latest: {{ display_number(object.funder_financial_year.funder.latest_year.scaling, format_str="{:,.0f}") }}
+        {% if object.funder.latest_year %}
+        Latest: {{ display_number(object.funder.latest_year.scaling, format_str="{:,.0f}") }}
         {% endif %}
     </td>
 </tr>
diff --git a/readme.md b/readme.md
index a43d44c..b452322 100644
--- a/readme.md
+++ b/readme.md
@@ -178,6 +178,16 @@ without fetching any new data.
 
 #### `python manage.py update financial-year`
 
+This runs a series of SQL commands to keep the financial data up to date. The commands are:
+
+1. Ensure every funder has a funder financial year for the current financial year
+2. Set the current year to the correct year
+3. Set the latest year to the correct year
+4. Recalculate aggregate values for funder financial years
+5. Update current funder financial years with the latest funder data
+6. Ensure that the current funder financial year has the correct tags
+7. Remove existing tags from the current funder financial year
+
 #### `python manage.py update grants`
 
 #### `python manage.py update grant-recipient-type`
@@ -203,3 +213,9 @@ Should instead use the endpoint at `/grantmakers/upload`.
 #### `python manage.py fetch fgt`
 
 Used to import data from the back catalogue of Foundation Giving Trends.
+
+## Export data as backup
+
+```sh
+dokku postgres:export uk-grantmaking-data-db > ~/uk-grantmaking-data-db-20251701.export
+```
diff --git a/ukgrantmaking/admin/funder.py b/ukgrantmaking/admin/funder.py
index 94f3f9a..dedcaea 100644
--- a/ukgrantmaking/admin/funder.py
+++ b/ukgrantmaking/admin/funder.py
@@ -1,4 +1,5 @@
 from django.contrib import admin
+from django.contrib.contenttypes.admin import GenericTabularInline
 from django.utils.html import format_html
 
 from ukgrantmaking.admin.csv_upload import CSVUploadModelAdmin
@@ -16,7 +17,7 @@ def funder_count(self, obj):
         return obj.funders.count()
 
 
-class FunderNoteInline(admin.TabularInline):
+class FunderNoteInline(GenericTabularInline):
     model = FunderNote
     extra = 0
     fields = (
diff --git a/ukgrantmaking/admin/funder_year.py b/ukgrantmaking/admin/funder_year.py
index 3c3f0eb..c4281a2 100644
--- a/ukgrantmaking/admin/funder_year.py
+++ b/ukgrantmaking/admin/funder_year.py
@@ -73,10 +73,6 @@ class FunderYearInline(admin.StackedInline):
                         "spending_grant_making_institutions_unknown_manual",
                     ),
                     ("spending_grant_making_institutions",),
-                    (
-                        "checked_by",
-                        "notes",
-                    ),
                 ],
             },
         ),
@@ -126,7 +122,6 @@ class FunderYearAdmin(CSVUploadModelAdmin):
         "income",
         "spending",
         "spending_charitable",
-        "checked",
     )
     show_facets = admin.ShowFacets.ALWAYS
     list_display_links = ("financial_year_end",)
@@ -136,8 +131,6 @@ class FunderYearAdmin(CSVUploadModelAdmin):
         "funder_financial_year__financial_year__fy",
         "funder_financial_year__segment",
         "funder_financial_year__category",
-        "checked",
-        ("checked_by", admin.EmptyFieldListFilter),
     )
     readonly_fields = (
         "income_registered",
@@ -162,8 +155,6 @@ class FunderYearAdmin(CSVUploadModelAdmin):
         "funds_restricted_registered",
         "funds_unrestricted_registered",
         "employees_registered",
-        "checked_on",
-        "checked",
         "date_added",
         "date_updated",
         # "funder_financial_year",
@@ -254,10 +245,6 @@ class FunderYearAdmin(CSVUploadModelAdmin):
             "Checked",
             {
                 "fields": [
-                    "checked",
-                    "checked_on",
-                    "checked_by",
-                    "notes",
                     "date_added",
                     "date_updated",
                 ]
diff --git a/ukgrantmaking/migrations/0101_alter_cleaningstatus_sort_by_and_more.py b/ukgrantmaking/migrations/0101_alter_cleaningstatus_sort_by_and_more.py
new file mode 100644
index 0000000..4c224cb
--- /dev/null
+++ b/ukgrantmaking/migrations/0101_alter_cleaningstatus_sort_by_and_more.py
@@ -0,0 +1,319 @@
+# Generated by Django 5.0.6 on 2025-01-17 15:55
+
+from django.db import migrations, models
+
+fields_to_replace = {
+    "funder_financial_year__funder__org_id": "funder__org_id",
+    "funder_financial_year__funder__name": "funder__name",
+    "funder_financial_year__funder__status": "funder__status",
+    "funder_financial_year__funder__date_of_registration": "funder__date_of_registration",
+    "funder_financial_year__funder__date_of_removal": "funder__date_of_removal",
+    "funder_financial_year__funder__active": "funder__active",
+    "funder_financial_year__funder__latest_year__scaling": "funder__latest_year__scaling",
+    "funder_financial_year__scaling": "scaling",
+    "funder_financial_year__tags__tag": "tags__tag",
+    "funder_financial_year__segment": "segment",
+    "funder_financial_year__category": "category",
+    "funder_financial_year__included": "included",
+    "funder_financial_year__makes_grants_to_individuals": "makes_grants_to_individuals",
+    "funder_financial_year__checked": "checked",
+    "funder_financial_year__checked_on": "checked_on",
+    "funder_financial_year__checked_by": "checked_by",
+    "funder_financial_year__notes": "notes",
+    "funder_financial_year__date_added": "date_added",
+    "funder_financial_year__date_updated": "date_updated",
+}
+
+
+def replace_fields(apps, schema_editor):
+    CleaningStatus = apps.get_model("ukgrantmaking", "CleaningStatus")
+    CleaningStatusQuery = apps.get_model("ukgrantmaking", "CleaningStatusQuery")
+
+    for row in CleaningStatus.objects.filter(sort_by__in=fields_to_replace.keys()):
+        row.sort_by = fields_to_replace[row.sort_by]
+        row.save()
+
+    for row in CleaningStatusQuery.objects.filter(field__in=fields_to_replace.keys()):
+        row.field = fields_to_replace[row.field]
+        row.save()
+
+
+def replace_fields_backwards(apps, schema_editor):
+    fields_to_replace_reverse = {v: k for k, v in fields_to_replace.items()}
+    CleaningStatus = apps.get_model("ukgrantmaking", "CleaningStatus")
+    CleaningStatusQuery = apps.get_model("ukgrantmaking", "CleaningStatusQuery")
+
+    for row in CleaningStatus.objects.filter(
+        sort_by__in=fields_to_replace_reverse.keys()
+    ):
+        row.sort_by = fields_to_replace_reverse[row.field]
+        row.save()
+
+    for row in CleaningStatusQuery.objects.filter(
+        field__in=fields_to_replace_reverse.keys()
+    ):
+        row.field = fields_to_replace_reverse[row.field]
+        row.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ukgrantmaking", "0100_auto_20250116_1755"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="cleaningstatus",
+            name="sort_by",
+            field=models.CharField(
+                choices=[
+                    (
+                        "Grantmaker",
+                        [
+                            ("financial_year_end", "Financial Year End"),
+                            ("income", "Income"),
+                            ("income_investment", "Income investment"),
+                            ("spending", "Spending"),
+                            ("spending_investment", "Spending investment"),
+                            ("spending_charitable", "Spending charitable"),
+                            ("spending_grant_making", "Spending grant making"),
+                            (
+                                "spending_grant_making_individuals",
+                                "Spending grant making individuals",
+                            ),
+                            (
+                                "spending_grant_making_institutions_charitable",
+                                "Spending grant making institutions charitable",
+                            ),
+                            (
+                                "spending_grant_making_institutions_noncharitable",
+                                "Spending grant making institutions noncharitable",
+                            ),
+                            (
+                                "spending_grant_making_institutions_unknown",
+                                "Spending grant making institutions unknown",
+                            ),
+                            (
+                                "spending_grant_making_institutions",
+                                "Spending grant making institutions",
+                            ),
+                            ("total_net_assets", "Total net assets"),
+                            ("funds", "Funds"),
+                            ("funds_endowment", "Funds endowment"),
+                            ("funds_restricted", "Funds restricted"),
+                            ("funds_unrestricted", "Funds unrestricted"),
+                            ("employees", "Employees"),
+                            ("checked", "Checked"),
+                            ("funder__org_id", "Funder ID"),
+                            ("funder__name", "Funder Name"),
+                            ("funder__status", "Funder status"),
+                            ("funder__date_of_registration", "Registration date"),
+                            ("funder__date_of_removal", "Removal date"),
+                            ("funder__active", "Funder Active"),
+                            (
+                                "funder__latest_year__scaling",
+                                "Grantmaker size (latest)",
+                            ),
+                            ("scaling", "Grantmaker size"),
+                            ("tags__tag", "Tags"),
+                            ("segment", "Segment"),
+                            ("category", "Category"),
+                            ("included", "Included"),
+                            (
+                                "makes_grants_to_individuals",
+                                "Makes grants to individuals",
+                            ),
+                            ("checked_on", "Checked on"),
+                            ("checked_by", "Checked by"),
+                            ("notes", "Notes"),
+                            ("date_added", "Date added"),
+                            ("date_updated", "Date updated"),
+                        ],
+                    ),
+                    (
+                        "Grant",
+                        [
+                            ("title", "Title"),
+                            ("description", "Description"),
+                            ("currency", "Currency"),
+                            ("amount_awarded", "Amount awarded"),
+                            ("amount_awarded_GBP", "Amount awarded GBP"),
+                            ("award_date", "Award date"),
+                            ("planned_dates_duration", "Planned dates duration"),
+                            ("planned_dates_startDate", "Planned dates startDate"),
+                            ("planned_dates_endDate", "Planned dates endDate"),
+                            ("recipient_organisation_id", "Recipient organisation id"),
+                            (
+                                "recipient_organisation_name",
+                                "Recipient organisation name",
+                            ),
+                            ("recipient_individual_id", "Recipient individual id"),
+                            ("recipient_individual_name", "Recipient individual name"),
+                            (
+                                "recipient_individual_primary_grant_reason",
+                                "Recipient individual primary grant reason",
+                            ),
+                            (
+                                "recipient_individual_secondary_grant_reason",
+                                "Recipient individual secondary grant reason",
+                            ),
+                            (
+                                "recipient_individual_grant_purpose",
+                                "Recipient individual grant purpose",
+                            ),
+                            ("recipient_type", "Recipient type"),
+                            ("funding_organisation_id", "Funding organisation id"),
+                            ("funding_organisation_name", "Funding organisation name"),
+                            ("funding_organisation_type", "Funding organisation type"),
+                            ("regrant_type", "Regrant type"),
+                            ("location_scope", "Location scope"),
+                            ("grant_programme_title", "Grant programme title"),
+                            ("publisher_prefix", "Publisher prefix"),
+                            ("publisher_name", "Publisher name"),
+                            ("license", "License"),
+                            ("recipient_location_rgn", "Recipient location rgn"),
+                            ("recipient_location_ctry", "Recipient location ctry"),
+                            ("beneficiary_location_rgn", "Beneficiary location rgn"),
+                            ("beneficiary_location_ctry", "Beneficiary location ctry"),
+                            ("funder", "Funder"),
+                            ("recipient", "Recipient"),
+                            ("recipient_type_manual", "Recipient type manual"),
+                            ("inclusion", "Inclusion"),
+                            ("notes", "Notes"),
+                            ("checked_by", "Checked by"),
+                            ("annual_amount", "Annual amount"),
+                        ],
+                    ),
+                ],
+                default="funder_financial_year__funder__latest_year__scaling",
+                max_length=255,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="cleaningstatusquery",
+            name="field",
+            field=models.CharField(
+                choices=[
+                    (
+                        "Grantmaker",
+                        [
+                            ("financial_year_end", "Financial Year End"),
+                            ("income", "Income"),
+                            ("income_investment", "Income investment"),
+                            ("spending", "Spending"),
+                            ("spending_investment", "Spending investment"),
+                            ("spending_charitable", "Spending charitable"),
+                            ("spending_grant_making", "Spending grant making"),
+                            (
+                                "spending_grant_making_individuals",
+                                "Spending grant making individuals",
+                            ),
+                            (
+                                "spending_grant_making_institutions_charitable",
+                                "Spending grant making institutions charitable",
+                            ),
+                            (
+                                "spending_grant_making_institutions_noncharitable",
+                                "Spending grant making institutions noncharitable",
+                            ),
+                            (
+                                "spending_grant_making_institutions_unknown",
+                                "Spending grant making institutions unknown",
+                            ),
+                            (
+                                "spending_grant_making_institutions",
+                                "Spending grant making institutions",
+                            ),
+                            ("total_net_assets", "Total net assets"),
+                            ("funds", "Funds"),
+                            ("funds_endowment", "Funds endowment"),
+                            ("funds_restricted", "Funds restricted"),
+                            ("funds_unrestricted", "Funds unrestricted"),
+                            ("employees", "Employees"),
+                            ("checked", "Checked"),
+                            ("funder__org_id", "Funder ID"),
+                            ("funder__name", "Funder Name"),
+                            ("funder__status", "Funder status"),
+                            ("funder__date_of_registration", "Registration date"),
+                            ("funder__date_of_removal", "Removal date"),
+                            ("funder__active", "Funder Active"),
+                            (
+                                "funder__latest_year__scaling",
+                                "Grantmaker size (latest)",
+                            ),
+                            ("scaling", "Grantmaker size"),
+                            ("tags__tag", "Tags"),
+                            ("segment", "Segment"),
+                            ("category", "Category"),
+                            ("included", "Included"),
+                            (
+                                "makes_grants_to_individuals",
+                                "Makes grants to individuals",
+                            ),
+                            ("checked_on", "Checked on"),
+                            ("checked_by", "Checked by"),
+                            ("notes", "Notes"),
+                            ("date_added", "Date added"),
+                            ("date_updated", "Date updated"),
+                        ],
+                    ),
+                    (
+                        "Grant",
+                        [
+                            ("title", "Title"),
+                            ("description", "Description"),
+                            ("currency", "Currency"),
+                            ("amount_awarded", "Amount awarded"),
+                            ("amount_awarded_GBP", "Amount awarded GBP"),
+                            ("award_date", "Award date"),
+                            ("planned_dates_duration", "Planned dates duration"),
+                            ("planned_dates_startDate", "Planned dates startDate"),
+                            ("planned_dates_endDate", "Planned dates endDate"),
+                            ("recipient_organisation_id", "Recipient organisation id"),
+                            (
+                                "recipient_organisation_name",
+                                "Recipient organisation name",
+                            ),
+                            ("recipient_individual_id", "Recipient individual id"),
+                            ("recipient_individual_name", "Recipient individual name"),
+                            (
+                                "recipient_individual_primary_grant_reason",
+                                "Recipient individual primary grant reason",
+                            ),
+                            (
+                                "recipient_individual_secondary_grant_reason",
+                                "Recipient individual secondary grant reason",
+                            ),
+                            (
+                                "recipient_individual_grant_purpose",
+                                "Recipient individual grant purpose",
+                            ),
+                            ("recipient_type", "Recipient type"),
+                            ("funding_organisation_id", "Funding organisation id"),
+                            ("funding_organisation_name", "Funding organisation name"),
+                            ("funding_organisation_type", "Funding organisation type"),
+                            ("regrant_type", "Regrant type"),
+                            ("location_scope", "Location scope"),
+                            ("grant_programme_title", "Grant programme title"),
+                            ("publisher_prefix", "Publisher prefix"),
+                            ("publisher_name", "Publisher name"),
+                            ("license", "License"),
+                            ("recipient_location_rgn", "Recipient location rgn"),
+                            ("recipient_location_ctry", "Recipient location ctry"),
+                            ("beneficiary_location_rgn", "Beneficiary location rgn"),
+                            ("beneficiary_location_ctry", "Beneficiary location ctry"),
+                            ("funder", "Funder"),
+                            ("recipient", "Recipient"),
+                            ("recipient_type_manual", "Recipient type manual"),
+                            ("inclusion", "Inclusion"),
+                            ("notes", "Notes"),
+                            ("checked_by", "Checked by"),
+                            ("annual_amount", "Annual amount"),
+                        ],
+                    ),
+                ],
+                max_length=255,
+            ),
+        ),
+        migrations.RunPython(replace_fields, replace_fields_backwards),
+    ]
diff --git a/ukgrantmaking/migrations/0102_remove_funderfinancialyear_notes_and_more.py b/ukgrantmaking/migrations/0102_remove_funderfinancialyear_notes_and_more.py
new file mode 100644
index 0000000..cf65657
--- /dev/null
+++ b/ukgrantmaking/migrations/0102_remove_funderfinancialyear_notes_and_more.py
@@ -0,0 +1,32 @@
+# Generated by Django 5.0.6 on 2025-01-17 16:41
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("contenttypes", "0002_remove_content_type_name"),
+        ("ukgrantmaking", "0101_alter_cleaningstatus_sort_by_and_more"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="fundernote",
+            name="content_type",
+            field=models.ForeignKey(
+                default=1,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="contenttypes.contenttype",
+            ),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name="fundernote",
+            name="object_id",
+            field=models.CharField(default="1", max_length=255),
+            preserve_default=False,
+        ),
+    ]
diff --git a/ukgrantmaking/migrations/0103_remove_funderfinancialyear_notes_and_more.py b/ukgrantmaking/migrations/0103_remove_funderfinancialyear_notes_and_more.py
new file mode 100644
index 0000000..06b0db4
--- /dev/null
+++ b/ukgrantmaking/migrations/0103_remove_funderfinancialyear_notes_and_more.py
@@ -0,0 +1,83 @@
+# Generated by Django 5.0.6 on 2025-01-17 16:54
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+def add_funder_note_content_type(apps, schema_editor):
+    FunderNote = apps.get_model("ukgrantmaking", "FunderNote")
+    ContentType = apps.get_model("contenttypes", "ContentType")
+    for note in FunderNote.objects.all():
+        note.content_type = ContentType.objects.get_for_model(note.funder)
+        note.object_id = note.funder.org_id
+        note.save()
+
+
+def reverse_add_funder_note_content_type(apps, schema_editor):
+    FunderNote = apps.get_model("ukgrantmaking", "FunderNote")
+    for note in FunderNote.objects.all():
+        note.funder = note.content_object
+        note.save()
+
+
+def copy_notes_from_funder_year(apps, schema_editor):
+    ContentType = apps.get_model("contenttypes", "ContentType")
+    FunderYear = apps.get_model("ukgrantmaking", "FunderYear")
+    FunderNote = apps.get_model("ukgrantmaking", "FunderNote")
+
+    funder_year_content_type = ContentType.objects.get_for_model(FunderYear)
+
+    for funder_year in FunderYear.objects.exclude(
+        models.Q(notes__isnull=True) | models.Q(notes__exact="")
+    ).select_related("funder_financial_year"):
+        FunderNote.objects.create(
+            content_type=funder_year_content_type,
+            object_id=funder_year.id,
+            funder=funder_year.funder_financial_year.funder,
+            note=funder_year.notes,
+            date_added=funder_year.checked_on,
+            date_updated=funder_year.checked_on,
+            added_by=funder_year.funder_financial_year.checked_by,
+        )
+        if funder_year.funder_financial_year.notes and (
+            funder_year.funder_financial_year.notes != funder_year.notes
+        ):
+            FunderNote.objects.create(
+                content_type=funder_year_content_type,
+                object_id=funder_year.id,
+                funder=funder_year.funder_financial_year.funder,
+                note=funder_year.funder_financial_year.notes,
+                date_added=funder_year.funder_financial_year.checked_on,
+                date_updated=funder_year.funder_financial_year.checked_on,
+                added_by=funder_year.funder_financial_year.checked_by,
+            )
+
+
+def reverse_copy_notes_from_funder_year(apps, schema_editor):
+    ContentType = apps.get_model("contenttypes", "ContentType")
+    FunderYear = apps.get_model("ukgrantmaking", "FunderYear")
+    FunderNote = apps.get_model("ukgrantmaking", "FunderNote")
+
+    funder_year_content_type = ContentType.objects.get_for_model(FunderYear)
+
+    for note in FunderNote.objects.filter(content_type=funder_year_content_type):
+        note.content_object.notes += note.note
+        note.content_object.save()
+        note.delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("contenttypes", "0002_remove_content_type_name"),
+        ("ukgrantmaking", "0102_remove_funderfinancialyear_notes_and_more"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            add_funder_note_content_type, reverse_add_funder_note_content_type
+        ),
+        migrations.RunPython(
+            copy_notes_from_funder_year, reverse_copy_notes_from_funder_year
+        ),
+    ]
diff --git a/ukgrantmaking/migrations/0104_remove_funderfinancialyear_notes_and_more.py b/ukgrantmaking/migrations/0104_remove_funderfinancialyear_notes_and_more.py
new file mode 100644
index 0000000..03441d5
--- /dev/null
+++ b/ukgrantmaking/migrations/0104_remove_funderfinancialyear_notes_and_more.py
@@ -0,0 +1,42 @@
+# Generated by Django 5.0.6 on 2025-01-17 16:55
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("contenttypes", "0002_remove_content_type_name"),
+        ("ukgrantmaking", "0103_remove_funderfinancialyear_notes_and_more"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="funderfinancialyear",
+            name="notes",
+        ),
+        migrations.RemoveField(
+            model_name="fundernote",
+            name="funder",
+        ),
+        migrations.RemoveField(
+            model_name="funderyear",
+            name="checked",
+        ),
+        migrations.RemoveField(
+            model_name="funderyear",
+            name="notes",
+        ),
+        migrations.RemoveField(
+            model_name="grant",
+            name="notes",
+        ),
+        migrations.AddIndex(
+            model_name="fundernote",
+            index=models.Index(
+                fields=["content_type", "object_id"],
+                name="ukgrantmaki_content_ff0f8b_idx",
+            ),
+        ),
+    ]
diff --git a/ukgrantmaking/models/cleaningstatus.py b/ukgrantmaking/models/cleaningstatus.py
index d2e4ab0..d4975c4 100644
--- a/ukgrantmaking/models/cleaningstatus.py
+++ b/ukgrantmaking/models/cleaningstatus.py
@@ -95,95 +95,95 @@ class CleaningStatusType(models.TextChoices):
     ("employees", "Employees", CleaningStatusType.GRANTMAKER, int),
     ("checked", "Checked", CleaningStatusType.GRANTMAKER, str),
     (
-        "funder_financial_year__funder__org_id",
+        "funder__org_id",
         "Funder ID",
         CleaningStatusType.GRANTMAKER,
         str,
     ),
     (
-        "funder_financial_year__funder__name",
+        "funder__name",
         "Funder Name",
         CleaningStatusType.GRANTMAKER,
         str,
     ),
     (
-        "funder_financial_year__funder__status",
+        "funder__status",
         "Funder status",
         CleaningStatusType.GRANTMAKER,
         str,
     ),
     (
-        "funder_financial_year__funder__date_of_registration",
+        "funder__date_of_registration",
         "Registration date",
         CleaningStatusType.GRANTMAKER,
         date,
     ),
     (
-        "funder_financial_year__funder__date_of_removal",
+        "funder__date_of_removal",
         "Removal date",
         CleaningStatusType.GRANTMAKER,
         date,
     ),
     (
-        "funder_financial_year__funder__active",
+        "funder__active",
         "Funder Active",
         CleaningStatusType.GRANTMAKER,
         bool,
     ),
     (
-        "funder_financial_year__funder__latest_year__scaling",
+        "funder__latest_year__scaling",
         "Grantmaker size (latest)",
         CleaningStatusType.GRANTMAKER,
         int,
     ),
     (
-        "funder_financial_year__scaling",
+        "scaling",
         "Grantmaker size",
         CleaningStatusType.GRANTMAKER,
         int,
     ),
     (
-        "funder_financial_year__tags__tag",
+        "tags__tag",
         "Tags",
         CleaningStatusType.GRANTMAKER,
         list[str],
     ),
-    ("funder_financial_year__segment", "Segment", CleaningStatusType.GRANTMAKER, str),
-    ("funder_financial_year__category", "Category", CleaningStatusType.GRANTMAKER, str),
+    ("segment", "Segment", CleaningStatusType.GRANTMAKER, str),
+    ("category", "Category", CleaningStatusType.GRANTMAKER, str),
     (
-        "funder_financial_year__included",
+        "included",
         "Included",
         CleaningStatusType.GRANTMAKER,
         bool,
     ),
     (
-        "funder_financial_year__makes_grants_to_individuals",
+        "makes_grants_to_individuals",
         "Makes grants to individuals",
         CleaningStatusType.GRANTMAKER,
         bool,
     ),
-    ("funder_financial_year__checked", "Checked", CleaningStatusType.GRANTMAKER, str),
+    ("checked", "Checked", CleaningStatusType.GRANTMAKER, str),
     (
-        "funder_financial_year__checked_on",
+        "checked_on",
         "Checked on",
         CleaningStatusType.GRANTMAKER,
         date,
     ),
     (
-        "funder_financial_year__checked_by",
+        "checked_by",
         "Checked by",
         CleaningStatusType.GRANTMAKER,
         str,
     ),
-    ("funder_financial_year__notes", "Notes", CleaningStatusType.GRANTMAKER, str),
+    ("notes", "Notes", CleaningStatusType.GRANTMAKER, str),
     (
-        "funder_financial_year__date_added",
+        "date_added",
         "Date added",
         CleaningStatusType.GRANTMAKER,
         date,
     ),
     (
-        "funder_financial_year__date_updated",
+        "date_updated",
         "Date updated",
         CleaningStatusType.GRANTMAKER,
         date,
@@ -395,7 +395,7 @@ class SortOrder(models.TextChoices):
     sort_by = models.CharField(
         max_length=255,
         choices=FIELD_CHOICES,
-        default="funder_financial_year__funder__latest_year__scaling",
+        default="funder__latest_year__scaling",
     )
     sort_order = models.CharField(
         max_length=1,
@@ -407,6 +407,7 @@ class SortOrder(models.TextChoices):
 
     def __init__(self, *args, **kwargs):
         self._query_results = {}
+        self._sql_query = None
         super().__init__(*args, **kwargs)
 
     def __str__(self):
@@ -462,8 +463,8 @@ def run(self, qs, exclude_cleaned=False):
             if self.type == CleaningStatusType.GRANTMAKER:
                 qs = qs.filter(
                     ~models.Q(
-                        funder_financial_year__checked="Checked",
-                        funder_financial_year__checked__isnull=False,
+                        checked="Checked",
+                        checked__isnull=False,
                     )
                 )
 
@@ -477,6 +478,7 @@ def run(self, qs, exclude_cleaned=False):
             qs = qs.distinct("id")
         if self.n > 0:
             qs = qs[: self.n]
+        self._sql_query = qs.query.__str__()
         self._query_results[query_hash] = list(qs)
         return self._query_results[query_hash]
 
@@ -484,9 +486,7 @@ def get_status(self, qs):
         results = self.run(qs)
         total = len(results)
         if self.type == CleaningStatusType.GRANTMAKER:
-            checked = len(
-                [r for r in results if r.funder_financial_year.checked == "Checked"]
-            )
+            checked = len([r for r in results if r.checked == "Checked"])
             return [
                 Meter(
                     **{
@@ -500,7 +500,7 @@ def get_status(self, qs):
                 #         "name": "Checked segment",
                 #         "total": total,
                 #         "value": self.run(
-                #             qs.filter(funder_financial_year__funder__status="Checked")
+                #             qs.filter(funder__status="Checked")
                 #         ).count(),
                 #     }
                 # ),
diff --git a/ukgrantmaking/models/funder.py b/ukgrantmaking/models/funder.py
index b32963d..7e564ff 100644
--- a/ukgrantmaking/models/funder.py
+++ b/ukgrantmaking/models/funder.py
@@ -1,5 +1,6 @@
 from django.conf import settings
 from django.contrib.admin.models import LogEntry
+from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
 from django.contrib.contenttypes.models import ContentType
 from django.db import models
 from django.db.models.functions import Coalesce, Left, Length, Right, StrIndex
@@ -41,7 +42,9 @@ def save(self, *args, **kwargs):
 
 
 class FunderNote(models.Model):
-    funder = models.ForeignKey("Funder", on_delete=models.CASCADE, related_name="notes")
+    content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
+    object_id = models.CharField(max_length=255)
+    content_object = GenericForeignKey("content_type", "object_id")
     note = MarkdownxField()
     date_added = models.DateTimeField(auto_now_add=True)
     date_updated = models.DateTimeField(auto_now=True)
@@ -57,6 +60,11 @@ class FunderNote(models.Model):
     def __str__(self):
         return self.note
 
+    class Meta:
+        indexes = [
+            models.Index(fields=["content_type", "object_id"]),
+        ]
+
 
 class Funder(models.Model):
     org_id = models.CharField(max_length=255, primary_key=True)
@@ -201,6 +209,7 @@ class Funder(models.Model):
     )
 
     tags = models.ManyToManyField(FunderTag, blank=True, related_name="funders")
+    notes = GenericRelation(FunderNote)
 
     name = models.GeneratedField(
         expression=Coalesce(
diff --git a/ukgrantmaking/models/funder_financial_year.py b/ukgrantmaking/models/funder_financial_year.py
index 74d1c1e..289baba 100644
--- a/ukgrantmaking/models/funder_financial_year.py
+++ b/ukgrantmaking/models/funder_financial_year.py
@@ -1,7 +1,6 @@
 from django.conf import settings
 from django.db import models
 from django.db.models.functions import Coalesce
-from markdownx.models import MarkdownxField
 
 from ukgrantmaking.models.funder_utils import (
     FUNDER_CATEGORIES,
@@ -106,7 +105,6 @@ class FunderFinancialYear(models.Model):
         null=True,
         blank=True,
     )
-    notes = MarkdownxField(null=True, blank=True)
     date_added = models.DateTimeField(
         auto_now_add=True, db_index=True, null=True, blank=True
     )
diff --git a/ukgrantmaking/models/funder_year.py b/ukgrantmaking/models/funder_year.py
index 3b3bddd..edda250 100644
--- a/ukgrantmaking/models/funder_year.py
+++ b/ukgrantmaking/models/funder_year.py
@@ -1,8 +1,8 @@
+from django.contrib.contenttypes.fields import GenericRelation
 from django.core.exceptions import FieldDoesNotExist
 from django.db import models
 from django.db.models.functions import Coalesce
 from django.db.models.lookups import IsNull
-from markdownx.models import MarkdownxField
 
 from ukgrantmaking.models.funder_financial_year import FunderFinancialYear
 from ukgrantmaking.models.funder_utils import EditableField
@@ -355,42 +355,10 @@ class FunderYear(models.Model):
 
     checked_on = models.DateTimeField(null=True, blank=True)
     checked_by = models.CharField(max_length=255, null=True, blank=True)
-    notes = MarkdownxField(null=True, blank=True)
+    notes = GenericRelation("FunderNote")
     date_added = models.DateTimeField(auto_now_add=True, db_index=True)
     date_updated = models.DateTimeField(auto_now=True)
 
-    checked = models.GeneratedField(
-        db_persist=True,
-        expression=models.Q(
-            models.lookups.IsNull(models.F("checked_by"), False),
-            models.lookups.IsNull(models.F("notes"), False),
-            models.lookups.IsNull(models.F("income_manual"), False),
-            models.lookups.IsNull(models.F("spending_manual"), False),
-            models.lookups.IsNull(models.F("spending_charitable_manual"), False),
-            models.lookups.IsNull(
-                models.F("spending_grant_making_individuals_manual"), False
-            ),
-            models.lookups.IsNull(
-                models.F("spending_grant_making_institutions_charitable_manual"), False
-            ),
-            models.lookups.IsNull(
-                models.F("spending_grant_making_institutions_noncharitable_manual"),
-                False,
-            ),
-            models.lookups.IsNull(
-                models.F("spending_grant_making_institutions_unknown_manual"), False
-            ),
-            models.lookups.IsNull(models.F("total_net_assets_manual"), False),
-            models.lookups.IsNull(models.F("funds_manual"), False),
-            models.lookups.IsNull(models.F("funds_endowment_manual"), False),
-            models.lookups.IsNull(models.F("funds_restricted_manual"), False),
-            models.lookups.IsNull(models.F("funds_unrestricted_manual"), False),
-            models.lookups.IsNull(models.F("employees_manual"), False),
-            _connector="OR",
-        ),
-        output_field=models.BooleanField(),
-    )
-
     class Meta:
         ordering = ["funder_financial_year", "-financial_year_end"]
         constraints = [
diff --git a/ukgrantmaking/models/grant.py b/ukgrantmaking/models/grant.py
index 48ed4e1..3f204d9 100644
--- a/ukgrantmaking/models/grant.py
+++ b/ukgrantmaking/models/grant.py
@@ -1,6 +1,6 @@
+from django.contrib.contenttypes.fields import GenericRelation
 from django.db import models
 from django.db.models.functions import Coalesce, Left, Length, Right, StrIndex
-from markdownx.models import MarkdownxField
 
 from ukgrantmaking.models.financial_years import FinancialYear
 
@@ -336,7 +336,7 @@ class GrantToIndividualsReason(models.TextChoices):
         blank=True,
         db_index=True,
     )
-    notes = MarkdownxField(null=True, blank=True)
+    notes = GenericRelation("FunderNote")
     checked_by = models.CharField(max_length=255, null=True, blank=True)
     financial_year = models.ForeignKey(
         FinancialYear,
diff --git a/ukgrantmaking/tests/conftest.py b/ukgrantmaking/tests/conftest.py
index e9ecd58..5a0a427 100644
--- a/ukgrantmaking/tests/conftest.py
+++ b/ukgrantmaking/tests/conftest.py
@@ -104,7 +104,7 @@ def complex_task(make_funder):
     )
     # add a condition to the task
     task.cleaningstatusquery_set.create(
-        field="funder_financial_year__funder__name",
+        field="funder__name",
         comparison=Comparison.EQUAL,
         value="Test Funder 1",
     )
diff --git a/ukgrantmaking/tests/test_funderyear_htmx.py b/ukgrantmaking/tests/test_funderyear_htmx.py
index 00e1339..b4cc81e 100644
--- a/ukgrantmaking/tests/test_funderyear_htmx.py
+++ b/ukgrantmaking/tests/test_funderyear_htmx.py
@@ -177,7 +177,7 @@ def test_edit_funderyear_py(
 
 def test_edit_funderyear_add_note(client_logged_in, funder, check_log_entry):
     fy = funder.current_year.funder_years.first()
-    assert fy.notes is None
+    assert fy.notes.count() == 0
 
     note = "Test note"
 
@@ -189,7 +189,8 @@ def test_edit_funderyear_add_note(client_logged_in, funder, check_log_entry):
     assert response.status_code == 200
 
     fy.refresh_from_db()
-    assert fy.notes == note
+    assert fy.notes.count() == 1
+    assert fy.notes.first().note == note
     assert (
         fy.financial_year_end.strftime("%Y-%m-%d")
         in check_log_entry(funder)[0].change_message
diff --git a/ukgrantmaking/views/grantmakers.py b/ukgrantmaking/views/grantmakers.py
index f7c0c0b..b25a3b3 100644
--- a/ukgrantmaking/views/grantmakers.py
+++ b/ukgrantmaking/views/grantmakers.py
@@ -50,11 +50,9 @@ def index(request):
 def task_index(request):
     current_fy = FinancialYear.objects.get(current=True)
     cleaning_tasks = CleaningStatus.objects.filter(type=CleaningStatusType.GRANTMAKER)
-    base_qs = FunderYear.objects.filter(
-        funder_financial_year__financial_year=current_fy
-    ).select_related(
-        "funder_financial_year__funder", "funder_financial_year__financial_year"
-    )
+    base_qs = FunderFinancialYear.objects.filter(
+        financial_year=current_fy
+    ).select_related("funder", "financial_year")
     statuses = {task.id: task.get_status(base_qs) for task in cleaning_tasks}
     return render(
         request,
@@ -72,11 +70,9 @@ def task_detail(request, task_id, filetype=None):
         ).get(id=task_id)
     except CleaningStatus.DoesNotExist:
         raise Http404("Task not found")
-    base_qs = FunderYear.objects.filter(
-        funder_financial_year__financial_year=current_fy
-    ).select_related(
-        "funder_financial_year__funder", "funder_financial_year__financial_year"
-    )
+    base_qs = FunderFinancialYear.objects.filter(
+        financial_year=current_fy
+    ).select_related("funder", "financial_year")
 
     exclude_cleaned = "exclude_cleaned" in request.GET
 
@@ -359,11 +355,12 @@ def edit_funderyear(funder_year: FunderYear, request, suffix: str = "cy"):
         if current_value != new_value:
             changed_values.append(f"{field.name} from {current_value} to {new_value}")
 
-    if "note" in request.POST:
-        existing_value = funder_year.notes
-        funder_year.notes = request.POST.get("note")
-        if existing_value != funder_year.notes:
-            changed_values.append(f"Note from {existing_value} to {funder_year.notes}")
+    if "note" in request.POST and request.POST.get("note"):
+        funder_year.notes.create(
+            added_by=request.user,
+            note=request.POST.get("note"),
+        )
+        changed_values.append(f"Note added {funder_year.notes}")
 
     if changed_values:
         funder_year.funder_financial_year.checked_on = timezone.now()