From 037876194557b8e444c0be4703a2e264e2d56b7a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Sep 2023 10:02:21 +0100 Subject: [PATCH 1/4] Bump gevent from 21.12.0 to 23.9.1 (#1053) Bumps [gevent](https://github.com/gevent/gevent) from 21.12.0 to 23.9.1. - [Release notes](https://github.com/gevent/gevent/releases) - [Changelog](https://github.com/gevent/gevent/blob/master/docs/changelog_pre.rst) - [Commits](https://github.com/gevent/gevent/compare/21.12.0...23.9.1) --- updated-dependencies: - dependency-name: gevent dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cd109a217..5a3798ba8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,7 +39,7 @@ elastic-apm==6.7.2 factory-boy==3.2.0 flower==1.2.0 freezegun==1.1.0 -gevent==21.12.0 +gevent==23.9.1 govuk-frontend-jinja @ git+https://github.com/alphagov/govuk-frontend-jinja.git@15845e4cca3a05df72c6e13ec6a7e35acc682f52 govuk-tech-docs-sphinx-theme==1.0.0 gunicorn==20.1.0 From 2b337308683b7e63aa59a26a6497e697c08cc369 Mon Sep 17 00:00:00 2001 From: Dale Cannon <118175145+dalecannon@users.noreply.github.com> Date: Tue, 26 Sep 2023 11:18:51 +0100 Subject: [PATCH 2/4] TP2000-1064 Fix quota origins in create measure journey (#1046) * Use active origins when setting cleaned_data * Fix page heading * Update test --- measures/forms.py | 4 ++- .../measures/create-geo-areas-formset.jinja | 18 ++++++++--- measures/tests/test_forms.py | 30 ++++++++++++------- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/measures/forms.py b/measures/forms.py index 0a17e6b10..c9ae7e85e 100644 --- a/measures/forms.py +++ b/measures/forms.py @@ -1148,7 +1148,9 @@ def clean(self): # Use quota order number origins and exclusions to set cleaned_data if self.order_number: - origins = self.order_number.quotaordernumberorigin_set.current() + origins = ( + self.order_number.quotaordernumberorigin_set.current().as_at_today_and_beyond() + ) cleaned_data["geo_areas_and_exclusions"] = [ { "geo_area": origin.geographical_area, diff --git a/measures/jinja2/measures/create-geo-areas-formset.jinja b/measures/jinja2/measures/create-geo-areas-formset.jinja index 5bbd1786c..99c5a33d6 100644 --- a/measures/jinja2/measures/create-geo-areas-formset.jinja +++ b/measures/jinja2/measures/create-geo-areas-formset.jinja @@ -3,12 +3,22 @@ {% from "components/summary-list/macro.njk" import govukSummaryList %} {% block breadcrumb %} - {% if request.path != "/" %} - {{ breadcrumbs(request, [{"text": "Review the geographical areas"}]) }} + {% if order_number %} + {% if request.path != "/" %} + {{ breadcrumbs(request, [{"text": "Review the geographical areas"}]) }} + {% endif %} + {% else %} + {{ super() }} {% endif %} - {% endblock %} +{% endblock %} -{% block page_title_heading %}Review the geographical areas{% endblock %} +{% block page_title_heading %} + {% if order_number %} + Review the geographical areas + {% else %} + {{ super() }} + {% endif %} +{% endblock %} {% block form %} {% if order_number and origins_and_exclusions %} diff --git a/measures/tests/test_forms.py b/measures/tests/test_forms.py index a518f1e71..ac3343734 100644 --- a/measures/tests/test_forms.py +++ b/measures/tests/test_forms.py @@ -471,18 +471,26 @@ def test_measure_forms_geo_area_invalid_data_error_messages(data, error, erga_om assert error in form.errors["geo_area"] -def test_measure_geographical_area_form_quota_order_number(): +def test_measure_geographical_area_form_quota_order_number(date_ranges): """Tests that `MeasureGeographicalAreaForm` sets `cleaned_data` with geographical data from quota order number if one is passed in `kwargs`.""" - origin_1 = factories.QuotaOrderNumberOriginFactory.create() - origin_2 = factories.QuotaOrderNumberOriginFactory.create( - order_number=origin_1.order_number, + old_origin = factories.QuotaOrderNumberOriginFactory.create( + valid_between=date_ranges.earlier, ) - origin_1_exclusions = factories.QuotaOrderNumberOriginExclusionFactory.create_batch( - 2, - origin=origin_1, + order_number = old_origin.order_number + + new_origin_1 = factories.QuotaOrderNumberOriginFactory.create( + order_number=order_number, + ) + new_origin_2 = factories.QuotaOrderNumberOriginFactory.create( + order_number=order_number, + ) + new_origin_1_exclusions = ( + factories.QuotaOrderNumberOriginExclusionFactory.create_batch( + 2, + origin=new_origin_1, + ) ) - order_number = origin_1.order_number with override_current_transaction(Transaction.objects.last()): form = forms.MeasureGeographicalAreaForm( @@ -495,11 +503,11 @@ def test_measure_geographical_area_form_quota_order_number(): expected_cleaned_data = [ { - "geo_area": origin_1.geographical_area, - "exclusions": list(origin_1.excluded_areas.all()), + "geo_area": new_origin_1.geographical_area, + "exclusions": list(new_origin_1.excluded_areas.all()), }, { - "geo_area": origin_2.geographical_area, + "geo_area": new_origin_2.geographical_area, "exclusions": [], }, ] From 48c1eee0788cf2ed06fbfbd5f871dfb93d797bd7 Mon Sep 17 00:00:00 2001 From: Dale Cannon <118175145+dalecannon@users.noreply.github.com> Date: Tue, 26 Sep 2023 11:33:47 +0100 Subject: [PATCH 3/4] TP2000-1030 Fix footnotes autocomplete selection (#1048) --- common/static/common/js/util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/static/common/js/util.js b/common/static/common/js/util.js index e20012f51..e891ba133 100644 --- a/common/static/common/js/util.js +++ b/common/static/common/js/util.js @@ -7,7 +7,7 @@ const nodeListForEach = (nodes, callback) => { } } -const newLine = /\n/g; +const newLine = /[\n\r]/g; const removeNewLine = (str) => str.replace(newLine, "") const cleanResults = (results) => { From b5e90efae6926f2cca4333d4a92360702186cb41 Mon Sep 17 00:00:00 2001 From: Paul Pepper <85895113+paulpepper-trade@users.noreply.github.com> Date: Wed, 27 Sep 2023 14:53:56 +0100 Subject: [PATCH 4/4] TP2000-1068 Envelope generation celery task info (#1051) * Add envelope task info to app-info page. * Better and consistent formatting. * App info page tests. --- common/jinja2/common/app_info.jinja | 113 ++++++++++++++++++++-------- common/tests/test_views.py | 30 +++++++- common/views.py | 71 ++++++++++++++--- 3 files changed, 167 insertions(+), 47 deletions(-) diff --git a/common/jinja2/common/app_info.jinja b/common/jinja2/common/app_info.jinja index 1080c062e..90b9acd6e 100644 --- a/common/jinja2/common/app_info.jinja +++ b/common/jinja2/common/app_info.jinja @@ -29,14 +29,57 @@ -
+ {% if request.user.is_superuser %} +
+
+

Deployment information

+ + {% set table_head = [ + {"text": "Name"}, + {"text": "Value"}, + {"text": "Description"}, + ] %} + + {% set table_rows = [ + [ + {"text": "GIT_BRANCH"}, + {"text": GIT_BRANCH}, + {"text": "Environment variable"}, + ], + [ + {"text": "GIT_COMMIT"}, + {"text": GIT_COMMIT}, + {"text": "Environment variable"}, + ], + [ + {"text": "APP_UPDATED_TIME"}, + {"text": APP_UPDATED_TIME}, + {"text": "Estimated application deploy time"}, + ], + [ + {"text": "LAST_TRANSACTION_TIME"}, + {"text": LAST_TRANSACTION_TIME}, + {"text": "Last transaction change time"}, + ], + ] %} + + {{ govukTable({ + "head": table_head, + "rows": table_rows + }) }} +
+
+ {% endif %} + + +

Active business rule checks

{% set table_head = [ {"text": "Workbasket ID"}, - {"text": "Time started"}, {"text": "Checks completed"}, + {"text": "Time started"}, {"text": "Celery task ID"}, ] %} {% set table_rows = [] %} @@ -46,15 +89,15 @@ {% for check in active_checks %} {{ table_rows.append([ {"text": check.workbasket_id}, - {"text": check.date_time_start}, {"text": check.checks_completed}, + {"text": check.date_time_start}, {"text": check.task_id}, ]) or "" }} {% endfor %} {% else %} {{ table_rows.append([ { - "text": "No active checks.", + "text": "No active business rule checks.", "colspan": 4, } ]) or "" }} @@ -76,39 +119,44 @@
- {% if request.user.is_superuser %} -
+
-

Deployment Information

+

Active envelope generation tasks

{% set table_head = [ - {"text": "Name"}, - {"text": "Value"}, - {"text": "Description"}, + {"text": "Packaged Workbasket ID"}, + {"text": "Workbasket ID"}, + {"text": "Time started"}, + {"text": "Celery task ID"}, ] %} + {% set table_rows = [] %} - {% set table_rows = [ - [ - {"text": "GIT_BRANCH"}, - {"text": GIT_BRANCH}, - {"text": "Environment variable"}, - ], - [ - {"text": "GIT_COMMIT"}, - {"text": GIT_COMMIT}, - {"text": "Environment variable"}, - ], - [ - {"text": "APP_UPDATED_TIME"}, - {"text": "{:%d %b %Y, %H:%M}".format(APP_UPDATED_TIME|localtime)}, - {"text": "Estimated application deploy time"}, - ], - [ - {"text": "LAST_TRANSACTION_TIME"}, - {"text": "{:%d %b %Y, %H:%M}".format(LAST_TRANSACTION_TIME|localtime)}, - {"text": "Last transaction change time"}, - ], - ] %} + {% if celery_healthy %} + {% if active_envelope_generation %} + {% for generation_info in active_envelope_generation %} + {{ table_rows.append([ + {"text": generation_info.packaged_workbasket_id}, + {"text": generation_info.workbasket_id}, + {"text": generation_info.date_time_start}, + {"text": generation_info.task_id}, + ]) or "" }} + {% endfor %} + {% else %} + {{ table_rows.append([ + { + "text": "No active envelope generation tasks.", + "colspan": 4, + } + ]) or "" }} + {% endif %} + {% else %} + {{ table_rows.append([ + { + "text": "Envelope generation task details are currently unavailable.", + "colspan": 4, + } + ]) or "" }} + {% endif %} {{ govukTable({ "head": table_head, @@ -116,6 +164,5 @@ }) }}
-{% endif %} {% endblock %} diff --git a/common/tests/test_views.py b/common/tests/test_views.py index 98cabe92f..2dfc9fa7c 100644 --- a/common/tests/test_views.py +++ b/common/tests/test_views.py @@ -118,13 +118,39 @@ def test_healthcheck_response(response, status_code, status): assert payload[1].tag == "response_time" -def test_app_info(valid_user_client): +def test_app_info_non_superuser(valid_user_client): + """Users without the superuser permission have a restricted view of + application information.""" response = valid_user_client.get(reverse("app-info")) assert response.status_code == 200 page = BeautifulSoup(str(response.content), "html.parser") - assert "Active business rule checks" in page.select("h2")[0].text + h2_elements = page.select(".info-section h2") + + assert len(h2_elements) == 2 + assert "Active business rule checks" in h2_elements[0].text + assert "Active envelope generation tasks" in h2_elements[1].text + + +def test_app_info_superuser(superuser_client, new_workbasket): + """ + Superusers should have an unrestricted view of application information. + + The new_workbasket fixture provides access to transaction information in the + deployment infomation section. + """ + response = superuser_client.get(reverse("app-info")) + + assert response.status_code == 200 + + page = BeautifulSoup(str(response.content), "html.parser") + h2_elements = page.select(".info-section h2") + + assert len(h2_elements) == 3 + assert "Deployment information" in h2_elements[0].text + assert "Active business rule checks" in h2_elements[1].text + assert "Active envelope generation tasks" in h2_elements[2].text def test_index_displays_footer_links(valid_user_client): diff --git a/common/views.py b/common/views.py index 74e829d01..8b95c115d 100644 --- a/common/views.py +++ b/common/views.py @@ -2,6 +2,7 @@ import os import time from datetime import datetime +from typing import Dict from typing import Optional from typing import Tuple from typing import Type @@ -41,6 +42,7 @@ from common.models import Transaction from common.pagination import build_pagination_list from common.validators import UpdateType +from publishing.models import PackagedWorkBasket from workbaskets.models import WorkBasket from workbaskets.views.mixins import WithCurrentWorkBasket @@ -129,16 +131,54 @@ class AppInfoView( TemplateView, ): template_name = "common/app_info.jinja" + DATETIME_FORMAT = "%d %b %Y, %H:%M" - def active_checks(self): - results = [] + def active_tasks(self) -> Dict: inspect = app.control.inspect() if not inspect: - return results + return {} active_tasks = inspect.active() if not active_tasks: - return results + return {} + + return active_tasks + + @staticmethod + def timestamp_to_datetime_string(timestamp): + return make_aware( + datetime.fromtimestamp(timestamp), + ).strftime(AppInfoView.DATETIME_FORMAT) + + def active_envelope_generation(self, active_tasks): + results = [] + + for _, task_info_list in active_tasks.items(): + for task_info in task_info_list: + if task_info.get("name") == "publishing.tasks.create_xml_envelope_file": + date_time_start = AppInfoView.timestamp_to_datetime_string( + task_info.get("time_start"), + ) + + packaged_workbasket_id = task_info.get("args", [""])[0] + packaged_workbasket = PackagedWorkBasket.objects.get( + id=packaged_workbasket_id, + ) + workbasket_id = packaged_workbasket.workbasket.id + + results.append( + { + "task_id": task_info.get("id"), + "packaged_workbasket_id": packaged_workbasket_id, + "workbasket_id": workbasket_id, + "date_time_start": date_time_start, + }, + ) + + return results + + def active_checks(self, active_tasks): + results = [] for _, task_info_list in active_tasks.items(): for task_info in task_info_list: @@ -146,11 +186,9 @@ def active_checks(self): task_info.get("name") == "workbaskets.tasks.call_check_workbasket_sync" ): - date_time_start = make_aware( - datetime.fromtimestamp( - task_info.get("time_start"), - ), - ).strftime("%Y-%m-%d, %H:%M:%S") + date_time_start = AppInfoView.timestamp_to_datetime_string( + task_info.get("time_start"), + ) workbasket_id = task_info.get("args", [""])[0] workbasket = WorkBasket.objects.get(id=workbasket_id) @@ -170,19 +208,28 @@ def active_checks(self): def get_context_data(self, **kwargs): data = super().get_context_data(**kwargs) try: - data["active_checks"] = self.active_checks() + active_tasks = self.active_tasks() data["celery_healthy"] = True + data["active_checks"] = self.active_checks(active_tasks) + data["active_envelope_generation"] = self.active_envelope_generation( + active_tasks, + ) except kombu.exceptions.OperationalError as oe: data["celery_healthy"] = False if self.request.user.is_superuser: data["GIT_BRANCH"] = os.getenv("GIT_BRANCH", "Unavailable") data["GIT_COMMIT"] = os.getenv("GIT_COMMIT", "Unavailable") - data["APP_UPDATED_TIME"] = datetime.fromtimestamp( + data["APP_UPDATED_TIME"] = AppInfoView.timestamp_to_datetime_string( os.path.getmtime(__file__), ) + last_transaction = Transaction.objects.order_by("updated_at").last() data["LAST_TRANSACTION_TIME"] = ( - Transaction.objects.order_by("-updated_at").first().updated_at + format( + last_transaction.updated_at.strftime(AppInfoView.DATETIME_FORMAT), + ) + if last_transaction + else "No transactions" ) return data