diff --git a/commcare_connect/opportunity/export.py b/commcare_connect/opportunity/export.py index 3e1cece8..cea1190d 100644 --- a/commcare_connect/opportunity/export.py +++ b/commcare_connect/opportunity/export.py @@ -6,15 +6,13 @@ from tablib import Dataset from commcare_connect.opportunity.forms import DateRanges -from commcare_connect.opportunity.helpers import ( - get_annotated_opportunity_access, - get_annotated_opportunity_access_deliver_status, -) from commcare_connect.opportunity.models import ( CatchmentArea, CompletedWork, Opportunity, OpportunityAccess, + OpportunityDeliverySummary, + UserInviteSummary, UserVisit, VisitValidationStatus, ) @@ -108,13 +106,13 @@ def export_empty_payment_table(opportunity: Opportunity) -> Dataset: def export_user_status_table(opportunity: Opportunity) -> Dataset: - access_objects = get_annotated_opportunity_access(opportunity) + access_objects = UserInviteSummary.objects.filter(opportunity=opportunity) table = UserStatusTable(access_objects, exclude=("date_popup", "view_profile")) return get_dataset(table, export_title="User status export") def export_deliver_status_table(opportunity: Opportunity) -> Dataset: - access_objects = get_annotated_opportunity_access_deliver_status(opportunity) + access_objects = OpportunityDeliverySummary.objects.filter(opportunity=opportunity) table = DeliverStatusTable(access_objects, exclude=("details", "date_popup")) return get_dataset(table, export_title="Deliver Status export") diff --git a/commcare_connect/opportunity/helpers.py b/commcare_connect/opportunity/helpers.py index 7cd144b6..0d36544c 100644 --- a/commcare_connect/opportunity/helpers.py +++ b/commcare_connect/opportunity/helpers.py @@ -1,133 +1,6 @@ from collections import namedtuple -from django.db.models import Case, Count, F, Max, Min, Q, Sum, Value, When - -from commcare_connect.opportunity.models import ( - CompletedWork, - CompletedWorkStatus, - Opportunity, - OpportunityAccess, - PaymentUnit, - UserInvite, - VisitValidationStatus, -) - - -def get_annotated_opportunity_access(opportunity: Opportunity): - learn_modules_count = opportunity.learn_app.learn_modules.count() - access_objects = ( - UserInvite.objects.filter(opportunity=opportunity) - .select_related("opportunity_access", "opportunity_access__opportunityclaim", "opportunity_access__user") - .annotate( - last_visit_date_d=Max( - "opportunity_access__user__uservisit__visit_date", - filter=Q(opportunity_access__user__uservisit__opportunity=opportunity) - & ~Q(opportunity_access__user__uservisit__status=VisitValidationStatus.trial), - ), - date_deliver_started=Min( - "opportunity_access__user__uservisit__visit_date", - filter=Q(opportunity_access__user__uservisit__opportunity=opportunity), - ), - passed_assessment=Sum( - Case( - When( - Q( - opportunity_access__user__assessments__opportunity=opportunity, - opportunity_access__user__assessments__passed=True, - ), - then=1, - ), - default=0, - ) - ), - completed_modules_count=Count( - "opportunity_access__user__completed_modules", - filter=Q(opportunity_access__user__completed_modules__opportunity=opportunity), - distinct=True, - ), - job_claimed=Case( - When( - Q(opportunity_access__opportunityclaim__isnull=False), - then="opportunity_access__opportunityclaim__date_claimed", - ) - ), - ) - .annotate( - date_learn_completed=Case( - When( - Q(completed_modules_count=learn_modules_count), - then=Max( - "opportunity_access__user__completed_modules__date", - filter=Q(opportunity_access__user__completed_modules__opportunity=opportunity), - ), - ) - ) - ) - .order_by("opportunity_access__user__name") - ) - - return access_objects - - -def get_annotated_opportunity_access_deliver_status(opportunity: Opportunity): - access_objects = [] - for payment_unit in opportunity.paymentunit_set.all(): - access_objects += ( - OpportunityAccess.objects.filter(opportunity=opportunity) - .select_related("user") - .annotate( - payment_unit=Value(payment_unit.name), - pending=Count( - "completedwork", - filter=Q( - completedwork__opportunity_access_id=F("pk"), - completedwork__payment_unit=payment_unit, - completedwork__status=CompletedWorkStatus.pending, - ), - distinct=True, - ), - approved=Count( - "completedwork", - filter=Q( - completedwork__opportunity_access_id=F("pk"), - completedwork__payment_unit=payment_unit, - completedwork__status=CompletedWorkStatus.approved, - ), - distinct=True, - ), - rejected=Count( - "completedwork", - filter=Q( - completedwork__opportunity_access_id=F("pk"), - completedwork__payment_unit=payment_unit, - completedwork__status=CompletedWorkStatus.rejected, - ), - distinct=True, - ), - over_limit=Count( - "completedwork", - filter=Q( - completedwork__opportunity_access_id=F("pk"), - completedwork__payment_unit=payment_unit, - completedwork__status=CompletedWorkStatus.over_limit, - ), - distinct=True, - ), - incomplete=Count( - "completedwork", - filter=Q( - completedwork__opportunity_access_id=F("pk"), - completedwork__payment_unit=payment_unit, - completedwork__status=CompletedWorkStatus.incomplete, - ), - distinct=True, - ), - completed=F("approved") + F("rejected") + F("pending") + F("over_limit"), - ) - .order_by("user__name") - ) - access_objects.sort(key=lambda a: a.user.name) - return access_objects +from commcare_connect.opportunity.models import CompletedWork, CompletedWorkStatus, Opportunity, PaymentUnit def get_payment_report_data(opportunity: Opportunity): diff --git a/commcare_connect/opportunity/migrations/0059_userinvitesummary.py b/commcare_connect/opportunity/migrations/0059_userinvitesummary.py new file mode 100644 index 00000000..dc6c1270 --- /dev/null +++ b/commcare_connect/opportunity/migrations/0059_userinvitesummary.py @@ -0,0 +1,215 @@ +# Generated by Django 4.2.5 on 2024-09-30 14:21 +import django +from django.db import migrations, models +from django_celery_beat.models import CrontabSchedule, PeriodicTask + + +def create_refresh_materialized_view_task(apps, schema_editor): + schedule, _ = CrontabSchedule.objects.get_or_create( + minute="0", + hour="0", # At midnight + day_of_week="*", + day_of_month="*", + month_of_year="*", + ) + PeriodicTask.objects.update_or_create( + crontab=schedule, + name="refresh_materialized_view", + task="commcare_connect.opportunity.tasks.refresh_materialized_view", + ) + + +def delete_refresh_materialized_view_task(apps, schema_editor): + PeriodicTask.objects.filter( + name="refresh_materialized_view", + task="commcare_connect.opportunity.tasks.refresh_materialized_view", + ).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("opportunity", "0058_paymentinvoice_payment_invoice"), + ] + + """This migration exists because we created a materialized view for the `UserInviteSummary` model. + Django automatically generates migrations for all models present in the application. + Since `managed = False` and `db_table is provided Django will not create or modify this table in the database. + However, we still need this migration in the migration script to prevent Django from generating it again when `makemigrations` is run in the future. + This ensures that the model is recognized without altering the actual database structure.""" + + operations = [ + migrations.CreateModel( + name="UserInviteSummary", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "status", + models.CharField( + choices=[ + ("sms_delivered", "SMS Delivered"), + ("sms_not_delivered", "SMS Not Delivered"), + ("accepted", "Accepted"), + ("invited", "Invited"), + ("not_found", "ConnectID Not Found"), + ], + default="invited", + max_length=50, + ), + ), + ("last_visit_date", models.DateTimeField(blank=True, null=True)), + ("date_deliver_started", models.DateTimeField(blank=True, null=True)), + ("passed_assessment", models.IntegerField(default=0)), + ("completed_modules_count", models.IntegerField(default=0)), + ("job_claimed", models.DateTimeField(null=True)), + ("date_learn_completed", models.DateTimeField(null=True)), + ( + "opportunity", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="opportunity.opportunity"), + ), + ( + "opportunity_access", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="opportunity.opportunityaccess", + ), + ), + ], + options={ + "db_table": "opportunity_userinvite_summary", + "managed": False, + }, + ), + migrations.RunSQL( + sql=""" + CREATE MATERIALIZED VIEW opportunity_userinvite_summary AS + WITH total_learning_modules AS ( + SELECT + app_id, + COUNT(*) AS total_modules_count + FROM + opportunity_learnmodule + GROUP BY + app_id + ) + SELECT + userinvite.id AS id, + userinvite.status AS status, + userinvite.opportunity_access_id AS opportunity_access_id, + userinvite.opportunity_id AS opportunity_id, + MAX(uservisit.visit_date) FILTER (WHERE uservisit.opportunity_id = opp.id AND uservisit.status != 'trial') AS last_visit_date, + MIN(uservisit.visit_date) FILTER (WHERE uservisit.opportunity_id = opp.id) AS date_deliver_started, + SUM(CASE WHEN assessment.opportunity_id = opp.id AND assessment.passed = TRUE THEN 1 ELSE 0 END) AS passed_assessment, + COUNT(DISTINCT completedmodule.id) FILTER (WHERE completedmodule.opportunity_id = opp.id) AS completed_modules_count, + CASE WHEN claim.id IS NOT NULL THEN claim.date_claimed END AS job_claimed, + CASE + WHEN COUNT(completedmodule.id) = learning_module.total_modules_count THEN + MAX(completedmodule.date) FILTER (WHERE completedmodule.opportunity_id = opp.id) + END AS date_learn_completed + FROM + opportunity_userinvite AS userinvite + JOIN + opportunity_opportunityaccess AS access ON userinvite.opportunity_access_id = access.id + JOIN + users_user AS _user ON access.user_id = _user.id + LEFT JOIN + opportunity_uservisit AS uservisit ON _user.id = uservisit.user_id + LEFT JOIN + opportunity_assessment AS assessment ON _user.id = assessment.user_id + LEFT JOIN + opportunity_completedmodule AS completedmodule ON completedmodule.user_id = _user.id + LEFT JOIN + opportunity_opportunityclaim AS claim ON access.id = claim.opportunity_access_id + JOIN + opportunity_opportunity AS opp ON userinvite.opportunity_id = opp.id + JOIN + total_learning_modules AS learning_module ON learning_module.app_id = opp.learn_app_id + GROUP BY + userinvite.id, _user.id, claim.id, learning_module.total_modules_count, userinvite.opportunity_access_id; + """, + reverse_sql="DROP MATERIALIZED VIEW IF EXISTS opportunity_userinvite_summary;", + ), + migrations.CreateModel( + name="OpportunityDeliverySummary", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("approved", models.IntegerField(default=0)), + ("pending", models.IntegerField(default=0)), + ("rejected", models.IntegerField(default=0)), + ("over_limit", models.IntegerField(default=0)), + ("incomplete", models.IntegerField(default=0)), + ("completed", models.IntegerField(default=0)), + ("payment_unit", models.CharField(max_length=255)), + ( + "opportunity", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="opportunity.opportunity"), + ), + ( + "user", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="users.user"), + ), + ( + "opportunity_access", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to="opportunity.opportunityaccess", + ), + ), + ], + options={ + "db_table": "opportunity_delivery_summary", + "managed": False, + }, + ), + migrations.RunSQL( + sql=""" + CREATE MATERIALIZED VIEW opportunity_delivery_summary AS + SELECT + access.id AS id, + access.id AS opportunity_access_id, + access.opportunity_id AS opportunity_id, + access.user_id AS user_id, + payment_unit.name AS payment_unit, + + COUNT(DISTINCT CASE WHEN completed_work.status = 'pending' + AND completed_work.opportunity_access_id = access.id + AND completed_work.payment_unit_id = payment_unit.id THEN completed_work.id END) AS pending, + + COUNT(DISTINCT CASE WHEN completed_work.status = 'approved' + AND completed_work.opportunity_access_id = access.id + AND completed_work.payment_unit_id = payment_unit.id THEN completed_work.id END) AS approved, + + COUNT(DISTINCT CASE WHEN completed_work.status = 'rejected' + AND completed_work.opportunity_access_id = access.id + AND completed_work.payment_unit_id = payment_unit.id THEN completed_work.id END) AS rejected, + + COUNT(DISTINCT CASE WHEN completed_work.status = 'over_limit' + AND completed_work.opportunity_access_id = access.id + AND completed_work.payment_unit_id = payment_unit.id THEN completed_work.id END) AS over_limit, + + COUNT(DISTINCT CASE WHEN completed_work.status = 'incomplete' + AND completed_work.opportunity_access_id = access.id + AND completed_work.payment_unit_id = payment_unit.id THEN completed_work.id END) AS incomplete, + + COALESCE( + COUNT(DISTINCT CASE WHEN completed_work.status IN ('approved', 'rejected', 'pending', 'over_limit') + AND completed_work.opportunity_access_id = access.id + AND completed_work.payment_unit_id = payment_unit.id THEN completed_work.id END), + 0 + ) AS completed + FROM + opportunity_opportunityaccess AS access + LEFT JOIN + opportunity_completedwork AS completed_work ON access.id = completed_work.opportunity_access_id + LEFT JOIN + opportunity_paymentunit AS payment_unit ON completed_work.payment_unit_id = payment_unit.id + INNER JOIN + users_user AS _user ON access.user_id = _user.id + GROUP BY + access.id, + _user.id, + payment_unit.name; + """, + reverse_sql="DROP MATERIALIZED VIEW IF EXISTS opportunity_delivery_summary;", + ), + migrations.RunPython(create_refresh_materialized_view_task, delete_refresh_materialized_view_task), + ] diff --git a/commcare_connect/opportunity/models.py b/commcare_connect/opportunity/models.py index 87d970c3..f976f689 100644 --- a/commcare_connect/opportunity/models.py +++ b/commcare_connect/opportunity/models.py @@ -683,3 +683,36 @@ class CatchmentArea(models.Model): class Meta: unique_together = ("site_code", "opportunity") + + +class UserInviteSummary(models.Model): + opportunity_access = models.ForeignKey(OpportunityAccess, null=True, on_delete=models.DO_NOTHING) + opportunity = models.ForeignKey(Opportunity, null=True, on_delete=models.CASCADE) + status = models.CharField(max_length=50, choices=UserInviteStatus.choices, default=UserInviteStatus.invited) + last_visit_date = models.DateTimeField(null=True, blank=True) + date_deliver_started = models.DateTimeField(null=True, blank=True) + passed_assessment = models.IntegerField(default=0) + completed_modules_count = models.IntegerField(default=0) + job_claimed = models.DateTimeField(null=True) + date_learn_completed = models.DateTimeField(null=True) + + class Meta: + managed = False + db_table = "opportunity_userinvite_summary" + + +class OpportunityDeliverySummary(models.Model): + opportunity_access = models.ForeignKey(OpportunityAccess, null=True, on_delete=models.DO_NOTHING) + opportunity = models.ForeignKey(Opportunity, null=True, on_delete=models.CASCADE) + user = models.ForeignKey(User, null=True, on_delete=models.CASCADE) + approved = models.IntegerField(default=0) + pending = models.IntegerField(default=0) + rejected = models.IntegerField(default=0) + over_limit = models.IntegerField(default=0) + incomplete = models.IntegerField(default=0) + completed = models.IntegerField(default=0) + payment_unit = models.CharField(max_length=255) + + class Meta: + managed = False + db_table = "opportunity_delivery_summary" diff --git a/commcare_connect/opportunity/tables.py b/commcare_connect/opportunity/tables.py index a856650f..248ef0d8 100644 --- a/commcare_connect/opportunity/tables.py +++ b/commcare_connect/opportunity/tables.py @@ -10,8 +10,8 @@ Payment, PaymentInvoice, PaymentUnit, - UserInvite, UserInviteStatus, + UserInviteSummary, UserVisit, VisitValidationStatus, ) @@ -152,11 +152,11 @@ class UserStatusTable(OrgContextTable): completed_learning = AggregateColumn(verbose_name="Completed Learning", accessor="date_learn_completed") passed_assessment = BooleanAggregateColumn(verbose_name="Passed Assessment") started_delivery = AggregateColumn(verbose_name="Started Delivery", accessor="date_deliver_started") - last_visit_date = columns.Column(accessor="last_visit_date_d") + last_visit_date = columns.Column(accessor="last_visit_date") view_profile = columns.Column("View Profile", empty_values=(), footer=lambda table: f"Invited: {len(table.rows)}") class Meta: - model = UserInvite + model = UserInviteSummary fields = ("status",) sequence = ( "display_name", diff --git a/commcare_connect/opportunity/tasks.py b/commcare_connect/opportunity/tasks.py index b5450cc1..daf4a09b 100644 --- a/commcare_connect/opportunity/tasks.py +++ b/commcare_connect/opportunity/tasks.py @@ -6,7 +6,7 @@ from django.conf import settings from django.core.files.base import ContentFile from django.core.files.storage import default_storage -from django.db import transaction +from django.db import connection, transaction from django.urls import reverse from django.utils.timezone import now from django.utils.translation import gettext @@ -339,3 +339,13 @@ def generate_catchment_area_export(opportunity_id: int, export_format: str): export_tmp_name = f"{now().isoformat()}_{opportunity.name}_catchment_area.{export_format}" save_export(dataset, export_tmp_name, export_format) return export_tmp_name + + +@celery_app.task() +def refresh_materialized_view(): + with connection.cursor() as cursor: + cursor.execute( + """REFRESH MATERIALIZED VIEW opportunity_userinvite_summary; + REFRESH MATERIALIZED VIEW opportunity_delivery_summary; + """ + ) diff --git a/commcare_connect/opportunity/tests/test_export.py b/commcare_connect/opportunity/tests/test_export.py index 32a94c55..bc4df061 100644 --- a/commcare_connect/opportunity/tests/test_export.py +++ b/commcare_connect/opportunity/tests/test_export.py @@ -13,6 +13,7 @@ ) from commcare_connect.opportunity.forms import DateRanges from commcare_connect.opportunity.models import Opportunity, UserInviteStatus, UserVisit +from commcare_connect.opportunity.tasks import refresh_materialized_view from commcare_connect.opportunity.tests.factories import ( AssessmentFactory, CatchmentAreaFactory, @@ -127,6 +128,7 @@ def test_export_user_status_table_no_data_only(opportunity: Opportunity): rows.append( (mobile_user.name, mobile_user.username, "Accepted", date.replace(tzinfo=None), "", False, "", "", "") ) + refresh_materialized_view() dataset = export_user_status_table(opportunity) prepared_test_dataset = _get_prepared_dataset_for_user_status_test(rows) assert prepared_test_dataset.export("csv") == dataset.export("csv") @@ -150,6 +152,7 @@ def test_export_user_status_table_learn_data_only(opportunity: Opportunity): rows.append( (mobile_user.name, mobile_user.username, "Accepted", date.replace(tzinfo=None), "", False, "", "", "") ) + refresh_materialized_view() dataset = export_user_status_table(opportunity) prepared_test_dataset = _get_prepared_dataset_for_user_status_test(rows) assert prepared_test_dataset.export("csv") == dataset.export("csv") @@ -191,6 +194,7 @@ def test_export_user_status_table_learn_assessment_data_only(opportunity: Opport "", ) ) + refresh_materialized_view() dataset = export_user_status_table(opportunity) prepared_test_dataset = _get_prepared_dataset_for_user_status_test(rows) assert prepared_test_dataset.export("csv") == dataset.export("csv") @@ -241,6 +245,7 @@ def test_export_user_status_table_data(opportunity: Opportunity): date.replace(tzinfo=None), ) ) + refresh_materialized_view() dataset = export_user_status_table(opportunity) prepared_test_dataset = _get_prepared_dataset_for_user_status_test(rows) assert prepared_test_dataset.export("csv") == dataset.export("csv") diff --git a/commcare_connect/opportunity/tests/test_helpers.py b/commcare_connect/opportunity/tests/test_helpers.py index c0528364..c667c69f 100644 --- a/commcare_connect/opportunity/tests/test_helpers.py +++ b/commcare_connect/opportunity/tests/test_helpers.py @@ -1,7 +1,7 @@ import pytest -from commcare_connect.opportunity.helpers import get_annotated_opportunity_access_deliver_status -from commcare_connect.opportunity.models import Opportunity +from commcare_connect.opportunity.models import Opportunity, OpportunityDeliverySummary +from commcare_connect.opportunity.tasks import refresh_materialized_view from commcare_connect.opportunity.tests.factories import ( CompletedWorkFactory, OpportunityAccessFactory, @@ -15,7 +15,8 @@ def test_deliver_status_query_no_visits(opportunity: Opportunity): mobile_users = MobileUserFactory.create_batch(5) for mobile_user in mobile_users: OpportunityAccessFactory(opportunity=opportunity, user=mobile_user, accepted=True) - access_objects = get_annotated_opportunity_access_deliver_status(opportunity) + refresh_materialized_view() + access_objects = OpportunityDeliverySummary.objects.filter(opportunity=opportunity) usernames = {user.username for user in mobile_users} for access in access_objects: @@ -41,7 +42,8 @@ def test_deliver_status_query(opportunity: Opportunity): count_by_status["completed"] = len(completed_works) - count_by_status["incomplete"] completed_work_counts[(mobile_user.username, pu.name)] = count_by_status - access_objects = get_annotated_opportunity_access_deliver_status(opportunity) + refresh_materialized_view() + access_objects = OpportunityDeliverySummary.objects.filter(opportunity=opportunity) for access in access_objects: username = access.user.username assert (username, access.payment_unit) in completed_work_counts @@ -61,7 +63,9 @@ def test_deliver_status_query_visits_another_opportunity(opportunity: Opportunit for mobile_user in mobile_users: OpportunityAccessFactory(opportunity=opportunity, user=mobile_user, accepted=True) CompletedWorkFactory.create_batch(5) - access_objects = get_annotated_opportunity_access_deliver_status(opportunity) + + refresh_materialized_view() + access_objects = OpportunityDeliverySummary.objects.filter(opportunity=opportunity) usernames = {user.username for user in mobile_users} for access in access_objects: assert access.user.username in usernames diff --git a/commcare_connect/opportunity/views.py b/commcare_connect/opportunity/views.py index 9b6c00aa..f5b8e6dd 100644 --- a/commcare_connect/opportunity/views.py +++ b/commcare_connect/opportunity/views.py @@ -43,11 +43,7 @@ SendMessageMobileUsersForm, VisitExportForm, ) -from commcare_connect.opportunity.helpers import ( - get_annotated_opportunity_access, - get_annotated_opportunity_access_deliver_status, - get_payment_report_data, -) +from commcare_connect.opportunity.helpers import get_payment_report_data from commcare_connect.opportunity.models import ( BlobMeta, CatchmentArea, @@ -60,10 +56,12 @@ OpportunityAccess, OpportunityClaim, OpportunityClaimLimit, + OpportunityDeliverySummary, OpportunityVerificationFlags, Payment, PaymentInvoice, PaymentUnit, + UserInviteSummary, UserVisit, VisitValidationStatus, ) @@ -478,8 +476,7 @@ def get_queryset(self): opportunity_id = self.kwargs["pk"] org_slug = self.kwargs["org_slug"] opportunity = get_opportunity_or_404(org_slug=org_slug, pk=opportunity_id) - access_objects = get_annotated_opportunity_access(opportunity) - return access_objects + return UserInviteSummary.objects.filter(opportunity=opportunity) @org_member_required @@ -653,8 +650,7 @@ def get_queryset(self): opportunity_id = self.kwargs["pk"] org_slug = self.kwargs["org_slug"] opportunity = get_opportunity_or_404(pk=opportunity_id, org_slug=org_slug) - access_objects = get_annotated_opportunity_access_deliver_status(opportunity) - return access_objects + return OpportunityDeliverySummary.objects.filter(opportunity=opportunity) @org_member_required