From ed89c16519386c9e99b2d6a733bfa02fe42e2086 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Mon, 2 Dec 2024 13:20:41 +0100 Subject: [PATCH 1/4] chore: use the most recent comment on observer exports --- apollo/submissions/services.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apollo/submissions/services.py b/apollo/submissions/services.py index a4d1124a3..217fe1257 100644 --- a/apollo/submissions/services.py +++ b/apollo/submissions/services.py @@ -134,6 +134,10 @@ def export_list(self, query, include_qa=False, include_group_timestamps=False): extra_data_columns = [""] * len(extra_fields) record = [submission.serial_no] if form.form_type == "SURVEY" else [] # noqa + most_recent_comment = SubmissionComment.query.filter_by( + submission=submission + ).order_by(SubmissionComment.id.desc()).first() + last_comment = (most_recent_comment.comment or "") if most_recent_comment else "" record.extend( [ @@ -169,9 +173,7 @@ def export_list(self, query, include_qa=False, include_group_timestamps=False): [submission.updated.strftime("%Y-%m-%d %H:%M:%S") if submission.updated else ""] + [1 if sample in submission.participant.samples else 0 for sample in samples] + [ - submission.comments[0].comment.replace("\n", "") # noqa - if submission.comments - else "", + last_comment.replace("\n", ""), submission.quarantine_status.value if submission.quarantine_status else "", ] ) From c481637c79ff89a619111cec5d4a2a9ba745c82a Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Mon, 2 Dec 2024 14:38:08 +0100 Subject: [PATCH 2/4] chore: tiny text cleanup --- apollo/submissions/services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apollo/submissions/services.py b/apollo/submissions/services.py index 217fe1257..b1698cfaa 100644 --- a/apollo/submissions/services.py +++ b/apollo/submissions/services.py @@ -173,7 +173,7 @@ def export_list(self, query, include_qa=False, include_group_timestamps=False): [submission.updated.strftime("%Y-%m-%d %H:%M:%S") if submission.updated else ""] + [1 if sample in submission.participant.samples else 0 for sample in samples] + [ - last_comment.replace("\n", ""), + last_comment.replace("\n", " ").strip(), submission.quarantine_status.value if submission.quarantine_status else "", ] ) From c919cf3c2ab99cec19ed216a3e0800af14d263ed Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Mon, 2 Dec 2024 16:02:36 +0100 Subject: [PATCH 3/4] chore: replace multiple queries --- apollo/submissions/services.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/apollo/submissions/services.py b/apollo/submissions/services.py index b1698cfaa..cd6670b2b 100644 --- a/apollo/submissions/services.py +++ b/apollo/submissions/services.py @@ -8,6 +8,7 @@ from geoalchemy2.shape import to_shape from apollo import constants +from apollo.core import db from apollo.dal.service import Service from apollo.locations.models import LocationType, LocationTypePath from apollo.participants.models import Sample @@ -118,6 +119,30 @@ def export_list(self, query, include_qa=False, include_group_timestamps=False): yield output.getvalue() output.close() + submission_ids = [i[0] for i in query.with_entities(Submission.id).all()] + latest_comments = ( + db.session.query( + SubmissionComment.submission_id, sa.func.max(SubmissionComment.submit_date).label("max_date") + ) + .group_by(SubmissionComment.submission_id) + .subquery() + ) + + sub_id_comment_pairs = ( + db.session.query(Submission.id, SubmissionComment.comment) + .join(latest_comments, Submission.id == latest_comments.c.submission_id) + .join( + SubmissionComment, + sa.and_( + SubmissionComment.submission_id == latest_comments.c.submission_id, + SubmissionComment.submit_date == latest_comments.c.max_date, + ), + ) + .where(Submission.id.in_(submission_ids)) + .order_by(SubmissionComment.submit_date.desc()) + ).all() + comment_map = {p[0]: p[1] for p in sub_id_comment_pairs} + for item in query: if export_qa: row_dict = item._asdict() @@ -134,10 +159,6 @@ def export_list(self, query, include_qa=False, include_group_timestamps=False): extra_data_columns = [""] * len(extra_fields) record = [submission.serial_no] if form.form_type == "SURVEY" else [] # noqa - most_recent_comment = SubmissionComment.query.filter_by( - submission=submission - ).order_by(SubmissionComment.id.desc()).first() - last_comment = (most_recent_comment.comment or "") if most_recent_comment else "" record.extend( [ @@ -173,7 +194,7 @@ def export_list(self, query, include_qa=False, include_group_timestamps=False): [submission.updated.strftime("%Y-%m-%d %H:%M:%S") if submission.updated else ""] + [1 if sample in submission.participant.samples else 0 for sample in samples] + [ - last_comment.replace("\n", " ").strip(), + comment_map.get(submission.id, "").replace("\n", " ").strip(), submission.quarantine_status.value if submission.quarantine_status else "", ] ) From af85112e8e43836cc7e4660da1843143e7b46fa5 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Thu, 5 Dec 2024 18:00:20 +0100 Subject: [PATCH 4/4] chore: refactor export --- apollo/submissions/services.py | 43 +++++++++++++--------------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/apollo/submissions/services.py b/apollo/submissions/services.py index cd6670b2b..f275fed91 100644 --- a/apollo/submissions/services.py +++ b/apollo/submissions/services.py @@ -8,7 +8,6 @@ from geoalchemy2.shape import to_shape from apollo import constants -from apollo.core import db from apollo.dal.service import Service from apollo.locations.models import LocationType, LocationTypePath from apollo.participants.models import Sample @@ -119,31 +118,21 @@ def export_list(self, query, include_qa=False, include_group_timestamps=False): yield output.getvalue() output.close() - submission_ids = [i[0] for i in query.with_entities(Submission.id).all()] - latest_comments = ( - db.session.query( - SubmissionComment.submission_id, sa.func.max(SubmissionComment.submit_date).label("max_date") - ) - .group_by(SubmissionComment.submission_id) - .subquery() - ) - - sub_id_comment_pairs = ( - db.session.query(Submission.id, SubmissionComment.comment) - .join(latest_comments, Submission.id == latest_comments.c.submission_id) - .join( - SubmissionComment, - sa.and_( - SubmissionComment.submission_id == latest_comments.c.submission_id, - SubmissionComment.submit_date == latest_comments.c.max_date, - ), - ) - .where(Submission.id.in_(submission_ids)) - .order_by(SubmissionComment.submit_date.desc()) - ).all() - comment_map = {p[0]: p[1] for p in sub_id_comment_pairs} - - for item in query: + latest_comments_subquery = sa.select( + SubmissionComment, + sa.func.row_number() + .over(partition_by=SubmissionComment.submission_id, order_by=SubmissionComment.submit_date.desc()) + .label("row_number"), + ).subquery() + + query2 = query.outerjoin( + latest_comments_subquery, + sa.and_( + Submission.id == latest_comments_subquery.c.submission_id, latest_comments_subquery.c.row_number == 1 + ), + ).with_entities(Submission, latest_comments_subquery.c.comment) + + for item, most_recent_comment in query2: if export_qa: row_dict = item._asdict() submission = row_dict["Submission"] @@ -194,7 +183,7 @@ def export_list(self, query, include_qa=False, include_group_timestamps=False): [submission.updated.strftime("%Y-%m-%d %H:%M:%S") if submission.updated else ""] + [1 if sample in submission.participant.samples else 0 for sample in samples] + [ - comment_map.get(submission.id, "").replace("\n", " ").strip(), + (most_recent_comment or "").replace("\n", " ").strip(), submission.quarantine_status.value if submission.quarantine_status else "", ] )