From 3fe22c88abddb9358221dd3167d1cdf6b53ef278 Mon Sep 17 00:00:00 2001 From: Raphael Odini Date: Fri, 23 Jun 2023 22:15:55 +0200 Subject: [PATCH 1/4] Stats: add add_count queryset on QuestionAnswerEvent & QuizAnswerEvent --- stats/models.py | 42 ++++++++++++++++++++++++++++++++++++++ stats/tests/test_models.py | 22 ++++++++++++++++++-- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/stats/models.py b/stats/models.py index c7466b26..bac0060b 100644 --- a/stats/models.py +++ b/stats/models.py @@ -26,6 +26,27 @@ def for_question(self, question_id): def from_quiz(self): return self.filter(source=constants.QUESTION_SOURCE_QUIZ) + def agg_count( + self, + since="total", + week_or_month_iso_number=None, + year=None, + ): + queryset = self + # since + if since not in constants.AGGREGATION_SINCE_CHOICE_LIST: + raise ValueError(f"DailyStat agg_count: must be one of {constants.AGGREGATION_SINCE_CHOICE_LIST}") + if since == "last_30_days": + queryset = queryset.filter(created__date__gte=(date.today() - timedelta(days=30))) + if since == "month": + queryset = queryset.filter(created__month=week_or_month_iso_number) + elif since == "week": + queryset = queryset.filter(created__week=week_or_month_iso_number) + if year: + queryset = queryset.filter(created__year=year) + # field + return queryset.count() + def agg_timeseries(self): queryset = self queryset = ( @@ -109,6 +130,27 @@ def for_quiz(self, quiz_id): def last_30_days(self): return self.filter(created__date__gte=(date.today() - timedelta(days=30))) + def agg_count( + self, + since="total", + week_or_month_iso_number=None, + year=None, + ): + queryset = self + # since + if since not in constants.AGGREGATION_SINCE_CHOICE_LIST: + raise ValueError(f"DailyStat agg_count: must be one of {constants.AGGREGATION_SINCE_CHOICE_LIST}") + if since == "last_30_days": + queryset = queryset.filter(created__date__gte=(date.today() - timedelta(days=30))) + if since == "month": + queryset = queryset.filter(created__month=week_or_month_iso_number) + elif since == "week": + queryset = queryset.filter(created__week=week_or_month_iso_number) + if year: + queryset = queryset.filter(created__year=year) + # field + return queryset.count() + def agg_timeseries(self, scale="day"): queryset = self # scale diff --git a/stats/tests/test_models.py b/stats/tests/test_models.py index 0dba70cc..0089967b 100644 --- a/stats/tests/test_models.py +++ b/stats/tests/test_models.py @@ -1,4 +1,7 @@ +from datetime import timedelta + from django.test import TestCase +from django.utils import timezone from core import constants from questions.factories import QuestionFactory @@ -15,6 +18,9 @@ ) +datetime_50_days_ago = timezone.now() - timedelta(days=50) + + class QuestionStatTest(TestCase): @classmethod def setUpTestData(cls): @@ -28,9 +34,16 @@ def setUpTestData(cls): cls.question_rm_1 = QuestionFactory(type=constants.QUESTION_TYPE_QCM_RM, answer_correct="ab") cls.question_rm_2 = QuestionFactory(type=constants.QUESTION_TYPE_QCM_RM, answer_correct="abc") cls.question_rm_3 = QuestionFactory(type=constants.QUESTION_TYPE_QCM_RM, answer_correct="abcd") - QuestionAnswerEvent.objects.create(question_id=cls.question_rm_1.id, choice="cd", source="question") + QuestionAnswerEvent.objects.create( + question_id=cls.question_rm_1.id, choice="cd", source="question", created=datetime_50_days_ago + ) cls.question_vf = QuestionFactory(type=constants.QUESTION_TYPE_VF, answer_correct="b") + def test_question_answer_event_agg_count(self): + self.assertEqual(QuestionAnswerEvent.objects.count(), 2) + self.assertEqual(QuestionAnswerEvent.objects.agg_count(), 2) + self.assertEqual(QuestionAnswerEvent.objects.agg_count("last_30_days"), 1) + def test_question_agg_stat_created(self): self.assertEqual(Question.objects.count(), 1 + 3 + 1) self.assertEqual(QuestionAggStat.objects.count(), 1 + 3 + 1) @@ -54,9 +67,14 @@ def setUpTestData(cls): cls.quiz_1 = QuizFactory(name="quiz 1") # questions=[cls.question_1.id] QuizQuestion.objects.create(quiz=cls.quiz_1, question=cls.question_1) QuestionAnswerEvent.objects.create(question_id=cls.question_1.id, choice="a", source="question") - QuizAnswerEvent.objects.create(quiz_id=cls.quiz_1.id, answer_success_count=1) + QuizAnswerEvent.objects.create(quiz_id=cls.quiz_1.id, answer_success_count=1, created=datetime_50_days_ago) QuizFeedbackEvent.objects.create(quiz_id=cls.quiz_1.id, choice="dislike") + def test_quiz_answer_event_agg_count(self): + self.assertEqual(QuizAnswerEvent.objects.count(), 1) + self.assertEqual(QuizAnswerEvent.objects.agg_count(), 1) + self.assertEqual(QuizAnswerEvent.objects.agg_count("last_30_days"), 0) + def test_answer_count(self): self.assertEqual(self.quiz_1.answer_count_agg, 1) From 60e20dd2d368d9b54e92e1a8f5937fbe75071088 Mon Sep 17 00:00:00 2001 From: Raphael Odini Date: Fri, 23 Jun 2023 22:17:11 +0200 Subject: [PATCH 2/4] First basic send_contributor_monthly_recap_email --- .../send_contributor_monthly_recap_email.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 users/management/commands/send_contributor_monthly_recap_email.py diff --git a/users/management/commands/send_contributor_monthly_recap_email.py b/users/management/commands/send_contributor_monthly_recap_email.py new file mode 100644 index 00000000..193071a0 --- /dev/null +++ b/users/management/commands/send_contributor_monthly_recap_email.py @@ -0,0 +1,56 @@ +from datetime import timedelta + +from django.core.management import BaseCommand +from django.utils import timezone + +from stats.models import QuestionAnswerEvent, QuizAnswerEvent +from users.models import User + + +class Command(BaseCommand): + """ + Command to send an e-mail to each contributor with its monthly stats + By default, it will compute on the last month (run ideally on the first day of the next month :) + + Usage: + python manage.py send_contributor_monthly_recap_email --dry-run + python manage.py send_contributor_monthly_recap_email + """ + + def add_arguments(self, parser): + parser.add_argument( + "--dry-run", dest="dry_run", action="store_true", help="Dry run (no sends nor changes to the DB)" + ) + + def handle(self, *args, **options): + print("=== send_contributor_monthly_recap_email running") + weekday = timezone.now() - timedelta(days=25) # last month + weekday_year = weekday.year + weekday_month = weekday.month + + contributors = User.objects.all_contributors() + print(f"{contributors.count()} contributors") + contributors_with_public_content = contributors.has_public_content() + print(f"{contributors_with_public_content.count()} contributors with public content") + + for user in contributors_with_public_content: + question_answer_count_month = QuestionAnswerEvent.objects.filter( + question__in=user.questions.public().validated() + ).agg_count(since="month", week_or_month_iso_number=weekday_month, year=weekday_year) + quiz_answer_count_month = QuizAnswerEvent.objects.filter( + quiz__in=user.quizs.public().published() + ).agg_count(since="month", week_or_month_iso_number=weekday_month, year=weekday_year) + + parameters = { + "QUESTION_PUBLIC_VALIDATED_COUNT": user.question_public_validated_count, + "QUIZ_PUBLIC_PUBLISHED_COUNT": user.quiz_public_published_count, + "QUESTION_ANSWER_COUNT_MONTH": question_answer_count_month, + "QUIZ_ANSWER_COUNT_MONTH": quiz_answer_count_month, + # "COMMENT_COUNT_MONTH": 0 + } + + if not options["dry_run"]: + print("user") + print(parameters) + # send email + # log metadata From 6f11046ba1cc2683e8eaaee377406c65de606aee Mon Sep 17 00:00:00 2001 From: Raphael Odini Date: Fri, 7 Jul 2023 23:14:56 +0200 Subject: [PATCH 3/4] Create Sendinblue template. Working end-to-end script --- app/settings.py | 1 + .../send_contributor_monthly_recap_email.py | 59 +++++++++++++------ 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/app/settings.py b/app/settings.py index 4f53f64b..87333cf4 100644 --- a/app/settings.py +++ b/app/settings.py @@ -266,6 +266,7 @@ SIB_NEWSLETTER_DOI_TEMPLATE_ID = os.getenv("SIB_NEWSLETTER_DOI_TEMPLATE_ID", 0) SIB_SMTP_ENDPOINT = "https://api.brevo.com/v3/smtp/email" +SIB_CONTRIBUTOR_MONTHLY_RECAP_TEMPLATE_ID = os.getenv("SIB_CONTRIBUTOR_MONTHLY_RECAP_TEMPLATE_ID", 0) # Errors diff --git a/users/management/commands/send_contributor_monthly_recap_email.py b/users/management/commands/send_contributor_monthly_recap_email.py index 193071a0..0540569b 100644 --- a/users/management/commands/send_contributor_monthly_recap_email.py +++ b/users/management/commands/send_contributor_monthly_recap_email.py @@ -1,9 +1,11 @@ from datetime import timedelta +from django.conf import settings from django.core.management import BaseCommand from django.utils import timezone -from stats.models import QuestionAnswerEvent, QuizAnswerEvent +from core.utils.sendinblue import send_transactional_email_with_template_id +from stats.models import QuizAnswerEvent from users.models import User @@ -30,27 +32,48 @@ def handle(self, *args, **options): contributors = User.objects.all_contributors() print(f"{contributors.count()} contributors") - contributors_with_public_content = contributors.has_public_content() - print(f"{contributors_with_public_content.count()} contributors with public content") + # for now we contact only contributors with a public_quiz + contributors_with_public_quiz = contributors.has_public_quiz() # has_public_content() + print(f"{contributors_with_public_quiz.count()} contributors with public quiz") - for user in contributors_with_public_content: - question_answer_count_month = QuestionAnswerEvent.objects.filter( - question__in=user.questions.public().validated() - ).agg_count(since="month", week_or_month_iso_number=weekday_month, year=weekday_year) - quiz_answer_count_month = QuizAnswerEvent.objects.filter( - quiz__in=user.quizs.public().published() - ).agg_count(since="month", week_or_month_iso_number=weekday_month, year=weekday_year) + for user in contributors_with_public_quiz: + quiz_published = user.quizs.public().published() + quiz_published_count = quiz_published.count() + quiz_answer_count_month = QuizAnswerEvent.objects.filter(quiz__in=quiz_published).agg_count( + since="month", week_or_month_iso_number=weekday_month, year=weekday_year + ) + # question_answer_count_month = QuestionAnswerEvent.objects.filter( + # question__in=user.questions.public().validated() + # ).agg_count(since="month", week_or_month_iso_number=weekday_month, year=weekday_year) + # quiz_comment_count_month = parameters = { - "QUESTION_PUBLIC_VALIDATED_COUNT": user.question_public_validated_count, - "QUIZ_PUBLIC_PUBLISHED_COUNT": user.quiz_public_published_count, - "QUESTION_ANSWER_COUNT_MONTH": question_answer_count_month, - "QUIZ_ANSWER_COUNT_MONTH": quiz_answer_count_month, - # "COMMENT_COUNT_MONTH": 0 + "firstName": user.first_name, + "lastMonth": weekday.strftime("%B %Y"), + "quizAnswerCountLastMonth": quiz_answer_count_month, + "quizCountString": f"ton quiz {quiz_published.first().name}" + if (quiz_published_count == 1) + else f"tes {quiz_published_count} quizs", + # "commentCountLastMonth": 5, } if not options["dry_run"]: - print("user") - print(parameters) # send email - # log metadata + send_transactional_email_with_template_id( + to_email=user.email, + to_name=user.full_name, + template_id=int(settings.SIB_CONTRIBUTOR_MONTHLY_RECAP_TEMPLATE_ID), + parameters=parameters, + ) + + # log email + log_item = { + "action": f"email_contributor_monthly_recap_{weekday_year}_{weekday_month}", + "email_to": user.email, + # "email_subject": email_subject, + # "email_body": email_body, + "email_timestamp": timezone.now().isoformat(), + "metadata": {"parameters": parameters}, + } + user.logs.append(log_item) + user.save() From 99af6624ed268e81f181346e618322995f9d3f1d Mon Sep 17 00:00:00 2001 From: Raphael Odini Date: Mon, 10 Jul 2023 17:09:29 +0200 Subject: [PATCH 4/4] Add comment count --- contributions/models.py | 22 ++++++++++++ .../send_contributor_monthly_recap_email.py | 36 ++++++++++++++----- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/contributions/models.py b/contributions/models.py index 18698293..1855dc43 100644 --- a/contributions/models.py +++ b/contributions/models.py @@ -11,6 +11,7 @@ from history.models import HistoryChangedFieldsAbstractModel from questions.models import Question from quizs.models import Quiz +from stats import constants as stat_constants class CommentQuerySet(models.QuerySet): @@ -58,6 +59,27 @@ def has_parent(self): def published(self): return self.filter(publish=True) + def agg_count( + self, + since="total", + week_or_month_iso_number=None, + year=None, + ): + queryset = self + # since + if since not in stat_constants.AGGREGATION_SINCE_CHOICE_LIST: + raise ValueError(f"DailyStat agg_count: must be one of {stat_constants.AGGREGATION_SINCE_CHOICE_LIST}") + if since == "last_30_days": + queryset = queryset.filter(created__date__gte=(date.today() - timedelta(days=30))) + if since == "month": + queryset = queryset.filter(created__month=week_or_month_iso_number) + elif since == "week": + queryset = queryset.filter(created__week=week_or_month_iso_number) + if year: + queryset = queryset.filter(created__year=year) + # field + return queryset.count() + class Comment(models.Model): COMMENT_CHOICE_FIELDS = ["type", "status"] diff --git a/users/management/commands/send_contributor_monthly_recap_email.py b/users/management/commands/send_contributor_monthly_recap_email.py index 0540569b..e19063de 100644 --- a/users/management/commands/send_contributor_monthly_recap_email.py +++ b/users/management/commands/send_contributor_monthly_recap_email.py @@ -4,6 +4,7 @@ from django.core.management import BaseCommand from django.utils import timezone +from contributions.models import Comment from core.utils.sendinblue import send_transactional_email_with_template_id from stats.models import QuizAnswerEvent from users.models import User @@ -37,24 +38,41 @@ def handle(self, *args, **options): print(f"{contributors_with_public_quiz.count()} contributors with public quiz") for user in contributors_with_public_quiz: - quiz_published = user.quizs.public().published() - quiz_published_count = quiz_published.count() - quiz_answer_count_month = QuizAnswerEvent.objects.filter(quiz__in=quiz_published).agg_count( + # quiz stats + quiz_public_published = user.quizs.public().published() + quiz_public_published_count = quiz_public_published.count() + quiz_answer_count_month = QuizAnswerEvent.objects.filter(quiz__in=quiz_public_published).agg_count( since="month", week_or_month_iso_number=weekday_month, year=weekday_year ) - # question_answer_count_month = QuestionAnswerEvent.objects.filter( + quiz_public_published_string = ( + f"ton quiz {quiz_public_published.first().name}" + if (quiz_public_published_count == 1) + else f"tes {quiz_public_published_count} quizs" + ) + # question stats + question_public_validated = user.questions.public().validated() + # question_public_validated_count = question_public_validated.count() + # quiz_answer_count_month = QuestionAnswerEvent.objects.filter( # question__in=user.questions.public().validated() # ).agg_count(since="month", week_or_month_iso_number=weekday_month, year=weekday_year) - # quiz_comment_count_month = + # comment stats + quiz_comment_count_month = ( + Comment.objects.exclude_contributor_work() + .filter(quiz__in=quiz_public_published) + .agg_count(since="month", week_or_month_iso_number=weekday_month, year=weekday_year) + ) + question_comment_count_month = ( + Comment.objects.exclude_contributor_work() + .filter(question__in=question_public_validated) + .agg_count(since="month", week_or_month_iso_number=weekday_month, year=weekday_year) + ) parameters = { "firstName": user.first_name, "lastMonth": weekday.strftime("%B %Y"), "quizAnswerCountLastMonth": quiz_answer_count_month, - "quizCountString": f"ton quiz {quiz_published.first().name}" - if (quiz_published_count == 1) - else f"tes {quiz_published_count} quizs", - # "commentCountLastMonth": 5, + "quizCountString": quiz_public_published_string, + "commentCountLastMonth": quiz_comment_count_month + question_comment_count_month, } if not options["dry_run"]: